Line data Source code
1 : /**
2 : Copyright (c) 2017-2022 Roman Katuntsev <sbkarr@stappler.org>
3 : Copyright (c) 2023 Stappler LLC <admin@stappler.dev>
4 :
5 : Permission is hereby granted, free of charge, to any person obtaining a copy
6 : of this software and associated documentation files (the "Software"), to deal
7 : in the Software without restriction, including without limitation the rights
8 : to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 : copies of the Software, and to permit persons to whom the Software is
10 : furnished to do so, subject to the following conditions:
11 :
12 : The above copyright notice and this permission notice shall be included in
13 : all copies or substantial portions of the Software.
14 :
15 : THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 : IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 : FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 : AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 : LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 : OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21 : THE SOFTWARE.
22 : **/
23 :
24 : #ifndef STAPPLER_CORE_MEMORY_SPMEMALLOC_H_
25 : #define STAPPLER_CORE_MEMORY_SPMEMALLOC_H_
26 :
27 : #include "SPMemPoolApi.h"
28 :
29 : namespace STAPPLER_VERSIONIZED stappler::memory {
30 :
31 : struct AllocBase {
32 8876438 : void * operator new (size_t size) noexcept { return ::operator new(size); }
33 : void * operator new (size_t size, const std::nothrow_t& tag) noexcept { return ::operator new(size); }
34 42 : void * operator new (size_t size, void* ptr) noexcept { return ::operator new(size, ptr); }
35 8877253 : void operator delete(void *ptr) noexcept { return ::operator delete(ptr); }
36 : };
37 :
38 : // Root class for pool allocated objects
39 : // Use with care
40 : struct AllocPool {
41 11223895 : void *operator new(size_t size) noexcept {
42 11223895 : return pool::alloc(pool::acquire(), size);
43 : }
44 842355 : void *operator new(size_t size, pool_t *pool) noexcept {
45 842355 : return pool::alloc(pool, size);
46 : }
47 3858796 : void *operator new(size_t size, void *mem) noexcept {
48 3858796 : return mem;
49 : }
50 796325 : void operator delete(void *) noexcept {
51 : // APR doesn't require to free object's memory
52 796325 : }
53 :
54 50 : static pool_t *getCurrentPool() {
55 50 : return pool::acquire();
56 : }
57 :
58 : static bool isCustomPool(pool_t *);
59 :
60 : template <typename T>
61 : static status_t cleanupObjectFromPool(void *data);
62 :
63 : template <typename T>
64 : static void registerCleanupDestructor(T *obj, pool_t *pool);
65 : };
66 :
67 : namespace {
68 : template< class...Args> struct Allocator_SelectFirst;
69 : template< class A, class ...Args> struct Allocator_SelectFirst<A,Args...>{ using type = A; };
70 : }
71 :
72 : template <typename Type>
73 : struct Allocator_protect_construct {
74 : static constexpr bool value = false; // !std::is_scalar<Type>::value;
75 : };
76 :
77 : template <class T>
78 : class Allocator {
79 : public:
80 : using pointer = T *;
81 : using const_pointer = const T *;
82 :
83 : using void_pointer = void *;
84 : using const_void_pointer = const void *;
85 :
86 : using reference = T &;
87 : using const_reference = const T &;
88 :
89 : using value_type = T;
90 :
91 : using size_type = size_t;
92 : using difference_type = ptrdiff_t;
93 :
94 : template <class U> struct rebind { using other = Allocator<U>; };
95 :
96 : // default alignment for pool_t is 8-bit, so, we can store up to 3 flags in pool pointer
97 :
98 : enum AllocFlag : uintptr_t {
99 : FirstFlag = 1,
100 : SecondFlag = 2,
101 : ThirdFlag = 4,
102 : BitMask = 7,
103 : };
104 :
105 : private:
106 37281728 : static pool_t *pool_ptr(pool_t *p) {
107 37281728 : return (pool_t *)(uintptr_t(p) & ~toInt(BitMask));
108 : }
109 :
110 : public:
111 : // Default allocator uses pool from top of thread's AllocStack
112 30627348 : Allocator() noexcept : pool(pool::acquire()) { }
113 258856 : Allocator(pool_t *p) noexcept : pool(p) { }
114 :
115 6547975 : template<class B> Allocator(const Allocator<B> &a) noexcept : pool(a.getPool()) { }
116 : template<class B> Allocator(Allocator<B> &&a) noexcept : pool(a.getPool()) { }
117 :
118 : template<class B> Allocator<T> & operator = (const Allocator<B> &a) noexcept { pool = pool_ptr(a.pool); return *this; }
119 : template<class B> Allocator<T> & operator = (Allocator<B> &&a) noexcept { pool = pool_ptr(a.pool); return *this; }
120 :
121 1704 : T * allocate(size_t n) {
122 1704 : size_t size = sizeof(T) * n;
123 3408 : return (T *)pool::alloc(pool_ptr(pool), size);
124 : }
125 :
126 : T * __allocate(size_t &n) {
127 : size_t size = sizeof(T) * n;
128 : auto ptr = (T *)pool::alloc(pool_ptr(pool), size);
129 : n = size / sizeof(T);
130 : return ptr;
131 : }
132 :
133 10351859 : T * __allocate(size_t n, size_t &bytes) {
134 10351859 : size_t size = sizeof(T) * n;
135 10351859 : auto ptr = (T *)pool::alloc(pool_ptr(pool), size);
136 10351485 : bytes = size;
137 10351485 : return ptr;
138 : }
139 :
140 2195846 : void deallocate(T *t, size_t n) {
141 2195846 : pool::free(pool_ptr(pool), t, n * sizeof(T));
142 2196052 : }
143 :
144 1277247 : void __deallocate(T *t, size_t n, size_t bytes) {
145 1277247 : pool::free(pool_ptr(pool), t, bytes);
146 1277204 : }
147 :
148 8423228 : template<class B> inline bool operator == (const Allocator<B> &p) const noexcept { return pool_ptr(p.pool) == pool_ptr(pool); }
149 : template<class B> inline bool operator != (const Allocator<B> &p) const noexcept { return pool_ptr(p.pool) != pool_ptr(pool); }
150 :
151 : inline pointer address(reference r) const noexcept { return &r; }
152 : inline const_pointer address(const_reference r) const noexcept { return &r; }
153 :
154 : size_type max_size() const noexcept { return maxOf<size_type>(); }
155 :
156 : template <typename ...Args>
157 1731787987 : void construct(pointer p, Args &&...args) {
158 : static_assert(std::is_constructible<T, Args...>::value, "Invalid arguments for constructor");
159 : if constexpr (std::is_constructible<T, Args...>::value) {
160 : if constexpr (sizeof...(Args) == 1) {
161 : if constexpr (std::is_trivially_copyable<T>::value && std::is_convertible_v<typename Allocator_SelectFirst<Args...>::type, const T &>) {
162 558179498 : auto construct_memcpy = [] (pointer p, const T &source) {
163 558179498 : memcpy(p, &source, sizeof(T));
164 : };
165 :
166 558173413 : construct_memcpy(p, std::forward<Args>(args)...);
167 558209684 : return;
168 : }
169 : }
170 :
171 : if constexpr (Allocator_protect_construct<T>::value) {
172 : auto selfP = pool_ptr(pool);
173 : auto p = memory::pool::acquire();
174 : if (p != selfP) {
175 : memory::pool::push(selfP);
176 : new ((T*)p) T(std::forward<Args>(args)...);
177 : memory::pool::pop();
178 : return;
179 : }
180 : }
181 1173614574 : new ((T*)p) T(std::forward<Args>(args)...);
182 : }
183 1173614565 : }
184 :
185 15142178 : void destroy(pointer p) {
186 : if constexpr (!std::is_destructible<T>::value || std::is_scalar<T>::value) {
187 : // do nothing
188 : } else {
189 : if constexpr (Allocator_protect_construct<T>::value) {
190 : auto selfP = pool_ptr(pool);
191 : auto pool = memory::pool::acquire();
192 : if (pool != selfP) {
193 : memory::pool::push(selfP);
194 : do { p->~T(); } while (0);
195 : memory::pool::pop();
196 : return;
197 : }
198 : }
199 :
200 13083407 : do { p->~T(); } while (0);
201 : }
202 15142181 : }
203 :
204 16920616 : void destroy(pointer p, size_t size) {
205 : if constexpr (!std::is_destructible<T>::value || std::is_scalar<T>::value) {
206 : // do nothing
207 : } else {
208 : if constexpr (Allocator_protect_construct<T>::value) {
209 : auto selfP = pool_ptr(pool);
210 : auto pool = memory::pool::acquire();
211 : if (pool != selfP) {
212 : memory::pool::push(selfP);
213 : for (size_t i = 0; i < size; ++i) {
214 : (p + i)->~T();
215 : }
216 : memory::pool::pop();
217 : return;
218 : }
219 : }
220 :
221 2232890 : for (size_t i = 0; i < size; ++i) {
222 2023743 : (p + i)->~T();
223 : }
224 : }
225 16920617 : }
226 :
227 24644 : operator pool_t * () const noexcept { return pool_ptr(pool); }
228 6552792 : pool_t *getPool() const noexcept { return pool_ptr(pool); }
229 :
230 494137574 : void copy(T *dest, const T *source, size_t count) noexcept {
231 : if constexpr (std::is_trivially_copyable<T>::value) {
232 494137574 : memmove(dest, source, count * sizeof(T));
233 : } else {
234 : if (dest == source) {
235 : return;
236 : } else if (uintptr_t(dest) > uintptr_t(source)) {
237 : for (size_t i = count; i > 0; i--) {
238 : construct(dest + i - 1, *(source + i - 1));
239 : }
240 : } else {
241 : for (size_t i = 0; i < count; i++) {
242 : construct(dest + i, *(source + i));
243 : }
244 : }
245 : }
246 494137574 : }
247 :
248 19463712 : void copy_rewrite(T *dest, size_t dcount, const T *source, size_t count) noexcept {
249 : if constexpr (std::is_trivially_copyable<T>::value) {
250 19077812 : memmove(dest, source, count * sizeof(T));
251 : } else {
252 385900 : if (dest == source) {
253 3475 : return;
254 382425 : } else if (uintptr_t(dest) > uintptr_t(source)) {
255 201423 : size_t i = count;
256 201423 : size_t m = std::min(count, dcount);
257 850935 : for (; i > m; i--) {
258 649512 : construct(dest + i - 1, *(source + i - 1));
259 : }
260 316814 : for (; i > 0; i--) {
261 115391 : destroy(dest + i - 1);
262 115391 : construct(dest + i - 1, *(source + i - 1));
263 : }
264 : } else {
265 181002 : size_t i = 0;
266 181002 : size_t m = std::min(count, dcount);
267 327236 : for (; i < m; ++ i) {
268 146234 : destroy(dest + i);
269 146234 : construct(dest + i, *(source + i));
270 : }
271 539740 : for (; i < count; ++ i) {
272 358738 : construct(dest + i, *(source + i));
273 : }
274 : }
275 : }
276 19077812 : }
277 :
278 1406419 : void move(T *dest, T *source, size_t count) noexcept {
279 : if constexpr (std::is_trivially_copyable<T>::value) {
280 1218948 : memmove(dest, source, count * sizeof(T));
281 : } else {
282 187471 : if (dest == source) {
283 0 : return;
284 187471 : } else if (uintptr_t(dest) > uintptr_t(source)) {
285 872600 : for (size_t i = count; i > 0; i--) {
286 713787 : construct(dest + i - 1, std::move(*(source + i - 1)));
287 713787 : destroy(source + i - 1);
288 : }
289 : } else {
290 293998 : for (size_t i = 0; i < count; i++) {
291 265340 : construct(dest + i, std::move(*(source + i)));
292 265340 : destroy(source + i);
293 : }
294 : }
295 : }
296 1218948 : }
297 7699551 : void move_rewrite(T *dest, size_t dcount, T *source, size_t count) noexcept {
298 : if constexpr (std::is_trivially_copyable<T>::value) {
299 7633026 : memmove(dest, source, count * sizeof(T));
300 : } else {
301 66525 : if (dest == source) {
302 0 : return;
303 66525 : } else if (uintptr_t(dest) > uintptr_t(source)) {
304 64386 : size_t i = count;
305 64386 : size_t m = std::min(count, dcount);
306 128772 : for (; i > m; i--) {
307 64386 : construct(dest + i - 1, std::move(*(source + i - 1)));
308 64386 : destroy(source + i - 1);
309 : }
310 64386 : for (; i > 0; i--) {
311 0 : destroy(dest + i - 1);
312 0 : construct(dest + i - 1, std::move(*(source + i - 1)));
313 0 : destroy(source + i - 1);
314 : }
315 : } else {
316 2139 : size_t i = 0;
317 2139 : size_t m = std::min(count, dcount);
318 2139 : for (; i < m; ++ i) {
319 0 : destroy(dest + i);
320 0 : construct(dest + i, std::move(*(source + i)));
321 0 : destroy(source + i);
322 : }
323 4278 : for (; i < count; ++ i) {
324 2139 : construct(dest + i, std::move(*(source + i)));
325 2139 : destroy(source + i);
326 : }
327 : }
328 : }
329 7633026 : }
330 :
331 4748088222 : bool test(AllocFlag f) const { return (uintptr_t(pool) & toInt(f)) != uintptr_t(0); }
332 25340335 : void set(AllocFlag f) { pool = (pool_t *)(uintptr_t(pool) | toInt(f)); }
333 39729088 : void reset(AllocFlag f) {pool = (pool_t *)(uintptr_t(pool) & ~toInt(f)); }
334 : void flip(AllocFlag f) { pool = (pool_t *)(uintptr_t(pool) ^ toInt(f)); }
335 :
336 : private:
337 : pool_t *pool = nullptr;
338 : };
339 :
340 : template <typename Value>
341 : struct Storage {
342 : struct Image { Value _value; };
343 :
344 : alignas(__alignof__(Image::_value)) uint8_t _storage[sizeof(Value)];
345 :
346 : Storage() noexcept { }
347 : Storage(nullptr_t) noexcept {}
348 :
349 45385754 : void * addr() noexcept { return static_cast<void *>(&_storage); }
350 457560155 : const void * addr() const noexcept { return static_cast<const void *>(&_storage); }
351 :
352 45383177 : Value * ptr() noexcept { return static_cast<Value *>(addr()); }
353 457654595 : const Value * ptr() const noexcept { return static_cast<const Value *>(addr()); }
354 :
355 : Value & ref() noexcept { return *ptr(); }
356 138303636 : const Value & ref() const noexcept { return *ptr(); }
357 : };
358 :
359 : template <typename T>
360 125 : inline status_t AllocPool::cleanupObjectFromPool(void *data) {
361 125 : delete ((T *)data);
362 125 : return SUCCESS;
363 : }
364 :
365 : template <typename T>
366 150 : inline void AllocPool::registerCleanupDestructor(T *obj, pool_t *pool) {
367 150 : pool::pre_cleanup_register(pool, (void *)obj, &(cleanupObjectFromPool<T>));
368 150 : }
369 :
370 : }
371 :
372 : #endif /* STAPPLER_CORE_MEMORY_SPMEMALLOC_H_ */
|
