Line data Source code
1 : /**
2 : Copyright (c) 2017-2022 Roman Katuntsev <sbkarr@stappler.org>
3 : Copyright (c) 2023 Stappler LLC <admin@stappler.dev>
4 :
5 : Permission is hereby granted, free of charge, to any person obtaining a copy
6 : of this software and associated documentation files (the "Software"), to deal
7 : in the Software without restriction, including without limitation the rights
8 : to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 : copies of the Software, and to permit persons to whom the Software is
10 : furnished to do so, subject to the following conditions:
11 :
12 : The above copyright notice and this permission notice shall be included in
13 : all copies or substantial portions of the Software.
14 :
15 : THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 : IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 : FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 : AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 : LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 : OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21 : THE SOFTWARE.
22 : **/
23 :
24 : #ifndef STAPPLER_CORE_MEMORY_SPMEMSTORAGEMEMIMPL_H_
25 : #define STAPPLER_CORE_MEMORY_SPMEMSTORAGEMEMIMPL_H_
26 :
27 : #include "SPMemAlloc.h"
28 :
29 : namespace STAPPLER_VERSIONIZED stappler::memory::impl {
30 :
31 : // small object optimization type (based on SSO-23: https://github.com/elliotgoodrich/SSO-23)
32 : template <typename Type, size_t ByteCount>
33 : struct mem_small {
34 : using self = mem_small<Type, ByteCount>;
35 : using pointer = Type *;
36 : using const_pointer = const Type *;
37 : using size_type = size_t;
38 : using allocator = Allocator<Type>;
39 :
40 294569252 : static constexpr size_type max_capacity() { return (sizeof(Type) < ByteCount) ? ((ByteCount - 1) / sizeof(Type)) : 0; }
41 :
42 16612093 : void assign(allocator &a, const_pointer ptr, size_type s) {
43 16612093 : const auto current = size();
44 16612092 : a.copy_rewrite(data(), size(), ptr, s);
45 16612094 : if (current > s) {
46 15997141 : a.destroy(data() + s, current - s);
47 : }
48 16612095 : set_size(s);
49 16612092 : }
50 :
51 7263198 : void move_assign(allocator &a, pointer source, size_type count) {
52 7263198 : const auto current = size();
53 7263200 : a.move_rewrite(data(), current, source, count);
54 7263197 : if (current > count) {
55 0 : a.destroy(data() + count, current - count);
56 : }
57 7263197 : set_size(count);
58 7263195 : }
59 :
60 15991475 : void force_clear() {
61 15991475 : set_size(0);
62 15991532 : }
63 :
64 40240986 : void drop_unused() {
65 40240986 : const auto unused = storage[ByteCount - 1];
66 40241084 : if (unused < max_capacity()) {
67 24249559 : const auto s = max_capacity() - unused;
68 : // data is already garbage, bypass -Wclass-memaccess
69 24249562 : memset((void *)(storage.data() + s * sizeof(Type)), 0, unused * sizeof(Type));
70 : }
71 40241090 : }
72 :
73 40240995 : void set_size(size_t s) {
74 40240995 : storage[ByteCount - 1] = uint8_t(max_capacity() - s);
75 40240996 : drop_unused();
76 40241082 : }
77 :
78 5921783 : size_t modify_size(intptr_t diff) {
79 5921783 : storage[ByteCount - 1] = uint8_t(max_capacity() - (size() + diff));
80 5921951 : return size();
81 : }
82 :
83 144655328 : size_t size() const { return max_capacity() - storage[ByteCount - 1]; }
84 :
85 11681222 : size_t capacity() const { return max_capacity(); }
86 :
87 79769329 : pointer data() { return (pointer)storage.data(); }
88 67787095 : const_pointer data() const { return (const_pointer)storage.data(); }
89 :
90 : std::array<uint8_t, ByteCount> storage;
91 : };
92 :
93 : template <typename Type, size_t Extra = 0>
94 : class mem_large {
95 : public:
96 : using self = mem_large<Type, Extra>;
97 : using pointer = Type *;
98 : using const_pointer = const Type *;
99 : using size_type = size_t;
100 : using allocator = Allocator<Type>;
101 :
102 28517909 : mem_large() = default;
103 : mem_large(const self &) = default;
104 : self & operator= (const self &other) = default;
105 :
106 0 : mem_large(self &&other) {
107 0 : _ptr = other._ptr;
108 0 : _used = other._used;
109 0 : _allocated = other._allocated;
110 0 : other._ptr = nullptr;
111 0 : other._used = 0;
112 0 : other._allocated = 0;
113 0 : }
114 :
115 29648563 : self & operator= (self &&other) {
116 29648563 : _ptr = other._ptr;
117 29648563 : _used = other._used;
118 29648563 : _allocated = other._allocated;
119 29648563 : other._ptr = nullptr;
120 29648563 : other._used = 0;
121 29648563 : other._allocated = 0;
122 29648563 : return *this;
123 : }
124 :
125 2846868 : void assign(allocator &a, const_pointer ptr, size_type count) {
126 2846868 : if (_allocated < count) {
127 2846868 : reserve(a, count);
128 : }
129 2846868 : a.copy_rewrite(data(), size(), ptr, count);
130 2846868 : if (_used > count) {
131 0 : a.destroy(data() + count, _used - count);
132 : }
133 2846868 : _used = count;
134 2846868 : drop_unused();
135 2846868 : }
136 :
137 436355 : void move_assign(allocator &a, pointer ptr, size_type count) {
138 436355 : if (_allocated < count) {
139 0 : reserve(a, count);
140 : }
141 436355 : a.move_rewrite(_ptr, _used, ptr, count);
142 436376 : if (_used > count) {
143 0 : a.destroy(data() + count, _used - count);
144 : }
145 436376 : _used = count;
146 436376 : drop_unused();
147 436393 : }
148 :
149 : void assign_weak(pointer ptr, size_type s) {
150 : _ptr = ptr;
151 : _used = s;
152 : _allocated = 0;
153 : }
154 :
155 42774 : void assign_weak(const_pointer ptr, size_type s) {
156 42774 : _ptr = const_cast<pointer>(ptr);
157 42774 : _used = s;
158 42774 : _allocated = 0;
159 42774 : }
160 :
161 : void assign_mem(pointer ptr, size_type s, size_type nalloc) {
162 : _ptr = ptr;
163 : _used = s;
164 : _allocated = nalloc - Extra;
165 : }
166 :
167 : bool is_weak() const noexcept {
168 : return _used > 0 && _allocated == 0;
169 : }
170 :
171 5102748 : void reserve(allocator &a, size_type s) {
172 5102748 : grow_alloc(a, s);
173 5101856 : drop_unused();
174 5102061 : }
175 :
176 8580276 : void clear_dealloc(allocator &a) {
177 8580276 : if (_ptr) {
178 955165 : if (_used) {
179 884135 : a.destroy(_ptr, _used);
180 : }
181 955148 : if (_allocated) {
182 912388 : a.deallocate(_ptr, _allocated + Extra);
183 : }
184 : }
185 8580202 : _ptr = nullptr;
186 8580202 : _used = 0;
187 8580202 : _allocated = 0;
188 8580202 : }
189 :
190 7777095 : void force_clear() {
191 7777095 : _ptr = nullptr;
192 7777095 : _used = 0;
193 7777095 : _allocated = 0;
194 7777095 : }
195 :
196 : pointer extract() {
197 : auto ret = _ptr;
198 : force_clear();
199 : return ret;
200 : }
201 :
202 8392884 : void drop_unused() {
203 8392884 : if (_allocated > 0 && _allocated >= _used && _ptr) {
204 : // data is already garbage, bypass -Wclass-memaccess
205 8389725 : memset((void *)(_ptr + _used), 0, (_allocated - _used + Extra) * sizeof(Type));
206 : }
207 8392884 : }
208 :
209 5102534 : void grow_alloc(allocator &a, size_type newsize) {
210 5102534 : size_t alloc_size = newsize + Extra;
211 :
212 : // use extra memory if provided by allocator
213 5102534 : size_t allocated = 0; // real memory block size returned
214 5102534 : auto ptr = a.__allocate(alloc_size, allocated);
215 5101987 : alloc_size = allocated / sizeof(Type);
216 :
217 5101987 : if (_used > 0 && _ptr) {
218 1282970 : a.move(ptr, _ptr, _used);
219 : }
220 :
221 5101848 : if (_ptr && _allocated > 0) {
222 1282866 : a.deallocate(_ptr, _allocated);
223 : }
224 :
225 5101878 : _ptr = ptr;
226 5101878 : _allocated = alloc_size - Extra;
227 5101878 : }
228 :
229 1038514509 : size_t modify_size(intptr_t diff) {
230 1038514509 : _used += diff;
231 1038514509 : return _used;
232 : }
233 :
234 8444980 : void set_size(size_t s) {
235 8444980 : if (s < _used) {
236 3978 : _used = s;
237 3978 : drop_unused();
238 : } else {
239 8441002 : _used = s;
240 : }
241 8444980 : }
242 :
243 1209182007 : size_t size() const noexcept { return _used; }
244 545220531 : size_t capacity() const noexcept { return _allocated; }
245 :
246 1813316471 : pointer data() noexcept { return _ptr; }
247 14905601 : const_pointer data() const noexcept { return _ptr; }
248 :
249 23112804 : bool empty() const noexcept { return _ptr == nullptr || _used == 0; }
250 :
251 : protected:
252 : pointer _ptr = nullptr;
253 : size_type _used = 0; // in elements
254 : size_type _allocated = 0; // in elements
255 : };
256 :
257 : template <typename Type, size_t Extra, bool UseSoo>
258 : class mem_soo_iface;
259 :
260 : template <typename Type, size_t Extra>
261 : class mem_soo_iface<Type, Extra, false> : public mem_large<Type, Extra> {
262 : public:
263 : using base = mem_large<Type, Extra>;
264 : using pointer = typename base::pointer;
265 : using const_pointer = typename base::const_pointer;
266 : using size_type = typename base::size_type;
267 : using allocator = typename base::allocator;
268 :
269 : static constexpr size_type get_soo_size() { return 0; }
270 :
271 413664 : mem_soo_iface(const allocator &alloc) : _allocator(alloc) { }
272 :
273 360406 : ~mem_soo_iface() noexcept { clear_dealloc(_allocator); }
274 :
275 4750 : void assign(const_pointer ptr, size_type size) {
276 4750 : reserve(size, false);
277 4750 : _allocator.copy_rewrite(_ptr, _used, ptr, size);
278 4750 : if (_used > size) {
279 0 : _allocator.destroy(data() + size, _used - size);
280 : }
281 4750 : _used = size;
282 4750 : drop_unused();
283 4750 : }
284 :
285 : using base::assign_weak;
286 : using base::assign_mem;
287 : using base::is_weak;
288 :
289 : using base::data;
290 : using base::size;
291 : using base::capacity;
292 :
293 : // reserve memory block, optimal for realloc/free
294 : // useful for small temporary buffers
295 : // this memory block can be reused by next temporary buffer of same size
296 : // so, no pool memory will be leaked
297 625 : pointer reserve_block_optimal() {
298 625 : auto target = mempool::custom::BlockThreshold / sizeof(Type) + 1;
299 625 : return reserve(target);
300 : }
301 :
302 2118043 : pointer reserve(size_type s, bool grow = false) {
303 2118043 : if (s > 0 && s > _allocated) {
304 236814 : auto newmem = (grow ? max(s, _allocated * 2) : s);
305 236814 : base::reserve(_allocator, newmem);
306 : }
307 2118042 : return _ptr;
308 : }
309 :
310 843555 : void clear() {
311 843555 : if (_used > 0 && _allocated > 0) {
312 29449 : if (_ptr) {
313 29449 : _allocator.destroy(_ptr, _used);
314 : }
315 : } else {
316 814106 : if (_allocated == 0) {
317 12399 : _ptr = nullptr;
318 : }
319 : }
320 843555 : _used = 0;
321 843555 : }
322 :
323 : using base::force_clear;
324 : using base::extract;
325 :
326 : protected:
327 78875 : void perform_move(mem_soo_iface &&other) {
328 78875 : *(static_cast<base *>(this)) = std::move(other);
329 78874 : }
330 :
331 : using base::clear_dealloc;
332 : using base::modify_size;
333 : using base::set_size;
334 : using base::drop_unused;
335 :
336 : using base::_ptr;
337 : using base::_allocated;
338 : using base::_used;
339 : allocator _allocator;
340 : };
341 :
342 : template <typename Type, size_t Extra>
343 : class mem_soo_iface<Type, Extra, true> {
344 : public:
345 : using pointer = Type *;
346 : using const_pointer = const Type *;
347 :
348 : using size_type = size_t;
349 : using allocator = Allocator<Type>;
350 :
351 : using large_mem = mem_large<Type, Extra>;
352 : using small_mem = mem_small<Type, sizeof(large_mem)>;
353 :
354 53325 : static constexpr size_type get_soo_size() { return small_mem::max_capacity(); }
355 :
356 28104204 : mem_soo_iface(const allocator &alloc) : _allocator(alloc) {
357 28104204 : set_large_flag();
358 28104271 : _large = large_mem();
359 28104664 : }
360 :
361 9287744 : ~mem_soo_iface() noexcept {
362 9287744 : if (is_large()) {
363 7067522 : _large.clear_dealloc(_allocator);
364 : }
365 9287655 : }
366 :
367 19900011 : void assign(const_pointer ptr, size_type size) {
368 19900011 : if (!ptr || size == 0) {
369 441044 : clear();
370 19458967 : } else if (size <= small_mem::max_capacity() && (is_small() || empty())) {
371 16612100 : set_small_flag();
372 16612097 : _small.assign(_allocator, ptr, size);
373 2846867 : } else if (size > small_mem::max_capacity() && (is_large() || empty())) {
374 2846868 : set_large_flag();
375 2846868 : _large.assign(_allocator, ptr, size);
376 : } else {
377 0 : if (is_small()) {
378 0 : large_mem new_large;
379 0 : new_large.assign(_allocator, ptr, size);
380 0 : set_large_flag_force();
381 0 : _large = std::move(new_large);
382 : } else {
383 0 : large_mem old_large(std::move(_large));
384 0 : _small.force_clear();
385 0 : set_small_flag();
386 0 : _small.assign(_allocator, ptr, size);
387 0 : old_large.clear_dealloc(_allocator);
388 : }
389 : }
390 19900009 : }
391 :
392 : void assign_weak(pointer ptr, size_type s) {
393 : if (s <= small_mem::max_capacity()) {
394 : assign(ptr, s);
395 : } else {
396 : set_large_flag_force();
397 : _large.assign_weak(ptr, s);
398 : }
399 : }
400 :
401 61820 : void assign_weak(const_pointer ptr, size_type s) {
402 61820 : if (s <= small_mem::max_capacity()) {
403 19049 : assign(ptr, s);
404 : } else {
405 42773 : set_large_flag_force();
406 42773 : _large.assign_weak(ptr, s);
407 : }
408 61820 : }
409 :
410 : void assign_mem(pointer ptr, size_type s, size_type nalloc) {
411 : set_large_flag_force();
412 : _large.assign_mem(ptr, s, nalloc);
413 : }
414 :
415 : // Проверка, владеем ли блоком памяти
416 : bool is_weak() const noexcept {
417 : return is_large() && _large.is_weak();
418 : }
419 :
420 548995554 : pointer reserve(size_type s, bool grow = false) {
421 548995554 : const auto _allocated = capacity();
422 548990967 : const auto _used = size();
423 548995516 : if (s > _allocated) {
424 3483708 : if (s <= small_mem::max_capacity() && _used <= small_mem::max_capacity()) {
425 1465084 : if (_allocated == 0) { // память пустая или CoW
426 1465091 : set_small_flag();
427 1465110 : if (_large.data() != nullptr) { // CoW
428 0 : _small.move_assign(_allocator, _large.data(), _large.size());
429 : } else { // empty
430 1465106 : _small.force_clear();
431 : }
432 : }
433 1465139 : return _small.data();
434 2019262 : } else if (s > 0) {
435 2019238 : auto newmem = (grow ? max(s, _allocated * 2) : s);
436 2019257 : if (is_small() && newmem > small_mem::max_capacity()) {
437 436519 : large_mem new_large;
438 436519 : new_large.reserve(_allocator, newmem);
439 436384 : new_large.move_assign(_allocator, _small.data(), _small.size());
440 436369 : set_large_flag();
441 436394 : _large = std::move(new_large);
442 : } else {
443 1583087 : _large.reserve(_allocator, newmem);
444 : }
445 2018816 : return _large.data();
446 : }
447 : }
448 545511832 : return data();
449 : }
450 :
451 7906630 : void clear() {
452 7906630 : const auto _used = size();
453 7906621 : const auto _allocated = capacity();
454 7906616 : auto _ptr = data();
455 7906610 : if (_used > 0 && _allocated > 0 && _ptr) {
456 6075 : _allocator.destroy(_ptr, _used);
457 : } else {
458 7900535 : if (_allocated == 0 && is_large()) { // проверяем и очищаем CoW
459 506747 : _large.force_clear();
460 : }
461 : }
462 :
463 7906610 : if (is_large()) {
464 637617 : _large.set_size(0);
465 : } else {
466 7268998 : set_large_flag();
467 7268998 : _large.force_clear();
468 : }
469 7906616 : }
470 :
471 1350 : void force_clear() {
472 1350 : if (is_small()) {
473 1050 : set_large_flag();
474 : }
475 1350 : _large.force_clear();
476 1350 : }
477 :
478 : pointer extract() {
479 : if (is_large()) {
480 : return _large.extract();
481 : } else {
482 : auto s = _small.size();
483 : auto ptr = _allocator.allocate(s + Extra);
484 : _allocator.move(ptr, _small.data(), s);
485 : if constexpr (Extra) {
486 : // zero-terminated, bypass -Wclass-memaccess
487 : memset((void *)(ptr + s), 0, Extra * sizeof(Type));
488 : }
489 : force_clear();
490 : return ptr;
491 : }
492 : }
493 :
494 1746785602 : pointer data() noexcept { return is_large() ? _large.data() : _small.data(); }
495 75614137 : const_pointer data() const noexcept { return is_large() ? _large.data() : _small.data(); }
496 :
497 1244470609 : size_type size() const noexcept { return is_large() ? _large.size() : _small.size(); }
498 556890706 : size_type capacity() const noexcept { return is_large() ? _large.capacity() : _small.capacity(); }
499 :
500 23801973 : bool empty() const noexcept { return is_large() ? _large.empty() : (_small.size() == 0); }
501 :
502 : protected:
503 8285832 : void perform_move(mem_soo_iface &&other) {
504 8285832 : if (other.is_small()) {
505 7259249 : set_small_flag();
506 7259245 : _small.force_clear();
507 7259246 : _small.move_assign(this->_allocator, other.data(), other.size());
508 7259238 : other._small.force_clear();
509 : } else {
510 1026583 : set_large_flag();
511 1026589 : _large = std::move(other._large);
512 : }
513 8285834 : }
514 :
515 8325802 : void clear_dealloc(allocator &a) {
516 8325802 : if (is_large()) {
517 1065611 : _large.clear_dealloc(a);
518 : } else {
519 7260189 : clear();
520 : }
521 8325803 : }
522 :
523 1040631553 : size_type modify_size(intptr_t diff) {
524 1040631553 : return is_large() ? _large.modify_size(diff) : _small.modify_size(diff);
525 : }
526 :
527 8181443 : void set_size(size_type s) {
528 8181443 : if (is_large()) {
529 7807123 : _large.set_size(s);
530 : } else {
531 374319 : _small.set_size(s);
532 : }
533 8181440 : }
534 :
535 : protected:
536 : allocator _allocator;
537 :
538 : private:
539 26924408 : bool is_small() const { return this->_allocator.test(allocator::FirstFlag); }
540 4721257334 : bool is_large() const { return !this->_allocator.test(allocator::FirstFlag); }
541 :
542 39729044 : void set_large_flag() {
543 39729044 : this->_allocator.reset(allocator::FirstFlag);
544 39729319 : }
545 42774 : void set_large_flag_force() {
546 42774 : clear(); // discard content
547 42773 : set_large_flag();
548 42773 : }
549 :
550 25340357 : void set_small_flag() {
551 25340357 : this->_allocator.set(allocator::FirstFlag);
552 25340376 : }
553 :
554 : union {
555 : large_mem _large;
556 : small_mem _small;
557 : };
558 : };
559 :
560 : }
561 :
562 : #endif /* STAPPLER_CORE_MEMORY_SPMEMSTORAGEMEMIMPL_H_ */
|
