1// <memory_resource> -*- C++ -*- 2 3// Copyright (C) 2018-2020 Free Software Foundation, Inc. 4// 5// This file is part of the GNU ISO C++ Library. This library is free 6// software; you can redistribute it and/or modify it under the 7// terms of the GNU General Public License as published by the 8// Free Software Foundation; either version 3, or (at your option) 9// any later version. 10 11// This library is distributed in the hope that it will be useful, 12// but WITHOUT ANY WARRANTY; without even the implied warranty of 13// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14// GNU General Public License for more details. 15 16// Under Section 7 of GPL version 3, you are granted additional 17// permissions described in the GCC Runtime Library Exception, version 18// 3.1, as published by the Free Software Foundation. 19 20// You should have received a copy of the GNU General Public License and 21// a copy of the GCC Runtime Library Exception along with this program; 22// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see 23// <http://www.gnu.org/licenses/>. 24 25/** @file include/memory_resource 26 * This is a Standard C++ Library header. 27 */ 28 29#ifndef _GLIBCXX_MEMORY_RESOURCE 30#define _GLIBCXX_MEMORY_RESOURCE 1 31 32#pragma GCC system_header 33 34#if __cplusplus >= 201703L 35 36#include <memory> // align, allocator_arg_t, __uses_alloc 37#include <utility> // pair, index_sequence 38#include <vector> // vector 39#include <cstddef> // size_t, max_align_t, byte 40#include <shared_mutex> // shared_mutex 41#include <bits/functexcept.h> 42#include <ext/numeric_traits.h> 43#include <debug/assertions.h> 44 45namespace std _GLIBCXX_VISIBILITY(default) 46{ 47_GLIBCXX_BEGIN_NAMESPACE_VERSION 48namespace pmr 49{ 50#ifdef _GLIBCXX_HAS_GTHREADS 51 // Header and all contents are present. 52# define __cpp_lib_memory_resource 201603 53#else 54 // The pmr::synchronized_pool_resource type is missing. 55# define __cpp_lib_memory_resource 1 56#endif 57 58 class memory_resource; 59 60#if __cplusplus == 201703L 61 template<typename _Tp> 62 class polymorphic_allocator; 63#else // C++20 64# define __cpp_lib_polymorphic_allocator 201902L 65 template<typename _Tp = std::byte> 66 class polymorphic_allocator; 67#endif 68 69 // Global memory resources 70 memory_resource* new_delete_resource() noexcept; 71 memory_resource* null_memory_resource() noexcept; 72 memory_resource* set_default_resource(memory_resource* __r) noexcept; 73 memory_resource* get_default_resource() noexcept 74 __attribute__((__returns_nonnull__)); 75 76 // Pool resource classes 77 struct pool_options; 78#ifdef _GLIBCXX_HAS_GTHREADS 79 class synchronized_pool_resource; 80#endif 81 class unsynchronized_pool_resource; 82 class monotonic_buffer_resource; 83 84 /// Class memory_resource 85 class memory_resource 86 { 87 static constexpr size_t _S_max_align = alignof(max_align_t); 88 89 public: 90 memory_resource() = default; 91 memory_resource(const memory_resource&) = default; 92 virtual ~memory_resource(); // key function 93 94 memory_resource& operator=(const memory_resource&) = default; 95 96 [[nodiscard]] 97 void* 98 allocate(size_t __bytes, size_t __alignment = _S_max_align) 99 __attribute__((__returns_nonnull__,__alloc_size__(2),__alloc_align__(3))) 100 { return do_allocate(__bytes, __alignment); } 101 102 void 103 deallocate(void* __p, size_t __bytes, size_t __alignment = _S_max_align) 104 __attribute__((__nonnull__)) 105 { return do_deallocate(__p, __bytes, __alignment); } 106 107 bool 108 is_equal(const memory_resource& __other) const noexcept 109 { return do_is_equal(__other); } 110 111 private: 112 virtual void* 113 do_allocate(size_t __bytes, size_t __alignment) = 0; 114 115 virtual void 116 do_deallocate(void* __p, size_t __bytes, size_t __alignment) = 0; 117 118 virtual bool 119 do_is_equal(const memory_resource& __other) const noexcept = 0; 120 }; 121 122 inline bool 123 operator==(const memory_resource& __a, const memory_resource& __b) noexcept 124 { return &__a == &__b || __a.is_equal(__b); } 125 126#if __cpp_impl_three_way_comparison < 201907L 127 inline bool 128 operator!=(const memory_resource& __a, const memory_resource& __b) noexcept 129 { return !(__a == __b); } 130#endif 131 132 // C++17 23.12.3 Class template polymorphic_allocator 133 template<typename _Tp> 134 class polymorphic_allocator 135 { 136 // _GLIBCXX_RESOLVE_LIB_DEFECTS 137 // 2975. Missing case for pair construction in polymorphic allocators 138 template<typename _Up> 139 struct __not_pair { using type = void; }; 140 141 template<typename _Up1, typename _Up2> 142 struct __not_pair<pair<_Up1, _Up2>> { }; 143 144 public: 145 using value_type = _Tp; 146 147 polymorphic_allocator() noexcept 148 : _M_resource(get_default_resource()) 149 { } 150 151 polymorphic_allocator(memory_resource* __r) noexcept 152 __attribute__((__nonnull__)) 153 : _M_resource(__r) 154 { _GLIBCXX_DEBUG_ASSERT(__r); } 155 156 polymorphic_allocator(const polymorphic_allocator& __other) = default; 157 158 template<typename _Up> 159 polymorphic_allocator(const polymorphic_allocator<_Up>& __x) noexcept 160 : _M_resource(__x.resource()) 161 { } 162 163 polymorphic_allocator& 164 operator=(const polymorphic_allocator&) = delete; 165 166 [[nodiscard]] 167 _Tp* 168 allocate(size_t __n) 169 __attribute__((__returns_nonnull__)) 170 { 171 if (__n > (__gnu_cxx::__int_traits<size_t>::__max / sizeof(_Tp))) 172 _GLIBCXX_THROW_OR_ABORT(bad_array_new_length()); 173 return static_cast<_Tp*>(_M_resource->allocate(__n * sizeof(_Tp), 174 alignof(_Tp))); 175 } 176 177 void 178 deallocate(_Tp* __p, size_t __n) noexcept 179 __attribute__((__nonnull__)) 180 { _M_resource->deallocate(__p, __n * sizeof(_Tp), alignof(_Tp)); } 181 182#if __cplusplus > 201703L 183 [[nodiscard]] void* 184 allocate_bytes(size_t __nbytes, 185 size_t __alignment = alignof(max_align_t)) 186 { return _M_resource->allocate(__nbytes, __alignment); } 187 188 void 189 deallocate_bytes(void* __p, size_t __nbytes, 190 size_t __alignment = alignof(max_align_t)) 191 { _M_resource->deallocate(__p, __nbytes, __alignment); } 192 193 template<typename _Up> 194 [[nodiscard]] _Up* 195 allocate_object(size_t __n = 1) 196 { 197 if ((__gnu_cxx::__int_traits<size_t>::__max / sizeof(_Up)) < __n) 198 _GLIBCXX_THROW_OR_ABORT(bad_array_new_length()); 199 return static_cast<_Up*>(allocate_bytes(__n * sizeof(_Up), 200 alignof(_Up))); 201 } 202 203 template<typename _Up> 204 void 205 deallocate_object(_Up* __p, size_t __n = 1) 206 { deallocate_bytes(__p, __n * sizeof(_Up), alignof(_Up)); } 207 208 template<typename _Up, typename... _CtorArgs> 209 [[nodiscard]] _Up* 210 new_object(_CtorArgs&&... __ctor_args) 211 { 212 _Up* __p = allocate_object<_Up>(); 213 __try 214 { 215 construct(__p, std::forward<_CtorArgs>(__ctor_args)...); 216 } 217 __catch (...) 218 { 219 deallocate_object(__p); 220 __throw_exception_again; 221 } 222 return __p; 223 } 224 225 template<typename _Up> 226 void 227 delete_object(_Up* __p) 228 { 229 destroy(__p); 230 deallocate_object(__p); 231 } 232#endif // C++2a 233 234#if __cplusplus == 201703L 235 template<typename _Tp1, typename... _Args> 236 __attribute__((__nonnull__)) 237 typename __not_pair<_Tp1>::type 238 construct(_Tp1* __p, _Args&&... __args) 239 { 240 // _GLIBCXX_RESOLVE_LIB_DEFECTS 241 // 2969. polymorphic_allocator::construct() shouldn't pass resource() 242 using __use_tag 243 = std::__uses_alloc_t<_Tp1, polymorphic_allocator, _Args...>; 244 if constexpr (is_base_of_v<__uses_alloc0, __use_tag>) 245 ::new(__p) _Tp1(std::forward<_Args>(__args)...); 246 else if constexpr (is_base_of_v<__uses_alloc1_, __use_tag>) 247 ::new(__p) _Tp1(allocator_arg, *this, 248 std::forward<_Args>(__args)...); 249 else 250 ::new(__p) _Tp1(std::forward<_Args>(__args)..., *this); 251 } 252 253 template<typename _Tp1, typename _Tp2, 254 typename... _Args1, typename... _Args2> 255 __attribute__((__nonnull__)) 256 void 257 construct(pair<_Tp1, _Tp2>* __p, piecewise_construct_t, 258 tuple<_Args1...> __x, tuple<_Args2...> __y) 259 { 260 auto __x_tag = 261 __use_alloc<_Tp1, polymorphic_allocator, _Args1...>(*this); 262 auto __y_tag = 263 __use_alloc<_Tp2, polymorphic_allocator, _Args2...>(*this); 264 index_sequence_for<_Args1...> __x_i; 265 index_sequence_for<_Args2...> __y_i; 266 267 ::new(__p) pair<_Tp1, _Tp2>(piecewise_construct, 268 _S_construct_p(__x_tag, __x_i, __x), 269 _S_construct_p(__y_tag, __y_i, __y)); 270 } 271 272 template<typename _Tp1, typename _Tp2> 273 __attribute__((__nonnull__)) 274 void 275 construct(pair<_Tp1, _Tp2>* __p) 276 { this->construct(__p, piecewise_construct, tuple<>(), tuple<>()); } 277 278 template<typename _Tp1, typename _Tp2, typename _Up, typename _Vp> 279 __attribute__((__nonnull__)) 280 void 281 construct(pair<_Tp1, _Tp2>* __p, _Up&& __x, _Vp&& __y) 282 { 283 this->construct(__p, piecewise_construct, 284 forward_as_tuple(std::forward<_Up>(__x)), 285 forward_as_tuple(std::forward<_Vp>(__y))); 286 } 287 288 template <typename _Tp1, typename _Tp2, typename _Up, typename _Vp> 289 __attribute__((__nonnull__)) 290 void 291 construct(pair<_Tp1, _Tp2>* __p, const std::pair<_Up, _Vp>& __pr) 292 { 293 this->construct(__p, piecewise_construct, 294 forward_as_tuple(__pr.first), 295 forward_as_tuple(__pr.second)); 296 } 297 298 template<typename _Tp1, typename _Tp2, typename _Up, typename _Vp> 299 __attribute__((__nonnull__)) 300 void 301 construct(pair<_Tp1, _Tp2>* __p, pair<_Up, _Vp>&& __pr) 302 { 303 this->construct(__p, piecewise_construct, 304 forward_as_tuple(std::forward<_Up>(__pr.first)), 305 forward_as_tuple(std::forward<_Vp>(__pr.second))); 306 } 307#else 308 template<typename _Tp1, typename... _Args> 309 __attribute__((__nonnull__)) 310 void 311 construct(_Tp1* __p, _Args&&... __args) 312 { 313 std::uninitialized_construct_using_allocator(__p, *this, 314 std::forward<_Args>(__args)...); 315 } 316#endif 317 318 template<typename _Up> 319 __attribute__((__nonnull__)) 320 void 321 destroy(_Up* __p) 322 { __p->~_Up(); } 323 324 polymorphic_allocator 325 select_on_container_copy_construction() const noexcept 326 { return polymorphic_allocator(); } 327 328 memory_resource* 329 resource() const noexcept 330 __attribute__((__returns_nonnull__)) 331 { return _M_resource; } 332 333 private: 334 using __uses_alloc1_ = __uses_alloc1<polymorphic_allocator>; 335 using __uses_alloc2_ = __uses_alloc2<polymorphic_allocator>; 336 337 template<typename _Ind, typename... _Args> 338 static tuple<_Args&&...> 339 _S_construct_p(__uses_alloc0, _Ind, tuple<_Args...>& __t) 340 { return std::move(__t); } 341 342 template<size_t... _Ind, typename... _Args> 343 static tuple<allocator_arg_t, polymorphic_allocator, _Args&&...> 344 _S_construct_p(__uses_alloc1_ __ua, index_sequence<_Ind...>, 345 tuple<_Args...>& __t) 346 { 347 return { 348 allocator_arg, *__ua._M_a, std::get<_Ind>(std::move(__t))... 349 }; 350 } 351 352 template<size_t... _Ind, typename... _Args> 353 static tuple<_Args&&..., polymorphic_allocator> 354 _S_construct_p(__uses_alloc2_ __ua, index_sequence<_Ind...>, 355 tuple<_Args...>& __t) 356 { return { std::get<_Ind>(std::move(__t))..., *__ua._M_a }; } 357 358 memory_resource* _M_resource; 359 }; 360 361 template<typename _Tp1, typename _Tp2> 362 inline bool 363 operator==(const polymorphic_allocator<_Tp1>& __a, 364 const polymorphic_allocator<_Tp2>& __b) noexcept 365 { return *__a.resource() == *__b.resource(); } 366 367#if __cpp_impl_three_way_comparison < 201907L 368 template<typename _Tp1, typename _Tp2> 369 inline bool 370 operator!=(const polymorphic_allocator<_Tp1>& __a, 371 const polymorphic_allocator<_Tp2>& __b) noexcept 372 { return !(__a == __b); } 373#endif 374 375 /// Parameters for tuning a pool resource's behaviour. 376 struct pool_options 377 { 378 /** @brief Upper limit on number of blocks in a chunk. 379 * 380 * A lower value prevents allocating huge chunks that could remain mostly 381 * unused, but means pools will need to replenished more frequently. 382 */ 383 size_t max_blocks_per_chunk = 0; 384 385 /* @brief Largest block size (in bytes) that should be served from pools. 386 * 387 * Larger allocations will be served directly by the upstream resource, 388 * not from one of the pools managed by the pool resource. 389 */ 390 size_t largest_required_pool_block = 0; 391 }; 392 393 // Common implementation details for un-/synchronized pool resources. 394 class __pool_resource 395 { 396 friend class synchronized_pool_resource; 397 friend class unsynchronized_pool_resource; 398 399 __pool_resource(const pool_options& __opts, memory_resource* __upstream); 400 401 ~__pool_resource(); 402 403 __pool_resource(const __pool_resource&) = delete; 404 __pool_resource& operator=(const __pool_resource&) = delete; 405 406 // Allocate a large unpooled block. 407 void* 408 allocate(size_t __bytes, size_t __alignment); 409 410 // Deallocate a large unpooled block. 411 void 412 deallocate(void* __p, size_t __bytes, size_t __alignment); 413 414 415 // Deallocate unpooled memory. 416 void release() noexcept; 417 418 memory_resource* resource() const noexcept 419 { return _M_unpooled.get_allocator().resource(); } 420 421 struct _Pool; 422 423 _Pool* _M_alloc_pools(); 424 425 const pool_options _M_opts; 426 427 struct _BigBlock; 428 // Collection of blocks too big for any pool, sorted by address. 429 // This also stores the only copy of the upstream memory resource pointer. 430 _GLIBCXX_STD_C::pmr::vector<_BigBlock> _M_unpooled; 431 432 const int _M_npools; 433 }; 434 435#ifdef _GLIBCXX_HAS_GTHREADS 436 /// A thread-safe memory resource that manages pools of fixed-size blocks. 437 class synchronized_pool_resource : public memory_resource 438 { 439 public: 440 synchronized_pool_resource(const pool_options& __opts, 441 memory_resource* __upstream) 442 __attribute__((__nonnull__)); 443 444 synchronized_pool_resource() 445 : synchronized_pool_resource(pool_options(), get_default_resource()) 446 { } 447 448 explicit 449 synchronized_pool_resource(memory_resource* __upstream) 450 __attribute__((__nonnull__)) 451 : synchronized_pool_resource(pool_options(), __upstream) 452 { } 453 454 explicit 455 synchronized_pool_resource(const pool_options& __opts) 456 : synchronized_pool_resource(__opts, get_default_resource()) { } 457 458 synchronized_pool_resource(const synchronized_pool_resource&) = delete; 459 460 virtual ~synchronized_pool_resource(); 461 462 synchronized_pool_resource& 463 operator=(const synchronized_pool_resource&) = delete; 464 465 void release(); 466 467 memory_resource* 468 upstream_resource() const noexcept 469 __attribute__((__returns_nonnull__)) 470 { return _M_impl.resource(); } 471 472 pool_options options() const noexcept { return _M_impl._M_opts; } 473 474 protected: 475 void* 476 do_allocate(size_t __bytes, size_t __alignment) override; 477 478 void 479 do_deallocate(void* __p, size_t __bytes, size_t __alignment) override; 480 481 bool 482 do_is_equal(const memory_resource& __other) const noexcept override 483 { return this == &__other; } 484 485 public: 486 // Thread-specific pools (only public for access by implementation details) 487 struct _TPools; 488 489 private: 490 _TPools* _M_alloc_tpools(lock_guard<shared_mutex>&); 491 _TPools* _M_alloc_shared_tpools(lock_guard<shared_mutex>&); 492 auto _M_thread_specific_pools() noexcept; 493 494 __pool_resource _M_impl; 495 __gthread_key_t _M_key; 496 // Linked list of thread-specific pools. All threads share _M_tpools[0]. 497 _TPools* _M_tpools = nullptr; 498 mutable shared_mutex _M_mx; 499 }; 500#endif 501 502 /// A non-thread-safe memory resource that manages pools of fixed-size blocks. 503 class unsynchronized_pool_resource : public memory_resource 504 { 505 public: 506 [[__gnu__::__nonnull__]] 507 unsynchronized_pool_resource(const pool_options& __opts, 508 memory_resource* __upstream); 509 510 unsynchronized_pool_resource() 511 : unsynchronized_pool_resource(pool_options(), get_default_resource()) 512 { } 513 514 [[__gnu__::__nonnull__]] 515 explicit 516 unsynchronized_pool_resource(memory_resource* __upstream) 517 : unsynchronized_pool_resource(pool_options(), __upstream) 518 { } 519 520 explicit 521 unsynchronized_pool_resource(const pool_options& __opts) 522 : unsynchronized_pool_resource(__opts, get_default_resource()) { } 523 524 unsynchronized_pool_resource(const unsynchronized_pool_resource&) = delete; 525 526 virtual ~unsynchronized_pool_resource(); 527 528 unsynchronized_pool_resource& 529 operator=(const unsynchronized_pool_resource&) = delete; 530 531 void release(); 532 533 [[__gnu__::__returns_nonnull__]] 534 memory_resource* 535 upstream_resource() const noexcept 536 { return _M_impl.resource(); } 537 538 pool_options options() const noexcept { return _M_impl._M_opts; } 539 540 protected: 541 void* 542 do_allocate(size_t __bytes, size_t __alignment) override; 543 544 void 545 do_deallocate(void* __p, size_t __bytes, size_t __alignment) override; 546 547 bool 548 do_is_equal(const memory_resource& __other) const noexcept override 549 { return this == &__other; } 550 551 private: 552 using _Pool = __pool_resource::_Pool; 553 554 auto _M_find_pool(size_t) noexcept; 555 556 __pool_resource _M_impl; 557 _Pool* _M_pools = nullptr; 558 }; 559 560 class monotonic_buffer_resource : public memory_resource 561 { 562 public: 563 explicit 564 monotonic_buffer_resource(memory_resource* __upstream) noexcept 565 __attribute__((__nonnull__)) 566 : _M_upstream(__upstream) 567 { _GLIBCXX_DEBUG_ASSERT(__upstream != nullptr); } 568 569 monotonic_buffer_resource(size_t __initial_size, 570 memory_resource* __upstream) noexcept 571 __attribute__((__nonnull__)) 572 : _M_next_bufsiz(__initial_size), 573 _M_upstream(__upstream) 574 { 575 _GLIBCXX_DEBUG_ASSERT(__upstream != nullptr); 576 _GLIBCXX_DEBUG_ASSERT(__initial_size > 0); 577 } 578 579 monotonic_buffer_resource(void* __buffer, size_t __buffer_size, 580 memory_resource* __upstream) noexcept 581 __attribute__((__nonnull__(4))) 582 : _M_current_buf(__buffer), _M_avail(__buffer_size), 583 _M_next_bufsiz(_S_next_bufsize(__buffer_size)), 584 _M_upstream(__upstream), 585 _M_orig_buf(__buffer), _M_orig_size(__buffer_size) 586 { 587 _GLIBCXX_DEBUG_ASSERT(__upstream != nullptr); 588 _GLIBCXX_DEBUG_ASSERT(__buffer != nullptr || __buffer_size == 0); 589 } 590 591 monotonic_buffer_resource() noexcept 592 : monotonic_buffer_resource(get_default_resource()) 593 { } 594 595 explicit 596 monotonic_buffer_resource(size_t __initial_size) noexcept 597 : monotonic_buffer_resource(__initial_size, get_default_resource()) 598 { } 599 600 monotonic_buffer_resource(void* __buffer, size_t __buffer_size) noexcept 601 : monotonic_buffer_resource(__buffer, __buffer_size, get_default_resource()) 602 { } 603 604 monotonic_buffer_resource(const monotonic_buffer_resource&) = delete; 605 606 virtual ~monotonic_buffer_resource(); // key function 607 608 monotonic_buffer_resource& 609 operator=(const monotonic_buffer_resource&) = delete; 610 611 void 612 release() noexcept 613 { 614 if (_M_head) 615 _M_release_buffers(); 616 617 // reset to initial state at contruction: 618 if ((_M_current_buf = _M_orig_buf)) 619 { 620 _M_avail = _M_orig_size; 621 _M_next_bufsiz = _S_next_bufsize(_M_orig_size); 622 } 623 else 624 { 625 _M_avail = 0; 626 _M_next_bufsiz = _M_orig_size; 627 } 628 } 629 630 memory_resource* 631 upstream_resource() const noexcept 632 __attribute__((__returns_nonnull__)) 633 { return _M_upstream; } 634 635 protected: 636 void* 637 do_allocate(size_t __bytes, size_t __alignment) override 638 { 639 if (__bytes == 0) 640 __bytes = 1; // Ensures we don't return the same pointer twice. 641 642 void* __p = std::align(__alignment, __bytes, _M_current_buf, _M_avail); 643 if (!__p) 644 { 645 _M_new_buffer(__bytes, __alignment); 646 __p = _M_current_buf; 647 } 648 _M_current_buf = (char*)_M_current_buf + __bytes; 649 _M_avail -= __bytes; 650 return __p; 651 } 652 653 void 654 do_deallocate(void*, size_t, size_t) override 655 { } 656 657 bool 658 do_is_equal(const memory_resource& __other) const noexcept override 659 { return this == &__other; } 660 661 private: 662 // Update _M_current_buf and _M_avail to refer to a new buffer with 663 // at least the specified size and alignment, allocated from upstream. 664 void 665 _M_new_buffer(size_t __bytes, size_t __alignment); 666 667 // Deallocate all buffers obtained from upstream. 668 void 669 _M_release_buffers() noexcept; 670 671 static size_t 672 _S_next_bufsize(size_t __buffer_size) noexcept 673 { 674 if (__buffer_size == 0) 675 __buffer_size = 1; 676 return __buffer_size * _S_growth_factor; 677 } 678 679 static constexpr size_t _S_init_bufsize = 128 * sizeof(void*); 680 static constexpr float _S_growth_factor = 1.5; 681 682 void* _M_current_buf = nullptr; 683 size_t _M_avail = 0; 684 size_t _M_next_bufsiz = _S_init_bufsize; 685 686 // Initial values set at construction and reused by release(): 687 memory_resource* const _M_upstream; 688 void* const _M_orig_buf = nullptr; 689 size_t const _M_orig_size = _M_next_bufsiz; 690 691 class _Chunk; 692 _Chunk* _M_head = nullptr; 693 }; 694 695} // namespace pmr 696_GLIBCXX_END_NAMESPACE_VERSION 697} // namespace std 698 699#endif // C++17 700#endif // _GLIBCXX_MEMORY_RESOURCE 701