1 #ifndef JEMALLOC_INTERNAL_UTIL_INLINES_H 2 #define JEMALLOC_INTERNAL_UTIL_INLINES_H 3 4 #ifndef JEMALLOC_ENABLE_INLINE 5 unsigned ffs_llu(unsigned long long bitmap); 6 unsigned ffs_lu(unsigned long bitmap); 7 unsigned ffs_u(unsigned bitmap); 8 unsigned ffs_zu(size_t bitmap); 9 unsigned ffs_u64(uint64_t bitmap); 10 unsigned ffs_u32(uint32_t bitmap); 11 uint64_t pow2_ceil_u64(uint64_t x); 12 uint32_t pow2_ceil_u32(uint32_t x); 13 size_t pow2_ceil_zu(size_t x); 14 unsigned lg_floor(size_t x); 15 void set_errno(int errnum); 16 int get_errno(void); 17 #endif 18 19 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_UTIL_C_)) 20 21 /* Sanity check. */ 22 #if !defined(JEMALLOC_INTERNAL_FFSLL) || !defined(JEMALLOC_INTERNAL_FFSL) \ 23 || !defined(JEMALLOC_INTERNAL_FFS) 24 # error JEMALLOC_INTERNAL_FFS{,L,LL} should have been defined by configure 25 #endif 26 27 JEMALLOC_ALWAYS_INLINE unsigned ffs_llu(unsigned long long bitmap)28ffs_llu(unsigned long long bitmap) 29 { 30 return (JEMALLOC_INTERNAL_FFSLL(bitmap)); 31 } 32 33 JEMALLOC_ALWAYS_INLINE unsigned ffs_lu(unsigned long bitmap)34ffs_lu(unsigned long bitmap) 35 { 36 return (JEMALLOC_INTERNAL_FFSL(bitmap)); 37 } 38 39 JEMALLOC_ALWAYS_INLINE unsigned ffs_u(unsigned bitmap)40ffs_u(unsigned bitmap) 41 { 42 return (JEMALLOC_INTERNAL_FFS(bitmap)); 43 } 44 45 JEMALLOC_ALWAYS_INLINE unsigned ffs_zu(size_t bitmap)46ffs_zu(size_t bitmap) 47 { 48 #if LG_SIZEOF_PTR == LG_SIZEOF_INT 49 return (ffs_u(bitmap)); 50 #elif LG_SIZEOF_PTR == LG_SIZEOF_LONG 51 return (ffs_lu(bitmap)); 52 #elif LG_SIZEOF_PTR == LG_SIZEOF_LONG_LONG 53 return (ffs_llu(bitmap)); 54 #else 55 #error No implementation for size_t ffs() 56 #endif 57 } 58 59 JEMALLOC_ALWAYS_INLINE unsigned ffs_u64(uint64_t bitmap)60ffs_u64(uint64_t bitmap) 61 { 62 #if LG_SIZEOF_LONG == 3 63 return (ffs_lu(bitmap)); 64 #elif LG_SIZEOF_LONG_LONG == 3 65 return (ffs_llu(bitmap)); 66 #else 67 #error No implementation for 64-bit ffs() 68 #endif 69 } 70 71 JEMALLOC_ALWAYS_INLINE unsigned ffs_u32(uint32_t bitmap)72ffs_u32(uint32_t bitmap) 73 { 74 #if LG_SIZEOF_INT == 2 75 return (ffs_u(bitmap)); 76 #else 77 #error No implementation for 32-bit ffs() 78 #endif 79 return (ffs_u(bitmap)); 80 } 81 82 JEMALLOC_INLINE uint64_t pow2_ceil_u64(uint64_t x)83pow2_ceil_u64(uint64_t x) 84 { 85 x--; 86 x |= x >> 1; 87 x |= x >> 2; 88 x |= x >> 4; 89 x |= x >> 8; 90 x |= x >> 16; 91 x |= x >> 32; 92 x++; 93 return (x); 94 } 95 96 JEMALLOC_INLINE uint32_t pow2_ceil_u32(uint32_t x)97pow2_ceil_u32(uint32_t x) 98 { 99 x--; 100 x |= x >> 1; 101 x |= x >> 2; 102 x |= x >> 4; 103 x |= x >> 8; 104 x |= x >> 16; 105 x++; 106 return (x); 107 } 108 109 /* Compute the smallest power of 2 that is >= x. */ 110 JEMALLOC_INLINE size_t pow2_ceil_zu(size_t x)111pow2_ceil_zu(size_t x) 112 { 113 #if (LG_SIZEOF_PTR == 3) 114 return (pow2_ceil_u64(x)); 115 #else 116 return (pow2_ceil_u32(x)); 117 #endif 118 } 119 120 #if (defined(__i386__) || defined(__amd64__) || defined(__x86_64__)) 121 JEMALLOC_INLINE unsigned lg_floor(size_t x)122lg_floor(size_t x) 123 { 124 size_t ret; 125 126 assert(x != 0); 127 128 __asm__ ("bsr %1, %0" 129 : "=r"(ret) // Outputs. 130 : "r"(x) // Inputs. 131 ); 132 assert(ret < UINT_MAX); 133 return ((unsigned)ret); 134 } 135 #elif (defined(_MSC_VER)) 136 JEMALLOC_INLINE unsigned lg_floor(size_t x)137lg_floor(size_t x) 138 { 139 unsigned long ret; 140 141 assert(x != 0); 142 143 #if (LG_SIZEOF_PTR == 3) 144 _BitScanReverse64(&ret, x); 145 #elif (LG_SIZEOF_PTR == 2) 146 _BitScanReverse(&ret, x); 147 #else 148 # error "Unsupported type size for lg_floor()" 149 #endif 150 assert(ret < UINT_MAX); 151 return ((unsigned)ret); 152 } 153 #elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ)) 154 JEMALLOC_INLINE unsigned lg_floor(size_t x)155lg_floor(size_t x) 156 { 157 assert(x != 0); 158 159 #if (LG_SIZEOF_PTR == LG_SIZEOF_INT) 160 return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clz(x)); 161 #elif (LG_SIZEOF_PTR == LG_SIZEOF_LONG) 162 return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clzl(x)); 163 #else 164 # error "Unsupported type size for lg_floor()" 165 #endif 166 } 167 #else 168 JEMALLOC_INLINE unsigned lg_floor(size_t x)169lg_floor(size_t x) 170 { 171 assert(x != 0); 172 173 x |= (x >> 1); 174 x |= (x >> 2); 175 x |= (x >> 4); 176 x |= (x >> 8); 177 x |= (x >> 16); 178 #if (LG_SIZEOF_PTR == 3) 179 x |= (x >> 32); 180 #endif 181 if (x == SIZE_T_MAX) 182 return ((8 << LG_SIZEOF_PTR) - 1); 183 x++; 184 return (ffs_zu(x) - 2); 185 } 186 #endif 187 188 /* Set error code. */ 189 JEMALLOC_INLINE void set_errno(int errnum)190set_errno(int errnum) 191 { 192 #ifdef _WIN32 193 SetLastError(errnum); 194 #else 195 errno = errnum; 196 #endif 197 } 198 199 /* Get last error code. */ 200 JEMALLOC_INLINE int get_errno(void)201get_errno(void) 202 { 203 #ifdef _WIN32 204 return (GetLastError()); 205 #else 206 return (errno); 207 #endif 208 } 209 #endif 210 211 #endif /* JEMALLOC_INTERNAL_UTIL_INLINES_H */ 212