1 #ifndef JEMALLOC_INTERNAL_PROF_INLINES_H
2 #define JEMALLOC_INTERNAL_PROF_INLINES_H
3
4 #ifndef JEMALLOC_ENABLE_INLINE
5 bool prof_active_get_unlocked(void);
6 bool prof_gdump_get_unlocked(void);
7 prof_tdata_t *prof_tdata_get(tsd_t *tsd, bool create);
8 prof_tctx_t *prof_tctx_get(tsdn_t *tsdn, const extent_t *extent,
9 const void *ptr);
10 void prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr,
11 size_t usize, prof_tctx_t *tctx);
12 void prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr,
13 prof_tctx_t *tctx);
14 bool prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update,
15 prof_tdata_t **tdata_out);
16 prof_tctx_t *prof_alloc_prep(tsd_t *tsd, size_t usize, bool prof_active,
17 bool update);
18 void prof_malloc(tsdn_t *tsdn, extent_t *extent, const void *ptr,
19 size_t usize, prof_tctx_t *tctx);
20 void prof_realloc(tsd_t *tsd, extent_t *extent, const void *ptr,
21 size_t usize, prof_tctx_t *tctx, bool prof_active, bool updated,
22 extent_t *old_extent, const void *old_ptr, size_t old_usize,
23 prof_tctx_t *old_tctx);
24 void prof_free(tsd_t *tsd, const extent_t *extent, const void *ptr,
25 size_t usize);
26 #endif
27
28 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PROF_C_))
29 JEMALLOC_ALWAYS_INLINE bool
prof_active_get_unlocked(void)30 prof_active_get_unlocked(void)
31 {
32 /*
33 * Even if opt_prof is true, sampling can be temporarily disabled by
34 * setting prof_active to false. No locking is used when reading
35 * prof_active in the fast path, so there are no guarantees regarding
36 * how long it will take for all threads to notice state changes.
37 */
38 return (prof_active);
39 }
40
41 JEMALLOC_ALWAYS_INLINE bool
prof_gdump_get_unlocked(void)42 prof_gdump_get_unlocked(void)
43 {
44 /*
45 * No locking is used when reading prof_gdump_val in the fast path, so
46 * there are no guarantees regarding how long it will take for all
47 * threads to notice state changes.
48 */
49 return (prof_gdump_val);
50 }
51
52 JEMALLOC_ALWAYS_INLINE prof_tdata_t *
prof_tdata_get(tsd_t * tsd,bool create)53 prof_tdata_get(tsd_t *tsd, bool create)
54 {
55 prof_tdata_t *tdata;
56
57 cassert(config_prof);
58
59 tdata = tsd_prof_tdata_get(tsd);
60 if (create) {
61 if (unlikely(tdata == NULL)) {
62 if (tsd_nominal(tsd)) {
63 tdata = prof_tdata_init(tsd);
64 tsd_prof_tdata_set(tsd, tdata);
65 }
66 } else if (unlikely(tdata->expired)) {
67 tdata = prof_tdata_reinit(tsd, tdata);
68 tsd_prof_tdata_set(tsd, tdata);
69 }
70 assert(tdata == NULL || tdata->attached);
71 }
72
73 return (tdata);
74 }
75
76 JEMALLOC_ALWAYS_INLINE prof_tctx_t *
prof_tctx_get(tsdn_t * tsdn,const extent_t * extent,const void * ptr)77 prof_tctx_get(tsdn_t *tsdn, const extent_t *extent, const void *ptr)
78 {
79 cassert(config_prof);
80 assert(ptr != NULL);
81
82 return (arena_prof_tctx_get(tsdn, extent, ptr));
83 }
84
85 JEMALLOC_ALWAYS_INLINE void
prof_tctx_set(tsdn_t * tsdn,extent_t * extent,const void * ptr,size_t usize,prof_tctx_t * tctx)86 prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr, size_t usize,
87 prof_tctx_t *tctx)
88 {
89 cassert(config_prof);
90 assert(ptr != NULL);
91
92 arena_prof_tctx_set(tsdn, extent, ptr, usize, tctx);
93 }
94
95 JEMALLOC_ALWAYS_INLINE void
prof_tctx_reset(tsdn_t * tsdn,extent_t * extent,const void * ptr,prof_tctx_t * tctx)96 prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr,
97 prof_tctx_t *tctx)
98 {
99 cassert(config_prof);
100 assert(ptr != NULL);
101
102 arena_prof_tctx_reset(tsdn, extent, ptr, tctx);
103 }
104
105 JEMALLOC_ALWAYS_INLINE bool
prof_sample_accum_update(tsd_t * tsd,size_t usize,bool update,prof_tdata_t ** tdata_out)106 prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update,
107 prof_tdata_t **tdata_out)
108 {
109 prof_tdata_t *tdata;
110
111 cassert(config_prof);
112
113 tdata = prof_tdata_get(tsd, true);
114 if (unlikely((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX))
115 tdata = NULL;
116
117 if (tdata_out != NULL)
118 *tdata_out = tdata;
119
120 if (unlikely(tdata == NULL))
121 return (true);
122
123 if (likely(tdata->bytes_until_sample >= usize)) {
124 if (update)
125 tdata->bytes_until_sample -= usize;
126 return (true);
127 } else {
128 /* Compute new sample threshold. */
129 if (update)
130 prof_sample_threshold_update(tdata);
131 return (!tdata->active);
132 }
133 }
134
135 JEMALLOC_ALWAYS_INLINE prof_tctx_t *
prof_alloc_prep(tsd_t * tsd,size_t usize,bool prof_active,bool update)136 prof_alloc_prep(tsd_t *tsd, size_t usize, bool prof_active, bool update)
137 {
138 prof_tctx_t *ret;
139 prof_tdata_t *tdata;
140 prof_bt_t bt;
141
142 assert(usize == s2u(usize));
143
144 if (!prof_active || likely(prof_sample_accum_update(tsd, usize, update,
145 &tdata)))
146 ret = (prof_tctx_t *)(uintptr_t)1U;
147 else {
148 bt_init(&bt, tdata->vec);
149 prof_backtrace(&bt);
150 ret = prof_lookup(tsd, &bt);
151 }
152
153 return (ret);
154 }
155
156 JEMALLOC_ALWAYS_INLINE void
prof_malloc(tsdn_t * tsdn,extent_t * extent,const void * ptr,size_t usize,prof_tctx_t * tctx)157 prof_malloc(tsdn_t *tsdn, extent_t *extent, const void *ptr, size_t usize,
158 prof_tctx_t *tctx)
159 {
160 cassert(config_prof);
161 assert(ptr != NULL);
162 assert(usize == isalloc(tsdn, extent, ptr));
163
164 if (unlikely((uintptr_t)tctx > (uintptr_t)1U))
165 prof_malloc_sample_object(tsdn, extent, ptr, usize, tctx);
166 else {
167 prof_tctx_set(tsdn, extent, ptr, usize,
168 (prof_tctx_t *)(uintptr_t)1U);
169 }
170 }
171
172 JEMALLOC_ALWAYS_INLINE void
prof_realloc(tsd_t * tsd,extent_t * extent,const void * ptr,size_t usize,prof_tctx_t * tctx,bool prof_active,bool updated,extent_t * old_extent,const void * old_ptr,size_t old_usize,prof_tctx_t * old_tctx)173 prof_realloc(tsd_t *tsd, extent_t *extent, const void *ptr, size_t usize,
174 prof_tctx_t *tctx, bool prof_active, bool updated, extent_t *old_extent,
175 const void *old_ptr, size_t old_usize, prof_tctx_t *old_tctx)
176 {
177 bool sampled, old_sampled, moved;
178
179 cassert(config_prof);
180 assert(ptr != NULL || (uintptr_t)tctx <= (uintptr_t)1U);
181
182 if (prof_active && !updated && ptr != NULL) {
183 assert(usize == isalloc(tsd_tsdn(tsd), extent, ptr));
184 if (prof_sample_accum_update(tsd, usize, true, NULL)) {
185 /*
186 * Don't sample. The usize passed to prof_alloc_prep()
187 * was larger than what actually got allocated, so a
188 * backtrace was captured for this allocation, even
189 * though its actual usize was insufficient to cross the
190 * sample threshold.
191 */
192 prof_alloc_rollback(tsd, tctx, true);
193 tctx = (prof_tctx_t *)(uintptr_t)1U;
194 }
195 }
196
197 sampled = ((uintptr_t)tctx > (uintptr_t)1U);
198 old_sampled = ((uintptr_t)old_tctx > (uintptr_t)1U);
199 moved = (ptr != old_ptr);
200
201 if (unlikely(sampled)) {
202 prof_malloc_sample_object(tsd_tsdn(tsd), extent, ptr, usize,
203 tctx);
204 } else if (moved) {
205 prof_tctx_set(tsd_tsdn(tsd), extent, ptr, usize,
206 (prof_tctx_t *)(uintptr_t)1U);
207 } else if (unlikely(old_sampled)) {
208 /*
209 * prof_tctx_set() would work for the !moved case as well, but
210 * prof_tctx_reset() is slightly cheaper, and the proper thing
211 * to do here in the presence of explicit knowledge re: moved
212 * state.
213 */
214 prof_tctx_reset(tsd_tsdn(tsd), extent, ptr, tctx);
215 } else {
216 assert((uintptr_t)prof_tctx_get(tsd_tsdn(tsd), extent, ptr) ==
217 (uintptr_t)1U);
218 }
219
220 /*
221 * The prof_free_sampled_object() call must come after the
222 * prof_malloc_sample_object() call, because tctx and old_tctx may be
223 * the same, in which case reversing the call order could cause the tctx
224 * to be prematurely destroyed as a side effect of momentarily zeroed
225 * counters.
226 */
227 if (unlikely(old_sampled)) {
228 prof_free_sampled_object(tsd, old_usize, old_tctx);
229 }
230 }
231
232 JEMALLOC_ALWAYS_INLINE void
prof_free(tsd_t * tsd,const extent_t * extent,const void * ptr,size_t usize)233 prof_free(tsd_t *tsd, const extent_t *extent, const void *ptr, size_t usize)
234 {
235 prof_tctx_t *tctx = prof_tctx_get(tsd_tsdn(tsd), extent, ptr);
236
237 cassert(config_prof);
238 assert(usize == isalloc(tsd_tsdn(tsd), extent, ptr));
239
240 if (unlikely((uintptr_t)tctx > (uintptr_t)1U))
241 prof_free_sampled_object(tsd, usize, tctx);
242 }
243 #endif
244
245 #endif /* JEMALLOC_INTERNAL_PROF_INLINES_H */
246