Lines Matching refs:cache
89 static void rds_ib_cache_xfer_to_ready(struct rds_ib_refill_cache *cache) in rds_ib_cache_xfer_to_ready() argument
93 tmp = xchg(&cache->xfer, NULL); in rds_ib_cache_xfer_to_ready()
95 if (cache->ready) in rds_ib_cache_xfer_to_ready()
96 list_splice_entire_tail(tmp, cache->ready); in rds_ib_cache_xfer_to_ready()
98 cache->ready = tmp; in rds_ib_cache_xfer_to_ready()
102 static int rds_ib_recv_alloc_cache(struct rds_ib_refill_cache *cache, gfp_t gfp) in rds_ib_recv_alloc_cache() argument
107 cache->percpu = alloc_percpu_gfp(struct rds_ib_cache_head, gfp); in rds_ib_recv_alloc_cache()
108 if (!cache->percpu) in rds_ib_recv_alloc_cache()
112 head = per_cpu_ptr(cache->percpu, cpu); in rds_ib_recv_alloc_cache()
116 cache->xfer = NULL; in rds_ib_recv_alloc_cache()
117 cache->ready = NULL; in rds_ib_recv_alloc_cache()
136 static void rds_ib_cache_splice_all_lists(struct rds_ib_refill_cache *cache, in rds_ib_cache_splice_all_lists() argument
143 head = per_cpu_ptr(cache->percpu, cpu); in rds_ib_cache_splice_all_lists()
150 if (cache->ready) { in rds_ib_cache_splice_all_lists()
151 list_splice_entire_tail(cache->ready, caller_list); in rds_ib_cache_splice_all_lists()
152 cache->ready = NULL; in rds_ib_cache_splice_all_lists()
188 struct rds_ib_refill_cache *cache);
189 static struct list_head *rds_ib_recv_cache_get(struct rds_ib_refill_cache *cache);
480 struct rds_ib_refill_cache *cache) in rds_ib_recv_cache_put() argument
487 chpfirst = __this_cpu_read(cache->percpu->first); in rds_ib_recv_cache_put()
493 __this_cpu_write(cache->percpu->first, new_item); in rds_ib_recv_cache_put()
494 __this_cpu_inc(cache->percpu->count); in rds_ib_recv_cache_put()
496 if (__this_cpu_read(cache->percpu->count) < RDS_IB_RECYCLE_BATCH_COUNT) in rds_ib_recv_cache_put()
506 old = xchg(&cache->xfer, NULL); in rds_ib_recv_cache_put()
509 old = cmpxchg(&cache->xfer, NULL, chpfirst); in rds_ib_recv_cache_put()
513 __this_cpu_write(cache->percpu->first, NULL); in rds_ib_recv_cache_put()
514 __this_cpu_write(cache->percpu->count, 0); in rds_ib_recv_cache_put()
519 static struct list_head *rds_ib_recv_cache_get(struct rds_ib_refill_cache *cache) in rds_ib_recv_cache_get() argument
521 struct list_head *head = cache->ready; in rds_ib_recv_cache_get()
525 cache->ready = head->next; in rds_ib_recv_cache_get()
528 cache->ready = NULL; in rds_ib_recv_cache_get()