Lines Matching refs:buffer
806 bool this_cpu_buffer_push(struct percpu_buffer *buffer, in this_cpu_buffer_push() argument
820 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); in this_cpu_buffer_push()
821 if (offset == buffer->c[cpu].buflen) in this_cpu_buffer_push()
824 targetptr_spec = (intptr_t *)&buffer->c[cpu].array[offset]; in this_cpu_buffer_push()
826 targetptr_final = &buffer->c[cpu].offset; in this_cpu_buffer_push()
841 struct percpu_buffer_node *this_cpu_buffer_pop(struct percpu_buffer *buffer, in this_cpu_buffer_pop() argument
854 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); in this_cpu_buffer_pop()
859 head = RSEQ_READ_ONCE(buffer->c[cpu].array[offset - 1]); in this_cpu_buffer_pop()
861 targetptr = (intptr_t *)&buffer->c[cpu].offset; in this_cpu_buffer_pop()
864 (intptr_t *)&buffer->c[cpu].array[offset - 1], in this_cpu_buffer_pop()
879 struct percpu_buffer_node *__percpu_buffer_pop(struct percpu_buffer *buffer, in __percpu_buffer_pop() argument
885 offset = buffer->c[cpu].offset; in __percpu_buffer_pop()
888 head = buffer->c[cpu].array[offset - 1]; in __percpu_buffer_pop()
889 buffer->c[cpu].offset = offset - 1; in __percpu_buffer_pop()
896 struct percpu_buffer *buffer = (struct percpu_buffer *)arg; in test_percpu_buffer_thread() local
905 node = this_cpu_buffer_pop(buffer, NULL); in test_percpu_buffer_thread()
909 if (!this_cpu_buffer_push(buffer, node, NULL)) { in test_percpu_buffer_thread()
930 struct percpu_buffer buffer; in test_percpu_buffer() local
934 memset(&buffer, 0, sizeof(buffer)); in test_percpu_buffer()
942 buffer.c[i].array = in test_percpu_buffer()
943 malloc(sizeof(*buffer.c[i].array) * CPU_SETSIZE * in test_percpu_buffer()
945 assert(buffer.c[i].array); in test_percpu_buffer()
946 buffer.c[i].buflen = CPU_SETSIZE * BUFFER_ITEM_PER_CPU; in test_percpu_buffer()
962 buffer.c[i].array[j - 1] = node; in test_percpu_buffer()
963 buffer.c[i].offset++; in test_percpu_buffer()
969 test_percpu_buffer_thread, &buffer); in test_percpu_buffer()
992 while ((node = __percpu_buffer_pop(&buffer, i))) { in test_percpu_buffer()
996 free(buffer.c[i].array); in test_percpu_buffer()
1007 bool this_cpu_memcpy_buffer_push(struct percpu_memcpy_buffer *buffer, in this_cpu_memcpy_buffer_push() argument
1022 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); in this_cpu_memcpy_buffer_push()
1023 if (offset == buffer->c[cpu].buflen) in this_cpu_memcpy_buffer_push()
1025 destptr = (char *)&buffer->c[cpu].array[offset]; in this_cpu_memcpy_buffer_push()
1030 targetptr_final = &buffer->c[cpu].offset; in this_cpu_memcpy_buffer_push()
1047 bool this_cpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer, in this_cpu_memcpy_buffer_pop() argument
1062 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); in this_cpu_memcpy_buffer_pop()
1066 srcptr = (char *)&buffer->c[cpu].array[offset - 1]; in this_cpu_memcpy_buffer_pop()
1070 targetptr_final = &buffer->c[cpu].offset; in this_cpu_memcpy_buffer_pop()
1089 bool __percpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer, in __percpu_memcpy_buffer_pop() argument
1095 offset = buffer->c[cpu].offset; in __percpu_memcpy_buffer_pop()
1098 memcpy(item, &buffer->c[cpu].array[offset - 1], sizeof(*item)); in __percpu_memcpy_buffer_pop()
1099 buffer->c[cpu].offset = offset - 1; in __percpu_memcpy_buffer_pop()
1106 struct percpu_memcpy_buffer *buffer = (struct percpu_memcpy_buffer *)arg; in test_percpu_memcpy_buffer_thread() local
1116 result = this_cpu_memcpy_buffer_pop(buffer, &item, NULL); in test_percpu_memcpy_buffer_thread()
1120 if (!this_cpu_memcpy_buffer_push(buffer, item, NULL)) { in test_percpu_memcpy_buffer_thread()
1141 struct percpu_memcpy_buffer buffer; in test_percpu_memcpy_buffer() local
1145 memset(&buffer, 0, sizeof(buffer)); in test_percpu_memcpy_buffer()
1153 buffer.c[i].array = in test_percpu_memcpy_buffer()
1154 malloc(sizeof(*buffer.c[i].array) * CPU_SETSIZE * in test_percpu_memcpy_buffer()
1156 assert(buffer.c[i].array); in test_percpu_memcpy_buffer()
1157 buffer.c[i].buflen = CPU_SETSIZE * MEMCPY_BUFFER_ITEM_PER_CPU; in test_percpu_memcpy_buffer()
1168 buffer.c[i].array[j - 1].data1 = j; in test_percpu_memcpy_buffer()
1169 buffer.c[i].array[j - 1].data2 = j + 1; in test_percpu_memcpy_buffer()
1170 buffer.c[i].offset++; in test_percpu_memcpy_buffer()
1177 &buffer); in test_percpu_memcpy_buffer()
1200 while (__percpu_memcpy_buffer_pop(&buffer, &item, i)) { in test_percpu_memcpy_buffer()
1204 free(buffer.c[i].array); in test_percpu_memcpy_buffer()