1/* Copyright (C) 2002-2006, 2007, 2009 Free Software Foundation, Inc.
2   This file is part of the GNU C Library.
3   Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
4
5   The GNU C Library is free software; you can redistribute it and/or
6   modify it under the terms of the GNU Lesser General Public
7   License as published by the Free Software Foundation; either
8   version 2.1 of the License, or (at your option) any later version.
9
10   The GNU C Library is distributed in the hope that it will be useful,
11   but WITHOUT ANY WARRANTY; without even the implied warranty of
12   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13   Lesser General Public License for more details.
14
15   You should have received a copy of the GNU Lesser General Public
16   License along with the GNU C Library; if not, see
17   <http://www.gnu.org/licenses/>.  */
18
19#include <sysdep.h>
20#include <pthread-errnos.h>
21#include <bits/kernel-features.h>
22#include <lowlevellock.h>
23#include <tcb-offsets.h>
24
25	.text
26
27#ifdef __ASSUME_PRIVATE_FUTEX
28# define LOAD_PRIVATE_FUTEX_WAIT(reg) \
29	movl	$(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
30# define LOAD_PRIVATE_FUTEX_WAKE(reg) \
31	movl	$(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg
32# define LOAD_FUTEX_WAIT(reg) \
33	xorl	$(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
34# define LOAD_FUTEX_WAIT_ABS(reg) \
35	xorl	$(FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME), reg
36# define LOAD_FUTEX_WAKE(reg) \
37	xorl	$(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg
38#else
39# if FUTEX_WAIT == 0
40#  define LOAD_PRIVATE_FUTEX_WAIT(reg) \
41	movl    %fs:PRIVATE_FUTEX, reg
42# else
43#  define LOAD_PRIVATE_FUTEX_WAIT(reg) \
44	movl	%fs:PRIVATE_FUTEX, reg ; \
45	orl	$FUTEX_WAIT, reg
46# endif
47# define LOAD_PRIVATE_FUTEX_WAKE(reg) \
48	movl    %fs:PRIVATE_FUTEX, reg ; \
49	orl     $FUTEX_WAKE, reg
50# if FUTEX_WAIT == 0
51#  define LOAD_FUTEX_WAIT(reg) \
52	xorl	$FUTEX_PRIVATE_FLAG, reg ; \
53	andl	%fs:PRIVATE_FUTEX, reg
54# else
55#  define LOAD_FUTEX_WAIT(reg) \
56	xorl	$FUTEX_PRIVATE_FLAG, reg ; \
57	andl	%fs:PRIVATE_FUTEX, reg ; \
58	orl	$FUTEX_WAIT, reg
59# endif
60# define LOAD_FUTEX_WAIT_ABS(reg) \
61	xorl	$FUTEX_PRIVATE_FLAG, reg ; \
62	andl	%fs:PRIVATE_FUTEX, reg ; \
63	orl	$FUTEX_WAIT_BITSET | FUTEX_CLOCK_REALTIME, reg
64# define LOAD_FUTEX_WAKE(reg) \
65	xorl	$FUTEX_PRIVATE_FLAG, reg ; \
66	andl	%fs:PRIVATE_FUTEX, reg ; \
67	orl	$FUTEX_WAKE, reg
68#endif
69
70
71/* For the calculation see asm/vsyscall.h.  */
72#define VSYSCALL_ADDR_vgettimeofday	0xffffffffff600000
73
74
75	.globl	__lll_lock_wait_private
76	.type	__lll_lock_wait_private,@function
77	.hidden	__lll_lock_wait_private
78#ifndef IS_IN_libpthread
79	.weak   __lll_lock_wait_private
80#endif
81	.align	16
82__lll_lock_wait_private:
83	cfi_startproc
84	pushq	%r10
85	cfi_adjust_cfa_offset(8)
86	pushq	%rdx
87	cfi_adjust_cfa_offset(8)
88	cfi_offset(%r10, -16)
89	cfi_offset(%rdx, -24)
90	xorq	%r10, %r10	/* No timeout.  */
91	movl	$2, %edx
92	LOAD_PRIVATE_FUTEX_WAIT (%esi)
93
94	cmpl	%edx, %eax	/* NB:	 %edx == 2 */
95	jne	2f
96
971:	movl	$SYS_futex, %eax
98	syscall
99
1002:	movl	%edx, %eax
101	xchgl	%eax, (%rdi)	/* NB:	 lock is implied */
102
103	testl	%eax, %eax
104	jnz	1b
105
106	popq	%rdx
107	cfi_adjust_cfa_offset(-8)
108	cfi_restore(%rdx)
109	popq	%r10
110	cfi_adjust_cfa_offset(-8)
111	cfi_restore(%r10)
112	retq
113	cfi_endproc
114	.size	__lll_lock_wait_private,.-__lll_lock_wait_private
115
116#ifdef NOT_IN_libc
117	.globl	__lll_lock_wait
118	.type	__lll_lock_wait,@function
119	.hidden	__lll_lock_wait
120	.align	16
121__lll_lock_wait:
122	cfi_startproc
123	pushq	%r10
124	cfi_adjust_cfa_offset(8)
125	pushq	%rdx
126	cfi_adjust_cfa_offset(8)
127	cfi_offset(%r10, -16)
128	cfi_offset(%rdx, -24)
129	xorq	%r10, %r10	/* No timeout.  */
130	movl	$2, %edx
131	LOAD_FUTEX_WAIT (%esi)
132
133	cmpl	%edx, %eax	/* NB:	 %edx == 2 */
134	jne	2f
135
1361:	movl	$SYS_futex, %eax
137	syscall
138
1392:	movl	%edx, %eax
140	xchgl	%eax, (%rdi)	/* NB:	 lock is implied */
141
142	testl	%eax, %eax
143	jnz	1b
144
145	popq	%rdx
146	cfi_adjust_cfa_offset(-8)
147	cfi_restore(%rdx)
148	popq	%r10
149	cfi_adjust_cfa_offset(-8)
150	cfi_restore(%r10)
151	retq
152	cfi_endproc
153	.size	__lll_lock_wait,.-__lll_lock_wait
154
155	/*      %rdi: futex
156		%rsi: flags
157		%rdx: timeout
158		%eax: futex value
159	*/
160	.globl	__lll_timedlock_wait
161	.type	__lll_timedlock_wait,@function
162	.hidden	__lll_timedlock_wait
163	.align	16
164__lll_timedlock_wait:
165	cfi_startproc
166# ifndef __ASSUME_FUTEX_CLOCK_REALTIME
167#  ifdef __PIC__
168	cmpl	$0, __have_futex_clock_realtime@GOTOFF(%rip)
169#  else
170	cmpl	$0, __have_futex_clock_realtime
171#  endif
172	je	.Lreltmo
173# endif
174
175	pushq	%r9
176	cfi_adjust_cfa_offset(8)
177	cfi_rel_offset(%r9, 0)
178	movq	%rdx, %r10
179	movl	$0xffffffff, %r9d
180	LOAD_FUTEX_WAIT_ABS (%esi)
181
182	movl	$2, %edx
183	cmpl	%edx, %eax
184	jne	2f
185
1861:	movl	$SYS_futex, %eax
187	movl	$2, %edx
188	syscall
189
1902:	xchgl	%edx, (%rdi)	/* NB:   lock is implied */
191
192	testl	%edx, %edx
193	jz	3f
194
195	cmpl	$-ETIMEDOUT, %eax
196	je	4f
197	cmpl	$-EINVAL, %eax
198	jne	1b
1994:	movl	%eax, %edx
200	negl	%edx
201
2023:	movl	%edx, %eax
203	popq	%r9
204	cfi_adjust_cfa_offset(-8)
205	cfi_restore(%r9)
206	retq
207
208# ifndef __ASSUME_FUTEX_CLOCK_REALTIME
209.Lreltmo:
210	/* Check for a valid timeout value.  */
211	cmpq	$1000000000, 8(%rdx)
212	jae	3f
213
214	pushq	%r8
215	cfi_adjust_cfa_offset(8)
216	pushq	%r9
217	cfi_adjust_cfa_offset(8)
218	pushq	%r12
219	cfi_adjust_cfa_offset(8)
220	pushq	%r13
221	cfi_adjust_cfa_offset(8)
222	pushq	%r14
223	cfi_adjust_cfa_offset(8)
224	cfi_offset(%r8, -16)
225	cfi_offset(%r9, -24)
226	cfi_offset(%r12, -32)
227	cfi_offset(%r13, -40)
228	cfi_offset(%r14, -48)
229	pushq	%rsi
230	cfi_adjust_cfa_offset(8)
231
232	/* Stack frame for the timespec and timeval structs.  */
233	subq	$24, %rsp
234	cfi_adjust_cfa_offset(24)
235
236	movq	%rdi, %r12
237	movq	%rdx, %r13
238
239	movl	$2, %edx
240	xchgl	%edx, (%r12)
241
242	testl	%edx, %edx
243	je	6f
244
2451:
246	/* Get current time.  */
247	movq	%rsp, %rdi
248	xorl	%esi, %esi
249	movq	$VSYSCALL_ADDR_vgettimeofday, %rax
250	/* This is a regular function call, all caller-save registers
251	   might be clobbered.  */
252	callq	*%rax
253
254	/* Compute relative timeout.  */
255	movq	8(%rsp), %rax
256	movl	$1000, %edi
257	mul	%rdi		/* Milli seconds to nano seconds.  */
258	movq	(%r13), %rdi
259	movq	8(%r13), %rsi
260	subq	(%rsp), %rdi
261	subq	%rax, %rsi
262	jns	4f
263	addq	$1000000000, %rsi
264	decq	%rdi
2654:	testq	%rdi, %rdi
266	js	2f		/* Time is already up.  */
267
268	/* Store relative timeout.  */
269	movq	%rdi, (%rsp)
270	movq	%rsi, 8(%rsp)
271
272	/* Futex call.  */
273	movl	$2, %edx
274	movl	$1, %eax
275	movq	%rsp, %r10
276	movl	24(%rsp), %esi
277	LOAD_FUTEX_WAIT (%esi)
278	movq	%r12, %rdi
279	movl	$SYS_futex, %eax
280	syscall
281
282	/* NB: %edx == 2 */
283	xchgl	%edx, (%r12)
284
285	testl	%edx, %edx
286	je	6f
287
288	cmpl	$-ETIMEDOUT, %eax
289	jne	1b
2902:	movl	$ETIMEDOUT, %edx
291
2926:	addq	$32, %rsp
293	cfi_adjust_cfa_offset(-32)
294	popq	%r14
295	cfi_adjust_cfa_offset(-8)
296	cfi_restore(%r14)
297	popq	%r13
298	cfi_adjust_cfa_offset(-8)
299	cfi_restore(%r13)
300	popq	%r12
301	cfi_adjust_cfa_offset(-8)
302	cfi_restore(%r12)
303	popq	%r9
304	cfi_adjust_cfa_offset(-8)
305	cfi_restore(%r9)
306	popq	%r8
307	cfi_adjust_cfa_offset(-8)
308	cfi_restore(%r8)
309	movl	%edx, %eax
310	retq
311
3123:	movl	$EINVAL, %eax
313	retq
314# endif
315	cfi_endproc
316	.size	__lll_timedlock_wait,.-__lll_timedlock_wait
317#endif
318
319
320	.globl	__lll_unlock_wake_private
321	.type	__lll_unlock_wake_private,@function
322	.hidden	__lll_unlock_wake_private
323#ifndef IS_IN_libpthread
324	.weak	__lll_unlock_wake_private
325#endif
326	.align	16
327__lll_unlock_wake_private:
328	cfi_startproc
329	pushq	%rsi
330	cfi_adjust_cfa_offset(8)
331	pushq	%rdx
332	cfi_adjust_cfa_offset(8)
333	cfi_offset(%rsi, -16)
334	cfi_offset(%rdx, -24)
335
336	movl	$0, (%rdi)
337	LOAD_PRIVATE_FUTEX_WAKE (%esi)
338	movl	$1, %edx	/* Wake one thread.  */
339	movl	$SYS_futex, %eax
340	syscall
341
342	popq	%rdx
343	cfi_adjust_cfa_offset(-8)
344	cfi_restore(%rdx)
345	popq	%rsi
346	cfi_adjust_cfa_offset(-8)
347	cfi_restore(%rsi)
348	retq
349	cfi_endproc
350	.size	__lll_unlock_wake_private,.-__lll_unlock_wake_private
351
352#ifdef NOT_IN_libc
353	.globl	__lll_unlock_wake
354	.type	__lll_unlock_wake,@function
355	.hidden	__lll_unlock_wake
356	.align	16
357__lll_unlock_wake:
358	cfi_startproc
359	pushq	%rsi
360	cfi_adjust_cfa_offset(8)
361	pushq	%rdx
362	cfi_adjust_cfa_offset(8)
363	cfi_offset(%rsi, -16)
364	cfi_offset(%rdx, -24)
365
366	movl	$0, (%rdi)
367	LOAD_FUTEX_WAKE (%esi)
368	movl	$1, %edx	/* Wake one thread.  */
369	movl	$SYS_futex, %eax
370	syscall
371
372	popq	%rdx
373	cfi_adjust_cfa_offset(-8)
374	cfi_restore(%rdx)
375	popq	%rsi
376	cfi_adjust_cfa_offset(-8)
377	cfi_restore(%rsi)
378	retq
379	cfi_endproc
380	.size	__lll_unlock_wake,.-__lll_unlock_wake
381
382	.globl	__lll_timedwait_tid
383	.type	__lll_timedwait_tid,@function
384	.hidden	__lll_timedwait_tid
385	.align	16
386__lll_timedwait_tid:
387	cfi_startproc
388	pushq	%r12
389	cfi_adjust_cfa_offset(8)
390	pushq	%r13
391	cfi_adjust_cfa_offset(8)
392	cfi_offset(%r12, -16)
393	cfi_offset(%r13, -24)
394
395	movq	%rdi, %r12
396	movq	%rsi, %r13
397
398	subq	$16, %rsp
399	cfi_adjust_cfa_offset(16)
400
401	/* Get current time.  */
4022:	movq	%rsp, %rdi
403	xorl	%esi, %esi
404	movq	$VSYSCALL_ADDR_vgettimeofday, %rax
405	callq	*%rax
406
407	/* Compute relative timeout.  */
408	movq	8(%rsp), %rax
409	movl	$1000, %edi
410	mul	%rdi		/* Milli seconds to nano seconds.  */
411	movq	(%r13), %rdi
412	movq	8(%r13), %rsi
413	subq	(%rsp), %rdi
414	subq	%rax, %rsi
415	jns	5f
416	addq	$1000000000, %rsi
417	decq	%rdi
4185:	testq	%rdi, %rdi
419	js	6f		/* Time is already up.  */
420
421	movq	%rdi, (%rsp)	/* Store relative timeout.  */
422	movq	%rsi, 8(%rsp)
423
424	movl	(%r12), %edx
425	testl	%edx, %edx
426	jz	4f
427
428	movq	%rsp, %r10
429	/* XXX The kernel so far uses global futex for the wakeup at
430	   all times.  */
431#if FUTEX_WAIT == 0
432	xorl	%esi, %esi
433#else
434	movl	$FUTEX_WAIT, %esi
435#endif
436	movq	%r12, %rdi
437	movl	$SYS_futex, %eax
438	syscall
439
440	cmpl	$0, (%rdi)
441	jne	1f
4424:	xorl	%eax, %eax
443
4448:	addq	$16, %rsp
445	cfi_adjust_cfa_offset(-16)
446	popq	%r13
447	cfi_adjust_cfa_offset(-8)
448	cfi_restore(%r13)
449	popq	%r12
450	cfi_adjust_cfa_offset(-8)
451	cfi_restore(%r12)
452	retq
453
454	cfi_adjust_cfa_offset(32)
4551:	cmpq	$-ETIMEDOUT, %rax
456	jne	2b
457
4586:	movl	$ETIMEDOUT, %eax
459	jmp	8b
460	cfi_endproc
461	.size	__lll_timedwait_tid,.-__lll_timedwait_tid
462#endif
463