1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * x86 TSC related functions
4 */
5 #ifndef _ASM_X86_TSC_H
6 #define _ASM_X86_TSC_H
7
8 #include <asm/asm.h>
9 #include <asm/cpufeature.h>
10 #include <asm/processor.h>
11 #include <asm/msr.h>
12
13 /**
14 * rdtsc() - returns the current TSC without ordering constraints
15 *
16 * rdtsc() returns the result of RDTSC as a 64-bit integer. The
17 * only ordering constraint it supplies is the ordering implied by
18 * "asm volatile": it will put the RDTSC in the place you expect. The
19 * CPU can and will speculatively execute that RDTSC, though, so the
20 * results can be non-monotonic if compared on different CPUs.
21 */
rdtsc(void)22 static __always_inline u64 rdtsc(void)
23 {
24 EAX_EDX_DECLARE_ARGS(val, low, high);
25
26 asm volatile("rdtsc" : EAX_EDX_RET(val, low, high));
27
28 return EAX_EDX_VAL(val, low, high);
29 }
30
31 /**
32 * rdtsc_ordered() - read the current TSC in program order
33 *
34 * rdtsc_ordered() returns the result of RDTSC as a 64-bit integer.
35 * It is ordered like a load to a global in-memory counter. It should
36 * be impossible to observe non-monotonic rdtsc_unordered() behavior
37 * across multiple CPUs as long as the TSC is synced.
38 */
rdtsc_ordered(void)39 static __always_inline u64 rdtsc_ordered(void)
40 {
41 EAX_EDX_DECLARE_ARGS(val, low, high);
42
43 /*
44 * The RDTSC instruction is not ordered relative to memory
45 * access. The Intel SDM and the AMD APM are both vague on this
46 * point, but empirically an RDTSC instruction can be
47 * speculatively executed before prior loads. An RDTSC
48 * immediately after an appropriate barrier appears to be
49 * ordered as a normal load, that is, it provides the same
50 * ordering guarantees as reading from a global memory location
51 * that some other imaginary CPU is updating continuously with a
52 * time stamp.
53 *
54 * Thus, use the preferred barrier on the respective CPU, aiming for
55 * RDTSCP as the default.
56 */
57 asm volatile(ALTERNATIVE_2("rdtsc",
58 "lfence; rdtsc", X86_FEATURE_LFENCE_RDTSC,
59 "rdtscp", X86_FEATURE_RDTSCP)
60 : EAX_EDX_RET(val, low, high)
61 /* RDTSCP clobbers ECX with MSR_TSC_AUX. */
62 :: "ecx");
63
64 return EAX_EDX_VAL(val, low, high);
65 }
66
67 /*
68 * Standard way to access the cycle counter.
69 */
70 typedef unsigned long long cycles_t;
71
72 extern unsigned int cpu_khz;
73 extern unsigned int tsc_khz;
74
75 extern void disable_TSC(void);
76
get_cycles(void)77 static inline cycles_t get_cycles(void)
78 {
79 if (!IS_ENABLED(CONFIG_X86_TSC) &&
80 !cpu_feature_enabled(X86_FEATURE_TSC))
81 return 0;
82 return rdtsc();
83 }
84 #define get_cycles get_cycles
85
86 extern void tsc_early_init(void);
87 extern void tsc_init(void);
88 extern void mark_tsc_unstable(char *reason);
89 extern int unsynchronized_tsc(void);
90 extern int check_tsc_unstable(void);
91 extern void mark_tsc_async_resets(char *reason);
92 extern unsigned long native_calibrate_cpu_early(void);
93 extern unsigned long native_calibrate_tsc(void);
94 extern unsigned long long native_sched_clock_from_tsc(u64 tsc);
95
96 extern int tsc_clocksource_reliable;
97 #ifdef CONFIG_X86_TSC
98 extern bool tsc_async_resets;
99 #else
100 # define tsc_async_resets false
101 #endif
102
103 /*
104 * Boot-time check whether the TSCs are synchronized across
105 * all CPUs/cores:
106 */
107 #ifdef CONFIG_X86_TSC
108 extern bool tsc_store_and_check_tsc_adjust(bool bootcpu);
109 extern void tsc_verify_tsc_adjust(bool resume);
110 extern void check_tsc_sync_target(void);
111 #else
tsc_store_and_check_tsc_adjust(bool bootcpu)112 static inline bool tsc_store_and_check_tsc_adjust(bool bootcpu) { return false; }
tsc_verify_tsc_adjust(bool resume)113 static inline void tsc_verify_tsc_adjust(bool resume) { }
check_tsc_sync_target(void)114 static inline void check_tsc_sync_target(void) { }
115 #endif
116
117 extern int notsc_setup(char *);
118 extern void tsc_save_sched_clock_state(void);
119 extern void tsc_restore_sched_clock_state(void);
120
121 unsigned long cpu_khz_from_msr(void);
122
123 #endif /* _ASM_X86_TSC_H */
124