1/*
2 * Relocate a kexec_image to its destination and call it.
3 *
4 * Copyright (C) 2013 Citrix Systems R&D Ltd.
5 *
6 * Portions derived from Linux's arch/x86/kernel/relocate_kernel_64.S.
7 *
8 *   Copyright (C) 2002-2005 Eric Biederman  <ebiederm@xmission.com>
9 *
10 * This source code is licensed under the GNU General Public License,
11 * Version 2.  See the file COPYING for more details.
12 */
13
14        .file __FILE__
15
16#include <xen/kimage.h>
17
18#include <asm/asm_defns.h>
19#include <asm/msr-index.h>
20#include <asm/page.h>
21#include <asm/machine_kexec.h>
22
23        .section .text.kexec, "ax", @progbits
24        .align PAGE_SIZE
25        .code64
26
27ENTRY(kexec_reloc)
28        /* %rdi - code page maddr */
29        /* %rsi - page table maddr */
30        /* %rdx - indirection page maddr */
31        /* %rcx - entry maddr (%rbp) */
32        /* %r8 - flags */
33
34        movq    %rcx, %rbp
35
36        /* Setup stack. */
37        leaq    (.Lreloc_stack_base - kexec_reloc)(%rdi), %rsp
38
39        /* Load reloc page table. */
40        movq    %rsi, %cr3
41
42        /* Jump to identity mapped code. */
43        leaq    (.L_identity_mapped - kexec_reloc)(%rdi), %rax
44        jmpq    *%rax
45
46.L_identity_mapped:
47        /*
48         * Set cr0 to a known state:
49         *  - Paging enabled
50         *  - Alignment check disabled
51         *  - Write protect disabled
52         *  - No task switch
53         *  - Don't do FP software emulation.
54         *  - Protected mode enabled
55         */
56        movq    %cr0, %rax
57        andl    $~(X86_CR0_AM | X86_CR0_WP | X86_CR0_TS | X86_CR0_EM), %eax
58        orl     $(X86_CR0_PG | X86_CR0_PE), %eax
59        movq    %rax, %cr0
60
61        /*
62         * Set cr4 to a known state:
63         *  - physical address extension enabled
64         */
65        movl    $X86_CR4_PAE, %eax
66        movq    %rax, %cr4
67
68        movq    %rdx, %rdi
69        call    relocate_pages
70
71        /* Need to switch to 32-bit mode? */
72        testq   $KEXEC_RELOC_FLAG_COMPAT, %r8
73        jnz     .L_call_32_bit
74
75.L_call_64_bit:
76        /* Call the image entry point.  This should never return. */
77        callq   *%rbp
78        ud2
79
80.L_call_32_bit:
81        /* Setup IDT. */
82        lidt    compat_mode_idt(%rip)
83
84        /* Load compat GDT. */
85        leaq    compat_mode_gdt(%rip), %rax
86        movq    %rax, (compat_mode_gdt_desc + 2)(%rip)
87        lgdt    compat_mode_gdt_desc(%rip)
88
89        /* Enter compatibility mode. */
90        lea     compatibility_mode(%rip), %rax
91        push    $0x10
92        push    %rax
93        lretq
94
95relocate_pages:
96        /* %rdi - indirection page maddr */
97        pushq   %rbx
98
99        cld
100        movq    %rdi, %rbx
101        xorl    %edi, %edi
102        xorl    %esi, %esi
103
104.L_next_entry: /* top, read another word for the indirection page */
105
106        movq    (%rbx), %rcx
107        addq    $8, %rbx
108.L_is_dest:
109        testb   $IND_DESTINATION, %cl
110        jz      .L_is_ind
111        movq    %rcx, %rdi
112        andq    $PAGE_MASK, %rdi
113        jmp     .L_next_entry
114.L_is_ind:
115        testb   $IND_INDIRECTION, %cl
116        jz      .L_is_done
117        movq    %rcx, %rbx
118        andq    $PAGE_MASK, %rbx
119        jmp     .L_next_entry
120.L_is_done:
121        testb   $IND_DONE, %cl
122        jnz     .L_done
123.L_is_source:
124        testb   $IND_SOURCE, %cl
125        jz      .L_is_zero
126        movq    %rcx, %rsi      /* For every source page do a copy */
127        andq    $PAGE_MASK, %rsi
128        movl    $(PAGE_SIZE / 8), %ecx
129        rep movsq
130        jmp     .L_next_entry
131.L_is_zero:
132        testb   $IND_ZERO, %cl
133        jz      .L_next_entry
134        movl    $(PAGE_SIZE / 8), %ecx  /* Zero the destination page. */
135        xorl    %eax, %eax
136        rep stosq
137        jmp     .L_next_entry
138.L_done:
139        popq    %rbx
140        ret
141
142        .code32
143
144compatibility_mode:
145        /* Setup some sane segments. */
146        movl    $0x0008, %eax
147        movl    %eax, %ds
148        movl    %eax, %es
149        movl    %eax, %fs
150        movl    %eax, %gs
151        movl    %eax, %ss
152
153        /* Disable paging and therefore leave 64 bit mode. */
154        movl    %cr0, %eax
155        andl    $~X86_CR0_PG, %eax
156        movl    %eax, %cr0
157
158        /* Disable long mode */
159        movl    $MSR_EFER, %ecx
160        rdmsr
161        andl    $~EFER_LME, %eax
162        wrmsr
163
164        /* Clear cr4 to disable PAE. */
165        xorl    %eax, %eax
166        movl    %eax, %cr4
167
168        /* Call the image entry point.  This should never return. */
169        call    *%ebp
170        ud2
171
172        .align 4
173compat_mode_gdt_desc:
174        .word .Lcompat_mode_gdt_end - compat_mode_gdt -1
175        .quad 0x0000000000000000     /* set in call_32_bit above */
176
177        .type compat_mode_gdt_desc, @object
178        .size compat_mode_gdt_desc, . - compat_mode_gdt_desc
179
180        .align 8
181compat_mode_gdt:
182        .quad 0x0000000000000000     /* null                              */
183        .quad 0x00cf93000000ffff     /* 0x0008 ring 0 data                */
184        .quad 0x00cf9b000000ffff     /* 0x0010 ring 0 code, compatibility */
185.Lcompat_mode_gdt_end:
186
187        .type compat_mode_gdt, @object
188        .size compat_mode_gdt, . - compat_mode_gdt
189
190compat_mode_idt:
191        .word 0                      /* limit */
192        .long 0                      /* base */
193
194        .type compat_mode_idt, @object
195        .size compat_mode_idt, . - compat_mode_idt
196
197        /*
198         * 16 words of stack are more than enough.
199         */
200        .align 8
201reloc_stack:
202        .fill 16,8,0
203.Lreloc_stack_base:
204
205        .type reloc_stack, @object
206        .size reloc_stack, . - reloc_stack
207