1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Guest memory management for KVM/s390 nested VMs.
4 *
5 * Copyright IBM Corp. 2008, 2020, 2024
6 *
7 * Author(s): Claudio Imbrenda <imbrenda@linux.ibm.com>
8 * Martin Schwidefsky <schwidefsky@de.ibm.com>
9 * David Hildenbrand <david@redhat.com>
10 * Janosch Frank <frankja@linux.vnet.ibm.com>
11 */
12
13 #include <linux/compiler.h>
14 #include <linux/kvm.h>
15 #include <linux/kvm_host.h>
16 #include <linux/pgtable.h>
17 #include <linux/pagemap.h>
18 #include <linux/mman.h>
19
20 #include <asm/lowcore.h>
21 #include <asm/gmap.h>
22 #include <asm/uv.h>
23
24 #include "kvm-s390.h"
25
26 /**
27 * gmap_find_shadow - find a specific asce in the list of shadow tables
28 * @parent: pointer to the parent gmap
29 * @asce: ASCE for which the shadow table is created
30 * @edat_level: edat level to be used for the shadow translation
31 *
32 * Returns the pointer to a gmap if a shadow table with the given asce is
33 * already available, ERR_PTR(-EAGAIN) if another one is just being created,
34 * otherwise NULL
35 *
36 * Context: Called with parent->shadow_lock held
37 */
gmap_find_shadow(struct gmap * parent,unsigned long asce,int edat_level)38 static struct gmap *gmap_find_shadow(struct gmap *parent, unsigned long asce, int edat_level)
39 {
40 struct gmap *sg;
41
42 lockdep_assert_held(&parent->shadow_lock);
43 list_for_each_entry(sg, &parent->children, list) {
44 if (!gmap_shadow_valid(sg, asce, edat_level))
45 continue;
46 if (!sg->initialized)
47 return ERR_PTR(-EAGAIN);
48 refcount_inc(&sg->ref_count);
49 return sg;
50 }
51 return NULL;
52 }
53
54 /**
55 * gmap_shadow - create/find a shadow guest address space
56 * @parent: pointer to the parent gmap
57 * @asce: ASCE for which the shadow table is created
58 * @edat_level: edat level to be used for the shadow translation
59 *
60 * The pages of the top level page table referred by the asce parameter
61 * will be set to read-only and marked in the PGSTEs of the kvm process.
62 * The shadow table will be removed automatically on any change to the
63 * PTE mapping for the source table.
64 *
65 * Returns a guest address space structure, ERR_PTR(-ENOMEM) if out of memory,
66 * ERR_PTR(-EAGAIN) if the caller has to retry and ERR_PTR(-EFAULT) if the
67 * parent gmap table could not be protected.
68 */
gmap_shadow(struct gmap * parent,unsigned long asce,int edat_level)69 struct gmap *gmap_shadow(struct gmap *parent, unsigned long asce, int edat_level)
70 {
71 struct gmap *sg, *new;
72 unsigned long limit;
73 int rc;
74
75 if (KVM_BUG_ON(parent->mm->context.allow_gmap_hpage_1m, (struct kvm *)parent->private) ||
76 KVM_BUG_ON(gmap_is_shadow(parent), (struct kvm *)parent->private))
77 return ERR_PTR(-EFAULT);
78 spin_lock(&parent->shadow_lock);
79 sg = gmap_find_shadow(parent, asce, edat_level);
80 spin_unlock(&parent->shadow_lock);
81 if (sg)
82 return sg;
83 /* Create a new shadow gmap */
84 limit = -1UL >> (33 - (((asce & _ASCE_TYPE_MASK) >> 2) * 11));
85 if (asce & _ASCE_REAL_SPACE)
86 limit = -1UL;
87 new = gmap_alloc(limit);
88 if (!new)
89 return ERR_PTR(-ENOMEM);
90 new->mm = parent->mm;
91 new->parent = gmap_get(parent);
92 new->private = parent->private;
93 new->orig_asce = asce;
94 new->edat_level = edat_level;
95 new->initialized = false;
96 spin_lock(&parent->shadow_lock);
97 /* Recheck if another CPU created the same shadow */
98 sg = gmap_find_shadow(parent, asce, edat_level);
99 if (sg) {
100 spin_unlock(&parent->shadow_lock);
101 gmap_free(new);
102 return sg;
103 }
104 if (asce & _ASCE_REAL_SPACE) {
105 /* only allow one real-space gmap shadow */
106 list_for_each_entry(sg, &parent->children, list) {
107 if (sg->orig_asce & _ASCE_REAL_SPACE) {
108 spin_lock(&sg->guest_table_lock);
109 gmap_unshadow(sg);
110 spin_unlock(&sg->guest_table_lock);
111 list_del(&sg->list);
112 gmap_put(sg);
113 break;
114 }
115 }
116 }
117 refcount_set(&new->ref_count, 2);
118 list_add(&new->list, &parent->children);
119 if (asce & _ASCE_REAL_SPACE) {
120 /* nothing to protect, return right away */
121 new->initialized = true;
122 spin_unlock(&parent->shadow_lock);
123 return new;
124 }
125 spin_unlock(&parent->shadow_lock);
126 /* protect after insertion, so it will get properly invalidated */
127 mmap_read_lock(parent->mm);
128 rc = __kvm_s390_mprotect_many(parent, asce & _ASCE_ORIGIN,
129 ((asce & _ASCE_TABLE_LENGTH) + 1),
130 PROT_READ, GMAP_NOTIFY_SHADOW);
131 mmap_read_unlock(parent->mm);
132 spin_lock(&parent->shadow_lock);
133 new->initialized = true;
134 if (rc) {
135 list_del(&new->list);
136 gmap_free(new);
137 new = ERR_PTR(rc);
138 }
139 spin_unlock(&parent->shadow_lock);
140 return new;
141 }
142