1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * shadow.c - Shadow Variables
4  *
5  * Copyright (C) 2014 Josh Poimboeuf <jpoimboe@redhat.com>
6  * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
7  * Copyright (C) 2017 Joe Lawrence <joe.lawrence@redhat.com>
8  */
9 
10 /**
11  * DOC: Shadow variable API concurrency notes:
12  *
13  * The shadow variable API provides a simple relationship between an
14  * <obj, id> pair and a pointer value.  It is the responsibility of the
15  * caller to provide any mutual exclusion required of the shadow data.
16  *
17  * Once a shadow variable is attached to its parent object via the
18  * klp_shadow_*alloc() API calls, it is considered live: any subsequent
19  * call to klp_shadow_get() may then return the shadow variable's data
20  * pointer.  Callers of klp_shadow_*alloc() should prepare shadow data
21  * accordingly.
22  *
23  * The klp_shadow_*alloc() API calls may allocate memory for new shadow
24  * variable structures.  Their implementation does not call kmalloc
25  * inside any spinlocks, but API callers should pass GFP flags according
26  * to their specific needs.
27  *
28  * The klp_shadow_hash is an RCU-enabled hashtable and is safe against
29  * concurrent klp_shadow_free() and klp_shadow_get() operations.
30  */
31 
32 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
33 
34 #include <linux/hashtable.h>
35 #include <linux/slab.h>
36 #include <linux/livepatch.h>
37 
38 static DEFINE_HASHTABLE(klp_shadow_hash, 12);
39 
40 /*
41  * klp_shadow_lock provides exclusive access to the klp_shadow_hash and
42  * the shadow variables it references.
43  */
44 static DEFINE_SPINLOCK(klp_shadow_lock);
45 
46 /**
47  * struct klp_shadow - shadow variable structure
48  * @node:	klp_shadow_hash hash table node
49  * @rcu_head:	RCU is used to safely free this structure
50  * @obj:	pointer to parent object
51  * @id:		data identifier
52  * @data:	data area
53  */
54 struct klp_shadow {
55 	struct hlist_node node;
56 	struct rcu_head rcu_head;
57 	void *obj;
58 	unsigned long id;
59 	char data[];
60 };
61 
62 /**
63  * klp_shadow_match() - verify a shadow variable matches given <obj, id>
64  * @shadow:	shadow variable to match
65  * @obj:	pointer to parent object
66  * @id:		data identifier
67  *
68  * Return: true if the shadow variable matches.
69  */
klp_shadow_match(struct klp_shadow * shadow,void * obj,unsigned long id)70 static inline bool klp_shadow_match(struct klp_shadow *shadow, void *obj,
71 				unsigned long id)
72 {
73 	return shadow->obj == obj && shadow->id == id;
74 }
75 
76 /**
77  * klp_shadow_get() - retrieve a shadow variable data pointer
78  * @obj:	pointer to parent object
79  * @id:		data identifier
80  *
81  * Return: the shadow variable data element, NULL on failure.
82  */
klp_shadow_get(void * obj,unsigned long id)83 void *klp_shadow_get(void *obj, unsigned long id)
84 {
85 	struct klp_shadow *shadow;
86 
87 	rcu_read_lock();
88 
89 	hash_for_each_possible_rcu(klp_shadow_hash, shadow, node,
90 				   (unsigned long)obj) {
91 
92 		if (klp_shadow_match(shadow, obj, id)) {
93 			rcu_read_unlock();
94 			return shadow->data;
95 		}
96 	}
97 
98 	rcu_read_unlock();
99 
100 	return NULL;
101 }
102 EXPORT_SYMBOL_GPL(klp_shadow_get);
103 
__klp_shadow_get_or_alloc(void * obj,unsigned long id,size_t size,gfp_t gfp_flags,klp_shadow_ctor_t ctor,void * ctor_data,bool warn_on_exist)104 static void *__klp_shadow_get_or_alloc(void *obj, unsigned long id,
105 				       size_t size, gfp_t gfp_flags,
106 				       klp_shadow_ctor_t ctor, void *ctor_data,
107 				       bool warn_on_exist)
108 {
109 	struct klp_shadow *new_shadow;
110 	void *shadow_data;
111 	unsigned long flags;
112 
113 	/* Check if the shadow variable already exists */
114 	shadow_data = klp_shadow_get(obj, id);
115 	if (shadow_data)
116 		goto exists;
117 
118 	/*
119 	 * Allocate a new shadow variable.  Fill it with zeroes by default.
120 	 * More complex setting can be done by @ctor function.  But it is
121 	 * called only when the buffer is really used (under klp_shadow_lock).
122 	 */
123 	new_shadow = kzalloc(size + sizeof(*new_shadow), gfp_flags);
124 	if (!new_shadow)
125 		return NULL;
126 
127 	/* Look for <obj, id> again under the lock */
128 	spin_lock_irqsave(&klp_shadow_lock, flags);
129 	shadow_data = klp_shadow_get(obj, id);
130 	if (unlikely(shadow_data)) {
131 		/*
132 		 * Shadow variable was found, throw away speculative
133 		 * allocation.
134 		 */
135 		spin_unlock_irqrestore(&klp_shadow_lock, flags);
136 		kfree(new_shadow);
137 		goto exists;
138 	}
139 
140 	new_shadow->obj = obj;
141 	new_shadow->id = id;
142 
143 	if (ctor) {
144 		int err;
145 
146 		err = ctor(obj, new_shadow->data, ctor_data);
147 		if (err) {
148 			spin_unlock_irqrestore(&klp_shadow_lock, flags);
149 			kfree(new_shadow);
150 			pr_err("Failed to construct shadow variable <%p, %lx> (%d)\n",
151 			       obj, id, err);
152 			return NULL;
153 		}
154 	}
155 
156 	/* No <obj, id> found, so attach the newly allocated one */
157 	hash_add_rcu(klp_shadow_hash, &new_shadow->node,
158 		     (unsigned long)new_shadow->obj);
159 	spin_unlock_irqrestore(&klp_shadow_lock, flags);
160 
161 	return new_shadow->data;
162 
163 exists:
164 	if (warn_on_exist) {
165 		WARN(1, "Duplicate shadow variable <%p, %lx>\n", obj, id);
166 		return NULL;
167 	}
168 
169 	return shadow_data;
170 }
171 
172 /**
173  * klp_shadow_alloc() - allocate and add a new shadow variable
174  * @obj:	pointer to parent object
175  * @id:		data identifier
176  * @size:	size of attached data
177  * @gfp_flags:	GFP mask for allocation
178  * @ctor:	custom constructor to initialize the shadow data (optional)
179  * @ctor_data:	pointer to any data needed by @ctor (optional)
180  *
181  * Allocates @size bytes for new shadow variable data using @gfp_flags.
182  * The data are zeroed by default.  They are further initialized by @ctor
183  * function if it is not NULL.  The new shadow variable is then added
184  * to the global hashtable.
185  *
186  * If an existing <obj, id> shadow variable can be found, this routine will
187  * issue a WARN, exit early and return NULL.
188  *
189  * This function guarantees that the constructor function is called only when
190  * the variable did not exist before.  The cost is that @ctor is called
191  * in atomic context under a spin lock.
192  *
193  * Return: the shadow variable data element, NULL on duplicate or
194  * failure.
195  */
klp_shadow_alloc(void * obj,unsigned long id,size_t size,gfp_t gfp_flags,klp_shadow_ctor_t ctor,void * ctor_data)196 void *klp_shadow_alloc(void *obj, unsigned long id,
197 		       size_t size, gfp_t gfp_flags,
198 		       klp_shadow_ctor_t ctor, void *ctor_data)
199 {
200 	return __klp_shadow_get_or_alloc(obj, id, size, gfp_flags,
201 					 ctor, ctor_data, true);
202 }
203 EXPORT_SYMBOL_GPL(klp_shadow_alloc);
204 
205 /**
206  * klp_shadow_get_or_alloc() - get existing or allocate a new shadow variable
207  * @obj:	pointer to parent object
208  * @id:		data identifier
209  * @size:	size of attached data
210  * @gfp_flags:	GFP mask for allocation
211  * @ctor:	custom constructor to initialize the shadow data (optional)
212  * @ctor_data:	pointer to any data needed by @ctor (optional)
213  *
214  * Returns a pointer to existing shadow data if an <obj, id> shadow
215  * variable is already present.  Otherwise, it creates a new shadow
216  * variable like klp_shadow_alloc().
217  *
218  * This function guarantees that only one shadow variable exists with the given
219  * @id for the given @obj.  It also guarantees that the constructor function
220  * will be called only when the variable did not exist before.  The cost is
221  * that @ctor is called in atomic context under a spin lock.
222  *
223  * Return: the shadow variable data element, NULL on failure.
224  */
klp_shadow_get_or_alloc(void * obj,unsigned long id,size_t size,gfp_t gfp_flags,klp_shadow_ctor_t ctor,void * ctor_data)225 void *klp_shadow_get_or_alloc(void *obj, unsigned long id,
226 			      size_t size, gfp_t gfp_flags,
227 			      klp_shadow_ctor_t ctor, void *ctor_data)
228 {
229 	return __klp_shadow_get_or_alloc(obj, id, size, gfp_flags,
230 					 ctor, ctor_data, false);
231 }
232 EXPORT_SYMBOL_GPL(klp_shadow_get_or_alloc);
233 
klp_shadow_free_struct(struct klp_shadow * shadow,klp_shadow_dtor_t dtor)234 static void klp_shadow_free_struct(struct klp_shadow *shadow,
235 				   klp_shadow_dtor_t dtor)
236 {
237 	hash_del_rcu(&shadow->node);
238 	if (dtor)
239 		dtor(shadow->obj, shadow->data);
240 	kfree_rcu(shadow, rcu_head);
241 }
242 
243 /**
244  * klp_shadow_free() - detach and free a <obj, id> shadow variable
245  * @obj:	pointer to parent object
246  * @id:		data identifier
247  * @dtor:	custom callback that can be used to unregister the variable
248  *		and/or free data that the shadow variable points to (optional)
249  *
250  * This function releases the memory for this <obj, id> shadow variable
251  * instance, callers should stop referencing it accordingly.
252  */
klp_shadow_free(void * obj,unsigned long id,klp_shadow_dtor_t dtor)253 void klp_shadow_free(void *obj, unsigned long id, klp_shadow_dtor_t dtor)
254 {
255 	struct klp_shadow *shadow;
256 	unsigned long flags;
257 
258 	spin_lock_irqsave(&klp_shadow_lock, flags);
259 
260 	/* Delete <obj, id> from hash */
261 	hash_for_each_possible(klp_shadow_hash, shadow, node,
262 			       (unsigned long)obj) {
263 
264 		if (klp_shadow_match(shadow, obj, id)) {
265 			klp_shadow_free_struct(shadow, dtor);
266 			break;
267 		}
268 	}
269 
270 	spin_unlock_irqrestore(&klp_shadow_lock, flags);
271 }
272 EXPORT_SYMBOL_GPL(klp_shadow_free);
273 
274 /**
275  * klp_shadow_free_all() - detach and free all <_, id> shadow variables
276  * @id:		data identifier
277  * @dtor:	custom callback that can be used to unregister the variable
278  *		and/or free data that the shadow variable points to (optional)
279  *
280  * This function releases the memory for all <_, id> shadow variable
281  * instances, callers should stop referencing them accordingly.
282  */
klp_shadow_free_all(unsigned long id,klp_shadow_dtor_t dtor)283 void klp_shadow_free_all(unsigned long id, klp_shadow_dtor_t dtor)
284 {
285 	struct klp_shadow *shadow;
286 	unsigned long flags;
287 	int i;
288 
289 	spin_lock_irqsave(&klp_shadow_lock, flags);
290 
291 	/* Delete all <_, id> from hash */
292 	hash_for_each(klp_shadow_hash, i, shadow, node) {
293 		if (klp_shadow_match(shadow, shadow->obj, id))
294 			klp_shadow_free_struct(shadow, dtor);
295 	}
296 
297 	spin_unlock_irqrestore(&klp_shadow_lock, flags);
298 }
299 EXPORT_SYMBOL_GPL(klp_shadow_free_all);
300