1 /*
2 * Copyright (c) 2015 Travis Geiselbrecht
3 *
4 * Use of this source code is governed by a MIT-style
5 * license that can be found in the LICENSE file or at
6 * https://opensource.org/licenses/MIT
7 */
8 #include <dev/virtio/net.h>
9
10 #include <stdlib.h>
11 #include <lk/debug.h>
12 #include <assert.h>
13 #include <lk/trace.h>
14 #include <lk/compiler.h>
15 #include <lk/list.h>
16 #include <string.h>
17 #include <lk/err.h>
18 #include <kernel/thread.h>
19 #include <kernel/event.h>
20 #include <kernel/spinlock.h>
21 #include <lib/pktbuf.h>
22 #include <lib/minip.h>
23
24 #define LOCAL_TRACE 0
25
26 struct virtio_net_config {
27 uint8_t mac[6];
28 uint16_t status;
29 uint16_t max_virtqueue_pairs;
30 };
31 STATIC_ASSERT(sizeof(struct virtio_net_config) == 10);
32
33 struct virtio_net_hdr {
34 uint8_t flags;
35 uint8_t gso_type;
36 uint16_t hdr_len;
37 uint16_t gso_size;
38 uint16_t csum_start;
39 uint16_t csum_offset;
40 uint16_t num_buffers; // unused in tx
41 };
42 STATIC_ASSERT(sizeof(struct virtio_net_hdr) == 12);
43
44 #define VIRTIO_NET_F_CSUM (1<<0)
45 #define VIRTIO_NET_F_GUEST_CSUM (1<<1)
46 #define VIRTIO_NET_F_CTRL_GUEST_OFFLOADS (1<<2)
47 #define VIRTIO_NET_F_MAC (1<<5)
48 #define VIRTIO_NET_F_GSO (1<<6)
49 #define VIRTIO_NET_F_GUEST_TSO4 (1<<7)
50 #define VIRTIO_NET_F_GUEST_TSO6 (1<<8)
51 #define VIRTIO_NET_F_GUEST_ECN (1<<9)
52 #define VIRTIO_NET_F_GUEST_UFO (1<<10)
53 #define VIRTIO_NET_F_HOST_TSO4 (1<<11)
54 #define VIRTIO_NET_F_HOST_TSO6 (1<<12)
55 #define VIRTIO_NET_F_HOST_ECN (1<<13)
56 #define VIRTIO_NET_F_HOST_UFO (1<<14)
57 #define VIRTIO_NET_F_MRG_RXBUF (1<<15)
58 #define VIRTIO_NET_F_STATUS (1<<16)
59 #define VIRTIO_NET_F_CTRL_VQ (1<<17)
60 #define VIRTIO_NET_F_CTRL_RX (1<<18)
61 #define VIRTIO_NET_F_CTRL_VLAN (1<<19)
62 #define VIRTIO_NET_F_GUEST_ANNOUNCE (1<<21)
63 #define VIRTIO_NET_F_MQ (1<<22)
64 #define VIRTIO_NET_F_CTRL_MAC_ADDR (1<<23)
65
66 #define VIRTIO_NET_S_LINK_UP (1<<0)
67 #define VIRTIO_NET_S_ANNOUNCE (1<<1)
68
69 #define TX_RING_SIZE 16
70 #define RX_RING_SIZE 16
71
72 #define RING_RX 0
73 #define RING_TX 1
74
75 #define VIRTIO_NET_MSS 1514
76
77 struct virtio_net_dev {
78 struct virtio_device *dev;
79 bool started;
80
81 struct virtio_net_config *config;
82
83 spin_lock_t lock;
84 event_t rx_event;
85
86 /* list of active tx/rx packets to be freed at irq time */
87 pktbuf_t *pending_tx_packet[TX_RING_SIZE];
88 pktbuf_t *pending_rx_packet[RX_RING_SIZE];
89
90 uint tx_pending_count;
91 struct list_node completed_rx_queue;
92 };
93
94 static enum handler_return virtio_net_irq_driver_callback(struct virtio_device *dev, uint ring, const struct vring_used_elem *e);
95 static int virtio_net_rx_worker(void *arg);
96 static status_t virtio_net_queue_rx(struct virtio_net_dev *ndev, pktbuf_t *p);
97
98 // XXX remove need for this
99 static struct virtio_net_dev *the_ndev;
100
dump_feature_bits(uint32_t feature)101 static void dump_feature_bits(uint32_t feature) {
102 printf("virtio-net host features (0x%x):", feature);
103 if (feature & VIRTIO_NET_F_CSUM) printf(" CSUM");
104 if (feature & VIRTIO_NET_F_GUEST_CSUM) printf(" GUEST_CSUM");
105 if (feature & VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) printf(" CTRL_GUEST_OFFLOADS");
106 if (feature & VIRTIO_NET_F_MAC) printf(" MAC");
107 if (feature & VIRTIO_NET_F_GSO) printf(" GSO");
108 if (feature & VIRTIO_NET_F_GUEST_TSO4) printf(" GUEST_TSO4");
109 if (feature & VIRTIO_NET_F_GUEST_TSO6) printf(" GUEST_TSO6");
110 if (feature & VIRTIO_NET_F_GUEST_ECN) printf(" GUEST_ECN");
111 if (feature & VIRTIO_NET_F_GUEST_UFO) printf(" GUEST_UFO");
112 if (feature & VIRTIO_NET_F_HOST_TSO4) printf(" HOST_TSO4");
113 if (feature & VIRTIO_NET_F_HOST_TSO6) printf(" HOST_TSO6");
114 if (feature & VIRTIO_NET_F_HOST_ECN) printf(" HOST_ECN");
115 if (feature & VIRTIO_NET_F_HOST_UFO) printf(" HOST_UFO");
116 if (feature & VIRTIO_NET_F_MRG_RXBUF) printf(" MRG_RXBUF");
117 if (feature & VIRTIO_NET_F_STATUS) printf(" STATUS");
118 if (feature & VIRTIO_NET_F_CTRL_VQ) printf(" CTRL_VQ");
119 if (feature & VIRTIO_NET_F_CTRL_RX) printf(" CTRL_RX");
120 if (feature & VIRTIO_NET_F_CTRL_VLAN) printf(" CTRL_VLAN");
121 if (feature & VIRTIO_NET_F_GUEST_ANNOUNCE) printf(" GUEST_ANNOUNCE");
122 if (feature & VIRTIO_NET_F_MQ) printf(" MQ");
123 if (feature & VIRTIO_NET_F_CTRL_MAC_ADDR) printf(" CTRL_MAC_ADDR");
124 printf("\n");
125 }
126
virtio_net_init(struct virtio_device * dev,uint32_t host_features)127 status_t virtio_net_init(struct virtio_device *dev, uint32_t host_features) {
128 LTRACEF("dev %p, host_features 0x%x\n", dev, host_features);
129
130 /* allocate a new net device */
131 struct virtio_net_dev *ndev = calloc(1, sizeof(struct virtio_net_dev));
132 if (!ndev)
133 return ERR_NO_MEMORY;
134
135 ndev->dev = dev;
136 dev->priv = ndev;
137 ndev->started = false;
138
139 ndev->lock = SPIN_LOCK_INITIAL_VALUE;
140 event_init(&ndev->rx_event, false, EVENT_FLAG_AUTOUNSIGNAL);
141 list_initialize(&ndev->completed_rx_queue);
142
143 ndev->config = (struct virtio_net_config *)dev->config_ptr;
144
145 /* ack and set the driver status bit */
146 virtio_status_acknowledge_driver(dev);
147
148 // XXX check features bits and ack/nak them
149 dump_feature_bits(host_features);
150
151 /* set our irq handler */
152 dev->irq_driver_callback = &virtio_net_irq_driver_callback;
153
154 /* set DRIVER_OK */
155 virtio_status_driver_ok(dev);
156
157 /* allocate a pair of virtio rings */
158 virtio_alloc_ring(dev, RING_RX, RX_RING_SIZE); // rx
159 virtio_alloc_ring(dev, RING_TX, TX_RING_SIZE); // tx
160
161 the_ndev = ndev;
162
163 return NO_ERROR;
164 }
165
virtio_net_start(void)166 status_t virtio_net_start(void) {
167 if (the_ndev->started)
168 return ERR_ALREADY_STARTED;
169
170 the_ndev->started = true;
171
172 /* start the rx worker thread */
173 thread_resume(thread_create("virtio_net_rx", &virtio_net_rx_worker, (void *)the_ndev, HIGH_PRIORITY, DEFAULT_STACK_SIZE));
174
175 /* queue up a bunch of rxes */
176 for (uint i = 0; i < RX_RING_SIZE - 1; i++) {
177 pktbuf_t *p = pktbuf_alloc();
178 if (p) {
179 virtio_net_queue_rx(the_ndev, p);
180 }
181 }
182
183 return NO_ERROR;
184 }
185
virtio_net_queue_tx_pktbuf(struct virtio_net_dev * ndev,pktbuf_t * p2)186 static status_t virtio_net_queue_tx_pktbuf(struct virtio_net_dev *ndev, pktbuf_t *p2) {
187 struct virtio_device *vdev = ndev->dev;
188
189 uint16_t i;
190 pktbuf_t *p;
191
192 DEBUG_ASSERT(ndev);
193
194 p = pktbuf_alloc();
195 if (!p)
196 return ERR_NO_MEMORY;
197
198 /* point our header to the base of the first pktbuf */
199 struct virtio_net_hdr *hdr = pktbuf_append(p, sizeof(struct virtio_net_hdr) - 2);
200 memset(hdr, 0, p->dlen);
201
202 spin_lock_saved_state_t state;
203 spin_lock_irqsave(&ndev->lock, state);
204
205 /* only queue if we have enough tx descriptors */
206 if (ndev->tx_pending_count + 2 > TX_RING_SIZE)
207 goto nodesc;
208
209 /* allocate a chain of descriptors for our transfer */
210 struct vring_desc *desc = virtio_alloc_desc_chain(vdev, RING_TX, 2, &i);
211 if (!desc) {
212 spin_unlock_irqrestore(&ndev->lock, state);
213
214 nodesc:
215 TRACEF("out of virtio tx descriptors, tx_pending_count %u\n", ndev->tx_pending_count);
216 pktbuf_free(p, true);
217
218 return ERR_NO_MEMORY;
219 }
220
221 ndev->tx_pending_count += 2;
222
223 /* save a pointer to our pktbufs for the irq handler to free */
224 LTRACEF("saving pointer to pkt in index %u and %u\n", i, desc->next);
225 DEBUG_ASSERT(ndev->pending_tx_packet[i] == NULL);
226 DEBUG_ASSERT(ndev->pending_tx_packet[desc->next] == NULL);
227 ndev->pending_tx_packet[i] = p;
228 ndev->pending_tx_packet[desc->next] = p2;
229
230 /* set up the descriptor pointing to the header */
231 desc->addr = pktbuf_data_phys(p);
232 desc->len = p->dlen;
233 desc->flags |= VRING_DESC_F_NEXT;
234
235 /* set up the descriptor pointing to the buffer */
236 desc = virtio_desc_index_to_desc(vdev, RING_TX, desc->next);
237 desc->addr = pktbuf_data_phys(p2);
238 desc->len = p2->dlen;
239 desc->flags = 0;
240
241 /* submit the transfer */
242 virtio_submit_chain(vdev, RING_TX, i);
243
244 /* kick it off */
245 virtio_kick(vdev, RING_TX);
246
247 spin_unlock_irqrestore(&ndev->lock, state);
248
249 return NO_ERROR;
250 }
251
252 /* variant of the above function that copies the buffer into a pktbuf before sending */
virtio_net_queue_tx(struct virtio_net_dev * ndev,const void * buf,size_t len)253 static status_t virtio_net_queue_tx(struct virtio_net_dev *ndev, const void *buf, size_t len) {
254 DEBUG_ASSERT(ndev);
255 DEBUG_ASSERT(buf);
256
257 pktbuf_t *p = pktbuf_alloc();
258 if (!p)
259 return ERR_NO_MEMORY;
260
261 /* copy the outgoing packet into the pktbuf */
262 p->data = p->buffer;
263 p->dlen = len;
264 memcpy(p->data, buf, len);
265
266 /* call through to the variant of the function that takes a pre-populated pktbuf */
267 status_t err = virtio_net_queue_tx_pktbuf(ndev, p);
268 if (err < 0) {
269 pktbuf_free(p, true);
270 }
271
272 return err;
273 }
274
virtio_net_queue_rx(struct virtio_net_dev * ndev,pktbuf_t * p)275 static status_t virtio_net_queue_rx(struct virtio_net_dev *ndev, pktbuf_t *p) {
276 struct virtio_device *vdev = ndev->dev;
277
278 DEBUG_ASSERT(ndev);
279 DEBUG_ASSERT(p);
280
281 /* point our header to the base of the pktbuf */
282 p->data = p->buffer;
283 struct virtio_net_hdr *hdr = (struct virtio_net_hdr *)p->data;
284 memset(hdr, 0, sizeof(struct virtio_net_hdr) - 2);
285
286 p->dlen = sizeof(struct virtio_net_hdr) - 2 + VIRTIO_NET_MSS;
287
288 spin_lock_saved_state_t state;
289 spin_lock_irqsave(&ndev->lock, state);
290
291 /* allocate a chain of descriptors for our transfer */
292 uint16_t i;
293 struct vring_desc *desc = virtio_alloc_desc_chain(vdev, RING_RX, 1, &i);
294 DEBUG_ASSERT(desc); /* shouldn't be possible not to have a descriptor ready */
295
296 /* save a pointer to our pktbufs for the irq handler to use */
297 DEBUG_ASSERT(ndev->pending_rx_packet[i] == NULL);
298 ndev->pending_rx_packet[i] = p;
299
300 /* set up the descriptor pointing to the header */
301 desc->addr = pktbuf_data_phys(p);
302 desc->len = p->dlen;
303 desc->flags = VRING_DESC_F_WRITE;
304
305 /* submit the transfer */
306 virtio_submit_chain(vdev, RING_RX, i);
307
308 /* kick it off */
309 virtio_kick(vdev, RING_RX);
310
311 spin_unlock_irqrestore(&ndev->lock, state);
312
313 return NO_ERROR;
314 }
315
virtio_net_irq_driver_callback(struct virtio_device * dev,uint ring,const struct vring_used_elem * e)316 static enum handler_return virtio_net_irq_driver_callback(struct virtio_device *dev, uint ring, const struct vring_used_elem *e) {
317 struct virtio_net_dev *ndev = (struct virtio_net_dev *)dev->priv;
318
319 LTRACEF("dev %p, ring %u, e %p, id %u, len %u\n", dev, ring, e, e->id, e->len);
320
321 spin_lock(&ndev->lock);
322
323 /* parse our descriptor chain, add back to the free queue */
324 uint16_t i = e->id;
325 for (;;) {
326 int next;
327 struct vring_desc *desc = virtio_desc_index_to_desc(dev, ring, i);
328
329 if (desc->flags & VRING_DESC_F_NEXT) {
330 next = desc->next;
331 } else {
332 /* end of chain */
333 next = -1;
334 }
335
336 virtio_free_desc(dev, ring, i);
337
338 if (ring == RING_RX) {
339 /* put the freed rx buffer in a queue */
340 pktbuf_t *p = ndev->pending_rx_packet[i];
341 ndev->pending_rx_packet[i] = NULL;
342
343 DEBUG_ASSERT(p);
344 LTRACEF("rx pktbuf %p filled\n", p);
345
346 /* trim the pktbuf according to the written length in the used element descriptor */
347 if (e->len > (sizeof(struct virtio_net_hdr) - 2 + VIRTIO_NET_MSS)) {
348 TRACEF("bad used len on RX %u\n", e->len);
349 p->dlen = 0;
350 } else {
351 p->dlen = e->len;
352 }
353
354 list_add_tail(&ndev->completed_rx_queue, &p->list);
355 } else { // ring == RING_TX
356 /* free the pktbuf associated with the tx packet we just consumed */
357 pktbuf_t *p = ndev->pending_tx_packet[i];
358 ndev->pending_tx_packet[i] = NULL;
359 ndev->tx_pending_count--;
360
361 DEBUG_ASSERT(p);
362 LTRACEF("freeing pktbuf %p\n", p);
363
364 pktbuf_free(p, false);
365 }
366
367 if (next < 0)
368 break;
369 i = next;
370 }
371
372 spin_unlock(&ndev->lock);
373
374 /* if rx ring, signal our event */
375 if (ring == 0) {
376 event_signal(&ndev->rx_event, false);
377 }
378
379 return INT_RESCHEDULE;
380 }
381
virtio_net_rx_worker(void * arg)382 static int virtio_net_rx_worker(void *arg) {
383 struct virtio_net_dev *ndev = (struct virtio_net_dev *)arg;
384
385 for (;;) {
386 event_wait(&ndev->rx_event);
387
388 /* pull some packets from the received queue */
389 for (;;) {
390 spin_lock_saved_state_t state;
391 spin_lock_irqsave(&ndev->lock, state);
392
393 pktbuf_t *p = list_remove_head_type(&ndev->completed_rx_queue, pktbuf_t, list);
394
395 spin_unlock_irqrestore(&ndev->lock, state);
396
397 if (!p)
398 break; /* nothing left in the queue, go back to waiting */
399
400 LTRACEF("got packet len %u\n", p->dlen);
401
402 /* process our packet */
403 struct virtio_net_hdr *hdr = pktbuf_consume(p, sizeof(struct virtio_net_hdr) - 2);
404 if (hdr) {
405 /* call up into the stack */
406 minip_rx_driver_callback(p);
407 }
408
409 /* requeue the pktbuf in the rx queue */
410 virtio_net_queue_rx(ndev, p);
411 }
412 }
413 return 0;
414 }
415
virtio_net_found(void)416 int virtio_net_found(void) {
417 return the_ndev ? 1 : 0;
418 }
419
virtio_net_get_mac_addr(uint8_t mac_addr[6])420 status_t virtio_net_get_mac_addr(uint8_t mac_addr[6]) {
421 if (!the_ndev)
422 return ERR_NOT_FOUND;
423
424 memcpy(mac_addr, the_ndev->config->mac, 6);
425
426 return NO_ERROR;
427 }
428
virtio_net_send_minip_pkt(pktbuf_t * p)429 status_t virtio_net_send_minip_pkt(pktbuf_t *p) {
430 LTRACEF("p %p, dlen %u, flags 0x%x\n", p, p->dlen, p->flags);
431
432 DEBUG_ASSERT(p && p->dlen);
433
434 if ((p->flags & PKTBUF_FLAG_EOF) == 0) {
435 /* can't handle multi part packets yet */
436 PANIC_UNIMPLEMENTED;
437
438 return ERR_NOT_IMPLEMENTED;
439 }
440
441 /* hand the pktbuf off to the nic, it owns the pktbuf from now on out unless it fails */
442 status_t err = virtio_net_queue_tx_pktbuf(the_ndev, p);
443 if (err < 0) {
444 pktbuf_free(p, true);
445 }
446
447 return err;
448 }
449
450