Lines Matching refs:worker
244 static void vhost_worker_queue(struct vhost_worker *worker, in vhost_worker_queue() argument
252 llist_add(&work->node, &worker->work_list); in vhost_worker_queue()
253 worker->ops->wakeup(worker); in vhost_worker_queue()
259 struct vhost_worker *worker; in vhost_vq_work_queue() local
263 worker = rcu_dereference(vq->worker); in vhost_vq_work_queue()
264 if (worker) { in vhost_vq_work_queue()
266 vhost_worker_queue(worker, work); in vhost_vq_work_queue()
280 static void __vhost_worker_flush(struct vhost_worker *worker) in __vhost_worker_flush() argument
284 if (!worker->attachment_cnt || worker->killed) in __vhost_worker_flush()
290 vhost_worker_queue(worker, &flush.work); in __vhost_worker_flush()
295 mutex_unlock(&worker->mutex); in __vhost_worker_flush()
297 mutex_lock(&worker->mutex); in __vhost_worker_flush()
300 static void vhost_worker_flush(struct vhost_worker *worker) in vhost_worker_flush() argument
302 mutex_lock(&worker->mutex); in vhost_worker_flush()
303 __vhost_worker_flush(worker); in vhost_worker_flush()
304 mutex_unlock(&worker->mutex); in vhost_worker_flush()
309 struct vhost_worker *worker; in vhost_dev_flush() local
312 xa_for_each(&dev->worker_xa, i, worker) in vhost_dev_flush()
313 vhost_worker_flush(worker); in vhost_dev_flush()
320 struct vhost_worker *worker; in vhost_vq_has_work() local
324 worker = rcu_dereference(vq->worker); in vhost_vq_has_work()
325 if (worker && !llist_empty(&worker->work_list)) in vhost_vq_has_work()
395 rcu_assign_pointer(vq->worker, NULL); in vhost_vq_reset()
402 struct vhost_worker *worker = data; in vhost_run_work_kthread_list() local
404 struct vhost_dev *dev = worker->dev; in vhost_run_work_kthread_list()
417 node = llist_del_all(&worker->work_list); in vhost_run_work_kthread_list()
427 kcov_remote_start_common(worker->kcov_handle); in vhost_run_work_kthread_list()
440 struct vhost_worker *worker = data; in vhost_run_work_list() local
444 node = llist_del_all(&worker->work_list); in vhost_run_work_list()
453 kcov_remote_start_common(worker->kcov_handle); in vhost_run_work_list()
465 struct vhost_worker *worker = data; in vhost_worker_killed() local
466 struct vhost_dev *dev = worker->dev; in vhost_worker_killed()
470 mutex_lock(&worker->mutex); in vhost_worker_killed()
471 worker->killed = true; in vhost_worker_killed()
477 if (worker == in vhost_worker_killed()
478 rcu_dereference_check(vq->worker, in vhost_worker_killed()
480 rcu_assign_pointer(vq->worker, NULL); in vhost_worker_killed()
486 worker->attachment_cnt -= attach_cnt; in vhost_worker_killed()
493 vhost_run_work_list(worker); in vhost_worker_killed()
494 mutex_unlock(&worker->mutex); in vhost_worker_killed()
651 static int vhost_attach_task_to_cgroups(struct vhost_worker *worker) in vhost_attach_task_to_cgroups() argument
659 vhost_worker_queue(worker, &attach.work); in vhost_attach_task_to_cgroups()
661 mutex_lock(&worker->mutex); in vhost_attach_task_to_cgroups()
667 saved_cnt = worker->attachment_cnt; in vhost_attach_task_to_cgroups()
668 worker->attachment_cnt = INT_MAX; in vhost_attach_task_to_cgroups()
669 __vhost_worker_flush(worker); in vhost_attach_task_to_cgroups()
670 worker->attachment_cnt = saved_cnt; in vhost_attach_task_to_cgroups()
672 mutex_unlock(&worker->mutex); in vhost_attach_task_to_cgroups()
715 struct vhost_worker *worker) in vhost_worker_destroy() argument
717 if (!worker) in vhost_worker_destroy()
720 WARN_ON(!llist_empty(&worker->work_list)); in vhost_worker_destroy()
721 xa_erase(&dev->worker_xa, worker->id); in vhost_worker_destroy()
722 worker->ops->stop(worker); in vhost_worker_destroy()
723 kfree(worker); in vhost_worker_destroy()
728 struct vhost_worker *worker; in vhost_workers_free() local
735 rcu_assign_pointer(dev->vqs[i]->worker, NULL); in vhost_workers_free()
740 xa_for_each(&dev->worker_xa, i, worker) in vhost_workers_free()
741 vhost_worker_destroy(dev, worker); in vhost_workers_free()
745 static void vhost_task_wakeup(struct vhost_worker *worker) in vhost_task_wakeup() argument
747 return vhost_task_wake(worker->vtsk); in vhost_task_wakeup()
750 static void vhost_kthread_wakeup(struct vhost_worker *worker) in vhost_kthread_wakeup() argument
752 wake_up_process(worker->kthread_task); in vhost_kthread_wakeup()
755 static void vhost_task_do_stop(struct vhost_worker *worker) in vhost_task_do_stop() argument
757 return vhost_task_stop(worker->vtsk); in vhost_task_do_stop()
760 static void vhost_kthread_do_stop(struct vhost_worker *worker) in vhost_kthread_do_stop() argument
762 kthread_stop(worker->kthread_task); in vhost_kthread_do_stop()
765 static int vhost_task_worker_create(struct vhost_worker *worker, in vhost_task_worker_create() argument
773 worker, name); in vhost_task_worker_create()
777 worker->vtsk = vtsk; in vhost_task_worker_create()
779 ret = xa_alloc(&dev->worker_xa, &id, worker, xa_limit_32b, GFP_KERNEL); in vhost_task_worker_create()
781 vhost_task_do_stop(worker); in vhost_task_worker_create()
784 worker->id = id; in vhost_task_worker_create()
788 static int vhost_kthread_worker_create(struct vhost_worker *worker, in vhost_kthread_worker_create() argument
795 task = kthread_create(vhost_run_work_kthread_list, worker, "%s", name); in vhost_kthread_worker_create()
799 worker->kthread_task = task; in vhost_kthread_worker_create()
801 ret = xa_alloc(&dev->worker_xa, &id, worker, xa_limit_32b, GFP_KERNEL); in vhost_kthread_worker_create()
805 ret = vhost_attach_task_to_cgroups(worker); in vhost_kthread_worker_create()
809 worker->id = id; in vhost_kthread_worker_create()
813 vhost_kthread_do_stop(worker); in vhost_kthread_worker_create()
831 struct vhost_worker *worker; in vhost_worker_create() local
837 worker = kzalloc(sizeof(*worker), GFP_KERNEL_ACCOUNT); in vhost_worker_create()
838 if (!worker) in vhost_worker_create()
841 worker->dev = dev; in vhost_worker_create()
842 worker->ops = ops; in vhost_worker_create()
845 mutex_init(&worker->mutex); in vhost_worker_create()
846 init_llist_head(&worker->work_list); in vhost_worker_create()
847 worker->kcov_handle = kcov_common_handle(); in vhost_worker_create()
848 ret = ops->create(worker, dev, name); in vhost_worker_create()
852 return worker; in vhost_worker_create()
855 kfree(worker); in vhost_worker_create()
861 struct vhost_worker *worker) in __vhost_vq_attach_worker() argument
865 mutex_lock(&worker->mutex); in __vhost_vq_attach_worker()
866 if (worker->killed) { in __vhost_vq_attach_worker()
867 mutex_unlock(&worker->mutex); in __vhost_vq_attach_worker()
873 old_worker = rcu_dereference_check(vq->worker, in __vhost_vq_attach_worker()
875 rcu_assign_pointer(vq->worker, worker); in __vhost_vq_attach_worker()
876 worker->attachment_cnt++; in __vhost_vq_attach_worker()
880 mutex_unlock(&worker->mutex); in __vhost_vq_attach_worker()
884 mutex_unlock(&worker->mutex); in __vhost_vq_attach_worker()
934 struct vhost_worker *worker; in vhost_vq_attach_worker() local
939 worker = xa_find(&dev->worker_xa, &index, UINT_MAX, XA_PRESENT); in vhost_vq_attach_worker()
940 if (!worker || worker->id != info->worker_id) in vhost_vq_attach_worker()
943 __vhost_vq_attach_worker(vq, worker); in vhost_vq_attach_worker()
951 struct vhost_worker *worker; in vhost_new_worker() local
953 worker = vhost_worker_create(dev); in vhost_new_worker()
954 if (!worker) in vhost_new_worker()
957 info->worker_id = worker->id; in vhost_new_worker()
966 struct vhost_worker *worker; in vhost_free_worker() local
968 worker = xa_find(&dev->worker_xa, &index, UINT_MAX, XA_PRESENT); in vhost_free_worker()
969 if (!worker || worker->id != info->worker_id) in vhost_free_worker()
972 mutex_lock(&worker->mutex); in vhost_free_worker()
973 if (worker->attachment_cnt || worker->killed) { in vhost_free_worker()
974 mutex_unlock(&worker->mutex); in vhost_free_worker()
982 __vhost_worker_flush(worker); in vhost_free_worker()
983 mutex_unlock(&worker->mutex); in vhost_free_worker()
985 vhost_worker_destroy(dev, worker); in vhost_free_worker()
1016 struct vhost_worker *worker; in vhost_worker_ioctl() local
1072 worker = rcu_dereference_check(vq->worker, in vhost_worker_ioctl()
1074 if (!worker) { in vhost_worker_ioctl()
1080 ring_worker.worker_id = worker->id; in vhost_worker_ioctl()
1097 struct vhost_worker *worker; in vhost_dev_set_owner() local
1119 worker = vhost_worker_create(dev); in vhost_dev_set_owner()
1120 if (!worker) { in vhost_dev_set_owner()
1126 __vhost_vq_attach_worker(dev->vqs[i], worker); in vhost_dev_set_owner()