Lines Matching refs:kvm
148 static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm);
154 __weak void kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm, in kvm_arch_mmu_notifier_invalidate_range() argument
159 __weak void kvm_arch_guest_memory_reclaimed(struct kvm *kvm) in kvm_arch_guest_memory_reclaimed() argument
292 bool kvm_make_vcpus_request_mask(struct kvm *kvm, unsigned int req, in kvm_make_vcpus_request_mask() argument
306 vcpu = kvm_get_vcpu(kvm, i); in kvm_make_vcpus_request_mask()
318 bool kvm_make_all_cpus_request_except(struct kvm *kvm, unsigned int req, in kvm_make_all_cpus_request_except() argument
332 kvm_for_each_vcpu(i, vcpu, kvm) { in kvm_make_all_cpus_request_except()
344 bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req) in kvm_make_all_cpus_request() argument
346 return kvm_make_all_cpus_request_except(kvm, req, NULL); in kvm_make_all_cpus_request()
351 void kvm_flush_remote_tlbs(struct kvm *kvm) in kvm_flush_remote_tlbs() argument
353 ++kvm->stat.generic.remote_tlb_flush_requests; in kvm_flush_remote_tlbs()
366 if (!kvm_arch_flush_remote_tlb(kvm) in kvm_flush_remote_tlbs()
367 || kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH)) in kvm_flush_remote_tlbs()
368 ++kvm->stat.generic.remote_tlb_flush; in kvm_flush_remote_tlbs()
373 static void kvm_flush_shadow_all(struct kvm *kvm) in kvm_flush_shadow_all() argument
375 kvm_arch_flush_shadow_all(kvm); in kvm_flush_shadow_all()
376 kvm_arch_guest_memory_reclaimed(kvm); in kvm_flush_shadow_all()
461 static void kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id) in kvm_vcpu_init() argument
465 vcpu->kvm = kvm; in kvm_vcpu_init()
501 void kvm_destroy_vcpus(struct kvm *kvm) in kvm_destroy_vcpus() argument
506 kvm_for_each_vcpu(i, vcpu, kvm) { in kvm_destroy_vcpus()
508 xa_erase(&kvm->vcpu_array, i); in kvm_destroy_vcpus()
511 atomic_set(&kvm->online_vcpus, 0); in kvm_destroy_vcpus()
516 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn) in mmu_notifier_to_kvm()
518 return container_of(mn, struct kvm, mmu_notifier); in mmu_notifier_to_kvm()
525 struct kvm *kvm = mmu_notifier_to_kvm(mn); in kvm_mmu_notifier_invalidate_range() local
528 idx = srcu_read_lock(&kvm->srcu); in kvm_mmu_notifier_invalidate_range()
529 kvm_arch_mmu_notifier_invalidate_range(kvm, start, end); in kvm_mmu_notifier_invalidate_range()
530 srcu_read_unlock(&kvm->srcu, idx); in kvm_mmu_notifier_invalidate_range()
533 typedef bool (*hva_handler_t)(struct kvm *kvm, struct kvm_gfn_range *range);
535 typedef void (*on_lock_fn_t)(struct kvm *kvm, unsigned long start,
538 typedef void (*on_unlock_fn_t)(struct kvm *kvm);
570 static __always_inline int __kvm_handle_hva_range(struct kvm *kvm, in __kvm_handle_hva_range() argument
587 idx = srcu_read_lock(&kvm->srcu); in __kvm_handle_hva_range()
592 slots = __kvm_memslots(kvm, i); in __kvm_handle_hva_range()
621 KVM_MMU_LOCK(kvm); in __kvm_handle_hva_range()
623 range->on_lock(kvm, range->start, range->end); in __kvm_handle_hva_range()
627 ret |= range->handler(kvm, &gfn_range); in __kvm_handle_hva_range()
632 kvm_flush_remote_tlbs(kvm); in __kvm_handle_hva_range()
635 KVM_MMU_UNLOCK(kvm); in __kvm_handle_hva_range()
637 range->on_unlock(kvm); in __kvm_handle_hva_range()
640 srcu_read_unlock(&kvm->srcu, idx); in __kvm_handle_hva_range()
652 struct kvm *kvm = mmu_notifier_to_kvm(mn); in kvm_handle_hva_range() local
664 return __kvm_handle_hva_range(kvm, &range); in kvm_handle_hva_range()
672 struct kvm *kvm = mmu_notifier_to_kvm(mn); in kvm_handle_hva_range_no_flush() local
684 return __kvm_handle_hva_range(kvm, &range); in kvm_handle_hva_range_no_flush()
691 struct kvm *kvm = mmu_notifier_to_kvm(mn); in kvm_mmu_notifier_change_pte() local
703 WARN_ON_ONCE(!READ_ONCE(kvm->mn_active_invalidate_count)); in kvm_mmu_notifier_change_pte()
704 if (!READ_ONCE(kvm->mmu_invalidate_in_progress)) in kvm_mmu_notifier_change_pte()
710 void kvm_mmu_invalidate_begin(struct kvm *kvm, unsigned long start, in kvm_mmu_invalidate_begin() argument
718 kvm->mmu_invalidate_in_progress++; in kvm_mmu_invalidate_begin()
719 if (likely(kvm->mmu_invalidate_in_progress == 1)) { in kvm_mmu_invalidate_begin()
720 kvm->mmu_invalidate_range_start = start; in kvm_mmu_invalidate_begin()
721 kvm->mmu_invalidate_range_end = end; in kvm_mmu_invalidate_begin()
732 kvm->mmu_invalidate_range_start = in kvm_mmu_invalidate_begin()
733 min(kvm->mmu_invalidate_range_start, start); in kvm_mmu_invalidate_begin()
734 kvm->mmu_invalidate_range_end = in kvm_mmu_invalidate_begin()
735 max(kvm->mmu_invalidate_range_end, end); in kvm_mmu_invalidate_begin()
742 struct kvm *kvm = mmu_notifier_to_kvm(mn); in kvm_mmu_notifier_invalidate_range_start() local
764 spin_lock(&kvm->mn_invalidate_lock); in kvm_mmu_notifier_invalidate_range_start()
765 kvm->mn_active_invalidate_count++; in kvm_mmu_notifier_invalidate_range_start()
766 spin_unlock(&kvm->mn_invalidate_lock); in kvm_mmu_notifier_invalidate_range_start()
778 gfn_to_pfn_cache_invalidate_start(kvm, range->start, range->end, in kvm_mmu_notifier_invalidate_range_start()
781 __kvm_handle_hva_range(kvm, &hva_range); in kvm_mmu_notifier_invalidate_range_start()
786 void kvm_mmu_invalidate_end(struct kvm *kvm, unsigned long start, in kvm_mmu_invalidate_end() argument
794 kvm->mmu_invalidate_seq++; in kvm_mmu_invalidate_end()
801 kvm->mmu_invalidate_in_progress--; in kvm_mmu_invalidate_end()
807 struct kvm *kvm = mmu_notifier_to_kvm(mn); in kvm_mmu_notifier_invalidate_range_end() local
820 __kvm_handle_hva_range(kvm, &hva_range); in kvm_mmu_notifier_invalidate_range_end()
823 spin_lock(&kvm->mn_invalidate_lock); in kvm_mmu_notifier_invalidate_range_end()
824 wake = (--kvm->mn_active_invalidate_count == 0); in kvm_mmu_notifier_invalidate_range_end()
825 spin_unlock(&kvm->mn_invalidate_lock); in kvm_mmu_notifier_invalidate_range_end()
832 rcuwait_wake_up(&kvm->mn_memslots_update_rcuwait); in kvm_mmu_notifier_invalidate_range_end()
834 BUG_ON(kvm->mmu_invalidate_in_progress < 0); in kvm_mmu_notifier_invalidate_range_end()
883 struct kvm *kvm = mmu_notifier_to_kvm(mn); in kvm_mmu_notifier_release() local
886 idx = srcu_read_lock(&kvm->srcu); in kvm_mmu_notifier_release()
887 kvm_flush_shadow_all(kvm); in kvm_mmu_notifier_release()
888 srcu_read_unlock(&kvm->srcu, idx); in kvm_mmu_notifier_release()
902 static int kvm_init_mmu_notifier(struct kvm *kvm) in kvm_init_mmu_notifier() argument
904 kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops; in kvm_init_mmu_notifier()
905 return mmu_notifier_register(&kvm->mmu_notifier, current->mm); in kvm_init_mmu_notifier()
910 static int kvm_init_mmu_notifier(struct kvm *kvm) in kvm_init_mmu_notifier() argument
922 struct kvm *kvm = container_of(bl, struct kvm, pm_notifier); in kvm_pm_notifier_call() local
924 return kvm_arch_pm_notifier(kvm, state); in kvm_pm_notifier_call()
927 static void kvm_init_pm_notifier(struct kvm *kvm) in kvm_init_pm_notifier() argument
929 kvm->pm_notifier.notifier_call = kvm_pm_notifier_call; in kvm_init_pm_notifier()
931 kvm->pm_notifier.priority = INT_MAX; in kvm_init_pm_notifier()
932 register_pm_notifier(&kvm->pm_notifier); in kvm_init_pm_notifier()
935 static void kvm_destroy_pm_notifier(struct kvm *kvm) in kvm_destroy_pm_notifier() argument
937 unregister_pm_notifier(&kvm->pm_notifier); in kvm_destroy_pm_notifier()
940 static void kvm_init_pm_notifier(struct kvm *kvm) in kvm_init_pm_notifier() argument
944 static void kvm_destroy_pm_notifier(struct kvm *kvm) in kvm_destroy_pm_notifier() argument
959 static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) in kvm_free_memslot() argument
963 kvm_arch_free_memslot(kvm, slot); in kvm_free_memslot()
968 static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots) in kvm_free_memslots() argument
984 kvm_free_memslot(kvm, memslot); in kvm_free_memslots()
1000 static void kvm_destroy_vm_debugfs(struct kvm *kvm) in kvm_destroy_vm_debugfs() argument
1006 if (IS_ERR(kvm->debugfs_dentry)) in kvm_destroy_vm_debugfs()
1009 debugfs_remove_recursive(kvm->debugfs_dentry); in kvm_destroy_vm_debugfs()
1011 if (kvm->debugfs_stat_data) { in kvm_destroy_vm_debugfs()
1013 kfree(kvm->debugfs_stat_data[i]); in kvm_destroy_vm_debugfs()
1014 kfree(kvm->debugfs_stat_data); in kvm_destroy_vm_debugfs()
1018 static int kvm_create_vm_debugfs(struct kvm *kvm, const char *fdname) in kvm_create_vm_debugfs() argument
1046 kvm->debugfs_dentry = dent; in kvm_create_vm_debugfs()
1047 kvm->debugfs_stat_data = kcalloc(kvm_debugfs_num_entries, in kvm_create_vm_debugfs()
1048 sizeof(*kvm->debugfs_stat_data), in kvm_create_vm_debugfs()
1050 if (!kvm->debugfs_stat_data) in kvm_create_vm_debugfs()
1059 stat_data->kvm = kvm; in kvm_create_vm_debugfs()
1062 kvm->debugfs_stat_data[i] = stat_data; in kvm_create_vm_debugfs()
1064 kvm->debugfs_dentry, stat_data, in kvm_create_vm_debugfs()
1074 stat_data->kvm = kvm; in kvm_create_vm_debugfs()
1077 kvm->debugfs_stat_data[i + kvm_vm_stats_header.num_desc] = stat_data; in kvm_create_vm_debugfs()
1079 kvm->debugfs_dentry, stat_data, in kvm_create_vm_debugfs()
1083 ret = kvm_arch_create_vm_debugfs(kvm); in kvm_create_vm_debugfs()
1089 kvm_destroy_vm_debugfs(kvm); in kvm_create_vm_debugfs()
1097 int __weak kvm_arch_post_init_vm(struct kvm *kvm) in kvm_arch_post_init_vm() argument
1106 void __weak kvm_arch_pre_destroy_vm(struct kvm *kvm) in kvm_arch_pre_destroy_vm() argument
1116 int __weak kvm_arch_create_vm_debugfs(struct kvm *kvm) in kvm_arch_create_vm_debugfs() argument
1121 static struct kvm *kvm_create_vm(unsigned long type, const char *fdname) in kvm_create_vm()
1123 struct kvm *kvm = kvm_arch_alloc_vm(); in kvm_create_vm() local
1128 if (!kvm) in kvm_create_vm()
1134 KVM_MMU_LOCK_INIT(kvm); in kvm_create_vm()
1136 kvm->mm = current->mm; in kvm_create_vm()
1137 kvm_eventfd_init(kvm); in kvm_create_vm()
1138 mutex_init(&kvm->lock); in kvm_create_vm()
1139 mutex_init(&kvm->irq_lock); in kvm_create_vm()
1140 mutex_init(&kvm->slots_lock); in kvm_create_vm()
1141 mutex_init(&kvm->slots_arch_lock); in kvm_create_vm()
1142 spin_lock_init(&kvm->mn_invalidate_lock); in kvm_create_vm()
1143 rcuwait_init(&kvm->mn_memslots_update_rcuwait); in kvm_create_vm()
1144 xa_init(&kvm->vcpu_array); in kvm_create_vm()
1146 INIT_LIST_HEAD(&kvm->gpc_list); in kvm_create_vm()
1147 spin_lock_init(&kvm->gpc_lock); in kvm_create_vm()
1149 INIT_LIST_HEAD(&kvm->devices); in kvm_create_vm()
1150 kvm->max_vcpus = KVM_MAX_VCPUS; in kvm_create_vm()
1158 kvm->debugfs_dentry = ERR_PTR(-ENOENT); in kvm_create_vm()
1160 snprintf(kvm->stats_id, sizeof(kvm->stats_id), "kvm-%d", in kvm_create_vm()
1163 if (init_srcu_struct(&kvm->srcu)) in kvm_create_vm()
1165 if (init_srcu_struct(&kvm->irq_srcu)) in kvm_create_vm()
1168 refcount_set(&kvm->users_count, 1); in kvm_create_vm()
1171 slots = &kvm->__memslots[i][j]; in kvm_create_vm()
1183 rcu_assign_pointer(kvm->memslots[i], &kvm->__memslots[i][0]); in kvm_create_vm()
1187 rcu_assign_pointer(kvm->buses[i], in kvm_create_vm()
1189 if (!kvm->buses[i]) in kvm_create_vm()
1193 r = kvm_arch_init_vm(kvm, type); in kvm_create_vm()
1202 INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list); in kvm_create_vm()
1205 r = kvm_init_mmu_notifier(kvm); in kvm_create_vm()
1209 r = kvm_coalesced_mmio_init(kvm); in kvm_create_vm()
1213 r = kvm_create_vm_debugfs(kvm, fdname); in kvm_create_vm()
1217 r = kvm_arch_post_init_vm(kvm); in kvm_create_vm()
1222 list_add(&kvm->vm_list, &vm_list); in kvm_create_vm()
1226 kvm_init_pm_notifier(kvm); in kvm_create_vm()
1228 return kvm; in kvm_create_vm()
1231 kvm_destroy_vm_debugfs(kvm); in kvm_create_vm()
1233 kvm_coalesced_mmio_free(kvm); in kvm_create_vm()
1236 if (kvm->mmu_notifier.ops) in kvm_create_vm()
1237 mmu_notifier_unregister(&kvm->mmu_notifier, current->mm); in kvm_create_vm()
1242 kvm_arch_destroy_vm(kvm); in kvm_create_vm()
1244 WARN_ON_ONCE(!refcount_dec_and_test(&kvm->users_count)); in kvm_create_vm()
1246 kfree(kvm_get_bus(kvm, i)); in kvm_create_vm()
1247 cleanup_srcu_struct(&kvm->irq_srcu); in kvm_create_vm()
1249 cleanup_srcu_struct(&kvm->srcu); in kvm_create_vm()
1251 kvm_arch_free_vm(kvm); in kvm_create_vm()
1257 static void kvm_destroy_devices(struct kvm *kvm) in kvm_destroy_devices() argument
1266 list_for_each_entry_safe(dev, tmp, &kvm->devices, vm_node) { in kvm_destroy_devices()
1272 static void kvm_destroy_vm(struct kvm *kvm) in kvm_destroy_vm() argument
1275 struct mm_struct *mm = kvm->mm; in kvm_destroy_vm()
1277 kvm_destroy_pm_notifier(kvm); in kvm_destroy_vm()
1278 kvm_uevent_notify_change(KVM_EVENT_DESTROY_VM, kvm); in kvm_destroy_vm()
1279 kvm_destroy_vm_debugfs(kvm); in kvm_destroy_vm()
1280 kvm_arch_sync_events(kvm); in kvm_destroy_vm()
1282 list_del(&kvm->vm_list); in kvm_destroy_vm()
1284 kvm_arch_pre_destroy_vm(kvm); in kvm_destroy_vm()
1286 kvm_free_irq_routing(kvm); in kvm_destroy_vm()
1288 struct kvm_io_bus *bus = kvm_get_bus(kvm, i); in kvm_destroy_vm()
1292 kvm->buses[i] = NULL; in kvm_destroy_vm()
1294 kvm_coalesced_mmio_free(kvm); in kvm_destroy_vm()
1296 mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm); in kvm_destroy_vm()
1305 WARN_ON(rcuwait_active(&kvm->mn_memslots_update_rcuwait)); in kvm_destroy_vm()
1306 kvm->mn_active_invalidate_count = 0; in kvm_destroy_vm()
1308 kvm_flush_shadow_all(kvm); in kvm_destroy_vm()
1310 kvm_arch_destroy_vm(kvm); in kvm_destroy_vm()
1311 kvm_destroy_devices(kvm); in kvm_destroy_vm()
1313 kvm_free_memslots(kvm, &kvm->__memslots[i][0]); in kvm_destroy_vm()
1314 kvm_free_memslots(kvm, &kvm->__memslots[i][1]); in kvm_destroy_vm()
1316 cleanup_srcu_struct(&kvm->irq_srcu); in kvm_destroy_vm()
1317 cleanup_srcu_struct(&kvm->srcu); in kvm_destroy_vm()
1318 kvm_arch_free_vm(kvm); in kvm_destroy_vm()
1325 void kvm_get_kvm(struct kvm *kvm) in kvm_get_kvm() argument
1327 refcount_inc(&kvm->users_count); in kvm_get_kvm()
1335 bool kvm_get_kvm_safe(struct kvm *kvm) in kvm_get_kvm_safe() argument
1337 return refcount_inc_not_zero(&kvm->users_count); in kvm_get_kvm_safe()
1341 void kvm_put_kvm(struct kvm *kvm) in kvm_put_kvm() argument
1343 if (refcount_dec_and_test(&kvm->users_count)) in kvm_put_kvm()
1344 kvm_destroy_vm(kvm); in kvm_put_kvm()
1355 void kvm_put_kvm_no_destroy(struct kvm *kvm) in kvm_put_kvm_no_destroy() argument
1357 WARN_ON(refcount_dec_and_test(&kvm->users_count)); in kvm_put_kvm_no_destroy()
1363 struct kvm *kvm = filp->private_data; in kvm_vm_release() local
1365 kvm_irqfd_release(kvm); in kvm_vm_release()
1367 kvm_put_kvm(kvm); in kvm_vm_release()
1386 static struct kvm_memslots *kvm_get_inactive_memslots(struct kvm *kvm, int as_id) in kvm_get_inactive_memslots() argument
1388 struct kvm_memslots *active = __kvm_memslots(kvm, as_id); in kvm_get_inactive_memslots()
1391 return &kvm->__memslots[as_id][node_idx_inactive]; in kvm_get_inactive_memslots()
1466 static void kvm_replace_memslot(struct kvm *kvm, in kvm_replace_memslot() argument
1471 struct kvm_memslots *slots = kvm_get_inactive_memslots(kvm, as_id); in kvm_replace_memslot()
1533 static void kvm_swap_active_memslots(struct kvm *kvm, int as_id) in kvm_swap_active_memslots() argument
1535 struct kvm_memslots *slots = kvm_get_inactive_memslots(kvm, as_id); in kvm_swap_active_memslots()
1538 u64 gen = __kvm_memslots(kvm, as_id)->generation; in kvm_swap_active_memslots()
1548 spin_lock(&kvm->mn_invalidate_lock); in kvm_swap_active_memslots()
1549 prepare_to_rcuwait(&kvm->mn_memslots_update_rcuwait); in kvm_swap_active_memslots()
1550 while (kvm->mn_active_invalidate_count) { in kvm_swap_active_memslots()
1552 spin_unlock(&kvm->mn_invalidate_lock); in kvm_swap_active_memslots()
1554 spin_lock(&kvm->mn_invalidate_lock); in kvm_swap_active_memslots()
1556 finish_rcuwait(&kvm->mn_memslots_update_rcuwait); in kvm_swap_active_memslots()
1557 rcu_assign_pointer(kvm->memslots[as_id], slots); in kvm_swap_active_memslots()
1558 spin_unlock(&kvm->mn_invalidate_lock); in kvm_swap_active_memslots()
1565 mutex_unlock(&kvm->slots_arch_lock); in kvm_swap_active_memslots()
1567 synchronize_srcu_expedited(&kvm->srcu); in kvm_swap_active_memslots()
1586 kvm_arch_memslots_updated(kvm, gen); in kvm_swap_active_memslots()
1591 static int kvm_prepare_memory_region(struct kvm *kvm, in kvm_prepare_memory_region() argument
1610 else if (kvm_use_dirty_bitmap(kvm)) { in kvm_prepare_memory_region()
1615 if (kvm_dirty_log_manual_protect_and_init_set(kvm)) in kvm_prepare_memory_region()
1620 r = kvm_arch_prepare_memory_region(kvm, old, new, change); in kvm_prepare_memory_region()
1629 static void kvm_commit_memory_region(struct kvm *kvm, in kvm_commit_memory_region() argument
1641 kvm->nr_memslot_pages -= old->npages; in kvm_commit_memory_region()
1643 kvm->nr_memslot_pages += new->npages; in kvm_commit_memory_region()
1647 atomic_set(&kvm->nr_memslots_dirty_logging, in kvm_commit_memory_region()
1648 atomic_read(&kvm->nr_memslots_dirty_logging) + change); in kvm_commit_memory_region()
1651 kvm_arch_commit_memory_region(kvm, old, new, change); in kvm_commit_memory_region()
1659 kvm_free_memslot(kvm, old); in kvm_commit_memory_region()
1691 static void kvm_activate_memslot(struct kvm *kvm, in kvm_activate_memslot() argument
1697 kvm_swap_active_memslots(kvm, as_id); in kvm_activate_memslot()
1700 kvm_replace_memslot(kvm, old, new); in kvm_activate_memslot()
1716 static void kvm_invalidate_memslot(struct kvm *kvm, in kvm_invalidate_memslot() argument
1727 kvm_replace_memslot(kvm, old, invalid_slot); in kvm_invalidate_memslot()
1734 kvm_swap_active_memslots(kvm, old->as_id); in kvm_invalidate_memslot()
1742 kvm_arch_flush_shadow_memslot(kvm, old); in kvm_invalidate_memslot()
1743 kvm_arch_guest_memory_reclaimed(kvm); in kvm_invalidate_memslot()
1746 mutex_lock(&kvm->slots_arch_lock); in kvm_invalidate_memslot()
1758 static void kvm_create_memslot(struct kvm *kvm, in kvm_create_memslot() argument
1762 kvm_replace_memslot(kvm, NULL, new); in kvm_create_memslot()
1763 kvm_activate_memslot(kvm, NULL, new); in kvm_create_memslot()
1766 static void kvm_delete_memslot(struct kvm *kvm, in kvm_delete_memslot() argument
1774 kvm_replace_memslot(kvm, old, NULL); in kvm_delete_memslot()
1775 kvm_activate_memslot(kvm, invalid_slot, NULL); in kvm_delete_memslot()
1778 static void kvm_move_memslot(struct kvm *kvm, in kvm_move_memslot() argument
1787 kvm_replace_memslot(kvm, old, new); in kvm_move_memslot()
1788 kvm_activate_memslot(kvm, invalid_slot, new); in kvm_move_memslot()
1791 static void kvm_update_flags_memslot(struct kvm *kvm, in kvm_update_flags_memslot() argument
1800 kvm_replace_memslot(kvm, old, new); in kvm_update_flags_memslot()
1801 kvm_activate_memslot(kvm, old, new); in kvm_update_flags_memslot()
1804 static int kvm_set_memslot(struct kvm *kvm, in kvm_set_memslot() argument
1826 mutex_lock(&kvm->slots_arch_lock); in kvm_set_memslot()
1844 mutex_unlock(&kvm->slots_arch_lock); in kvm_set_memslot()
1847 kvm_invalidate_memslot(kvm, old, invalid_slot); in kvm_set_memslot()
1850 r = kvm_prepare_memory_region(kvm, old, new, change); in kvm_set_memslot()
1859 kvm_activate_memslot(kvm, invalid_slot, old); in kvm_set_memslot()
1862 mutex_unlock(&kvm->slots_arch_lock); in kvm_set_memslot()
1875 kvm_create_memslot(kvm, new); in kvm_set_memslot()
1877 kvm_delete_memslot(kvm, old, invalid_slot); in kvm_set_memslot()
1879 kvm_move_memslot(kvm, old, new, invalid_slot); in kvm_set_memslot()
1881 kvm_update_flags_memslot(kvm, old, new); in kvm_set_memslot()
1894 kvm_commit_memory_region(kvm, old, new, change); in kvm_set_memslot()
1920 int __kvm_set_memory_region(struct kvm *kvm, in __kvm_set_memory_region() argument
1957 slots = __kvm_memslots(kvm, as_id); in __kvm_set_memory_region()
1969 if (WARN_ON_ONCE(kvm->nr_memslot_pages < old->npages)) in __kvm_set_memory_region()
1972 return kvm_set_memslot(kvm, old, NULL, KVM_MR_DELETE); in __kvm_set_memory_region()
1985 if ((kvm->nr_memslot_pages + npages) < kvm->nr_memslot_pages) in __kvm_set_memory_region()
2017 r = kvm_set_memslot(kvm, old, new, change); in __kvm_set_memory_region()
2024 int kvm_set_memory_region(struct kvm *kvm, in kvm_set_memory_region() argument
2029 mutex_lock(&kvm->slots_lock); in kvm_set_memory_region()
2030 r = __kvm_set_memory_region(kvm, mem); in kvm_set_memory_region()
2031 mutex_unlock(&kvm->slots_lock); in kvm_set_memory_region()
2036 static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, in kvm_vm_ioctl_set_memory_region() argument
2042 return kvm_set_memory_region(kvm, mem); in kvm_vm_ioctl_set_memory_region()
2053 int kvm_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log, in kvm_get_dirty_log() argument
2062 if (!kvm_use_dirty_bitmap(kvm)) in kvm_get_dirty_log()
2073 slots = __kvm_memslots(kvm, as_id); in kvm_get_dirty_log()
2078 kvm_arch_sync_dirty_log(kvm, *memslot); in kvm_get_dirty_log()
2116 static int kvm_get_dirty_log_protect(struct kvm *kvm, struct kvm_dirty_log *log) in kvm_get_dirty_log_protect() argument
2127 if (!kvm_use_dirty_bitmap(kvm)) in kvm_get_dirty_log_protect()
2135 slots = __kvm_memslots(kvm, as_id); in kvm_get_dirty_log_protect()
2142 kvm_arch_sync_dirty_log(kvm, memslot); in kvm_get_dirty_log_protect()
2146 if (kvm->manual_dirty_log_protect) { in kvm_get_dirty_log_protect()
2160 KVM_MMU_LOCK(kvm); in kvm_get_dirty_log_protect()
2173 kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, in kvm_get_dirty_log_protect()
2176 KVM_MMU_UNLOCK(kvm); in kvm_get_dirty_log_protect()
2180 kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); in kvm_get_dirty_log_protect()
2207 static int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, in kvm_vm_ioctl_get_dirty_log() argument
2212 mutex_lock(&kvm->slots_lock); in kvm_vm_ioctl_get_dirty_log()
2214 r = kvm_get_dirty_log_protect(kvm, log); in kvm_vm_ioctl_get_dirty_log()
2216 mutex_unlock(&kvm->slots_lock); in kvm_vm_ioctl_get_dirty_log()
2226 static int kvm_clear_dirty_log_protect(struct kvm *kvm, in kvm_clear_dirty_log_protect() argument
2239 if (!kvm_use_dirty_bitmap(kvm)) in kvm_clear_dirty_log_protect()
2250 slots = __kvm_memslots(kvm, as_id); in kvm_clear_dirty_log_protect()
2264 kvm_arch_sync_dirty_log(kvm, memslot); in kvm_clear_dirty_log_protect()
2271 KVM_MMU_LOCK(kvm); in kvm_clear_dirty_log_protect()
2290 kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, in kvm_clear_dirty_log_protect()
2294 KVM_MMU_UNLOCK(kvm); in kvm_clear_dirty_log_protect()
2297 kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); in kvm_clear_dirty_log_protect()
2302 static int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm, in kvm_vm_ioctl_clear_dirty_log() argument
2307 mutex_lock(&kvm->slots_lock); in kvm_vm_ioctl_clear_dirty_log()
2309 r = kvm_clear_dirty_log_protect(kvm, log); in kvm_vm_ioctl_clear_dirty_log()
2311 mutex_unlock(&kvm->slots_lock); in kvm_vm_ioctl_clear_dirty_log()
2316 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn) in gfn_to_memslot() argument
2318 return __gfn_to_memslot(kvm_memslots(kvm), gfn); in gfn_to_memslot()
2355 bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn) in kvm_is_visible_gfn() argument
2357 struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn); in kvm_is_visible_gfn()
2428 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn) in gfn_to_hva() argument
2430 return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL); in gfn_to_hva()
2459 unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable) in gfn_to_hva_prot() argument
2461 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); in gfn_to_hva_prot()
2735 kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault, in gfn_to_pfn_prot() argument
2738 return __gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn, false, false, in gfn_to_pfn_prot()
2763 kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn) in gfn_to_pfn() argument
2765 return gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn); in gfn_to_pfn()
2798 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) in gfn_to_page() argument
2803 pfn = gfn_to_pfn(kvm, gfn); in gfn_to_page()
2833 pfn = gfn_to_pfn(vcpu->kvm, gfn); in kvm_vcpu_map()
3000 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, in kvm_read_guest_page() argument
3003 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); in kvm_read_guest_page()
3018 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len) in kvm_read_guest() argument
3026 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg); in kvm_read_guest()
3086 static int __kvm_write_guest_page(struct kvm *kvm, in __kvm_write_guest_page() argument
3099 mark_page_dirty_in_slot(kvm, memslot, gfn); in __kvm_write_guest_page()
3103 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, in kvm_write_guest_page() argument
3106 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); in kvm_write_guest_page()
3108 return __kvm_write_guest_page(kvm, slot, gfn, data, offset, len); in kvm_write_guest_page()
3117 return __kvm_write_guest_page(vcpu->kvm, slot, gfn, data, offset, len); in kvm_vcpu_write_guest_page()
3121 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, in kvm_write_guest() argument
3130 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg); in kvm_write_guest()
3204 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc, in kvm_gfn_to_hva_cache_init() argument
3207 struct kvm_memslots *slots = kvm_memslots(kvm); in kvm_gfn_to_hva_cache_init()
3212 int kvm_write_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, in kvm_write_guest_offset_cached() argument
3216 struct kvm_memslots *slots = kvm_memslots(kvm); in kvm_write_guest_offset_cached()
3232 return kvm_write_guest(kvm, gpa, data, len); in kvm_write_guest_offset_cached()
3237 mark_page_dirty_in_slot(kvm, ghc->memslot, gpa >> PAGE_SHIFT); in kvm_write_guest_offset_cached()
3243 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, in kvm_write_guest_cached() argument
3246 return kvm_write_guest_offset_cached(kvm, ghc, data, 0, len); in kvm_write_guest_cached()
3250 int kvm_read_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, in kvm_read_guest_offset_cached() argument
3254 struct kvm_memslots *slots = kvm_memslots(kvm); in kvm_read_guest_offset_cached()
3270 return kvm_read_guest(kvm, gpa, data, len); in kvm_read_guest_offset_cached()
3280 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, in kvm_read_guest_cached() argument
3283 return kvm_read_guest_offset_cached(kvm, ghc, data, 0, len); in kvm_read_guest_cached()
3287 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len) in kvm_clear_guest() argument
3296 ret = kvm_write_guest_page(kvm, gfn, zero_page, offset, len); in kvm_clear_guest()
3307 void mark_page_dirty_in_slot(struct kvm *kvm, in mark_page_dirty_in_slot() argument
3314 if (WARN_ON_ONCE(vcpu && vcpu->kvm != kvm)) in mark_page_dirty_in_slot()
3317 WARN_ON_ONCE(!vcpu && !kvm_arch_allow_write_without_running_vcpu(kvm)); in mark_page_dirty_in_slot()
3324 if (kvm->dirty_ring_size && vcpu) in mark_page_dirty_in_slot()
3332 void mark_page_dirty(struct kvm *kvm, gfn_t gfn) in mark_page_dirty() argument
3336 memslot = gfn_to_memslot(kvm, gfn); in mark_page_dirty()
3337 mark_page_dirty_in_slot(kvm, memslot, gfn); in mark_page_dirty()
3346 mark_page_dirty_in_slot(vcpu->kvm, memslot, gfn); in kvm_vcpu_mark_page_dirty()
3414 int idx = srcu_read_lock(&vcpu->kvm->srcu); in kvm_vcpu_check_block()
3427 srcu_read_unlock(&vcpu->kvm->srcu, idx); in kvm_vcpu_check_block()
3492 struct kvm *kvm = vcpu->kvm; in kvm_vcpu_max_halt_poll_ns() local
3494 if (kvm->override_halt_poll_ns) { in kvm_vcpu_max_halt_poll_ns()
3502 return READ_ONCE(kvm->max_halt_poll_ns); in kvm_vcpu_max_halt_poll_ns()
3730 struct kvm *kvm = me->kvm; in kvm_vcpu_on_spin() local
3732 int last_boosted_vcpu = me->kvm->last_boosted_vcpu; in kvm_vcpu_on_spin()
3747 kvm_for_each_vcpu(i, vcpu, kvm) { in kvm_vcpu_on_spin()
3768 kvm->last_boosted_vcpu = i; in kvm_vcpu_on_spin()
3784 static bool kvm_page_in_dirty_ring(struct kvm *kvm, unsigned long pgoff) in kvm_page_in_dirty_ring() argument
3789 kvm->dirty_ring_size / PAGE_SIZE); in kvm_page_in_dirty_ring()
3808 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring); in kvm_vcpu_fault()
3810 else if (kvm_page_in_dirty_ring(vcpu->kvm, vmf->pgoff)) in kvm_vcpu_fault()
3830 if ((kvm_page_in_dirty_ring(vcpu->kvm, vma->vm_pgoff) || in kvm_vcpu_mmap()
3831 kvm_page_in_dirty_ring(vcpu->kvm, vma->vm_pgoff + pages - 1)) && in kvm_vcpu_mmap()
3843 kvm_put_kvm(vcpu->kvm); in kvm_vcpu_release()
3886 vcpu->kvm->debugfs_dentry); in kvm_create_vcpu_debugfs()
3897 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id) in kvm_vm_ioctl_create_vcpu() argument
3906 mutex_lock(&kvm->lock); in kvm_vm_ioctl_create_vcpu()
3907 if (kvm->created_vcpus >= kvm->max_vcpus) { in kvm_vm_ioctl_create_vcpu()
3908 mutex_unlock(&kvm->lock); in kvm_vm_ioctl_create_vcpu()
3912 r = kvm_arch_vcpu_precreate(kvm, id); in kvm_vm_ioctl_create_vcpu()
3914 mutex_unlock(&kvm->lock); in kvm_vm_ioctl_create_vcpu()
3918 kvm->created_vcpus++; in kvm_vm_ioctl_create_vcpu()
3919 mutex_unlock(&kvm->lock); in kvm_vm_ioctl_create_vcpu()
3935 kvm_vcpu_init(vcpu, kvm, id); in kvm_vm_ioctl_create_vcpu()
3941 if (kvm->dirty_ring_size) { in kvm_vm_ioctl_create_vcpu()
3943 id, kvm->dirty_ring_size); in kvm_vm_ioctl_create_vcpu()
3948 mutex_lock(&kvm->lock); in kvm_vm_ioctl_create_vcpu()
3956 if (kvm_get_vcpu_by_id(kvm, id)) { in kvm_vm_ioctl_create_vcpu()
3961 vcpu->vcpu_idx = atomic_read(&kvm->online_vcpus); in kvm_vm_ioctl_create_vcpu()
3962 r = xa_insert(&kvm->vcpu_array, vcpu->vcpu_idx, vcpu, GFP_KERNEL_ACCOUNT); in kvm_vm_ioctl_create_vcpu()
3968 kvm_get_kvm(kvm); in kvm_vm_ioctl_create_vcpu()
3971 xa_erase(&kvm->vcpu_array, vcpu->vcpu_idx); in kvm_vm_ioctl_create_vcpu()
3972 kvm_put_kvm_no_destroy(kvm); in kvm_vm_ioctl_create_vcpu()
3981 atomic_inc(&kvm->online_vcpus); in kvm_vm_ioctl_create_vcpu()
3983 mutex_unlock(&kvm->lock); in kvm_vm_ioctl_create_vcpu()
3989 mutex_unlock(&kvm->lock); in kvm_vm_ioctl_create_vcpu()
3998 mutex_lock(&kvm->lock); in kvm_vm_ioctl_create_vcpu()
3999 kvm->created_vcpus--; in kvm_vm_ioctl_create_vcpu()
4000 mutex_unlock(&kvm->lock); in kvm_vm_ioctl_create_vcpu()
4062 if (vcpu->kvm->mm != current->mm || vcpu->kvm->vm_dead) in kvm_vcpu_ioctl()
4272 if (vcpu->kvm->mm != current->mm || vcpu->kvm->vm_dead) in kvm_vcpu_compat_ioctl()
4338 if (dev->kvm->mm != current->mm || dev->kvm->vm_dead) in kvm_device_ioctl()
4359 struct kvm *kvm = dev->kvm; in kvm_device_release() local
4362 mutex_lock(&kvm->lock); in kvm_device_release()
4365 mutex_unlock(&kvm->lock); in kvm_device_release()
4368 kvm_put_kvm(kvm); in kvm_device_release()
4412 static int kvm_ioctl_create_device(struct kvm *kvm, in kvm_ioctl_create_device() argument
4437 dev->kvm = kvm; in kvm_ioctl_create_device()
4439 mutex_lock(&kvm->lock); in kvm_ioctl_create_device()
4442 mutex_unlock(&kvm->lock); in kvm_ioctl_create_device()
4446 list_add(&dev->vm_node, &kvm->devices); in kvm_ioctl_create_device()
4447 mutex_unlock(&kvm->lock); in kvm_ioctl_create_device()
4452 kvm_get_kvm(kvm); in kvm_ioctl_create_device()
4455 kvm_put_kvm_no_destroy(kvm); in kvm_ioctl_create_device()
4456 mutex_lock(&kvm->lock); in kvm_ioctl_create_device()
4460 mutex_unlock(&kvm->lock); in kvm_ioctl_create_device()
4470 static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) in kvm_vm_ioctl_check_extension_generic() argument
4530 return kvm_vm_ioctl_check_extension(kvm, arg); in kvm_vm_ioctl_check_extension_generic()
4533 static int kvm_vm_ioctl_enable_dirty_log_ring(struct kvm *kvm, u32 size) in kvm_vm_ioctl_enable_dirty_log_ring() argument
4554 if (kvm->dirty_ring_size) in kvm_vm_ioctl_enable_dirty_log_ring()
4557 mutex_lock(&kvm->lock); in kvm_vm_ioctl_enable_dirty_log_ring()
4559 if (kvm->created_vcpus) { in kvm_vm_ioctl_enable_dirty_log_ring()
4563 kvm->dirty_ring_size = size; in kvm_vm_ioctl_enable_dirty_log_ring()
4567 mutex_unlock(&kvm->lock); in kvm_vm_ioctl_enable_dirty_log_ring()
4571 static int kvm_vm_ioctl_reset_dirty_pages(struct kvm *kvm) in kvm_vm_ioctl_reset_dirty_pages() argument
4577 if (!kvm->dirty_ring_size) in kvm_vm_ioctl_reset_dirty_pages()
4580 mutex_lock(&kvm->slots_lock); in kvm_vm_ioctl_reset_dirty_pages()
4582 kvm_for_each_vcpu(i, vcpu, kvm) in kvm_vm_ioctl_reset_dirty_pages()
4583 cleared += kvm_dirty_ring_reset(vcpu->kvm, &vcpu->dirty_ring); in kvm_vm_ioctl_reset_dirty_pages()
4585 mutex_unlock(&kvm->slots_lock); in kvm_vm_ioctl_reset_dirty_pages()
4588 kvm_flush_remote_tlbs(kvm); in kvm_vm_ioctl_reset_dirty_pages()
4593 int __attribute__((weak)) kvm_vm_ioctl_enable_cap(struct kvm *kvm, in kvm_vm_ioctl_enable_cap() argument
4599 static bool kvm_are_all_memslots_empty(struct kvm *kvm) in kvm_are_all_memslots_empty() argument
4603 lockdep_assert_held(&kvm->slots_lock); in kvm_are_all_memslots_empty()
4606 if (!kvm_memslots_empty(__kvm_memslots(kvm, i))) in kvm_are_all_memslots_empty()
4613 static int kvm_vm_ioctl_enable_cap_generic(struct kvm *kvm, in kvm_vm_ioctl_enable_cap_generic() argument
4626 kvm->manual_dirty_log_protect = cap->args[0]; in kvm_vm_ioctl_enable_cap_generic()
4634 kvm->max_halt_poll_ns = cap->args[0]; in kvm_vm_ioctl_enable_cap_generic()
4643 kvm->override_halt_poll_ns = true; in kvm_vm_ioctl_enable_cap_generic()
4649 if (!kvm_vm_ioctl_check_extension_generic(kvm, cap->cap)) in kvm_vm_ioctl_enable_cap_generic()
4652 return kvm_vm_ioctl_enable_dirty_log_ring(kvm, cap->args[0]); in kvm_vm_ioctl_enable_cap_generic()
4657 !kvm->dirty_ring_size || cap->flags) in kvm_vm_ioctl_enable_cap_generic()
4660 mutex_lock(&kvm->slots_lock); in kvm_vm_ioctl_enable_cap_generic()
4667 if (kvm_are_all_memslots_empty(kvm)) { in kvm_vm_ioctl_enable_cap_generic()
4668 kvm->dirty_ring_with_bitmap = true; in kvm_vm_ioctl_enable_cap_generic()
4672 mutex_unlock(&kvm->slots_lock); in kvm_vm_ioctl_enable_cap_generic()
4677 return kvm_vm_ioctl_enable_cap(kvm, cap); in kvm_vm_ioctl_enable_cap_generic()
4684 struct kvm *kvm = file->private_data; in kvm_vm_stats_read() local
4686 return kvm_stats_read(kvm->stats_id, &kvm_vm_stats_header, in kvm_vm_stats_read()
4687 &kvm_vm_stats_desc[0], &kvm->stat, in kvm_vm_stats_read()
4688 sizeof(kvm->stat), user_buffer, size, offset); in kvm_vm_stats_read()
4696 static int kvm_vm_ioctl_get_stats_fd(struct kvm *kvm) in kvm_vm_ioctl_get_stats_fd() argument
4706 &kvm_vm_stats_fops, kvm, O_RDONLY); in kvm_vm_ioctl_get_stats_fd()
4720 struct kvm *kvm = filp->private_data; in kvm_vm_ioctl() local
4724 if (kvm->mm != current->mm || kvm->vm_dead) in kvm_vm_ioctl()
4728 r = kvm_vm_ioctl_create_vcpu(kvm, arg); in kvm_vm_ioctl()
4736 r = kvm_vm_ioctl_enable_cap_generic(kvm, &cap); in kvm_vm_ioctl()
4747 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem); in kvm_vm_ioctl()
4756 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); in kvm_vm_ioctl()
4766 r = kvm_vm_ioctl_clear_dirty_log(kvm, &log); in kvm_vm_ioctl()
4777 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone); in kvm_vm_ioctl()
4786 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone); in kvm_vm_ioctl()
4796 r = kvm_irqfd(kvm, &data); in kvm_vm_ioctl()
4805 r = kvm_ioeventfd(kvm, &data); in kvm_vm_ioctl()
4815 r = kvm_send_userspace_msi(kvm, &msi); in kvm_vm_ioctl()
4828 r = kvm_vm_ioctl_irq_line(kvm, &irq_event, in kvm_vm_ioctl()
4853 if (!kvm_arch_can_set_irq_routing(kvm)) in kvm_vm_ioctl()
4869 r = kvm_set_irq_routing(kvm, entries, routing.nr, in kvm_vm_ioctl()
4882 r = kvm_ioctl_create_device(kvm, &cd); in kvm_vm_ioctl()
4894 r = kvm_vm_ioctl_check_extension_generic(kvm, arg); in kvm_vm_ioctl()
4897 r = kvm_vm_ioctl_reset_dirty_pages(kvm); in kvm_vm_ioctl()
4900 r = kvm_vm_ioctl_get_stats_fd(kvm); in kvm_vm_ioctl()
4938 struct kvm *kvm = filp->private_data; in kvm_vm_compat_ioctl() local
4941 if (kvm->mm != current->mm || kvm->vm_dead) in kvm_vm_compat_ioctl()
4963 r = kvm_vm_ioctl_clear_dirty_log(kvm, &log); in kvm_vm_compat_ioctl()
4979 r = kvm_vm_ioctl_get_dirty_log(kvm, &log); in kvm_vm_compat_ioctl()
5006 struct kvm *kvm; in kvm_dev_ioctl_create_vm() local
5015 kvm = kvm_create_vm(type, fdname); in kvm_dev_ioctl_create_vm()
5016 if (IS_ERR(kvm)) { in kvm_dev_ioctl_create_vm()
5017 r = PTR_ERR(kvm); in kvm_dev_ioctl_create_vm()
5021 file = anon_inode_getfile("kvm-vm", &kvm_vm_fops, kvm, O_RDWR); in kvm_dev_ioctl_create_vm()
5033 kvm_uevent_notify_change(KVM_EVENT_CREATE_VM, kvm); in kvm_dev_ioctl_create_vm()
5039 kvm_put_kvm(kvm); in kvm_dev_ioctl_create_vm()
5376 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); in kvm_io_bus_write()
5396 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); in kvm_io_bus_write_cookie()
5447 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu); in kvm_io_bus_read()
5455 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, in kvm_io_bus_register_dev() argument
5462 bus = kvm_get_bus(kvm, bus_idx); in kvm_io_bus_register_dev()
5490 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); in kvm_io_bus_register_dev()
5491 synchronize_srcu_expedited(&kvm->srcu); in kvm_io_bus_register_dev()
5497 int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, in kvm_io_bus_unregister_dev() argument
5503 lockdep_assert_held(&kvm->slots_lock); in kvm_io_bus_unregister_dev()
5505 bus = kvm_get_bus(kvm, bus_idx); in kvm_io_bus_unregister_dev()
5527 rcu_assign_pointer(kvm->buses[bus_idx], new_bus); in kvm_io_bus_unregister_dev()
5528 synchronize_srcu_expedited(&kvm->srcu); in kvm_io_bus_unregister_dev()
5544 struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx, in kvm_io_bus_get_dev() argument
5551 srcu_idx = srcu_read_lock(&kvm->srcu); in kvm_io_bus_get_dev()
5553 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu); in kvm_io_bus_get_dev()
5564 srcu_read_unlock(&kvm->srcu, srcu_idx); in kvm_io_bus_get_dev()
5583 if (!kvm_get_kvm_safe(stat_data->kvm)) in kvm_debugfs_open()
5590 kvm_put_kvm(stat_data->kvm); in kvm_debugfs_open()
5601 kvm_put_kvm(stat_data->kvm); in kvm_debugfs_release()
5606 static int kvm_get_stat_per_vm(struct kvm *kvm, size_t offset, u64 *val) in kvm_get_stat_per_vm() argument
5608 *val = *(u64 *)((void *)(&kvm->stat) + offset); in kvm_get_stat_per_vm()
5613 static int kvm_clear_stat_per_vm(struct kvm *kvm, size_t offset) in kvm_clear_stat_per_vm() argument
5615 *(u64 *)((void *)(&kvm->stat) + offset) = 0; in kvm_clear_stat_per_vm()
5620 static int kvm_get_stat_per_vcpu(struct kvm *kvm, size_t offset, u64 *val) in kvm_get_stat_per_vcpu() argument
5627 kvm_for_each_vcpu(i, vcpu, kvm) in kvm_get_stat_per_vcpu()
5633 static int kvm_clear_stat_per_vcpu(struct kvm *kvm, size_t offset) in kvm_clear_stat_per_vcpu() argument
5638 kvm_for_each_vcpu(i, vcpu, kvm) in kvm_clear_stat_per_vcpu()
5651 r = kvm_get_stat_per_vm(stat_data->kvm, in kvm_stat_data_get()
5655 r = kvm_get_stat_per_vcpu(stat_data->kvm, in kvm_stat_data_get()
5673 r = kvm_clear_stat_per_vm(stat_data->kvm, in kvm_stat_data_clear()
5677 r = kvm_clear_stat_per_vcpu(stat_data->kvm, in kvm_stat_data_clear()
5704 struct kvm *kvm; in vm_stat_get() local
5709 list_for_each_entry(kvm, &vm_list, vm_list) { in vm_stat_get()
5710 kvm_get_stat_per_vm(kvm, offset, &tmp_val); in vm_stat_get()
5720 struct kvm *kvm; in vm_stat_clear() local
5726 list_for_each_entry(kvm, &vm_list, vm_list) { in vm_stat_clear()
5727 kvm_clear_stat_per_vm(kvm, offset); in vm_stat_clear()
5740 struct kvm *kvm; in vcpu_stat_get() local
5745 list_for_each_entry(kvm, &vm_list, vm_list) { in vcpu_stat_get()
5746 kvm_get_stat_per_vcpu(kvm, offset, &tmp_val); in vcpu_stat_get()
5756 struct kvm *kvm; in vcpu_stat_clear() local
5762 list_for_each_entry(kvm, &vm_list, vm_list) { in vcpu_stat_clear()
5763 kvm_clear_stat_per_vcpu(kvm, offset); in vcpu_stat_clear()
5774 static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm) in kvm_uevent_notify_change() argument
5779 if (!kvm_dev.this_device || !kvm) in kvm_uevent_notify_change()
5802 kvm->userspace_pid = task_pid_nr(current); in kvm_uevent_notify_change()
5806 add_uevent_var(env, "PID=%d", kvm->userspace_pid); in kvm_uevent_notify_change()
5808 if (!IS_ERR(kvm->debugfs_dentry)) { in kvm_uevent_notify_change()
5812 tmp = dentry_path_raw(kvm->debugfs_dentry, p, PATH_MAX); in kvm_uevent_notify_change()
6076 struct kvm *kvm; member
6092 struct kvm *kvm = init_context->kvm; in kvm_vm_worker_thread() local
6124 err = thread_fn(kvm, data); in kvm_vm_worker_thread()
6148 int kvm_vm_create_worker_thread(struct kvm *kvm, kvm_vm_thread_fn_t thread_fn, in kvm_vm_create_worker_thread() argument
6156 init_context.kvm = kvm; in kvm_vm_create_worker_thread()