Lines Matching refs:gtt

569 	const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in _ppgtt_get_root_entry()
596 	const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in _ppgtt_set_root_entry()
612 	const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in ggtt_get_guest_entry()
624 	const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in ggtt_set_guest_entry()
635 	const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in ggtt_get_host_entry()
645 	const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in ggtt_set_host_entry()
671 	const struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;  in ppgtt_spt_get_entry()
700 	const struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;  in ppgtt_spt_set_entry()
763 	radix_tree_delete(&spt->vgpu->gtt.spt_tree, spt->shadow_page.mfn);  in ppgtt_free_spt()
784 	radix_tree_for_each_slot(slot, &vgpu->gtt.spt_tree, &iter, 0) {  in ppgtt_free_all_spt()
832 	return radix_tree_lookup(&vgpu->gtt.spt_tree, mfn);  in intel_vgpu_find_spt_by_mfn()
874 	ret = radix_tree_insert(&vgpu->gtt.spt_tree, spt->shadow_page.mfn, spt);  in ppgtt_alloc_spt()
928 		    spt->vgpu->gvt->gtt.pte_ops->test_present(e))
934 		    spt->vgpu->gvt->gtt.pte_ops->test_present(e))
963 	const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_invalidate_spt_by_shadow_entry()
984 			vgpu->gtt.scratch_pt[cur_pt_type].page_mfn)  in ppgtt_invalidate_spt_by_shadow_entry()
1000 	const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_invalidate_pte()
1008 	if (!pfn || pfn == vgpu->gtt.scratch_pt[type].page_mfn)  in ppgtt_invalidate_pte()
1088 	const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_populate_spt_by_guest_entry()
1152 	const struct intel_gvt_gtt_pte_ops *ops = s->vgpu->gvt->gtt.pte_ops;  in ppgtt_generate_shadow_entry()
1175 	const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in is_2MB_gtt_possible()
1193 	const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in split_2MB_gtt_entry()
1249 	const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in split_64KB_gtt_entry()
1280 	const struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_populate_shadow_entry()
1333 	const struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;  in ppgtt_populate_spt()
1355 				ops->set_pfn(&se, gvt->gtt.scratch_mfn);  in ppgtt_populate_spt()
1376 	const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_handle_guest_entry_removal()
1389 	    vgpu->gtt.scratch_pt[spt->shadow_page.type].page_mfn)  in ppgtt_handle_guest_entry_removal()
1457 	const struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;  in sync_oos_page()
1509 	list_move_tail(&oos_page->list, &gvt->gtt.oos_page_free_list_head);  in detach_oos_page()
1529 	list_move_tail(&oos_page->list, &gvt->gtt.oos_page_use_list_head);  in attach_oos_page()
1555 	struct intel_gvt_gtt *gtt = &gvt->gtt;  in ppgtt_allocate_oos_page()  local
1561 	if (list_empty(&gtt->oos_page_free_list_head)) {  in ppgtt_allocate_oos_page()
1562 		oos_page = container_of(gtt->oos_page_use_list_head.next,  in ppgtt_allocate_oos_page()
1571 		oos_page = container_of(gtt->oos_page_free_list_head.next,  in ppgtt_allocate_oos_page()
1586 	list_add_tail(&oos_page->vm_list, &spt->vgpu->gtt.oos_page_list_head);  in ppgtt_set_guest_page_oos()
1609 	list_for_each_safe(pos, n, &vgpu->gtt.oos_page_list_head) {  in intel_vgpu_sync_oos_pages()
1628 	const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_handle_guest_write_page_table()
1660 					vgpu->gtt.scratch_pt[type].page_mfn);  in ppgtt_handle_guest_write_page_table()
1667 				     vgpu->gtt.scratch_pt[type].page_mfn);  in ppgtt_handle_guest_write_page_table()
1671 				     vgpu->gtt.scratch_pt[type].page_mfn);  in ppgtt_handle_guest_write_page_table()
1700 			&spt->vgpu->gtt.post_shadow_list_head);  in ppgtt_set_post_shadow()
1721 	list_for_each_safe(pos, n, &vgpu->gtt.post_shadow_list_head) {  in intel_vgpu_flush_post_shadow()
1745 	const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_handle_guest_write_page_table_bytes()
1779 			ops->set_pfn(&se, vgpu->gtt.scratch_pt[type].page_mfn);  in ppgtt_handle_guest_write_page_table_bytes()
1809 	struct intel_gvt_gtt *gtt = &gvt->gtt;  in invalidate_ppgtt_mm()  local
1810 	const struct intel_gvt_gtt_pte_ops *ops = gtt->pte_ops;  in invalidate_ppgtt_mm()
1839 	struct intel_gvt_gtt *gtt = &gvt->gtt;  in shadow_ppgtt_mm()  local
1840 	const struct intel_gvt_gtt_pte_ops *ops = gtt->pte_ops;  in shadow_ppgtt_mm()
1943 	list_add_tail(&mm->ppgtt_mm.list, &vgpu->gtt.ppgtt_mm_list_head);  in intel_vgpu_create_ppgtt_mm()
1945 	mutex_lock(&gvt->gtt.ppgtt_mm_lock);  in intel_vgpu_create_ppgtt_mm()
1946 	list_add_tail(&mm->ppgtt_mm.lru_list, &gvt->gtt.ppgtt_mm_lru_list_head);  in intel_vgpu_create_ppgtt_mm()
1947 	mutex_unlock(&gvt->gtt.ppgtt_mm_lock);  in intel_vgpu_create_ppgtt_mm()
2007 		mutex_lock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock);  in _intel_vgpu_mm_release()
2009 		mutex_unlock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock);  in _intel_vgpu_mm_release()
2054 		mutex_lock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock);  in intel_vgpu_pin_mm()
2056 			       &mm->vgpu->gvt->gtt.ppgtt_mm_lru_list_head);  in intel_vgpu_pin_mm()
2057 		mutex_unlock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock);  in intel_vgpu_pin_mm()
2068 	mutex_lock(&gvt->gtt.ppgtt_mm_lock);  in reclaim_one_ppgtt_mm()
2070 	list_for_each_safe(pos, n, &gvt->gtt.ppgtt_mm_lru_list_head) {  in reclaim_one_ppgtt_mm()
2077 		mutex_unlock(&gvt->gtt.ppgtt_mm_lock);  in reclaim_one_ppgtt_mm()
2081 	mutex_unlock(&gvt->gtt.ppgtt_mm_lock);  in reclaim_one_ppgtt_mm()
2092 	const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_get_next_level_entry()
2121 	const struct intel_gvt_gtt_pte_ops *pte_ops = gvt->gtt.pte_ops;  in intel_vgpu_gma_to_gpa()
2122 	const struct intel_gvt_gtt_gma_ops *gma_ops = gvt->gtt.gma_ops;  in intel_vgpu_gma_to_gpa()
2194 	struct intel_vgpu_mm *ggtt_mm = vgpu->gtt.ggtt_mm;  in emulate_ggtt_mmio_read()
2246 	const struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops;  in ggtt_invalidate_pte()
2250 	if (pfn != vgpu->gvt->gtt.scratch_mfn)  in ggtt_invalidate_pte()
2259 	struct intel_vgpu_mm *ggtt_mm = vgpu->gtt.ggtt_mm;  in emulate_ggtt_mmio_write()
2260 	const struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;  in emulate_ggtt_mmio_write()
2338 			ops->set_pfn(&m, gvt->gtt.scratch_mfn);  in emulate_ggtt_mmio_write()
2350 			ops->set_pfn(&m, gvt->gtt.scratch_mfn);  in emulate_ggtt_mmio_write()
2354 		ops->set_pfn(&m, gvt->gtt.scratch_mfn);  in emulate_ggtt_mmio_write()
2414 	struct intel_vgpu_gtt *gtt = &vgpu->gtt;  in alloc_scratch_pages()  local
2415 	const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in alloc_scratch_pages()
2439 	gtt->scratch_pt[type].page_mfn =  in alloc_scratch_pages()
2441 	gtt->scratch_pt[type].page = virt_to_page(scratch_pt);  in alloc_scratch_pages()
2443 			vgpu->id, type, gtt->scratch_pt[type].page_mfn);  in alloc_scratch_pages()
2458 		ops->set_pfn(&se, gtt->scratch_pt[type - 1].page_mfn);  in alloc_scratch_pages()
2481 		if (vgpu->gtt.scratch_pt[i].page != NULL) {  in release_scratch_page_tree()
2482 			daddr = (dma_addr_t)(vgpu->gtt.scratch_pt[i].page_mfn <<  in release_scratch_page_tree()
2485 			__free_page(vgpu->gtt.scratch_pt[i].page);  in release_scratch_page_tree()
2486 			vgpu->gtt.scratch_pt[i].page = NULL;  in release_scratch_page_tree()
2487 			vgpu->gtt.scratch_pt[i].page_mfn = 0;  in release_scratch_page_tree()
2523 	struct intel_vgpu_gtt *gtt = &vgpu->gtt;  in intel_vgpu_init_gtt()  local
2525 	INIT_RADIX_TREE(&gtt->spt_tree, GFP_KERNEL);  in intel_vgpu_init_gtt()
2527 	INIT_LIST_HEAD(&gtt->ppgtt_mm_list_head);  in intel_vgpu_init_gtt()
2528 	INIT_LIST_HEAD(&gtt->oos_page_list_head);  in intel_vgpu_init_gtt()
2529 	INIT_LIST_HEAD(&gtt->post_shadow_list_head);  in intel_vgpu_init_gtt()
2531 	gtt->ggtt_mm = intel_vgpu_create_ggtt_mm(vgpu);  in intel_vgpu_init_gtt()
2532 	if (IS_ERR(gtt->ggtt_mm)) {  in intel_vgpu_init_gtt()
2534 		return PTR_ERR(gtt->ggtt_mm);  in intel_vgpu_init_gtt()
2539 	INIT_LIST_HEAD(&gtt->ggtt_mm->ggtt_mm.partial_pte_list);  in intel_vgpu_init_gtt()
2549 	list_for_each_safe(pos, n, &vgpu->gtt.ppgtt_mm_list_head) {  in intel_vgpu_destroy_all_ppgtt_mm()
2554 	if (GEM_WARN_ON(!list_empty(&vgpu->gtt.ppgtt_mm_list_head)))  in intel_vgpu_destroy_all_ppgtt_mm()
2557 	if (GEM_WARN_ON(!radix_tree_empty(&vgpu->gtt.spt_tree))) {  in intel_vgpu_destroy_all_ppgtt_mm()
2568 				 &vgpu->gtt.ggtt_mm->ggtt_mm.partial_pte_list,  in intel_vgpu_destroy_ggtt_mm()
2574 	intel_vgpu_destroy_mm(vgpu->gtt.ggtt_mm);  in intel_vgpu_destroy_ggtt_mm()
2575 	vgpu->gtt.ggtt_mm = NULL;  in intel_vgpu_destroy_ggtt_mm()
2597 	struct intel_gvt_gtt *gtt = &gvt->gtt;  in clean_spt_oos()  local
2601 	WARN(!list_empty(&gtt->oos_page_use_list_head),  in clean_spt_oos()
2604 	list_for_each_safe(pos, n, &gtt->oos_page_free_list_head) {  in clean_spt_oos()
2614 	struct intel_gvt_gtt *gtt = &gvt->gtt;  in setup_spt_oos()  local
2619 	INIT_LIST_HEAD(&gtt->oos_page_free_list_head);  in setup_spt_oos()
2620 	INIT_LIST_HEAD(&gtt->oos_page_use_list_head);  in setup_spt_oos()
2638 		list_add_tail(&oos_page->list, &gtt->oos_page_free_list_head);  in setup_spt_oos()
2665 	list_for_each(pos, &vgpu->gtt.ppgtt_mm_list_head) {  in intel_vgpu_find_ppgtt_mm()
2754 	gvt->gtt.pte_ops = &gen8_gtt_pte_ops;  in intel_gvt_init_gtt()
2755 	gvt->gtt.gma_ops = &gen8_gtt_gma_ops;  in intel_gvt_init_gtt()
2771 	gvt->gtt.scratch_page = virt_to_page(page);  in intel_gvt_init_gtt()
2772 	gvt->gtt.scratch_mfn = (unsigned long)(daddr >> I915_GTT_PAGE_SHIFT);  in intel_gvt_init_gtt()
2779 			__free_page(gvt->gtt.scratch_page);  in intel_gvt_init_gtt()
2783 	INIT_LIST_HEAD(&gvt->gtt.ppgtt_mm_lru_list_head);  in intel_gvt_init_gtt()
2784 	mutex_init(&gvt->gtt.ppgtt_mm_lock);  in intel_gvt_init_gtt()
2799 	dma_addr_t daddr = (dma_addr_t)(gvt->gtt.scratch_mfn <<  in intel_gvt_clean_gtt()
2804 	__free_page(gvt->gtt.scratch_page);  in intel_gvt_clean_gtt()
2822 	list_for_each_safe(pos, n, &vgpu->gtt.ppgtt_mm_list_head) {  in intel_vgpu_invalidate_ppgtt()
2825 			mutex_lock(&vgpu->gvt->gtt.ppgtt_mm_lock);  in intel_vgpu_invalidate_ppgtt()
2827 			mutex_unlock(&vgpu->gvt->gtt.ppgtt_mm_lock);  in intel_vgpu_invalidate_ppgtt()
2846 	const struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops;  in intel_vgpu_reset_ggtt()
2852 	pte_ops->set_pfn(&entry, gvt->gtt.scratch_mfn);  in intel_vgpu_reset_ggtt()
2859 			ggtt_get_host_entry(vgpu->gtt.ggtt_mm, &old_entry, index);  in intel_vgpu_reset_ggtt()
2862 		ggtt_set_host_entry(vgpu->gtt.ggtt_mm, &entry, index++);  in intel_vgpu_reset_ggtt()
2869 			ggtt_get_host_entry(vgpu->gtt.ggtt_mm, &old_entry, index);  in intel_vgpu_reset_ggtt()
2872 		ggtt_set_host_entry(vgpu->gtt.ggtt_mm, &entry, index++);  in intel_vgpu_reset_ggtt()
2914 		mm = vgpu->gtt.ggtt_mm;  in intel_gvt_restore_ggtt()