Lines Matching refs:head
105 spin_lock_irqsave(&ns->head->requeue_lock, flags); in nvme_failover_req()
107 bio_set_dev(bio, ns->head->disk->part0); in nvme_failover_req()
121 blk_steal_bios(&ns->head->requeue_list, req); in nvme_failover_req()
122 spin_unlock_irqrestore(&ns->head->requeue_lock, flags); in nvme_failover_req()
126 kblockd_schedule_work(&ns->head->requeue_work); in nvme_failover_req()
132 struct gendisk *disk = ns->head->disk; in nvme_mpath_start_request()
134 if (READ_ONCE(ns->head->subsys->iopolicy) == NVME_IOPOLICY_QD) { in nvme_mpath_start_request()
157 bdev_end_io_acct(ns->head->disk->part0, req_op(rq), in nvme_mpath_end_request()
169 if (!ns->head->disk) in nvme_kick_requeue_lists()
171 kblockd_schedule_work(&ns->head->requeue_work); in nvme_kick_requeue_lists()
173 disk_uevent(ns->head->disk, KOBJ_CHANGE); in nvme_kick_requeue_lists()
189 struct nvme_ns_head *head = ns->head; in nvme_mpath_clear_current_path() local
193 if (!head) in nvme_mpath_clear_current_path()
197 if (ns == rcu_access_pointer(head->current_path[node])) { in nvme_mpath_clear_current_path()
198 rcu_assign_pointer(head->current_path[node], NULL); in nvme_mpath_clear_current_path()
214 kblockd_schedule_work(&ns->head->requeue_work); in nvme_mpath_clear_ctrl_paths()
221 struct nvme_ns_head *head = ns->head; in nvme_mpath_revalidate_paths() local
222 sector_t capacity = get_capacity(head->disk); in nvme_mpath_revalidate_paths()
226 srcu_idx = srcu_read_lock(&head->srcu); in nvme_mpath_revalidate_paths()
227 list_for_each_entry_rcu(ns, &head->list, siblings) { in nvme_mpath_revalidate_paths()
231 srcu_read_unlock(&head->srcu, srcu_idx); in nvme_mpath_revalidate_paths()
234 rcu_assign_pointer(head->current_path[node], NULL); in nvme_mpath_revalidate_paths()
235 kblockd_schedule_work(&head->requeue_work); in nvme_mpath_revalidate_paths()
255 static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node) in __nvme_find_path() argument
260 list_for_each_entry_rcu(ns, &head->list, siblings) { in __nvme_find_path()
265 READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_NUMA) in __nvme_find_path()
291 rcu_assign_pointer(head->current_path[node], found); in __nvme_find_path()
295 static struct nvme_ns *nvme_next_ns(struct nvme_ns_head *head, in nvme_next_ns() argument
298 ns = list_next_or_null_rcu(&head->list, &ns->siblings, struct nvme_ns, in nvme_next_ns()
302 return list_first_or_null_rcu(&head->list, struct nvme_ns, siblings); in nvme_next_ns()
305 static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head) in nvme_round_robin_path() argument
309 struct nvme_ns *old = srcu_dereference(head->current_path[node], in nvme_round_robin_path()
310 &head->srcu); in nvme_round_robin_path()
313 return __nvme_find_path(head, node); in nvme_round_robin_path()
315 if (list_is_singular(&head->list)) { in nvme_round_robin_path()
321 for (ns = nvme_next_ns(head, old); in nvme_round_robin_path()
323 ns = nvme_next_ns(head, ns)) { in nvme_round_robin_path()
349 rcu_assign_pointer(head->current_path[node], found); in nvme_round_robin_path()
353 static struct nvme_ns *nvme_queue_depth_path(struct nvme_ns_head *head) in nvme_queue_depth_path() argument
359 list_for_each_entry_rcu(ns, &head->list, siblings) { in nvme_queue_depth_path()
395 static struct nvme_ns *nvme_numa_path(struct nvme_ns_head *head) in nvme_numa_path() argument
400 ns = srcu_dereference(head->current_path[node], &head->srcu); in nvme_numa_path()
402 return __nvme_find_path(head, node); in nvme_numa_path()
404 return __nvme_find_path(head, node); in nvme_numa_path()
408 inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head) in nvme_find_path() argument
410 switch (READ_ONCE(head->subsys->iopolicy)) { in nvme_find_path()
412 return nvme_queue_depth_path(head); in nvme_find_path()
414 return nvme_round_robin_path(head); in nvme_find_path()
416 return nvme_numa_path(head); in nvme_find_path()
420 static bool nvme_available_path(struct nvme_ns_head *head) in nvme_available_path() argument
424 if (!test_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) in nvme_available_path()
427 list_for_each_entry_rcu(ns, &head->list, siblings) { in nvme_available_path()
444 struct nvme_ns_head *head = bio->bi_bdev->bd_disk->private_data; in nvme_ns_head_submit_bio() local
445 struct device *dev = disk_to_dev(head->disk); in nvme_ns_head_submit_bio()
458 srcu_idx = srcu_read_lock(&head->srcu); in nvme_ns_head_submit_bio()
459 ns = nvme_find_path(head); in nvme_ns_head_submit_bio()
463 trace_block_bio_remap(bio, disk_devt(ns->head->disk), in nvme_ns_head_submit_bio()
466 } else if (nvme_available_path(head)) { in nvme_ns_head_submit_bio()
469 spin_lock_irq(&head->requeue_lock); in nvme_ns_head_submit_bio()
470 bio_list_add(&head->requeue_list, bio); in nvme_ns_head_submit_bio()
471 spin_unlock_irq(&head->requeue_lock); in nvme_ns_head_submit_bio()
478 srcu_read_unlock(&head->srcu, srcu_idx); in nvme_ns_head_submit_bio()
496 struct nvme_ns_head *head = disk->private_data; in nvme_ns_head_get_unique_id() local
500 srcu_idx = srcu_read_lock(&head->srcu); in nvme_ns_head_get_unique_id()
501 ns = nvme_find_path(head); in nvme_ns_head_get_unique_id()
504 srcu_read_unlock(&head->srcu, srcu_idx); in nvme_ns_head_get_unique_id()
512 struct nvme_ns_head *head = disk->private_data; in nvme_ns_head_report_zones() local
516 srcu_idx = srcu_read_lock(&head->srcu); in nvme_ns_head_report_zones()
517 ns = nvme_find_path(head); in nvme_ns_head_report_zones()
520 srcu_read_unlock(&head->srcu, srcu_idx); in nvme_ns_head_report_zones()
568 static int nvme_add_ns_head_cdev(struct nvme_ns_head *head) in nvme_add_ns_head_cdev() argument
572 head->cdev_device.parent = &head->subsys->dev; in nvme_add_ns_head_cdev()
573 ret = dev_set_name(&head->cdev_device, "ng%dn%d", in nvme_add_ns_head_cdev()
574 head->subsys->instance, head->instance); in nvme_add_ns_head_cdev()
577 ret = nvme_cdev_add(&head->cdev, &head->cdev_device, in nvme_add_ns_head_cdev()
584 struct nvme_ns_head *head = in nvme_partition_scan_work() local
588 &head->disk->state))) in nvme_partition_scan_work()
591 mutex_lock(&head->disk->open_mutex); in nvme_partition_scan_work()
592 bdev_disk_changed(head->disk, false); in nvme_partition_scan_work()
593 mutex_unlock(&head->disk->open_mutex); in nvme_partition_scan_work()
598 struct nvme_ns_head *head = in nvme_requeue_work() local
602 spin_lock_irq(&head->requeue_lock); in nvme_requeue_work()
603 next = bio_list_get(&head->requeue_list); in nvme_requeue_work()
604 spin_unlock_irq(&head->requeue_lock); in nvme_requeue_work()
614 int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head) in nvme_mpath_alloc_disk() argument
618 mutex_init(&head->lock); in nvme_mpath_alloc_disk()
619 bio_list_init(&head->requeue_list); in nvme_mpath_alloc_disk()
620 spin_lock_init(&head->requeue_lock); in nvme_mpath_alloc_disk()
621 INIT_WORK(&head->requeue_work, nvme_requeue_work); in nvme_mpath_alloc_disk()
622 INIT_WORK(&head->partition_scan_work, nvme_partition_scan_work); in nvme_mpath_alloc_disk()
630 !nvme_is_unique_nsid(ctrl, head) || !multipath) in nvme_mpath_alloc_disk()
636 if (head->ids.csi == NVME_CSI_ZNS) in nvme_mpath_alloc_disk()
641 head->disk = blk_alloc_disk(&lim, ctrl->numa_node); in nvme_mpath_alloc_disk()
642 if (IS_ERR(head->disk)) in nvme_mpath_alloc_disk()
643 return PTR_ERR(head->disk); in nvme_mpath_alloc_disk()
644 head->disk->fops = &nvme_ns_head_ops; in nvme_mpath_alloc_disk()
645 head->disk->private_data = head; in nvme_mpath_alloc_disk()
655 set_bit(GD_SUPPRESS_PART_SCAN, &head->disk->state); in nvme_mpath_alloc_disk()
656 sprintf(head->disk->disk_name, "nvme%dn%d", in nvme_mpath_alloc_disk()
657 ctrl->subsys->instance, head->instance); in nvme_mpath_alloc_disk()
663 struct nvme_ns_head *head = ns->head; in nvme_mpath_set_live() local
666 if (!head->disk) in nvme_mpath_set_live()
674 if (!test_and_set_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) { in nvme_mpath_set_live()
675 rc = device_add_disk(&head->subsys->dev, head->disk, in nvme_mpath_set_live()
678 clear_bit(NVME_NSHEAD_DISK_LIVE, &head->flags); in nvme_mpath_set_live()
681 nvme_add_ns_head_cdev(head); in nvme_mpath_set_live()
682 kblockd_schedule_work(&head->partition_scan_work); in nvme_mpath_set_live()
685 mutex_lock(&head->lock); in nvme_mpath_set_live()
689 srcu_idx = srcu_read_lock(&head->srcu); in nvme_mpath_set_live()
691 __nvme_find_path(head, node); in nvme_mpath_set_live()
692 srcu_read_unlock(&head->srcu, srcu_idx); in nvme_mpath_set_live()
694 mutex_unlock(&head->lock); in nvme_mpath_set_live()
696 synchronize_srcu(&head->srcu); in nvme_mpath_set_live()
697 kblockd_schedule_work(&head->requeue_work); in nvme_mpath_set_live()
792 if (ns->head->ns_id < nsid) in nvme_update_ana_state()
794 if (ns->head->ns_id == nsid) in nvme_update_ana_state()
798 if (ns->head->ns_id > nsid) in nvme_update_ana_state()
991 if (blk_queue_is_zoned(ns->queue) && ns->head->disk) in nvme_mpath_add_disk()
992 ns->head->disk->nr_zones = ns->disk->nr_zones; in nvme_mpath_add_disk()
996 void nvme_mpath_shutdown_disk(struct nvme_ns_head *head) in nvme_mpath_shutdown_disk() argument
998 if (!head->disk) in nvme_mpath_shutdown_disk()
1000 if (test_and_clear_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) { in nvme_mpath_shutdown_disk()
1001 nvme_cdev_del(&head->cdev, &head->cdev_device); in nvme_mpath_shutdown_disk()
1006 synchronize_srcu(&head->srcu); in nvme_mpath_shutdown_disk()
1007 kblockd_schedule_work(&head->requeue_work); in nvme_mpath_shutdown_disk()
1008 del_gendisk(head->disk); in nvme_mpath_shutdown_disk()
1012 void nvme_mpath_remove_disk(struct nvme_ns_head *head) in nvme_mpath_remove_disk() argument
1014 if (!head->disk) in nvme_mpath_remove_disk()
1017 kblockd_schedule_work(&head->requeue_work); in nvme_mpath_remove_disk()
1018 flush_work(&head->requeue_work); in nvme_mpath_remove_disk()
1019 flush_work(&head->partition_scan_work); in nvme_mpath_remove_disk()
1020 put_disk(head->disk); in nvme_mpath_remove_disk()