Lines Matching refs:hdev
43 struct hl_device *hdev = cs->ctx->hdev; in hl_hw_queue_update_ci() local
47 if (hdev->disabled) in hl_hw_queue_update_ci()
50 q = &hdev->kernel_queues[0]; in hl_hw_queue_update_ci()
53 if (!hdev->asic_prop.max_queues || q->queue_type == QUEUE_TYPE_HW) in hl_hw_queue_update_ci()
61 for (i = 0 ; i < hdev->asic_prop.max_queues ; i++, q++) { in hl_hw_queue_update_ci()
83 void hl_hw_queue_submit_bd(struct hl_device *hdev, struct hl_hw_queue *q, in hl_hw_queue_submit_bd() argument
95 hdev->asic_funcs->ring_doorbell(hdev, q->hw_queue_id, q->pi); in hl_hw_queue_submit_bd()
116 static int ext_queue_sanity_checks(struct hl_device *hdev, in ext_queue_sanity_checks() argument
121 &hdev->completion_queue[q->cq_id].free_slots_cnt; in ext_queue_sanity_checks()
128 dev_dbg(hdev->dev, "Queue %d doesn't have room for %d CBs\n", in ext_queue_sanity_checks()
142 dev_dbg(hdev->dev, "No space for %d on CQ %d\n", in ext_queue_sanity_checks()
165 static int int_queue_sanity_checks(struct hl_device *hdev, in int_queue_sanity_checks() argument
172 dev_err(hdev->dev, in int_queue_sanity_checks()
182 dev_dbg(hdev->dev, "Queue %d doesn't have room for %d CBs\n", in int_queue_sanity_checks()
200 static int hw_queue_sanity_checks(struct hl_device *hdev, struct hl_hw_queue *q, in hw_queue_sanity_checks() argument
209 dev_dbg(hdev->dev, "Queue %d doesn't have room for %d CBs\n", in hw_queue_sanity_checks()
228 int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id, in hl_hw_queue_send_cb_no_cmpl() argument
231 struct hl_hw_queue *q = &hdev->kernel_queues[hw_queue_id]; in hl_hw_queue_send_cb_no_cmpl()
234 hdev->asic_funcs->hw_queues_lock(hdev); in hl_hw_queue_send_cb_no_cmpl()
236 if (hdev->disabled) { in hl_hw_queue_send_cb_no_cmpl()
247 rc = ext_queue_sanity_checks(hdev, q, 1, false); in hl_hw_queue_send_cb_no_cmpl()
252 hl_hw_queue_submit_bd(hdev, q, 0, cb_size, cb_ptr); in hl_hw_queue_send_cb_no_cmpl()
255 hdev->asic_funcs->hw_queues_unlock(hdev); in hl_hw_queue_send_cb_no_cmpl()
270 struct hl_device *hdev = job->cs->ctx->hdev; in ext_queue_schedule_job() local
271 struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id]; in ext_queue_schedule_job()
307 cq = &hdev->completion_queue[q->cq_id]; in ext_queue_schedule_job()
310 hdev->asic_funcs->add_end_of_cb_packets(hdev, cb->kernel_address, len, in ext_queue_schedule_job()
321 hl_hw_queue_submit_bd(hdev, q, ctl, len, ptr); in ext_queue_schedule_job()
334 struct hl_device *hdev = job->cs->ctx->hdev; in int_queue_schedule_job() local
335 struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id]; in int_queue_schedule_job()
355 hdev->asic_funcs->pqe_write(hdev, pi, &bd); in int_queue_schedule_job()
357 hdev->asic_funcs->ring_doorbell(hdev, q->hw_queue_id, q->pi); in int_queue_schedule_job()
370 struct hl_device *hdev = job->cs->ctx->hdev; in hw_queue_schedule_job() local
371 struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id]; in hw_queue_schedule_job()
381 offset = job->cs->sequence & (hdev->asic_prop.max_pending_cs - 1); in hw_queue_schedule_job()
400 hl_hw_queue_submit_bd(hdev, q, ctl, len, ptr); in hw_queue_schedule_job()
403 static int init_signal_cs(struct hl_device *hdev, in init_signal_cs() argument
412 prop = &hdev->kernel_queues[q_idx].sync_stream_prop; in init_signal_cs()
418 dev_dbg(hdev->dev, in init_signal_cs()
426 hdev->asic_funcs->gen_signal_cb(hdev, job->patched_cb, in init_signal_cs()
429 rc = hl_cs_signal_sob_wraparound_handler(hdev, q_idx, &hw_sob, 1, in init_signal_cs()
435 void hl_hw_queue_encaps_sig_set_sob_info(struct hl_device *hdev, in hl_hw_queue_encaps_sig_set_sob_info() argument
459 static int init_wait_cs(struct hl_device *hdev, struct hl_cs *cs, in init_wait_cs() argument
468 prop = &hdev->kernel_queues[q_idx].sync_stream_prop; in init_wait_cs()
479 hl_hw_queue_encaps_sig_set_sob_info(hdev, cs, job, cs_cmpl); in init_wait_cs()
481 …dev_dbg(hdev->dev, "Wait for encaps signals handle, qidx(%u), CS sequence(%llu), sob val: 0x%x, of… in init_wait_cs()
515 dev_dbg(hdev->dev, in init_wait_cs()
528 hdev->asic_funcs->gen_wait_cb(hdev, &wait_prop); in init_wait_cs()
546 struct hl_device *hdev = ctx->hdev; in init_signal_wait_cs() local
557 rc = init_signal_cs(hdev, job, cs_cmpl); in init_signal_wait_cs()
559 rc = init_wait_cs(hdev, cs, job, cs_cmpl); in init_signal_wait_cs()
565 (struct hl_device *hdev, struct hl_cs *cs) in encaps_sig_first_staged_cs_handler() argument
574 mgr = &hdev->compute_ctx->sig_mgr; in encaps_sig_first_staged_cs_handler()
599 …dev_dbg(hdev->dev, "CS seq (%llu) added to encaps signal handler id (%u), count(%u), qidx(%u), sob… in encaps_sig_first_staged_cs_handler()
607 dev_err(hdev->dev, "encaps handle id(%u) wasn't found!\n", in encaps_sig_first_staged_cs_handler()
626 struct hl_device *hdev = ctx->hdev; in hl_hw_queue_schedule_cs() local
633 cntr = &hdev->aggregated_cs_counters; in hl_hw_queue_schedule_cs()
635 hdev->asic_funcs->hw_queues_lock(hdev); in hl_hw_queue_schedule_cs()
637 if (!hl_device_operational(hdev, &status)) { in hl_hw_queue_schedule_cs()
640 dev_err(hdev->dev, in hl_hw_queue_schedule_cs()
641 "device is %s, CS rejected!\n", hdev->status[status]); in hl_hw_queue_schedule_cs()
646 max_queues = hdev->asic_prop.max_queues; in hl_hw_queue_schedule_cs()
648 q = &hdev->kernel_queues[0]; in hl_hw_queue_schedule_cs()
653 rc = ext_queue_sanity_checks(hdev, q, in hl_hw_queue_schedule_cs()
659 rc = int_queue_sanity_checks(hdev, q, in hl_hw_queue_schedule_cs()
663 rc = hw_queue_sanity_checks(hdev, q, in hl_hw_queue_schedule_cs()
667 dev_err(hdev->dev, "Queue type %d is invalid\n", in hl_hw_queue_schedule_cs()
690 rc = hdev->asic_funcs->collective_wait_init_cs(cs); in hl_hw_queue_schedule_cs()
697 rc = encaps_sig_first_staged_cs_handler(hdev, cs); in hl_hw_queue_schedule_cs()
702 spin_lock(&hdev->cs_mirror_lock); in hl_hw_queue_schedule_cs()
708 staged_cs = hl_staged_cs_find_first(hdev, cs->staged_sequence); in hl_hw_queue_schedule_cs()
710 dev_err(hdev->dev, in hl_hw_queue_schedule_cs()
717 if (is_staged_cs_last_exists(hdev, staged_cs)) { in hl_hw_queue_schedule_cs()
718 dev_err(hdev->dev, in hl_hw_queue_schedule_cs()
728 if (hdev->supports_wait_for_multi_cs) in hl_hw_queue_schedule_cs()
733 list_add_tail(&cs->mirror_node, &hdev->cs_mirror_list); in hl_hw_queue_schedule_cs()
736 first_entry = list_first_entry(&hdev->cs_mirror_list, in hl_hw_queue_schedule_cs()
738 if ((hdev->timeout_jiffies != MAX_SCHEDULE_TIMEOUT) && in hl_hw_queue_schedule_cs()
745 spin_unlock(&hdev->cs_mirror_lock); in hl_hw_queue_schedule_cs()
767 spin_unlock(&hdev->cs_mirror_lock); in hl_hw_queue_schedule_cs()
769 q = &hdev->kernel_queues[0]; in hl_hw_queue_schedule_cs()
774 &hdev->completion_queue[i].free_slots_cnt; in hl_hw_queue_schedule_cs()
781 hdev->asic_funcs->hw_queues_unlock(hdev); in hl_hw_queue_schedule_cs()
792 void hl_hw_queue_inc_ci_kernel(struct hl_device *hdev, u32 hw_queue_id) in hl_hw_queue_inc_ci_kernel() argument
794 struct hl_hw_queue *q = &hdev->kernel_queues[hw_queue_id]; in hl_hw_queue_inc_ci_kernel()
799 static int ext_and_cpu_queue_init(struct hl_device *hdev, struct hl_hw_queue *q, in ext_and_cpu_queue_init() argument
806 p = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev, in ext_and_cpu_queue_init()
810 p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, in ext_and_cpu_queue_init()
823 dev_err(hdev->dev, in ext_and_cpu_queue_init()
838 hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, in ext_and_cpu_queue_init()
842 hdev->asic_funcs->asic_dma_free_coherent(hdev, in ext_and_cpu_queue_init()
850 static int int_queue_init(struct hl_device *hdev, struct hl_hw_queue *q) in int_queue_init() argument
854 p = hdev->asic_funcs->get_int_queue_base(hdev, q->hw_queue_id, in int_queue_init()
857 dev_err(hdev->dev, in int_queue_init()
870 static int cpu_queue_init(struct hl_device *hdev, struct hl_hw_queue *q) in cpu_queue_init() argument
872 return ext_and_cpu_queue_init(hdev, q, true); in cpu_queue_init()
875 static int ext_queue_init(struct hl_device *hdev, struct hl_hw_queue *q) in ext_queue_init() argument
877 return ext_and_cpu_queue_init(hdev, q, false); in ext_queue_init()
880 static int hw_queue_init(struct hl_device *hdev, struct hl_hw_queue *q) in hw_queue_init() argument
884 p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, in hw_queue_init()
900 static void sync_stream_queue_init(struct hl_device *hdev, u32 q_idx) in sync_stream_queue_init() argument
903 struct asic_fixed_properties *prop = &hdev->asic_prop; in sync_stream_queue_init()
907 sync_stream_prop = &hdev->kernel_queues[q_idx].sync_stream_prop; in sync_stream_queue_init()
914 if (hdev->kernel_queues[q_idx].collective_mode == in sync_stream_queue_init()
916 reserved_mon_idx = hdev->collective_mon_idx; in sync_stream_queue_init()
926 hdev->collective_mon_idx += HL_COLLECTIVE_RSVD_MSTR_MONS; in sync_stream_queue_init()
927 } else if (hdev->kernel_queues[q_idx].collective_mode == in sync_stream_queue_init()
929 reserved_mon_idx = hdev->collective_mon_idx++; in sync_stream_queue_init()
936 if (!hdev->kernel_queues[q_idx].supports_sync_stream) in sync_stream_queue_init()
939 queue_idx = hdev->sync_stream_queue_idx++; in sync_stream_queue_init()
950 hw_sob->hdev = hdev; in sync_stream_queue_init()
953 hdev->asic_funcs->get_sob_addr(hdev, hw_sob->sob_id); in sync_stream_queue_init()
959 static void sync_stream_queue_reset(struct hl_device *hdev, u32 q_idx) in sync_stream_queue_reset() argument
962 &hdev->kernel_queues[q_idx].sync_stream_prop; in sync_stream_queue_reset()
983 static int queue_init(struct hl_device *hdev, struct hl_hw_queue *q, in queue_init() argument
992 rc = ext_queue_init(hdev, q); in queue_init()
995 rc = int_queue_init(hdev, q); in queue_init()
998 rc = cpu_queue_init(hdev, q); in queue_init()
1001 rc = hw_queue_init(hdev, q); in queue_init()
1007 dev_crit(hdev->dev, "wrong queue type %d during init\n", in queue_init()
1013 sync_stream_queue_init(hdev, q->hw_queue_id); in queue_init()
1031 static void queue_fini(struct hl_device *hdev, struct hl_hw_queue *q) in queue_fini() argument
1060 hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, in queue_fini()
1064 hdev->asic_funcs->asic_dma_free_coherent(hdev, in queue_fini()
1070 int hl_hw_queues_create(struct hl_device *hdev) in hl_hw_queues_create() argument
1072 struct asic_fixed_properties *asic = &hdev->asic_prop; in hl_hw_queues_create()
1076 hdev->kernel_queues = kcalloc(asic->max_queues, in hl_hw_queues_create()
1077 sizeof(*hdev->kernel_queues), GFP_KERNEL); in hl_hw_queues_create()
1079 if (!hdev->kernel_queues) { in hl_hw_queues_create()
1080 dev_err(hdev->dev, "Not enough memory for H/W queues\n"); in hl_hw_queues_create()
1085 for (i = 0, q_ready_cnt = 0, q = hdev->kernel_queues; in hl_hw_queues_create()
1092 rc = queue_init(hdev, q, i); in hl_hw_queues_create()
1094 dev_err(hdev->dev, in hl_hw_queues_create()
1103 for (i = 0, q = hdev->kernel_queues ; i < q_ready_cnt ; i++, q++) in hl_hw_queues_create()
1104 queue_fini(hdev, q); in hl_hw_queues_create()
1106 kfree(hdev->kernel_queues); in hl_hw_queues_create()
1111 void hl_hw_queues_destroy(struct hl_device *hdev) in hl_hw_queues_destroy() argument
1114 u32 max_queues = hdev->asic_prop.max_queues; in hl_hw_queues_destroy()
1117 for (i = 0, q = hdev->kernel_queues ; i < max_queues ; i++, q++) in hl_hw_queues_destroy()
1118 queue_fini(hdev, q); in hl_hw_queues_destroy()
1120 kfree(hdev->kernel_queues); in hl_hw_queues_destroy()
1123 void hl_hw_queue_reset(struct hl_device *hdev, bool hard_reset) in hl_hw_queue_reset() argument
1126 u32 max_queues = hdev->asic_prop.max_queues; in hl_hw_queue_reset()
1129 for (i = 0, q = hdev->kernel_queues ; i < max_queues ; i++, q++) { in hl_hw_queue_reset()
1137 sync_stream_queue_reset(hdev, q->hw_queue_id); in hl_hw_queue_reset()