Lines Matching refs:its
124 #define is_v4(its) (!!((its)->typer & GITS_TYPER_VLPIS)) argument
125 #define is_v4_1(its) (!!((its)->typer & GITS_TYPER_VMAPP)) argument
126 #define device_ids(its) (FIELD_GET(GITS_TYPER_DEVBITS, (its)->typer) + 1) argument
165 struct its_node *its; member
207 static bool require_its_list_vmovp(struct its_vm *vm, struct its_node *its) in require_its_list_vmovp() argument
209 return (gic_rdists->has_rvpeid || vm->vlpi_count[its->list_nr]); in require_its_list_vmovp()
214 struct its_node *its; in get_its_list() local
217 list_for_each_entry(its, &its_nodes, entry) { in get_its_list()
218 if (!is_v4(its)) in get_its_list()
221 if (require_its_list_vmovp(vm, its)) in get_its_list()
222 __set_bit(its->list_nr, &its_list); in get_its_list()
237 struct its_node *its = its_dev->its; in dev_event_to_col() local
239 return its->collections + its_dev->event_map.col_map[event]; in dev_event_to_col()
308 static struct its_vpe *valid_vpe(struct its_node *its, struct its_vpe *vpe) in valid_vpe() argument
310 if (valid_col(its->collections + vpe->col_idx)) in valid_vpe()
594 static struct its_collection *its_build_mapd_cmd(struct its_node *its, in its_build_mapd_cmd() argument
615 static struct its_collection *its_build_mapc_cmd(struct its_node *its, in its_build_mapc_cmd() argument
629 static struct its_collection *its_build_mapti_cmd(struct its_node *its, in its_build_mapti_cmd() argument
649 static struct its_collection *its_build_movi_cmd(struct its_node *its, in its_build_movi_cmd() argument
668 static struct its_collection *its_build_discard_cmd(struct its_node *its, in its_build_discard_cmd() argument
686 static struct its_collection *its_build_inv_cmd(struct its_node *its, in its_build_inv_cmd() argument
704 static struct its_collection *its_build_int_cmd(struct its_node *its, in its_build_int_cmd() argument
722 static struct its_collection *its_build_clear_cmd(struct its_node *its, in its_build_clear_cmd() argument
740 static struct its_collection *its_build_invall_cmd(struct its_node *its, in its_build_invall_cmd() argument
752 static struct its_vpe *its_build_vinvall_cmd(struct its_node *its, in its_build_vinvall_cmd() argument
761 return valid_vpe(its, desc->its_vinvall_cmd.vpe); in its_build_vinvall_cmd()
764 static struct its_vpe *its_build_vmapp_cmd(struct its_node *its, in its_build_vmapp_cmd() argument
777 if (is_v4_1(its)) { in its_build_vmapp_cmd()
786 target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset; in its_build_vmapp_cmd()
792 if (!is_v4_1(its)) in its_build_vmapp_cmd()
814 return valid_vpe(its, desc->its_vmapp_cmd.vpe); in its_build_vmapp_cmd()
817 static struct its_vpe *its_build_vmapti_cmd(struct its_node *its, in its_build_vmapti_cmd() argument
823 if (!is_v4_1(its) && desc->its_vmapti_cmd.db_enabled) in its_build_vmapti_cmd()
837 return valid_vpe(its, desc->its_vmapti_cmd.vpe); in its_build_vmapti_cmd()
840 static struct its_vpe *its_build_vmovi_cmd(struct its_node *its, in its_build_vmovi_cmd() argument
846 if (!is_v4_1(its) && desc->its_vmovi_cmd.db_enabled) in its_build_vmovi_cmd()
860 return valid_vpe(its, desc->its_vmovi_cmd.vpe); in its_build_vmovi_cmd()
863 static struct its_vpe *its_build_vmovp_cmd(struct its_node *its, in its_build_vmovp_cmd() argument
869 target = desc->its_vmovp_cmd.col->target_address + its->vlpi_redist_offset; in its_build_vmovp_cmd()
876 if (is_v4_1(its)) { in its_build_vmovp_cmd()
883 return valid_vpe(its, desc->its_vmovp_cmd.vpe); in its_build_vmovp_cmd()
886 static struct its_vpe *its_build_vinv_cmd(struct its_node *its, in its_build_vinv_cmd() argument
901 return valid_vpe(its, map->vpe); in its_build_vinv_cmd()
904 static struct its_vpe *its_build_vint_cmd(struct its_node *its, in its_build_vint_cmd() argument
919 return valid_vpe(its, map->vpe); in its_build_vint_cmd()
922 static struct its_vpe *its_build_vclear_cmd(struct its_node *its, in its_build_vclear_cmd() argument
937 return valid_vpe(its, map->vpe); in its_build_vclear_cmd()
940 static struct its_vpe *its_build_invdb_cmd(struct its_node *its, in its_build_invdb_cmd() argument
944 if (WARN_ON(!is_v4_1(its))) in its_build_invdb_cmd()
952 return valid_vpe(its, desc->its_invdb_cmd.vpe); in its_build_invdb_cmd()
955 static struct its_vpe *its_build_vsgi_cmd(struct its_node *its, in its_build_vsgi_cmd() argument
959 if (WARN_ON(!is_v4_1(its))) in its_build_vsgi_cmd()
972 return valid_vpe(its, desc->its_vsgi_cmd.vpe); in its_build_vsgi_cmd()
975 static u64 its_cmd_ptr_to_offset(struct its_node *its, in its_cmd_ptr_to_offset() argument
978 return (ptr - its->cmd_base) * sizeof(*ptr); in its_cmd_ptr_to_offset()
981 static int its_queue_full(struct its_node *its) in its_queue_full() argument
986 widx = its->cmd_write - its->cmd_base; in its_queue_full()
987 ridx = readl_relaxed(its->base + GITS_CREADR) / sizeof(struct its_cmd_block); in its_queue_full()
996 static struct its_cmd_block *its_allocate_entry(struct its_node *its) in its_allocate_entry() argument
1001 while (its_queue_full(its)) { in its_allocate_entry()
1011 cmd = its->cmd_write++; in its_allocate_entry()
1014 if (its->cmd_write == (its->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES)) in its_allocate_entry()
1015 its->cmd_write = its->cmd_base; in its_allocate_entry()
1026 static struct its_cmd_block *its_post_commands(struct its_node *its) in its_post_commands() argument
1028 u64 wr = its_cmd_ptr_to_offset(its, its->cmd_write); in its_post_commands()
1030 writel_relaxed(wr, its->base + GITS_CWRITER); in its_post_commands()
1032 return its->cmd_write; in its_post_commands()
1035 static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd) in its_flush_cmd() argument
1041 if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING) in its_flush_cmd()
1047 static int its_wait_for_range_completion(struct its_node *its, in its_wait_for_range_completion() argument
1055 to_idx = its_cmd_ptr_to_offset(its, to); in its_wait_for_range_completion()
1064 rd_idx = readl_relaxed(its->base + GITS_CREADR); in its_wait_for_range_completion()
1094 void name(struct its_node *its, \
1103 raw_spin_lock_irqsave(&its->lock, flags); \
1105 cmd = its_allocate_entry(its); \
1107 raw_spin_unlock_irqrestore(&its->lock, flags); \
1110 sync_obj = builder(its, cmd, desc); \
1111 its_flush_cmd(its, cmd); \
1114 sync_cmd = its_allocate_entry(its); \
1118 buildfn(its, sync_cmd, sync_obj); \
1119 its_flush_cmd(its, sync_cmd); \
1123 rd_idx = readl_relaxed(its->base + GITS_CREADR); \
1124 next_cmd = its_post_commands(its); \
1125 raw_spin_unlock_irqrestore(&its->lock, flags); \
1127 if (its_wait_for_range_completion(its, rd_idx, next_cmd)) \
1131 static void its_build_sync_cmd(struct its_node *its, in its_build_sync_cmd() argument
1144 static void its_build_vsync_cmd(struct its_node *its, in BUILD_SINGLE_CMD_FUNC()
1164 its_send_single_command(dev->its, its_build_int_cmd, &desc); in BUILD_SINGLE_CMD_FUNC()
1174 its_send_single_command(dev->its, its_build_clear_cmd, &desc); in its_send_clear()
1184 its_send_single_command(dev->its, its_build_inv_cmd, &desc); in its_send_inv()
1194 its_send_single_command(dev->its, its_build_mapd_cmd, &desc); in its_send_mapd()
1197 static void its_send_mapc(struct its_node *its, struct its_collection *col, in its_send_mapc() argument
1205 its_send_single_command(its, its_build_mapc_cmd, &desc); in its_send_mapc()
1216 its_send_single_command(dev->its, its_build_mapti_cmd, &desc); in its_send_mapti()
1228 its_send_single_command(dev->its, its_build_movi_cmd, &desc); in its_send_movi()
1238 its_send_single_command(dev->its, its_build_discard_cmd, &desc); in its_send_discard()
1241 static void its_send_invall(struct its_node *its, struct its_collection *col) in its_send_invall() argument
1247 its_send_single_command(its, its_build_invall_cmd, &desc); in its_send_invall()
1261 its_send_single_vcommand(dev->its, its_build_vmapti_cmd, &desc); in its_send_vmapti()
1274 its_send_single_vcommand(dev->its, its_build_vmovi_cmd, &desc); in its_send_vmovi()
1277 static void its_send_vmapp(struct its_node *its, in its_send_vmapp() argument
1284 desc.its_vmapp_cmd.col = &its->collections[vpe->col_idx]; in its_send_vmapp()
1286 its_send_single_vcommand(its, its_build_vmapp_cmd, &desc); in its_send_vmapp()
1292 struct its_node *its; in its_send_vmovp() local
1299 its = list_first_entry(&its_nodes, struct its_node, entry); in its_send_vmovp()
1300 desc.its_vmovp_cmd.col = &its->collections[col_id]; in its_send_vmovp()
1301 its_send_single_vcommand(its, its_build_vmovp_cmd, &desc); in its_send_vmovp()
1319 list_for_each_entry(its, &its_nodes, entry) { in its_send_vmovp()
1320 if (!is_v4(its)) in its_send_vmovp()
1323 if (!require_its_list_vmovp(vpe->its_vm, its)) in its_send_vmovp()
1326 desc.its_vmovp_cmd.col = &its->collections[col_id]; in its_send_vmovp()
1327 its_send_single_vcommand(its, its_build_vmovp_cmd, &desc); in its_send_vmovp()
1333 static void its_send_vinvall(struct its_node *its, struct its_vpe *vpe) in its_send_vinvall() argument
1338 its_send_single_vcommand(its, its_build_vinvall_cmd, &desc); in its_send_vinvall()
1352 its_send_single_vcommand(dev->its, its_build_vinv_cmd, &desc); in its_send_vinv()
1366 its_send_single_vcommand(dev->its, its_build_vint_cmd, &desc); in its_send_vint()
1380 its_send_single_vcommand(dev->its, its_build_vclear_cmd, &desc); in its_send_vclear()
1383 static void its_send_invdb(struct its_node *its, struct its_vpe *vpe) in its_send_invdb() argument
1388 its_send_single_vcommand(its, its_build_invdb_cmd, &desc); in its_send_invdb()
1445 WARN_ON(!is_v4_1(its_dev->its)); in direct_lpi_inv()
1471 (is_v4_1(its_dev->its) || !irqd_is_forwarded_to_vcpu(d))) in lpi_update_config()
1489 if (is_v4_1(its_dev->its)) in its_vlpi_set_doorbell()
1582 node = its_dev->its->numa_node; in its_select_cpu()
1614 if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144)) in its_select_cpu()
1632 if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) && in its_select_cpu()
1670 target_col = &its_dev->its->collections[cpu]; in its_set_affinity()
1687 struct its_node *its = its_dev->its; in its_irq_get_msi_base() local
1689 return its->phys_base + GITS_TRANSLATER; in its_irq_get_msi_base()
1695 struct its_node *its; in its_irq_compose_msi_msg() local
1698 its = its_dev->its; in its_irq_compose_msi_msg()
1699 addr = its->get_msi_base(its_dev); in its_irq_compose_msi_msg()
1758 static void its_map_vm(struct its_node *its, struct its_vm *vm) in its_map_vm() argument
1771 vm->vlpi_count[its->list_nr]++; in its_map_vm()
1773 if (vm->vlpi_count[its->list_nr] == 1) { in its_map_vm()
1782 its_send_vmapp(its, vpe, true); in its_map_vm()
1783 its_send_vinvall(its, vpe); in its_map_vm()
1791 static void its_unmap_vm(struct its_node *its, struct its_vm *vm) in its_unmap_vm() argument
1801 if (!--vm->vlpi_count[its->list_nr]) { in its_unmap_vm()
1805 its_send_vmapp(its, vm->vpes[i], false); in its_unmap_vm()
1847 its_map_vm(its_dev->its, info->map->vm); in its_vlpi_map()
1920 its_unmap_vm(its_dev->its, its_dev->event_map.vm); in its_vlpi_unmap()
1958 if (!is_v4(its_dev->its)) in its_irq_set_vcpu_affinity()
2278 static u64 its_read_baser(struct its_node *its, struct its_baser *baser) in its_read_baser() argument
2280 u32 idx = baser - its->tables; in its_read_baser()
2282 return gits_read_baser(its->base + GITS_BASER + (idx << 3)); in its_read_baser()
2285 static void its_write_baser(struct its_node *its, struct its_baser *baser, in its_write_baser() argument
2288 u32 idx = baser - its->tables; in its_write_baser()
2290 gits_write_baser(val, its->base + GITS_BASER + (idx << 3)); in its_write_baser()
2291 baser->val = its_read_baser(its, baser); in its_write_baser()
2294 static int its_setup_baser(struct its_node *its, struct its_baser *baser, in its_setup_baser() argument
2297 u64 val = its_read_baser(its, baser); in its_setup_baser()
2309 &its->phys_base, its_base_type_string[type], in its_setup_baser()
2315 page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, order); in its_setup_baser()
2359 its_write_baser(its, baser, val); in its_setup_baser()
2380 &its->phys_base, its_base_type_string[type], in its_setup_baser()
2392 &its->phys_base, (int)(PAGE_ORDER_TO_SIZE(order) / (int)tmp), in its_setup_baser()
2401 static bool its_parse_indirect_baser(struct its_node *its, in its_parse_indirect_baser() argument
2405 u64 tmp = its_read_baser(its, baser); in its_parse_indirect_baser()
2419 its_write_baser(its, baser, val | GITS_BASER_INDIRECT); in its_parse_indirect_baser()
2447 &its->phys_base, its_base_type_string[type], in its_parse_indirect_baser()
2448 device_ids(its), ids); in its_parse_indirect_baser()
2466 static u32 compute_its_aff(struct its_node *its) in compute_its_aff() argument
2476 svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer); in compute_its_aff()
2478 val |= FIELD_PREP(GICR_TYPER_AFFINITY, its->mpidr); in compute_its_aff()
2484 struct its_node *its; in find_sibling_its() local
2492 list_for_each_entry(its, &its_nodes, entry) { in find_sibling_its()
2495 if (!is_v4_1(its) || its == cur_its) in find_sibling_its()
2498 if (!FIELD_GET(GITS_TYPER_SVPET, its->typer)) in find_sibling_its()
2501 if (aff != compute_its_aff(its)) in find_sibling_its()
2505 baser = its->tables[2].val; in find_sibling_its()
2509 return its; in find_sibling_its()
2515 static void its_free_tables(struct its_node *its) in its_free_tables() argument
2520 if (its->tables[i].base) { in its_free_tables()
2521 free_pages((unsigned long)its->tables[i].base, in its_free_tables()
2522 its->tables[i].order); in its_free_tables()
2523 its->tables[i].base = NULL; in its_free_tables()
2528 static int its_probe_baser_psz(struct its_node *its, struct its_baser *baser) in its_probe_baser_psz() argument
2535 val = its_read_baser(its, baser); in its_probe_baser_psz()
2554 its_write_baser(its, baser, val); in its_probe_baser_psz()
2576 static int its_alloc_tables(struct its_node *its) in its_alloc_tables() argument
2582 if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375) in its_alloc_tables()
2587 struct its_baser *baser = its->tables + i; in its_alloc_tables()
2588 u64 val = its_read_baser(its, baser); in its_alloc_tables()
2596 if (its_probe_baser_psz(its, baser)) { in its_alloc_tables()
2597 its_free_tables(its); in its_alloc_tables()
2605 indirect = its_parse_indirect_baser(its, baser, &order, in its_alloc_tables()
2606 device_ids(its)); in its_alloc_tables()
2610 if (is_v4_1(its)) { in its_alloc_tables()
2614 if ((sibling = find_sibling_its(its))) { in its_alloc_tables()
2616 its_write_baser(its, baser, baser->val); in its_alloc_tables()
2621 indirect = its_parse_indirect_baser(its, baser, &order, in its_alloc_tables()
2626 err = its_setup_baser(its, baser, cache, shr, order, indirect); in its_alloc_tables()
2628 its_free_tables(its); in its_alloc_tables()
2642 struct its_node *its; in inherit_vpe_l1_table_from_its() local
2649 list_for_each_entry(its, &its_nodes, entry) { in inherit_vpe_l1_table_from_its()
2652 if (!is_v4_1(its)) in inherit_vpe_l1_table_from_its()
2655 if (!FIELD_GET(GITS_TYPER_SVPET, its->typer)) in inherit_vpe_l1_table_from_its()
2658 if (aff != compute_its_aff(its)) in inherit_vpe_l1_table_from_its()
2662 baser = its->tables[2].val; in inherit_vpe_l1_table_from_its()
2667 gic_data_rdist()->vpe_l1_base = its->tables[2].base; in inherit_vpe_l1_table_from_its()
2925 static int its_alloc_collections(struct its_node *its) in its_alloc_collections() argument
2929 its->collections = kcalloc(nr_cpu_ids, sizeof(*its->collections), in its_alloc_collections()
2931 if (!its->collections) in its_alloc_collections()
2935 its->collections[i].target_address = ~0ULL; in its_alloc_collections()
3183 static void its_cpu_init_collection(struct its_node *its) in its_cpu_init_collection() argument
3189 if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) { in its_cpu_init_collection()
3193 if (its->numa_node != NUMA_NO_NODE && in its_cpu_init_collection()
3194 its->numa_node != of_node_to_nid(cpu_node)) in its_cpu_init_collection()
3202 if (gic_read_typer(its->base + GITS_TYPER) & GITS_TYPER_PTA) { in its_cpu_init_collection()
3215 its->collections[cpu].target_address = target; in its_cpu_init_collection()
3216 its->collections[cpu].col_id = cpu; in its_cpu_init_collection()
3218 its_send_mapc(its, &its->collections[cpu], 1); in its_cpu_init_collection()
3219 its_send_invall(its, &its->collections[cpu]); in its_cpu_init_collection()
3224 struct its_node *its; in its_cpu_init_collections() local
3228 list_for_each_entry(its, &its_nodes, entry) in its_cpu_init_collections()
3229 its_cpu_init_collection(its); in its_cpu_init_collections()
3234 static struct its_device *its_find_device(struct its_node *its, u32 dev_id) in its_find_device() argument
3239 raw_spin_lock_irqsave(&its->lock, flags); in its_find_device()
3241 list_for_each_entry(tmp, &its->its_device_list, entry) { in its_find_device()
3248 raw_spin_unlock_irqrestore(&its->lock, flags); in its_find_device()
3253 static struct its_baser *its_get_baser(struct its_node *its, u32 type) in its_get_baser() argument
3258 if (GITS_BASER_TYPE(its->tables[i].val) == type) in its_get_baser()
3259 return &its->tables[i]; in its_get_baser()
3265 static bool its_alloc_table_entry(struct its_node *its, in its_alloc_table_entry() argument
3286 page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, in its_alloc_table_entry()
3308 static bool its_alloc_device_table(struct its_node *its, u32 dev_id) in its_alloc_device_table() argument
3312 baser = its_get_baser(its, GITS_BASER_TYPE_DEVICE); in its_alloc_device_table()
3316 return (ilog2(dev_id) < device_ids(its)); in its_alloc_device_table()
3318 return its_alloc_table_entry(its, baser, dev_id); in its_alloc_device_table()
3323 struct its_node *its; in its_alloc_vpe_table() local
3333 list_for_each_entry(its, &its_nodes, entry) { in its_alloc_vpe_table()
3336 if (!is_v4(its)) in its_alloc_vpe_table()
3339 baser = its_get_baser(its, GITS_BASER_TYPE_VCPU); in its_alloc_vpe_table()
3343 if (!its_alloc_table_entry(its, baser, vpe_id)) in its_alloc_vpe_table()
3363 static struct its_device *its_create_device(struct its_node *its, u32 dev_id, in its_create_device() argument
3376 if (!its_alloc_device_table(its, dev_id)) in its_create_device()
3388 sz = nr_ites * (FIELD_GET(GITS_TYPER_ITT_ENTRY_SIZE, its->typer) + 1); in its_create_device()
3390 itt = kzalloc_node(sz, GFP_KERNEL, its->numa_node); in its_create_device()
3412 dev->its = its; in its_create_device()
3423 raw_spin_lock_irqsave(&its->lock, flags); in its_create_device()
3424 list_add(&dev->entry, &its->its_device_list); in its_create_device()
3425 raw_spin_unlock_irqrestore(&its->lock, flags); in its_create_device()
3437 raw_spin_lock_irqsave(&its_dev->its->lock, flags); in its_free_device()
3439 raw_spin_unlock_irqrestore(&its_dev->its->lock, flags); in its_free_device()
3464 struct its_node *its; in its_msi_prepare() local
3479 its = msi_info->data; in its_msi_prepare()
3483 vpe_proxy.dev->its == its && in its_msi_prepare()
3491 mutex_lock(&its->dev_alloc_lock); in its_msi_prepare()
3492 its_dev = its_find_device(its, dev_id); in its_msi_prepare()
3504 its_dev = its_create_device(its, dev_id, nvec, true); in its_msi_prepare()
3515 mutex_unlock(&its->dev_alloc_lock); in its_msi_prepare()
3553 struct its_node *its = its_dev->its; in its_irq_domain_alloc() local
3563 err = iommu_dma_prepare_msi(info->desc, its->get_msi_base(its_dev)); in its_irq_domain_alloc()
3621 struct its_node *its = its_dev->its; in its_irq_domain_free() local
3635 mutex_lock(&its->dev_alloc_lock); in its_irq_domain_free()
3653 mutex_unlock(&its->dev_alloc_lock); in its_irq_domain_free()
3771 target_col = &vpe_proxy.dev->its->collections[to]; in its_vpe_db_proxy_move()
3882 struct its_node *its; in its_vpe_invall() local
3884 list_for_each_entry(its, &its_nodes, entry) { in its_vpe_invall()
3885 if (!is_v4(its)) in its_vpe_invall()
3888 if (its_list_map && !vpe->its_vm->vlpi_count[its->list_nr]) in its_vpe_invall()
3895 its_send_vinvall(its, vpe); in its_vpe_invall()
4024 static struct its_node *its = NULL; in find_4_1_its() local
4026 if (!its) { in find_4_1_its()
4027 list_for_each_entry(its, &its_nodes, entry) { in find_4_1_its()
4028 if (is_v4_1(its)) in find_4_1_its()
4029 return its; in find_4_1_its()
4033 its = NULL; in find_4_1_its()
4036 return its; in find_4_1_its()
4042 struct its_node *its; in its_vpe_4_1_send_inv() local
4049 its = find_4_1_its(); in its_vpe_4_1_send_inv()
4050 if (its) in its_vpe_4_1_send_inv()
4051 its_send_invdb(its, vpe); in its_vpe_4_1_send_inv()
4233 struct its_node *its = find_4_1_its(); in its_sgi_set_irqchip_state() local
4238 writeq_relaxed(val, its->sgir_base + GITS_SGIR - SZ_128K); in its_sgi_set_irqchip_state()
4532 struct its_node *its; in its_vpe_irq_domain_activate() local
4545 list_for_each_entry(its, &its_nodes, entry) { in its_vpe_irq_domain_activate()
4546 if (!is_v4(its)) in its_vpe_irq_domain_activate()
4549 its_send_vmapp(its, vpe, true); in its_vpe_irq_domain_activate()
4550 its_send_vinvall(its, vpe); in its_vpe_irq_domain_activate()
4562 struct its_node *its; in its_vpe_irq_domain_deactivate() local
4571 list_for_each_entry(its, &its_nodes, entry) { in its_vpe_irq_domain_deactivate()
4572 if (!is_v4(its)) in its_vpe_irq_domain_deactivate()
4575 its_send_vmapp(its, vpe, false); in its_vpe_irq_domain_deactivate()
4630 struct its_node *its = data; in its_enable_quirk_cavium_22375() local
4633 its->typer &= ~GITS_TYPER_DEVBITS; in its_enable_quirk_cavium_22375()
4634 its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, 20 - 1); in its_enable_quirk_cavium_22375()
4635 its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_22375; in its_enable_quirk_cavium_22375()
4642 struct its_node *its = data; in its_enable_quirk_cavium_23144() local
4644 its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_23144; in its_enable_quirk_cavium_23144()
4651 struct its_node *its = data; in its_enable_quirk_qdf2400_e0065() local
4654 its->typer &= ~GITS_TYPER_ITT_ENTRY_SIZE; in its_enable_quirk_qdf2400_e0065()
4655 its->typer |= FIELD_PREP(GITS_TYPER_ITT_ENTRY_SIZE, 16 - 1); in its_enable_quirk_qdf2400_e0065()
4662 struct its_node *its = its_dev->its; in its_irq_get_msi_base_pre_its() local
4671 return its->pre_its_base + (its_dev->device_id << 2); in its_irq_get_msi_base_pre_its()
4676 struct its_node *its = data; in its_enable_quirk_socionext_synquacer() local
4680 if (!fwnode_property_read_u32_array(its->fwnode_handle, in its_enable_quirk_socionext_synquacer()
4685 its->pre_its_base = pre_its_window[0]; in its_enable_quirk_socionext_synquacer()
4686 its->get_msi_base = its_irq_get_msi_base_pre_its; in its_enable_quirk_socionext_synquacer()
4689 if (device_ids(its) > ids) { in its_enable_quirk_socionext_synquacer()
4690 its->typer &= ~GITS_TYPER_DEVBITS; in its_enable_quirk_socionext_synquacer()
4691 its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, ids - 1); in its_enable_quirk_socionext_synquacer()
4695 its->msi_domain_flags &= ~IRQ_DOMAIN_FLAG_ISOLATED_MSI; in its_enable_quirk_socionext_synquacer()
4703 struct its_node *its = data; in its_enable_quirk_hip07_161600802() local
4709 its->vlpi_redist_offset = SZ_128K; in its_enable_quirk_hip07_161600802()
4763 static void its_enable_quirks(struct its_node *its) in its_enable_quirks() argument
4765 u32 iidr = readl_relaxed(its->base + GITS_IIDR); in its_enable_quirks()
4767 gic_enable_quirks(iidr, its_quirks, its); in its_enable_quirks()
4772 struct its_node *its; in its_save_disable() local
4776 list_for_each_entry(its, &its_nodes, entry) { in its_save_disable()
4779 base = its->base; in its_save_disable()
4780 its->ctlr_save = readl_relaxed(base + GITS_CTLR); in its_save_disable()
4784 &its->phys_base, err); in its_save_disable()
4785 writel_relaxed(its->ctlr_save, base + GITS_CTLR); in its_save_disable()
4789 its->cbaser_save = gits_read_cbaser(base + GITS_CBASER); in its_save_disable()
4794 list_for_each_entry_continue_reverse(its, &its_nodes, entry) { in its_save_disable()
4797 base = its->base; in its_save_disable()
4798 writel_relaxed(its->ctlr_save, base + GITS_CTLR); in its_save_disable()
4808 struct its_node *its; in its_restore_enable() local
4812 list_for_each_entry(its, &its_nodes, entry) { in its_restore_enable()
4816 base = its->base; in its_restore_enable()
4830 &its->phys_base, ret); in its_restore_enable()
4834 gits_write_cbaser(its->cbaser_save, base + GITS_CBASER); in its_restore_enable()
4840 its->cmd_write = its->cmd_base; in its_restore_enable()
4845 struct its_baser *baser = &its->tables[i]; in its_restore_enable()
4850 its_write_baser(its, baser, baser->val); in its_restore_enable()
4852 writel_relaxed(its->ctlr_save, base + GITS_CTLR); in its_restore_enable()
4859 if (its->collections[smp_processor_id()].col_id < in its_restore_enable()
4861 its_cpu_init_collection(its); in its_restore_enable()
4903 static int its_init_domain(struct fwnode_handle *handle, struct its_node *its) in its_init_domain() argument
4913 info->data = its; in its_init_domain()
4916 its->msi_domain_flags, 0, in its_init_domain()
4931 struct its_node *its; in its_init_vpe_domain() local
4941 its = list_first_entry(&its_nodes, struct its_node, entry); in its_init_vpe_domain()
4950 devid = GENMASK(device_ids(its) - 1, 0); in its_init_vpe_domain()
4951 vpe_proxy.dev = its_create_device(its, devid, entries, false); in its_init_vpe_domain()
5009 struct its_node *its; in its_probe_one() local
5022 its = kzalloc(sizeof(*its), GFP_KERNEL); in its_probe_one()
5023 if (!its) { in its_probe_one()
5028 raw_spin_lock_init(&its->lock); in its_probe_one()
5029 mutex_init(&its->dev_alloc_lock); in its_probe_one()
5030 INIT_LIST_HEAD(&its->entry); in its_probe_one()
5031 INIT_LIST_HEAD(&its->its_device_list); in its_probe_one()
5033 its->typer = typer; in its_probe_one()
5034 its->base = its_base; in its_probe_one()
5035 its->phys_base = res->start; in its_probe_one()
5036 if (is_v4(its)) { in its_probe_one()
5042 its->list_nr = err; in its_probe_one()
5050 if (is_v4_1(its)) { in its_probe_one()
5053 its->sgir_base = ioremap(res->start + SZ_128K, SZ_64K); in its_probe_one()
5054 if (!its->sgir_base) { in its_probe_one()
5059 its->mpidr = readl_relaxed(its_base + GITS_MPIDR); in its_probe_one()
5062 &res->start, its->mpidr, svpet); in its_probe_one()
5066 its->numa_node = numa_node; in its_probe_one()
5068 page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, in its_probe_one()
5074 its->cmd_base = (void *)page_address(page); in its_probe_one()
5075 its->cmd_write = its->cmd_base; in its_probe_one()
5076 its->fwnode_handle = handle; in its_probe_one()
5077 its->get_msi_base = its_irq_get_msi_base; in its_probe_one()
5078 its->msi_domain_flags = IRQ_DOMAIN_FLAG_ISOLATED_MSI; in its_probe_one()
5080 its_enable_quirks(its); in its_probe_one()
5082 err = its_alloc_tables(its); in its_probe_one()
5086 err = its_alloc_collections(its); in its_probe_one()
5090 baser = (virt_to_phys(its->cmd_base) | in its_probe_one()
5096 gits_write_cbaser(baser, its->base + GITS_CBASER); in its_probe_one()
5097 tmp = gits_read_cbaser(its->base + GITS_CBASER); in its_probe_one()
5109 gits_write_cbaser(baser, its->base + GITS_CBASER); in its_probe_one()
5112 its->flags |= ITS_FLAGS_CMDQ_NEEDS_FLUSHING; in its_probe_one()
5115 gits_write_cwriter(0, its->base + GITS_CWRITER); in its_probe_one()
5116 ctlr = readl_relaxed(its->base + GITS_CTLR); in its_probe_one()
5118 if (is_v4(its)) in its_probe_one()
5120 writel_relaxed(ctlr, its->base + GITS_CTLR); in its_probe_one()
5122 err = its_init_domain(handle, its); in its_probe_one()
5127 list_add(&its->entry, &its_nodes); in its_probe_one()
5133 its_free_tables(its); in its_probe_one()
5135 free_pages((unsigned long)its->cmd_base, get_order(ITS_CMD_QUEUE_SZ)); in its_probe_one()
5137 if (its->sgir_base) in its_probe_one()
5138 iounmap(its->sgir_base); in its_probe_one()
5140 kfree(its); in its_probe_one()
5553 struct its_node *its; in its_init() local
5576 list_for_each_entry(its, &its_nodes, entry) { in its_init()
5577 has_v4 |= is_v4(its); in its_init()
5578 has_v4_1 |= is_v4_1(its); in its_init()