Lines Matching +full:non +full:- +full:interleave
1 // SPDX-License-Identifier: GPL-2.0-or-later
22 case DF2: return FIELD_GET(DF2_DST_FABRIC_ID, ctx->map.limit); in get_dst_fabric_id()
23 case DF3: return FIELD_GET(DF3_DST_FABRIC_ID, ctx->map.limit); in get_dst_fabric_id()
24 case DF3p5: return FIELD_GET(DF3p5_DST_FABRIC_ID, ctx->map.limit); in get_dst_fabric_id()
25 case DF4: return FIELD_GET(DF4_DST_FABRIC_ID, ctx->map.ctl); in get_dst_fabric_id()
26 case DF4p5: return FIELD_GET(DF4p5_DST_FABRIC_ID, ctx->map.ctl); in get_dst_fabric_id()
38 * # of interleave bits (n): 3
39 * starting interleave bit (p): 8
41 * expanded address bits: [20+n : n+p][n+p-1 : p][p-1 : 0]
46 return expand_bits(ctx->map.intlv_bit_pos, in make_space_for_coh_st_id_at_intlv_bit()
47 ctx->map.total_intlv_bits, in make_space_for_coh_st_id_at_intlv_bit()
48 ctx->ret_addr); in make_space_for_coh_st_id_at_intlv_bit()
54 * Second gap is the remaining N-1 bits at bit 12.
58 * # of interleave bits (n): 3
59 * starting interleave bit (p): 8
62 * expanded address bits: [20+1 : p+1][p][p-1 : 0]
66 * r = n - 1; remaining interleave bits
67 * expanded address bits: [21+r : 12+r][12+r-1: 12][11 : 0]
72 /* Make a single space at the interleave bit. */ in make_space_for_coh_st_id_split_2_1()
73 u64 denorm_addr = expand_bits(ctx->map.intlv_bit_pos, 1, ctx->ret_addr); in make_space_for_coh_st_id_split_2_1()
75 /* Done if there's only a single interleave bit. */ in make_space_for_coh_st_id_split_2_1()
76 if (ctx->map.total_intlv_bits <= 1) in make_space_for_coh_st_id_split_2_1()
79 /* Make spaces for the remaining interleave bits starting at bit 12. */ in make_space_for_coh_st_id_split_2_1()
80 return expand_bits(12, ctx->map.total_intlv_bits - 1, denorm_addr); in make_space_for_coh_st_id_split_2_1()
86 * 8 channels -> bits [10:8]
87 * 16 channels -> bits [11:8]
88 * 32 channels -> bits [14,11:8]
90 * 1 die -> N/A
91 * 2 dies -> bit [12]
92 * 4 dies -> bits [13:12]
96 u8 num_intlv_bits = ilog2(ctx->map.num_intlv_chan); in make_space_for_coh_st_id_mi300()
99 if (ctx->map.intlv_bit_pos != 8) { in make_space_for_coh_st_id_mi300()
100 pr_debug("Invalid interleave bit: %u", ctx->map.intlv_bit_pos); in make_space_for_coh_st_id_mi300()
105 denorm_addr = expand_bits(8, min(num_intlv_bits, 4), ctx->ret_addr); in make_space_for_coh_st_id_mi300()
108 denorm_addr = expand_bits(12, ilog2(ctx->map.num_intlv_dies), denorm_addr); in make_space_for_coh_st_id_mi300()
119 * to make a gap where the interleave bits will be inserted.
123 switch (ctx->map.intlv_mode) { in make_space_for_coh_st_id()
158 u8 num_socket_intlv_bits = ilog2(ctx->map.num_intlv_sockets); in get_coh_st_id_df2()
159 u8 num_die_intlv_bits = ilog2(ctx->map.num_intlv_dies); in get_coh_st_id_df2()
163 coh_st_id = ctx->coh_st_fabric_id - get_dst_fabric_id(ctx); in get_coh_st_id_df2()
165 /* Channel interleave bits */ in get_coh_st_id_df2()
166 num_intlv_bits = order_base_2(ctx->map.num_intlv_chan); in get_coh_st_id_df2()
167 mask = GENMASK(num_intlv_bits - 1, 0); in get_coh_st_id_df2()
170 /* Die interleave bits */ in get_coh_st_id_df2()
174 mask = GENMASK(num_die_intlv_bits - 1, 0); in get_coh_st_id_df2()
175 die_bits = ctx->coh_st_fabric_id & df_cfg.die_id_mask; in get_coh_st_id_df2()
182 /* Socket interleave bits */ in get_coh_st_id_df2()
186 mask = GENMASK(num_socket_intlv_bits - 1, 0); in get_coh_st_id_df2()
187 socket_bits = ctx->coh_st_fabric_id & df_cfg.socket_id_mask; in get_coh_st_id_df2()
199 * Start with the original component mask and the number of interleave in get_coh_st_id_df4()
202 u8 num_intlv_bits = ilog2(ctx->map.num_intlv_chan); in get_coh_st_id_df4()
208 u16 coh_st_id = ctx->coh_st_fabric_id & mask; in get_coh_st_id_df4()
214 coh_st_id -= get_dst_fabric_id(ctx) & mask; in get_coh_st_id_df4()
220 mask = GENMASK(num_intlv_bits - 1, 0); in get_coh_st_id_df4()
224 if (ctx->map.num_intlv_sockets <= 1) in get_coh_st_id_df4()
232 num_intlv_bits = ilog2(ctx->map.num_intlv_sockets); in get_coh_st_id_df4()
236 mask = GENMASK(num_intlv_bits - 1, 0); in get_coh_st_id_df4()
238 /* Get the socket interleave bits from the original Coherent Station Fabric ID. */ in get_coh_st_id_df4()
239 socket_bits = (ctx->coh_st_fabric_id & df_cfg.socket_id_mask) >> df_cfg.socket_id_shift; in get_coh_st_id_df4()
262 ctx->coh_st_fabric_id -= get_dst_fabric_id(ctx); in get_coh_st_id_mi300()
264 die_id = (ctx->coh_st_fabric_id & df_cfg.die_id_mask) >> df_cfg.die_id_shift; in get_coh_st_id_mi300()
266 channel_bits = FIELD_GET(GENMASK(3, 0), ctx->coh_st_fabric_id); in get_coh_st_id_mi300()
267 stack_bit = FIELD_GET(BIT(4), ctx->coh_st_fabric_id) << 6; in get_coh_st_id_mi300()
274 * Derive the correct Coherent Station ID that represents the interleave bits
276 * interleave mode, number of interleaved channels/dies/sockets, and
277 * other system/mode-specific bit swizzling.
284 switch (ctx->map.intlv_mode) { in calculate_coh_st_id()
315 return ctx->coh_st_fabric_id - get_dst_fabric_id(ctx); in calculate_coh_st_id()
325 return denorm_addr | (coh_st_id << ctx->map.intlv_bit_pos); in insert_coh_st_id_at_intlv_bit()
330 /* Insert coh_st_id[0] at the interleave bit. */ in insert_coh_st_id_split_2_1()
331 denorm_addr |= (coh_st_id & BIT(0)) << ctx->map.intlv_bit_pos; in insert_coh_st_id_split_2_1()
355 switch (ctx->map.intlv_mode) { in insert_coh_st_id()
392 * MI300 systems have a fixed, hardware-defined physical-to-logical
408 if (ctx->inst_id >= ARRAY_SIZE(phy_to_log_coh_st_map_mi300)) { in get_logical_coh_st_fabric_id_mi300()
413 return phy_to_log_coh_st_map_mi300[ctx->inst_id] | (ctx->node_id << df_cfg.node_id_shift); in get_logical_coh_st_fabric_id_mi300()
421 u16 phys_fabric_id = ctx->coh_st_fabric_id; in get_logical_coh_st_fabric_id()
427 if (!FIELD_GET(DF4_REMAP_EN, ctx->map.ctl) && in get_logical_coh_st_fabric_id()
428 ctx->map.intlv_mode != DF3_6CHAN) in get_logical_coh_st_fabric_id()
439 if (ctx->map.remap_array[log_fabric_id] == component_id) in get_logical_coh_st_fabric_id()
458 hash_ctl_64k = FIELD_GET(DF4_HASH_CTL_64K, ctx->map.ctl); in get_logical_coh_st_fabric_id_for_current_spa()
459 hash_ctl_2M = FIELD_GET(DF4_HASH_CTL_2M, ctx->map.ctl); in get_logical_coh_st_fabric_id_for_current_spa()
460 hash_ctl_1G = FIELD_GET(DF4_HASH_CTL_1G, ctx->map.ctl); in get_logical_coh_st_fabric_id_for_current_spa()
461 hash_ctl_1T = FIELD_GET(DF4p5_HASH_CTL_1T, ctx->map.ctl); in get_logical_coh_st_fabric_id_for_current_spa()
463 hash_pa8 = FIELD_GET(BIT_ULL(8), denorm_ctx->current_spa); in get_logical_coh_st_fabric_id_for_current_spa()
464 hash_pa8 ^= FIELD_GET(BIT_ULL(14), denorm_ctx->current_spa); in get_logical_coh_st_fabric_id_for_current_spa()
465 hash_pa8 ^= FIELD_GET(BIT_ULL(16), denorm_ctx->current_spa) & hash_ctl_64k; in get_logical_coh_st_fabric_id_for_current_spa()
466 hash_pa8 ^= FIELD_GET(BIT_ULL(21), denorm_ctx->current_spa) & hash_ctl_2M; in get_logical_coh_st_fabric_id_for_current_spa()
467 hash_pa8 ^= FIELD_GET(BIT_ULL(30), denorm_ctx->current_spa) & hash_ctl_1G; in get_logical_coh_st_fabric_id_for_current_spa()
468 hash_pa8 ^= FIELD_GET(BIT_ULL(40), denorm_ctx->current_spa) & hash_ctl_1T; in get_logical_coh_st_fabric_id_for_current_spa()
470 hash_pa9 = FIELD_GET(BIT_ULL(9), denorm_ctx->current_spa); in get_logical_coh_st_fabric_id_for_current_spa()
471 hash_pa9 ^= FIELD_GET(BIT_ULL(17), denorm_ctx->current_spa) & hash_ctl_64k; in get_logical_coh_st_fabric_id_for_current_spa()
472 hash_pa9 ^= FIELD_GET(BIT_ULL(22), denorm_ctx->current_spa) & hash_ctl_2M; in get_logical_coh_st_fabric_id_for_current_spa()
473 hash_pa9 ^= FIELD_GET(BIT_ULL(31), denorm_ctx->current_spa) & hash_ctl_1G; in get_logical_coh_st_fabric_id_for_current_spa()
474 hash_pa9 ^= FIELD_GET(BIT_ULL(41), denorm_ctx->current_spa) & hash_ctl_1T; in get_logical_coh_st_fabric_id_for_current_spa()
476 hash_pa12 = FIELD_GET(BIT_ULL(12), denorm_ctx->current_spa); in get_logical_coh_st_fabric_id_for_current_spa()
477 hash_pa12 ^= FIELD_GET(BIT_ULL(18), denorm_ctx->current_spa) & hash_ctl_64k; in get_logical_coh_st_fabric_id_for_current_spa()
478 hash_pa12 ^= FIELD_GET(BIT_ULL(23), denorm_ctx->current_spa) & hash_ctl_2M; in get_logical_coh_st_fabric_id_for_current_spa()
479 hash_pa12 ^= FIELD_GET(BIT_ULL(32), denorm_ctx->current_spa) & hash_ctl_1G; in get_logical_coh_st_fabric_id_for_current_spa()
480 hash_pa12 ^= FIELD_GET(BIT_ULL(42), denorm_ctx->current_spa) & hash_ctl_1T; in get_logical_coh_st_fabric_id_for_current_spa()
482 hash_pa13 = FIELD_GET(BIT_ULL(13), denorm_ctx->current_spa); in get_logical_coh_st_fabric_id_for_current_spa()
483 hash_pa13 ^= FIELD_GET(BIT_ULL(19), denorm_ctx->current_spa) & hash_ctl_64k; in get_logical_coh_st_fabric_id_for_current_spa()
484 hash_pa13 ^= FIELD_GET(BIT_ULL(24), denorm_ctx->current_spa) & hash_ctl_2M; in get_logical_coh_st_fabric_id_for_current_spa()
485 hash_pa13 ^= FIELD_GET(BIT_ULL(33), denorm_ctx->current_spa) & hash_ctl_1G; in get_logical_coh_st_fabric_id_for_current_spa()
486 hash_pa13 ^= FIELD_GET(BIT_ULL(43), denorm_ctx->current_spa) & hash_ctl_1T; in get_logical_coh_st_fabric_id_for_current_spa()
488 switch (ctx->map.intlv_mode) { in get_logical_coh_st_fabric_id_for_current_spa()
490 cs_id = FIELD_GET(GENMASK_ULL(63, 13), denorm_ctx->current_spa) << 3; in get_logical_coh_st_fabric_id_for_current_spa()
491 cs_id %= denorm_ctx->mod_value; in get_logical_coh_st_fabric_id_for_current_spa()
498 cs_id = FIELD_GET(GENMASK_ULL(63, 14), denorm_ctx->current_spa) << 4; in get_logical_coh_st_fabric_id_for_current_spa()
499 cs_id %= denorm_ctx->mod_value; in get_logical_coh_st_fabric_id_for_current_spa()
506 cs_id = FIELD_GET(GENMASK_ULL(63, 12), denorm_ctx->current_spa) << 2; in get_logical_coh_st_fabric_id_for_current_spa()
507 cs_id %= denorm_ctx->mod_value; in get_logical_coh_st_fabric_id_for_current_spa()
513 cs_id = FIELD_GET(GENMASK_ULL(63, 13), denorm_ctx->current_spa) << 3; in get_logical_coh_st_fabric_id_for_current_spa()
514 cs_id %= denorm_ctx->mod_value; in get_logical_coh_st_fabric_id_for_current_spa()
521 cs_id = FIELD_GET(GENMASK_ULL(63, 12), denorm_ctx->current_spa) << 2; in get_logical_coh_st_fabric_id_for_current_spa()
522 cs_id |= (FIELD_GET(BIT_ULL(9), denorm_ctx->current_spa) << 1); in get_logical_coh_st_fabric_id_for_current_spa()
523 cs_id %= denorm_ctx->mod_value; in get_logical_coh_st_fabric_id_for_current_spa()
530 cs_id = FIELD_GET(GENMASK_ULL(63, 12), denorm_ctx->current_spa) << 2; in get_logical_coh_st_fabric_id_for_current_spa()
531 cs_id %= denorm_ctx->mod_value; in get_logical_coh_st_fabric_id_for_current_spa()
538 cs_id = FIELD_GET(GENMASK_ULL(63, 12), denorm_ctx->current_spa) << 2; in get_logical_coh_st_fabric_id_for_current_spa()
539 cs_id |= FIELD_GET(GENMASK_ULL(9, 8), denorm_ctx->current_spa); in get_logical_coh_st_fabric_id_for_current_spa()
540 cs_id %= denorm_ctx->mod_value; in get_logical_coh_st_fabric_id_for_current_spa()
545 cs_id = FIELD_GET(GENMASK_ULL(63, 12), denorm_ctx->current_spa) << 2; in get_logical_coh_st_fabric_id_for_current_spa()
546 cs_id |= FIELD_GET(BIT_ULL(8), denorm_ctx->current_spa) << 1; in get_logical_coh_st_fabric_id_for_current_spa()
547 cs_id %= denorm_ctx->mod_value; in get_logical_coh_st_fabric_id_for_current_spa()
570 * This is required for non-power-of-two and other interleaving modes. in denorm_addr_common()
572 ctx->coh_st_fabric_id = get_logical_coh_st_fabric_id(ctx); in denorm_addr_common()
576 ctx->ret_addr = insert_coh_st_id(ctx, denorm_addr, coh_st_id); in denorm_addr_common()
582 u16 coh_st_id = ctx->coh_st_fabric_id & df_cfg.component_id_mask; in denorm_addr_df3_6chan()
583 u8 total_intlv_bits = ctx->map.total_intlv_bits; in denorm_addr_df3_6chan()
584 u8 low_bit, intlv_bit = ctx->map.intlv_bit_pos; in denorm_addr_df3_6chan()
586 u8 np2_bits = ctx->map.np2_bits; in denorm_addr_df3_6chan()
588 if (ctx->map.intlv_mode != DF3_6CHAN) in denorm_addr_df3_6chan()
589 return -EINVAL; in denorm_addr_df3_6chan()
597 * Rounded up to next power-of-2: 8GB in denorm_addr_df3_6chan()
601 * Get the two most-significant interleave bits from the in denorm_addr_df3_6chan()
604 * [15 + np2_bits - total_intlv_bits : 14 + np2_bits - total_intlv_bits] in denorm_addr_df3_6chan()
606 low_bit = 14 + np2_bits - total_intlv_bits; in denorm_addr_df3_6chan()
607 msb_intlv_bits = ctx->ret_addr >> low_bit; in denorm_addr_df3_6chan()
617 /* Get the remaining interleave bits from the input address. */ in denorm_addr_df3_6chan()
618 temp_addr_b = GENMASK_ULL(low_bit - 1, intlv_bit) & ctx->ret_addr; in denorm_addr_df3_6chan()
631 phys_addr_msb *= 3 - addr_mod + msb_coh_st_id; in denorm_addr_df3_6chan()
635 temp_addr_b |= phys_addr_msb << (low_bit - total_intlv_bits - intlv_bit); in denorm_addr_df3_6chan()
641 temp_addr_b = GENMASK_ULL(63, intlv_bit) & ctx->ret_addr; in denorm_addr_df3_6chan()
645 temp_addr_a = GENMASK_ULL(intlv_bit - 1, 0) & ctx->ret_addr; in denorm_addr_df3_6chan()
648 ctx->ret_addr = temp_addr_a | temp_addr_b; in denorm_addr_df3_6chan()
649 ctx->ret_addr |= coh_st_id << intlv_bit; in denorm_addr_df3_6chan()
662 switch (ctx->map.intlv_mode) { in denorm_addr_df4_np2()
685 return -EINVAL; in denorm_addr_df4_np2()
688 if (ctx->map.num_intlv_sockets == 1) { in denorm_addr_df4_np2()
689 hash_pa8 = BIT_ULL(shift_value) & ctx->ret_addr; in denorm_addr_df4_np2()
690 temp_addr_a = remove_bits(shift_value, shift_value, ctx->ret_addr); in denorm_addr_df4_np2()
692 hash_pa8 = ctx->coh_st_fabric_id & df_cfg.socket_id_mask; in denorm_addr_df4_np2()
693 temp_addr_a = ctx->ret_addr; in denorm_addr_df4_np2()
700 if (ctx->map.intlv_mode == DF4_NPS2_6CHAN_HASH || in denorm_addr_df4_np2()
701 ctx->map.intlv_mode == DF4_NPS1_10CHAN_HASH) { in denorm_addr_df4_np2()
703 } else if (ctx->map.intlv_mode == DF4_NPS1_12CHAN_HASH) { in denorm_addr_df4_np2()
711 shift_value += 1 - ilog2(ctx->map.num_intlv_sockets); in denorm_addr_df4_np2()
712 temp_addr_b = GENMASK_ULL(63, shift_value) & ctx->ret_addr; in denorm_addr_df4_np2()
720 * e.g. NP4_3CHAN -> [0, 1, 2] [6, 7, 8] in denorm_addr_df4_np2()
724 * e.g. NP2_5CHAN -> [0, 1, 2, 3, 4] [5, 6, 7, 8, 9] in denorm_addr_df4_np2()
730 * log_coh_st_offset = 8 - 5 = 3 in denorm_addr_df4_np2()
732 log_coh_st_offset = (ctx->coh_st_fabric_id & mask) - (get_dst_fabric_id(ctx) & mask); in denorm_addr_df4_np2()
757 group_offset = mod_value - 1; in denorm_addr_df4_np2()
759 group_offset--; in denorm_addr_df4_np2()
769 ctx->ret_addr = temp_addr_a | temp_addr_b; in denorm_addr_df4_np2()
772 hash_ctl_64k = FIELD_GET(DF4_HASH_CTL_64K, ctx->map.ctl); in denorm_addr_df4_np2()
773 hash_ctl_2M = FIELD_GET(DF4_HASH_CTL_2M, ctx->map.ctl); in denorm_addr_df4_np2()
774 hash_ctl_1G = FIELD_GET(DF4_HASH_CTL_1G, ctx->map.ctl); in denorm_addr_df4_np2()
777 hashed_bit ^= FIELD_GET(BIT_ULL(14), ctx->ret_addr); in denorm_addr_df4_np2()
778 hashed_bit ^= FIELD_GET(BIT_ULL(16), ctx->ret_addr) & hash_ctl_64k; in denorm_addr_df4_np2()
779 hashed_bit ^= FIELD_GET(BIT_ULL(21), ctx->ret_addr) & hash_ctl_2M; in denorm_addr_df4_np2()
780 hashed_bit ^= FIELD_GET(BIT_ULL(30), ctx->ret_addr) & hash_ctl_1G; in denorm_addr_df4_np2()
782 ctx->ret_addr |= hashed_bit << 8; in denorm_addr_df4_np2()
785 if (ctx->map.intlv_mode == DF4_NPS4_3CHAN_HASH || in denorm_addr_df4_np2()
786 ctx->map.intlv_mode == DF4_NPS2_5CHAN_HASH) in denorm_addr_df4_np2()
790 if (ctx->map.intlv_mode == DF4_NPS1_12CHAN_HASH) in denorm_addr_df4_np2()
795 hashed_bit ^= FIELD_GET(BIT_ULL(18), ctx->ret_addr) & hash_ctl_64k; in denorm_addr_df4_np2()
796 hashed_bit ^= FIELD_GET(BIT_ULL(23), ctx->ret_addr) & hash_ctl_2M; in denorm_addr_df4_np2()
797 hashed_bit ^= FIELD_GET(BIT_ULL(32), ctx->ret_addr) & hash_ctl_1G; in denorm_addr_df4_np2()
799 ctx->ret_addr |= hashed_bit << 13; in denorm_addr_df4_np2()
802 if (ctx->map.intlv_mode != DF4_NPS1_12CHAN_HASH) in denorm_addr_df4_np2()
806 hashed_bit ^= FIELD_GET(BIT_ULL(17), ctx->ret_addr) & hash_ctl_64k; in denorm_addr_df4_np2()
807 hashed_bit ^= FIELD_GET(BIT_ULL(22), ctx->ret_addr) & hash_ctl_2M; in denorm_addr_df4_np2()
808 hashed_bit ^= FIELD_GET(BIT_ULL(31), ctx->ret_addr) & hash_ctl_1G; in denorm_addr_df4_np2()
810 ctx->ret_addr |= hashed_bit << 12; in denorm_addr_df4_np2()
819 switch (ctx->map.intlv_mode) { in normalize_addr_df4p5_np2()
843 switch (ctx->map.intlv_mode) { in normalize_addr_df4p5_np2()
845 temp_addr_b = FIELD_GET(GENMASK_ULL(63, 13), addr) / denorm_ctx->mod_value; in normalize_addr_df4p5_np2()
850 temp_addr_b = FIELD_GET(GENMASK_ULL(63, 14), addr) / denorm_ctx->mod_value; in normalize_addr_df4p5_np2()
855 temp_addr_b = FIELD_GET(GENMASK_ULL(63, 12), addr) / denorm_ctx->mod_value; in normalize_addr_df4p5_np2()
860 temp_addr_b = FIELD_GET(GENMASK_ULL(63, 13), addr) / denorm_ctx->mod_value; in normalize_addr_df4p5_np2()
868 temp_addr_b /= denorm_ctx->mod_value; in normalize_addr_df4p5_np2()
874 temp_addr_b = FIELD_GET(GENMASK_ULL(63, 12), addr) / denorm_ctx->mod_value; in normalize_addr_df4p5_np2()
882 temp_addr_b /= denorm_ctx->mod_value; in normalize_addr_df4p5_np2()
890 temp_addr_b /= denorm_ctx->mod_value; in normalize_addr_df4p5_np2()
899 return denorm_ctx->base_denorm_addr | temp_addr_a | temp_addr_b; in normalize_addr_df4p5_np2()
907 if (!denorm_ctx->rehash_vector) in recalculate_hashed_bits_df4p5_np2()
910 hash_ctl_64k = FIELD_GET(DF4_HASH_CTL_64K, ctx->map.ctl); in recalculate_hashed_bits_df4p5_np2()
911 hash_ctl_2M = FIELD_GET(DF4_HASH_CTL_2M, ctx->map.ctl); in recalculate_hashed_bits_df4p5_np2()
912 hash_ctl_1G = FIELD_GET(DF4_HASH_CTL_1G, ctx->map.ctl); in recalculate_hashed_bits_df4p5_np2()
913 hash_ctl_1T = FIELD_GET(DF4p5_HASH_CTL_1T, ctx->map.ctl); in recalculate_hashed_bits_df4p5_np2()
915 if (denorm_ctx->rehash_vector & BIT_ULL(8)) { in recalculate_hashed_bits_df4p5_np2()
916 hashed_bit = FIELD_GET(BIT_ULL(8), denorm_ctx->current_spa); in recalculate_hashed_bits_df4p5_np2()
917 hashed_bit ^= FIELD_GET(BIT_ULL(14), denorm_ctx->current_spa); in recalculate_hashed_bits_df4p5_np2()
918 hashed_bit ^= FIELD_GET(BIT_ULL(16), denorm_ctx->current_spa) & hash_ctl_64k; in recalculate_hashed_bits_df4p5_np2()
919 hashed_bit ^= FIELD_GET(BIT_ULL(21), denorm_ctx->current_spa) & hash_ctl_2M; in recalculate_hashed_bits_df4p5_np2()
920 hashed_bit ^= FIELD_GET(BIT_ULL(30), denorm_ctx->current_spa) & hash_ctl_1G; in recalculate_hashed_bits_df4p5_np2()
921 hashed_bit ^= FIELD_GET(BIT_ULL(40), denorm_ctx->current_spa) & hash_ctl_1T; in recalculate_hashed_bits_df4p5_np2()
923 if (FIELD_GET(BIT_ULL(8), denorm_ctx->current_spa) != hashed_bit) in recalculate_hashed_bits_df4p5_np2()
924 denorm_ctx->current_spa ^= BIT_ULL(8); in recalculate_hashed_bits_df4p5_np2()
927 if (denorm_ctx->rehash_vector & BIT_ULL(9)) { in recalculate_hashed_bits_df4p5_np2()
928 hashed_bit = FIELD_GET(BIT_ULL(9), denorm_ctx->current_spa); in recalculate_hashed_bits_df4p5_np2()
929 hashed_bit ^= FIELD_GET(BIT_ULL(17), denorm_ctx->current_spa) & hash_ctl_64k; in recalculate_hashed_bits_df4p5_np2()
930 hashed_bit ^= FIELD_GET(BIT_ULL(22), denorm_ctx->current_spa) & hash_ctl_2M; in recalculate_hashed_bits_df4p5_np2()
931 hashed_bit ^= FIELD_GET(BIT_ULL(31), denorm_ctx->current_spa) & hash_ctl_1G; in recalculate_hashed_bits_df4p5_np2()
932 hashed_bit ^= FIELD_GET(BIT_ULL(41), denorm_ctx->current_spa) & hash_ctl_1T; in recalculate_hashed_bits_df4p5_np2()
934 if (FIELD_GET(BIT_ULL(9), denorm_ctx->current_spa) != hashed_bit) in recalculate_hashed_bits_df4p5_np2()
935 denorm_ctx->current_spa ^= BIT_ULL(9); in recalculate_hashed_bits_df4p5_np2()
938 if (denorm_ctx->rehash_vector & BIT_ULL(12)) { in recalculate_hashed_bits_df4p5_np2()
939 hashed_bit = FIELD_GET(BIT_ULL(12), denorm_ctx->current_spa); in recalculate_hashed_bits_df4p5_np2()
940 hashed_bit ^= FIELD_GET(BIT_ULL(18), denorm_ctx->current_spa) & hash_ctl_64k; in recalculate_hashed_bits_df4p5_np2()
941 hashed_bit ^= FIELD_GET(BIT_ULL(23), denorm_ctx->current_spa) & hash_ctl_2M; in recalculate_hashed_bits_df4p5_np2()
942 hashed_bit ^= FIELD_GET(BIT_ULL(32), denorm_ctx->current_spa) & hash_ctl_1G; in recalculate_hashed_bits_df4p5_np2()
943 hashed_bit ^= FIELD_GET(BIT_ULL(42), denorm_ctx->current_spa) & hash_ctl_1T; in recalculate_hashed_bits_df4p5_np2()
945 if (FIELD_GET(BIT_ULL(12), denorm_ctx->current_spa) != hashed_bit) in recalculate_hashed_bits_df4p5_np2()
946 denorm_ctx->current_spa ^= BIT_ULL(12); in recalculate_hashed_bits_df4p5_np2()
949 if (denorm_ctx->rehash_vector & BIT_ULL(13)) { in recalculate_hashed_bits_df4p5_np2()
950 hashed_bit = FIELD_GET(BIT_ULL(13), denorm_ctx->current_spa); in recalculate_hashed_bits_df4p5_np2()
951 hashed_bit ^= FIELD_GET(BIT_ULL(19), denorm_ctx->current_spa) & hash_ctl_64k; in recalculate_hashed_bits_df4p5_np2()
952 hashed_bit ^= FIELD_GET(BIT_ULL(24), denorm_ctx->current_spa) & hash_ctl_2M; in recalculate_hashed_bits_df4p5_np2()
953 hashed_bit ^= FIELD_GET(BIT_ULL(33), denorm_ctx->current_spa) & hash_ctl_1G; in recalculate_hashed_bits_df4p5_np2()
954 hashed_bit ^= FIELD_GET(BIT_ULL(43), denorm_ctx->current_spa) & hash_ctl_1T; in recalculate_hashed_bits_df4p5_np2()
956 if (FIELD_GET(BIT_ULL(13), denorm_ctx->current_spa) != hashed_bit) in recalculate_hashed_bits_df4p5_np2()
957 denorm_ctx->current_spa ^= BIT_ULL(13); in recalculate_hashed_bits_df4p5_np2()
972 atl_debug(ctx, " expected fabric id = 0x%x\n", denorm_ctx->coh_st_fabric_id); in match_logical_coh_st_fabric_id()
974 return denorm_ctx->coh_st_fabric_id == id; in match_logical_coh_st_fabric_id()
979 u64 addr = remove_base_and_hole(ctx, denorm_ctx->current_spa); in match_norm_addr()
989 atl_debug(ctx, " expected normalized addr = 0x%016llx\n", ctx->ret_addr); in match_norm_addr()
991 return addr == ctx->ret_addr; in match_norm_addr()
999 denorm_ctx->div_addr *= denorm_ctx->mod_value; in check_permutations()
1003 * of the dropped remainder. This will be either 0-3 or 0-5 depending in check_permutations()
1004 * on the interleave mode. The low order bits represent the in check_permutations()
1006 * 1, 2, or 3 bits depending on the interleave mode. in check_permutations()
1008 num_perms = denorm_ctx->mod_value << denorm_ctx->perm_shift; in check_permutations()
1011 denorm_addr = denorm_ctx->base_denorm_addr; in check_permutations()
1012 dropped_remainder = test_perm >> denorm_ctx->perm_shift; in check_permutations()
1013 temp_addr = denorm_ctx->div_addr + dropped_remainder; in check_permutations()
1015 switch (ctx->map.intlv_mode) { in check_permutations()
1051 return -EINVAL; in check_permutations()
1054 switch (ctx->map.intlv_mode) { in check_permutations()
1090 return -EINVAL; in check_permutations()
1093 denorm_ctx->current_spa = add_base_and_hole(ctx, denorm_addr); in check_permutations()
1097 denorm_ctx->current_spa); in check_permutations()
1105 if (denorm_ctx->resolved_spa == INVALID_SPA || in check_permutations()
1106 denorm_ctx->current_spa > denorm_ctx->resolved_spa) in check_permutations()
1107 denorm_ctx->resolved_spa = denorm_ctx->current_spa; in check_permutations()
1110 if (denorm_ctx->resolved_spa == INVALID_SPA) { in check_permutations()
1112 ctx->ret_addr); in check_permutations()
1113 return -EINVAL; in check_permutations()
1117 ctx->ret_addr = remove_base_and_hole(ctx, denorm_ctx->resolved_spa); in check_permutations()
1124 denorm_ctx->current_spa = INVALID_SPA; in init_df4p5_denorm_ctx()
1125 denorm_ctx->resolved_spa = INVALID_SPA; in init_df4p5_denorm_ctx()
1127 switch (ctx->map.intlv_mode) { in init_df4p5_denorm_ctx()
1129 denorm_ctx->perm_shift = 3; in init_df4p5_denorm_ctx()
1130 denorm_ctx->rehash_vector = BIT(8) | BIT(9) | BIT(12); in init_df4p5_denorm_ctx()
1134 denorm_ctx->perm_shift = 3; in init_df4p5_denorm_ctx()
1135 denorm_ctx->rehash_vector = BIT(8) | BIT(12) | BIT(13); in init_df4p5_denorm_ctx()
1139 denorm_ctx->perm_shift = 2; in init_df4p5_denorm_ctx()
1140 denorm_ctx->rehash_vector = BIT(8); in init_df4p5_denorm_ctx()
1144 denorm_ctx->perm_shift = 2; in init_df4p5_denorm_ctx()
1145 denorm_ctx->rehash_vector = BIT(8) | BIT(12); in init_df4p5_denorm_ctx()
1152 denorm_ctx->perm_shift = 1; in init_df4p5_denorm_ctx()
1153 denorm_ctx->rehash_vector = BIT(8); in init_df4p5_denorm_ctx()
1158 denorm_ctx->perm_shift = 2; in init_df4p5_denorm_ctx()
1159 denorm_ctx->rehash_vector = 0; in init_df4p5_denorm_ctx()
1164 denorm_ctx->perm_shift = 1; in init_df4p5_denorm_ctx()
1165 denorm_ctx->rehash_vector = 0; in init_df4p5_denorm_ctx()
1170 return -EINVAL; in init_df4p5_denorm_ctx()
1173 denorm_ctx->base_denorm_addr = FIELD_GET(GENMASK_ULL(7, 0), ctx->ret_addr); in init_df4p5_denorm_ctx()
1175 switch (ctx->map.intlv_mode) { in init_df4p5_denorm_ctx()
1182 denorm_ctx->base_denorm_addr |= FIELD_GET(GENMASK_ULL(9, 8), ctx->ret_addr) << 10; in init_df4p5_denorm_ctx()
1183 denorm_ctx->div_addr = FIELD_GET(GENMASK_ULL(63, 10), ctx->ret_addr); in init_df4p5_denorm_ctx()
1192 denorm_ctx->base_denorm_addr |= FIELD_GET(GENMASK_ULL(10, 8), ctx->ret_addr) << 9; in init_df4p5_denorm_ctx()
1193 denorm_ctx->div_addr = FIELD_GET(GENMASK_ULL(63, 11), ctx->ret_addr); in init_df4p5_denorm_ctx()
1198 return -EINVAL; in init_df4p5_denorm_ctx()
1201 if (ctx->map.num_intlv_chan % 3 == 0) in init_df4p5_denorm_ctx()
1202 denorm_ctx->mod_value = 3; in init_df4p5_denorm_ctx()
1204 denorm_ctx->mod_value = 5; in init_df4p5_denorm_ctx()
1206 denorm_ctx->coh_st_fabric_id = get_logical_coh_st_fabric_id(ctx) - get_dst_fabric_id(ctx); in init_df4p5_denorm_ctx()
1209 atl_debug(ctx, " mod_value = %d", denorm_ctx->mod_value); in init_df4p5_denorm_ctx()
1210 atl_debug(ctx, " perm_shift = %d", denorm_ctx->perm_shift); in init_df4p5_denorm_ctx()
1211 atl_debug(ctx, " rehash_vector = 0x%x", denorm_ctx->rehash_vector); in init_df4p5_denorm_ctx()
1212 atl_debug(ctx, " base_denorm_addr = 0x%016llx", denorm_ctx->base_denorm_addr); in init_df4p5_denorm_ctx()
1213 atl_debug(ctx, " div_addr = 0x%016llx", denorm_ctx->div_addr); in init_df4p5_denorm_ctx()
1214 atl_debug(ctx, " coh_st_fabric_id = 0x%x", denorm_ctx->coh_st_fabric_id); in init_df4p5_denorm_ctx()
1221 * normalized address. The exact bits will differ between interleave modes, but
1241 atl_debug(ctx, "Denormalizing DF 4.5 normalized address 0x%016llx", ctx->ret_addr); in denorm_addr_df4p5_np2()
1252 switch (ctx->map.intlv_mode) { in denormalize_address()