194 		dev_err(dev, "table %u not support multi-hop addressing!\n",  in get_hem_table_config()
203 			   struct hns_roce_hem_table *table, unsigned long *obj,  in hns_roce_calc_hem_mhop()  argument
212 	if (get_hem_table_config(hr_dev, mhop, table->type))  in hns_roce_calc_hem_mhop()
222 	bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);  in hns_roce_calc_hem_mhop()
224 	chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size :  in hns_roce_calc_hem_mhop()
226 	table_idx = *obj / (chunk_size / table->obj_size);  in hns_roce_calc_hem_mhop()
241 		dev_err(dev, "table %u not support hop_num = %u!\n",  in hns_roce_calc_hem_mhop()
242 			table->type, mhop->hop_num);  in hns_roce_calc_hem_mhop()
299 			   struct hns_roce_hem_table *table, unsigned long obj,  in calc_hem_config()  argument
310 	ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, mhop);  in calc_hem_config()
318 	bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);  in calc_hem_config()
334 		ibdev_err(ibdev, "table %u not support mhop.hop_num = %u!\n",  in calc_hem_config()
335 			  table->type, mhop->hop_num);  in calc_hem_config()
339 	if (unlikely(index->buf >= table->num_hem)) {  in calc_hem_config()
340 		ibdev_err(ibdev, "table %u exceed hem limt idx %llu, max %lu!\n",  in calc_hem_config()
341 			  table->type, index->buf, table->num_hem);  in calc_hem_config()
349 			  struct hns_roce_hem_table *table,  in free_mhop_hem()  argument
357 		hns_roce_free_hem(hr_dev, table->hem[index->buf]);  in free_mhop_hem()
358 		table->hem[index->buf] = NULL;  in free_mhop_hem()
362 		dma_free_coherent(dev, bt_size, table->bt_l1[index->l1],  in free_mhop_hem()
363 				  table->bt_l1_dma_addr[index->l1]);  in free_mhop_hem()
364 		table->bt_l1[index->l1] = NULL;  in free_mhop_hem()
368 		dma_free_coherent(dev, bt_size, table->bt_l0[index->l0],  in free_mhop_hem()
369 				  table->bt_l0_dma_addr[index->l0]);  in free_mhop_hem()
370 		table->bt_l0[index->l0] = NULL;  in free_mhop_hem()
375 			  struct hns_roce_hem_table *table,  in alloc_mhop_hem()  argument
387 	if ((check_whether_bt_num_3(table->type, mhop->hop_num) ||  in alloc_mhop_hem()
388 	     check_whether_bt_num_2(table->type, mhop->hop_num)) &&  in alloc_mhop_hem()
389 	     !table->bt_l0[index->l0]) {  in alloc_mhop_hem()
390 		table->bt_l0[index->l0] = dma_alloc_coherent(dev, bt_size,  in alloc_mhop_hem()
391 					    &table->bt_l0_dma_addr[index->l0],  in alloc_mhop_hem()
393 		if (!table->bt_l0[index->l0]) {  in alloc_mhop_hem()
401 	if (check_whether_bt_num_3(table->type, mhop->hop_num) &&  in alloc_mhop_hem()
402 	    !table->bt_l1[index->l1])  {  in alloc_mhop_hem()
403 		table->bt_l1[index->l1] = dma_alloc_coherent(dev, bt_size,  in alloc_mhop_hem()
404 					    &table->bt_l1_dma_addr[index->l1],  in alloc_mhop_hem()
406 		if (!table->bt_l1[index->l1]) {  in alloc_mhop_hem()
411 		*(table->bt_l0[index->l0] + mhop->l1_idx) =  in alloc_mhop_hem()
412 					       table->bt_l1_dma_addr[index->l1];  in alloc_mhop_hem()
419 	size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : bt_size;  in alloc_mhop_hem()
421 	table->hem[index->buf] = hns_roce_alloc_hem(hr_dev, size, flag);  in alloc_mhop_hem()
422 	if (!table->hem[index->buf]) {  in alloc_mhop_hem()
428 	bt_ba = table->hem[index->buf]->dma;  in alloc_mhop_hem()
430 	if (table->type < HEM_TYPE_MTT) {  in alloc_mhop_hem()
432 			*(table->bt_l1[index->l1] + mhop->l2_idx) = bt_ba;  in alloc_mhop_hem()
434 			*(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba;  in alloc_mhop_hem()
436 		*(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba;  in alloc_mhop_hem()
441 	free_mhop_hem(hr_dev, table, mhop, index);  in alloc_mhop_hem()
447 			struct hns_roce_hem_table *table, unsigned long obj,  in set_mhop_hem()  argument
456 		ret = hr_dev->hw->set_hem(hr_dev, table, obj, 0);  in set_mhop_hem()
464 		ret = hr_dev->hw->set_hem(hr_dev, table, obj, 1);  in set_mhop_hem()
476 		ret = hr_dev->hw->set_hem(hr_dev, table, obj, step_idx);  in set_mhop_hem()
485 				   struct hns_roce_hem_table *table,  in hns_roce_table_mhop_get()  argument
493 	ret = calc_hem_config(hr_dev, table, obj, &mhop, &index);  in hns_roce_table_mhop_get()
499 	mutex_lock(&table->mutex);  in hns_roce_table_mhop_get()
500 	if (table->hem[index.buf]) {  in hns_roce_table_mhop_get()
501 		refcount_inc(&table->hem[index.buf]->refcount);  in hns_roce_table_mhop_get()
505 	ret = alloc_mhop_hem(hr_dev, table, &mhop, &index);  in hns_roce_table_mhop_get()
512 	if (table->type < HEM_TYPE_MTT) {  in hns_roce_table_mhop_get()
513 		ret = set_mhop_hem(hr_dev, table, obj, &mhop, &index);  in hns_roce_table_mhop_get()
520 	refcount_set(&table->hem[index.buf]->refcount, 1);  in hns_roce_table_mhop_get()
524 	free_mhop_hem(hr_dev, table, &mhop, &index);  in hns_roce_table_mhop_get()
526 	mutex_unlock(&table->mutex);  in hns_roce_table_mhop_get()
531 		       struct hns_roce_hem_table *table, unsigned long obj)  in hns_roce_table_get()  argument
537 	if (hns_roce_check_whether_mhop(hr_dev, table->type))  in hns_roce_table_get()
538 		return hns_roce_table_mhop_get(hr_dev, table, obj);  in hns_roce_table_get()
540 	i = obj / (table->table_chunk_size / table->obj_size);  in hns_roce_table_get()
542 	mutex_lock(&table->mutex);  in hns_roce_table_get()
544 	if (table->hem[i]) {  in hns_roce_table_get()
545 		refcount_inc(&table->hem[i]->refcount);  in hns_roce_table_get()
549 	table->hem[i] = hns_roce_alloc_hem(hr_dev,  in hns_roce_table_get()
550 				       table->table_chunk_size,  in hns_roce_table_get()
552 	if (!table->hem[i]) {  in hns_roce_table_get()
558 	ret = hr_dev->hw->set_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT);  in hns_roce_table_get()
560 		hns_roce_free_hem(hr_dev, table->hem[i]);  in hns_roce_table_get()
561 		table->hem[i] = NULL;  in hns_roce_table_get()
567 	refcount_set(&table->hem[i]->refcount, 1);  in hns_roce_table_get()
569 	mutex_unlock(&table->mutex);  in hns_roce_table_get()
574 			   struct hns_roce_hem_table *table, unsigned long obj,  in clear_mhop_hem()  argument
586 	if (check_whether_bt_num_2(table->type, hop_num)) {  in clear_mhop_hem()
587 		if (hns_roce_check_hem_null(table->hem, index->buf,  in clear_mhop_hem()
588 					    chunk_ba_num, table->num_hem))  in clear_mhop_hem()
590 	} else if (check_whether_bt_num_3(table->type, hop_num)) {  in clear_mhop_hem()
591 		if (hns_roce_check_hem_null(table->hem, index->buf,  in clear_mhop_hem()
592 					    chunk_ba_num, table->num_hem)) {  in clear_mhop_hem()
594 			if (hns_roce_check_bt_null(table->bt_l1, index->l1,  in clear_mhop_hem()
600 	if (table->type < HEM_TYPE_MTT) {  in clear_mhop_hem()
606 		ret = hr_dev->hw->clear_hem(hr_dev, table, obj, step_idx);  in clear_mhop_hem()
612 			ret = hr_dev->hw->clear_hem(hr_dev, table, obj, 1);  in clear_mhop_hem()
619 			ret = hr_dev->hw->clear_hem(hr_dev, table, obj, 0);  in clear_mhop_hem()
628 				    struct hns_roce_hem_table *table,  in hns_roce_table_mhop_put()  argument
637 	ret = calc_hem_config(hr_dev, table, obj, &mhop, &index);  in hns_roce_table_mhop_put()
644 		mutex_lock(&table->mutex);  in hns_roce_table_mhop_put()
645 	else if (!refcount_dec_and_mutex_lock(&table->hem[index.buf]->refcount,  in hns_roce_table_mhop_put()
646 					      &table->mutex))  in hns_roce_table_mhop_put()
649 	clear_mhop_hem(hr_dev, table, obj, &mhop, &index);  in hns_roce_table_mhop_put()
650 	free_mhop_hem(hr_dev, table, &mhop, &index);  in hns_roce_table_mhop_put()
652 	mutex_unlock(&table->mutex);  in hns_roce_table_mhop_put()
656 			struct hns_roce_hem_table *table, unsigned long obj)  in hns_roce_table_put()  argument
662 	if (hns_roce_check_whether_mhop(hr_dev, table->type)) {  in hns_roce_table_put()
663 		hns_roce_table_mhop_put(hr_dev, table, obj, 1);  in hns_roce_table_put()
667 	i = obj / (table->table_chunk_size / table->obj_size);  in hns_roce_table_put()
669 	if (!refcount_dec_and_mutex_lock(&table->hem[i]->refcount,  in hns_roce_table_put()
670 					 &table->mutex))  in hns_roce_table_put()
673 	ret = hr_dev->hw->clear_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT);  in hns_roce_table_put()
678 	hns_roce_free_hem(hr_dev, table->hem[i]);  in hns_roce_table_put()
679 	table->hem[i] = NULL;  in hns_roce_table_put()
681 	mutex_unlock(&table->mutex);  in hns_roce_table_put()
685 			  struct hns_roce_hem_table *table,  in hns_roce_table_find()  argument
698 	mutex_lock(&table->mutex);  in hns_roce_table_find()
700 	if (!hns_roce_check_whether_mhop(hr_dev, table->type)) {  in hns_roce_table_find()
701 		obj_per_chunk = table->table_chunk_size / table->obj_size;  in hns_roce_table_find()
702 		hem = table->hem[obj / obj_per_chunk];  in hns_roce_table_find()
704 		dma_offset = offset = idx_offset * table->obj_size;  in hns_roce_table_find()
708 		if (hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop))  in hns_roce_table_find()
719 		hem = table->hem[hem_idx];  in hns_roce_table_find()
732 	mutex_unlock(&table->mutex);  in hns_roce_table_find()
737 			    struct hns_roce_hem_table *table, u32 type,  in hns_roce_init_hem_table()  argument
744 		table->table_chunk_size = hr_dev->caps.chunk_sz;  in hns_roce_init_hem_table()
745 		obj_per_chunk = table->table_chunk_size / obj_size;  in hns_roce_init_hem_table()
748 		table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL);  in hns_roce_init_hem_table()
749 		if (!table->hem)  in hns_roce_init_hem_table()
774 		table->hem = kcalloc(num_hem, sizeof(*table->hem),  in hns_roce_init_hem_table()
776 		if (!table->hem)  in hns_roce_init_hem_table()
783 			table->bt_l1 = kcalloc(num_bt_l1,  in hns_roce_init_hem_table()
784 					       sizeof(*table->bt_l1),  in hns_roce_init_hem_table()
786 			if (!table->bt_l1)  in hns_roce_init_hem_table()
789 			table->bt_l1_dma_addr = kcalloc(num_bt_l1,  in hns_roce_init_hem_table()
790 						 sizeof(*table->bt_l1_dma_addr),  in hns_roce_init_hem_table()
793 			if (!table->bt_l1_dma_addr)  in hns_roce_init_hem_table()
799 			table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0),  in hns_roce_init_hem_table()
801 			if (!table->bt_l0)  in hns_roce_init_hem_table()
804 			table->bt_l0_dma_addr = kcalloc(num_bt_l0,  in hns_roce_init_hem_table()
805 						 sizeof(*table->bt_l0_dma_addr),  in hns_roce_init_hem_table()
807 			if (!table->bt_l0_dma_addr)  in hns_roce_init_hem_table()
812 	table->type = type;  in hns_roce_init_hem_table()
813 	table->num_hem = num_hem;  in hns_roce_init_hem_table()
814 	table->obj_size = obj_size;  in hns_roce_init_hem_table()
815 	mutex_init(&table->mutex);  in hns_roce_init_hem_table()
820 	kfree(table->bt_l0);  in hns_roce_init_hem_table()
821 	table->bt_l0 = NULL;  in hns_roce_init_hem_table()
824 	kfree(table->bt_l1_dma_addr);  in hns_roce_init_hem_table()
825 	table->bt_l1_dma_addr = NULL;  in hns_roce_init_hem_table()
828 	kfree(table->bt_l1);  in hns_roce_init_hem_table()
829 	table->bt_l1 = NULL;  in hns_roce_init_hem_table()
832 	kfree(table->hem);  in hns_roce_init_hem_table()
833 	table->hem = NULL;  in hns_roce_init_hem_table()
840 					    struct hns_roce_hem_table *table)  in hns_roce_cleanup_mhop_hem_table()  argument
847 	if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop))  in hns_roce_cleanup_mhop_hem_table()
849 	buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size :  in hns_roce_cleanup_mhop_hem_table()
852 	for (i = 0; i < table->num_hem; ++i) {  in hns_roce_cleanup_mhop_hem_table()
853 		obj = i * buf_chunk_size / table->obj_size;  in hns_roce_cleanup_mhop_hem_table()
854 		if (table->hem[i])  in hns_roce_cleanup_mhop_hem_table()
855 			hns_roce_table_mhop_put(hr_dev, table, obj, 0);  in hns_roce_cleanup_mhop_hem_table()
858 	kfree(table->hem);  in hns_roce_cleanup_mhop_hem_table()
859 	table->hem = NULL;  in hns_roce_cleanup_mhop_hem_table()
860 	kfree(table->bt_l1);  in hns_roce_cleanup_mhop_hem_table()
861 	table->bt_l1 = NULL;  in hns_roce_cleanup_mhop_hem_table()
862 	kfree(table->bt_l1_dma_addr);  in hns_roce_cleanup_mhop_hem_table()
863 	table->bt_l1_dma_addr = NULL;  in hns_roce_cleanup_mhop_hem_table()
864 	kfree(table->bt_l0);  in hns_roce_cleanup_mhop_hem_table()
865 	table->bt_l0 = NULL;  in hns_roce_cleanup_mhop_hem_table()
866 	kfree(table->bt_l0_dma_addr);  in hns_roce_cleanup_mhop_hem_table()
867 	table->bt_l0_dma_addr = NULL;  in hns_roce_cleanup_mhop_hem_table()
871 				struct hns_roce_hem_table *table)  in hns_roce_cleanup_hem_table()  argument
878 	if (hns_roce_check_whether_mhop(hr_dev, table->type)) {  in hns_roce_cleanup_hem_table()
879 		hns_roce_cleanup_mhop_hem_table(hr_dev, table);  in hns_roce_cleanup_hem_table()
880 		mutex_destroy(&table->mutex);  in hns_roce_cleanup_hem_table()
884 	for (i = 0; i < table->num_hem; ++i)  in hns_roce_cleanup_hem_table()
885 		if (table->hem[i]) {  in hns_roce_cleanup_hem_table()
886 			obj = i * table->table_chunk_size / table->obj_size;  in hns_roce_cleanup_hem_table()
887 			ret = hr_dev->hw->clear_hem(hr_dev, table, obj, 0);  in hns_roce_cleanup_hem_table()
892 			hns_roce_free_hem(hr_dev, table->hem[i]);  in hns_roce_cleanup_hem_table()
895 	mutex_destroy(&table->mutex);  in hns_roce_cleanup_hem_table()
896 	kfree(table->hem);  in hns_roce_cleanup_hem_table()
903 					   &hr_dev->srq_table.table);  in hns_roce_cleanup_hem()
904 	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table);  in hns_roce_cleanup_hem()
996 /* assign L0 table address to hem from root bt */
1029 	 * hopnum    base address table levels  in hem_list_is_bottom_bt()
1041  * @bt_level: base address table level
1348 /* construct the base address table and link them by address hop config */