112 	struct perf_output_handle handle; /* AUX buffer output handle */
819 	/* Use AUX buffer. No need to allocate it by ourself */  in __hw_perf_event_init()
1201 	 * AUX buffer is used when in diagnostic sampling mode.  in hw_perf_event_update()
1279 static inline unsigned long aux_sdb_index(struct aux_buffer *aux,  in aux_sdb_index()  argument
1282 	return i % aux->sfb.num_sdb;  in aux_sdb_index()
1290 static inline unsigned long aux_sdb_num_alert(struct aux_buffer *aux)  in aux_sdb_num_alert()  argument
1292 	return aux_sdb_num(aux->head, aux->alert_mark);  in aux_sdb_num_alert()
1295 static inline unsigned long aux_sdb_num_empty(struct aux_buffer *aux)  in aux_sdb_num_empty()  argument
1297 	return aux_sdb_num(aux->head, aux->empty_mark);  in aux_sdb_num_empty()
1303 static struct hws_trailer_entry *aux_sdb_trailer(struct aux_buffer *aux,  in aux_sdb_trailer()  argument
1308 	index = aux_sdb_index(aux, index);  in aux_sdb_trailer()
1309 	sdb = aux->sdb_index[index];  in aux_sdb_trailer()
1315  * disabled. Collect the full SDBs in AUX buffer which have not reached
1325 	struct aux_buffer *aux;  in aux_output_end()  local
1328 	aux = perf_get_aux(handle);  in aux_output_end()
1329 	if (!aux)  in aux_output_end()
1332 	range_scan = aux_sdb_num_alert(aux);  in aux_output_end()
1333 	for (i = 0, idx = aux->head; i < range_scan; i++, idx++) {  in aux_output_end()
1334 		te = aux_sdb_trailer(aux, idx);  in aux_output_end()
1342 	te = aux_sdb_trailer(aux, aux->alert_mark);  in aux_output_end()
1356 			    struct aux_buffer *aux,  in aux_output_begin()  argument
1365 	aux->head = handle->head >> PAGE_SHIFT;  in aux_output_begin()
1371 	 * SDBs between aux->head and aux->empty_mark are already ready  in aux_output_begin()
1374 	if (range > aux_sdb_num_empty(aux)) {  in aux_output_begin()
1375 		range_scan = range - aux_sdb_num_empty(aux);  in aux_output_begin()
1376 		idx = aux->empty_mark + 1;  in aux_output_begin()
1378 			te = aux_sdb_trailer(aux, idx);  in aux_output_begin()
1384 		aux->empty_mark = aux->head + range - 1;  in aux_output_begin()
1388 	aux->alert_mark = aux->head + range/2 - 1;  in aux_output_begin()
1389 	te = aux_sdb_trailer(aux, aux->alert_mark);  in aux_output_begin()
1393 	head = aux_sdb_index(aux, aux->head);  in aux_output_begin()
1394 	base = aux->sdbt_index[head / CPUM_SF_SDB_PER_TABLE];  in aux_output_begin()
1397 	cpuhw->lsctl.dear = virt_to_phys((void *)aux->sdb_index[head]);  in aux_output_begin()
1408 static bool aux_set_alert(struct aux_buffer *aux, unsigned long alert_index,  in aux_set_alert()  argument
1414 	te = aux_sdb_trailer(aux, alert_index);  in aux_set_alert()
1437  * @aux:	The AUX buffer to set
1438  * @range:	The range of SDBs to scan started from aux->head
1441  * Set alert indicator on the SDB at index of aux->alert_mark. If this SDB is
1446  * Scan the SDBs in AUX buffer from behind aux->empty_mark. They are used
1449  * If aux->alert_mark fall in this area, just set it. Overflow count is
1452  * SDBs between aux->head and aux->empty_mark are already reset at last time.
1457 static bool aux_reset_buffer(struct aux_buffer *aux, unsigned long range,  in aux_reset_buffer()  argument
1465 	if (range <= aux_sdb_num_empty(aux))  in aux_reset_buffer()
1471 		return aux_set_alert(aux, aux->alert_mark, overflow);  in aux_reset_buffer()
1473 	if (aux->alert_mark <= aux->empty_mark)  in aux_reset_buffer()
1478 		if (!aux_set_alert(aux, aux->alert_mark, overflow))  in aux_reset_buffer()
1486 	range_scan = range - aux_sdb_num_empty(aux);  in aux_reset_buffer()
1487 	idx = aux->empty_mark + 1;  in aux_reset_buffer()
1489 		te = aux_sdb_trailer(aux, idx);  in aux_reset_buffer()
1497 			if (idx == aux->alert_mark)  in aux_reset_buffer()
1507 	aux->empty_mark = aux->head + range - 1;  in aux_reset_buffer()
1517 	struct aux_buffer *aux;  in hw_collect_aux()  local
1524 	aux = perf_get_aux(handle);  in hw_collect_aux()
1525 	if (!aux)  in hw_collect_aux()
1529 	size = aux_sdb_num_alert(aux) << PAGE_SHIFT;  in hw_collect_aux()
1534 	num_sdb = aux->sfb.num_sdb;  in hw_collect_aux()
1537 		aux = perf_aux_output_begin(handle, cpuhw->event);  in hw_collect_aux()
1539 			pr_err("The AUX buffer with %lu pages for the "  in hw_collect_aux()
1544 		if (!aux)  in hw_collect_aux()
1548 		aux->head = handle->head >> PAGE_SHIFT;  in hw_collect_aux()
1551 			aux->alert_mark = aux->head;  in hw_collect_aux()
1553 			aux->alert_mark = aux->head + range/2 - 1;  in hw_collect_aux()
1555 		if (aux_reset_buffer(aux, range, &overflow)) {  in hw_collect_aux()
1562 			pr_err("Sample data caused the AUX buffer with %lu "  in hw_collect_aux()
1563 			       "pages to overflow\n", aux->sfb.num_sdb);  in hw_collect_aux()
1565 			size = aux_sdb_num_alert(aux) << PAGE_SHIFT;  in hw_collect_aux()
1572  * Callback when freeing AUX buffers.
1576 	struct aux_buffer *aux = data;  in aux_buffer_free()  local
1579 	if (!aux)  in aux_buffer_free()
1583 	num_sdbt = aux->sfb.num_sdbt;  in aux_buffer_free()
1585 		free_page(aux->sdbt_index[i]);  in aux_buffer_free()
1587 	kfree(aux->sdbt_index);  in aux_buffer_free()
1588 	kfree(aux->sdb_index);  in aux_buffer_free()
1589 	kfree(aux);  in aux_buffer_free()
1604  * aux_buffer_setup() - Setup AUX buffer for diagnostic mode sampling
1610  * This is the callback when setup an event using AUX buffer. Perf tool can
1612  * for basic samples, AUX buffer belongs to the event. It is scheduled with
1615  * Return the private AUX buffer structure if success or NULL if fails.
1621 	struct aux_buffer *aux;  in aux_buffer_setup()  local
1629 		pr_err("AUX buffer size (%i pages) is larger than the "  in aux_buffer_setup()
1634 		pr_err("AUX buffer size (%i pages) is less than the "  in aux_buffer_setup()
1641 	aux = kzalloc(sizeof(struct aux_buffer), GFP_KERNEL);  in aux_buffer_setup()
1642 	if (!aux)  in aux_buffer_setup()
1644 	sfb = &aux->sfb;  in aux_buffer_setup()
1648 	aux->sdbt_index = kmalloc_array(n_sdbt, sizeof(void *), GFP_KERNEL);  in aux_buffer_setup()
1649 	if (!aux->sdbt_index)  in aux_buffer_setup()
1653 	aux->sdb_index = kmalloc_array(nr_pages, sizeof(void *), GFP_KERNEL);  in aux_buffer_setup()
1654 	if (!aux->sdb_index)  in aux_buffer_setup()
1662 	aux->sdbt_index[sfb->num_sdbt++] = (unsigned long)sfb->sdbt;  in aux_buffer_setup()
1666 	 * Link the provided pages of AUX buffer to SDBT.  in aux_buffer_setup()
1674 			aux->sdbt_index[sfb->num_sdbt++] = (unsigned long)new;  in aux_buffer_setup()
1681 		aux->sdb_index[i] = (unsigned long)pages[i];  in aux_buffer_setup()
1695 	aux->empty_mark = sfb->num_sdb - 1;  in aux_buffer_setup()
1697 	return aux;  in aux_buffer_setup()
1700 	/* SDBs (AUX buffer pages) are freed by caller */  in aux_buffer_setup()
1702 		free_page(aux->sdbt_index[i]);  in aux_buffer_setup()
1703 	kfree(aux->sdb_index);  in aux_buffer_setup()
1705 	kfree(aux->sdbt_index);  in aux_buffer_setup()
1707 	kfree(aux);  in aux_buffer_setup()
1792 	struct aux_buffer *aux;  in cpumsf_pmu_add()  local
1826 		aux = perf_aux_output_begin(&cpuhw->handle, event);  in cpumsf_pmu_add()
1827 		if (!aux) {  in cpumsf_pmu_add()
1831 		err = aux_output_begin(&cpuhw->handle, aux, cpuhw);  in cpumsf_pmu_add()