arm64/crypto/crct10dif-ce-core.S

2 // Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions
14 // Implement fast CRC-T10DIF computation with SSE and PCLMULQDQ instructions
62 //  /white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
69 	.arch		armv8-a+crypto
277 CPU_LE(	rev64		v2.16b, v2.16b			)
285 CPU_LE(	ext		v2.16b, v2.16b, v2.16b, #8	)
292 	// XOR the first 16 data *bits* with the initial CRC value.
305 	// While >= 128 data bytes remain (not counting v0-v7), fold the 128
306 	// bytes v0-v7 into them, storing the result back into v0-v7.
309 	fold_32_bytes	\p, v2, v3
316 	// Now fold the 112 bytes in v0-v6 into the 16 bytes in v7.
324 	fold_16_bytes	\p, v2, v6
336 	adds		len, len, #(128-16)
367 	ldr		q0, [buf, #-16]
371 	// v1 = high order part of second chunk: v7 left-shifted by 'len' bytes.
374 	ld1		{v2.16b}, [x4]
375 	tbl		v1.16b, {v7.16b}, v2.16b
377 	// v3 = first chunk: v7 right-shifted by '16-len' bytes.
379 	eor		v2.16b, v2.16b, v3.16b
380 	tbl		v3.16b, {v7.16b}, v2.16b
382 	// Convert to 8-bit masks: 'len' 0x00 bytes, then '16-len' 0xff bytes.
383 	sshr		v2.16b, v2.16b, #7
385 	// v2 = second chunk: 'len' bytes from v0 (low-order bytes),
386 	// then '16-len' bytes from v1 (high-order bytes).
387 	bsl		v2.16b, v1.16b, v0.16b
393 	eor		v7.16b, v7.16b, v2.16b
396 	// Reduce the 128-bit value M(x), stored in v7, to the final 16-bit CRC.
398 	movi		v2.16b, #0		// init zero register
405 	// x^64.  This produces a 128-bit value congruent to x^64 * M(x) and
407 	ext		v0.16b, v2.16b, v7.16b, #8
411 	// Fold the high 32 bits into the low 96 bits.  This produces a 96-bit
413 	ext		v1.16b, v0.16b, v2.16b, #12	// extract high 32 bits
414 	mov		v0.s[3], v2.s[0]	// zero high 32 bits
446 	// XOR the first 16 data *bits* with the initial CRC value.
451 	// Load the fold-across-16-bytes constants.
507 // For 1 <= len <= 15, the 16-byte vector beginning at &byteshift_table[16 -
509 // ..., 0x80} XOR the index vector to shift right by '16 - len' bytes.