arm/crypto/crct10dif-ce-core.S

2 // Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions instructions
14 // Implement fast CRC-T10DIF computation with SSE and PCLMULQDQ instructions
62 //  /white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
75 	.arch		armv8-a
76 	.fpu		crypto-neon-fp-armv8
118 	vld1.64		{q11-q12}, [buf]!
155 // Assumes len >= 16.
167 	vld1.64		{q0-q1}, [buf]!
168 	vld1.64		{q2-q3}, [buf]!
169 	vld1.64		{q4-q5}, [buf]!
170 	vld1.64		{q6-q7}, [buf]!
188 	// XOR the first 16 data *bits* with the initial CRC value.
200 	// While >= 128 data bytes remain (not counting q0-q7), fold the 128
201 	// bytes q0-q7 into them, storing the result back into q0-q7.
210 	// Now fold the 112 bytes in q0-q6 into the 16 bytes in q7.
221 	// Fold across 16 bytes.
226 	// Then subtract 16 to simplify the termination condition of the
228 	adds		len, len, #(128-16)
230 	// While >= 16 data bytes remain (not counting q7), fold the 16 bytes q7
241 	subs		len, len, #16
245 	// Add 16 to get the correct number of data bytes remaining in 0...15
246 	// (not counting q7), following the previous extra subtraction by 16.
247 	adds		len, len, #16
251 	// Reduce the last '16 + len' bytes where 1 <= len <= 15 and the first
252 	// 16 bytes are in q7 and the rest are the remaining data in 'buf'.  To
255 	// chunk of 16 bytes, then fold the first chunk into the second.
257 	// q0 = last 16 original data bytes
259 	sub		buf, buf, #16
264 	// q1 = high order part of second chunk: q7 left-shifted by 'len' bytes.
265 	__adrl		r3, .Lbyteshift_table + 16
268 	vtbl.8		q1l, {q7l-q7h}, q2l
269 	vtbl.8		q1h, {q7l-q7h}, q2h
271 	// q3 = first chunk: q7 right-shifted by '16-len' bytes.
274 	vtbl.8		q3l, {q7l-q7h}, q2l
275 	vtbl.8		q3h, {q7l-q7h}, q2h
277 	// Convert to 8-bit masks: 'len' 0x00 bytes, then '16-len' 0xff bytes.
280 	// q2 = second chunk: 'len' bytes from q0 (low-order bytes),
281 	// then '16-len' bytes from q1 (high-order bytes).
291 	// Reduce the 128-bit value M(x), stored in q7, to the final 16-bit CRC.
296 	// Fold the high 64 bits into the low 64 bits, while also multiplying by
297 	// x^64.  This produces a 128-bit value congruent to x^64 * M(x) and
298 	// whose low 48 bits are 0.
299 	vmull.p64	q0, q7h, FOLD_CONST_H	// high bits * x^48 * (x^80 mod G(x))
300 	veor.8		q0h, q0h, q7l		// + low bits * x^64
302 	// Fold the high 32 bits into the low 96 bits.  This produces a 96-bit
303 	// value congruent to x^64 * M(x) and whose low 48 bits are 0.
305 	vmov		s4, s3			// extract high 32 bits
306 	vmov		s3, s5			// zero high 32 bits
307 	vmull.p64	q1, q1l, FOLD_CONST_L	// high 32 bits * x^48 * (x^48 mod G(x))
308 	veor.8		q0, q0, q1		// + low bits
314 	vmull.p64	q1, q0h, FOLD_CONST_H	// high 32 bits * floor(x^48 / G(x))
318 	veor.8		q0l, q0l, q1l		// + low 16 nonzero bits
319 	// Final CRC value (x^16 * M(x)) mod G(x) is in low 16 bits of q0.
325 	// Checksumming a buffer of length 16...255 bytes
329 	// Load the first 16 data bytes.
334 	// XOR the first 16 data *bits* with the initial CRC value.
339 	// Load the fold-across-16-bytes constants.
342 	cmp		len, #16
343 	beq		.Lreduce_final_16_bytes		// len == 16
345 	addlt		len, len, #16
354 // G(x) = x^16 + x^15 + x^11 + x^9 + x^8 + x^7 + x^5 + x^4 + x^2 + x^1 + x^0
374 // For 1 <= len <= 15, the 16-byte vector beginning at &byteshift_table[16 -
376 // ..., 0x80} XOR the index vector to shift right by '16 - len' bytes.