Lines Matching +full:9 +full:- +full:inch
5 * This source code is licensed under both the BSD-style license (found in the
8 * You may select, at your option, one of the above-listed licenses.
14 /*-*******************************************************
50 /*-*************************************************************
63 bpPtr->lastBlock = cBlockHeader & 1; in ZSTD_getcBlockSize()
64 bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3); in ZSTD_getcBlockSize()
65 bpPtr->origSize = cSize; /* only useful for RLE */ in ZSTD_getcBlockSize()
66 if (bpPtr->blockType == bt_rle) return 1; in ZSTD_getcBlockSize()
67 RETURN_ERROR_IF(bpPtr->blockType == bt_reserved, corruption_detected, ""); in ZSTD_getcBlockSize()
79 dctx->litBuffer = (BYTE*)dst + ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH; in ZSTD_allocateLiteralsBuffer()
80 dctx->litBufferEnd = dctx->litBuffer + litSize; in ZSTD_allocateLiteralsBuffer()
81 dctx->litBufferLocation = ZSTD_in_dst; in ZSTD_allocateLiteralsBuffer()
88 …dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OV… in ZSTD_allocateLiteralsBuffer()
89 dctx->litBufferEnd = dctx->litBuffer + litSize - ZSTD_LITBUFFEREXTRASIZE; in ZSTD_allocateLiteralsBuffer()
93 dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize; in ZSTD_allocateLiteralsBuffer()
94 dctx->litBufferEnd = (BYTE*)dst + expectedWriteSize; in ZSTD_allocateLiteralsBuffer()
96 dctx->litBufferLocation = ZSTD_split; in ZSTD_allocateLiteralsBuffer()
101 dctx->litBuffer = dctx->litExtraBuffer; in ZSTD_allocateLiteralsBuffer()
102 dctx->litBufferEnd = dctx->litBuffer + litSize; in ZSTD_allocateLiteralsBuffer()
103 dctx->litBufferLocation = ZSTD_not_in_dst; in ZSTD_allocateLiteralsBuffer()
115 * stored in dctx->litExtraBuffer to help keep it "ahead" of the current output write.
132 DEBUGLOG(5, "set_repeat flag : re-using stats from previous compressed literals block"); in ZSTD_decodeLiteralsBlock()
133 RETURN_ERROR_IF(dctx->litEntropy==0, dictionary_corrupted, ""); in ZSTD_decodeLiteralsBlock()
147 /* 2 - 2 - 10 - 10 */ in ZSTD_decodeLiteralsBlock()
154 /* 2 - 2 - 14 - 14 */ in ZSTD_decodeLiteralsBlock()
160 /* 2 - 2 - 18 - 18 */ in ZSTD_decodeLiteralsBlock()
173 if (dctx->ddictIsCold && (litSize > 768 /* heuristic */)) { in ZSTD_decodeLiteralsBlock()
174 PREFETCH_AREA(dctx->HUFptr, sizeof(dctx->entropy.hufTable)); in ZSTD_decodeLiteralsBlock()
180 dctx->litBuffer, litSize, istart+lhSize, litCSize, in ZSTD_decodeLiteralsBlock()
181 dctx->HUFptr, ZSTD_DCtx_get_bmi2(dctx)); in ZSTD_decodeLiteralsBlock()
184 dctx->litBuffer, litSize, istart+lhSize, litCSize, in ZSTD_decodeLiteralsBlock()
185 dctx->HUFptr, ZSTD_DCtx_get_bmi2(dctx)); in ZSTD_decodeLiteralsBlock()
191 dctx->entropy.hufTable, dctx->litBuffer, litSize, in ZSTD_decodeLiteralsBlock()
192 istart+lhSize, litCSize, dctx->workspace, in ZSTD_decodeLiteralsBlock()
193 sizeof(dctx->workspace)); in ZSTD_decodeLiteralsBlock()
196 dctx->entropy.hufTable, dctx->litBuffer, litSize, in ZSTD_decodeLiteralsBlock()
197 istart+lhSize, litCSize, dctx->workspace, in ZSTD_decodeLiteralsBlock()
198 sizeof(dctx->workspace), ZSTD_DCtx_get_bmi2(dctx)); in ZSTD_decodeLiteralsBlock()
202 dctx->entropy.hufTable, dctx->litBuffer, litSize, in ZSTD_decodeLiteralsBlock()
203 istart+lhSize, litCSize, dctx->workspace, in ZSTD_decodeLiteralsBlock()
204 sizeof(dctx->workspace), ZSTD_DCtx_get_bmi2(dctx)); in ZSTD_decodeLiteralsBlock()
207 if (dctx->litBufferLocation == ZSTD_split) in ZSTD_decodeLiteralsBlock()
209 …ZSTD_memcpy(dctx->litExtraBuffer, dctx->litBufferEnd - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTR… in ZSTD_decodeLiteralsBlock()
210 …ZSTD_memmove(dctx->litBuffer + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH, dctx->litBuffer, lit… in ZSTD_decodeLiteralsBlock()
211 dctx->litBuffer += ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH; in ZSTD_decodeLiteralsBlock()
212 dctx->litBufferEnd -= WILDCOPY_OVERLENGTH; in ZSTD_decodeLiteralsBlock()
217 dctx->litPtr = dctx->litBuffer; in ZSTD_decodeLiteralsBlock()
218 dctx->litSize = litSize; in ZSTD_decodeLiteralsBlock()
219 dctx->litEntropy = 1; in ZSTD_decodeLiteralsBlock()
220 if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable; in ZSTD_decodeLiteralsBlock()
249 if (dctx->litBufferLocation == ZSTD_split) in ZSTD_decodeLiteralsBlock()
251 … ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize - ZSTD_LITBUFFEREXTRASIZE); in ZSTD_decodeLiteralsBlock()
252 …ZSTD_memcpy(dctx->litExtraBuffer, istart + lhSize + litSize - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUF… in ZSTD_decodeLiteralsBlock()
256 ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize); in ZSTD_decodeLiteralsBlock()
258 dctx->litPtr = dctx->litBuffer; in ZSTD_decodeLiteralsBlock()
259 dctx->litSize = litSize; in ZSTD_decodeLiteralsBlock()
263 dctx->litPtr = istart+lhSize; in ZSTD_decodeLiteralsBlock()
264 dctx->litSize = litSize; in ZSTD_decodeLiteralsBlock()
265 dctx->litBufferEnd = dctx->litPtr + litSize; in ZSTD_decodeLiteralsBlock()
266 dctx->litBufferLocation = ZSTD_not_in_dst; in ZSTD_decodeLiteralsBlock()
294 if (dctx->litBufferLocation == ZSTD_split) in ZSTD_decodeLiteralsBlock()
296 ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize - ZSTD_LITBUFFEREXTRASIZE); in ZSTD_decodeLiteralsBlock()
297 ZSTD_memset(dctx->litExtraBuffer, istart[lhSize], ZSTD_LITBUFFEREXTRASIZE); in ZSTD_decodeLiteralsBlock()
301 ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize); in ZSTD_decodeLiteralsBlock()
303 dctx->litPtr = dctx->litBuffer; in ZSTD_decodeLiteralsBlock()
304 dctx->litSize = litSize; in ZSTD_decodeLiteralsBlock()
314 …* These are pre-calculated FSE decoding tables using default distributions as defined in specifica…
315 …* https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#default-distributio…
317 * - start from default distributions, present in /lib/common/zstd_internal.h
318 * - generate tables normally, using ZSTD_buildFSETable()
319 * - printout the content of tables
320 * - pretify output, report below, test with fuzzer to ensure it's correct */
329 { 0, 0, 5, 7}, { 0, 0, 5, 9},
346 { 32, 7, 5, 128}, { 0, 9, 6, 512},
351 { 32, 0, 5, 9}, { 32, 0, 5, 11},
365 { 0, 9, 5, 509}, { 0, 15, 5,32765},
389 { 0, 0, 5, 8}, { 0, 0, 5, 9},
397 { 0, 7, 6, 131}, { 0, 9, 6, 515},
400 { 32, 0, 5, 9}, { 0, 0, 5, 10},
428 DTableH->tableLog = 0; in ZSTD_buildSeqTable_rle()
429 DTableH->fastMode = 0; in ZSTD_buildSeqTable_rle()
431 cell->nbBits = 0; in ZSTD_buildSeqTable_rle()
432 cell->nextState = 0; in ZSTD_buildSeqTable_rle()
434 cell->nbAdditionalBits = nbAddBits; in ZSTD_buildSeqTable_rle()
435 cell->baseValue = baseValue; in ZSTD_buildSeqTable_rle()
455 U32 highThreshold = tableSize - 1; in ZSTD_buildFSETable_body()
467 { S16 const largeLimit= (S16)(1 << (tableLog-1)); in ZSTD_buildFSETable_body()
470 if (normalizedCounter[s]==-1) { in ZSTD_buildFSETable_body()
471 tableDecode[highThreshold--].baseValue = s; in ZSTD_buildFSETable_body()
484 * no low probability (-1 count) symbols. When compressing in ZSTD_buildFSETable_body()
488 if (highThreshold == tableSize - 1) { in ZSTD_buildFSETable_body()
489 size_t const tableMask = tableSize-1; in ZSTD_buildFSETable_body()
495 * our buffer to handle the over-write. in ZSTD_buildFSETable_body()
534 U32 const tableMask = tableSize-1; in ZSTD_buildFSETable_body()
554 tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32(nextState) ); in ZSTD_buildFSETable_body()
555 tableDecode[u].nextState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize); in ZSTD_buildFSETable_body()
679 nbSeq = ((nbSeq-0x80)<<8) + *ip++; in ZSTD_decodeSeqHeaders()
692 { size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr, in ZSTD_decodeSeqHeaders()
694 ip, iend-ip, in ZSTD_decodeSeqHeaders()
696 LL_defaultDTable, dctx->fseEntropy, in ZSTD_decodeSeqHeaders()
697 dctx->ddictIsCold, nbSeq, in ZSTD_decodeSeqHeaders()
698 dctx->workspace, sizeof(dctx->workspace), in ZSTD_decodeSeqHeaders()
704 { size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr, in ZSTD_decodeSeqHeaders()
706 ip, iend-ip, in ZSTD_decodeSeqHeaders()
708 OF_defaultDTable, dctx->fseEntropy, in ZSTD_decodeSeqHeaders()
709 dctx->ddictIsCold, nbSeq, in ZSTD_decodeSeqHeaders()
710 dctx->workspace, sizeof(dctx->workspace), in ZSTD_decodeSeqHeaders()
716 { size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr, in ZSTD_decodeSeqHeaders()
718 ip, iend-ip, in ZSTD_decodeSeqHeaders()
720 ML_defaultDTable, dctx->fseEntropy, in ZSTD_decodeSeqHeaders()
721 dctx->ddictIsCold, nbSeq, in ZSTD_decodeSeqHeaders()
722 dctx->workspace, sizeof(dctx->workspace), in ZSTD_decodeSeqHeaders()
729 return ip-istart; in ZSTD_decodeSeqHeaders()
757 * Postcondition: *op - *op >= 8
764 static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */ in ZSTD_overlapCopy8()
772 *ip -= sub2; in ZSTD_overlapCopy8()
778 assert(*op - *ip >= 8); in ZSTD_overlapCopy8()
788 …* - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VEC…
789 * - ZSTD_overlap_src_before_dst: The src and dst may overlap and may be any distance apart.
793 ptrdiff_t const diff = op - ip; in ZSTD_safecopy()
796 assert((ovtype == ZSTD_no_overlap && (diff <= -8 || diff >= 8 || op >= oend_w)) || in ZSTD_safecopy()
808 length -= 8; in ZSTD_safecopy()
809 assert(op - ip >= 8); in ZSTD_safecopy()
821 ZSTD_wildcopy(op, ip, oend_w - op, ovtype); in ZSTD_safecopy()
822 ip += oend_w - op; in ZSTD_safecopy()
823 op += oend_w - op; in ZSTD_safecopy()
830 …* This version allows overlap with dst before src, or handles the non-overlap case with dst after …
833 ptrdiff_t const diff = op - ip; in ZSTD_safecopyDstBeforeSrc()
836 if (length < 8 || diff > -8) { in ZSTD_safecopyDstBeforeSrc()
842 if (op <= oend - WILDCOPY_OVERLENGTH && diff < -WILDCOPY_VECLEN) { in ZSTD_safecopyDstBeforeSrc()
843 ZSTD_wildcopy(op, ip, oend - WILDCOPY_OVERLENGTH - op, ZSTD_no_overlap); in ZSTD_safecopyDstBeforeSrc()
844 ip += oend - WILDCOPY_OVERLENGTH - op; in ZSTD_safecopyDstBeforeSrc()
845 op += oend - WILDCOPY_OVERLENGTH - op; in ZSTD_safecopyDstBeforeSrc()
869 const BYTE* match = oLitEnd - sequence.offset; in ZSTD_execSequenceEnd()
870 BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; in ZSTD_execSequenceEnd()
872 /* bounds checks : careful of address space overflow in 32-bit mode */ in ZSTD_execSequenceEnd()
873 …RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit withi… in ZSTD_execSequenceEnd()
874 …RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to re… in ZSTD_execSequenceEnd()
884 if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { in ZSTD_execSequenceEnd()
886 … RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); in ZSTD_execSequenceEnd()
887 match = dictEnd - (prefixStart - match); in ZSTD_execSequenceEnd()
893 { size_t const length1 = dictEnd - match; in ZSTD_execSequenceEnd()
896 sequence.matchLength -= length1; in ZSTD_execSequenceEnd()
916 const BYTE* match = oLitEnd - sequence.offset; in ZSTD_execSequenceEndSplitLitBuffer()
919 /* bounds checks : careful of address space overflow in 32-bit mode */ in ZSTD_execSequenceEndSplitLitBuffer()
920 …RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit withi… in ZSTD_execSequenceEndSplitLitBuffer()
921 …RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to re… in ZSTD_execSequenceEndSplitLitBuffer()
932 if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { in ZSTD_execSequenceEndSplitLitBuffer()
934 … RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); in ZSTD_execSequenceEndSplitLitBuffer()
935 match = dictEnd - (prefixStart - match); in ZSTD_execSequenceEndSplitLitBuffer()
941 { size_t const length1 = dictEnd - match; in ZSTD_execSequenceEndSplitLitBuffer()
944 sequence.matchLength -= length1; in ZSTD_execSequenceEndSplitLitBuffer()
960 BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ in ZSTD_execSequence()
961 …BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; /* risk : address space underflow on oend=NULL … in ZSTD_execSequence()
963 const BYTE* match = oLitEnd - sequence.offset; in ZSTD_execSequence()
968 * - Read beyond end of literals in ZSTD_execSequence()
969 * - Match end is within WILDCOPY_OVERLIMIT of oend in ZSTD_execSequence()
970 * - 32-bit mode and the match length overflows in ZSTD_execSequence()
975 (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) in ZSTD_execSequence()
980 assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */); in ZSTD_execSequence()
987 * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9. in ZSTD_execSequence()
988 * We likely don't need the full 32-byte wildcopy. in ZSTD_execSequence()
993 ZSTD_wildcopy(op + 16, (*litPtr) + 16, sequence.litLength - 16, ZSTD_no_overlap); in ZSTD_execSequence()
999 if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { in ZSTD_execSequence()
1000 /* offset beyond prefix -> go into extDict */ in ZSTD_execSequence()
1001 …RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected,… in ZSTD_execSequence()
1002 match = dictEnd + (match - prefixStart); in ZSTD_execSequence()
1008 { size_t const length1 = dictEnd - match; in ZSTD_execSequence()
1011 sequence.matchLength -= length1; in ZSTD_execSequence()
1040 ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8, ZSTD_overlap_src_before_dst); in ZSTD_execSequence()
1053 BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ in ZSTD_execSequenceSplitLitBuffer()
1055 const BYTE* match = oLitEnd - sequence.offset; in ZSTD_execSequenceSplitLitBuffer()
1060 * - Read beyond end of literals in ZSTD_execSequenceSplitLitBuffer()
1061 * - Match end is within WILDCOPY_OVERLIMIT of oend in ZSTD_execSequenceSplitLitBuffer()
1062 * - 32-bit mode and the match length overflows in ZSTD_execSequenceSplitLitBuffer()
1067 (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) in ZSTD_execSequenceSplitLitBuffer()
1072 assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */); in ZSTD_execSequenceSplitLitBuffer()
1079 * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9. in ZSTD_execSequenceSplitLitBuffer()
1080 * We likely don't need the full 32-byte wildcopy. in ZSTD_execSequenceSplitLitBuffer()
1085 ZSTD_wildcopy(op+16, (*litPtr)+16, sequence.litLength-16, ZSTD_no_overlap); in ZSTD_execSequenceSplitLitBuffer()
1091 if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { in ZSTD_execSequenceSplitLitBuffer()
1092 /* offset beyond prefix -> go into extDict */ in ZSTD_execSequenceSplitLitBuffer()
1093 …RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected,… in ZSTD_execSequenceSplitLitBuffer()
1094 match = dictEnd + (match - prefixStart); in ZSTD_execSequenceSplitLitBuffer()
1100 { size_t const length1 = dictEnd - match; in ZSTD_execSequenceSplitLitBuffer()
1103 sequence.matchLength -= length1; in ZSTD_execSequenceSplitLitBuffer()
1131 ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); in ZSTD_execSequenceSplitLitBuffer()
1142 DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); in ZSTD_initFseState()
1144 (U32)DStatePtr->state, DTableH->tableLog); in ZSTD_initFseState()
1146 DStatePtr->table = dt + 1; in ZSTD_initFseState()
1153 DStatePtr->state = nextState + lowBits; in ZSTD_updateFseStateWithDInfo()
1156 /* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum
1157 * offset bits. But we can only read at most (STREAM_ACCUMULATOR_MIN_32 - 1)
1163 ? ZSTD_WINDOWLOG_MAX_32 - STREAM_ACCUMULATOR_MIN_32 \
1172 const ZSTD_seqSymbol* const llDInfo = seqState->stateLL.table + seqState->stateLL.state;
1173 const ZSTD_seqSymbol* const mlDInfo = seqState->stateML.table + seqState->stateML.state;
1174 const ZSTD_seqSymbol* const ofDInfo = seqState->stateOffb.table + seqState->stateOffb.state;
1175 seq.matchLength = mlDInfo->baseValue;
1176 seq.litLength = llDInfo->baseValue;
1177 { U32 const ofBase = ofDInfo->baseValue;
1178 BYTE const llBits = llDInfo->nbAdditionalBits;
1179 BYTE const mlBits = mlDInfo->nbAdditionalBits;
1180 BYTE const ofBits = ofDInfo->nbAdditionalBits;
1183 U16 const llNext = llDInfo->nextState;
1184 U16 const mlNext = mlDInfo->nextState;
1185 U16 const ofNext = ofDInfo->nextState;
1186 U32 const llnbBits = llDInfo->nbBits;
1187 U32 const mlnbBits = mlDInfo->nbBits;
1188 U32 const ofnbBits = ofDInfo->nbBits;
1191 * valuable to mark likelyness for clang, it gives around 3-4% of
1206 U32 const extraBits = ofBits - MIN(ofBits, 32 - seqState->DStream.bitsConsumed);
1207 … offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits);
1208 BIT_reloadDStream(&seqState->DStream);
1209 if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits);
1212 …offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX…
1213 if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream);
1215 seqState->prevOffset[2] = seqState->prevOffset[1];
1216 seqState->prevOffset[1] = seqState->prevOffset[0];
1217 seqState->prevOffset[0] = offset;
1219 U32 const ll0 = (llDInfo->baseValue == 0);
1221 offset = seqState->prevOffset[ll0];
1222 seqState->prevOffset[1] = seqState->prevOffset[!ll0];
1223 seqState->prevOffset[0] = offset;
1225 offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1);
1226 … { size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset];
1228 if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1];
1229 seqState->prevOffset[1] = seqState->prevOffset[0];
1230 seqState->prevOffset[0] = offset = temp;
1240 seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/);
1242 … if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32))
1243 BIT_reloadDStream(&seqState->DStream);
1244 …if (MEM_64bits() && UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog)))
1245 BIT_reloadDStream(&seqState->DStream);
1246 /* Ensure there are enough bits to read the rest of data in 64-bit mode. */
1254 seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/);
1257 BIT_reloadDStream(&seqState->DStream);
1262 …ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llNext, llnbBits); /* <= …
1263 …ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlNext, mlnbBits); /* <= …
1264 if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */
1265 …ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofNext, ofnbBits); /* <= …
1274 size_t const windowSize = dctx->fParams.windowSize;
1276 if (dctx->dictContentEndForFuzzing == NULL) return 0;
1278 if (prefixStart == dctx->dictContentBeginForFuzzing) return 1;
1279 /* Dictionary is not our ext-dict. */
1280 if (dctx->dictEnd != dctx->dictContentEndForFuzzing) return 0;
1282 if ((size_t)(oLitEnd - prefixStart) >= windowSize) return 0;
1294 size_t const windowSize = dctx->fParams.windowSize;
1300 assert((size_t)(oend - op) >= sequenceSize);
1303 …size_t const dictSize = (size_t)((char const*)dctx->dictContentEndForFuzzing - (char const*)dctx->…
1305 assert(seq.offset <= (size_t)(oLitEnd - virtualStart));
1333 const BYTE* litPtr = dctx->litPtr;
1334 const BYTE* litBufferEnd = dctx->litBufferEnd;
1335 const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart);
1336 const BYTE* const vBase = (const BYTE*) (dctx->virtualStart);
1337 const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
1344 dctx->fseEntropy = 1;
1345 { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
1347 ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)),
1349 ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
1350 ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
1351 ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
1364 * zstd compiled with gcc-9 on an Intel i9-9900k shows 10% decompression
1372 * perf stat -e cycles -e instructions -e idq.all_dsb_cycles_any_uops \
1373 * -e idq.all_mite_cycles_any_uops -- ./zstd -tq test.zst
1380 * - Kabylake: Macbook Pro (15-inch, 2019) 2.4 GHz Intel Core i9
1381 * Use Instruments->Counters to get DSB/MITE cycles.
1385 * - Coffeelake: Intel i9-9900k
1386 * - Coffeelake: Intel i7-9700k
1390 * - Haswell
1391 * - Broadwell: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GH
1392 * - Skylake
1395 … * - ZSTD_decompressSequences_bodySplitLitBuffer - presplit section of the literal buffer
1396 … * - ZSTD_decompressSequences_bodySplitLitBuffer - postsplit section of the literal buffer
1397 * - ZSTD_decompressSequences_body
1409 /* good for gcc-7, gcc-9, and gcc-11 */
1415 /* good for gcc-8 and gcc-10 */
1423 for (; litPtr + sequence.litLength <= dctx->litBufferEnd; ) {
1424 …ize = ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence.litLength - WILDCOPY_OVERLENGTH,…
1433 if (UNLIKELY(!--nbSeq))
1441 const size_t leftoverLit = dctx->litBufferEnd - litPtr;
1444 …RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit withi…
1446 sequence.litLength -= leftoverLit;
1449 litPtr = dctx->litExtraBuffer;
1450 litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
1451 dctx->litBufferLocation = ZSTD_not_in_dst;
1462 if (--nbSeq)
1475 /* worse for gcc-7 better for gcc-8, gcc-9, and gcc-10 and clang */
1499 if (UNLIKELY(!--nbSeq))
1510 … { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
1514 …if (dctx->litBufferLocation == ZSTD_split) /* split hasn't been reached yet, first get dst then c…
1516 size_t const lastLLSize = litBufferEnd - litPtr;
1517 RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, "");
1522 litPtr = dctx->litExtraBuffer;
1523 litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
1524 dctx->litBufferLocation = ZSTD_not_in_dst;
1526 { size_t const lastLLSize = litBufferEnd - litPtr;
1527 RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, "");
1534 return op-ostart;
1548 …BYTE* const oend = dctx->litBufferLocation == ZSTD_not_in_dst ? ostart + maxDstSize : dctx->litBuf…
1550 const BYTE* litPtr = dctx->litPtr;
1551 const BYTE* const litEnd = litPtr + dctx->litSize;
1552 const BYTE* const prefixStart = (const BYTE*)(dctx->prefixStart);
1553 const BYTE* const vBase = (const BYTE*)(dctx->virtualStart);
1554 const BYTE* const dictEnd = (const BYTE*)(dctx->dictEnd);
1561 dctx->fseEntropy = 1;
1562 { U32 i; for (i = 0; i < ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
1564 ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend - ip)),
1566 ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
1567 ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
1568 ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
1601 if (UNLIKELY(!--nbSeq))
1611 … { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
1615 { size_t const lastLLSize = litEnd - litPtr;
1616 RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, "");
1623 return op-ostart;
1655 …const BYTE* const match = matchBase + prefetchPos - sequence.offset; /* note : this operation can …
1664 * It's generally employed when block contains a significant portion of long-distance matches
1677 … BYTE* const oend = dctx->litBufferLocation == ZSTD_in_dst ? dctx->litBuffer : ostart + maxDstSize;
1679 const BYTE* litPtr = dctx->litPtr;
1680 const BYTE* litBufferEnd = dctx->litBufferEnd;
1681 const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart);
1682 const BYTE* const dictStart = (const BYTE*) (dctx->virtualStart);
1683 const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
1689 #define STORED_SEQS_MASK (STORED_SEQS-1)
1695 size_t prefetchPos = (size_t)(op-prefixStart); /* track position relative to prefixStart */
1697 dctx->fseEntropy = 1;
1698 { int i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
1702 ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)),
1704 ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
1705 ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
1706 ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
1721 …if (dctx->litBufferLocation == ZSTD_split && litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_S…
1724 const size_t leftoverLit = dctx->litBufferEnd - litPtr;
1727 …RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit withi…
1729 sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength -= leftoverLit;
1732 litPtr = dctx->litExtraBuffer;
1733 litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
1734 dctx->litBufferLocation = ZSTD_not_in_dst;
1735 …oneSeqSize = ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &l…
1738 …if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQ…
1749 oneSeqSize = dctx->litBufferLocation == ZSTD_split ?
1750 …oend, litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength - WILDCOPY_OVERLENG…
1751 …ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBuf…
1754 …if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQ…
1766 seqNb -= seqAdvance;
1769 … if (dctx->litBufferLocation == ZSTD_split && litPtr + sequence->litLength > dctx->litBufferEnd)
1771 const size_t leftoverLit = dctx->litBufferEnd - litPtr;
1774 …RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit withi…
1776 sequence->litLength -= leftoverLit;
1779 litPtr = dctx->litExtraBuffer;
1780 litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
1781 dctx->litBufferLocation = ZSTD_not_in_dst;
1794 size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ?
1795 …ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence->litLength - WILDCOPY_OVERLENGTH, *seq…
1807 … { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
1811 …if (dctx->litBufferLocation == ZSTD_split) /* first deplete literal buffer in dst, then copy litE…
1813 size_t const lastLLSize = litBufferEnd - litPtr;
1814 RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, "");
1819 litPtr = dctx->litExtraBuffer;
1820 litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
1822 { size_t const lastLLSize = litBufferEnd - litPtr;
1823 RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, "");
1830 return op-ostart;
1971 total <<= (OffFSELog - tableLog); /* scale to OffFSELog */
1985 * We don't expect that to be the case in 64-bit mode.
1987 * (note: but it could be evaluated from current-lowLimit)
1989 …ZSTD_longOffset_e const isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (!frame || (dctx->fPara…
1999 srcSize -= litCSize;
2004 /* These macros control at build-time which decompressor implementation
2010 int usePrefetchDecoder = dctx->ddictIsCold;
2016 srcSize -= seqHSize;
2023 && (!frame || (dctx->fParams.windowSize > (1<<24)))
2025 U32 const shareLongOffsets = ZSTD_getLongOffsetsShare(dctx->OFTptr);
2031 dctx->ddictIsCold = 0;
2043 if (dctx->litBufferLocation == ZSTD_split)
2054 if (dst != dctx->previousDstEnd && dstSize > 0) { /* not contiguous */
2055 dctx->dictEnd = dctx->previousDstEnd;
2056 …dctx->virtualStart = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx-…
2057 dctx->prefixStart = dst;
2058 dctx->previousDstEnd = dst;
2070 dctx->previousDstEnd = (char*)dst + dSize;