packages/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

!59 zstd 1.5.0 update to 1.5.2

Browse Source 
From: @zhoupengcheng11 
Reviewed-by: @lvying6 
Signed-off-by: @lvying6
This commit is contained in:
openeuler-ci-bot 2023-02-02 09:35:19 +00:00 committed by Gitee
commit e08a183510
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
17 changed files with 97 additions and 1157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

154
patch-9-add-test-c-result-print.patch → add-test-c-result-print.patch
View File

@ -1,7 +1,25 @@
diff -Nur zstd-1.5.0/tests/decodecorpus.c zstd-1.5.0-new/tests/decodecorpus.c
--- zstd-1.5.0/tests/decodecorpus.c 2021-05-14 22:59:34.000000000 +0800
+++ zstd-1.5.0-new/tests/decodecorpus.c 2021-12-09 14:36:53.425004688 +0800
@@ -1541,6 +1541,7 @@
From 7409f10fed50978ad5dff36540dc0a7aae688688 Mon Sep 17 00:00:00 2001
From: zhangying <zhangying103@huawei.com>
Date: Thu, 2 Feb 2023 14:59:55 +0800
Subject: [PATCH] zstd-add-test-c-result-print.patch
Displays statistics about the execution of all zstd test cases.
---
tests/decodecorpus.c | 17 +++++++++--
tests/fullbench.c | 61 ++++++++++++++++++++++++++++++++++---
tests/fuzzer.c | 16 ++++++++++
tests/invalidDictionaries.c | 10 ++++++
tests/legacy.c | 10 ++++--
tests/playTests.sh | 2 ++
tests/poolTests.c | 24 ++++++++++++---
tests/zstreamtest.c | 18 +++++++++++
8 files changed, 144 insertions(+), 14 deletions(-)
diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c
index 1037a36..5d053aa 100644
--- a/tests/decodecorpus.c
+++ b/tests/decodecorpus.c
@@ -1538,6 +1538,7 @@ static int runTestMode(U32 seed, unsigned numFiles, unsigned const testDurationS
if (numFiles == 0 && !testDurationS) numFiles = 1;
@ -9,7 +27,7 @@ diff -Nur zstd-1.5.0/tests/decodecorpus.c zstd-1.5.0-new/tests/decodecorpus.c
DISPLAY("seed: %u\n", (unsigned)seed);
for (fnum = 0; fnum < numFiles || UTIL_clockSpanMicro(startClock) < maxClockSpan; fnum++) {
@@ -1552,12 +1553,13 @@
@@ -1549,12 +1550,13 @@ static int runTestMode(U32 seed, unsigned numFiles, unsigned const testDurationS
{ int const ret = (genType == gt_frame) ?
runFrameTest(&seed) :
runBlockTest(&seed);
@ -24,7 +42,7 @@ diff -Nur zstd-1.5.0/tests/decodecorpus.c zstd-1.5.0-new/tests/decodecorpus.c
return 0;
}
@@ -1571,6 +1573,7 @@
@@ -1568,6 +1570,7 @@ static int generateFile(U32 seed, const char* const path,
{
frame_t fr;
@ -32,7 +50,7 @@ diff -Nur zstd-1.5.0/tests/decodecorpus.c zstd-1.5.0-new/tests/decodecorpus.c
DISPLAY("seed: %u\n", (unsigned)seed);
{ dictInfo const info = initDictInfo(0, 0, NULL, 0);
@@ -1584,6 +1587,7 @@
@@ -1581,6 +1584,7 @@ static int generateFile(U32 seed, const char* const path,
if (origPath) {
outputBuffer(fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart, origPath);
}
@ -40,7 +58,7 @@ diff -Nur zstd-1.5.0/tests/decodecorpus.c zstd-1.5.0-new/tests/decodecorpus.c
return 0;
}
@@ -1593,6 +1597,7 @@
@@ -1590,6 +1594,7 @@ static int generateCorpus(U32 seed, unsigned numFiles, const char* const path,
char outPath[MAX_PATH];
unsigned fnum;
@ -48,7 +66,7 @@ diff -Nur zstd-1.5.0/tests/decodecorpus.c zstd-1.5.0-new/tests/decodecorpus.c
DISPLAY("seed: %u\n", (unsigned)seed);
for (fnum = 0; fnum < numFiles; fnum++) {
@@ -1610,6 +1615,7 @@
@@ -1607,6 +1612,7 @@ static int generateCorpus(U32 seed, unsigned numFiles, const char* const path,
if (snprintf(outPath, MAX_PATH, "%s/z%06u.zst", path, fnum) + 1 > MAX_PATH) {
DISPLAY("Error: path too long\n");
@ -56,7 +74,7 @@ diff -Nur zstd-1.5.0/tests/decodecorpus.c zstd-1.5.0-new/tests/decodecorpus.c
return 1;
}
outputBuffer(fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, outPath);
@@ -1617,6 +1623,7 @@
@@ -1614,6 +1620,7 @@ static int generateCorpus(U32 seed, unsigned numFiles, const char* const path,
if (origPath) {
if (snprintf(outPath, MAX_PATH, "%s/z%06u", origPath, fnum) + 1 > MAX_PATH) {
DISPLAY("Error: path too long\n");
@ -64,7 +82,7 @@ diff -Nur zstd-1.5.0/tests/decodecorpus.c zstd-1.5.0-new/tests/decodecorpus.c
return 1;
}
outputBuffer(fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart, outPath);
@@ -1624,7 +1631,7 @@
@@ -1621,7 +1628,7 @@ static int generateCorpus(U32 seed, unsigned numFiles, const char* const path,
}
DISPLAY("\r%u/%u \n", fnum, numFiles);
@ -73,7 +91,7 @@ diff -Nur zstd-1.5.0/tests/decodecorpus.c zstd-1.5.0-new/tests/decodecorpus.c
return 0;
}
@@ -1637,8 +1644,11 @@
@@ -1634,8 +1641,11 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const
U32 const dictID = RAND(&seed);
int errorDetected = 0;
@ -85,7 +103,7 @@ diff -Nur zstd-1.5.0/tests/decodecorpus.c zstd-1.5.0-new/tests/decodecorpus.c
return 1;
}
@@ -1646,6 +1656,7 @@
@@ -1643,6 +1653,7 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const
fullDict = malloc(dictSize);
if (fullDict == NULL) {
DISPLAY("Error: could not allocate space for full dictionary.\n");
@ -93,7 +111,7 @@ diff -Nur zstd-1.5.0/tests/decodecorpus.c zstd-1.5.0-new/tests/decodecorpus.c
return 1;
}
@@ -1714,6 +1725,8 @@
@@ -1711,6 +1722,8 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const
}
dictCleanup:
@ -102,10 +120,11 @@ diff -Nur zstd-1.5.0/tests/decodecorpus.c zstd-1.5.0-new/tests/decodecorpus.c
free(fullDict);
return errorDetected;
}
diff -Nur zstd-1.5.0/tests/fullbench.c zstd-1.5.0-new/tests/fullbench.c
--- zstd-1.5.0/tests/fullbench.c 2021-05-14 22:59:34.000000000 +0800
+++ zstd-1.5.0-new/tests/fullbench.c 2021-12-09 16:32:14.897179654 +0800
@@ -474,7 +474,7 @@
diff --git a/tests/fullbench.c b/tests/fullbench.c
index b55ff76..3a6ce2b 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -500,7 +500,7 @@ static int benchMem(unsigned benchNb,
benchFunction = local_ZSTD_compress_generic_T2_end; benchName = "compress_generic, -T2, end";
break;
default :
@ -114,7 +133,7 @@ diff -Nur zstd-1.5.0/tests/fullbench.c zstd-1.5.0-new/tests/fullbench.c
}
/* Allocation */
@@ -672,8 +672,15 @@
@@ -700,8 +700,15 @@ static int benchSample(U32 benchNb,
size_t benchedSize, double compressibility,
int cLevel, ZSTD_compressionParameters cparams)
{
@ -130,7 +149,7 @@ diff -Nur zstd-1.5.0/tests/fullbench.c zstd-1.5.0-new/tests/fullbench.c
if (!origBuff) { DISPLAY("\nError: not enough memory!\n"); return 12; }
/* Fill buffer */
@@ -683,12 +690,31 @@
@@ -711,12 +718,31 @@ static int benchSample(U32 benchNb,
DISPLAY("\r%70s\r", "");
DISPLAY(" Sample %u bytes : \n", (unsigned)benchedSize);
if (benchNb) {
@ -164,7 +183,7 @@ diff -Nur zstd-1.5.0/tests/fullbench.c zstd-1.5.0-new/tests/fullbench.c
free(origBuff);
return 0;
}
@@ -700,6 +726,12 @@
@@ -728,6 +754,12 @@ static int benchFiles(U32 benchNb,
{
/* Loop for each file */
int fileIdx;
@ -177,7 +196,7 @@ diff -Nur zstd-1.5.0/tests/fullbench.c zstd-1.5.0-new/tests/fullbench.c
for (fileIdx=0; fileIdx<nbFiles; fileIdx++) {
const char* const inFileName = fileNamesTable[fileIdx];
FILE* const inFile = fopen( inFileName, "rb" );
@@ -741,15 +773,34 @@
@@ -769,15 +801,34 @@ static int benchFiles(U32 benchNb,
DISPLAY("\r%70s\r", ""); /* blank line */
DISPLAY(" %s : \n", inFileName);
if (benchNb) {
@ -214,10 +233,11 @@ diff -Nur zstd-1.5.0/tests/fullbench.c zstd-1.5.0-new/tests/fullbench.c
return 0;
}
diff -Nur zstd-1.5.0/tests/fuzzer.c zstd-1.5.0-new/tests/fuzzer.c
--- zstd-1.5.0/tests/fuzzer.c 2021-05-14 22:59:34.000000000 +0800
+++ zstd-1.5.0-new/tests/fuzzer.c 2021-12-09 09:48:15.108283586 +0800
@@ -472,6 +472,8 @@
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index d168d65..f6460d3 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -472,6 +472,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
unsigned testNb=0;
size_t cSize;
@ -226,7 +246,7 @@ diff -Nur zstd-1.5.0/tests/fuzzer.c zstd-1.5.0-new/tests/fuzzer.c
/* Create compressible noise */
if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
DISPLAY("Not enough memory, aborting\n");
@@ -3426,6 +3428,9 @@
@@ -3509,6 +3511,9 @@ static int basicUnitTests(U32 const seed, double compressibility)
#endif /* ZSTD_MULTITHREAD */
_end:
@ -236,7 +256,7 @@ diff -Nur zstd-1.5.0/tests/fuzzer.c zstd-1.5.0-new/tests/fuzzer.c
free(CNBuffer);
free(compressedBuffer);
free(decodedBuffer);
@@ -3448,6 +3453,8 @@
@@ -3531,6 +3536,8 @@ static int longUnitTests(U32 const seed, double compressibility)
unsigned testNb=0;
size_t cSize;
@ -245,7 +265,7 @@ diff -Nur zstd-1.5.0/tests/fuzzer.c zstd-1.5.0-new/tests/fuzzer.c
/* Create compressible noise */
if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
DISPLAY("Not enough memory, aborting\n");
@@ -3626,6 +3633,10 @@
@@ -3708,6 +3715,10 @@ static int longUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "OK \n");
_end:
@ -256,7 +276,7 @@ diff -Nur zstd-1.5.0/tests/fuzzer.c zstd-1.5.0-new/tests/fuzzer.c
free(CNBuffer);
free(compressedBuffer);
free(decodedBuffer);
@@ -3711,6 +3722,8 @@
@@ -3793,6 +3804,8 @@ static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, U32 const
U64 const maxClockSpan = maxDurationS * SEC_TO_MICRO;
int const cLevelLimiter = bigTests ? 3 : 2;
@ -265,7 +285,7 @@ diff -Nur zstd-1.5.0/tests/fuzzer.c zstd-1.5.0-new/tests/fuzzer.c
/* allocation */
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
@@ -3974,6 +3987,9 @@
@@ -4056,6 +4069,9 @@ static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, U32 const
DISPLAY("\r%u fuzzer tests completed \n", testNb-1);
_cleanup:
@ -275,9 +295,10 @@ diff -Nur zstd-1.5.0/tests/fuzzer.c zstd-1.5.0-new/tests/fuzzer.c
ZSTD_freeCCtx(refCtx);
ZSTD_freeCCtx(ctx);
ZSTD_freeDCtx(dctx);
diff -Nur zstd-1.5.0/tests/invalidDictionaries.c zstd-1.5.0-new/tests/invalidDictionaries.c
--- zstd-1.5.0/tests/invalidDictionaries.c 2021-05-14 22:59:34.000000000 +0800
+++ zstd-1.5.0-new/tests/invalidDictionaries.c 2021-12-09 10:44:46.819104816 +0800
diff --git a/tests/invalidDictionaries.c b/tests/invalidDictionaries.c
index b71f741..bb2b536 100644
--- a/tests/invalidDictionaries.c
+++ b/tests/invalidDictionaries.c
@@ -9,6 +9,7 @@
*/
@ -286,7 +307,7 @@ diff -Nur zstd-1.5.0/tests/invalidDictionaries.c zstd-1.5.0-new/tests/invalidDic
#include "zstd.h"
static const char invalidRepCode[] = {
@@ -38,23 +39,32 @@
@@ -38,23 +39,32 @@ static const dictionary dictionaries[] = {
{NULL, 0},
};
@ -319,10 +340,11 @@ diff -Nur zstd-1.5.0/tests/invalidDictionaries.c zstd-1.5.0-new/tests/invalidDic
(void)argc;
(void)argv;
return 0;
diff -Nur zstd-1.5.0/tests/legacy.c zstd-1.5.0-new/tests/legacy.c
--- zstd-1.5.0/tests/legacy.c 2021-05-14 22:59:34.000000000 +0800
+++ zstd-1.5.0-new/tests/legacy.c 2021-12-09 11:05:49.790480128 +0800
@@ -160,14 +160,18 @@
diff --git a/tests/legacy.c b/tests/legacy.c
index ac4938f..5ebb1cc 100644
--- a/tests/legacy.c
+++ b/tests/legacy.c
@@ -160,14 +160,18 @@ static int testFrameDecoding(void)
int main(void)
{
@ -344,18 +366,20 @@ diff -Nur zstd-1.5.0/tests/legacy.c zstd-1.5.0-new/tests/legacy.c
return 0;
}
diff -Nur zstd-1.5.0/tests/playTests.sh zstd-1.5.0-new/tests/playTests.sh
--- zstd-1.5.0/tests/playTests.sh 2021-05-14 22:59:34.000000000 +0800
+++ zstd-1.5.0-new/tests/playTests.sh 2021-12-13 15:05:29.027885006 +0800
@@ -1664,3 +1664,5 @@
diff --git a/tests/playTests.sh b/tests/playTests.sh
index b7a3d88..c152162 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -1743,3 +1743,5 @@ test -f dictionary
rm -f tmp* dictionary
rm -f tmp*
+
+println "all zstd cli tests passed"
diff -Nur zstd-1.5.0/tests/poolTests.c zstd-1.5.0-new/tests/poolTests.c
--- zstd-1.5.0/tests/poolTests.c 2021-05-14 22:59:34.000000000 +0800
+++ zstd-1.5.0-new/tests/poolTests.c 2021-12-09 15:07:25.250876163 +0800
diff --git a/tests/poolTests.c b/tests/poolTests.c
index 08f31c0..972e5c9 100644
--- a/tests/poolTests.c
+++ b/tests/poolTests.c
@@ -24,6 +24,7 @@
} while (0)
#define ASSERT_FALSE(p) ASSERT_TRUE(!(p))
@ -364,7 +388,7 @@ diff -Nur zstd-1.5.0/tests/poolTests.c zstd-1.5.0-new/tests/poolTests.c
struct data {
ZSTD_pthread_mutex_t mutex;
@@ -225,11 +226,16 @@
@@ -225,11 +226,16 @@ static int testAbruptEnding(void) {
int main(int argc, const char **argv) {
size_t numThreads;
@ -377,15 +401,15 @@ diff -Nur zstd-1.5.0/tests/poolTests.c zstd-1.5.0-new/tests/poolTests.c
+
if (POOL_create(0, 1)) { /* should not be possible */
printf("FAIL: should not create POOL with 0 threads\n");
+ DISPLAY("==== results; passed: 0/1, failed: 0/1, skipped: 0/1 ====\n");
+ DISPLAY("==== results: passed: 0/1, failed: 0/1, skipped: 0/1 ====\n");
return 1;
}
@@ -240,32 +246,42 @@
@@ -240,32 +246,42 @@ int main(int argc, const char **argv) {
(unsigned)queueSize, (unsigned)numThreads);
if (testOrder(numThreads, queueSize)) {
printf("FAIL: testOrder\n");
+ DISPLAY("==== results; passed: %d/%d, failed: 1/%d, skipped: 0/%d ====\n",
+ DISPLAY("==== results: passed: %d/%d, failed: 1/%d, skipped: 0/%d ====\n",
+ testNb, (testNb+1), (testNb+1), (testNb+1));
return 1;
}
@ -393,7 +417,7 @@ diff -Nur zstd-1.5.0/tests/poolTests.c zstd-1.5.0-new/tests/poolTests.c
+ printf("SUCCESS: testOrder\n");testNb++;
if (testWait(numThreads, queueSize)) {
printf("FAIL: testWait\n");
+ DISPLAY("==== results; passed: %d/%d, failed: 1/%d, skipped: 0/%d ====\n",
+ DISPLAY("==== results: passed: %d/%d, failed: 1/%d, skipped: 0/%d ====\n",
+ testNb, (testNb+1), (testNb+1), (testNb+1));
return 1;
}
@ -404,7 +428,7 @@ diff -Nur zstd-1.5.0/tests/poolTests.c zstd-1.5.0-new/tests/poolTests.c
if (testThreadReduction()) {
printf("FAIL: thread reduction not effective \n");
+ DISPLAY("==== results; passed: %d/%d, failed: 1/%d, skipped: 0/%d ====\n",
+ DISPLAY("==== results: passed: %d/%d, failed: 1/%d, skipped: 0/%d ====\n",
+ testNb, (testNb+1), (testNb+1), (testNb+1));
return 1;
} else {
@ -414,7 +438,7 @@ diff -Nur zstd-1.5.0/tests/poolTests.c zstd-1.5.0-new/tests/poolTests.c
if (testAbruptEnding()) {
printf("FAIL: jobs in queue not completed on early end \n");
+ DISPLAY("==== results; passed: %d/%d, failed: 1/%d, skipped: 0/%d ====\n",
+ DISPLAY("==== results: passed: %d/%d, failed: 1/%d, skipped: 0/%d ====\n",
+ testNb, (testNb+1), (testNb+1), (testNb+1));
return 1;
} else {
@ -423,15 +447,16 @@ diff -Nur zstd-1.5.0/tests/poolTests.c zstd-1.5.0-new/tests/poolTests.c
}
printf("PASS: all POOL tests\n");
+ DISPLAY("==== results; passed: %d/%d, failed: 0/%d, skipped: 0/%d ====\n",
+ DISPLAY("==== results: passed: %d/%d, failed: 0/%d, skipped: 0/%d ====\n",
+ testNb, testNb, testNb, testNb);
return 0;
}
diff -Nur zstd-1.5.0/tests/zstreamtest.c zstd-1.5.0-new/tests/zstreamtest.c
--- zstd-1.5.0/tests/zstreamtest.c 2021-05-14 22:59:34.000000000 +0800
+++ zstd-1.5.0-new/tests/zstreamtest.c 2021-12-09 10:16:25.467602447 +0800
@@ -282,6 +282,8 @@
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 72fd72e..be300b6 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -282,6 +282,8 @@ static int basicUnitTests(U32 seed, double compressibility)
size_t const dictSize = 128 KB;
unsigned dictID = 0;
@ -440,7 +465,7 @@ diff -Nur zstd-1.5.0/tests/zstreamtest.c zstd-1.5.0-new/tests/zstreamtest.c
/* Create compressible test buffer */
if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd || !mtctx) {
DISPLAY("Not enough memory, aborting \n");
@@ -1691,6 +1693,10 @@
@@ -1691,6 +1693,10 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "OK \n");
_end:
@ -451,7 +476,7 @@ diff -Nur zstd-1.5.0/tests/zstreamtest.c zstd-1.5.0-new/tests/zstreamtest.c
FUZ_freeDictionary(dictionary);
ZSTD_freeCStream(zc);
ZSTD_freeDStream(zd);
@@ -1778,6 +1784,8 @@
@@ -1778,6 +1784,8 @@ static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double co
U32 oldTestLog = 0;
U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel() : g_cLevelMax_smallTests;
@ -460,7 +485,7 @@ diff -Nur zstd-1.5.0/tests/zstreamtest.c zstd-1.5.0-new/tests/zstreamtest.c
/* allocations */
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
@@ -1989,6 +1997,10 @@
@@ -1989,6 +1997,10 @@ static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double co
DISPLAY("\r%u fuzzer tests completed \n", testNb);
_cleanup:
@ -471,7 +496,7 @@ diff -Nur zstd-1.5.0/tests/zstreamtest.c zstd-1.5.0-new/tests/zstreamtest.c
ZSTD_freeCStream(zc);
ZSTD_freeDStream(zd);
ZSTD_freeDStream(zd_noise);
@@ -2049,6 +2061,8 @@
@@ -2049,6 +2061,8 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest,
U32 const nbThreadsMax = bigTests ? 4 : 2;
ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
@ -480,7 +505,7 @@ diff -Nur zstd-1.5.0/tests/zstreamtest.c zstd-1.5.0-new/tests/zstreamtest.c
/* allocations */
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
@@ -2423,6 +2437,10 @@
@@ -2423,6 +2437,10 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest,
DISPLAY("\r%u fuzzer tests completed \n", testNb-1);
_cleanup:
@ -491,3 +516,6 @@ diff -Nur zstd-1.5.0/tests/zstreamtest.c zstd-1.5.0-new/tests/zstreamtest.c
ZSTD_freeCCtx(zc);
ZSTD_freeDStream(zd);
ZSTD_freeDStream(zd_noise);
--
2.33.0

149
backport-zstd-1.5.0-patch-1-set-mtime-on-output-files.patch
View File

@ -1,149 +0,0 @@
diff -Nur zstd-1.5.0/programs/fileio.c new-zstd-1.5.0/programs/fileio.c
--- zstd-1.5.0/programs/fileio.c 2021-05-14 22:59:34.000000000 +0800
+++ new-zstd-1.5.0/programs/fileio.c 2021-10-25 10:30:36.546222652 +0800
@@ -1638,6 +1638,7 @@
int closeDstFile = 0;
int result;
stat_t statbuf;
+ int transferMTime = 0;
assert(ress.srcFile != NULL);
if (ress.dstFile == NULL) {
int dstFilePermissions = DEFAULT_FILE_PERMISSIONS;
@@ -1645,6 +1646,7 @@
&& UTIL_stat(srcFileName, &statbuf)
&& UTIL_isRegularFileStat(&statbuf) ) {
dstFilePermissions = statbuf.st_mode;
+ transferMTime = 1;
}
closeDstFile = 1;
@@ -1671,6 +1673,12 @@
DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno));
result=1;
}
+
+ if (transferMTime) {
+ UTIL_utime(dstFileName, &statbuf);
+ }
+
+
if ( (result != 0) /* operation failure */
&& strcmp(dstFileName, stdoutmark) /* special case : don't remove() stdout */
) {
@@ -2537,6 +2545,7 @@
int result;
stat_t statbuf;
int releaseDstFile = 0;
+ int transferMTime = 0;
if ((ress.dstFile == NULL) && (prefs->testMode==0)) {
int dstFilePermissions = DEFAULT_FILE_PERMISSIONS;
@@ -2544,6 +2553,7 @@
&& UTIL_stat(srcFileName, &statbuf)
&& UTIL_isRegularFileStat(&statbuf) ) {
dstFilePermissions = statbuf.st_mode;
+ transferMTime = 1;
}
releaseDstFile = 1;
@@ -2568,7 +2578,9 @@
DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno));
result = 1;
}
-
+ if (transferMTime) {
+ UTIL_utime(dstFileName, &statbuf);
+ }
if ( (result != 0) /* operation failure */
&& strcmp(dstFileName, stdoutmark) /* special case : don't remove() stdout */
) {
diff -Nur zstd-1.5.0/programs/util.c new-zstd-1.5.0/programs/util.c
--- zstd-1.5.0/programs/util.c 2021-05-14 22:59:34.000000000 +0800
+++ new-zstd-1.5.0/programs/util.c 2021-10-25 10:30:43.246350653 +0800
@@ -31,6 +31,7 @@
# if PLATFORM_POSIX_VERSION < 200809L || !defined(st_mtime)
# include <utime.h> /* utime */
# else
+# include <utime.h>
# include <fcntl.h> /* AT_FDCWD */
# include <sys/stat.h> /* utimensat */
# endif
@@ -159,6 +160,31 @@
return chmod(filename, permissions);
}
+/*set access and modification times */
+int UTIL_utime(const char* filename, const stat_t *statbuf)
+{
+ int ret;
+ /* We check that st_mtime is a macro here in order to give us confidence
+ * that struct stat has a struct timespec st_mtim member. We need this
+ * check because there are some platforms that claim to be POSIX 2008
+ * compliant but which do not have st_mtim...
+ * */
+#if (PLATFORM_POSIX_VERSION >= 200809L) && defined(st_time)
+ /* (atime, mtime)*/
+ struct timespec timebuf[2] = { {0, UTIME_NOW} };
+ timebuf[1] = statbuf->st_mtim;
+ ret = utimensat(AT_FDCWD, filename, timebuf, 0);
+#else
+ struct utimbuf timebuf;
+ timebuf.actime = time(NULL);
+ timebuf.modtime = statbuf->st_mtime;
+ ret = utime(filename, &timebuf);
+#endif
+ errno = 0;
+ return ret;
+}
+
+
int UTIL_setFileStat(const char *filename, const stat_t *statbuf)
{
int res = 0;
@@ -168,25 +194,7 @@
return -1;
/* set access and modification times */
- /* We check that st_mtime is a macro here in order to give us confidence
- * that struct stat has a struct timespec st_mtim member. We need this
- * check because there are some platforms that claim to be POSIX 2008
- * compliant but which do not have st_mtim... */
-#if (PLATFORM_POSIX_VERSION >= 200809L) && defined(st_mtime)
- {
- /* (atime, mtime) */
- struct timespec timebuf[2] = { {0, UTIME_NOW} };
- timebuf[1] = statbuf->st_mtim;
- res += utimensat(AT_FDCWD, filename, timebuf, 0);
- }
-#else
- {
- struct utimbuf timebuf;
- timebuf.actime = time(NULL);
- timebuf.modtime = statbuf->st_mtime;
- res += utime(filename, &timebuf);
- }
-#endif
+ res += UTIL_utime(filename, statbuf);
#if !defined(_WIN32)
res += chown(filename, statbuf->st_uid, statbuf->st_gid); /* Copy ownership */
diff -Nur zstd-1.5.0/programs/util.h new-zstd-1.5.0/programs/util.h
--- zstd-1.5.0/programs/util.h 2021-05-14 22:59:34.000000000 +0800
+++ new-zstd-1.5.0/programs/util.h 2021-10-25 10:30:43.246350653 +0800
@@ -135,6 +135,16 @@
*/
int UTIL_setFileStat(const char* filename, const stat_t* statbuf);
+/**
+ * Set atime to now and mtime to the st_mtim in statbuf.
+ *
+ * Directly wraps utime() or utimensat(). Returns -1 on error.
+ * Does not validate filename is valid.
+ * */
+int UTIL_utime(const char* filename, const stat_t *statbuf);
+
+
+
/*
* These helpers operate on a pre-populated stat_t, i.e., the result of
* calling one of the above functions.

53
backport-zstd-1.5.0-patch-10-fix-entropy-repeat-mode-bug.patch
View File

@ -1,53 +0,0 @@
From 923e5ad3f5573cd68f792ebad49c24ecaa0c3ad0 Mon Sep 17 00:00:00 2001
From: Sen Huang <senhuang96@fb.com>
Date: Mon, 7 Jun 2021 00:32:03 -0700
Subject: [PATCH 0721/1000] Fix entropy repeat mode bug
---
lib/compress/zstd_compress.c | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 6afecfa1..dafe3cbd 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3332,6 +3332,7 @@ static size_t ZSTD_estimateBlockSize(const BYTE* literals, size_t litSize,
*/
static size_t ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(seqStore_t* seqStore, const ZSTD_CCtx* zc) {
ZSTD_entropyCTablesMetadata_t entropyMetadata;
+ DEBUGLOG(6, "ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize()");
FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(seqStore,
&zc->blockState.prevCBlock->entropy,
&zc->blockState.nextCBlock->entropy,
@@ -3510,9 +3511,6 @@ static size_t ZSTD_compressSeqStore_singleBlock(ZSTD_CCtx* zc, seqStore_t* const
return 0;
}
- if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid)
- zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check;
-
if (cSeqsSize == 0) {
cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, srcSize, lastBlock);
FORWARD_IF_ERROR(cSize, "Nocompress block failed");
@@ -3529,6 +3527,10 @@ static size_t ZSTD_compressSeqStore_singleBlock(ZSTD_CCtx* zc, seqStore_t* const
cSize = ZSTD_blockHeaderSize + cSeqsSize;
DEBUGLOG(4, "Writing out compressed block, size: %zu", cSize);
}
+
+ if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid)
+ zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check;
+
return cSize;
}
@@ -3564,6 +3566,7 @@ static void ZSTD_deriveBlockSplitsHelper(seqStoreSplits* splits, size_t startIdx
size_t midIdx = (startIdx + endIdx)/2;
if (endIdx - startIdx < MIN_SEQUENCES_BLOCK_SPLITTING || splits->idx >= MAX_NB_SPLITS) {
+ DEBUGLOG(6, "ZSTD_deriveBlockSplitsHelper: Too few sequences");
return;
}
ZSTD_deriveSeqStoreChunk(&fullSeqStoreChunk, origSeqStore, startIdx, endIdx);
--
2.23.0

98
backport-zstd-1.5.0-patch-11-Fix-progress-flag-to-properly-control-progress-display-and-default.patch
View File

@ -1,98 +0,0 @@
From d2f31b662779f3c13871d54868e9d5839343856d Mon Sep 17 00:00:00 2001
From: Binh Vo <binhvo@fb.com>
Date: Mon, 7 Jun 2021 11:50:22 -0400
Subject: [PATCH 0726/1000] Fix --progress flag to properly control progress
display and default progress display on when using -v
---
programs/fileio.c | 33 ++++++++++++++++-----------------
tests/playTests.sh | 6 +++++-
2 files changed, 21 insertions(+), 18 deletions(-)
diff --git a/programs/fileio.c b/programs/fileio.c
index 5693ac39..a67c0fbf 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -2155,6 +2155,7 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput,
ZSTD_inBuffer inBuff = { ress->srcBuffer, ress->srcBufferLoaded, 0 };
ZSTD_outBuffer outBuff= { ress->dstBuffer, ress->dstBufferSize, 0 };
size_t const readSizeHint = ZSTD_decompressStream(ress->dctx, &outBuff, &inBuff);
+ const int displayLevel = (!fCtx->hasStdoutOutput || g_display_prefs.progressSetting == FIO_ps_always) ? 1 : 2;
if (ZSTD_isError(readSizeHint)) {
DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n",
srcFileName, ZSTD_getErrorName(readSizeHint));
@@ -2165,21 +2166,19 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput,
/* Write block */
storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, prefs, storedSkips);
frameSize += outBuff.pos;
- if (!fCtx->hasStdoutOutput || g_display_prefs.progressSetting == FIO_ps_always) {
- if (fCtx->nbFilesTotal > 1) {
- size_t srcFileNameSize = strlen(srcFileName);
- if (srcFileNameSize > 18) {
- const char* truncatedSrcFileName = srcFileName + srcFileNameSize - 15;
- DISPLAYUPDATE(2, "\rDecompress: %2u/%2u files. Current: ...%s : %u MB... ",
- fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) );
- } else {
- DISPLAYUPDATE(2, "\rDecompress: %2u/%2u files. Current: %s : %u MB... ",
- fCtx->currFileIdx+1, fCtx->nbFilesTotal, srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) );
- }
+ if (fCtx->nbFilesTotal > 1) {
+ size_t srcFileNameSize = strlen(srcFileName);
+ if (srcFileNameSize > 18) {
+ const char* truncatedSrcFileName = srcFileName + srcFileNameSize - 15;
+ DISPLAYUPDATE(displayLevel, "\rDecompress: %2u/%2u files. Current: ...%s : %u MB... ",
+ fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) );
} else {
- DISPLAYUPDATE(2, "\r%-20.20s : %u MB... ",
- srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) );
+ DISPLAYUPDATE(displayLevel, "\rDecompress: %2u/%2u files. Current: %s : %u MB... ",
+ fCtx->currFileIdx+1, fCtx->nbFilesTotal, srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) );
}
+ } else {
+ DISPLAYUPDATE(displayLevel, "\r%-20.20s : %u MB... ",
+ srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) );
}
if (inBuff.pos > 0) {
@@ -2513,10 +2512,10 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx,
fCtx->totalBytesOutput += (size_t)filesize;
DISPLAYLEVEL(2, "\r%79s\r", "");
/* No status message in pipe mode (stdin - stdout) or multi-files mode */
- if (g_display_prefs.displayLevel >= 2) {
- if (fCtx->nbFilesTotal <= 1 || g_display_prefs.displayLevel >= 3) {
- DISPLAYLEVEL(2, "%-20s: %llu bytes \n", srcFileName, filesize);
- }
+ if ((g_display_prefs.displayLevel >= 2 && fCtx->nbFilesTotal <= 1) ||
+ g_display_prefs.displayLevel >= 3 ||
+ g_display_prefs.progressSetting == FIO_ps_always) {
+ DISPLAYLEVEL(1, "\r%-20s: %llu bytes \n", srcFileName, filesize);
}
return 0;
diff --git a/tests/playTests.sh b/tests/playTests.sh
index f57f61f3..25293900 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -263,6 +263,10 @@ zstd -q -f tmpro
println "test: --no-progress flag"
zstd tmpro -c --no-progress | zstd -d -f -o "$INTOVOID" --no-progress
zstd tmpro -cv --no-progress | zstd -dv -f -o "$INTOVOID" --no-progress
+println "test: --progress flag"
+zstd tmpro -c | zstd -d -f -o "$INTOVOID" --progress 2>&1 | grep -E "[A-Za-z0-9._ ]+: [0-9]+ bytes"
+zstd tmpro -c | zstd -d -f -q -o "$INTOVOID" --progress 2>&1 | grep -E "[A-Za-z0-9._ ]+: [0-9]+ bytes"
+zstd tmpro -c | zstd -d -f -v -o "$INTOVOID" 2>&1 | grep -E "[A-Za-z0-9._ ]+: [0-9]+ bytes"
rm -f tmpro tmpro.zst
println "test: overwrite input file (must fail)"
zstd tmp -fo tmp && die "zstd compression overwrote the input file"
@@ -1612,7 +1616,7 @@ roundTripTest -g600M -P50 "1 --single-thread --long --zstd=wlog=29,clog=28"
if [ -n "$hasMT" ]
then
- println "\n===> zstdmt long round-trip tests "
+ println "\n===> zstdmt long round-trip tests "
roundTripTest -g80000000 -P99 "19 -T2" " "
roundTripTest -g5000000000 -P99 "1 -T2" " "
roundTripTest -g500000000 -P97 "1 -T999" " "
--
2.23.0

28
backport-zstd-1.5.0-patch-12-Z_PREFIX-zError-function.patch
View File

@ -1,28 +0,0 @@
From e00412f6b595c517ddc9ab44e810454e156dfb77 Mon Sep 17 00:00:00 2001
From: koala <869829183@qq.com>
Date: Fri, 11 Jun 2021 19:29:27 +0800
Subject: [PATCH 0757/1000] Z_PREFIX zError function
When a project use zError function,linker can not find z_zError function
---
zlibWrapper/zstd_zlibwrapper.c | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c
index ceb23937..adb231f0 100644
--- a/zlibWrapper/zstd_zlibwrapper.c
+++ b/zlibWrapper/zstd_zlibwrapper.c
@@ -1189,3 +1189,10 @@ ZEXTERN const z_crc_t FAR * ZEXPORT z_get_crc_table OF((void))
return get_crc_table();
}
#endif
+
+ /* Error function */
+ZEXTERN const char * ZEXPORT z_zError OF((int err))
+{
+ /* Just use zlib Error function */
+ return zError(err);
+}
--
2.23.0

53
backport-zstd-1.5.0-patch-13-fix-Add-missing-bounds-checks-during-compression.patch
View File

@ -1,53 +0,0 @@
From 05b6773fbcce1075edbe498a821f9a41249cf384 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Mon, 14 Jun 2021 11:25:55 -0700
Subject: [PATCH 0759/1000] [fix] Add missing bounds checks during compression
* The block splitter missed a bounds check, so when the buffer is too small it
passes an erroneously large size to `ZSTD_entropyCompressSeqStore()`, which
can then write the compressed data past the end of the buffer. This is a new
regression in v1.5.0 when the block splitter is enabled. It is either enabled
explicitly, or implicitly when using the optimal parser and `ZSTD_compress2()`
or `ZSTD_compressStream*()`.
* `HUF_writeCTable_wksp()` omits a bounds check when calling
`HUF_compressWeights()`. If it is called with `dstCapacity == 0` it will pass
an erroneously large size to `HUF_compressWeights()`, which can then write
past the end of the buffer. This bug has been present for ages. However, I
believe that zstd cannot trigger the bug, because it never calls
`HUF_compress*()` with `dstCapacity == 0` because of [this check][1].
Credit to: Oss-Fuzz
[1]: https://github.com/facebook/zstd/blob/89127e5ee2f3c1e141668fa6d4ee91245f05d132/lib/compress/zstd_compress_literals.c#L100
---
lib/compress/huf_compress.c | 1 +
lib/compress/zstd_compress.c | 1 +
2 files changed, 2 insertions(+)
diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c
index 485906e6..e9cb0bd5 100644
--- a/lib/compress/huf_compress.c
+++ b/lib/compress/huf_compress.c
@@ -133,6 +133,7 @@ size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize,
wksp->huffWeight[n] = wksp->bitsToWeight[CTable[n].nbBits];
/* attempt weights compression by FSE */
+ if (maxDstSize < 1) return ERROR(dstSize_tooSmall);
{ CHECK_V_F(hSize, HUF_compressWeights(op+1, maxDstSize-1, wksp->huffWeight, maxSymbolValue, &wksp->wksp, sizeof(wksp->wksp)) );
if ((hSize>1) & (hSize < maxSymbolValue/2)) { /* FSE compressed */
op[0] = (BYTE)hSize;
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 70f16935..9e814e31 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3486,6 +3486,7 @@ static size_t ZSTD_compressSeqStore_singleBlock(ZSTD_CCtx* zc, seqStore_t* const
if (isPartition)
ZSTD_seqStore_resolveOffCodes(dRep, cRep, seqStore, (U32)(seqStore->sequences - seqStore->sequencesStart));
+ RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall, "Block header doesn't fit");
cSeqsSize = ZSTD_entropyCompressSeqStore(seqStore,
&zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy,
&zc->appliedParams,
--
2.23.0

35
backport-zstd-1.5.0-patch-2-add-tests-set-mtime-on-output-files.patch
View File

@ -1,35 +0,0 @@
diff -Nur zstd-1.5.0/tests/playTests.sh new-zstd-1.5.0/tests/playTests.sh
--- zstd-1.5.0/tests/playTests.sh 2021-05-14 22:59:34.000000000 +0800
+++ new-zstd-1.5.0/tests/playTests.sh 2021-11-03 11:22:19.974962223 +0800
@@ -124,6 +124,13 @@
Darwin | FreeBSD | OpenBSD | NetBSD) MTIME="stat -f %m" ;;
esac
+assertSameMTime() {
+ MT1=$($MTIME "$1")
+ MT2=$($MTIME "$2")
+ echo MTIME $MT1 $MT2
+ [ "$MT1" = "$MT2" ] || die "mtime on $1 doesn't match mtime on $2 ($MT1 != $MT2)"
+}
+
GET_PERMS="stat -c %a"
case "$UNAME" in
Darwin | FreeBSD | OpenBSD | NetBSD) GET_PERMS="stat -f %Lp" ;;
@@ -583,6 +590,17 @@
rm -f tmp.img tmp.img.zst tmp.img.copy
fi
+println "\n==> zstd created file timestamp tests"
+datagen > tmp
+touch -m -t 200001010000.00 tmp
+println "test: copy mtime in file -> file compression"
+zstd -f tmp -o tmp.zst
+assertSameMTime tmp tmp.zst
+println "test: copy mtime in file -> file decopmresssion"
+zstd -f -d tmp.zst -o tmp.out
+assertSameMTime tmp.zst tmp.out
+rm -f tmp
+
println "\n===> compress multiple files into an output directory, --output-dir-flat"
println henlo > tmp1
mkdir tmpInputTestDir

13
backport-zstd-1.5.0-patch-3-remove-invalid-test.patch
View File

@ -1,13 +0,0 @@
--- zstd-1.5.0/build/meson/tests/meson.build 2021-05-14 22:59:34.000000000 +0800
+++ zstd.bak/build/meson/tests/meson.build 2021-11-08 17:12:17.225379241 +0800
@@ -186,10 +186,6 @@
zstreamtest,
args: ['-v', ZSTREAM_TESTTIME] + FUZZER_FLAGS,
timeout: 240)
-test('test-zstream-2',
- zstreamtest,
- args: ['-mt', '-t1', ZSTREAM_TESTTIME] + FUZZER_FLAGS,
- timeout: 120)
test('test-zstream-3',
zstreamtest,
args: ['--newapi', '-t1', ZSTREAM_TESTTIME] + FUZZER_FLAGS,

399
backport-zstd-1.5.0-patch-4-limit-train-samples.patch
View File

@ -1,399 +0,0 @@
diff -Nur zstd-1.5.0/lib/dictBuilder/cover.c new-zstd/lib/dictBuilder/cover.c
--- zstd-1.5.0/lib/dictBuilder/cover.c 2021-05-14 22:59:34.000000000 +0800
+++ new-zstd/lib/dictBuilder/cover.c 2021-11-16 09:49:50.933861667 +0800
@@ -40,6 +40,14 @@
/*-*************************************
* Constants
***************************************/
+
+/**
+ * There are 32bit indexes used to ref samples, so limit samples size to 4GB
+ * on 64bit builds
+ * For 32bit builds we choose 1 GB
+ * Most 32bit platforms have 2 GB user-mode addressable space and we allocate a large
+ * contigous buffer, so 1GB is already a high limit
+ */
#define COVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB))
#define COVER_DEFAULT_SPLITPOINT 1.0
diff -Nur zstd-1.5.0/lib/dictBuilder/fastcover.c new-zstd/lib/dictBuilder/fastcover.c
--- zstd-1.5.0/lib/dictBuilder/fastcover.c 2021-05-14 22:59:34.000000000 +0800
+++ new-zstd/lib/dictBuilder/fastcover.c 2021-11-16 09:52:36.621087274 +0800
@@ -32,6 +32,14 @@
/*-*************************************
* Constants
***************************************/
+/**
+ * There are 32bit indexes used to ref samples, so limit samples size to 4GB
+ * on 64bit builds
+ * For 32bit builds we choose 1 GB
+ * Most 32bit platforms have 2 GB user-mode addressable space and we allocate a large
+ * contigous buffer, so 1GB is already a high limit
+ */
+
#define FASTCOVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB))
#define FASTCOVER_MAX_F 31
#define FASTCOVER_MAX_ACCEL 10
diff -Nur zstd-1.5.0/programs/dibio.c new-zstd/programs/dibio.c
--- zstd-1.5.0/programs/dibio.c 2021-05-14 22:59:34.000000000 +0800
+++ new-zstd/programs/dibio.c 2021-11-16 15:04:46.351257422 +0800
@@ -49,7 +49,7 @@
static const size_t g_maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_t)(512 MB) << sizeof(size_t));
#define NOISELENGTH 32
-
+#define MAX_SAMPLES_SIZE (2 GB) /*training dadaset limited to 2GB*/
/*-*************************************
* Console display
@@ -88,6 +88,15 @@
#undef MIN
#define MIN(a,b) ((a) < (b) ? (a) : (b))
+/**
+ Returns the size of a file.
+ If error returns -1.
+*/
+static S64 DiB_getFileSize (const char * fileName)
+{
+ U64 const fileSize = UTIL_getFileSize(fileName);
+ return (fileSize == UTIL_FILESIZE_UNKNOWN) ? -1 : (S64)fileSize;
+}
/* ********************************************************
* File related operations
@@ -101,47 +110,66 @@
* *bufferSizePtr is modified, it provides the amount data loaded within buffer.
* sampleSizes is filled with the size of each sample.
*/
-static unsigned DiB_loadFiles(void* buffer, size_t* bufferSizePtr,
- size_t* sampleSizes, unsigned sstSize,
- const char** fileNamesTable, unsigned nbFiles, size_t targetChunkSize,
- unsigned displayLevel)
+static int DiB_loadFiles(
+ void* buffer, size_t* bufferSizePtr,
+ size_t* sampleSizes, int sstSize,
+ const char** fileNamesTable, int nbFiles,
+ size_t targetChunkSize, int displayLevel)
{
char* const buff = (char*)buffer;
- size_t pos = 0;
- unsigned nbLoadedChunks = 0, fileIndex;
-
- for (fileIndex=0; fileIndex<nbFiles; fileIndex++) {
- const char* const fileName = fileNamesTable[fileIndex];
- unsigned long long const fs64 = UTIL_getFileSize(fileName);
- unsigned long long remainingToLoad = (fs64 == UTIL_FILESIZE_UNKNOWN) ? 0 : fs64;
- U32 const nbChunks = targetChunkSize ? (U32)((fs64 + (targetChunkSize-1)) / targetChunkSize) : 1;
- U64 const chunkSize = targetChunkSize ? MIN(targetChunkSize, fs64) : fs64;
- size_t const maxChunkSize = (size_t)MIN(chunkSize, SAMPLESIZE_MAX);
- U32 cnb;
- FILE* const f = fopen(fileName, "rb");
- if (f==NULL) EXM_THROW(10, "zstd: dictBuilder: %s %s ", fileName, strerror(errno));
- DISPLAYUPDATE(2, "Loading %s... \r", fileName);
- for (cnb=0; cnb<nbChunks; cnb++) {
- size_t const toLoad = (size_t)MIN(maxChunkSize, remainingToLoad);
- if (toLoad > *bufferSizePtr-pos) break;
- { size_t const readSize = fread(buff+pos, 1, toLoad, f);
- if (readSize != toLoad) EXM_THROW(11, "Pb reading %s", fileName);
- pos += readSize;
- sampleSizes[nbLoadedChunks++] = toLoad;
- remainingToLoad -= targetChunkSize;
- if (nbLoadedChunks == sstSize) { /* no more space left in sampleSizes table */
- fileIndex = nbFiles; /* stop there */
- break;
- }
- if (toLoad < targetChunkSize) {
- fseek(f, (long)(targetChunkSize - toLoad), SEEK_CUR);
- } } }
- fclose(f);
+ size_t totalDataLoaded = 0;
+ int nbSamplesLoaded = 0;
+ int fileIndex = 0;
+ FILE * f = NULL;
+ assert(targetChunkSize <= SAMPLESIZE_MAX);
+
+ while ( nbSamplesLoaded < sstSize && fileIndex < nbFiles ) {
+ size_t fileDataLoaded;
+ S64 const fileSize = DiB_getFileSize(fileNamesTable[fileIndex]);
+ if (fileSize <= 0)
+ continue;
+
+ f = fopen( fileNamesTable[fileIndex], "rb");
+ if (f == NULL)
+ EXM_THROW(10, "zstd: dictBuilder: %s %s ",fileNamesTable[fileIndex], strerror(errno));
+ DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[fileIndex]);
+
+ /* Load the first chunk of data from the file */
+ fileDataLoaded = targetChunkSize > 0 ?
+ (size_t)MIN(fileSize, (S64)targetChunkSize) :
+ (size_t)MIN(fileSize, SAMPLESIZE_MAX);
+ if (totalDataLoaded + fileDataLoaded > *bufferSizePtr)
+ break;
+ if (fread( buff+totalDataLoaded, 1, fileDataLoaded, f) != fileDataLoaded)
+ EXM_THROW(11, "Pb reading %s", fileNamesTable[fileIndex]);
+ sampleSizes[nbSamplesLoaded++] = fileDataLoaded;
+ totalDataLoaded += fileDataLoaded;
+
+ /* If file-chunking is enabled, load the rest if the file as more samples */
+ if (targetChunkSize > 0) {
+ while( (S64)fileDataLoaded < fileSize && nbSamplesLoaded < sstSize ) {
+ size_t const chunkSize = MIN((size_t)(fileSize-fileDataLoaded), targetChunkSize);
+ if (totalDataLoaded + chunkSize > *bufferSizePtr) /* buffer is full */
+ break;
+ if (fread( buff+totalDataLoaded, 1, chunkSize, f) != chunkSize)
+ EXM_THROW(11, "Pb reading %s", fileNamesTable[fileIndex]);
+ sampleSizes[nbSamplesLoaded++] = chunkSize;
+ totalDataLoaded += chunkSize;
+ fileDataLoaded += chunkSize;
+ }
+ }
+ fileIndex += 1;
+ fclose(f);
+ f = NULL;
}
+ if (f != NULL)
+ fclose(f);
+
DISPLAYLEVEL(2, "\r%79s\r", "");
- *bufferSizePtr = pos;
- DISPLAYLEVEL(4, "loaded : %u KB \n", (unsigned)(pos >> 10))
- return nbLoadedChunks;
+ DISPLAYLEVEL(4, "Loaded %d KB total taraining data, %d nb samples \n",
+ (int)(totalDataLoaded / (1 KB)), nbSamplesLoaded);
+ *bufferSizePtr = totalDataLoaded;
+ return nbSamplesLoaded;
}
#define DiB_rotl32(x,r) ((x << r) | (x >> (32 - r)))
@@ -225,9 +253,9 @@
typedef struct {
- U64 totalSizeToLoad;
- unsigned oneSampleTooLarge;
- unsigned nbSamples;
+ S64 totalSizeToLoad;
+ int nbSamples;
+ int oneSampleTooLarge;
} fileStats;
/*! DiB_fileStats() :
@@ -235,45 +263,86 @@
* provides the amount of data to be loaded and the resulting nb of samples.
* This is useful primarily for allocation purpose => sample buffer, and sample sizes table.
*/
-static fileStats DiB_fileStats(const char** fileNamesTable, unsigned nbFiles, size_t chunkSize, unsigned displayLevel)
+static fileStats DiB_fileStats(const char** fileNamesTable, int nbFiles, size_t chunkSize, int displayLevel)
{
fileStats fs;
- unsigned n;
+ int n;
memset(&fs, 0, sizeof(fs));
+
+ // We addume that if chunking is requsted, the chunk size is < SAMPLESIZE_MAX
+ assert( chunkSize <= SAMPLESIZE_MAX );
+
for (n=0; n<nbFiles; n++) {
- U64 const fileSize = UTIL_getFileSize(fileNamesTable[n]);
- U64 const srcSize = (fileSize == UTIL_FILESIZE_UNKNOWN) ? 0 : fileSize;
- U32 const nbSamples = (U32)(chunkSize ? (srcSize + (chunkSize-1)) / chunkSize : 1);
- U64 const chunkToLoad = chunkSize ? MIN(chunkSize, srcSize) : srcSize;
- size_t const cappedChunkSize = (size_t)MIN(chunkToLoad, SAMPLESIZE_MAX);
- fs.totalSizeToLoad += cappedChunkSize * nbSamples;
- fs.oneSampleTooLarge |= (chunkSize > 2*SAMPLESIZE_MAX);
- fs.nbSamples += nbSamples;
+ S64 const fileSize = DiB_getFileSize(fileNamesTable[n]);
+ //TODO: is there a minimum sample size? what if the file is 1-byte?
+ if (fileSize == 0) {
+ DISPLAYLEVEL(3, "Sample file '%s' has zero size, skipping...\n", fileNamesTable[n]);
+ continue;
+ }
+
+ /* the case where we are breaking up files in sample chunks */
+ if (chunkSize > 0)
+ {
+ // TODO: is there a minmum sample size? can we have a 1-byte sample?
+ fs.nbSamples += (int)((fileSize + chunkSize -1) / chunkSize);
+ fs.totalSizeToLoad += fileSize;
+ }
+ else {
+ /* the case where one file is one sample */
+ if (fileSize > SAMPLESIZE_MAX) {
+ /* falg excessively large sample files */
+ fs.oneSampleTooLarge |= (fileSize > 2*SAMPLESIZE_MAX);
+
+ /* Limt to the first SAMPLESIZE_MAX (128KB) of the file */
+ DISPLAYLEVEL(3, "Sample file '%s' is too large, limiting to %d KB",
+ fileNamesTable[n], SAMPLESIZE_MAX / (1 KB));
+ }
+ fs.nbSamples += 1;
+ fs.totalSizeToLoad += MIN(fileSize, SAMPLESIZE_MAX);
+ }
}
- DISPLAYLEVEL(4, "Preparing to load : %u KB \n", (unsigned)(fs.totalSizeToLoad >> 10));
+ DISPLAYLEVEL(4, "Found training data %d files, %d samples\n", nbFiles, (int)(fs.totalSizeToLoad / (1 KB)), fs.nbSamples);
return fs;
}
-int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
- const char** fileNamesTable, unsigned nbFiles, size_t chunkSize,
+int DiB_trainFromFiles(const char* dictFileName, size_t maxDictSize,
+ const char** fileNamesTable, int nbFiles, size_t chunkSize,
ZDICT_legacy_params_t* params, ZDICT_cover_params_t* coverParams,
ZDICT_fastCover_params_t* fastCoverParams, int optimize)
{
- unsigned const displayLevel = params ? params->zParams.notificationLevel :
- coverParams ? coverParams->zParams.notificationLevel :
- fastCoverParams ? fastCoverParams->zParams.notificationLevel :
- 0; /* should never happen */
+ fileStats fs;
+ size_t* sampleSizes;
+ int nbSamplesLoaded;
+ size_t loadedSize;
+ void* srcBuffer;
void* const dictBuffer = malloc(maxDictSize);
- fileStats const fs = DiB_fileStats(fileNamesTable, nbFiles, chunkSize, displayLevel);
- size_t* const sampleSizes = (size_t*)malloc(fs.nbSamples * sizeof(size_t));
- size_t const memMult = params ? MEMMULT :
- coverParams ? COVER_MEMMULT:
- FASTCOVER_MEMMULT;
- size_t const maxMem = DiB_findMaxMem(fs.totalSizeToLoad * memMult) / memMult;
- size_t loadedSize = (size_t) MIN ((unsigned long long)maxMem, fs.totalSizeToLoad);
- void* const srcBuffer = malloc(loadedSize+NOISELENGTH);
int result = 0;
+
+ int const displayLevel = params ? params->zParams.notificationLevel :
+ coverParams ? coverParams->zParams.notificationLevel :
+ fastCoverParams ? fastCoverParams->zParams.notificationLevel : 0;
+ /* Shuffle input files before we start assessing hao much sample data to load.
+ The purpose of the shuffle is to pick random samples when the sample
+ set is large than what we can load in memory*/
+ DISPLAYLEVEL(3, "shuffling input files\n");
+ DiB_shuffle(fileNamesTable, nbFiles);
+
+ /* Figure out how much samples data to load with how samples*/
+ fs = DiB_fileStats(fileNamesTable, nbFiles, chunkSize, displayLevel);
+
+ {
+ int const memMult = params ? MEMMULT :
+ coverParams ? COVER_MEMMULT:
+ FASTCOVER_MEMMULT;
+ size_t const maxMem = DiB_findMaxMem(fs.totalSizeToLoad * memMult) / memMult;
+ /* Limit the Size of the training data to the free memory */
+ /* Limit the Size of the training data to the 2GB */
+ /* TODO: there is oportunity to stop DiB_fileStats() early when the data limit is reached */
+ loadedSize = (size_t)MIN( MIN((S64)maxMem, fs.totalSizeToLoad), MAX_SAMPLES_SIZE );
+ srcBuffer = malloc(loadedSize+NOISELENGTH);
+ sampleSizes = (size_t*)malloc(fs.nbSamples * sizeof(size_t));
+ }
/* Checks */
if ((!sampleSizes) || (!srcBuffer) || (!dictBuffer))
@@ -289,31 +358,31 @@
DISPLAYLEVEL(2, "! Alternatively, split files into fixed-size blocks representative of samples, with -B# \n");
EXM_THROW(14, "nb of samples too low"); /* we now clearly forbid this case */
}
- if (fs.totalSizeToLoad < (unsigned long long)maxDictSize * 8) {
+ if (fs.totalSizeToLoad < (S64)maxDictSize * 8) {
DISPLAYLEVEL(2, "! Warning : data size of samples too small for target dictionary size \n");
DISPLAYLEVEL(2, "! Samples should be about 100x larger than target dictionary size \n");
}
/* init */
- if (loadedSize < fs.totalSizeToLoad)
- DISPLAYLEVEL(1, "Not enough memory; training on %u MB only...\n", (unsigned)(loadedSize >> 20));
+ if ((S64)loadedSize < fs.totalSizeToLoad)
+ DISPLAYLEVEL(1, "Training samples set too large (%u MB); training on %u MB only...\n",
+ (unsigned)(fs.totalSizeToLoad / (1 MB)),
+ (unsigned)(loadedSize / (1 MB)));
/* Load input buffer */
- DISPLAYLEVEL(3, "Shuffling input files\n");
- DiB_shuffle(fileNamesTable, nbFiles);
-
- DiB_loadFiles(srcBuffer, &loadedSize, sampleSizes, fs.nbSamples, fileNamesTable, nbFiles, chunkSize, displayLevel);
-
+ nbSamplesLoaded = DiB_loadFiles(
+ srcBuffer, &loadedSize, sampleSizes, fs.nbSamples,
+ fileNamesTable, nbFiles, chunkSize, displayLevel);
{ size_t dictSize;
if (params) {
DiB_fillNoise((char*)srcBuffer + loadedSize, NOISELENGTH); /* guard band, for end of buffer condition */
dictSize = ZDICT_trainFromBuffer_legacy(dictBuffer, maxDictSize,
- srcBuffer, sampleSizes, fs.nbSamples,
+ srcBuffer, sampleSizes, nbSamplesLoaded,
*params);
} else if (coverParams) {
if (optimize) {
dictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, maxDictSize,
- srcBuffer, sampleSizes, fs.nbSamples,
+ srcBuffer, sampleSizes, nbSamplesLoaded,
coverParams);
if (!ZDICT_isError(dictSize)) {
unsigned splitPercentage = (unsigned)(coverParams->splitPoint * 100);
@@ -322,13 +391,13 @@
}
} else {
dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, maxDictSize, srcBuffer,
- sampleSizes, fs.nbSamples, *coverParams);
+ sampleSizes, nbSamplesLoaded, *coverParams);
}
} else {
assert(fastCoverParams != NULL);
if (optimize) {
dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(dictBuffer, maxDictSize,
- srcBuffer, sampleSizes, fs.nbSamples,
+ srcBuffer, sampleSizes, nbSamplesLoaded,
fastCoverParams);
if (!ZDICT_isError(dictSize)) {
unsigned splitPercentage = (unsigned)(fastCoverParams->splitPoint * 100);
@@ -338,7 +407,7 @@
}
} else {
dictSize = ZDICT_trainFromBuffer_fastCover(dictBuffer, maxDictSize, srcBuffer,
- sampleSizes, fs.nbSamples, *fastCoverParams);
+ sampleSizes, nbSamplesLoaded, *fastCoverParams);
}
}
if (ZDICT_isError(dictSize)) {
diff -Nur zstd-1.5.0/programs/dibio.h new-zstd/programs/dibio.h
--- zstd-1.5.0/programs/dibio.h 2021-05-14 22:59:34.000000000 +0800
+++ new-zstd/programs/dibio.h 2021-11-16 14:27:26.675384927 +0800
@@ -31,8 +31,8 @@
`parameters` is optional and can be provided with values set to 0, meaning "default".
@return : 0 == ok. Any other : error.
*/
-int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
- const char** fileNamesTable, unsigned nbFiles, size_t chunkSize,
+int DiB_trainFromFiles(const char* dictFileName, size_t maxDictSize,
+ const char** fileNamesTable, int nbFiles, size_t chunkSize,
ZDICT_legacy_params_t* params, ZDICT_cover_params_t* coverParams,
ZDICT_fastCover_params_t* fastCoverParams, int optimize);
diff -Nur zstd-1.5.0/programs/zstdcli.c new-zstd/programs/zstdcli.c
--- zstd-1.5.0/programs/zstdcli.c 2021-05-14 22:59:34.000000000 +0800
+++ new-zstd/programs/zstdcli.c 2021-11-16 14:32:31.813357256 +0800
@@ -1253,18 +1253,18 @@
int const optimize = !coverParams.k || !coverParams.d;
coverParams.nbThreads = (unsigned)nbWorkers;
coverParams.zParams = zParams;
- operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (unsigned)filenames->tableSize, blockSize, NULL, &coverParams, NULL, optimize);
+ operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, blockSize, NULL, &coverParams, NULL, optimize);
} else if (dict == fastCover) {
int const optimize = !fastCoverParams.k || !fastCoverParams.d;
fastCoverParams.nbThreads = (unsigned)nbWorkers;
fastCoverParams.zParams = zParams;
- operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (unsigned)filenames->tableSize, blockSize, NULL, NULL, &fastCoverParams, optimize);
+ operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, blockSize, NULL, NULL, &fastCoverParams, optimize);
} else {
ZDICT_legacy_params_t dictParams;
memset(&dictParams, 0, sizeof(dictParams));
dictParams.selectivityLevel = dictSelect;
dictParams.zParams = zParams;
- operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (unsigned)filenames->tableSize, blockSize, &dictParams, NULL, NULL, 0);
+ operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, blockSize, &dictParams, NULL, NULL, 0);
}
#else
(void)dictCLevel; (void)dictSelect; (void)dictID; (void)maxDictSize; /* not used when ZSTD_NODICT set */

39
backport-zstd-1.5.0-patch-6-fix-a-determinism-bug-with-the-DUBT.patch
View File

@ -1,39 +0,0 @@
diff -Nur zstd-1.5.0/lib/compress/zstd_compress_internal.h new-zstd/lib/compress/zstd_compress_internal.h
--- zstd-1.5.0/lib/compress/zstd_compress_internal.h 2021-05-14 22:59:34.000000000 +0800
+++ new-zstd/lib/compress/zstd_compress_internal.h 2021-11-22 20:00:33.612480981 +0800
@@ -199,6 +199,8 @@
*/
} ZSTD_window_t;
+#define ZSTD_WINDOW_START_INDEX 2
+
typedef struct ZSTD_matchState_t ZSTD_matchState_t;
#define ZSTD_ROW_HASH_CACHE_SIZE 8 /* Size of prefetching hash cache for row-based matchfinder */
@@ -884,9 +886,9 @@
MEM_STATIC U32 ZSTD_window_isEmpty(ZSTD_window_t const window)
{
- return window.dictLimit == 1 &&
- window.lowLimit == 1 &&
- (window.nextSrc - window.base) == 1;
+ return window.dictLimit == ZSTD_WINDOW_START_INDEX &&
+ window.lowLimit == ZSTD_WINDOW_START_INDEX &&
+ (window.nextSrc - window.base) == ZSTD_WINDOW_START_INDEX;
}
/**
@@ -1151,9 +1153,10 @@
ZSTD_memset(window, 0, sizeof(*window));
window->base = (BYTE const*)"";
window->dictBase = (BYTE const*)"";
- window->dictLimit = 1; /* start from 1, so that 1st position is valid */
- window->lowLimit = 1; /* it ensures first and later CCtx usages compress the same */
- window->nextSrc = window->base + 1; /* see issue #1241 */
+ ZSTD_STATIC_ASSERT(ZSTD_DUBT_UNSORTED_MARK < ZSTD_WINDOW_START_INDEX);
+ window->dictLimit = ZSTD_WINDOW_START_INDEX; /* start from >0, so that 1st position is valid */
+ window->lowLimit = ZSTD_WINDOW_START_INDEX; /* it ensures first and later CCtx usages compress the same */
+ window->nextSrc = window->base + ZSTD_WINDOW_START_INDEX; /* see issue #1241 */
window->nbOverflowCorrections = 0;
}

56
patch-14-fix-pooltests-result-print.patch
View File

@ -1,56 +0,0 @@
diff -Nur zstd-1.5.0/tests/poolTests.c zstd-1.5.0-new/tests/poolTests.c
--- zstd-1.5.0/tests/poolTests.c 2021-12-15 14:27:22.047664997 +0800
+++ zstd-1.5.0-new/tests/poolTests.c 2021-12-15 14:40:08.926050471 +0800
@@ -235,7 +235,7 @@
if (POOL_create(0, 1)) { /* should not be possible */
printf("FAIL: should not create POOL with 0 threads\n");
- DISPLAY("==== results; passed: 0/1, failed: 0/1, skipped: 0/1 ====\n");
+ DISPLAY("==== results: passed: 0/1, failed: 0/1, skipped: 0/1 ====\n");
return 1;
}
@@ -246,14 +246,14 @@
(unsigned)queueSize, (unsigned)numThreads);
if (testOrder(numThreads, queueSize)) {
printf("FAIL: testOrder\n");
- DISPLAY("==== results; passed: %d/%d, failed: 1/%d, skipped: 0/%d ====\n",
+ DISPLAY("==== results: passed: %d/%d, failed: 1/%d, skipped: 0/%d ====\n",
testNb, (testNb+1), (testNb+1), (testNb+1));
return 1;
}
printf("SUCCESS: testOrder\n");testNb++;
if (testWait(numThreads, queueSize)) {
printf("FAIL: testWait\n");
- DISPLAY("==== results; passed: %d/%d, failed: 1/%d, skipped: 0/%d ====\n",
+ DISPLAY("==== results: passed: %d/%d, failed: 1/%d, skipped: 0/%d ====\n",
testNb, (testNb+1), (testNb+1), (testNb+1));
return 1;
}
@@ -263,7 +263,7 @@
if (testThreadReduction()) {
printf("FAIL: thread reduction not effective \n");
- DISPLAY("==== results; passed: %d/%d, failed: 1/%d, skipped: 0/%d ====\n",
+ DISPLAY("==== results: passed: %d/%d, failed: 1/%d, skipped: 0/%d ====\n",
testNb, (testNb+1), (testNb+1), (testNb+1));
return 1;
} else {
@@ -272,7 +272,7 @@
if (testAbruptEnding()) {
printf("FAIL: jobs in queue not completed on early end \n");
- DISPLAY("==== results; passed: %d/%d, failed: 1/%d, skipped: 0/%d ====\n",
+ DISPLAY("==== results: passed: %d/%d, failed: 1/%d, skipped: 0/%d ====\n",
testNb, (testNb+1), (testNb+1), (testNb+1));
return 1;
} else {
@@ -280,7 +280,7 @@
}
printf("PASS: all POOL tests\n");
- DISPLAY("==== results; passed: %d/%d, failed: 0/%d, skipped: 0/%d ====\n",
+ DISPLAY("==== results: passed: %d/%d, failed: 0/%d, skipped: 0/%d ====\n",
testNb, testNb, testNb, testNb);
return 0;

14
patch-5-add-test-case-survive-a-list-of-files-which-long-file-name-length.patch
View File

@ -1,14 +0,0 @@
diff -Nur zstd-1.5.0/tests/playTests.sh new-zstd/tests/playTests.sh
--- zstd-1.5.0/tests/playTests.sh 2021-05-14 22:59:34.000000000 +0800
+++ new-zstd/tests/playTests.sh 2021-11-18 16:44:28.963975598 +0800
@@ -689,6 +689,10 @@
test -f tmp3
test -f tmp4
+println "test : survive a list of files which long file name length (--filelist=FILE)"
+datagen -g51M > tmp_badList
+zstd -f --filelist=tmp_badList && die "should have failed : list file name length is too long" # can trigger the file list file name length is too long
+
println "test : survive a list of files which is text garbage (--filelist=FILE)"
datagen > tmp_badList
zstd -f --filelist=tmp_badList && die "should have failed : list is text garbage"

105
patch-7-add-test-case.patch
View File

@ -1,105 +0,0 @@
diff -Nur zstd-1.5.0/tests/playTests.sh new-zstd/tests/playTests.sh
--- zstd-1.5.0/tests/playTests.sh 2021-05-14 22:59:34.000000000 +0800
+++ new-zstd/tests/playTests.sh 2021-12-03 11:21:43.067003307 +0800
@@ -176,10 +176,15 @@
println "\n===> simple tests "
-
+zstd -h
+zstd -H
+zstd -V
datagen > tmp
println "test : basic compression "
zstd -f tmp # trivial compression case, creates tmp.zst
+zstd -f -z tmp # trivial compression case, creates tmp.zst
+zstd -f -k tmp # trivial compression case, creates tmp.zst
+zstd -f -C tmp # trivial compression case, creates tmp.zst
println "test : basic decompression"
zstd -df tmp.zst # trivial decompression case (overwrites tmp)
println "test : too large compression level => auto-fix"
@@ -188,6 +193,7 @@
println "test : --fast aka negative compression levels"
zstd --fast -f tmp # == -1
zstd --fast=3 -f tmp # == -3
+zstd --fast3 -f tmp && die "Invalid charcter following --fast"
zstd --fast=200000 -f tmp # too low compression level, automatic fixed
zstd --fast=5000000000 -f tmp && die "too large numeric value : must fail"
zstd -c --fast=0 tmp > $INTOVOID && die "--fast must not accept value 0"
@@ -719,6 +725,7 @@
println "test : show-default-cparams regular"
datagen > tmp
zstd --show-default-cparams -f tmp
+zstd --show-default-cparams -d tmp.zst && die "can't use --show-default-cparams in decompression mode"
rm -rf tmp*
println "test : show-default-cparams recursive"
@@ -757,6 +764,7 @@
cat hello.tmp world.tmp > helloworld.tmp
zstd -c hello.tmp > hello.zst
zstd -c world.tmp > world.zst
+zstd -c hello.tmp world.tmp > helloworld.zst
cat hello.zst world.zst > helloworld.zst
zstd -dc helloworld.zst > result.tmp
cat result.tmp
@@ -909,7 +917,13 @@
cat tmp | zstd -14 -f --size-hint=10950 | zstd -t # slightly too low
cat tmp | zstd -14 -f --size-hint=22000 | zstd -t # considerably too high
cat tmp | zstd -14 -f --size-hint=5500 | zstd -t # considerably too low
-
+println "test : allows and interprets K, KB, KiB, M, MB amd MiB suffix"
+cat tmp | zstd -14 -f --size-hint=11K | zstd -t # slightly too high
+cat tmp | zstd -14 -f --size-hint=11KB | zstd -t
+cat tmp | zstd -14 -f --size-hint=11KiB | zstd -t
+cat tmp | zstd -14 -f --size-hint=1M | zstd -t
+cat tmp | zstd -14 -f --size-hint=1MB | zstd -t
+cat tmp | zstd -14 -f --size-hint=1MiB | zstd -t
println "\n===> dictionary tests "
@@ -1014,6 +1028,7 @@
zstd --train-fastcover=steps=1,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict
zstd --train-fastcover=steps=1,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict1
zstd --train-fastcover=steps=1,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict2
+zstd --train-fastcover=shrink=5,steps=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict3
println "- Create dictionary with size limit"
zstd --train-fastcover=steps=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K
println "- Create dictionary using all samples for both training and testing"
@@ -1514,6 +1529,7 @@
then
println "\n===> adaptive mode "
roundTripTest -g270000000 " --adapt"
+ roundTripTest -g270000000 " --adapt=" && die "invalid compression parameter"
roundTripTest -g27000000 " --adapt=min=1,max=4"
println "===> test: --adapt must fail on incoherent bounds "
datagen > tmp
@@ -1557,6 +1573,7 @@
datagen -g550000000 -P0 > tmp_dict
datagen -g100000000 -P1 > tmp_patch
zstd --long=30 -1f --patch-from tmp_dict tmp_patch
+zstd --long30 -1f --patch-from tmp_dict tmp_patch && die "invalid character fllowing --long"
zstd --long=30 -df --patch-from tmp_dict tmp_patch.zst -o tmp_patch_recon
$DIFF -s tmp_patch_recon tmp_patch
rm -rf tmp*
@@ -1630,11 +1647,13 @@
println "\n===> cover dictionary builder : advanced options "
TESTFILE="$PRGDIR"/zstdcli.c
+datagen
datagen > tmpDict
println "- Create first dictionary"
zstd --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict
cp "$TESTFILE" tmp
zstd -f tmp -D tmpDict
+zstd -f tmp -D tmpDict --patch-from=tmpDict && die "can't use -D and --patch-from=# at the same time"
zstd -d tmp.zst -D tmpDict -fo result
$DIFF "$TESTFILE" result
zstd --train-cover=k=56,d=8 && die "Create dictionary without input file (should error)"
@@ -1645,6 +1664,7 @@
zstd --train-cover=steps=256,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict
zstd --train-cover=steps=256,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict1
zstd --train-cover=steps=256,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict2
+zstd --train-cover=shrink=5,steps=256 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict3
println "- Create dictionary with short dictID"
zstd --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1
cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"

35
patch-8-fix-extra-newline-gets-printes-out-when-compressing-multiple-files.patch
View File

@ -1,35 +0,0 @@
diff -Nur zstd-1.5.0/programs/fileio.c new-zstd/programs/fileio.c
--- zstd-1.5.0/programs/fileio.c 2021-05-14 22:59:34.000000000 +0800
+++ new-zstd/programs/fileio.c 2021-12-06 16:16:11.466665414 +0800
@@ -889,26 +889,25 @@
if (fCtx->nbFilesTotal > 1 && !prefs->overwrite) {
if (g_display_prefs.displayLevel <= displayLevelCutoff) {
if (prefs->removeSrcFile) {
- DISPLAYLEVEL(1, "zstd: Aborting... not deleting files and processing into dst: %s", outFileName);
+ DISPLAYLEVEL(1, "zstd: Aborting... not deleting files and processing into dst: %s\n", outFileName);
error = 1;
}
} else {
if (!strcmp(outFileName, stdoutmark)) {
- DISPLAYLEVEL(2, "zstd: WARNING: all input files will be processed and concatenated into stdout. ");
+ DISPLAYLEVEL(2, "zstd: WARNING: all input files will be processed and concatenated into stdout. \n");
} else {
- DISPLAYLEVEL(2, "zstd: WARNING: all input files will be processed and concatenated into a single output file: %s ", outFileName);
+ DISPLAYLEVEL(2, "zstd: WARNING: all input files will be processed and concatenated into a single output file: %s \n", outFileName);
}
- DISPLAYLEVEL(2, "\nThe concatenated output CANNOT regenerate the original directory tree. ")
+ DISPLAYLEVEL(2, "The concatenated output CANNOT regenerate the original directory tree. \n")
if (prefs->removeSrcFile) {
if (fCtx->hasStdoutOutput) {
- DISPLAYLEVEL(1, "\nAborting. Use -f if you really want to delete the files and output to stdout");
+ DISPLAYLEVEL(1, "Aborting. Use -f if you really want to delete the files and output to stdout\n");
error = 1;
} else {
error = g_display_prefs.displayLevel > displayLevelCutoff && UTIL_requireUserConfirmation("This is a destructive operation. Proceed? (y/n): ", "Aborting...", "yY", fCtx->hasStdinInput);
}
}
}
- DISPLAY("\n");
}
return error;
}

BIN
zstd-1.5.0.tar.gz
View File

Binary file not shown.

BIN
zstd-1.5.2.tar.gz Normal file
View File

Binary file not shown.

23
zstd.spec
View File

@ -1,27 +1,14 @@
%bcond_without pzstd
Name: zstd
Version: 1.5.0
Release: 18
Version: 1.5.2
Release: 1
Summary: A fast lossless compression algorithm
License: BSD and GPLv2
URL: https://github.com/facebook/zstd
Source0: https://github.com/facebook/zstd/archive/v%{version}.tar.gz#/%{name}-%{version}.tar.gz
Patch1: backport-zstd-1.5.0-patch-1-set-mtime-on-output-files.patch
Patch2: backport-zstd-1.5.0-patch-2-add-tests-set-mtime-on-output-files.patch
Patch3: backport-zstd-1.5.0-patch-3-remove-invalid-test.patch
Patch4: backport-zstd-1.5.0-patch-4-limit-train-samples.patch
Patch5: patch-5-add-test-case-survive-a-list-of-files-which-long-file-name-length.patch
Patch6: backport-zstd-1.5.0-patch-6-fix-a-determinism-bug-with-the-DUBT.patch
Patch7: patch-7-add-test-case.patch
Patch8: patch-8-fix-extra-newline-gets-printes-out-when-compressing-multiple-files.patch
Patch9: patch-9-add-test-c-result-print.patch
Patch10: backport-zstd-1.5.0-patch-10-fix-entropy-repeat-mode-bug.patch
Patch11: backport-zstd-1.5.0-patch-11-Fix-progress-flag-to-properly-control-progress-display-and-default.patch
Patch12: backport-zstd-1.5.0-patch-12-Z_PREFIX-zError-function.patch
Patch13: backport-zstd-1.5.0-patch-13-fix-Add-missing-bounds-checks-during-compression.patch
Patch14: patch-14-fix-pooltests-result-print.patch
Patch9000: add-test-c-result-print.patch
BuildRequires: gtest-devel gcc-c++ pkg-config
@ -100,10 +87,12 @@ install -D -m644 programs/zstd.1 %{buildroot}%{_mandir}/man1/pzstd.1
%{_mandir}/man1/*.1*
%changelog
* Tue Jan 31 2023 zhoupengcheng <zhoupengcheng11@huawei.com> - 1.5.2.1
* zstd update to 1.5.2
* Tue Oct 25 2022 yanglongkang <yanglongkang@h-partners.com> - 1.5.0.18
* rebuild for next release
* Wed Dec 15 2021 liushiyuan <liushiyuan2@huawei.com> - 1.5.0.17
* fix test-pool result print