Skip to content

Commit 82f8898

Browse files
committed
Add configurable input buffer size for decompression (--ibuf-size)
This feature addresses performance issues when decompressing large files on mechanical hard drives with excessive disk seek operations. Changes: - Add 'inputBufferSize' field to FIO_prefs_t in fileio_types.h - Implement FIO_setInputBufferSize() setter function - Use configured buffer size in FIO_createDResources() - Add --ibuf-size command-line option with help text The option allows users to specify input buffer size (default: 0 for automatic). For large files on slow drives, using --ibuf-size=1024M to --ibuf-size=5120M can significantly improve throughput by reducing disk seek operations. Backward compatible: defaults to ZSTD_DStreamInSize when not specified.
1 parent 92505de commit 82f8898

File tree

4 files changed

+13
-1
lines changed

4 files changed

+13
-1
lines changed

programs/fileio.c

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -573,6 +573,7 @@ FIO_prefs_t* FIO_createPreferences(void)
573573
ret->allowBlockDevices = 0;
574574
ret->asyncIO = AIO_supported();
575575
ret->passThrough = -1;
576+
ret->inputBufferSize = 0; /* 0 = use default */
576577
return ret;
577578
}
578579

@@ -682,6 +683,10 @@ void FIO_setSrcSizeHint(FIO_prefs_t* const prefs, size_t srcSizeHint) {
682683
prefs->srcSizeHint = (int)MIN((size_t)INT_MAX, srcSizeHint);
683684
}
684685

686+
void FIO_setInputBufferSize(FIO_prefs_t* const prefs, size_t inputBufferSize) {
687+
prefs->inputBufferSize = inputBufferSize;
688+
}
689+
685690
void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode) {
686691
prefs->testMode = (testMode!=0);
687692
}
@@ -2611,7 +2616,8 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi
26112616
}
26122617

26132618
ress.writeCtx = AIO_WritePool_create(prefs, ZSTD_DStreamOutSize());
2614-
ress.readCtx = AIO_ReadPool_create(prefs, ZSTD_DStreamInSize());
2619+
ress.readCtx = AIO_ReadPool_create(prefs,
2620+
prefs->inputBufferSize > 0 ? prefs->inputBufferSize : ZSTD_DStreamInSize());
26152621
return ress;
26162622
}
26172623

programs/fileio.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable);
8787
void FIO_setStreamSrcSize(FIO_prefs_t* const prefs, size_t streamSrcSize);
8888
void FIO_setTargetCBlockSize(FIO_prefs_t* const prefs, size_t targetCBlockSize);
8989
void FIO_setSrcSizeHint(FIO_prefs_t* const prefs, size_t srcSizeHint);
90+
void FIO_setInputBufferSize(FIO_prefs_t* const prefs, size_t inputBufferSize);
9091
void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode);
9192
void FIO_setLiteralCompressionMode(
9293
FIO_prefs_t* const prefs,

programs/fileio_types.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ typedef struct FIO_prefs_s {
5959
int removeSrcFile;
6060
int overwrite;
6161
int asyncIO;
62+
size_t inputBufferSize; /* size of the input buffer for decompression (0 = default) */
6263

6364
/* Computation resources preferences */
6465
unsigned memLimit;

programs/zstdcli.c

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -245,6 +245,7 @@ static void usageAdvanced(const char* programName)
245245
DISPLAYOUT(" --exclude-compressed Only compress files that are not already compressed.\n\n");
246246

247247
DISPLAYOUT(" --stream-size=# Specify size of streaming input from STDIN.\n");
248+
DISPLAYOUT(" --ibuf-size=# Specify input buffer size for decompression. Helps with large files on slow disks.\n");
248249
DISPLAYOUT(" --size-hint=# Optimize compression parameters for streaming input of approximately size #.\n");
249250
DISPLAYOUT(" --target-compressed-block-size=#\n");
250251
DISPLAYOUT(" Generate compressed blocks of approximately # size.\n\n");
@@ -912,6 +913,7 @@ int main(int argCount, const char* argv[])
912913
size_t streamSrcSize = 0;
913914
size_t targetCBlockSize = 0;
914915
size_t srcSizeHint = 0;
916+
size_t inputBufferSize = 0;
915917
size_t nbInputFileNames = 0;
916918
int dictCLevel = g_defaultDictCLevel;
917919
unsigned dictSelect = g_defaultSelectivityLevel;
@@ -1096,6 +1098,7 @@ int main(int argCount, const char* argv[])
10961098
if (longCommandWArg(&argument, "--dictID")) { NEXT_UINT32(dictID); continue; }
10971099
if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) { badUsage(programName, originalArgument); CLEAN_RETURN(1); } ; cType = FIO_zstdCompression; continue; }
10981100
if (longCommandWArg(&argument, "--stream-size")) { NEXT_TSIZE(streamSrcSize); continue; }
1101+
if (longCommandWArg(&argument, "--ibuf-size")) { NEXT_TSIZE(inputBufferSize); continue; }
10991102
if (longCommandWArg(&argument, "--target-compressed-block-size")) { NEXT_TSIZE(targetCBlockSize); continue; }
11001103
if (longCommandWArg(&argument, "--size-hint")) { NEXT_TSIZE(srcSizeHint); continue; }
11011104
if (longCommandWArg(&argument, "--output-dir-flat")) {
@@ -1615,6 +1618,7 @@ int main(int argCount, const char* argv[])
16151618
FIO_setStreamSrcSize(prefs, streamSrcSize);
16161619
FIO_setTargetCBlockSize(prefs, targetCBlockSize);
16171620
FIO_setSrcSizeHint(prefs, srcSizeHint);
1621+
FIO_setInputBufferSize(prefs, inputBufferSize);
16181622
FIO_setLiteralCompressionMode(prefs, literalCompressionMode);
16191623
FIO_setSparseWrite(prefs, 0);
16201624
if (adaptMin > cLevel) cLevel = adaptMin;

0 commit comments

Comments
 (0)