From 7e376e857f8d45df601fbdb41d8b5d3492a25052 Mon Sep 17 00:00:00 2001 From: Ciprian Dorin Craciun Date: Tue, 21 Dec 2021 20:30:34 +0200 Subject: [PATCH] [archiver] Store in the compression cache also if the compression is acceptable. --- sources/cmd/archiver/archiver.go | 53 ++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/sources/cmd/archiver/archiver.go b/sources/cmd/archiver/archiver.go index a6ae74f..92abe52 100644 --- a/sources/cmd/archiver/archiver.go +++ b/sources/cmd/archiver/archiver.go @@ -601,6 +601,7 @@ func prepareDataContent (_context *context, _pathResolved string, _pathInArchive if _compressAlgorithm != "identity" { var _dataCompressed []byte + var _dataCompressedZero bool var _dataCompressedCached bool _cacheBucketName := fmt.Sprintf ("%s:%d", _compressAlgorithm, _context.compressLevel) @@ -614,13 +615,19 @@ func prepareDataContent (_context *context, _pathResolved string, _pathInArchive if _cacheBucket != nil { _dataCompressed = _cacheBucket.Get ([]byte (_fingerprintContent)) _dataCompressedCached = _dataCompressed != nil + if _dataCompressedCached { + _dataCompressedZero = len (_dataCompressed) == 0 + if _dataCompressedZero { + _dataCompressed = nil + } + } } if _error := _cacheTxn.Rollback (); _error != nil { AbortError (_error, "[a06cfe46] unexpected compression cache error!") } } - if _dataCompressed == nil { + if _dataCompressed == nil && !_dataCompressedZero { if _dataContent == nil { if _data_0, _error := _dataContentRead (); _error == nil { _dataContent = _data_0 @@ -639,22 +646,24 @@ func prepareDataContent (_context *context, _pathResolved string, _pathInArchive _dataCompressedDelta := _dataUncompressedSize - _dataCompressedSize _dataCompressedRatio := float32 (_dataCompressedDelta) * 100 / float32 (_dataUncompressedSize) _accepted := false - _accepted = _accepted || ((_dataUncompressedSize > (1024 * 1024)) && (_dataCompressedRatio >= 5)) - _accepted = _accepted || ((_dataUncompressedSize > (64 * 1024)) && (_dataCompressedRatio >= 10)) - _accepted = _accepted || ((_dataUncompressedSize > (32 * 1024)) && (_dataCompressedRatio >= 15)) - _accepted = _accepted || ((_dataUncompressedSize > (16 * 1024)) && (_dataCompressedRatio >= 20)) - _accepted = _accepted || ((_dataUncompressedSize > (8 * 1024)) && (_dataCompressedRatio >= 25)) - _accepted = _accepted || ((_dataUncompressedSize > (4 * 1024)) && (_dataCompressedRatio >= 30)) - _accepted = _accepted || ((_dataUncompressedSize > (2 * 1024)) && (_dataCompressedRatio >= 35)) - _accepted = _accepted || ((_dataUncompressedSize > (1 * 1024)) && (_dataCompressedRatio >= 40)) - _accepted = _accepted || (_dataCompressedRatio >= 90) + if !_dataCompressedZero { + _accepted = _accepted || ((_dataUncompressedSize > (1024 * 1024)) && (_dataCompressedRatio >= 5)) + _accepted = _accepted || ((_dataUncompressedSize > (64 * 1024)) && (_dataCompressedRatio >= 10)) + _accepted = _accepted || ((_dataUncompressedSize > (32 * 1024)) && (_dataCompressedRatio >= 15)) + _accepted = _accepted || ((_dataUncompressedSize > (16 * 1024)) && (_dataCompressedRatio >= 20)) + _accepted = _accepted || ((_dataUncompressedSize > (8 * 1024)) && (_dataCompressedRatio >= 25)) + _accepted = _accepted || ((_dataUncompressedSize > (4 * 1024)) && (_dataCompressedRatio >= 30)) + _accepted = _accepted || ((_dataUncompressedSize > (2 * 1024)) && (_dataCompressedRatio >= 35)) + _accepted = _accepted || ((_dataUncompressedSize > (1 * 1024)) && (_dataCompressedRatio >= 40)) + _accepted = _accepted || (_dataCompressedRatio >= 90) + } if _accepted { _dataContent = _dataCompressed _dataEncoding = _compressEncoding _dataSize = _dataCompressedSize } - if (_context.compressCache != nil) && !_dataCompressedCached && _accepted { + if (_context.compressCache != nil) && !_dataCompressedCached { _cacheTxn, _error := _context.compressCache.Begin (true) if _error != nil { AbortError (_error, "[ddbe6a70] unexpected compression cache error!") @@ -668,7 +677,11 @@ func prepareDataContent (_context *context, _pathResolved string, _pathInArchive } } _cacheBucket.FillPercent = 0.9 - if _error := _cacheBucket.Put ([]byte (_fingerprintContent), _dataCompressed); _error != nil { + _dataCompressed_1 := _dataCompressed + if !_accepted { + _dataCompressed_1 = []byte {} + } + if _error := _cacheBucket.Put ([]byte (_fingerprintContent), _dataCompressed_1); _error != nil { AbortError (_error, "[51d57220] unexpected compression cache error!") } if _error := _cacheTxn.Commit (); _error != nil { @@ -676,13 +689,15 @@ func prepareDataContent (_context *context, _pathResolved string, _pathInArchive } } - if _dataSize < _dataUncompressedSize { - if _context.debug { - log.Printf ("[dd] [271e48d6] compress -- %.1f%% %d (%d) `%s`\n", _dataCompressedRatio, _dataCompressedSize, _dataCompressedDelta, _pathInArchive) - } - } else { - if _context.debug || _context.progress { - log.Printf ("[dd] [2174c2d6] compress-NOK -- %.1f%% %d (%d) `%s`\n", _dataCompressedRatio, _dataCompressedSize, _dataCompressedDelta, _pathInArchive) + if !_dataCompressedZero { + if _dataSize < _dataUncompressedSize { + if _context.debug { + log.Printf ("[dd] [271e48d6] compress -- %.1f%% %d (%d) `%s`\n", _dataCompressedRatio, _dataCompressedSize, _dataCompressedDelta, _pathInArchive) + } + } else { + if _context.debug { + log.Printf ("[dd] [2174c2d6] compress-NOK -- %.1f%% %d (%d) `%s`\n", _dataCompressedRatio, _dataCompressedSize, _dataCompressedDelta, _pathInArchive) + } } }