Nulo/kawipiko
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

[archiver] Add support for progress reporting and statistics.

Browse source
This commit is contained in:
Ciprian Dorin Craciun 2021-11-18 18:15:16 +02:00
parent 7aae2fc9f5
commit b295bbafb8
1 changed files with 95 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

104
sources/cmd/archiver/archiver.go
View file

@ -17,6 +17,7 @@ import "os"
import "sort"
import "strings"
import "syscall"
import "time"
import "github.com/colinmarc/cdb"
@ -28,19 +29,30 @@ import . "github.com/volution/kawipiko/lib/archiver"
type context struct {
cdbWriter *cdb.Writer
cdbWriteCount int
cdbWriteKeySize int
cdbWriteDataSize int
storedFilePaths []string
storedFolderPaths []string
storedDataMeta map[string]bool
storedDataContent map[string]bool
storedDataContentMeta map[string]map[string]string
storedFiles map[[2]uint64][2]string
archivedReferences uint
compress string
dataUncompressedCount int
dataUncompressedSize int
dataCompressedCount int
dataCompressedSize int
includeIndex bool
includeStripped bool
includeCache bool
includeEtag bool
includeFileListing bool
includeFolderListing bool
progress bool
progressStarted time.Time
progressLast time.Time
debug bool
}
@ -284,6 +296,9 @@ func archiveDataContent (_context *context, _fingerprintContent string, _dataCon
if _error := _context.cdbWriter.Put ([]byte (_key), _dataContent); _error != nil {
return _error
}
_context.cdbWriteCount += 1
_context.cdbWriteKeySize += len (_key)
_context.cdbWriteDataSize += len (_dataContent)
}
_context.storedDataContent[_fingerprintContent] = true
@ -306,6 +321,9 @@ func archiveDataMeta (_context *context, _fingerprintMeta string, _dataMeta []by
if _error := _context.cdbWriter.Put ([]byte (_key), _dataMeta); _error != nil {
return _error
}
_context.cdbWriteCount += 1
_context.cdbWriteKeySize += len (_key)
_context.cdbWriteDataSize += len (_dataMeta)
}
_context.storedDataMeta[_fingerprintMeta] = true
@ -326,16 +344,42 @@ func archiveReference (_context *context, _namespace string, _pathInArchive stri
default :
return fmt.Errorf ("[051a102a]")
}
_context.archivedReferences += 1
_key := fmt.Sprintf ("%s:%s", _namespace, _pathInArchive)
if _context.debug {
log.Printf ("[ ] reference ++ `%s` :: `%s` -> `%s` ~ `%s`\n", _namespace, _pathInArchive, _fingerprintContent[:16], _fingerprintMeta[:16])
}
_key := fmt.Sprintf ("%s:%s", _namespace, _pathInArchive)
_fingerprints := fmt.Sprintf ("%s:%s", _fingerprintContent, _fingerprintMeta)
if _error := _context.cdbWriter.Put ([]byte (_key), []byte (_fingerprints)); _error != nil {
return _error
}
_context.cdbWriteCount += 1
_context.cdbWriteKeySize += len (_key)
_context.cdbWriteDataSize += len (_fingerprints)
if _context.progress {
if _context.archivedReferences <= 1 {
_context.progressLast = time.Now ()
}
if ((_context.archivedReferences % 1000) == 0) || (((_context.archivedReferences % 10) == 0) && (time.Since (_context.progressLast) .Seconds () >= 6)) {
log.Printf ("[ ] pogress -- %0.2f minutes -- %d files, %d folders, %0.2f MiB (%0.2f MiB/s) -- %d compressed (%0.2f MiB, %0.2f%%) -- %d records (%0.2f MiB)\n",
time.Since (_context.progressStarted) .Minutes (),
len (_context.storedFilePaths),
len (_context.storedFolderPaths),
float32 (_context.dataUncompressedSize) / 1024 / 1024,
float64 (_context.dataUncompressedSize) / 1024 / 1024 / (time.Since (_context.progressStarted) .Seconds () + 0.001),
_context.dataCompressedCount,
float32 (_context.dataUncompressedSize - _context.dataCompressedSize) / 1024 / 1024,
(float32 (_context.dataUncompressedSize - _context.dataCompressedSize) / float32 (_context.dataUncompressedSize) * 100),
_context.cdbWriteCount,
float32 (_context.cdbWriteKeySize + _context.cdbWriteDataSize) / 1024 / 1024,
)
_context.progressLast = time.Now ()
}
}
return nil
}
@ -410,6 +454,13 @@ func prepareDataContent (_context *context, _pathResolved string, _pathInArchive
}
}
_context.dataUncompressedSize += _dataUncompressedSize
_context.dataUncompressedCount += 1
_context.dataCompressedSize += _dataSize
if _dataSize != _dataUncompressedSize {
_context.dataCompressedCount += 1
}
_dataMeta := make (map[string]string, 16)
// _dataMeta["Content-Length"] = fmt.Sprintf ("%d", _dataSize)
@ -527,8 +578,11 @@ func walkPath (_context *context, _pathResolved string, _pathInArchive string, _
if _stream, _error := os.Open (_pathResolved); _error == nil {
defer _stream.Close ()
_loop : for {
switch _buffer, _error := _stream.Readdir (128); _error {
switch _buffer, _error := _stream.Readdir (1024); _error {
case nil :
if _context.debug && (len (_childsStat) > 0) {
log.Printf ("[ ] folder ~~ `%s` ~~ %d\n", _pathInArchive, len (_childsStat))
}
for _, _childStat := range _buffer {
_childName := _childStat.Name ()
@ -641,6 +695,7 @@ func main_0 () (error) {
var _includeEtag bool
var _includeFileListing bool
var _includeFolderListing bool
var _progress bool
var _debug bool
{
@ -675,6 +730,7 @@ func main_0 () (error) {
--exclude-file-listing
--include-folder-listing
--progress
--debug
** for details see:
@ -692,6 +748,7 @@ func main_0 () (error) {
_includeEtag_0 := _flags.Bool ("include-etag", false, "")
_excludeFileListing_0 := _flags.Bool ("exclude-file-listing", false, "")
_includeFolderListing_0 := _flags.Bool ("include-folder-listing", false, "")
_progress_0 := _flags.Bool ("progress", false, "")
_debug_0 := _flags.Bool ("debug", false, "")
FlagsParse (_flags, 0, 0)
@ -705,6 +762,7 @@ func main_0 () (error) {
_includeEtag = *_includeEtag_0
_includeFileListing = ! *_excludeFileListing_0
_includeFolderListing = *_includeFolderListing_0
_progress = *_progress_0
_debug = *_debug_0
if _sourcesFolder == "" {
@ -723,10 +781,6 @@ func main_0 () (error) {
AbortError (_error, "[85234ba0] failed creating archive (while opening)!")
}
if _error := _cdbWriter.Put ([]byte (NamespaceSchemaVersion), []byte (CurrentSchemaVersion)); _error != nil {
AbortError (_error, "[43228812] failed writing archive!")
}
_context := & context {
cdbWriter : _cdbWriter,
storedFilePaths : make ([]string, 0, 16 * 1024),
@ -742,9 +796,19 @@ func main_0 () (error) {
includeEtag : _includeEtag,
includeFileListing : _includeFileListing,
includeFolderListing : _includeFolderListing,
progress : _progress,
debug : _debug,
}
_context.progressStarted = time.Now ()
if _error := _context.cdbWriter.Put ([]byte (NamespaceSchemaVersion), []byte (CurrentSchemaVersion)); _error != nil {
AbortError (_error, "[43228812] failed writing archive!")
}
_context.cdbWriteCount += 1
_context.cdbWriteKeySize += len (NamespaceSchemaVersion)
_context.cdbWriteDataSize += len (CurrentSchemaVersion)
if _, _error := walkPath (_context, _sourcesFolder, "/", filepath.Base (_sourcesFolder), nil, true); _error != nil {
AbortError (_error, "[b6a19ef4] failed walking folder!")
}
@ -755,9 +819,12 @@ func main_0 () (error) {
_buffer = append (_buffer, _path ...)
_buffer = append (_buffer, '\n')
}
if _error := _cdbWriter.Put ([]byte (NamespaceFilesIndex), _buffer); _error != nil {
if _error := _context.cdbWriter.Put ([]byte (NamespaceFilesIndex), _buffer); _error != nil {
AbortError (_error, "[1dbdde05] failed writing archive!")
}
_context.cdbWriteCount += 1
_context.cdbWriteKeySize += len (NamespaceFilesIndex)
_context.cdbWriteDataSize += len (_buffer)
}
if _includeFolderListing {
@ -766,14 +833,33 @@ func main_0 () (error) {
_buffer = append (_buffer, _path ...)
_buffer = append (_buffer, '\n')
}
if _error := _cdbWriter.Put ([]byte (NamespaceFoldersIndex), _buffer); _error != nil {
if _error := _context.cdbWriter.Put ([]byte (NamespaceFoldersIndex), _buffer); _error != nil {
AbortError (_error, "[e2dd2de0] failed writing archive!")
}
_context.cdbWriteCount += 1
_context.cdbWriteKeySize += len (NamespaceFilesIndex)
_context.cdbWriteDataSize += len (_buffer)
}
if _error := _cdbWriter.Close (); _error != nil {
if _error := _context.cdbWriter.Close (); _error != nil {
AbortError (_error, "[bbfb8478] failed creating archive (while closing)!")
}
_context.cdbWriter = nil
if true {
log.Printf ("[ ] completed -- %0.2f minutes -- %d files, %d folders, %0.2f MiB (%0.2f MiB/s) -- %d compressed (%0.2f MiB, %0.2f%%) -- %d records (%0.2f MiB)\n",
time.Since (_context.progressStarted) .Minutes (),
len (_context.storedFilePaths),
len (_context.storedFolderPaths),
float32 (_context.dataUncompressedSize) / 1024 / 1024,
float64 (_context.dataUncompressedSize) / 1024 / 1024 / (time.Since (_context.progressStarted) .Seconds () + 0.001),
_context.dataCompressedCount,
float32 (_context.dataUncompressedSize - _context.dataCompressedSize) / 1024 / 1024,
(float32 (_context.dataUncompressedSize - _context.dataCompressedSize) / float32 (_context.dataUncompressedSize) * 100),
_context.cdbWriteCount,
float32 (_context.cdbWriteKeySize + _context.cdbWriteDataSize) / 1024 / 1024,
)
}
return nil
}