Nulo/kawipiko
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

[archiver] Add support for including/excluding index and metadata

Browse source
This commit is contained in:
Ciprian Dorin Craciun 2018-11-15 20:46:38 +02:00
parent 26cf131270
commit 2df17ee939
2 changed files with 64 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
documentation/readme.rst
View file

@ -138,6 +138,8 @@ The project provides two binaries:
--sources <path> --sources <path>
--archive <path> --archive <path>
--compress <gzip | brotli | identity> --compress <gzip | brotli | identity>
--exclude-index
--include-metadata
--debug --debug

19
sources/cmd/archiver.go
View file

@ -31,6 +31,8 @@ type context struct {
storedData map[string]bool storedData map[string]bool
storedFiles map[[2]uint64]string storedFiles map[[2]uint64]string
compress string compress string
includeIndex bool
includeMetadata bool
debug bool debug bool
} }
@ -105,6 +107,7 @@ func archiveFolder (_context *context, _pathResolved string, _pathInArchive stri
} }
_entries := make ([]Entry, 0, len (_names)) _entries := make ([]Entry, 0, len (_names))
if _context.includeMetadata {
for _, _name := range _names { for _, _name := range _names {
_entry := Entry { _entry := Entry {
Name : _name, Name : _name,
@ -120,8 +123,10 @@ func archiveFolder (_context *context, _pathResolved string, _pathInArchive stri
} }
_entries = append (_entries, _entry) _entries = append (_entries, _entry)
} }
}
_indexNames := make ([]string, 0, 4) _indexNames := make ([]string, 0, 4)
if _context.includeIndex {
var _indexNameFirst string var _indexNameFirst string
for _, _indexName := range IndexNames { for _, _indexName := range IndexNames {
_indexNameFound := sort.SearchStrings (_names, _indexName) _indexNameFound := sort.SearchStrings (_names, _indexName)
@ -149,12 +154,13 @@ func archiveFolder (_context *context, _pathResolved string, _pathInArchive stri
} }
archiveFile (_context, _indexPathResolved, _indexPathInArchive, _indexNameFirst) archiveFile (_context, _indexPathResolved, _indexPathInArchive, _indexNameFirst)
} }
}
if _context.includeMetadata {
_folder := Folder { _folder := Folder {
Entries : _entries, Entries : _entries,
Indices : _indexNames, Indices : _indexNames,
} }
if _data, _error := json.Marshal (&_folder); _error == nil { if _data, _error := json.Marshal (&_folder); _error == nil {
if _, _, _error := archiveData (_context, NamespaceFoldersContent, _pathInArchive, "", _data, MimeTypeJson); _error != nil { if _, _, _error := archiveData (_context, NamespaceFoldersContent, _pathInArchive, "", _data, MimeTypeJson); _error != nil {
return _error return _error
@ -162,6 +168,7 @@ func archiveFolder (_context *context, _pathResolved string, _pathInArchive stri
} else { } else {
return _error return _error
} }
}
return nil return nil
} }
@ -385,6 +392,8 @@ func main_0 () (error) {
var _sourcesFolder string var _sourcesFolder string
var _archiveFile string var _archiveFile string
var _compress string var _compress string
var _includeIndex bool
var _includeMetadata bool
var _debug bool var _debug bool
{ {
@ -397,6 +406,8 @@ cdb-http-archiver
--sources <path> --sources <path>
--archive <path> --archive <path>
--compress <gzip | brotli | identity> --compress <gzip | brotli | identity>
--exclude-index
--include-metadata
--debug --debug
`) `)
} }
@ -404,6 +415,8 @@ cdb-http-archiver
_sourcesFolder_0 := _flags.String ("sources", "", "") _sourcesFolder_0 := _flags.String ("sources", "", "")
_archiveFile_0 := _flags.String ("archive", "", "") _archiveFile_0 := _flags.String ("archive", "", "")
_compress_0 := _flags.String ("compress", "", "") _compress_0 := _flags.String ("compress", "", "")
_excludeIndex_0 := _flags.Bool ("exclude-index", false, "")
_includeMetadata_0 := _flags.Bool ("include-metadata", false, "")
_debug_0 := _flags.Bool ("debug", false, "") _debug_0 := _flags.Bool ("debug", false, "")
FlagsParse (_flags, 0, 0) FlagsParse (_flags, 0, 0)
@ -411,6 +424,8 @@ cdb-http-archiver
_sourcesFolder = *_sourcesFolder_0 _sourcesFolder = *_sourcesFolder_0
_archiveFile = *_archiveFile_0 _archiveFile = *_archiveFile_0
_compress = *_compress_0 _compress = *_compress_0
_includeIndex = ! *_excludeIndex_0
_includeMetadata = *_includeMetadata_0
_debug = *_debug_0 _debug = *_debug_0
if _sourcesFolder == "" { if _sourcesFolder == "" {
@ -434,6 +449,8 @@ cdb-http-archiver
storedData : make (map[string]bool, 16 * 1024), storedData : make (map[string]bool, 16 * 1024),
storedFiles : make (map[[2]uint64]string, 16 * 1024), storedFiles : make (map[[2]uint64]string, 16 * 1024),
compress : _compress, compress : _compress,
includeIndex : _includeIndex,
includeMetadata : _includeMetadata,
debug : _debug, debug : _debug,
} }