From d70f8e39ac8abfa5a89ad5f2dbe00a203b816df9 Mon Sep 17 00:00:00 2001 From: Ciprian Dorin Craciun Date: Wed, 14 Nov 2018 21:28:25 +0200 Subject: [PATCH] [archiver][server] Move `index.*` handling out of `server` and into `archiver` --- sources/cmd/archiver.go | 70 ++++++++++++++++++++++++-------- sources/cmd/server.go | 35 ++-------------- sources/lib/archiver/index.go | 13 ++++++ sources/lib/common/metadata.go | 14 ++++++- sources/lib/common/namespaces.go | 3 +- 5 files changed, 82 insertions(+), 53 deletions(-) create mode 100644 sources/lib/archiver/index.go diff --git a/sources/cmd/archiver.go b/sources/cmd/archiver.go index 6cfe400..0a88279 100644 --- a/sources/cmd/archiver.go +++ b/sources/cmd/archiver.go @@ -37,7 +37,7 @@ type context struct { -func archiveFile (_context *context, _pathResolved string, _pathInArchive string, _name string, _stat os.FileInfo) (error) { +func archiveFile (_context *context, _pathResolved string, _pathInArchive string, _name string) (error) { var _file *os.File if _file_0, _error := os.Open (_pathResolved); _error == nil { @@ -100,7 +100,8 @@ func archiveFolder (_context *context, _pathResolved string, _pathInArchive stri } type Folder struct { - Entries []Entry `json:"entries"` + Entries []Entry `json:"entries",omitempty` + Indices []string `json:"indices",omitempty` } _entries := make ([]Entry, 0, len (_names)) @@ -120,12 +121,42 @@ func archiveFolder (_context *context, _pathResolved string, _pathInArchive stri _entries = append (_entries, _entry) } + _indexNames := make ([]string, 0, 4) + var _indexNameFirst string + for _, _indexName := range IndexNames { + _indexNameFound := sort.SearchStrings (_names, _indexName) + if _indexNameFound == len (_names) { + continue + } + if _names[_indexNameFound] != _indexName { + continue + } + _stat := _stats[_indexName] + _statMode := _stat.Mode () + if ! _statMode.IsRegular () { + continue + } + if _indexNameFirst == "" { + _indexNameFirst = _indexName + } + _indexNames = append (_indexNames, _indexName) + } + if _indexNameFirst != "" { + _indexPathResolved := filepath.Join (_pathResolved, _indexNameFirst) + _indexPathInArchive := _pathInArchive + "/" + if _pathInArchive == "/" { + _indexPathInArchive = "/" + } + archiveFile (_context, _indexPathResolved, _indexPathInArchive, _indexNameFirst) + } + _folder := Folder { Entries : _entries, + Indices : _indexNames, } if _data, _error := json.Marshal (&_folder); _error == nil { - if _, _, _error := archiveData (_context, NamespaceFoldersEntries, _pathInArchive, "", _data, MimeTypeJson); _error != nil { + if _, _, _error := archiveData (_context, NamespaceFoldersContent, _pathInArchive, "", _data, MimeTypeJson); _error != nil { return _error } } else { @@ -202,7 +233,9 @@ func archiveData (_context *context, _namespace string, _pathInArchive string, _ _context.storedData[_fingerprint] = true } else { - log.Printf ("[ ] == %s", _fingerprint) + if _context.debug { + log.Printf ("[ ] == %s", _fingerprint) + } } if _error := archiveDataReference (_context, _namespace, _pathInArchive, _fingerprint); _error != nil { @@ -231,7 +264,7 @@ func archiveDataReference (_context *context, _namespace string, _pathInArchive -func walkPath (_context *context, _path string, _prefix string, _name string, _recursed map[string]uint) (error) { +func walkPath (_context *context, _path string, _prefix string, _name string, _recursed map[string]uint) (os.FileInfo, error) { if _recursed == nil { _recursed = make (map[string]uint, 128) @@ -243,7 +276,7 @@ func walkPath (_context *context, _path string, _prefix string, _name string, _r if _stat_0, _error := os.Lstat (_path); _error == nil { _stat = _stat_0 } else { - return _error + return nil, _error } _isSymlink := false @@ -252,7 +285,7 @@ func walkPath (_context *context, _path string, _prefix string, _name string, _r if _stat_0, _error := os.Stat (_path); _error == nil { _stat = _stat_0 } else { - return _error + return nil, _error } } @@ -261,7 +294,7 @@ func walkPath (_context *context, _path string, _prefix string, _name string, _r if _resolved, _error := filepath.EvalSymlinks (_path); _error == nil { _pathResolved = _resolved } else { - return _error + return nil, _error } } else { _pathResolved = _path @@ -276,14 +309,14 @@ func walkPath (_context *context, _path string, _prefix string, _name string, _r if _context.debug { log.Printf ("[ ] ## %s\n", _pathInArchive) } - return archiveFile (_context, _pathResolved, _pathInArchive, _name, _stat) + return _stat, archiveFile (_context, _pathResolved, _pathInArchive, _name) } else if _stat.Mode () .IsDir () { _wasRecursed, _ := _recursed[_pathResolved] if _wasRecursed > 0 { log.Printf ("[ww] [2e1744c9] detected directory loop for `%s` resolving to `%s`; ignoring!\n", _path, _pathResolved) - return nil + return _stat, nil } _recursed[_pathResolved] = _wasRecursed + 1 @@ -303,15 +336,16 @@ func walkPath (_context *context, _path string, _prefix string, _name string, _r for _, _stat := range _buffer { _name := _stat.Name () _names = append (_names, _name) - _stats[_name] = _stat - if _error := walkPath (_context, filepath.Join (_path, _name), _prefix, _name, _recursed); _error != nil { - return _error + if _stat, _error := walkPath (_context, filepath.Join (_path, _name), _prefix, _name, _recursed); _error == nil { + _stats[_name] = _stat + } else { + return nil, _error } } case io.EOF : break _loop default : - return _error + return nil, _error } } } @@ -326,14 +360,14 @@ func walkPath (_context *context, _path string, _prefix string, _name string, _r log.Printf ("[ ] <> %s\n", _pathInArchive) } if _error := archiveFolder (_context, _pathResolved, _pathInArchive, _names, _stats); _error != nil { - return _error + return nil, _error } _recursed[_pathResolved] = _wasRecursed - return nil + return _stat, nil } else { - return fmt.Errorf ("[d9b836d7] unexpected file type for `%s`: `%s`!", _path, _stat.Mode ()) + return nil, fmt.Errorf ("[d9b836d7] unexpected file type for `%s`: `%s`!", _path, _stat.Mode ()) } } @@ -392,7 +426,7 @@ func main_0 () (error) { debug : _debug, } - if _error := walkPath (_context, _sourcesFolder, "/", "", nil); _error != nil { + if _, _error := walkPath (_context, _sourcesFolder, "/", "", nil); _error != nil { AbortError (_error, "[b6a19ef4] failed walking folder!") } diff --git a/sources/cmd/server.go b/sources/cmd/server.go index 663bf3a..66b8a34 100644 --- a/sources/cmd/server.go +++ b/sources/cmd/server.go @@ -88,50 +88,21 @@ func (_server *server) HandleHTTP (_context *fasthttp.RequestCtx) () { var _fingerprint []byte { - _path_0 := _path - if _pathHasSlash { - _path_0 = _path[: _pathLen - 1] - } - _found : for _, _namespace := range []string {NamespaceFilesContent, NamespaceFoldersContent, NamespaceFoldersEntries} { + _found : for _, _namespace := range []string {NamespaceFilesContent, NamespaceFoldersContent} { _key := _keyBuffer[:0] _key = append (_key, _namespace ...) _key = append (_key, ':') - _key = append (_key, _path_0 ...) + _key = append (_key, _path ...) if _value, _error := _server.cdbReader.GetWithCdbHash (_key); _error == nil { if _value != nil { _fingerprint = _value - if (_namespace == NamespaceFoldersContent) || (_namespace == NamespaceFoldersEntries) { + if (_namespace == NamespaceFoldersContent) { if !_pathIsRoot && !_pathHasSlash { _path = append (_path, '/') _server.ServeRedirect (_context, http.StatusTemporaryRedirect, _path) return } } - if _namespace == NamespaceFoldersEntries { - for _, _indexName := range []string { - "index.html", "index.htm", - "index.xhtml", "index.xht", - "index.txt", - "index.json", - "index.xml", - } { - _key := _keyBuffer[:0] - _key = append (_key, NamespaceFilesContent ...) - _key = append (_key, ':') - if !_pathIsRoot { - _key = append (_key, _path_0 ...) - } - _key = append (_key, '/') - _key = append (_key, _indexName ...) - if _value, _error := _server.cdbReader.GetWithCdbHash (_key); _error == nil { - _fingerprint = _value - break _found - } else { - _server.ServeError (_context, http.StatusInternalServerError, _error) - return - } - } - } break _found } } else { diff --git a/sources/lib/archiver/index.go b/sources/lib/archiver/index.go new file mode 100644 index 0000000..75b0d95 --- /dev/null +++ b/sources/lib/archiver/index.go @@ -0,0 +1,13 @@ + + +package archiver + + +var IndexNames []string = []string { + "index.html", "index.htm", + "index.xhtml", "index.xht", + "index.txt", + "index.json", + "index.xml", + } + diff --git a/sources/lib/common/metadata.go b/sources/lib/common/metadata.go index eecac68..43717d8 100644 --- a/sources/lib/common/metadata.go +++ b/sources/lib/common/metadata.go @@ -6,15 +6,27 @@ package common import "bytes" import "fmt" import "regexp" +import "sort" func MetadataEncode (_metadata map[string]string) ([]byte, error) { + _metadataArray := make ([][2]string, 0, len (_metadata)) + for _key, _value := range _metadata { + _metadataArray = append (_metadataArray, [2]string {_key, _value}) + } + sort.Slice (_metadataArray, + func (i int, j int) (bool) { + return _metadataArray[i][0] < _metadataArray[j][0] + }) + _buffer := & bytes.Buffer {} - for _key, _value := range _metadata { + for _, _metadata := range _metadataArray { + _key := _metadata[0] + _value := _metadata[1] if ! metadataKeyRegex.MatchString (_key) { return nil, fmt.Errorf ("[2f761e02] invalid metadata key: `%s`", _key) } diff --git a/sources/lib/common/namespaces.go b/sources/lib/common/namespaces.go index 33c9453..28245cd 100644 --- a/sources/lib/common/namespaces.go +++ b/sources/lib/common/namespaces.go @@ -6,8 +6,7 @@ package common const NamespaceFilesContent = "files:content" -const NamespaceFoldersContent = "folders:content" -const NamespaceFoldersEntries = "folders:entries" +const NamespaceFoldersContent = "folders:entries" const NamespaceDataContent = "data:content" const NamespaceDataMetadata = "data:metadata"