Nulo/kawipiko
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

[archiver] Update server to use optimized keys for data / metadata.

Browse source
This commit is contained in:
Ciprian Dorin Craciun 2021-12-19 14:06:39 +02:00
parent bc707d43bc
commit cb7914929c
1 changed files with 107 additions and 75 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

182
sources/cmd/server/server.go
View file

@ -49,16 +49,16 @@ type server struct {
httpTls2Server *http.Server httpTls2Server *http.Server
httpQuicServer *http3.Server httpQuicServer *http3.Server
cdbReader *cdb.CDB cdbReader *cdb.CDB
cachedFileFingerprints map[string][]byte cachedReferences map[string][2]uint64
cachedDataMeta map[string][]byte cachedDataMeta map[uint64][]byte
cachedDataContent map[string][]byte cachedDataContent map[uint64][]byte
securityHeadersEnabled bool securityHeadersEnabled bool
securityHeadersTls bool securityHeadersTls bool
http1Disabled bool http1Disabled bool
http2Disabled bool http2Disabled bool
http3AltSvc string http3AltSvc string
debug bool
quiet bool quiet bool
debug bool
dummy bool dummy bool
dummyEmpty bool dummyEmpty bool
dummyDelay time.Duration dummyDelay time.Duration
@ -96,9 +96,9 @@ func (_server *server) ServeUnwrapped (_context *fasthttp.RequestCtx) () {
_pathBuffer := [512]byte {} _pathBuffer := [256]byte {}
_keyBufferLarge := [512]byte {} _keyBufferLarge := [256 + 16]byte {}
_keyBufferSmall := [128]byte {} _keyBufferSmall := [8]byte {}
_path := _pathBuffer[:0] _path := _pathBuffer[:0]
_path = append (_path, _requestUri ...) _path = append (_path, _requestUri ...)
@ -152,7 +152,9 @@ func (_server *server) ServeUnwrapped (_context *fasthttp.RequestCtx) () {
} }
} }
var _fingerprints []byte var _referencesFound bool
var _referencesValues [2]uint64
var _referencesBuffer []byte
var _namespaceAndPathSuffixes = [][2]string { var _namespaceAndPathSuffixes = [][2]string {
{NamespaceFilesContent, ""}, {NamespaceFilesContent, ""},
@ -160,7 +162,7 @@ func (_server *server) ServeUnwrapped (_context *fasthttp.RequestCtx) () {
{NamespaceFoldersContent, ""}, {NamespaceFoldersContent, ""},
} }
if _fingerprints == nil { if !_referencesFound {
_loop_1 : for _namespaceAndPathSuffixIndex := range _namespaceAndPathSuffixes { _loop_1 : for _namespaceAndPathSuffixIndex := range _namespaceAndPathSuffixes {
_namespaceAndPathSuffix := _namespaceAndPathSuffixes[_namespaceAndPathSuffixIndex] _namespaceAndPathSuffix := _namespaceAndPathSuffixes[_namespaceAndPathSuffixIndex]
_namespace := _namespaceAndPathSuffix[0] _namespace := _namespaceAndPathSuffix[0]
@ -178,11 +180,11 @@ func (_server *server) ServeUnwrapped (_context *fasthttp.RequestCtx) () {
} }
_pathSuffixHasSlash := (len (_pathSuffix) != 0) && (_pathSuffix[0] == '/') _pathSuffixHasSlash := (len (_pathSuffix) != 0) && (_pathSuffix[0] == '/')
if _server.cachedFileFingerprints != nil { if _server.cachedReferences != nil {
_key := _keyBufferLarge[:0] _key := _keyBufferLarge[:0]
_key = append (_key, _path ...) _key = append (_key, _path ...)
_key = append (_key, _pathSuffix ...) _key = append (_key, _pathSuffix ...)
_fingerprints, _ = _server.cachedFileFingerprints[BytesToString (*NoEscapeBytes (&_key))] _referencesValues, _referencesFound = _server.cachedReferences[BytesToString (*NoEscapeBytes (&_key))]
} else { } else {
_key := _keyBufferLarge[:0] _key := _keyBufferLarge[:0]
_key = append (_key, _namespace ...) _key = append (_key, _namespace ...)
@ -190,14 +192,15 @@ func (_server *server) ServeUnwrapped (_context *fasthttp.RequestCtx) () {
_key = append (_key, _path ...) _key = append (_key, _path ...)
_key = append (_key, _pathSuffix ...) _key = append (_key, _pathSuffix ...)
if _value, _error := _server.cdbReader.GetWithCdbHash (_key); _error == nil { if _value, _error := _server.cdbReader.GetWithCdbHash (_key); _error == nil {
_fingerprints = _value _referencesBuffer = _value
_referencesFound = _value != nil
} else { } else {
_server.ServeError (_context, http.StatusInternalServerError, _error, false) _server.ServeError (_context, http.StatusInternalServerError, _error, false)
return return
} }
} }
if _fingerprints != nil { if _referencesFound {
if ((_namespace == NamespaceFoldersContent) || _pathSuffixHasSlash) && (!_pathIsRoot && !_pathHasSlash) { if ((_namespace == NamespaceFoldersContent) || _pathSuffixHasSlash) && (!_pathIsRoot && !_pathHasSlash) {
_path = append (_path, '/') _path = append (_path, '/')
_server.ServeRedirect (_context, http.StatusTemporaryRedirect, _path, true) _server.ServeRedirect (_context, http.StatusTemporaryRedirect, _path, true)
@ -208,24 +211,24 @@ func (_server *server) ServeUnwrapped (_context *fasthttp.RequestCtx) () {
} }
} }
if _fingerprints == nil { if !_referencesFound {
if bytes.Equal (StringToBytes ("/favicon.ico"), _path) { if bytes.Equal (StringToBytes ("/favicon.ico"), _path) {
_server.ServeStatic (_context, http.StatusOK, FaviconData, FaviconContentType, FaviconContentEncoding, true) _server.ServeStatic (_context, http.StatusOK, FaviconData, FaviconContentType, FaviconContentEncoding, true)
return return
} }
} }
if _fingerprints == nil { if !_referencesFound {
_loop_2 : for _loop_2 : for
_pathLimit := bytes.LastIndexByte (_path, '/'); _pathLimit := bytes.LastIndexByte (_path, '/');
_pathLimit >= 0; _pathLimit >= 0;
_pathLimit = bytes.LastIndexByte (_path[: _pathLimit], '/') { _pathLimit = bytes.LastIndexByte (_path[: _pathLimit], '/') {
if _server.cachedFileFingerprints != nil { if _server.cachedReferences != nil {
_key := _keyBufferLarge[:0] _key := _keyBufferLarge[:0]
_key = append (_key, _path[: _pathLimit] ...) _key = append (_key, _path[: _pathLimit] ...)
_key = append (_key, "/*" ...) _key = append (_key, "/*" ...)
_fingerprints, _ = _server.cachedFileFingerprints[BytesToString (*NoEscapeBytes (&_key))] _referencesValues, _referencesFound = _server.cachedReferences[BytesToString (*NoEscapeBytes (&_key))]
} else { } else {
_key := _keyBufferLarge[:0] _key := _keyBufferLarge[:0]
_key = append (_key, NamespaceFilesContent ...) _key = append (_key, NamespaceFilesContent ...)
@ -233,20 +236,21 @@ func (_server *server) ServeUnwrapped (_context *fasthttp.RequestCtx) () {
_key = append (_key, _path[: _pathLimit] ...) _key = append (_key, _path[: _pathLimit] ...)
_key = append (_key, "/*" ...) _key = append (_key, "/*" ...)
if _value, _error := _server.cdbReader.GetWithCdbHash (_key); _error == nil { if _value, _error := _server.cdbReader.GetWithCdbHash (_key); _error == nil {
_fingerprints = _value _referencesBuffer = _value
_referencesFound = _value != nil
} else { } else {
_server.ServeError (_context, http.StatusInternalServerError, _error, false) _server.ServeError (_context, http.StatusInternalServerError, _error, false)
return return
} }
} }
if _fingerprints != nil { if _referencesFound {
break _loop_2 break _loop_2
} }
} }
} }
if _fingerprints == nil { if !_referencesFound {
if !_server.quiet { if !_server.quiet {
log.Printf ("[ww] [7416f61d] [http-x..] not found `%s`!\n", *_requestUriString) log.Printf ("[ww] [7416f61d] [http-x..] not found `%s`!\n", *_requestUriString)
} }
@ -254,25 +258,30 @@ func (_server *server) ServeUnwrapped (_context *fasthttp.RequestCtx) () {
return return
} }
_fingerprintsSplit := bytes.IndexByte (_fingerprints, ':') var _keyMeta, _keyData uint64
if _fingerprintsSplit < 0 { if _referencesBuffer != nil {
if !_server.quiet { if _keyMeta_0, _keyData_0, _error := DecodeKeysPair (_referencesBuffer); _error == nil {
_keyMeta = _keyMeta_0
_keyData = _keyData_0
} else {
log.Printf ("[ee] [7ee6c981] [cdb.....] invalid data fingerprints for `%s`!\n", *_requestUriString) log.Printf ("[ee] [7ee6c981] [cdb.....] invalid data fingerprints for `%s`!\n", *_requestUriString)
_server.ServeError (_context, http.StatusInternalServerError, nil, false)
return
} }
_server.ServeError (_context, http.StatusInternalServerError, nil, false) } else {
return _keyMeta = _referencesValues[0]
_keyData = _referencesValues[1]
} }
_fingerprintMeta := _fingerprints[:_fingerprintsSplit]
_fingerprintContent := _fingerprints[_fingerprintsSplit + 1:]
var _data []byte var _data []byte
if _server.cachedDataContent != nil { if _server.cachedDataContent != nil {
_data, _ = _server.cachedDataContent[BytesToString (_fingerprintContent)] _data, _ = _server.cachedDataContent[_keyData]
} else { } else {
_key := _keyBufferSmall[:0] _key := _keyBufferSmall[:8]
_key = append (_key, NamespaceDataContent ...) if _error := EncodeKeyToBytes_0 (NamespaceDataContent, _keyData, _key); _error != nil {
_key = append (_key, ':') _server.ServeError (_context, http.StatusInternalServerError, _error, false)
_key = append (_key, _fingerprintContent ...) return
}
if _value, _error := _server.cdbReader.GetWithCdbHash (_key); _error == nil { if _value, _error := _server.cdbReader.GetWithCdbHash (_key); _error == nil {
_data = _value _data = _value
} else { } else {
@ -290,12 +299,13 @@ func (_server *server) ServeUnwrapped (_context *fasthttp.RequestCtx) () {
var _dataMetaRaw []byte var _dataMetaRaw []byte
if _server.cachedDataMeta != nil { if _server.cachedDataMeta != nil {
_dataMetaRaw, _ = _server.cachedDataMeta[BytesToString (_fingerprintMeta)] _dataMetaRaw, _ = _server.cachedDataMeta[_keyMeta]
} else { } else {
_key := _keyBufferSmall[:0] _key := _keyBufferSmall[:8]
_key = append (_key, NamespaceDataMetadata ...) if _error := EncodeKeyToBytes_0 (NamespaceDataMetadata, _keyMeta, _key); _error != nil {
_key = append (_key, ':') _server.ServeError (_context, http.StatusInternalServerError, _error, false)
_key = append (_key, _fingerprintMeta ...) return
}
if _value, _error := _server.cdbReader.GetWithCdbHash (_key); _error == nil { if _value, _error := _server.cdbReader.GetWithCdbHash (_key); _error == nil {
_dataMetaRaw = _value _dataMetaRaw = _value
} else { } else {
@ -659,8 +669,9 @@ func main_0 () (error) {
var _processes uint var _processes uint
var _threads uint var _threads uint
var _slave uint var _slave uint
var _debug bool
var _quiet bool var _quiet bool
var _report bool
var _debug bool
var _dummy bool var _dummy bool
var _dummyEmpty bool var _dummyEmpty bool
var _dummyDelay time.Duration var _dummyDelay time.Duration
@ -704,8 +715,9 @@ func main_0 () (error) {
_processes_0 := _flags.Uint ("processes", 0, "") _processes_0 := _flags.Uint ("processes", 0, "")
_threads_0 := _flags.Uint ("threads", 0, "") _threads_0 := _flags.Uint ("threads", 0, "")
_slave_0 := _flags.Uint ("slave", 0, "") _slave_0 := _flags.Uint ("slave", 0, "")
_debug_0 := _flags.Bool ("debug", false, "")
_quiet_0 := _flags.Bool ("quiet", false, "") _quiet_0 := _flags.Bool ("quiet", false, "")
_report_0 := _flags.Bool ("report", false, "")
_debug_0 := _flags.Bool ("debug", false, "")
_dummy_0 := _flags.Bool ("dummy", false, "") _dummy_0 := _flags.Bool ("dummy", false, "")
_dummyEmpty_0 := _flags.Bool ("dummy-empty", false, "") _dummyEmpty_0 := _flags.Bool ("dummy-empty", false, "")
_dummyDelay_0 := _flags.Duration ("dummy-delay", 0, "") _dummyDelay_0 := _flags.Duration ("dummy-delay", 0, "")
@ -737,8 +749,9 @@ func main_0 () (error) {
_processes = *_processes_0 _processes = *_processes_0
_threads = *_threads_0 _threads = *_threads_0
_slave = *_slave_0 _slave = *_slave_0
_quiet = *_quiet_0 && !*_debug_0
_report = *_report_0
_debug = *_debug_0 _debug = *_debug_0
_quiet = *_quiet_0 && !_debug
_dummy = *_dummy_0 _dummy = *_dummy_0
_dummyEmpty = *_dummyEmpty_0 _dummyEmpty = *_dummyEmpty_0
_dummyDelay = *_dummyDelay_0 _dummyDelay = *_dummyDelay_0
@ -989,6 +1002,9 @@ func main_0 () (error) {
if _quiet { if _quiet {
_processArguments = append (_processArguments, "--quiet") _processArguments = append (_processArguments, "--quiet")
} }
if _report {
_processArguments = append (_processArguments, "--report")
}
if _debug { if _debug {
_processArguments = append (_processArguments, "--debug") _processArguments = append (_processArguments, "--debug")
} }
@ -1221,24 +1237,30 @@ func main_0 () (error) {
var _cachedFileFingerprints map[string][]byte var _cachedReferences map[string][2]uint64
if _indexPaths { if _indexPaths {
_cachedFileFingerprints = make (map[string][]byte, 128 * 1024) _cachedReferences = make (map[string][2]uint64, 128 * 1024)
} }
var _cachedDataMeta map[string][]byte var _cachedDataMeta map[uint64][]byte
if _indexDataMeta { if _indexDataMeta {
_cachedDataMeta = make (map[string][]byte, 128 * 1024) _cachedDataMeta = make (map[uint64][]byte, 128 * 1024)
} }
var _cachedDataContent map[string][]byte var _cachedDataContent map[uint64][]byte
if _indexDataContent { if _indexDataContent {
_cachedDataContent = make (map[string][]byte, 128 * 1024) _cachedDataContent = make (map[uint64][]byte, 128 * 1024)
} }
if _indexPaths || _indexDataMeta || _indexDataContent { if _indexPaths || _indexDataMeta || _indexDataContent {
if !_quiet { if !_quiet {
log.Printf ("[ii] [fa5338fd] [cdb.....] indexing archive...\n") log.Printf ("[ii] [fa5338fd] [cdb.....] indexing archive...\n")
} }
if _filesIndex, _error := _cdbReader.GetWithCdbHash ([]byte (NamespaceFilesIndex)); _error == nil { var _filesIndexKey string
if _key_0, _error := PrepareKeyToString (NamespaceFilesIndex, 1); _error == nil {
_filesIndexKey = _key_0
} else {
AbortError (_error, "[5289ee67] [cdb.....] failed indexing archive!")
}
if _filesIndex, _error := _cdbReader.GetWithCdbHash (StringToBytes (_filesIndexKey)); _error == nil {
if _filesIndex != nil { if _filesIndex != nil {
_keyBuffer := [1024]byte {} _keyBuffer := [1024]byte {}
for { for {
@ -1251,23 +1273,15 @@ func main_0 () (error) {
} }
_filePath := _filesIndex[: _offset] _filePath := _filesIndex[: _offset]
_filesIndex = _filesIndex[_offset + 1 :] _filesIndex = _filesIndex[_offset + 1 :]
var _fingerprints []byte var _fileReferences []byte
var _fingerprintContent []byte
var _fingerprintMeta []byte
{ {
_key := _keyBuffer[:0] _key := _keyBuffer[:0]
_key = append (_key, NamespaceFilesContent ...) _key = append (_key, NamespaceFilesContent ...)
_key = append (_key, ':') _key = append (_key, ':')
_key = append (_key, _filePath ...) _key = append (_key, _filePath ...)
if _fingerprints_0, _error := _cdbReader.GetWithCdbHash (_key); _error == nil { if _references_0, _error := _cdbReader.GetWithCdbHash (_key); _error == nil {
if _fingerprints_0 != nil { if _references_0 != nil {
_fingerprints = _fingerprints_0 _fileReferences = _references_0
_fingerprintsSplit := bytes.IndexByte (_fingerprints, ':')
if _fingerprintsSplit < 0 {
AbortError (nil, "[aa6e678f] [cdb.....] failed indexing archive!")
}
_fingerprintMeta = _fingerprints[:_fingerprintsSplit]
_fingerprintContent = _fingerprints[_fingerprintsSplit + 1:]
} else { } else {
AbortError (_error, "[460b3cf1] [cdb.....] failed indexing archive!") AbortError (_error, "[460b3cf1] [cdb.....] failed indexing archive!")
} }
@ -1275,18 +1289,25 @@ func main_0 () (error) {
AbortError (_error, "[216f2075] [cdb.....] failed indexing archive!") AbortError (_error, "[216f2075] [cdb.....] failed indexing archive!")
} }
} }
var _keyDataMeta, _keyDataContent uint64
if _keyDataMeta_0, _keyDataContent_0, _error := DecodeKeysPair (_fileReferences); _error == nil {
_keyDataMeta = _keyDataMeta_0
_keyDataContent = _keyDataContent_0
} else {
AbortError (_error, "[7d1a366f] [cdb.....] failed indexing archive!")
}
if _indexPaths { if _indexPaths {
_cachedFileFingerprints[BytesToString (_filePath)] = _fingerprints _cachedReferences[BytesToString (_filePath)] = [2]uint64 { _keyDataMeta, _keyDataContent }
} }
if _indexDataMeta { if _indexDataMeta {
if _, _wasCached := _cachedDataMeta[BytesToString (_fingerprintMeta)]; !_wasCached { if _, _wasCached := _cachedDataMeta[_keyDataMeta]; !_wasCached {
_key := _keyBuffer[:0] _key := _keyBuffer[:8]
_key = append (_key, NamespaceDataMetadata ...) if _error := EncodeKeyToBytes_0 (NamespaceDataMetadata, _keyDataMeta, _key); _error != nil {
_key = append (_key, ':') AbortError (_error, "[b8cd07f4] [cdb.....] failed indexing archive!")
_key = append (_key, _fingerprintMeta ...) }
if _dataMeta, _error := _cdbReader.GetWithCdbHash (_key); _error == nil { if _dataMeta, _error := _cdbReader.GetWithCdbHash (_key); _error == nil {
if _dataMeta != nil { if _dataMeta != nil {
_cachedDataMeta[BytesToString (_fingerprintMeta)] = _dataMeta _cachedDataMeta[_keyDataMeta] = _dataMeta
} else { } else {
AbortError (_error, "[6df556bf] [cdb.....] failed indexing archive!") AbortError (_error, "[6df556bf] [cdb.....] failed indexing archive!")
} }
@ -1296,14 +1317,14 @@ func main_0 () (error) {
} }
} }
if _indexDataContent { if _indexDataContent {
if _, _wasCached := _cachedDataContent[BytesToString (_fingerprintContent)]; !_wasCached { if _, _wasCached := _cachedDataContent[_keyDataContent]; !_wasCached {
_key := _keyBuffer[:0] _key := _keyBuffer[:8]
_key = append (_key, NamespaceDataContent ...) if _error := EncodeKeyToBytes_0 (NamespaceDataContent, _keyDataContent, _key); _error != nil {
_key = append (_key, ':') AbortError (_error, "[580e387e] [cdb.....] failed indexing archive!")
_key = append (_key, _fingerprintContent ...) }
if _dataContent, _error := _cdbReader.GetWithCdbHash (_key); _error == nil { if _dataContent, _error := _cdbReader.GetWithCdbHash (_key); _error == nil {
if _dataContent != nil { if _dataContent != nil {
_cachedDataContent[BytesToString (_fingerprintContent)] = _dataContent _cachedDataContent[_keyDataContent] = _dataContent
} else { } else {
AbortError (_error, "[4e27fe46] [cdb.....] failed indexing archive!") AbortError (_error, "[4e27fe46] [cdb.....] failed indexing archive!")
} }
@ -1313,12 +1334,23 @@ func main_0 () (error) {
} }
} }
} }
if !_quiet {
if _indexPaths {
log.Printf ("[ii] [6b7ec5d9] [cdb.....] cached %d file references;\n", len (_cachedReferences))
}
if _indexDataMeta {
log.Printf ("[ii] [5ec4f113] [cdb.....] cached %d meta-data blocks;\n", len (_cachedDataMeta))
}
if _indexDataContent {
log.Printf ("[ii] [d9680a2f] [cdb.....] cached %d content blocks;\n", len (_cachedDataContent))
}
}
} else { } else {
log.Printf ("[ww] [30314f31] [cdb.....] missing archive files index; ignoring!\n") log.Printf ("[ww] [30314f31] [cdb.....] missing archive files index; ignoring!\n")
_indexPaths = false _indexPaths = false
_indexDataMeta = false _indexDataMeta = false
_indexDataContent = false _indexDataContent = false
_cachedFileFingerprints = nil _cachedReferences = nil
_cachedDataMeta = nil _cachedDataMeta = nil
_cachedDataContent = nil _cachedDataContent = nil
} }
@ -1387,7 +1419,7 @@ func main_0 () (error) {
_server := & server { _server := & server {
cdbReader : _cdbReader, cdbReader : _cdbReader,
cachedFileFingerprints : _cachedFileFingerprints, cachedReferences : _cachedReferences,
cachedDataMeta : _cachedDataMeta, cachedDataMeta : _cachedDataMeta,
cachedDataContent : _cachedDataContent, cachedDataContent : _cachedDataContent,
securityHeadersTls : _securityHeadersTls, securityHeadersTls : _securityHeadersTls,
@ -1898,7 +1930,7 @@ func main_0 () (error) {
} () } ()
} }
if _reportStatsEnabled { if _report && _reportStatsEnabled {
_reportStatsQuiet = _quiet _reportStatsQuiet = _quiet
go func () () { go func () () {
reportStatsLoop () reportStatsLoop ()