From 9f0f14bd5633cfb4e0d0dc872ecfb9655f67288e Mon Sep 17 00:00:00 2001 From: Ciprian Dorin Craciun Date: Sun, 19 Dec 2021 19:18:22 +0200 Subject: [PATCH] [common] Implement efficient binary encoding for headers (currently supporting only canonical values). --- sources/lib/common/headers.go | 38 ++++++++- sources/lib/common/keys.go | 70 +++++++++++---- sources/lib/common/metadata.go | 141 +++++++++++++++++++++++++++++-- sources/lib/common/namespaces.go | 21 ++++- 4 files changed, 244 insertions(+), 26 deletions(-) diff --git a/sources/lib/common/headers.go b/sources/lib/common/headers.go index 6ab68d0..b2ef0d3 100644 --- a/sources/lib/common/headers.go +++ b/sources/lib/common/headers.go @@ -47,10 +47,17 @@ func CanonicalHeaderValueRegister (_value string) () { var CanonicalHeaderNamesMap map[string]string +var CanonicalHeaderNamesToKey map[string]uint64 +var CanonicalHeaderNamesFromKey map[uint64]string + var CanonicalHeaderValuesMap map[string]string +var CanonicalHeaderValuesToKey map[string]uint64 +var CanonicalHeaderValuesFromKey map[uint64]string var CanonicalHeaderValuesArraysMap map[string][]string + + var CanonicalHeaderNames = []string { "Accept", "Accept-CH", @@ -101,7 +108,6 @@ var CanonicalHeaderNames = []string { "Expires", "Feature-Policy", "Forwarded", - "From", "Host", "If-Match", "If-Modified-Since", @@ -206,7 +212,10 @@ func init () { CanonicalHeaderNamesMap = make (map[string]string, len (CanonicalHeaderNames) * 4) - for _, _header := range CanonicalHeaderNames { + CanonicalHeaderNamesToKey = make (map[string]uint64, len (CanonicalHeaderNames) * 4) + CanonicalHeaderNamesFromKey = make (map[uint64]string, len (CanonicalHeaderNames)) + + for _index, _header := range CanonicalHeaderNames { _http := http.CanonicalHeaderKey (_header) _toLower := strings.ToLower (_header) @@ -216,10 +225,21 @@ func init () { panic (fmt.Sprintf ("[f0dffe23] invalid duplicate header `%s`", _header)) } + _key, _error := PrepareKey (NamespaceHeaderName, uint64 (_index + 1)) + if _error != nil { + panic (_error) + } + CanonicalHeaderNamesMap[_header] = _header CanonicalHeaderNamesMap[_toLower] = _header CanonicalHeaderNamesMap[_toUpper] = _header CanonicalHeaderNamesMap[_http] = _header + + CanonicalHeaderNamesToKey[_header] = _key + CanonicalHeaderNamesToKey[_toLower] = _key + CanonicalHeaderNamesToKey[_toUpper] = _key + CanonicalHeaderNamesToKey[_http] = _key + CanonicalHeaderNamesFromKey[_key] = _header } @@ -227,9 +247,21 @@ func init () { CanonicalHeaderValuesMap = make (map[string]string, len (CanonicalHeaderValues)) CanonicalHeaderValuesArraysMap = make (map[string][]string, len (CanonicalHeaderValues)) - for _, _value := range CanonicalHeaderValues { + CanonicalHeaderValuesToKey = make (map[string]uint64, len (CanonicalHeaderValues)) + CanonicalHeaderValuesFromKey = make (map[uint64]string, len (CanonicalHeaderValues)) + + for _index, _value := range CanonicalHeaderValues { + + _key, _error := PrepareKey (NamespaceHeaderValue, uint64 (_index + 1)) + if _error != nil { + panic (_error) + } + CanonicalHeaderValuesMap[_value] = _value CanonicalHeaderValuesArraysMap[_value] = []string { _value } + + CanonicalHeaderValuesToKey[_value] = _key + CanonicalHeaderValuesFromKey[_key] = _value } } diff --git a/sources/lib/common/keys.go b/sources/lib/common/keys.go index dfe17a8..e75d9b3 100644 --- a/sources/lib/common/keys.go +++ b/sources/lib/common/keys.go @@ -41,17 +41,17 @@ func EncodeKeyToBytes_0 (_namespace string, _key uint64, _buffer []byte) (error) if len (_buffer) != 8 { return fmt.Errorf ("[890eef13] invalid key buffer length!") } - _prefix := keyNamespacePrefix (_namespace) + _prefix := KeyNamespacePrefix (_namespace) if _prefix == 0 { return fmt.Errorf ("[feece73b] invalid key namespace `%s`!", _namespace) } - if (_key << 8) == 0 { - return fmt.Errorf ("[c8fa7817] invalid key zero!") - } _keyPrefix := byte (_key >> 56) if _keyPrefix != _prefix { return fmt.Errorf ("[85a5c362] invalid key prefix `%0x` for `%d`!", _key, _keyPrefix) } + if (_key << 8) == 0 { + return fmt.Errorf ("[c8fa7817] invalid key zero!") + } binary.BigEndian.PutUint64 (_buffer, _key) return nil } @@ -66,7 +66,7 @@ func PrepareKey (_namespace string, _key uint64) (uint64, error) { if _key >= (1 << 24) { return 0, fmt.Errorf ("[aba09b4d] invalid key value `%d`!", _key) } - _prefix := keyNamespacePrefix (_namespace) + _prefix := KeyNamespacePrefix (_namespace) if _prefix == 0 { return 0, fmt.Errorf ("[feece73b] invalid key namespace `%s`!", _namespace) } @@ -80,14 +80,16 @@ func PrepareKey (_namespace string, _key uint64) (uint64, error) { -func keyNamespacePrefix (_namespace string) (byte) { +func KeyNamespacePrefix (_namespace string) (byte) { switch _namespace { - case NamespaceFilesContent : return 'f' - case NamespaceFilesIndex : return 'F' - case NamespaceFoldersContent : return 'l' - case NamespaceFoldersIndex : return 'L' - case NamespaceDataContent : return 'd' - case NamespaceDataMetadata : return 'm' + case NamespaceFilesContent : return NamespaceFilesContentPrefix + case NamespaceFilesIndex : return NamespaceFilesIndexPrefix + case NamespaceFoldersContent : return NamespaceFoldersContentPrefix + case NamespaceFoldersIndex : return NamespaceFoldersIndexPrefix + case NamespaceDataContent : return NamespaceDataContentPrefix + case NamespaceDataMetadata : return NamespaceDataMetadataPrefix + case NamespaceHeaderName : return NamespaceHeaderNamePrefix + case NamespaceHeaderValue : return NamespaceHeaderValuePrefix default : return '0' } } @@ -97,13 +99,25 @@ func keyNamespacePrefix (_namespace string) (byte) { func EncodeKeysPairToString (_namespace1 string, _key1 uint64, _namespace2 string, _key2 uint64) (string, error) { var _buffer [16]byte - if _error := EncodeKeyToBytes_0 (_namespace1, _key1, _buffer[0:8]); _error != nil { + if _error := EncodeKeysPairToBytes_0 (_namespace1, _key1, _namespace2, _key2, _buffer[:]); _error == nil { + return string (_buffer[:]), nil + } else { return "", _error } +} + + +func EncodeKeysPairToBytes_0 (_namespace1 string, _key1 uint64, _namespace2 string, _key2 uint64, _buffer []byte) (error) { + if len (_buffer) != 16 { + return fmt.Errorf ("[c6f09bfb] invalid keys buffer length!") + } + if _error := EncodeKeyToBytes_0 (_namespace1, _key1, _buffer[0:8]); _error != nil { + return _error + } if _error := EncodeKeyToBytes_0 (_namespace2, _key2, _buffer[8:16]); _error != nil { - return "", _error + return _error } - return string (_buffer[:]), nil + return nil } @@ -118,3 +132,29 @@ func DecodeKeysPair (_buffer []byte) (uint64, uint64, error) { return _key1, _key2, nil } + +func DecodeKey (_namespace string, _buffer []byte) (uint64, error) { + if len (_buffer) != 8 { + return 0, fmt.Errorf ("[5096a85e] invalid key buffer length!") + } + _prefix := KeyNamespacePrefix (_namespace) + if _prefix == 0 { + return 0, fmt.Errorf ("[7fcffef8] invalid key namespace `%s`!", _namespace) + } + _key := DecodeKey_9 (_buffer) + _keyPrefix := byte (_key >> 56) + if _keyPrefix != _prefix { + return 0, fmt.Errorf ("[a8a94763] invalid key prefix `%0x` for `%d`!", _key, _keyPrefix) + } + if (_key << 8) == 0 { + return 0, fmt.Errorf ("[481159b4] invalid key zero!") + } + return _key, nil +} + + +func DecodeKey_9 (_buffer []byte) (uint64) { + _key := binary.BigEndian.Uint64 (_buffer) + return _key +} + diff --git a/sources/lib/common/metadata.go b/sources/lib/common/metadata.go index fb5dfbe..a357f2e 100644 --- a/sources/lib/common/metadata.go +++ b/sources/lib/common/metadata.go @@ -11,10 +11,17 @@ import "sort" -func MetadataEncode (_metadata map[string]string) ([]byte, error) { +func MetadataEncodeHttp (_metadata map[string]string) ([]byte, error) { + + if len (_metadata) > 128 { + return nil, fmt.Errorf ("[c56fce8f] invalid metadata: too large!") + } _metadataArray := make ([][2]string, 0, len (_metadata)) for _key, _value := range _metadata { + if _value == "" { + continue + } _metadataArray = append (_metadataArray, [2]string {_key, _value}) } sort.Slice (_metadataArray, @@ -30,9 +37,6 @@ func MetadataEncode (_metadata map[string]string) ([]byte, error) { if ! metadataKeyRegex.MatchString (_key) { return nil, fmt.Errorf ("[2f761e02] invalid metadata key: `%s`", _key) } - if _value == "" { - continue - } if ! metadataValueRegex.MatchString (_value) { return nil, fmt.Errorf ("[e8faf5bd] invalid metadata value: `%s`", _value) } @@ -49,12 +53,12 @@ func MetadataEncode (_metadata map[string]string) ([]byte, error) { -func MetadataDecode (_data []byte) ([][2]string, error) { +func MetadataDecodeHttp (_data []byte) ([][2]string, error) { _metadata := make ([][2]string, 0, 16) _metadataAppend := func (_key []byte, _value []byte) () { _metadata = append (_metadata, [2]string { string (_key), string (_value) }) } - if _error := MetadataDecodeIterate (_data, _metadataAppend); _error != nil { + if _error := MetadataDecodeHttpIterate (_data, _metadataAppend); _error != nil { return nil, _error } else { return _metadata, nil @@ -64,7 +68,7 @@ func MetadataDecode (_data []byte) ([][2]string, error) { -func MetadataDecodeIterate (_data []byte, _callback func ([]byte, []byte) ()) (error) { +func MetadataDecodeHttpIterate (_data []byte, _callback func ([]byte, []byte) ()) (error) { _dataSize := len (_data) _headerOffset := 0 @@ -105,6 +109,129 @@ func MetadataDecodeIterate (_data []byte, _callback func ([]byte, []byte) ()) (e +func MetadataEncodeBinary (_metadata map[string]string) ([]byte, error) { + + if len (_metadata) > 128 { + return nil, fmt.Errorf ("[2249daa0] invalid metadata: too large!") + } + + _metadataArray := make ([][2]string, 0, len (_metadata)) + for _key, _value := range _metadata { + if _value == "" { + continue + } + _metadataArray = append (_metadataArray, [2]string {_key, _value}) + } + sort.Slice (_metadataArray, + func (i int, j int) (bool) { + return _metadataArray[i][0] < _metadataArray[j][0] + }) + + _buffer := & bytes.Buffer {} + + for _, _metadata := range _metadataArray { + _key := _metadata[0] + _value := _metadata[1] + if ! metadataKeyRegex.MatchString (_key) { + return nil, fmt.Errorf ("[9c53ceb6] invalid metadata key: `%s`", _key) + } + if ! metadataValueRegex.MatchString (_value) { + return nil, fmt.Errorf ("[f932f38f] invalid metadata value: `%s`", _value) + } + + _keyId, _keyFound := CanonicalHeaderNamesToKey[_key] + _valueId, _valueFound := CanonicalHeaderValuesToKey[_value] + if !_keyFound { + return nil, fmt.Errorf ("[a2a62863] invalid metadata key: `%s` (not canonical)", _key) + } + if !_valueFound { + return nil, fmt.Errorf ("[5ed34411] invalid metadata value: `%s` (not canonical)", _value) + } + var _pairBuffer [16]byte + if _error := EncodeKeysPairToBytes_0 (NamespaceHeaderName, _keyId, NamespaceHeaderValue, _valueId, _pairBuffer[:]); _error != nil { + return nil, _error + } + _buffer.Write (_pairBuffer[:]) + } + + _data := _buffer.Bytes () + return _data, nil +} + + + + +func MetadataDecodeBinary (_data []byte) ([][2]string, error) { + _metadata := make ([][2]string, 0, 16) + _metadataAppend := func (_key []byte, _value []byte) () { + _metadata = append (_metadata, [2]string { string (_key), string (_value) }) + } + if _error := MetadataDecodeBinaryIterate (_data, _metadataAppend); _error != nil { + return nil, _error + } else { + return _metadata, nil + } +} + + + + +func MetadataDecodeBinaryIterate (_data []byte, _callback func ([]byte, []byte) ()) (error) { + + _dataLimit := len (_data) + _dataOffset := 0 + + for { + + _sliceSize := _dataLimit - _dataOffset + if _sliceSize == 0 { + return nil + } + _slice := _data[_dataOffset:] + + if _slice[0] != NamespaceHeaderNamePrefix { + return fmt.Errorf ("[f49c93cb] invalid metadata encoding") + } + if _sliceSize < 8 { + return fmt.Errorf ("[e8d008dc] invalid metadata encoding") + } + var _key []byte + if _key_0, _found := CanonicalHeaderNamesFromKey[DecodeKey_9 (_slice[0:8])]; _found { + _key = StringToBytes (_key_0) + } else { + return fmt.Errorf ("[7aa09c0f] invalid metadata encoding") + } + + _dataOffset += 8 + + _sliceSize = _dataLimit - _dataOffset + if _sliceSize == 0 { + return fmt.Errorf ("[77c8bef7] invalid metadata encoding") + } + _slice = _data[_dataOffset:] + + if _slice[0] != NamespaceHeaderValuePrefix { + return fmt.Errorf ("[2b43651c] invalid metadata encoding") + } + if _sliceSize < 8 { + return fmt.Errorf ("[7cd40b03] invalid metadata encoding") + } + var _value []byte + if _value_0, _found := CanonicalHeaderValuesFromKey[DecodeKey_9 (_slice[0:8])]; _found { + _value = StringToBytes (_value_0) + } else { + return fmt.Errorf ("[334e65ef] invalid metadata encoding") + } + + _dataOffset += 8 + + _callback (_key, _value) + } +} + + + + var metadataKeyRegex = regexp.MustCompile (`\A(?:[A-Z0-9](?:[a-z0-9]?[a-z]+)(?:-[A-Z0-9](?:[a-z0-9]?[a-z]+))*)|ETag\z`) var metadataValueRegex = regexp.MustCompile (`\A[[:graph:]](?: ?[[:graph:]]+)*\z`) diff --git a/sources/lib/common/namespaces.go b/sources/lib/common/namespaces.go index c0c6cd0..f025ee7 100644 --- a/sources/lib/common/namespaces.go +++ b/sources/lib/common/namespaces.go @@ -8,11 +8,30 @@ package common const NamespaceSchemaVersion = "schema" const CurrentSchemaVersion = "kawipiko-2021b" + const NamespaceFilesContent = "files-content" +const NamespaceFilesContentPrefix byte = 'f' + const NamespaceFilesIndex = "files-index" +const NamespaceFilesIndexPrefix byte = 'F' + const NamespaceFoldersContent = "folders-content" +const NamespaceFoldersContentPrefix byte = 'l' + const NamespaceFoldersIndex = "folders-index" +const NamespaceFoldersIndexPrefix byte = 'L' + + +const NamespaceDataMetadata = "resource-metadata" +const NamespaceDataMetadataPrefix byte = 'm' const NamespaceDataContent = "resource-data" -const NamespaceDataMetadata = "resource-metadata" +const NamespaceDataContentPrefix byte = 'd' + + +const NamespaceHeaderName = "header-name" +const NamespaceHeaderNamePrefix byte = 'h' + +const NamespaceHeaderValue = "header-value" +const NamespaceHeaderValuePrefix byte = 'H'