[common] Implement efficient binary encoding for headers (currently supporting only canonical values).

sources/lib/common/headers.go

@@ -47,10 +47,17 @@ func CanonicalHeaderValueRegister (_value string) () {
 
 
 var CanonicalHeaderNamesMap map[string]string
+var CanonicalHeaderNamesToKey map[string]uint64
+var CanonicalHeaderNamesFromKey map[uint64]string
+
 var CanonicalHeaderValuesMap map[string]string
+var CanonicalHeaderValuesToKey map[string]uint64
+var CanonicalHeaderValuesFromKey map[uint64]string
 var CanonicalHeaderValuesArraysMap map[string][]string
 
 
+
+
 var CanonicalHeaderNames = []string {
 		"Accept",
 		"Accept-CH",
@@ -101,7 +108,6 @@ var CanonicalHeaderNames = []string {
 		"Expires",
 		"Feature-Policy",
 		"Forwarded",
-		"From",
 		"Host",
 		"If-Match",
 		"If-Modified-Since",
@@ -206,7 +212,10 @@ func init () {
 	
 	
 	CanonicalHeaderNamesMap = make (map[string]string, len (CanonicalHeaderNames) * 4)
-	for _, _header := range CanonicalHeaderNames {
+	CanonicalHeaderNamesToKey = make (map[string]uint64, len (CanonicalHeaderNames) * 4)
+	CanonicalHeaderNamesFromKey = make (map[uint64]string, len (CanonicalHeaderNames))
+	
+	for _index, _header := range CanonicalHeaderNames {
 		
 		_http := http.CanonicalHeaderKey (_header)
 		_toLower := strings.ToLower (_header)
@@ -216,10 +225,21 @@ func init () {
 			panic (fmt.Sprintf ("[f0dffe23]  invalid duplicate header `%s`", _header))
 		}
 		
+		_key, _error := PrepareKey (NamespaceHeaderName, uint64 (_index + 1))
+		if _error != nil {
+			panic (_error)
+		}
+		
 		CanonicalHeaderNamesMap[_header] = _header
 		CanonicalHeaderNamesMap[_toLower] = _header
 		CanonicalHeaderNamesMap[_toUpper] = _header
 		CanonicalHeaderNamesMap[_http] = _header
+		
+		CanonicalHeaderNamesToKey[_header] = _key
+		CanonicalHeaderNamesToKey[_toLower] = _key
+		CanonicalHeaderNamesToKey[_toUpper] = _key
+		CanonicalHeaderNamesToKey[_http] = _key
+		CanonicalHeaderNamesFromKey[_key] = _header
 	}
 	
 	
@@ -227,9 +247,21 @@ func init () {
 	
 	CanonicalHeaderValuesMap = make (map[string]string, len (CanonicalHeaderValues))
 	CanonicalHeaderValuesArraysMap = make (map[string][]string, len (CanonicalHeaderValues))
-	for _, _value := range CanonicalHeaderValues {
+	CanonicalHeaderValuesToKey = make (map[string]uint64, len (CanonicalHeaderValues))
+	CanonicalHeaderValuesFromKey = make (map[uint64]string, len (CanonicalHeaderValues))
+	
+	for _index, _value := range CanonicalHeaderValues {
+		
+		_key, _error := PrepareKey (NamespaceHeaderValue, uint64 (_index + 1))
+		if _error != nil {
+			panic (_error)
+		}
+		
 		CanonicalHeaderValuesMap[_value] = _value
 		CanonicalHeaderValuesArraysMap[_value] = []string { _value }
+		
+		CanonicalHeaderValuesToKey[_value] = _key
+		CanonicalHeaderValuesFromKey[_key] = _value
 	}
 }
 

sources/lib/common/keys.go

@@ -41,17 +41,17 @@ func EncodeKeyToBytes_0 (_namespace string, _key uint64, _buffer []byte) (error)
 	if len (_buffer) != 8 {
 		return fmt.Errorf ("[890eef13]  invalid key buffer length!")
 	}
-	_prefix := keyNamespacePrefix (_namespace)
+	_prefix := KeyNamespacePrefix (_namespace)
 	if _prefix == 0 {
 		return fmt.Errorf ("[feece73b]  invalid key namespace `%s`!", _namespace)
 	}
-	if (_key << 8) == 0 {
-		return fmt.Errorf ("[c8fa7817]  invalid key zero!")
-	}
 	_keyPrefix := byte (_key >> 56)
 	if _keyPrefix != _prefix {
 		return fmt.Errorf ("[85a5c362]  invalid key prefix `%0x` for `%d`!", _key, _keyPrefix)
 	}
+	if (_key << 8) == 0 {
+		return fmt.Errorf ("[c8fa7817]  invalid key zero!")
+	}
 	binary.BigEndian.PutUint64 (_buffer, _key)
 	return nil
 }
@@ -66,7 +66,7 @@ func PrepareKey (_namespace string, _key uint64) (uint64, error) {
 	if _key >= (1 << 24) {
 		return 0, fmt.Errorf ("[aba09b4d]  invalid key value `%d`!", _key)
 	}
-	_prefix := keyNamespacePrefix (_namespace)
+	_prefix := KeyNamespacePrefix (_namespace)
 	if _prefix == 0 {
 		return 0, fmt.Errorf ("[feece73b]  invalid key namespace `%s`!", _namespace)
 	}
@@ -80,14 +80,16 @@ func PrepareKey (_namespace string, _key uint64) (uint64, error) {
 
 
 
-func keyNamespacePrefix (_namespace string) (byte) {
+func KeyNamespacePrefix (_namespace string) (byte) {
 	switch _namespace {
-		case NamespaceFilesContent : return 'f'
+		case NamespaceFilesContent : return NamespaceFilesContentPrefix
-		case NamespaceFilesIndex : return 'F'
+		case NamespaceFilesIndex : return NamespaceFilesIndexPrefix
-		case NamespaceFoldersContent : return 'l'
+		case NamespaceFoldersContent : return NamespaceFoldersContentPrefix
-		case NamespaceFoldersIndex : return 'L'
+		case NamespaceFoldersIndex : return NamespaceFoldersIndexPrefix
-		case NamespaceDataContent : return 'd'
+		case NamespaceDataContent : return NamespaceDataContentPrefix
-		case NamespaceDataMetadata : return 'm'
+		case NamespaceDataMetadata : return NamespaceDataMetadataPrefix
+		case NamespaceHeaderName : return NamespaceHeaderNamePrefix
+		case NamespaceHeaderValue : return NamespaceHeaderValuePrefix
 		default : return '0'
 	}
 }
@@ -97,13 +99,25 @@ func keyNamespacePrefix (_namespace string) (byte) {
 
 func EncodeKeysPairToString (_namespace1 string, _key1 uint64, _namespace2 string, _key2 uint64) (string, error) {
 	var _buffer [16]byte
-	if _error := EncodeKeyToBytes_0 (_namespace1, _key1, _buffer[0:8]); _error != nil {
+	if _error := EncodeKeysPairToBytes_0 (_namespace1, _key1, _namespace2, _key2, _buffer[:]); _error == nil {
+		return string (_buffer[:]), nil
+	} else {
 		return "", _error
 	}
+}
+
+
+func EncodeKeysPairToBytes_0 (_namespace1 string, _key1 uint64, _namespace2 string, _key2 uint64, _buffer []byte) (error) {
+	if len (_buffer) != 16 {
+		return fmt.Errorf ("[c6f09bfb]  invalid keys buffer length!")
+	}
+	if _error := EncodeKeyToBytes_0 (_namespace1, _key1, _buffer[0:8]); _error != nil {
+		return _error
+	}
 	if _error := EncodeKeyToBytes_0 (_namespace2, _key2, _buffer[8:16]); _error != nil {
-		return "", _error
+		return _error
 	}
-	return string (_buffer[:]), nil
+	return nil
 }
 
 
@@ -118,3 +132,29 @@ func DecodeKeysPair (_buffer []byte) (uint64, uint64, error) {
 	return _key1, _key2, nil
 }
 
+
+func DecodeKey (_namespace string, _buffer []byte) (uint64, error) {
+	if len (_buffer) != 8 {
+		return 0, fmt.Errorf ("[5096a85e]  invalid key buffer length!")
+	}
+	_prefix := KeyNamespacePrefix (_namespace)
+	if _prefix == 0 {
+		return 0, fmt.Errorf ("[7fcffef8]  invalid key namespace `%s`!", _namespace)
+	}
+	_key := DecodeKey_9 (_buffer)
+	_keyPrefix := byte (_key >> 56)
+	if _keyPrefix != _prefix {
+		return 0, fmt.Errorf ("[a8a94763]  invalid key prefix `%0x` for `%d`!", _key, _keyPrefix)
+	}
+	if (_key << 8) == 0 {
+		return 0, fmt.Errorf ("[481159b4]  invalid key zero!")
+	}
+	return _key, nil
+}
+
+
+func DecodeKey_9 (_buffer []byte) (uint64) {
+	_key := binary.BigEndian.Uint64 (_buffer)
+	return _key
+}
+

sources/lib/common/metadata.go

@@ -11,10 +11,17 @@ import "sort"
 
 
 
-func MetadataEncode (_metadata map[string]string) ([]byte, error) {
+func MetadataEncodeHttp (_metadata map[string]string) ([]byte, error) {
+	
+	if len (_metadata) > 128 {
+		return nil, fmt.Errorf ("[c56fce8f]  invalid metadata:  too large!")
+	}
 	
 	_metadataArray := make ([][2]string, 0, len (_metadata))
 	for _key, _value := range _metadata {
+		if _value == "" {
+			continue
+		}
 		_metadataArray = append (_metadataArray, [2]string {_key, _value})
 	}
 	sort.Slice (_metadataArray,
@@ -30,9 +37,6 @@ func MetadataEncode (_metadata map[string]string) ([]byte, error) {
 		if ! metadataKeyRegex.MatchString (_key) {
 			return nil, fmt.Errorf ("[2f761e02]  invalid metadata key:  `%s`", _key)
 		}
-		if _value == "" {
-			continue
-		}
 		if ! metadataValueRegex.MatchString (_value) {
 			return nil, fmt.Errorf ("[e8faf5bd]  invalid metadata value:  `%s`", _value)
 		}
@@ -49,12 +53,12 @@ func MetadataEncode (_metadata map[string]string) ([]byte, error) {
 
 
 
-func MetadataDecode (_data []byte) ([][2]string, error) {
+func MetadataDecodeHttp (_data []byte) ([][2]string, error) {
 	_metadata := make ([][2]string, 0, 16)
 	_metadataAppend := func (_key []byte, _value []byte) () {
 		_metadata = append (_metadata, [2]string { string (_key), string (_value) })
 	}
-	if _error := MetadataDecodeIterate (_data, _metadataAppend); _error != nil {
+	if _error := MetadataDecodeHttpIterate (_data, _metadataAppend); _error != nil {
 		return nil, _error
 	} else {
 		return _metadata, nil
@@ -64,7 +68,7 @@ func MetadataDecode (_data []byte) ([][2]string, error) {
 
 
 
-func MetadataDecodeIterate (_data []byte, _callback func ([]byte, []byte) ()) (error) {
+func MetadataDecodeHttpIterate (_data []byte, _callback func ([]byte, []byte) ()) (error) {
 	
 	_dataSize := len (_data)
 	_headerOffset := 0
@@ -105,6 +109,129 @@ func MetadataDecodeIterate (_data []byte, _callback func ([]byte, []byte) ()) (e
 
 
 
+func MetadataEncodeBinary (_metadata map[string]string) ([]byte, error) {
+	
+	if len (_metadata) > 128 {
+		return nil, fmt.Errorf ("[2249daa0]  invalid metadata:  too large!")
+	}
+	
+	_metadataArray := make ([][2]string, 0, len (_metadata))
+	for _key, _value := range _metadata {
+		if _value == "" {
+			continue
+		}
+		_metadataArray = append (_metadataArray, [2]string {_key, _value})
+	}
+	sort.Slice (_metadataArray,
+			func (i int, j int) (bool) {
+				return _metadataArray[i][0] < _metadataArray[j][0]
+			})
+	
+	_buffer := & bytes.Buffer {}
+	
+	for _, _metadata := range _metadataArray {
+		_key := _metadata[0]
+		_value := _metadata[1]
+		if ! metadataKeyRegex.MatchString (_key) {
+			return nil, fmt.Errorf ("[9c53ceb6]  invalid metadata key:  `%s`", _key)
+		}
+		if ! metadataValueRegex.MatchString (_value) {
+			return nil, fmt.Errorf ("[f932f38f]  invalid metadata value:  `%s`", _value)
+		}
+		
+		_keyId, _keyFound := CanonicalHeaderNamesToKey[_key]
+		_valueId, _valueFound := CanonicalHeaderValuesToKey[_value]
+		if !_keyFound {
+			return nil, fmt.Errorf ("[a2a62863]  invalid metadata key:  `%s` (not canonical)", _key)
+		}
+		if !_valueFound {
+			return nil, fmt.Errorf ("[5ed34411]  invalid metadata value:  `%s` (not canonical)", _value)
+		}
+		var _pairBuffer [16]byte
+		if _error := EncodeKeysPairToBytes_0 (NamespaceHeaderName, _keyId, NamespaceHeaderValue, _valueId, _pairBuffer[:]); _error != nil {
+			return nil, _error
+		}
+		_buffer.Write (_pairBuffer[:])
+	}
+	
+	_data := _buffer.Bytes ()
+	return _data, nil
+}
+
+
+
+
+func MetadataDecodeBinary (_data []byte) ([][2]string, error) {
+	_metadata := make ([][2]string, 0, 16)
+	_metadataAppend := func (_key []byte, _value []byte) () {
+		_metadata = append (_metadata, [2]string { string (_key), string (_value) })
+	}
+	if _error := MetadataDecodeBinaryIterate (_data, _metadataAppend); _error != nil {
+		return nil, _error
+	} else {
+		return _metadata, nil
+	}
+}
+
+
+
+
+func MetadataDecodeBinaryIterate (_data []byte, _callback func ([]byte, []byte) ()) (error) {
+	
+	_dataLimit := len (_data)
+	_dataOffset := 0
+	
+	for {
+		
+		_sliceSize := _dataLimit - _dataOffset
+		if _sliceSize == 0 {
+			return nil
+		}
+		_slice := _data[_dataOffset:]
+		
+		if _slice[0] != NamespaceHeaderNamePrefix {
+			return fmt.Errorf ("[f49c93cb]  invalid metadata encoding")
+		}
+		if _sliceSize < 8 {
+			return fmt.Errorf ("[e8d008dc]  invalid metadata encoding")
+		}
+		var _key []byte
+		if _key_0, _found := CanonicalHeaderNamesFromKey[DecodeKey_9 (_slice[0:8])]; _found {
+			_key = StringToBytes (_key_0)
+		} else {
+			return fmt.Errorf ("[7aa09c0f]  invalid metadata encoding")
+		}
+		
+		_dataOffset += 8
+		
+		_sliceSize = _dataLimit - _dataOffset
+		if _sliceSize == 0 {
+			return fmt.Errorf ("[77c8bef7]  invalid metadata encoding")
+		}
+		_slice = _data[_dataOffset:]
+		
+		if _slice[0] != NamespaceHeaderValuePrefix {
+			return fmt.Errorf ("[2b43651c]  invalid metadata encoding")
+		}
+		if _sliceSize < 8 {
+			return fmt.Errorf ("[7cd40b03]  invalid metadata encoding")
+		}
+		var _value []byte
+		if _value_0, _found := CanonicalHeaderValuesFromKey[DecodeKey_9 (_slice[0:8])]; _found {
+			_value = StringToBytes (_value_0)
+		} else {
+			return fmt.Errorf ("[334e65ef]  invalid metadata encoding")
+		}
+		
+		_dataOffset += 8
+		
+		_callback (_key, _value)
+	}
+}
+
+
+
+
 var metadataKeyRegex = regexp.MustCompile (`\A(?:[A-Z0-9](?:[a-z0-9]?[a-z]+)(?:-[A-Z0-9](?:[a-z0-9]?[a-z]+))*)|ETag\z`)
 var metadataValueRegex = regexp.MustCompile (`\A[[:graph:]](?: ?[[:graph:]]+)*\z`)
 

sources/lib/common/namespaces.go

@@ -8,11 +8,30 @@ package common
 const NamespaceSchemaVersion = "schema"
 const CurrentSchemaVersion = "kawipiko-2021b"
 
+
 const NamespaceFilesContent = "files-content"
+const NamespaceFilesContentPrefix byte = 'f'
+
 const NamespaceFilesIndex = "files-index"
+const NamespaceFilesIndexPrefix byte = 'F'
+
 const NamespaceFoldersContent = "folders-content"
+const NamespaceFoldersContentPrefix byte = 'l'
+
 const NamespaceFoldersIndex = "folders-index"
+const NamespaceFoldersIndexPrefix byte = 'L'
+
+
+const NamespaceDataMetadata = "resource-metadata"
+const NamespaceDataMetadataPrefix byte = 'm'
 
 const NamespaceDataContent = "resource-data"
-const NamespaceDataMetadata = "resource-metadata"
+const NamespaceDataContentPrefix byte = 'd'
+
+
+const NamespaceHeaderName = "header-name"
+const NamespaceHeaderNamePrefix byte = 'h'
+
+const NamespaceHeaderValue = "header-value"
+const NamespaceHeaderValuePrefix byte = 'H'