Nulo/kawipiko
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

[common] Implement efficient binary encoding for headers (currently supporting only canonical values).

Browse source
This commit is contained in:
Ciprian Dorin Craciun 2021-12-19 19:18:22 +02:00
parent 874228ec49
commit 9f0f14bd56
4 changed files with 244 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
sources/lib/common/headers.go
View file

@ -47,10 +47,17 @@ func CanonicalHeaderValueRegister (_value string) () {
var CanonicalHeaderNamesMap map[string]string
var CanonicalHeaderNamesToKey map[string]uint64
var CanonicalHeaderNamesFromKey map[uint64]string
var CanonicalHeaderValuesMap map[string]string
var CanonicalHeaderValuesToKey map[string]uint64
var CanonicalHeaderValuesFromKey map[uint64]string
var CanonicalHeaderValuesArraysMap map[string][]string
var CanonicalHeaderNames = []string {
"Accept",
"Accept-CH",
@ -101,7 +108,6 @@ var CanonicalHeaderNames = []string {
"Expires",
"Feature-Policy",
"Forwarded",
"From",
"Host",
"If-Match",
"If-Modified-Since",
@ -206,7 +212,10 @@ func init () {
CanonicalHeaderNamesMap = make (map[string]string, len (CanonicalHeaderNames) * 4)
for _, _header := range CanonicalHeaderNames {
CanonicalHeaderNamesToKey = make (map[string]uint64, len (CanonicalHeaderNames) * 4)
CanonicalHeaderNamesFromKey = make (map[uint64]string, len (CanonicalHeaderNames))
for _index, _header := range CanonicalHeaderNames {
_http := http.CanonicalHeaderKey (_header)
_toLower := strings.ToLower (_header)
@ -216,10 +225,21 @@ func init () {
panic (fmt.Sprintf ("[f0dffe23] invalid duplicate header `%s`", _header))
}
_key, _error := PrepareKey (NamespaceHeaderName, uint64 (_index + 1))
if _error != nil {
panic (_error)
}
CanonicalHeaderNamesMap[_header] = _header
CanonicalHeaderNamesMap[_toLower] = _header
CanonicalHeaderNamesMap[_toUpper] = _header
CanonicalHeaderNamesMap[_http] = _header
CanonicalHeaderNamesToKey[_header] = _key
CanonicalHeaderNamesToKey[_toLower] = _key
CanonicalHeaderNamesToKey[_toUpper] = _key
CanonicalHeaderNamesToKey[_http] = _key
CanonicalHeaderNamesFromKey[_key] = _header
}
@ -227,9 +247,21 @@ func init () {
CanonicalHeaderValuesMap = make (map[string]string, len (CanonicalHeaderValues))
CanonicalHeaderValuesArraysMap = make (map[string][]string, len (CanonicalHeaderValues))
for _, _value := range CanonicalHeaderValues {
CanonicalHeaderValuesToKey = make (map[string]uint64, len (CanonicalHeaderValues))
CanonicalHeaderValuesFromKey = make (map[uint64]string, len (CanonicalHeaderValues))
for _index, _value := range CanonicalHeaderValues {
_key, _error := PrepareKey (NamespaceHeaderValue, uint64 (_index + 1))
if _error != nil {
panic (_error)
}
CanonicalHeaderValuesMap[_value] = _value
CanonicalHeaderValuesArraysMap[_value] = []string { _value }
CanonicalHeaderValuesToKey[_value] = _key
CanonicalHeaderValuesFromKey[_key] = _value
}
}

70
sources/lib/common/keys.go
View file

@ -41,17 +41,17 @@ func EncodeKeyToBytes_0 (_namespace string, _key uint64, _buffer []byte) (error)
if len (_buffer) != 8 {
return fmt.Errorf ("[890eef13] invalid key buffer length!")
}
_prefix := keyNamespacePrefix (_namespace)
_prefix := KeyNamespacePrefix (_namespace)
if _prefix == 0 {
return fmt.Errorf ("[feece73b] invalid key namespace `%s`!", _namespace)
}
if (_key << 8) == 0 {
return fmt.Errorf ("[c8fa7817] invalid key zero!")
}
_keyPrefix := byte (_key >> 56)
if _keyPrefix != _prefix {
return fmt.Errorf ("[85a5c362] invalid key prefix `%0x` for `%d`!", _key, _keyPrefix)
}
if (_key << 8) == 0 {
return fmt.Errorf ("[c8fa7817] invalid key zero!")
}
binary.BigEndian.PutUint64 (_buffer, _key)
return nil
}
@ -66,7 +66,7 @@ func PrepareKey (_namespace string, _key uint64) (uint64, error) {
if _key >= (1 << 24) {
return 0, fmt.Errorf ("[aba09b4d] invalid key value `%d`!", _key)
}
_prefix := keyNamespacePrefix (_namespace)
_prefix := KeyNamespacePrefix (_namespace)
if _prefix == 0 {
return 0, fmt.Errorf ("[feece73b] invalid key namespace `%s`!", _namespace)
}
@ -80,14 +80,16 @@ func PrepareKey (_namespace string, _key uint64) (uint64, error) {
func keyNamespacePrefix (_namespace string) (byte) {
func KeyNamespacePrefix (_namespace string) (byte) {
switch _namespace {
case NamespaceFilesContent : return 'f'
case NamespaceFilesIndex : return 'F'
case NamespaceFoldersContent : return 'l'
case NamespaceFoldersIndex : return 'L'
case NamespaceDataContent : return 'd'
case NamespaceDataMetadata : return 'm'
case NamespaceFilesContent : return NamespaceFilesContentPrefix
case NamespaceFilesIndex : return NamespaceFilesIndexPrefix
case NamespaceFoldersContent : return NamespaceFoldersContentPrefix
case NamespaceFoldersIndex : return NamespaceFoldersIndexPrefix
case NamespaceDataContent : return NamespaceDataContentPrefix
case NamespaceDataMetadata : return NamespaceDataMetadataPrefix
case NamespaceHeaderName : return NamespaceHeaderNamePrefix
case NamespaceHeaderValue : return NamespaceHeaderValuePrefix
default : return '0'
}
}
@ -97,13 +99,25 @@ func keyNamespacePrefix (_namespace string) (byte) {
func EncodeKeysPairToString (_namespace1 string, _key1 uint64, _namespace2 string, _key2 uint64) (string, error) {
var _buffer [16]byte
if _error := EncodeKeyToBytes_0 (_namespace1, _key1, _buffer[0:8]); _error != nil {
if _error := EncodeKeysPairToBytes_0 (_namespace1, _key1, _namespace2, _key2, _buffer[:]); _error == nil {
return string (_buffer[:]), nil
} else {
return "", _error
}
}
func EncodeKeysPairToBytes_0 (_namespace1 string, _key1 uint64, _namespace2 string, _key2 uint64, _buffer []byte) (error) {
if len (_buffer) != 16 {
return fmt.Errorf ("[c6f09bfb] invalid keys buffer length!")
}
if _error := EncodeKeyToBytes_0 (_namespace1, _key1, _buffer[0:8]); _error != nil {
return _error
}
if _error := EncodeKeyToBytes_0 (_namespace2, _key2, _buffer[8:16]); _error != nil {
return "", _error
return _error
}
return string (_buffer[:]), nil
return nil
}
@ -118,3 +132,29 @@ func DecodeKeysPair (_buffer []byte) (uint64, uint64, error) {
return _key1, _key2, nil
}
func DecodeKey (_namespace string, _buffer []byte) (uint64, error) {
if len (_buffer) != 8 {
return 0, fmt.Errorf ("[5096a85e] invalid key buffer length!")
}
_prefix := KeyNamespacePrefix (_namespace)
if _prefix == 0 {
return 0, fmt.Errorf ("[7fcffef8] invalid key namespace `%s`!", _namespace)
}
_key := DecodeKey_9 (_buffer)
_keyPrefix := byte (_key >> 56)
if _keyPrefix != _prefix {
return 0, fmt.Errorf ("[a8a94763] invalid key prefix `%0x` for `%d`!", _key, _keyPrefix)
}
if (_key << 8) == 0 {
return 0, fmt.Errorf ("[481159b4] invalid key zero!")
}
return _key, nil
}
func DecodeKey_9 (_buffer []byte) (uint64) {
_key := binary.BigEndian.Uint64 (_buffer)
return _key
}

141
sources/lib/common/metadata.go
View file

@ -11,10 +11,17 @@ import "sort"
func MetadataEncode (_metadata map[string]string) ([]byte, error) {
func MetadataEncodeHttp (_metadata map[string]string) ([]byte, error) {
if len (_metadata) > 128 {
return nil, fmt.Errorf ("[c56fce8f] invalid metadata: too large!")
}
_metadataArray := make ([][2]string, 0, len (_metadata))
for _key, _value := range _metadata {
if _value == "" {
continue
}
_metadataArray = append (_metadataArray, [2]string {_key, _value})
}
sort.Slice (_metadataArray,
@ -30,9 +37,6 @@ func MetadataEncode (_metadata map[string]string) ([]byte, error) {
if ! metadataKeyRegex.MatchString (_key) {
return nil, fmt.Errorf ("[2f761e02] invalid metadata key: `%s`", _key)
}
if _value == "" {
continue
}
if ! metadataValueRegex.MatchString (_value) {
return nil, fmt.Errorf ("[e8faf5bd] invalid metadata value: `%s`", _value)
}
@ -49,12 +53,12 @@ func MetadataEncode (_metadata map[string]string) ([]byte, error) {
func MetadataDecode (_data []byte) ([][2]string, error) {
func MetadataDecodeHttp (_data []byte) ([][2]string, error) {
_metadata := make ([][2]string, 0, 16)
_metadataAppend := func (_key []byte, _value []byte) () {
_metadata = append (_metadata, [2]string { string (_key), string (_value) })
}
if _error := MetadataDecodeIterate (_data, _metadataAppend); _error != nil {
if _error := MetadataDecodeHttpIterate (_data, _metadataAppend); _error != nil {
return nil, _error
} else {
return _metadata, nil
@ -64,7 +68,7 @@ func MetadataDecode (_data []byte) ([][2]string, error) {
func MetadataDecodeIterate (_data []byte, _callback func ([]byte, []byte) ()) (error) {
func MetadataDecodeHttpIterate (_data []byte, _callback func ([]byte, []byte) ()) (error) {
_dataSize := len (_data)
_headerOffset := 0
@ -105,6 +109,129 @@ func MetadataDecodeIterate (_data []byte, _callback func ([]byte, []byte) ()) (e
func MetadataEncodeBinary (_metadata map[string]string) ([]byte, error) {
if len (_metadata) > 128 {
return nil, fmt.Errorf ("[2249daa0] invalid metadata: too large!")
}
_metadataArray := make ([][2]string, 0, len (_metadata))
for _key, _value := range _metadata {
if _value == "" {
continue
}
_metadataArray = append (_metadataArray, [2]string {_key, _value})
}
sort.Slice (_metadataArray,
func (i int, j int) (bool) {
return _metadataArray[i][0] < _metadataArray[j][0]
})
_buffer := & bytes.Buffer {}
for _, _metadata := range _metadataArray {
_key := _metadata[0]
_value := _metadata[1]
if ! metadataKeyRegex.MatchString (_key) {
return nil, fmt.Errorf ("[9c53ceb6] invalid metadata key: `%s`", _key)
}
if ! metadataValueRegex.MatchString (_value) {
return nil, fmt.Errorf ("[f932f38f] invalid metadata value: `%s`", _value)
}
_keyId, _keyFound := CanonicalHeaderNamesToKey[_key]
_valueId, _valueFound := CanonicalHeaderValuesToKey[_value]
if !_keyFound {
return nil, fmt.Errorf ("[a2a62863] invalid metadata key: `%s` (not canonical)", _key)
}
if !_valueFound {
return nil, fmt.Errorf ("[5ed34411] invalid metadata value: `%s` (not canonical)", _value)
}
var _pairBuffer [16]byte
if _error := EncodeKeysPairToBytes_0 (NamespaceHeaderName, _keyId, NamespaceHeaderValue, _valueId, _pairBuffer[:]); _error != nil {
return nil, _error
}
_buffer.Write (_pairBuffer[:])
}
_data := _buffer.Bytes ()
return _data, nil
}
func MetadataDecodeBinary (_data []byte) ([][2]string, error) {
_metadata := make ([][2]string, 0, 16)
_metadataAppend := func (_key []byte, _value []byte) () {
_metadata = append (_metadata, [2]string { string (_key), string (_value) })
}
if _error := MetadataDecodeBinaryIterate (_data, _metadataAppend); _error != nil {
return nil, _error
} else {
return _metadata, nil
}
}
func MetadataDecodeBinaryIterate (_data []byte, _callback func ([]byte, []byte) ()) (error) {
_dataLimit := len (_data)
_dataOffset := 0
for {
_sliceSize := _dataLimit - _dataOffset
if _sliceSize == 0 {
return nil
}
_slice := _data[_dataOffset:]
if _slice[0] != NamespaceHeaderNamePrefix {
return fmt.Errorf ("[f49c93cb] invalid metadata encoding")
}
if _sliceSize < 8 {
return fmt.Errorf ("[e8d008dc] invalid metadata encoding")
}
var _key []byte
if _key_0, _found := CanonicalHeaderNamesFromKey[DecodeKey_9 (_slice[0:8])]; _found {
_key = StringToBytes (_key_0)
} else {
return fmt.Errorf ("[7aa09c0f] invalid metadata encoding")
}
_dataOffset += 8
_sliceSize = _dataLimit - _dataOffset
if _sliceSize == 0 {
return fmt.Errorf ("[77c8bef7] invalid metadata encoding")
}
_slice = _data[_dataOffset:]
if _slice[0] != NamespaceHeaderValuePrefix {
return fmt.Errorf ("[2b43651c] invalid metadata encoding")
}
if _sliceSize < 8 {
return fmt.Errorf ("[7cd40b03] invalid metadata encoding")
}
var _value []byte
if _value_0, _found := CanonicalHeaderValuesFromKey[DecodeKey_9 (_slice[0:8])]; _found {
_value = StringToBytes (_value_0)
} else {
return fmt.Errorf ("[334e65ef] invalid metadata encoding")
}
_dataOffset += 8
_callback (_key, _value)
}
}
var metadataKeyRegex = regexp.MustCompile (`\A(?:[A-Z0-9](?:[a-z0-9]?[a-z]+)(?:-[A-Z0-9](?:[a-z0-9]?[a-z]+))*)|ETag\z`)
var metadataValueRegex = regexp.MustCompile (`\A[[:graph:]](?: ?[[:graph:]]+)*\z`)

21
sources/lib/common/namespaces.go
View file

@ -8,11 +8,30 @@ package common
const NamespaceSchemaVersion = "schema"
const CurrentSchemaVersion = "kawipiko-2021b"
const NamespaceFilesContent = "files-content"
const NamespaceFilesContentPrefix byte = 'f'
const NamespaceFilesIndex = "files-index"
const NamespaceFilesIndexPrefix byte = 'F'
const NamespaceFoldersContent = "folders-content"
const NamespaceFoldersContentPrefix byte = 'l'
const NamespaceFoldersIndex = "folders-index"
const NamespaceFoldersIndexPrefix byte = 'L'
const NamespaceDataMetadata = "resource-metadata"
const NamespaceDataMetadataPrefix byte = 'm'
const NamespaceDataContent = "resource-data"
const NamespaceDataMetadata = "resource-metadata"
const NamespaceDataContentPrefix byte = 'd'
const NamespaceHeaderName = "header-name"
const NamespaceHeaderNamePrefix byte = 'h'
const NamespaceHeaderValue = "header-value"
const NamespaceHeaderValuePrefix byte = 'H'