From 8f8b9d6f9ab33bac41d38f26c1c6d4c0a77b5809 Mon Sep 17 00:00:00 2001 From: Ciprian Dorin Craciun Date: Tue, 13 Aug 2019 21:08:16 +0300 Subject: [PATCH] [sources] Move archiver and server under proper packages --- sources/cmd/archiver.go | 770 +-------------------- sources/cmd/archiver/archiver.go | 780 ++++++++++++++++++++++ sources/cmd/server.go | 1062 +---------------------------- sources/cmd/server/server.go | 1072 ++++++++++++++++++++++++++++++ sources/lib/common/main.go | 2 +- 5 files changed, 1857 insertions(+), 1829 deletions(-) create mode 100644 sources/cmd/archiver/archiver.go create mode 100644 sources/cmd/server/server.go diff --git a/sources/cmd/archiver.go b/sources/cmd/archiver.go index 7fb6249..d7b3aef 100644 --- a/sources/cmd/archiver.go +++ b/sources/cmd/archiver.go @@ -3,778 +3,12 @@ package main -import "crypto/sha256" -import "encoding/hex" -import "encoding/json" -import "flag" -import "fmt" -import "io" -import "io/ioutil" -import "log" -import "net/http" -import "path/filepath" -import "os" -import "sort" -import "strings" -import "syscall" - -import "github.com/colinmarc/cdb" - -import . "github.com/volution/kawipiko/lib/common" -import . "github.com/volution/kawipiko/lib/archiver" - - - - -type context struct { - cdbWriter *cdb.Writer - storedFilePaths []string - storedFolderPaths []string - storedDataMeta map[string]bool - storedDataContent map[string]bool - storedDataContentMeta map[string]map[string]string - storedFiles map[[2]uint64][2]string - compress string - includeIndex bool - includeStripped bool - includeCache bool - includeEtag bool - includeFileListing bool - includeFolderListing bool - debug bool -} - - - - -func archiveFile (_context *context, _pathResolved string, _pathInArchive string, _name string) (error) { - - var _file *os.File - if _file_0, _error := os.Open (_pathResolved); _error == nil { - _file = _file_0 - } else { - return _error - } - - defer _file.Close () - - var _fileId [2]uint64 - if _stat, _error := _file.Stat (); _error == nil { - _stat := _stat.Sys() - if _stat, _ok := _stat.(*syscall.Stat_t); _ok { - _fileId = [2]uint64 { _stat.Dev, _stat.Ino } - } else { - return fmt.Errorf ("[6578d2d7] failed `stat`-ing: `%s`!", _pathResolved) - } - } - - var _wasStored bool - var _fingerprintContent string - var _fingerprintMeta string - if _fingerprints, _wasStored_0 := _context.storedFiles[_fileId]; _wasStored_0 { - _fingerprintContent = _fingerprints[0] - _fingerprintMeta = _fingerprints[1] - _wasStored = true - } - - var _dataContent []byte - var _dataMeta map[string]string - var _dataMetaRaw []byte - - if ! _wasStored { - - if _dataContent_0, _error := ioutil.ReadAll (_file); _error == nil { - _dataContent = _dataContent_0 - } else { - return _error - } - - if _fingerprintContent_0, _dataContent_0, _dataMeta_0, _error := prepareDataContent (_context, _pathResolved, _pathInArchive, _name, _dataContent, ""); _error != nil { - return _error - } else { - _fingerprintContent = _fingerprintContent_0 - _dataContent = _dataContent_0 - _dataMeta = _dataMeta_0 - } - if _fingerprintMeta_0, _dataMetaRaw_0, _error := prepareDataMeta (_context, _dataMeta); _error != nil { - return _error - } else { - _fingerprintMeta = _fingerprintMeta_0 - _dataMetaRaw = _dataMetaRaw_0 - } - } - - if _context.includeStripped { - for _, _suffix := range StripSuffixes { - if strings.HasSuffix (_pathInArchive, _suffix) { - _pathInArchive := _pathInArchive [: len (_pathInArchive) - len (_suffix)] - if _error := archiveReference (_context, NamespaceFilesContent, _pathInArchive, _fingerprintContent, _fingerprintMeta); _error != nil { - return _error - } - break - } - } - } - - if _error := archiveReference (_context, NamespaceFilesContent, _pathInArchive, _fingerprintContent, _fingerprintMeta); _error != nil { - return _error - } - if _dataMetaRaw != nil { - if _error := archiveDataMeta (_context, _fingerprintMeta, _dataMetaRaw); _error != nil { - return _error - } - } - if _dataContent != nil { - if _error := archiveDataContent (_context, _fingerprintContent, _dataContent); _error != nil { - return _error - } - } - - if ! _wasStored { - _context.storedFiles[_fileId] = [2]string { _fingerprintContent, _fingerprintMeta } - } - - return nil -} - - - - -func archiveFolder (_context *context, _pathResolved string, _pathInArchive string, _names []string, _stats map[string]os.FileInfo) (error) { - - type Entry struct { - Name string `json:"name,omitempty"` - Type string `json:"type,omitempty"` - Size uint64 `json:"size,omitempty"` - } - - type Folder struct { - Entries []Entry `json:"entries,omitempty"` - Indices []string `json:"indices,omitempty"` - } - - _entries := make ([]Entry, 0, len (_names)) - if _context.includeFolderListing { - for _, _name := range _names { - _entry := Entry { - Name : _name, - Type : "unknown", - } - _stat := _stats[_name] - _statMode := _stat.Mode () - if _statMode.IsRegular () { - _entry.Type = "file" - _entry.Size = uint64 (_stat.Size ()) - } else if _statMode.IsDir () { - _entry.Type = "folder" - } - _entries = append (_entries, _entry) - } - } - - _indexNames := make ([]string, 0, 4) - if _context.includeIndex { - var _indexNameFirst string - for _, _indexName := range IndexNames { - _indexNameFound := sort.SearchStrings (_names, _indexName) - if _indexNameFound == len (_names) { - continue - } - if _names[_indexNameFound] != _indexName { - continue - } - _stat := _stats[_indexName] - _statMode := _stat.Mode () - if ! _statMode.IsRegular () { - continue - } - if _indexNameFirst == "" { - _indexNameFirst = _indexName - } - _indexNames = append (_indexNames, _indexName) - } - if _indexNameFirst != "" { - _indexPathResolved := filepath.Join (_pathResolved, _indexNameFirst) - _indexPathInArchive := _pathInArchive + "/" - if _pathInArchive == "/" { - _indexPathInArchive = "/" - } - archiveFile (_context, _indexPathResolved, _indexPathInArchive, _indexNameFirst) - } - } - - if ! _context.includeFolderListing { - return nil - } - - _folder := Folder { - Entries : _entries, - Indices : _indexNames, - } - - var _data []byte - if _data_0, _error := json.Marshal (&_folder); _error == nil { - _data = _data_0 - } else { - return _error - } - - if _, _, _error := archiveReferenceAndData (_context, NamespaceFoldersContent, _pathResolved, _pathInArchive, "", _data, MimeTypeJson); _error != nil { - return _error - } - - return nil -} - - - - -func archiveReferenceAndData (_context *context, _namespace string, _pathResolved string, _pathInArchive string, _name string, _dataContent []byte, _dataType string) (string, string, error) { - - var _fingerprintContent string - var _fingerprintMeta string - var _dataMeta map[string]string - var _dataMetaRaw []byte - - if _fingerprintContent_0, _dataContent_0, _dataMeta_0, _error := prepareDataContent (_context, _pathResolved, _pathInArchive, _name, _dataContent, _dataType); _error != nil { - return "", "", _error - } else { - _fingerprintContent = _fingerprintContent_0 - _dataContent = _dataContent_0 - _dataMeta = _dataMeta_0 - } - if _fingerprintMeta_0, _dataMetaRaw_0, _error := prepareDataMeta (_context, _dataMeta); _error != nil { - return "", "", _error - } else { - _fingerprintMeta = _fingerprintMeta_0 - _dataMetaRaw = _dataMetaRaw_0 - } - - if _error := archiveReference (_context, _namespace, _pathInArchive, _fingerprintContent, _fingerprintMeta); _error != nil { - return "", "", _error - } - if _dataMetaRaw != nil { - if _error := archiveDataMeta (_context, _fingerprintMeta, _dataMetaRaw); _error != nil { - return "", "", _error - } - } - if _dataContent != nil { - if _error := archiveDataContent (_context, _fingerprintContent, _dataContent); _error != nil { - return "", "", _error - } - } - - return _fingerprintContent, _fingerprintMeta, nil -} - - - - -func archiveDataContent (_context *context, _fingerprintContent string, _dataContent []byte) (error) { - - if _wasStored, _ := _context.storedDataContent[_fingerprintContent]; _wasStored { - return fmt.Errorf ("[256cde78] data content already stored: `%s`!", _fingerprintContent) - } - - { - _key := fmt.Sprintf ("%s:%s", NamespaceDataContent, _fingerprintContent) - if _context.debug { - log.Printf ("[ ] data-content ++ `%s`\n", _key) - } - if _error := _context.cdbWriter.Put ([]byte (_key), _dataContent); _error != nil { - return _error - } - } - - _context.storedDataContent[_fingerprintContent] = true - - return nil -} - - -func archiveDataMeta (_context *context, _fingerprintMeta string, _dataMeta []byte) (error) { - - if _wasStored, _ := _context.storedDataMeta[_fingerprintMeta]; _wasStored { - return fmt.Errorf ("[2918c4e2] data meta already stored: `%s`!", _fingerprintMeta) - } - - { - _key := fmt.Sprintf ("%s:%s", NamespaceDataMetadata, _fingerprintMeta) - if _context.debug { - log.Printf ("[ ] data-meta ++ `%s`\n", _key) - } - if _error := _context.cdbWriter.Put ([]byte (_key), _dataMeta); _error != nil { - return _error - } - } - - _context.storedDataMeta[_fingerprintMeta] = true - - return nil -} - - - - -func archiveReference (_context *context, _namespace string, _pathInArchive string, _fingerprintContent string, _fingerprintMeta string) (error) { - - switch _namespace { - case NamespaceFilesContent : - _context.storedFilePaths = append (_context.storedFilePaths, _pathInArchive) - case NamespaceFoldersContent : - _context.storedFolderPaths = append (_context.storedFolderPaths, _pathInArchive) - default : - return fmt.Errorf ("[051a102a]") - } - - _key := fmt.Sprintf ("%s:%s", _namespace, _pathInArchive) - if _context.debug { - log.Printf ("[ ] reference ++ `%s` :: `%s` -> `%s` ~ `%s`\n", _namespace, _pathInArchive, _fingerprintContent[:16], _fingerprintMeta[:16]) - } - - _fingerprints := fmt.Sprintf ("%s:%s", _fingerprintContent, _fingerprintMeta) - if _error := _context.cdbWriter.Put ([]byte (_key), []byte (_fingerprints)); _error != nil { - return _error - } - - return nil -} - - - - -func prepareDataContent (_context *context, _pathResolved string, _pathInArchive string, _name string, _dataContent []byte, _dataType string) (string, []byte, map[string]string, error) { - - _fingerprintContentRaw := sha256.Sum256 (_dataContent) - _fingerprintContent := hex.EncodeToString (_fingerprintContentRaw[:]) - - if _wasStored, _ := _context.storedDataContent[_fingerprintContent]; _wasStored { - _dataMeta := _context.storedDataContentMeta[_fingerprintContent] - return _fingerprintContent, nil, _dataMeta, nil - } - - if (_dataType == "") && (_name != "") { - _extension := filepath.Ext (_name) - if _extension != "" { - _extension = _extension[1:] - } - _dataType, _ = MimeTypesByExtension[_extension] - } - if _dataType == "" { - _dataType = http.DetectContentType (_dataContent) - } - if _dataType == "" { - _dataType = MimeTypeRaw - } - - _dataEncoding := "identity" - _dataUncompressedSize := len (_dataContent) - _dataSize := _dataUncompressedSize - if _dataSize > 512 { - if _dataContent_0, _dataEncoding_0, _error := Compress (_dataContent, _context.compress); _error == nil { - if _dataEncoding_0 != "identity" { - _dataCompressedSize := len (_dataContent_0) - _dataCompressedDelta := _dataUncompressedSize - _dataCompressedSize - _dataCompressedRatio := (_dataCompressedDelta * 100) / _dataUncompressedSize - _accepted := false - _accepted = _accepted || ((_dataUncompressedSize > (1024 * 1024)) && (_dataCompressedRatio >= 5)) - _accepted = _accepted || ((_dataUncompressedSize > (64 * 1024)) && (_dataCompressedRatio >= 10)) - _accepted = _accepted || ((_dataUncompressedSize > (32 * 1024)) && (_dataCompressedRatio >= 15)) - _accepted = _accepted || ((_dataUncompressedSize > (16 * 1024)) && (_dataCompressedRatio >= 20)) - _accepted = _accepted || ((_dataUncompressedSize > (8 * 1024)) && (_dataCompressedRatio >= 25)) - _accepted = _accepted || ((_dataUncompressedSize > (4 * 1024)) && (_dataCompressedRatio >= 30)) - _accepted = _accepted || ((_dataUncompressedSize > (2 * 1024)) && (_dataCompressedRatio >= 35)) - _accepted = _accepted || ((_dataUncompressedSize > (1 * 1024)) && (_dataCompressedRatio >= 40)) - _accepted = _accepted || (_dataCompressedRatio >= 90) - if _accepted { - _dataContent = _dataContent_0 - _dataEncoding = _dataEncoding_0 - _dataSize = _dataCompressedSize - } - if _dataSize < _dataUncompressedSize { - if _context.debug { - log.Printf ("[ ] compress %02d %8d %8d `%s`\n", _dataCompressedRatio, _dataUncompressedSize, _dataCompressedDelta, _pathInArchive) - } - } else { - if _context.debug { - log.Printf ("[ ] compress-NOK %02d %8d %8d `%s`\n", _dataCompressedRatio, _dataUncompressedSize, _dataCompressedDelta, _pathInArchive) - } - } - } - } else { - return "", nil, nil, _error - } - } else { - if _context.debug && (_context.compress != "identity") { - log.Printf ("[ ] compress-NOK %8d `%s`\n", _dataUncompressedSize, _pathInArchive) - } - } - - _dataMeta := make (map[string]string, 16) - - // _dataMeta["Content-Length"] = fmt.Sprintf ("%d", _dataSize) - _dataMeta["Content-Type"] = _dataType - _dataMeta["Content-Encoding"] = _dataEncoding - - if _context.includeCache { - _dataMeta["Cache-Control"] = "public, immutable, max-age=3600" - } - if _context.includeEtag { - _dataMeta["ETag"] = _fingerprintContent - } - - _context.storedDataContentMeta[_fingerprintContent] = _dataMeta - - return _fingerprintContent, _dataContent, _dataMeta, nil -} - - -func prepareDataMeta (_context *context, _dataMeta map[string]string) (string, []byte, error) { - - var _dataMetaRaw []byte - if _dataMetaRaw_0, _error := MetadataEncode (_dataMeta); _error == nil { - _dataMetaRaw = _dataMetaRaw_0 - } else { - return "", nil, _error - } - - _fingerprintMetaRaw := sha256.Sum256 (_dataMetaRaw) - _fingerprintMeta := hex.EncodeToString (_fingerprintMetaRaw[:]) - - if _wasStored, _ := _context.storedDataMeta[_fingerprintMeta]; _wasStored { - return _fingerprintMeta, nil, nil - } - - return _fingerprintMeta, _dataMetaRaw, nil -} - - - - -func walkPath (_context *context, _pathResolved string, _pathInArchive string, _name string, _recursed map[string]uint, _recurse bool) (os.FileInfo, error) { - - if _recursed == nil { - _recursed = make (map[string]uint, 128) - } - - _pathOriginal := _pathResolved - - var _stat os.FileInfo - if _stat_0, _error := os.Lstat (_pathResolved); _error == nil { - _stat = _stat_0 - } else { - return nil, _error - } - _statMode := _stat.Mode () - - _isSymlink := false - if (_stat.Mode () & os.ModeSymlink) != 0 { - _isSymlink = true - if _stat_0, _error := os.Stat (_pathResolved); _error == nil { - _stat = _stat_0 - } else { - return nil, _error - } - } - _statMode = _stat.Mode () - - if ! _recurse { - return _stat, nil - } - - if _isSymlink { - if _pathResolved_0, _error := filepath.EvalSymlinks (_pathResolved); _error == nil { - _pathResolved = _pathResolved_0 - } else { - return nil, _error - } - } - - if _isSymlink && _context.debug { - log.Printf ("[ ] symlink :: `%s` -> `%s`\n", _pathInArchive, _pathResolved) - } - - - if _statMode.IsRegular () { - - if _context.debug { - log.Printf ("[ ] file :: `%s`\n", _pathInArchive) - } - if _error := archiveFile (_context, _pathResolved, _pathInArchive, _name); _error != nil { - return nil, _error - } - return _stat, nil - - } else if _statMode.IsDir () { - - _wasRecursed, _ := _recursed[_pathResolved] - if _wasRecursed > 0 { - log.Printf ("[ww] [2e1744c9] detected directory loop for `%s` resolving to `%s`; ignoring!\n", _pathOriginal, _pathResolved) - return _stat, nil - } - _recursed[_pathResolved] = _wasRecursed + 1 - - _childsName := make ([]string, 0, 16) - _childsPathResolved := make (map[string]string, 16) - _childsPathInArchive := make (map[string]string, 16) - _childsStat := make (map[string]os.FileInfo, 16) - - var _wildcardName string - - if _context.debug { - log.Printf ("[ ] folder >> `%s`\n", _pathInArchive) - } - if _stream, _error := os.Open (_pathResolved); _error == nil { - defer _stream.Close () - _loop : for { - switch _buffer, _error := _stream.Readdir (128); _error { - case nil : - for _, _childStat := range _buffer { - - _childName := _childStat.Name () - _childPathResolved := filepath.Join (_pathResolved, _childName) - _childPathInArchive := filepath.Join (_pathInArchive, _childName) - - if _childStat_0, _error := walkPath (_context, _childPathResolved, _childPathInArchive, _childName, _recursed, false); _error == nil { - _childStat = _childStat_0 - } else { - return nil, _error - } - - _childsPathResolved[_childName] = _childPathResolved - _childsPathInArchive[_childName] = _childPathInArchive - _childsStat[_childName] = _childStat - - if strings.HasPrefix (_childName, "_wildcard.") { - _wildcardName = _childName - continue - } - if ShouldSkipName (_childName) { - if _context.debug { - log.Printf ("[ ] skip !! `%s`\n", _childPathInArchive) - } - continue - } - - _childsName = append (_childsName, _childName) - } - case io.EOF : - break _loop - default : - return nil, _error - } - } - } - if _context.debug { - log.Printf ("[ ] folder << `%s`\n", _pathInArchive) - } - - sort.Strings (_childsName) - - if _context.debug { - log.Printf ("[ ] folder :: `%s`\n", _pathInArchive) - } - if _error := archiveFolder (_context, _pathResolved, _pathInArchive, _childsName, _childsStat); _error != nil { - return nil, _error - } - - if _wildcardName != "" { - _childPathInArchive := filepath.Join (_pathInArchive, "*") - if _, _error := walkPath (_context, _childsPathResolved[_wildcardName], _childPathInArchive, _wildcardName, _recursed, true); _error != nil { - return nil, _error - } - } - - if _context.debug { - log.Printf ("[ ] folder >> `%s`\n", _pathInArchive) - } - for _, _childName := range _childsName { - if _childsStat[_childName] .Mode () .IsRegular () { - if _, _error := walkPath (_context, _childsPathResolved[_childName], _childsPathInArchive[_childName], _childName, _recursed, true); _error != nil { - return nil, _error - } - } - } - for _, _childName := range _childsName { - if _childsStat[_childName] .Mode () .IsDir () { - if _, _error := walkPath (_context, _childsPathResolved[_childName], _childsPathInArchive[_childName], _childName, _recursed, true); _error != nil { - return nil, _error - } - } - } - for _, _childName := range _childsName { - if (! _childsStat[_childName] .Mode () .IsRegular ()) && (! _childsStat[_childName] .Mode () .IsDir ()) { - if _, _error := walkPath (_context, _childsPathResolved[_childName], _childsPathInArchive[_childName], _childName, _recursed, true); _error != nil { - return nil, _error - } - } - } - if _context.debug { - log.Printf ("[ ] folder << `%s`\n", _pathInArchive) - } - - _recursed[_pathResolved] = _wasRecursed - return _stat, nil - - } else { - return nil, fmt.Errorf ("[d9b836d7] unexpected file type for `%s`: `%s`!", _pathResolved, _statMode) - } -} +import . "github.com/volution/kawipiko/cmd/archiver" func main () () { - Main (main_0) -} - - -func main_0 () (error) { - - - var _sourcesFolder string - var _archiveFile string - var _compress string - var _includeIndex bool - var _includeStripped bool - var _includeCache bool - var _includeEtag bool - var _includeFileListing bool - var _includeFolderListing bool - var _debug bool - - { - _flags := flag.NewFlagSet ("kawipiko-archiver", flag.ContinueOnError) - - _flags.Usage = func () () { - fmt.Fprintf (os.Stderr, "%s", -` - ==== kawipiko -- blazingly fast static HTTP server ==== - - | Documentation, issues and sources: - | * https://github.com/volution/kawipiko - | Authors: - | * Ciprian Dorin Craciun - | ciprian@volution.ro - | ciprian.craciun@gmail.com - | https://volution.ro/ciprian - ----------------------------------------------------------- - - kawipiko-archiver - - --sources - - --archive - --compress - - --exclude-index - --exclude-strip - --exclude-cache - --include-etag - - --exclude-file-listing - --include-folder-listing - - --debug - - ** for details see: - https://github.com/volution/kawipiko#kawipiko-archiver - -`) - } - - _sourcesFolder_0 := _flags.String ("sources", "", "") - _archiveFile_0 := _flags.String ("archive", "", "") - _compress_0 := _flags.String ("compress", "", "") - _excludeIndex_0 := _flags.Bool ("exclude-index", false, "") - _excludeStripped_0 := _flags.Bool ("exclude-strip", false, "") - _excludeCache_0 := _flags.Bool ("exclude-cache", false, "") - _includeEtag_0 := _flags.Bool ("include-etag", false, "") - _excludeFileListing_0 := _flags.Bool ("exclude-file-listing", false, "") - _includeFolderListing_0 := _flags.Bool ("include-folder-listing", false, "") - _debug_0 := _flags.Bool ("debug", false, "") - - FlagsParse (_flags, 0, 0) - - _sourcesFolder = *_sourcesFolder_0 - _archiveFile = *_archiveFile_0 - _compress = *_compress_0 - _includeIndex = ! *_excludeIndex_0 - _includeStripped = ! *_excludeStripped_0 - _includeCache = ! *_excludeCache_0 - _includeEtag = *_includeEtag_0 - _includeFileListing = ! *_excludeFileListing_0 - _includeFolderListing = *_includeFolderListing_0 - _debug = *_debug_0 - - if _sourcesFolder == "" { - AbortError (nil, "[515ee462] expected sources folder argument!") - } - if _archiveFile == "" { - AbortError (nil, "[5e8da985] expected archive file argument!") - } - } - - - var _cdbWriter *cdb.Writer - if _cdbWriter_0, _error := cdb.Create (_archiveFile); _error == nil { - _cdbWriter = _cdbWriter_0 - } else { - AbortError (_error, "[85234ba0] failed creating archive (while opening)!") - } - - if _error := _cdbWriter.Put ([]byte (NamespaceSchemaVersion), []byte (CurrentSchemaVersion)); _error != nil { - AbortError (_error, "[43228812] failed writing archive!") - } - - _context := & context { - cdbWriter : _cdbWriter, - storedFilePaths : make ([]string, 0, 16 * 1024), - storedFolderPaths : make ([]string, 0, 16 * 1024), - storedDataMeta : make (map[string]bool, 16 * 1024), - storedDataContent : make (map[string]bool, 16 * 1024), - storedDataContentMeta : make (map[string]map[string]string, 16 * 1024), - storedFiles : make (map[[2]uint64][2]string, 16 * 1024), - compress : _compress, - includeIndex : _includeIndex, - includeStripped : _includeStripped, - includeCache : _includeCache, - includeEtag : _includeEtag, - includeFileListing : _includeFileListing, - includeFolderListing : _includeFolderListing, - debug : _debug, - } - - if _, _error := walkPath (_context, _sourcesFolder, "/", filepath.Base (_sourcesFolder), nil, true); _error != nil { - AbortError (_error, "[b6a19ef4] failed walking folder!") - } - - if _includeFileListing { - _buffer := make ([]byte, 0, 1024 * 1024) - for _, _path := range _context.storedFilePaths { - _buffer = append (_buffer, _path ...) - _buffer = append (_buffer, '\n') - } - if _error := _cdbWriter.Put ([]byte (NamespaceFilesIndex), _buffer); _error != nil { - AbortError (_error, "[1dbdde05] failed writing archive!") - } - } - - if _includeFolderListing { - _buffer := make ([]byte, 0, 1024 * 1024) - for _, _path := range _context.storedFolderPaths { - _buffer = append (_buffer, _path ...) - _buffer = append (_buffer, '\n') - } - if _error := _cdbWriter.Put ([]byte (NamespaceFoldersIndex), _buffer); _error != nil { - AbortError (_error, "[e2dd2de0] failed writing archive!") - } - } - - if _error := _cdbWriter.Close (); _error != nil { - AbortError (_error, "[bbfb8478] failed creating archive (while closing)!") - } - - return nil + Main () } diff --git a/sources/cmd/archiver/archiver.go b/sources/cmd/archiver/archiver.go new file mode 100644 index 0000000..6f4e7b0 --- /dev/null +++ b/sources/cmd/archiver/archiver.go @@ -0,0 +1,780 @@ + + +package archiver + + +import "crypto/sha256" +import "encoding/hex" +import "encoding/json" +import "flag" +import "fmt" +import "io" +import "io/ioutil" +import "log" +import "net/http" +import "path/filepath" +import "os" +import "sort" +import "strings" +import "syscall" + +import "github.com/colinmarc/cdb" + +import . "github.com/volution/kawipiko/lib/common" +import . "github.com/volution/kawipiko/lib/archiver" + + + + +type context struct { + cdbWriter *cdb.Writer + storedFilePaths []string + storedFolderPaths []string + storedDataMeta map[string]bool + storedDataContent map[string]bool + storedDataContentMeta map[string]map[string]string + storedFiles map[[2]uint64][2]string + compress string + includeIndex bool + includeStripped bool + includeCache bool + includeEtag bool + includeFileListing bool + includeFolderListing bool + debug bool +} + + + + +func archiveFile (_context *context, _pathResolved string, _pathInArchive string, _name string) (error) { + + var _file *os.File + if _file_0, _error := os.Open (_pathResolved); _error == nil { + _file = _file_0 + } else { + return _error + } + + defer _file.Close () + + var _fileId [2]uint64 + if _stat, _error := _file.Stat (); _error == nil { + _stat := _stat.Sys() + if _stat, _ok := _stat.(*syscall.Stat_t); _ok { + _fileId = [2]uint64 { _stat.Dev, _stat.Ino } + } else { + return fmt.Errorf ("[6578d2d7] failed `stat`-ing: `%s`!", _pathResolved) + } + } + + var _wasStored bool + var _fingerprintContent string + var _fingerprintMeta string + if _fingerprints, _wasStored_0 := _context.storedFiles[_fileId]; _wasStored_0 { + _fingerprintContent = _fingerprints[0] + _fingerprintMeta = _fingerprints[1] + _wasStored = true + } + + var _dataContent []byte + var _dataMeta map[string]string + var _dataMetaRaw []byte + + if ! _wasStored { + + if _dataContent_0, _error := ioutil.ReadAll (_file); _error == nil { + _dataContent = _dataContent_0 + } else { + return _error + } + + if _fingerprintContent_0, _dataContent_0, _dataMeta_0, _error := prepareDataContent (_context, _pathResolved, _pathInArchive, _name, _dataContent, ""); _error != nil { + return _error + } else { + _fingerprintContent = _fingerprintContent_0 + _dataContent = _dataContent_0 + _dataMeta = _dataMeta_0 + } + if _fingerprintMeta_0, _dataMetaRaw_0, _error := prepareDataMeta (_context, _dataMeta); _error != nil { + return _error + } else { + _fingerprintMeta = _fingerprintMeta_0 + _dataMetaRaw = _dataMetaRaw_0 + } + } + + if _context.includeStripped { + for _, _suffix := range StripSuffixes { + if strings.HasSuffix (_pathInArchive, _suffix) { + _pathInArchive := _pathInArchive [: len (_pathInArchive) - len (_suffix)] + if _error := archiveReference (_context, NamespaceFilesContent, _pathInArchive, _fingerprintContent, _fingerprintMeta); _error != nil { + return _error + } + break + } + } + } + + if _error := archiveReference (_context, NamespaceFilesContent, _pathInArchive, _fingerprintContent, _fingerprintMeta); _error != nil { + return _error + } + if _dataMetaRaw != nil { + if _error := archiveDataMeta (_context, _fingerprintMeta, _dataMetaRaw); _error != nil { + return _error + } + } + if _dataContent != nil { + if _error := archiveDataContent (_context, _fingerprintContent, _dataContent); _error != nil { + return _error + } + } + + if ! _wasStored { + _context.storedFiles[_fileId] = [2]string { _fingerprintContent, _fingerprintMeta } + } + + return nil +} + + + + +func archiveFolder (_context *context, _pathResolved string, _pathInArchive string, _names []string, _stats map[string]os.FileInfo) (error) { + + type Entry struct { + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + Size uint64 `json:"size,omitempty"` + } + + type Folder struct { + Entries []Entry `json:"entries,omitempty"` + Indices []string `json:"indices,omitempty"` + } + + _entries := make ([]Entry, 0, len (_names)) + if _context.includeFolderListing { + for _, _name := range _names { + _entry := Entry { + Name : _name, + Type : "unknown", + } + _stat := _stats[_name] + _statMode := _stat.Mode () + if _statMode.IsRegular () { + _entry.Type = "file" + _entry.Size = uint64 (_stat.Size ()) + } else if _statMode.IsDir () { + _entry.Type = "folder" + } + _entries = append (_entries, _entry) + } + } + + _indexNames := make ([]string, 0, 4) + if _context.includeIndex { + var _indexNameFirst string + for _, _indexName := range IndexNames { + _indexNameFound := sort.SearchStrings (_names, _indexName) + if _indexNameFound == len (_names) { + continue + } + if _names[_indexNameFound] != _indexName { + continue + } + _stat := _stats[_indexName] + _statMode := _stat.Mode () + if ! _statMode.IsRegular () { + continue + } + if _indexNameFirst == "" { + _indexNameFirst = _indexName + } + _indexNames = append (_indexNames, _indexName) + } + if _indexNameFirst != "" { + _indexPathResolved := filepath.Join (_pathResolved, _indexNameFirst) + _indexPathInArchive := _pathInArchive + "/" + if _pathInArchive == "/" { + _indexPathInArchive = "/" + } + archiveFile (_context, _indexPathResolved, _indexPathInArchive, _indexNameFirst) + } + } + + if ! _context.includeFolderListing { + return nil + } + + _folder := Folder { + Entries : _entries, + Indices : _indexNames, + } + + var _data []byte + if _data_0, _error := json.Marshal (&_folder); _error == nil { + _data = _data_0 + } else { + return _error + } + + if _, _, _error := archiveReferenceAndData (_context, NamespaceFoldersContent, _pathResolved, _pathInArchive, "", _data, MimeTypeJson); _error != nil { + return _error + } + + return nil +} + + + + +func archiveReferenceAndData (_context *context, _namespace string, _pathResolved string, _pathInArchive string, _name string, _dataContent []byte, _dataType string) (string, string, error) { + + var _fingerprintContent string + var _fingerprintMeta string + var _dataMeta map[string]string + var _dataMetaRaw []byte + + if _fingerprintContent_0, _dataContent_0, _dataMeta_0, _error := prepareDataContent (_context, _pathResolved, _pathInArchive, _name, _dataContent, _dataType); _error != nil { + return "", "", _error + } else { + _fingerprintContent = _fingerprintContent_0 + _dataContent = _dataContent_0 + _dataMeta = _dataMeta_0 + } + if _fingerprintMeta_0, _dataMetaRaw_0, _error := prepareDataMeta (_context, _dataMeta); _error != nil { + return "", "", _error + } else { + _fingerprintMeta = _fingerprintMeta_0 + _dataMetaRaw = _dataMetaRaw_0 + } + + if _error := archiveReference (_context, _namespace, _pathInArchive, _fingerprintContent, _fingerprintMeta); _error != nil { + return "", "", _error + } + if _dataMetaRaw != nil { + if _error := archiveDataMeta (_context, _fingerprintMeta, _dataMetaRaw); _error != nil { + return "", "", _error + } + } + if _dataContent != nil { + if _error := archiveDataContent (_context, _fingerprintContent, _dataContent); _error != nil { + return "", "", _error + } + } + + return _fingerprintContent, _fingerprintMeta, nil +} + + + + +func archiveDataContent (_context *context, _fingerprintContent string, _dataContent []byte) (error) { + + if _wasStored, _ := _context.storedDataContent[_fingerprintContent]; _wasStored { + return fmt.Errorf ("[256cde78] data content already stored: `%s`!", _fingerprintContent) + } + + { + _key := fmt.Sprintf ("%s:%s", NamespaceDataContent, _fingerprintContent) + if _context.debug { + log.Printf ("[ ] data-content ++ `%s`\n", _key) + } + if _error := _context.cdbWriter.Put ([]byte (_key), _dataContent); _error != nil { + return _error + } + } + + _context.storedDataContent[_fingerprintContent] = true + + return nil +} + + +func archiveDataMeta (_context *context, _fingerprintMeta string, _dataMeta []byte) (error) { + + if _wasStored, _ := _context.storedDataMeta[_fingerprintMeta]; _wasStored { + return fmt.Errorf ("[2918c4e2] data meta already stored: `%s`!", _fingerprintMeta) + } + + { + _key := fmt.Sprintf ("%s:%s", NamespaceDataMetadata, _fingerprintMeta) + if _context.debug { + log.Printf ("[ ] data-meta ++ `%s`\n", _key) + } + if _error := _context.cdbWriter.Put ([]byte (_key), _dataMeta); _error != nil { + return _error + } + } + + _context.storedDataMeta[_fingerprintMeta] = true + + return nil +} + + + + +func archiveReference (_context *context, _namespace string, _pathInArchive string, _fingerprintContent string, _fingerprintMeta string) (error) { + + switch _namespace { + case NamespaceFilesContent : + _context.storedFilePaths = append (_context.storedFilePaths, _pathInArchive) + case NamespaceFoldersContent : + _context.storedFolderPaths = append (_context.storedFolderPaths, _pathInArchive) + default : + return fmt.Errorf ("[051a102a]") + } + + _key := fmt.Sprintf ("%s:%s", _namespace, _pathInArchive) + if _context.debug { + log.Printf ("[ ] reference ++ `%s` :: `%s` -> `%s` ~ `%s`\n", _namespace, _pathInArchive, _fingerprintContent[:16], _fingerprintMeta[:16]) + } + + _fingerprints := fmt.Sprintf ("%s:%s", _fingerprintContent, _fingerprintMeta) + if _error := _context.cdbWriter.Put ([]byte (_key), []byte (_fingerprints)); _error != nil { + return _error + } + + return nil +} + + + + +func prepareDataContent (_context *context, _pathResolved string, _pathInArchive string, _name string, _dataContent []byte, _dataType string) (string, []byte, map[string]string, error) { + + _fingerprintContentRaw := sha256.Sum256 (_dataContent) + _fingerprintContent := hex.EncodeToString (_fingerprintContentRaw[:]) + + if _wasStored, _ := _context.storedDataContent[_fingerprintContent]; _wasStored { + _dataMeta := _context.storedDataContentMeta[_fingerprintContent] + return _fingerprintContent, nil, _dataMeta, nil + } + + if (_dataType == "") && (_name != "") { + _extension := filepath.Ext (_name) + if _extension != "" { + _extension = _extension[1:] + } + _dataType, _ = MimeTypesByExtension[_extension] + } + if _dataType == "" { + _dataType = http.DetectContentType (_dataContent) + } + if _dataType == "" { + _dataType = MimeTypeRaw + } + + _dataEncoding := "identity" + _dataUncompressedSize := len (_dataContent) + _dataSize := _dataUncompressedSize + if _dataSize > 512 { + if _dataContent_0, _dataEncoding_0, _error := Compress (_dataContent, _context.compress); _error == nil { + if _dataEncoding_0 != "identity" { + _dataCompressedSize := len (_dataContent_0) + _dataCompressedDelta := _dataUncompressedSize - _dataCompressedSize + _dataCompressedRatio := (_dataCompressedDelta * 100) / _dataUncompressedSize + _accepted := false + _accepted = _accepted || ((_dataUncompressedSize > (1024 * 1024)) && (_dataCompressedRatio >= 5)) + _accepted = _accepted || ((_dataUncompressedSize > (64 * 1024)) && (_dataCompressedRatio >= 10)) + _accepted = _accepted || ((_dataUncompressedSize > (32 * 1024)) && (_dataCompressedRatio >= 15)) + _accepted = _accepted || ((_dataUncompressedSize > (16 * 1024)) && (_dataCompressedRatio >= 20)) + _accepted = _accepted || ((_dataUncompressedSize > (8 * 1024)) && (_dataCompressedRatio >= 25)) + _accepted = _accepted || ((_dataUncompressedSize > (4 * 1024)) && (_dataCompressedRatio >= 30)) + _accepted = _accepted || ((_dataUncompressedSize > (2 * 1024)) && (_dataCompressedRatio >= 35)) + _accepted = _accepted || ((_dataUncompressedSize > (1 * 1024)) && (_dataCompressedRatio >= 40)) + _accepted = _accepted || (_dataCompressedRatio >= 90) + if _accepted { + _dataContent = _dataContent_0 + _dataEncoding = _dataEncoding_0 + _dataSize = _dataCompressedSize + } + if _dataSize < _dataUncompressedSize { + if _context.debug { + log.Printf ("[ ] compress %02d %8d %8d `%s`\n", _dataCompressedRatio, _dataUncompressedSize, _dataCompressedDelta, _pathInArchive) + } + } else { + if _context.debug { + log.Printf ("[ ] compress-NOK %02d %8d %8d `%s`\n", _dataCompressedRatio, _dataUncompressedSize, _dataCompressedDelta, _pathInArchive) + } + } + } + } else { + return "", nil, nil, _error + } + } else { + if _context.debug && (_context.compress != "identity") { + log.Printf ("[ ] compress-NOK %8d `%s`\n", _dataUncompressedSize, _pathInArchive) + } + } + + _dataMeta := make (map[string]string, 16) + + // _dataMeta["Content-Length"] = fmt.Sprintf ("%d", _dataSize) + _dataMeta["Content-Type"] = _dataType + _dataMeta["Content-Encoding"] = _dataEncoding + + if _context.includeCache { + _dataMeta["Cache-Control"] = "public, immutable, max-age=3600" + } + if _context.includeEtag { + _dataMeta["ETag"] = _fingerprintContent + } + + _context.storedDataContentMeta[_fingerprintContent] = _dataMeta + + return _fingerprintContent, _dataContent, _dataMeta, nil +} + + +func prepareDataMeta (_context *context, _dataMeta map[string]string) (string, []byte, error) { + + var _dataMetaRaw []byte + if _dataMetaRaw_0, _error := MetadataEncode (_dataMeta); _error == nil { + _dataMetaRaw = _dataMetaRaw_0 + } else { + return "", nil, _error + } + + _fingerprintMetaRaw := sha256.Sum256 (_dataMetaRaw) + _fingerprintMeta := hex.EncodeToString (_fingerprintMetaRaw[:]) + + if _wasStored, _ := _context.storedDataMeta[_fingerprintMeta]; _wasStored { + return _fingerprintMeta, nil, nil + } + + return _fingerprintMeta, _dataMetaRaw, nil +} + + + + +func walkPath (_context *context, _pathResolved string, _pathInArchive string, _name string, _recursed map[string]uint, _recurse bool) (os.FileInfo, error) { + + if _recursed == nil { + _recursed = make (map[string]uint, 128) + } + + _pathOriginal := _pathResolved + + var _stat os.FileInfo + if _stat_0, _error := os.Lstat (_pathResolved); _error == nil { + _stat = _stat_0 + } else { + return nil, _error + } + _statMode := _stat.Mode () + + _isSymlink := false + if (_stat.Mode () & os.ModeSymlink) != 0 { + _isSymlink = true + if _stat_0, _error := os.Stat (_pathResolved); _error == nil { + _stat = _stat_0 + } else { + return nil, _error + } + } + _statMode = _stat.Mode () + + if ! _recurse { + return _stat, nil + } + + if _isSymlink { + if _pathResolved_0, _error := filepath.EvalSymlinks (_pathResolved); _error == nil { + _pathResolved = _pathResolved_0 + } else { + return nil, _error + } + } + + if _isSymlink && _context.debug { + log.Printf ("[ ] symlink :: `%s` -> `%s`\n", _pathInArchive, _pathResolved) + } + + + if _statMode.IsRegular () { + + if _context.debug { + log.Printf ("[ ] file :: `%s`\n", _pathInArchive) + } + if _error := archiveFile (_context, _pathResolved, _pathInArchive, _name); _error != nil { + return nil, _error + } + return _stat, nil + + } else if _statMode.IsDir () { + + _wasRecursed, _ := _recursed[_pathResolved] + if _wasRecursed > 0 { + log.Printf ("[ww] [2e1744c9] detected directory loop for `%s` resolving to `%s`; ignoring!\n", _pathOriginal, _pathResolved) + return _stat, nil + } + _recursed[_pathResolved] = _wasRecursed + 1 + + _childsName := make ([]string, 0, 16) + _childsPathResolved := make (map[string]string, 16) + _childsPathInArchive := make (map[string]string, 16) + _childsStat := make (map[string]os.FileInfo, 16) + + var _wildcardName string + + if _context.debug { + log.Printf ("[ ] folder >> `%s`\n", _pathInArchive) + } + if _stream, _error := os.Open (_pathResolved); _error == nil { + defer _stream.Close () + _loop : for { + switch _buffer, _error := _stream.Readdir (128); _error { + case nil : + for _, _childStat := range _buffer { + + _childName := _childStat.Name () + _childPathResolved := filepath.Join (_pathResolved, _childName) + _childPathInArchive := filepath.Join (_pathInArchive, _childName) + + if _childStat_0, _error := walkPath (_context, _childPathResolved, _childPathInArchive, _childName, _recursed, false); _error == nil { + _childStat = _childStat_0 + } else { + return nil, _error + } + + _childsPathResolved[_childName] = _childPathResolved + _childsPathInArchive[_childName] = _childPathInArchive + _childsStat[_childName] = _childStat + + if strings.HasPrefix (_childName, "_wildcard.") { + _wildcardName = _childName + continue + } + if ShouldSkipName (_childName) { + if _context.debug { + log.Printf ("[ ] skip !! `%s`\n", _childPathInArchive) + } + continue + } + + _childsName = append (_childsName, _childName) + } + case io.EOF : + break _loop + default : + return nil, _error + } + } + } + if _context.debug { + log.Printf ("[ ] folder << `%s`\n", _pathInArchive) + } + + sort.Strings (_childsName) + + if _context.debug { + log.Printf ("[ ] folder :: `%s`\n", _pathInArchive) + } + if _error := archiveFolder (_context, _pathResolved, _pathInArchive, _childsName, _childsStat); _error != nil { + return nil, _error + } + + if _wildcardName != "" { + _childPathInArchive := filepath.Join (_pathInArchive, "*") + if _, _error := walkPath (_context, _childsPathResolved[_wildcardName], _childPathInArchive, _wildcardName, _recursed, true); _error != nil { + return nil, _error + } + } + + if _context.debug { + log.Printf ("[ ] folder >> `%s`\n", _pathInArchive) + } + for _, _childName := range _childsName { + if _childsStat[_childName] .Mode () .IsRegular () { + if _, _error := walkPath (_context, _childsPathResolved[_childName], _childsPathInArchive[_childName], _childName, _recursed, true); _error != nil { + return nil, _error + } + } + } + for _, _childName := range _childsName { + if _childsStat[_childName] .Mode () .IsDir () { + if _, _error := walkPath (_context, _childsPathResolved[_childName], _childsPathInArchive[_childName], _childName, _recursed, true); _error != nil { + return nil, _error + } + } + } + for _, _childName := range _childsName { + if (! _childsStat[_childName] .Mode () .IsRegular ()) && (! _childsStat[_childName] .Mode () .IsDir ()) { + if _, _error := walkPath (_context, _childsPathResolved[_childName], _childsPathInArchive[_childName], _childName, _recursed, true); _error != nil { + return nil, _error + } + } + } + if _context.debug { + log.Printf ("[ ] folder << `%s`\n", _pathInArchive) + } + + _recursed[_pathResolved] = _wasRecursed + return _stat, nil + + } else { + return nil, fmt.Errorf ("[d9b836d7] unexpected file type for `%s`: `%s`!", _pathResolved, _statMode) + } +} + + + + +func Main () () { + Main_0 (main_0) +} + + +func main_0 () (error) { + + + var _sourcesFolder string + var _archiveFile string + var _compress string + var _includeIndex bool + var _includeStripped bool + var _includeCache bool + var _includeEtag bool + var _includeFileListing bool + var _includeFolderListing bool + var _debug bool + + { + _flags := flag.NewFlagSet ("kawipiko-archiver", flag.ContinueOnError) + + _flags.Usage = func () () { + fmt.Fprintf (os.Stderr, "%s", +` + ==== kawipiko -- blazingly fast static HTTP server ==== + + | Documentation, issues and sources: + | * https://github.com/volution/kawipiko + | Authors: + | * Ciprian Dorin Craciun + | ciprian@volution.ro + | ciprian.craciun@gmail.com + | https://volution.ro/ciprian + ----------------------------------------------------------- + + kawipiko-archiver + + --sources + + --archive + --compress + + --exclude-index + --exclude-strip + --exclude-cache + --include-etag + + --exclude-file-listing + --include-folder-listing + + --debug + + ** for details see: + https://github.com/volution/kawipiko#kawipiko-archiver + +`) + } + + _sourcesFolder_0 := _flags.String ("sources", "", "") + _archiveFile_0 := _flags.String ("archive", "", "") + _compress_0 := _flags.String ("compress", "", "") + _excludeIndex_0 := _flags.Bool ("exclude-index", false, "") + _excludeStripped_0 := _flags.Bool ("exclude-strip", false, "") + _excludeCache_0 := _flags.Bool ("exclude-cache", false, "") + _includeEtag_0 := _flags.Bool ("include-etag", false, "") + _excludeFileListing_0 := _flags.Bool ("exclude-file-listing", false, "") + _includeFolderListing_0 := _flags.Bool ("include-folder-listing", false, "") + _debug_0 := _flags.Bool ("debug", false, "") + + FlagsParse (_flags, 0, 0) + + _sourcesFolder = *_sourcesFolder_0 + _archiveFile = *_archiveFile_0 + _compress = *_compress_0 + _includeIndex = ! *_excludeIndex_0 + _includeStripped = ! *_excludeStripped_0 + _includeCache = ! *_excludeCache_0 + _includeEtag = *_includeEtag_0 + _includeFileListing = ! *_excludeFileListing_0 + _includeFolderListing = *_includeFolderListing_0 + _debug = *_debug_0 + + if _sourcesFolder == "" { + AbortError (nil, "[515ee462] expected sources folder argument!") + } + if _archiveFile == "" { + AbortError (nil, "[5e8da985] expected archive file argument!") + } + } + + + var _cdbWriter *cdb.Writer + if _cdbWriter_0, _error := cdb.Create (_archiveFile); _error == nil { + _cdbWriter = _cdbWriter_0 + } else { + AbortError (_error, "[85234ba0] failed creating archive (while opening)!") + } + + if _error := _cdbWriter.Put ([]byte (NamespaceSchemaVersion), []byte (CurrentSchemaVersion)); _error != nil { + AbortError (_error, "[43228812] failed writing archive!") + } + + _context := & context { + cdbWriter : _cdbWriter, + storedFilePaths : make ([]string, 0, 16 * 1024), + storedFolderPaths : make ([]string, 0, 16 * 1024), + storedDataMeta : make (map[string]bool, 16 * 1024), + storedDataContent : make (map[string]bool, 16 * 1024), + storedDataContentMeta : make (map[string]map[string]string, 16 * 1024), + storedFiles : make (map[[2]uint64][2]string, 16 * 1024), + compress : _compress, + includeIndex : _includeIndex, + includeStripped : _includeStripped, + includeCache : _includeCache, + includeEtag : _includeEtag, + includeFileListing : _includeFileListing, + includeFolderListing : _includeFolderListing, + debug : _debug, + } + + if _, _error := walkPath (_context, _sourcesFolder, "/", filepath.Base (_sourcesFolder), nil, true); _error != nil { + AbortError (_error, "[b6a19ef4] failed walking folder!") + } + + if _includeFileListing { + _buffer := make ([]byte, 0, 1024 * 1024) + for _, _path := range _context.storedFilePaths { + _buffer = append (_buffer, _path ...) + _buffer = append (_buffer, '\n') + } + if _error := _cdbWriter.Put ([]byte (NamespaceFilesIndex), _buffer); _error != nil { + AbortError (_error, "[1dbdde05] failed writing archive!") + } + } + + if _includeFolderListing { + _buffer := make ([]byte, 0, 1024 * 1024) + for _, _path := range _context.storedFolderPaths { + _buffer = append (_buffer, _path ...) + _buffer = append (_buffer, '\n') + } + if _error := _cdbWriter.Put ([]byte (NamespaceFoldersIndex), _buffer); _error != nil { + AbortError (_error, "[e2dd2de0] failed writing archive!") + } + } + + if _error := _cdbWriter.Close (); _error != nil { + AbortError (_error, "[bbfb8478] failed creating archive (while closing)!") + } + + return nil +} + diff --git a/sources/cmd/server.go b/sources/cmd/server.go index 961652c..678577c 100644 --- a/sources/cmd/server.go +++ b/sources/cmd/server.go @@ -3,1070 +3,12 @@ package main -import "bytes" -import "flag" -import "fmt" -import "io" -import "log" -import "net" -import "net/http" -import "os" -import "os/signal" -import "runtime" -import "runtime/debug" -import "runtime/pprof" -import "strconv" -import "sync" -import "syscall" -import "time" -import "unsafe" - -import "github.com/colinmarc/cdb" - -import "github.com/valyala/fasthttp" -import "github.com/valyala/fasthttp/reuseport" - -import . "github.com/volution/kawipiko/lib/common" -import . "github.com/volution/kawipiko/lib/server" - - - - -type server struct { - httpServer *fasthttp.Server - cdbReader *cdb.CDB - cachedFileFingerprints map[string][]byte - cachedDataMeta map[string][]byte - cachedDataContent map[string][]byte - securityHeadersEnabled bool - securityHeadersTls bool - debug bool - dummy bool -} - - - - -func (_server *server) Serve (_context *fasthttp.RequestCtx) () { - - // _request := (*fasthttp.Request) (NoEscape (unsafe.Pointer (&_context.Request))) - _requestHeaders := (*fasthttp.RequestHeader) (NoEscape (unsafe.Pointer (&_context.Request.Header))) - _response := (*fasthttp.Response) (NoEscape (unsafe.Pointer (&_context.Response))) - _responseHeaders := (*fasthttp.ResponseHeader) (NoEscape (unsafe.Pointer (&_context.Response.Header))) - - _keyBuffer := [1024]byte {} - _pathBuffer := [1024]byte {} - - _method := _requestHeaders.Method () - - _path := _pathBuffer[:0] - _path = append (_path, _requestHeaders.RequestURI () ...) - if _pathLimit := bytes.IndexByte (_path, '?'); _pathLimit > 0 { - _path = _path[: _pathLimit] - } - // FIXME: Decode path according to `decodeArgAppendNoPlus`! - - _pathLen := len (_path) - _pathIsRoot := _pathLen == 1 - _pathHasSlash := !_pathIsRoot && (_path[_pathLen - 1] == '/') - - if ! bytes.Equal (StringToBytes (http.MethodGet), _method) { - log.Printf ("[ww] [bce7a75b] invalid method `%s` for `%s`!\n", _requestHeaders.Method (), _requestHeaders.RequestURI ()) - _server.ServeError (_context, http.StatusMethodNotAllowed, nil, true) - return - } - if (_pathLen == 0) || (_path[0] != '/') { - log.Printf ("[ww] [fa6b1923] invalid path `%s`!\n", _requestHeaders.RequestURI ()) - _server.ServeError (_context, http.StatusBadRequest, nil, true) - return - } - - if bytes.HasPrefix (_path, StringToBytes ("/__/")) { - if bytes.Equal (_path, StringToBytes ("/__/heartbeat")) || bytes.HasPrefix (_path, StringToBytes ("/__/heartbeat/")) { - _server.ServeStatic (_context, http.StatusOK, HeartbeatDataOk, HeartbeatContentType, HeartbeatContentEncoding, false) - return - } else if bytes.Equal (_path, StringToBytes ("/__/about")) { - _server.ServeStatic (_context, http.StatusOK, AboutBannerData, AboutBannerContentType, AboutBannerContentEncoding, true) - return - } else if bytes.HasPrefix (_path, StringToBytes ("/__/errors/banners/")) { - _code := _path[len ("/__/errors/banners/") :] - if _code, _error := strconv.Atoi (BytesToString (*NoEscapeBytes (&_code))); _error == nil { - _banner, _bannerFound := ErrorBannersData[uint (_code)] - if (_code > 0) && _bannerFound { - _server.ServeStatic (_context, http.StatusOK, _banner, ErrorBannerContentType, ErrorBannerContentEncoding, true) - return - } - } - _server.ServeError (_context, http.StatusNotFound, nil, true) - return - } else { - _server.ServeError (_context, http.StatusNotFound, nil, true) - return - } - } - - var _fingerprints []byte - - var _namespaceAndPathSuffixes = [][2]string { - {NamespaceFilesContent, ""}, - {NamespaceFilesContent, "/"}, - {NamespaceFoldersContent, ""}, - } - - if _fingerprints == nil { - _loop_1 : for _namespaceAndPathSuffixIndex := range _namespaceAndPathSuffixes { - _namespaceAndPathSuffix := _namespaceAndPathSuffixes[_namespaceAndPathSuffixIndex] - _namespace := _namespaceAndPathSuffix[0] - _pathSuffix := _namespaceAndPathSuffix[1] - - switch { - case !_pathIsRoot && !_pathHasSlash : - break - case _pathSuffix == "/" : - continue _loop_1 - case _pathSuffix == "" : - break - case _pathSuffix[0] == '/' : - _pathSuffix = _pathSuffix[1:] - } - _pathSuffixHasSlash := (len (_pathSuffix) != 0) && (_pathSuffix[0] == '/') - - if _server.cachedFileFingerprints != nil { - _key := _keyBuffer[:0] - _key = append (_key, _path ...) - _key = append (_key, _pathSuffix ...) - _fingerprints, _ = _server.cachedFileFingerprints[BytesToString (_key)] - } else { - _key := _keyBuffer[:0] - _key = append (_key, _namespace ...) - _key = append (_key, ':') - _key = append (_key, _path ...) - _key = append (_key, _pathSuffix ...) - if _value, _error := _server.cdbReader.GetWithCdbHash (_key); _error == nil { - _fingerprints = _value - } else { - _server.ServeError (_context, http.StatusInternalServerError, _error, false) - return - } - } - - if _fingerprints != nil { - if ((_namespace == NamespaceFoldersContent) || _pathSuffixHasSlash) && (!_pathIsRoot && !_pathHasSlash) { - _path = append (_path, '/') - _server.ServeRedirect (_context, http.StatusTemporaryRedirect, _path, true) - return - } - break _loop_1 - } - } - } - - if _fingerprints == nil { - if bytes.Equal (StringToBytes ("/favicon.ico"), _path) { - _server.ServeStatic (_context, http.StatusOK, FaviconData, FaviconContentType, FaviconContentEncoding, true) - return - } - } - - if _fingerprints == nil { - _loop_2 : for - _pathLimit := bytes.LastIndexByte (_path, '/'); - _pathLimit >= 0; - _pathLimit = bytes.LastIndexByte (_path[: _pathLimit], '/') { - - if _server.cachedFileFingerprints != nil { - _key := _keyBuffer[:0] - _key = append (_key, _path[: _pathLimit] ...) - _key = append (_key, "/*" ...) - _fingerprints, _ = _server.cachedFileFingerprints[BytesToString (_key)] - } else { - _key := _keyBuffer[:0] - _key = append (_key, NamespaceFilesContent ...) - _key = append (_key, ':') - _key = append (_key, _path[: _pathLimit] ...) - _key = append (_key, "/*" ...) - if _value, _error := _server.cdbReader.GetWithCdbHash (_key); _error == nil { - _fingerprints = _value - } else { - _server.ServeError (_context, http.StatusInternalServerError, _error, false) - return - } - } - - if _fingerprints != nil { - break _loop_2 - } - } - } - - if _fingerprints == nil { - log.Printf ("[ww] [7416f61d] not found `%s`!\n", _requestHeaders.RequestURI ()) - _server.ServeError (_context, http.StatusNotFound, nil, true) - return - } - - if len (_fingerprints) != 129 { - log.Printf ("[ee] [7ee6c981] invalid data fingerprints for `%s`!\n", _requestHeaders.RequestURI ()) - _server.ServeError (_context, http.StatusInternalServerError, nil, false) - return - } - _fingerprintContent := _fingerprints[0:64] - _fingerprintMeta := _fingerprints[65:129] - - var _data []byte - if _server.cachedDataContent != nil { - _data, _ = _server.cachedDataContent[BytesToString (_fingerprintContent)] - } else { - _key := _keyBuffer[:0] - _key = append (_key, NamespaceDataContent ...) - _key = append (_key, ':') - _key = append (_key, _fingerprintContent ...) - if _value, _error := _server.cdbReader.GetWithCdbHash (_key); _error == nil { - _data = _value - } else { - _server.ServeError (_context, http.StatusInternalServerError, _error, false) - return - } - } - if _data == nil { - log.Printf ("[ee] [0165c193] missing data content for `%s`!\n", _requestHeaders.RequestURI ()) - _server.ServeError (_context, http.StatusInternalServerError, nil, false) - return - } - - var _dataMetaRaw []byte - if _server.cachedDataMeta != nil { - _dataMetaRaw, _ = _server.cachedDataMeta[BytesToString (_fingerprintMeta)] - } else { - _key := _keyBuffer[:0] - _key = append (_key, NamespaceDataMetadata ...) - _key = append (_key, ':') - _key = append (_key, _fingerprintMeta ...) - if _value, _error := _server.cdbReader.GetWithCdbHash (_key); _error == nil { - _dataMetaRaw = _value - } else { - _server.ServeError (_context, http.StatusInternalServerError, _error, false) - return - } - } - if _dataMetaRaw == nil { - log.Printf ("[ee] [e8702411] missing data metadata for `%s`!\n", _requestHeaders.RequestURI ()) - _server.ServeError (_context, http.StatusInternalServerError, nil, false) - return - } - - _responseStatus := http.StatusOK - - _responseHeaders.AddRawLines (_dataMetaRaw) - -// FIXME: Re-enable this part! -// _handleHeader := func (_name []byte, _value []byte) { -// if _name[0] != '_' { -// _responseHeaders.AddRawKv (_name, _value) -// } else { -// switch BytesToString (_name) { -// case "!Status" : -// if _value, _error := strconv.Atoi (BytesToString (_value)); _error == nil { -// if (_value >= 200) && (_value <= 599) { -// _responseStatus = _value -// } else { -// log.Printf ("[c2f7ec36] invalid data metadata for `%s`!\n", _requestHeaders.RequestURI ()) -// _responseStatus = http.StatusInternalServerError -// } -// } else { -// log.Printf ("[beedae55] invalid data metadata for `%s`!\n", _requestHeaders.RequestURI ()) -// _responseStatus = http.StatusInternalServerError -// } -// default : -// log.Printf ("[7acc7d90] invalid data metadata for `%s`!\n", _requestHeaders.RequestURI ()) -// } -// } -// } -// if _error := MetadataDecodeIterate (_dataMetaRaw, _handleHeader); _error != nil { -// _server.ServeError (_context, http.StatusInternalServerError, _error, false) -// return -// } - - if _server.securityHeadersEnabled { - if _server.securityHeadersTls { - const _lines = ( - "Strict-Transport-Security: max-age=31536000" + "\r\n" + - "Content-Security-Policy: upgrade-insecure-requests" + "\r\n") - _responseHeaders.AddRawLines (StringToBytes (_lines)) - } - { - const _lines = ( - "Referrer-Policy: strict-origin-when-cross-origin" + "\r\n" + - "X-Content-Type-Options: nosniff" + "\r\n" + - "X-XSS-Protection: 1; mode=block" + "\r\n" + - "X-Frame-Options: sameorigin" + "\r\n") - _responseHeaders.AddRawLines (StringToBytes (_lines)) - } - } - - if _server.debug { - log.Printf ("[dd] [b15f3cad] serving for `%s`...\n", _requestHeaders.RequestURI ()) - } - - _response.SetStatusCode (_responseStatus) - _response.SetBodyRaw (_data) -} - - - - -func (_server *server) ServeStatic (_context *fasthttp.RequestCtx, _status uint, _data []byte, _contentType string, _contentEncoding string, _cache bool) () { - - _response := (*fasthttp.Response) (NoEscape (unsafe.Pointer (&_context.Response))) - _responseHeaders := (*fasthttp.ResponseHeader) (NoEscape (unsafe.Pointer (&_context.Response.Header))) - - _responseHeaders.AddRawKv (StringToBytes ("Content-Type"), StringToBytes (_contentType)) - _responseHeaders.AddRawKv (StringToBytes ("Content-Encoding"), StringToBytes (_contentEncoding)) - - if _cache { - _responseHeaders.AddRawLines (StringToBytes ("Cache-Control: public, immutable, max-age=3600\r\n")) - } else { - _responseHeaders.AddRawLines (StringToBytes ("Cache-Control: private, no-cache, no-store\r\n")) - } - - _response.SetStatusCode (int (_status)) - _response.SetBodyRaw (_data) -} - - -func (_server *server) ServeRedirect (_context *fasthttp.RequestCtx, _status uint, _path []byte, _cache bool) () { - - _response := (*fasthttp.Response) (NoEscape (unsafe.Pointer (&_context.Response))) - _responseHeaders := (*fasthttp.ResponseHeader) (NoEscape (unsafe.Pointer (&_context.Response.Header))) - - _responseHeaders.SetCanonical (StringToBytes ("Location"), _path) - - if _cache { - _responseHeaders.AddRawLines (StringToBytes ("Cache-Control: public, immutable, max-age=3600\r\n")) - } else { - _responseHeaders.AddRawLines (StringToBytes ("Cache-Control: private, no-cache, no-store\r\n")) - } - - _response.SetStatusCode (int (_status)) -} - - -func (_server *server) ServeError (_context *fasthttp.RequestCtx, _status uint, _error error, _cache bool) () { - - _response := (*fasthttp.Response) (NoEscape (unsafe.Pointer (&_context.Response))) - _responseHeaders := (*fasthttp.ResponseHeader) (NoEscape (unsafe.Pointer (&_context.Response.Header))) - - _responseHeaders.AddRawKv (StringToBytes ("Content-Type"), StringToBytes (ErrorBannerContentType)) - _responseHeaders.AddRawKv (StringToBytes ("Content-Encoding"), StringToBytes (ErrorBannerContentEncoding)) - - if _cache { - _responseHeaders.AddRawLines (StringToBytes ("Cache-Control: public, immutable, max-age=3600\r\n")) - } else { - _responseHeaders.AddRawLines (StringToBytes ("Cache-Control: private, no-cache, no-store\r\n")) - } - - if _banner, _bannerFound := ErrorBannersData[_status]; _bannerFound { - _response.SetBodyRaw (_banner) - } - - _response.SetStatusCode (int (_status)) - - LogError (_error, "") -} - - -func (_server *server) ServeDummy (_context *fasthttp.RequestCtx) () { - if false { - _server.ServeStatic (_context, http.StatusOK, DummyData, DummyContentType, DummyContentEncoding, false) - } else { - ServeDummyRaw (_context) - } -} - -func ServeDummyRaw (_context *fasthttp.RequestCtx) () { - _context.Response.Header.SetRaw (DummyMeta) - _context.Response.SetBodyRaw (DummyData) -} +import . "github.com/volution/kawipiko/cmd/server" func main () () { - Main (main_0) -} - - -func main_0 () (error) { - - - var _bind string - var _archivePath string - var _archiveInmem bool - var _archiveMmap bool - var _archivePreload bool - var _indexPaths bool - var _indexDataMeta bool - var _indexDataContent bool - var _securityHeadersEnabled bool - var _securityHeadersTls bool - var _timeoutDisabled bool - var _processes uint - var _threads uint - var _slave uint - var _debug bool - var _dummy bool - var _isFirst bool - var _isMaster bool - - var _profileCpu string - var _profileMem string - - { - _flags := flag.NewFlagSet ("kawipiko-server", flag.ContinueOnError) - - _flags.Usage = func () () { - fmt.Fprintf (os.Stderr, "%s", -` - ==== kawipiko -- blazingly fast static HTTP server ==== - - | Documentation, issues and sources: - | * https://github.com/volution/kawipiko - | Authors: - | * Ciprian Dorin Craciun - | ciprian@volution.ro - | ciprian.craciun@gmail.com - | https://volution.ro/ciprian - ----------------------------------------------------------- - - kawipiko-server - - --bind : - - --processes (of slave processes) - --threads (of threads per process) - - --archive - --archive-inmem (memory-loaded archive file) - --archive-mmap (memory-mapped archive file) - --archive-preload (preload archive file) - - --index-all - --index-paths - --index-data-meta - --index-data-content - - --security-headers-tls - --security-headers-disable - --timeout-disable - - --profile-cpu - --profile-mem - - --debug - --dummy - - ** for details see: - https://github.com/volution/kawipiko#kawipiko-server - -`) - } - - _bind_0 := _flags.String ("bind", "", "") - _archivePath_0 := _flags.String ("archive", "", "") - _archiveInmem_0 := _flags.Bool ("archive-inmem", false, "") - _archiveMmap_0 := _flags.Bool ("archive-mmap", false, "") - _archivePreload_0 := _flags.Bool ("archive-preload", false, "") - _indexAll_0 := _flags.Bool ("index-all", false, "") - _indexPaths_0 := _flags.Bool ("index-paths", false, "") - _indexDataMeta_0 := _flags.Bool ("index-data-meta", false, "") - _indexDataContent_0 := _flags.Bool ("index-data-content", false, "") - _timeoutDisabled_0 := _flags.Bool ("timeout-disable", false, "") - _securityHeadersTls_0 := _flags.Bool ("security-headers-tls", false, "") - _securityHeadersDisabled_0 := _flags.Bool ("security-headers-disable", false, "") - _processes_0 := _flags.Uint ("processes", 0, "") - _threads_0 := _flags.Uint ("threads", 0, "") - _slave_0 := _flags.Uint ("slave", 0, "") - _profileCpu_0 := _flags.String ("profile-cpu", "", "") - _profileMem_0 := _flags.String ("profile-mem", "", "") - _debug_0 := _flags.Bool ("debug", false, "") - _dummy_0 := _flags.Bool ("dummy", false, "") - - FlagsParse (_flags, 0, 0) - - _bind = *_bind_0 - _archivePath = *_archivePath_0 - _archiveInmem = *_archiveInmem_0 - _archiveMmap = *_archiveMmap_0 - _archivePreload = *_archivePreload_0 - _indexAll := *_indexAll_0 - _indexPaths = _indexAll || *_indexPaths_0 - _indexDataMeta = _indexAll || *_indexDataMeta_0 - _indexDataContent = _indexAll || *_indexDataContent_0 - _securityHeadersTls = *_securityHeadersTls_0 - _securityHeadersEnabled = ! *_securityHeadersDisabled_0 - _timeoutDisabled = *_timeoutDisabled_0 - _processes = *_processes_0 - _threads = *_threads_0 - _slave = *_slave_0 - _debug = *_debug_0 - _dummy = *_dummy_0 - - _profileCpu = *_profileCpu_0 - _profileMem = *_profileMem_0 - - if _slave == 0 { - _isMaster = true - } - if _slave <= 1 { - _isFirst = true - } - - if _bind == "" { - AbortError (nil, "[6edd9512] expected bind address argument!") - } - - if !_dummy { - if _archivePath == "" { - AbortError (nil, "[eefe1a38] expected archive file argument!") - } - if _archiveInmem && _archiveMmap { - AbortError (nil, "[a2101041] archive 'memory-loaded' and 'memory-mapped' are mutually exclusive!") - } - if _archiveInmem && _archivePreload { - log.Printf ("[ww] [3e8a40e4] archive 'memory-loaded' implies preloading!\n") - _archivePreload = false - } - } else { - if _isMaster { - log.Printf ("[ww] [8e014192] running in dummy mode; all archive related arguments are ignored!\n") - } - _archivePath = "" - _archiveInmem = false - _archiveMmap = false - _archivePreload = false - _indexAll = false - _indexPaths = false - _indexDataMeta = false - _indexDataContent = false - } - - if (_processes > 1) && ((_profileCpu != "") || (_profileMem != "")) { - AbortError (nil, "[cd18d250] multi-process and profiling are mutually exclusive!") - } - - if _processes < 1 { - _processes = 1 - } - if _threads < 1 { - _threads = 1 - } - - if _processes > 1024 { - AbortError (nil, "[45736c1d] maximum number of allowed processes is 1024!") - } - if _threads > 1024 { - AbortError (nil, "[c5df3c8d] maximum number of allowed threads is 1024!") - } - if (_processes * _threads) > 1024 { - AbortError (nil, "[b0177488] maximum number of allowed threads in total is 1024!") - } - } - - - runtime.GOMAXPROCS (int (_threads)) - - debug.SetGCPercent (50) - debug.SetMaxThreads (int (128 * (_threads / 64 + 1))) - debug.SetMaxStack (16 * 1024) - - - _httpServerReduceMemory := false - - if false { - if _error := syscall.Setrlimit (syscall.RLIMIT_DATA, & syscall.Rlimit { Max : 4 * 1024 * 1024 * 1024 }); _error != nil { - AbortError (_error, "[f661b4fe] failed to configure limits!") - } - } - - - if _processes > 1 { - - log.Printf ("[ii] [06f8c944] sub-processes starting (`%d` processes with `%d` threads each)...\n", _processes, _threads) - - _processesJoin := & sync.WaitGroup {} - - _processesPid := make ([]*os.Process, _processes) - - _processName := os.Args[0] - _processArguments := make ([]string, 0, len (os.Args)) - _processArguments = append (_processArguments, - "--bind", _bind, - ) - if _archivePath != "" { - _processArguments = append (_processArguments, "--archive", _archivePath) - } - if _archiveInmem { - _processArguments = append (_processArguments, "--archive-inmem") - } - if _archiveMmap { - _processArguments = append (_processArguments, "--archive-mmap") - } - if _archivePreload { - _processArguments = append (_processArguments, "--archive-preload") - } - if _indexPaths { - _processArguments = append (_processArguments, "--index-paths") - } - if _indexDataMeta { - _processArguments = append (_processArguments, "--index-data-meta") - } - if _indexDataContent { - _processArguments = append (_processArguments, "--index-data-content") - } - if _securityHeadersTls { - _processArguments = append (_processArguments, "--security-headers-tls") - } - if !_securityHeadersEnabled { - _processArguments = append (_processArguments, "--security-headers-disable") - } - if _timeoutDisabled { - _processArguments = append (_processArguments, "--timeout-disable") - } - if _debug { - _processArguments = append (_processArguments, "--debug") - } - if _dummy { - _processArguments = append (_processArguments, "--dummy") - } - _processArguments = append (_processArguments, "--threads", fmt.Sprintf ("%d", _threads)) - - _processAttributes := & os.ProcAttr { - Env : []string {}, - Files : []*os.File { - os.Stdin, - os.Stdout, - os.Stderr, - }, - Sys : nil, - } - - for _processIndex, _ := range _processesPid { - _processArguments := append ([]string { _processName, "--slave", fmt.Sprintf ("%d", _processIndex + 1) }, _processArguments ...) - if _processPid, _error := os.StartProcess (_processName, _processArguments, _processAttributes); _error == nil { - _processesJoin.Add (1) - _processesPid[_processIndex] = _processPid - if _debug { - log.Printf ("[ii] [63cb22f8] sub-process `%d` started (with `%d` threads);\n", _processPid.Pid, _threads) - } - go func (_index int, _processPid *os.Process) () { - if _processStatus, _error := _processPid.Wait (); _error == nil { - if _processStatus.Success () { - if _debug { - log.Printf ("[ii] [66b60b81] sub-process `%d` succeeded;\n", _processPid.Pid) - } - } else { - log.Printf ("[ww] [5d25046b] sub-process `%d` failed: `%s`; ignoring!\n", _processPid.Pid, _processStatus) - } - } else { - LogError (_error, fmt.Sprintf ("[f1bfc927] failed waiting for sub-process `%d`; ignoring!", _processPid.Pid)) - } - _processesPid[_processIndex] = nil - _processesJoin.Done () - } (_processIndex, _processPid) - } else { - LogError (_error, "[8892b34d] failed starting sub-process; ignoring!") - } - } - - { - _signals := make (chan os.Signal, 32) - signal.Notify (_signals, syscall.SIGINT, syscall.SIGTERM) - go func () () { - for { - _signal := <- _signals - if _debug { - log.Printf ("[ii] [a9243ecb] signaling sub-processes...\n") - } - for _, _processPid := range _processesPid { - if _processPid != nil { - if _error := _processPid.Signal (_signal); _error != nil { - LogError (_error, fmt.Sprintf ("[ab681164] failed signaling sub-process `%d`; ignoring!", _processPid.Pid)) - } - } - } - } - } () - } - - _processesJoin.Wait () - - if _debug { - log.Printf ("[ii] [b949bafc] sub-processes terminated;\n") - } - - return nil - } - - - if _isMaster { - log.Printf ("[ii] [6602a54a] starting (with `%d` threads)...\n", _threads) - } - - - var _cdbReader *cdb.CDB - if _archivePath != "" { - - if _debug || _isFirst { - log.Printf ("[ii] [3b788396] opening archive file `%s`...\n", _archivePath) - } - - var _cdbFile *os.File - if _cdbFile_0, _error := os.Open (_archivePath); _error == nil { - _cdbFile = _cdbFile_0 - } else { - AbortError (_error, "[9e0b5ed3] failed opening archive file!") - } - - var _cdbFileSize int - { - var _cdbFileSize_0 int64 - if _cdbFileStat, _error := _cdbFile.Stat (); _error == nil { - _cdbFileSize_0 = _cdbFileStat.Size () - } else { - AbortError (_error, "[0ccf0a3b] failed opening archive file!") - } - if _cdbFileSize_0 < 1024 { - AbortError (nil, "[6635a2a8] failed opening archive: file is too small (or empty)!") - } - if _cdbFileSize_0 >= (2 * 1024 * 1024 * 1024) { - AbortError (nil, "[545bf6ce] failed opening archive: file is too large!") - } - _cdbFileSize = int (_cdbFileSize_0) - } - - if _archivePreload { - if _debug { - log.Printf ("[ii] [13f4ebf7] preloading archive file...\n") - } - _buffer := [16 * 1024]byte {} - _loop : for { - switch _, _error := _cdbFile.Read (_buffer[:]); _error { - case io.EOF : - break _loop - case nil : - continue _loop - default : - AbortError (_error, "[a1c3b922] failed preloading archive file...\n") - } - } - } - - if _archiveInmem || _archiveMmap { - - var _cdbData []byte - - if _archiveInmem { - - if _debug { - log.Printf ("[ii] [216e584b] opening memory-loaded archive...\n") - } - - _cdbData = make ([]byte, _cdbFileSize) - if _, _error := io.ReadFull (_cdbFile, _cdbData); _error != nil { - AbortError (_error, "[73039784] failed loading archive file!") - } - - } else if _archiveMmap { - - if _debug { - log.Printf ("[ii] [f47fae8a] opening memory-mapped archive...\n") - } - - if _cdbData_0, _error := syscall.Mmap (int (_cdbFile.Fd ()), 0, int (_cdbFileSize), syscall.PROT_READ, syscall.MAP_SHARED); _error == nil { - _cdbData = _cdbData_0 - } else { - AbortError (_error, "[c0e2632c] failed mapping archive file!") - } - - if _archivePreload { - if _debug { - log.Printf ("[ii] [d96b06c9] preloading memory-loaded archive...\n") - } - _buffer := [16 * 1024]byte {} - _bufferOffset := 0 - for { - if _bufferOffset == _cdbFileSize { - break - } - _bufferOffset += copy (_buffer[:], _cdbData[_bufferOffset:]) - } - } - - } else { - panic ("e4fffcd8") - } - - if _error := _cdbFile.Close (); _error != nil { - AbortError (_error, "[5e0449c2] failed closing archive file!") - } - - if _cdbReader_0, _error := cdb.NewFromBufferWithHasher (_cdbData, nil); _error == nil { - _cdbReader = _cdbReader_0 - } else { - AbortError (_error, "[27e4813e] failed opening archive!") - } - - } else { - - if _debug || _isFirst { - log.Printf ("[ww] [dd697a66] using `read`-based archive (with significant performance impact)!\n") - } - - if _cdbReader_0, _error := cdb.NewFromReaderWithHasher (_cdbFile, nil); _error == nil { - _cdbReader = _cdbReader_0 - } else { - AbortError (_error, "[35832022] failed opening archive!") - } - - } - - if _schemaVersion, _error := _cdbReader.GetWithCdbHash ([]byte (NamespaceSchemaVersion)); _error == nil { - if _schemaVersion == nil { - AbortError (nil, "[09316866] missing archive schema version!") - } else if string (_schemaVersion) != CurrentSchemaVersion { - AbortError (nil, "[e6482cf7] invalid archive schema version!") - } - } else { - AbortError (_error, "[87cae197] failed opening archive!") - } - } - - - var _cachedFileFingerprints map[string][]byte - if _indexPaths { - _cachedFileFingerprints = make (map[string][]byte, 128 * 1024) - } - var _cachedDataMeta map[string][]byte - if _indexDataMeta { - _cachedDataMeta = make (map[string][]byte, 128 * 1024) - } - var _cachedDataContent map[string][]byte - if _indexDataContent { - _cachedDataContent = make (map[string][]byte, 128 * 1024) - } - - if _indexPaths || _indexDataMeta || _indexDataContent { - if _debug { - log.Printf ("[ii] [fa5338fd] indexing archive...\n") - } - if _filesIndex, _error := _cdbReader.GetWithCdbHash ([]byte (NamespaceFilesIndex)); _error == nil { - if _filesIndex != nil { - _keyBuffer := [1024]byte {} - for { - _offset := bytes.IndexByte (_filesIndex, '\n') - if _offset == 0 { - continue - } - if _offset == -1 { - break - } - _filePath := _filesIndex[: _offset] - _filesIndex = _filesIndex[_offset + 1 :] - var _fingerprints []byte - var _fingerprintContent []byte - var _fingerprintMeta []byte - { - _key := _keyBuffer[:0] - _key = append (_key, NamespaceFilesContent ...) - _key = append (_key, ':') - _key = append (_key, _filePath ...) - if _fingerprints_0, _error := _cdbReader.GetWithCdbHash (_key); _error == nil { - if _fingerprints_0 != nil { - _fingerprints = _fingerprints_0 - _fingerprintContent = _fingerprints[0:64] - _fingerprintMeta = _fingerprints[65:129] - } else { - AbortError (_error, "[460b3cf1] failed indexing archive!") - } - } else { - AbortError (_error, "[216f2075] failed indexing archive!") - } - } - if _indexPaths { - _cachedFileFingerprints[BytesToString (_filePath)] = _fingerprints - } - if _indexDataMeta { - if _, _wasCached := _cachedDataMeta[BytesToString (_fingerprintMeta)]; !_wasCached { - _key := _keyBuffer[:0] - _key = append (_key, NamespaceDataMetadata ...) - _key = append (_key, ':') - _key = append (_key, _fingerprintMeta ...) - if _dataMeta, _error := _cdbReader.GetWithCdbHash (_key); _error == nil { - if _dataMeta != nil { - _cachedDataMeta[BytesToString (_fingerprintMeta)] = _dataMeta - } else { - AbortError (_error, "[6df556bf] failed indexing archive!") - } - } else { - AbortError (_error, "[0d730134] failed indexing archive!") - } - } - } - if _indexDataContent { - if _, _wasCached := _cachedDataContent[BytesToString (_fingerprintContent)]; !_wasCached { - _key := _keyBuffer[:0] - _key = append (_key, NamespaceDataContent ...) - _key = append (_key, ':') - _key = append (_key, _fingerprintContent ...) - if _dataContent, _error := _cdbReader.GetWithCdbHash (_key); _error == nil { - if _dataContent != nil { - _cachedDataContent[BytesToString (_fingerprintContent)] = _dataContent - } else { - AbortError (_error, "[4e27fe46] failed indexing archive!") - } - } else { - AbortError (_error, "[532845ad] failed indexing archive!") - } - } - } - } - } else { - log.Printf ("[ww] [30314f31] missing archive files index; ignoring!\n") - } - } else { - AbortError (_error, "[82299b3d] failed indexing arcdive!") - } - } - - if _indexPaths && _indexDataMeta && _indexDataContent { - if _error := _cdbReader.Close (); _error == nil { - _cdbReader = nil - } else { - AbortError (_error, "[d7aa79e1] failed closing archive!") - } - } - - - _server := & server { - httpServer : nil, - cdbReader : _cdbReader, - cachedFileFingerprints : _cachedFileFingerprints, - cachedDataMeta : _cachedDataMeta, - cachedDataContent : _cachedDataContent, - securityHeadersTls : _securityHeadersTls, - securityHeadersEnabled : _securityHeadersEnabled, - debug : _debug, - dummy : _dummy, - } - - - if _profileCpu != "" { - log.Printf ("[ii] [70c210f3] profiling CPU to `%s`...\n", _profileCpu) - _stream, _error := os.Create (_profileCpu) - if _error != nil { - AbortError (_error, "[fd4e0009] failed opening CPU profile!") - } - _error = pprof.StartCPUProfile (_stream) - if _error != nil { - AbortError (_error, "[ac721629] failed starting CPU profile!") - } - defer pprof.StopCPUProfile () - } - if _profileMem != "" { - log.Printf ("[ii] [9196ee90] profiling MEM to `%s`...\n", _profileMem) - _stream, _error := os.Create (_profileMem) - if _error != nil { - AbortError (_error, "[907d08b5] failed opening MEM profile!") - } - _profile := pprof.Lookup ("heap") - defer func () () { - runtime.GC () - if _profile != nil { - if _error := _profile.WriteTo (_stream, 0); _error != nil { - AbortError (_error, "[4b1e5112] failed writing MEM profile!") - } - } else { - AbortError (nil, "[385dc8f0] failed loading MEM profile!") - } - _stream.Close () - } () - } - - - _httpServer := & fasthttp.Server { - - Name : "kawipiko", - Handler : _server.Serve, - GetOnly : true, - - NoDefaultServerHeader : true, - NoDefaultContentType : true, - NoDefaultDate : true, - DisableHeaderNamesNormalizing : true, - - Concurrency : 16 * 1024 + 128, - MaxRequestsPerConn : 256 * 1024, - - ReadBufferSize : 16 * 1024, - WriteBufferSize : 16 * 1024, - MaxRequestBodySize : 16 * 1024, - - ReadTimeout : 30 * time.Second, - WriteTimeout : 30 * time.Second, - IdleTimeout : 360 * time.Second, - - TCPKeepalive : true, - TCPKeepalivePeriod : 60 * time.Second, - - ReduceMemoryUsage : _httpServerReduceMemory, - - } - - if _timeoutDisabled { - _httpServer.ReadTimeout = 0 - _httpServer.WriteTimeout = 0 - _httpServer.IdleTimeout = 0 - } - - if _dummy { - _httpServer.Handler = _server.ServeDummy - } - - _server.httpServer = _httpServer - - - { - _signals := make (chan os.Signal, 32) - signal.Notify (_signals, syscall.SIGINT, syscall.SIGTERM) - go func () () { - <- _signals - if _debug { - log.Printf ("[ii] [691cb695] shutingdown...\n") - } - _server.httpServer.Shutdown () - } () - } - - - if _debug || _isFirst { - log.Printf ("[ii] [f11e4e37] listening on `http://%s/`;\n", _bind) - } - - var _httpListener net.Listener - if _httpListener_0, _error := reuseport.Listen ("tcp4", _bind); _error == nil { - _httpListener = _httpListener_0 - } else { - AbortError (_error, "[d5f51e9f] failed starting listener!") - } - - if _error := _httpServer.Serve (_httpListener); _error != nil { - AbortError (_error, "[44f45c67] failed executing server!") - } - - - if _debug { - defer log.Printf ("[ii] [a49175db] done!\n") - } - return nil + Main () } diff --git a/sources/cmd/server/server.go b/sources/cmd/server/server.go new file mode 100644 index 0000000..e47a222 --- /dev/null +++ b/sources/cmd/server/server.go @@ -0,0 +1,1072 @@ + + +package server + + +import "bytes" +import "flag" +import "fmt" +import "io" +import "log" +import "net" +import "net/http" +import "os" +import "os/signal" +import "runtime" +import "runtime/debug" +import "runtime/pprof" +import "strconv" +import "sync" +import "syscall" +import "time" +import "unsafe" + +import "github.com/colinmarc/cdb" + +import "github.com/valyala/fasthttp" +import "github.com/valyala/fasthttp/reuseport" + +import . "github.com/volution/kawipiko/lib/common" +import . "github.com/volution/kawipiko/lib/server" + + + + +type server struct { + httpServer *fasthttp.Server + cdbReader *cdb.CDB + cachedFileFingerprints map[string][]byte + cachedDataMeta map[string][]byte + cachedDataContent map[string][]byte + securityHeadersEnabled bool + securityHeadersTls bool + debug bool + dummy bool +} + + + + +func (_server *server) Serve (_context *fasthttp.RequestCtx) () { + + // _request := (*fasthttp.Request) (NoEscape (unsafe.Pointer (&_context.Request))) + _requestHeaders := (*fasthttp.RequestHeader) (NoEscape (unsafe.Pointer (&_context.Request.Header))) + _response := (*fasthttp.Response) (NoEscape (unsafe.Pointer (&_context.Response))) + _responseHeaders := (*fasthttp.ResponseHeader) (NoEscape (unsafe.Pointer (&_context.Response.Header))) + + _keyBuffer := [1024]byte {} + _pathBuffer := [1024]byte {} + + _method := _requestHeaders.Method () + + _path := _pathBuffer[:0] + _path = append (_path, _requestHeaders.RequestURI () ...) + if _pathLimit := bytes.IndexByte (_path, '?'); _pathLimit > 0 { + _path = _path[: _pathLimit] + } + // FIXME: Decode path according to `decodeArgAppendNoPlus`! + + _pathLen := len (_path) + _pathIsRoot := _pathLen == 1 + _pathHasSlash := !_pathIsRoot && (_path[_pathLen - 1] == '/') + + if ! bytes.Equal (StringToBytes (http.MethodGet), _method) { + log.Printf ("[ww] [bce7a75b] invalid method `%s` for `%s`!\n", _requestHeaders.Method (), _requestHeaders.RequestURI ()) + _server.ServeError (_context, http.StatusMethodNotAllowed, nil, true) + return + } + if (_pathLen == 0) || (_path[0] != '/') { + log.Printf ("[ww] [fa6b1923] invalid path `%s`!\n", _requestHeaders.RequestURI ()) + _server.ServeError (_context, http.StatusBadRequest, nil, true) + return + } + + if bytes.HasPrefix (_path, StringToBytes ("/__/")) { + if bytes.Equal (_path, StringToBytes ("/__/heartbeat")) || bytes.HasPrefix (_path, StringToBytes ("/__/heartbeat/")) { + _server.ServeStatic (_context, http.StatusOK, HeartbeatDataOk, HeartbeatContentType, HeartbeatContentEncoding, false) + return + } else if bytes.Equal (_path, StringToBytes ("/__/about")) { + _server.ServeStatic (_context, http.StatusOK, AboutBannerData, AboutBannerContentType, AboutBannerContentEncoding, true) + return + } else if bytes.HasPrefix (_path, StringToBytes ("/__/errors/banners/")) { + _code := _path[len ("/__/errors/banners/") :] + if _code, _error := strconv.Atoi (BytesToString (*NoEscapeBytes (&_code))); _error == nil { + _banner, _bannerFound := ErrorBannersData[uint (_code)] + if (_code > 0) && _bannerFound { + _server.ServeStatic (_context, http.StatusOK, _banner, ErrorBannerContentType, ErrorBannerContentEncoding, true) + return + } + } + _server.ServeError (_context, http.StatusNotFound, nil, true) + return + } else { + _server.ServeError (_context, http.StatusNotFound, nil, true) + return + } + } + + var _fingerprints []byte + + var _namespaceAndPathSuffixes = [][2]string { + {NamespaceFilesContent, ""}, + {NamespaceFilesContent, "/"}, + {NamespaceFoldersContent, ""}, + } + + if _fingerprints == nil { + _loop_1 : for _namespaceAndPathSuffixIndex := range _namespaceAndPathSuffixes { + _namespaceAndPathSuffix := _namespaceAndPathSuffixes[_namespaceAndPathSuffixIndex] + _namespace := _namespaceAndPathSuffix[0] + _pathSuffix := _namespaceAndPathSuffix[1] + + switch { + case !_pathIsRoot && !_pathHasSlash : + break + case _pathSuffix == "/" : + continue _loop_1 + case _pathSuffix == "" : + break + case _pathSuffix[0] == '/' : + _pathSuffix = _pathSuffix[1:] + } + _pathSuffixHasSlash := (len (_pathSuffix) != 0) && (_pathSuffix[0] == '/') + + if _server.cachedFileFingerprints != nil { + _key := _keyBuffer[:0] + _key = append (_key, _path ...) + _key = append (_key, _pathSuffix ...) + _fingerprints, _ = _server.cachedFileFingerprints[BytesToString (_key)] + } else { + _key := _keyBuffer[:0] + _key = append (_key, _namespace ...) + _key = append (_key, ':') + _key = append (_key, _path ...) + _key = append (_key, _pathSuffix ...) + if _value, _error := _server.cdbReader.GetWithCdbHash (_key); _error == nil { + _fingerprints = _value + } else { + _server.ServeError (_context, http.StatusInternalServerError, _error, false) + return + } + } + + if _fingerprints != nil { + if ((_namespace == NamespaceFoldersContent) || _pathSuffixHasSlash) && (!_pathIsRoot && !_pathHasSlash) { + _path = append (_path, '/') + _server.ServeRedirect (_context, http.StatusTemporaryRedirect, _path, true) + return + } + break _loop_1 + } + } + } + + if _fingerprints == nil { + if bytes.Equal (StringToBytes ("/favicon.ico"), _path) { + _server.ServeStatic (_context, http.StatusOK, FaviconData, FaviconContentType, FaviconContentEncoding, true) + return + } + } + + if _fingerprints == nil { + _loop_2 : for + _pathLimit := bytes.LastIndexByte (_path, '/'); + _pathLimit >= 0; + _pathLimit = bytes.LastIndexByte (_path[: _pathLimit], '/') { + + if _server.cachedFileFingerprints != nil { + _key := _keyBuffer[:0] + _key = append (_key, _path[: _pathLimit] ...) + _key = append (_key, "/*" ...) + _fingerprints, _ = _server.cachedFileFingerprints[BytesToString (_key)] + } else { + _key := _keyBuffer[:0] + _key = append (_key, NamespaceFilesContent ...) + _key = append (_key, ':') + _key = append (_key, _path[: _pathLimit] ...) + _key = append (_key, "/*" ...) + if _value, _error := _server.cdbReader.GetWithCdbHash (_key); _error == nil { + _fingerprints = _value + } else { + _server.ServeError (_context, http.StatusInternalServerError, _error, false) + return + } + } + + if _fingerprints != nil { + break _loop_2 + } + } + } + + if _fingerprints == nil { + log.Printf ("[ww] [7416f61d] not found `%s`!\n", _requestHeaders.RequestURI ()) + _server.ServeError (_context, http.StatusNotFound, nil, true) + return + } + + if len (_fingerprints) != 129 { + log.Printf ("[ee] [7ee6c981] invalid data fingerprints for `%s`!\n", _requestHeaders.RequestURI ()) + _server.ServeError (_context, http.StatusInternalServerError, nil, false) + return + } + _fingerprintContent := _fingerprints[0:64] + _fingerprintMeta := _fingerprints[65:129] + + var _data []byte + if _server.cachedDataContent != nil { + _data, _ = _server.cachedDataContent[BytesToString (_fingerprintContent)] + } else { + _key := _keyBuffer[:0] + _key = append (_key, NamespaceDataContent ...) + _key = append (_key, ':') + _key = append (_key, _fingerprintContent ...) + if _value, _error := _server.cdbReader.GetWithCdbHash (_key); _error == nil { + _data = _value + } else { + _server.ServeError (_context, http.StatusInternalServerError, _error, false) + return + } + } + if _data == nil { + log.Printf ("[ee] [0165c193] missing data content for `%s`!\n", _requestHeaders.RequestURI ()) + _server.ServeError (_context, http.StatusInternalServerError, nil, false) + return + } + + var _dataMetaRaw []byte + if _server.cachedDataMeta != nil { + _dataMetaRaw, _ = _server.cachedDataMeta[BytesToString (_fingerprintMeta)] + } else { + _key := _keyBuffer[:0] + _key = append (_key, NamespaceDataMetadata ...) + _key = append (_key, ':') + _key = append (_key, _fingerprintMeta ...) + if _value, _error := _server.cdbReader.GetWithCdbHash (_key); _error == nil { + _dataMetaRaw = _value + } else { + _server.ServeError (_context, http.StatusInternalServerError, _error, false) + return + } + } + if _dataMetaRaw == nil { + log.Printf ("[ee] [e8702411] missing data metadata for `%s`!\n", _requestHeaders.RequestURI ()) + _server.ServeError (_context, http.StatusInternalServerError, nil, false) + return + } + + _responseStatus := http.StatusOK + + _responseHeaders.AddRawLines (_dataMetaRaw) + +// FIXME: Re-enable this part! +// _handleHeader := func (_name []byte, _value []byte) { +// if _name[0] != '_' { +// _responseHeaders.AddRawKv (_name, _value) +// } else { +// switch BytesToString (_name) { +// case "!Status" : +// if _value, _error := strconv.Atoi (BytesToString (_value)); _error == nil { +// if (_value >= 200) && (_value <= 599) { +// _responseStatus = _value +// } else { +// log.Printf ("[c2f7ec36] invalid data metadata for `%s`!\n", _requestHeaders.RequestURI ()) +// _responseStatus = http.StatusInternalServerError +// } +// } else { +// log.Printf ("[beedae55] invalid data metadata for `%s`!\n", _requestHeaders.RequestURI ()) +// _responseStatus = http.StatusInternalServerError +// } +// default : +// log.Printf ("[7acc7d90] invalid data metadata for `%s`!\n", _requestHeaders.RequestURI ()) +// } +// } +// } +// if _error := MetadataDecodeIterate (_dataMetaRaw, _handleHeader); _error != nil { +// _server.ServeError (_context, http.StatusInternalServerError, _error, false) +// return +// } + + if _server.securityHeadersEnabled { + if _server.securityHeadersTls { + const _lines = ( + "Strict-Transport-Security: max-age=31536000" + "\r\n" + + "Content-Security-Policy: upgrade-insecure-requests" + "\r\n") + _responseHeaders.AddRawLines (StringToBytes (_lines)) + } + { + const _lines = ( + "Referrer-Policy: strict-origin-when-cross-origin" + "\r\n" + + "X-Content-Type-Options: nosniff" + "\r\n" + + "X-XSS-Protection: 1; mode=block" + "\r\n" + + "X-Frame-Options: sameorigin" + "\r\n") + _responseHeaders.AddRawLines (StringToBytes (_lines)) + } + } + + if _server.debug { + log.Printf ("[dd] [b15f3cad] serving for `%s`...\n", _requestHeaders.RequestURI ()) + } + + _response.SetStatusCode (_responseStatus) + _response.SetBodyRaw (_data) +} + + + + +func (_server *server) ServeStatic (_context *fasthttp.RequestCtx, _status uint, _data []byte, _contentType string, _contentEncoding string, _cache bool) () { + + _response := (*fasthttp.Response) (NoEscape (unsafe.Pointer (&_context.Response))) + _responseHeaders := (*fasthttp.ResponseHeader) (NoEscape (unsafe.Pointer (&_context.Response.Header))) + + _responseHeaders.AddRawKv (StringToBytes ("Content-Type"), StringToBytes (_contentType)) + _responseHeaders.AddRawKv (StringToBytes ("Content-Encoding"), StringToBytes (_contentEncoding)) + + if _cache { + _responseHeaders.AddRawLines (StringToBytes ("Cache-Control: public, immutable, max-age=3600\r\n")) + } else { + _responseHeaders.AddRawLines (StringToBytes ("Cache-Control: private, no-cache, no-store\r\n")) + } + + _response.SetStatusCode (int (_status)) + _response.SetBodyRaw (_data) +} + + +func (_server *server) ServeRedirect (_context *fasthttp.RequestCtx, _status uint, _path []byte, _cache bool) () { + + _response := (*fasthttp.Response) (NoEscape (unsafe.Pointer (&_context.Response))) + _responseHeaders := (*fasthttp.ResponseHeader) (NoEscape (unsafe.Pointer (&_context.Response.Header))) + + _responseHeaders.SetCanonical (StringToBytes ("Location"), _path) + + if _cache { + _responseHeaders.AddRawLines (StringToBytes ("Cache-Control: public, immutable, max-age=3600\r\n")) + } else { + _responseHeaders.AddRawLines (StringToBytes ("Cache-Control: private, no-cache, no-store\r\n")) + } + + _response.SetStatusCode (int (_status)) +} + + +func (_server *server) ServeError (_context *fasthttp.RequestCtx, _status uint, _error error, _cache bool) () { + + _response := (*fasthttp.Response) (NoEscape (unsafe.Pointer (&_context.Response))) + _responseHeaders := (*fasthttp.ResponseHeader) (NoEscape (unsafe.Pointer (&_context.Response.Header))) + + _responseHeaders.AddRawKv (StringToBytes ("Content-Type"), StringToBytes (ErrorBannerContentType)) + _responseHeaders.AddRawKv (StringToBytes ("Content-Encoding"), StringToBytes (ErrorBannerContentEncoding)) + + if _cache { + _responseHeaders.AddRawLines (StringToBytes ("Cache-Control: public, immutable, max-age=3600\r\n")) + } else { + _responseHeaders.AddRawLines (StringToBytes ("Cache-Control: private, no-cache, no-store\r\n")) + } + + if _banner, _bannerFound := ErrorBannersData[_status]; _bannerFound { + _response.SetBodyRaw (_banner) + } + + _response.SetStatusCode (int (_status)) + + LogError (_error, "") +} + + +func (_server *server) ServeDummy (_context *fasthttp.RequestCtx) () { + if false { + _server.ServeStatic (_context, http.StatusOK, DummyData, DummyContentType, DummyContentEncoding, false) + } else { + ServeDummyRaw (_context) + } +} + +func ServeDummyRaw (_context *fasthttp.RequestCtx) () { + _context.Response.Header.SetRaw (DummyMeta) + _context.Response.SetBodyRaw (DummyData) +} + + + + +func Main () () { + Main_0 (main_0) +} + + +func main_0 () (error) { + + + var _bind string + var _archivePath string + var _archiveInmem bool + var _archiveMmap bool + var _archivePreload bool + var _indexPaths bool + var _indexDataMeta bool + var _indexDataContent bool + var _securityHeadersEnabled bool + var _securityHeadersTls bool + var _timeoutDisabled bool + var _processes uint + var _threads uint + var _slave uint + var _debug bool + var _dummy bool + var _isFirst bool + var _isMaster bool + + var _profileCpu string + var _profileMem string + + { + _flags := flag.NewFlagSet ("kawipiko-server", flag.ContinueOnError) + + _flags.Usage = func () () { + fmt.Fprintf (os.Stderr, "%s", +` + ==== kawipiko -- blazingly fast static HTTP server ==== + + | Documentation, issues and sources: + | * https://github.com/volution/kawipiko + | Authors: + | * Ciprian Dorin Craciun + | ciprian@volution.ro + | ciprian.craciun@gmail.com + | https://volution.ro/ciprian + ----------------------------------------------------------- + + kawipiko-server + + --bind : + + --processes (of slave processes) + --threads (of threads per process) + + --archive + --archive-inmem (memory-loaded archive file) + --archive-mmap (memory-mapped archive file) + --archive-preload (preload archive file) + + --index-all + --index-paths + --index-data-meta + --index-data-content + + --security-headers-tls + --security-headers-disable + --timeout-disable + + --profile-cpu + --profile-mem + + --debug + --dummy + + ** for details see: + https://github.com/volution/kawipiko#kawipiko-server + +`) + } + + _bind_0 := _flags.String ("bind", "", "") + _archivePath_0 := _flags.String ("archive", "", "") + _archiveInmem_0 := _flags.Bool ("archive-inmem", false, "") + _archiveMmap_0 := _flags.Bool ("archive-mmap", false, "") + _archivePreload_0 := _flags.Bool ("archive-preload", false, "") + _indexAll_0 := _flags.Bool ("index-all", false, "") + _indexPaths_0 := _flags.Bool ("index-paths", false, "") + _indexDataMeta_0 := _flags.Bool ("index-data-meta", false, "") + _indexDataContent_0 := _flags.Bool ("index-data-content", false, "") + _timeoutDisabled_0 := _flags.Bool ("timeout-disable", false, "") + _securityHeadersTls_0 := _flags.Bool ("security-headers-tls", false, "") + _securityHeadersDisabled_0 := _flags.Bool ("security-headers-disable", false, "") + _processes_0 := _flags.Uint ("processes", 0, "") + _threads_0 := _flags.Uint ("threads", 0, "") + _slave_0 := _flags.Uint ("slave", 0, "") + _profileCpu_0 := _flags.String ("profile-cpu", "", "") + _profileMem_0 := _flags.String ("profile-mem", "", "") + _debug_0 := _flags.Bool ("debug", false, "") + _dummy_0 := _flags.Bool ("dummy", false, "") + + FlagsParse (_flags, 0, 0) + + _bind = *_bind_0 + _archivePath = *_archivePath_0 + _archiveInmem = *_archiveInmem_0 + _archiveMmap = *_archiveMmap_0 + _archivePreload = *_archivePreload_0 + _indexAll := *_indexAll_0 + _indexPaths = _indexAll || *_indexPaths_0 + _indexDataMeta = _indexAll || *_indexDataMeta_0 + _indexDataContent = _indexAll || *_indexDataContent_0 + _securityHeadersTls = *_securityHeadersTls_0 + _securityHeadersEnabled = ! *_securityHeadersDisabled_0 + _timeoutDisabled = *_timeoutDisabled_0 + _processes = *_processes_0 + _threads = *_threads_0 + _slave = *_slave_0 + _debug = *_debug_0 + _dummy = *_dummy_0 + + _profileCpu = *_profileCpu_0 + _profileMem = *_profileMem_0 + + if _slave == 0 { + _isMaster = true + } + if _slave <= 1 { + _isFirst = true + } + + if _bind == "" { + AbortError (nil, "[6edd9512] expected bind address argument!") + } + + if !_dummy { + if _archivePath == "" { + AbortError (nil, "[eefe1a38] expected archive file argument!") + } + if _archiveInmem && _archiveMmap { + AbortError (nil, "[a2101041] archive 'memory-loaded' and 'memory-mapped' are mutually exclusive!") + } + if _archiveInmem && _archivePreload { + log.Printf ("[ww] [3e8a40e4] archive 'memory-loaded' implies preloading!\n") + _archivePreload = false + } + } else { + if _isMaster { + log.Printf ("[ww] [8e014192] running in dummy mode; all archive related arguments are ignored!\n") + } + _archivePath = "" + _archiveInmem = false + _archiveMmap = false + _archivePreload = false + _indexAll = false + _indexPaths = false + _indexDataMeta = false + _indexDataContent = false + } + + if (_processes > 1) && ((_profileCpu != "") || (_profileMem != "")) { + AbortError (nil, "[cd18d250] multi-process and profiling are mutually exclusive!") + } + + if _processes < 1 { + _processes = 1 + } + if _threads < 1 { + _threads = 1 + } + + if _processes > 1024 { + AbortError (nil, "[45736c1d] maximum number of allowed processes is 1024!") + } + if _threads > 1024 { + AbortError (nil, "[c5df3c8d] maximum number of allowed threads is 1024!") + } + if (_processes * _threads) > 1024 { + AbortError (nil, "[b0177488] maximum number of allowed threads in total is 1024!") + } + } + + + runtime.GOMAXPROCS (int (_threads)) + + debug.SetGCPercent (50) + debug.SetMaxThreads (int (128 * (_threads / 64 + 1))) + debug.SetMaxStack (16 * 1024) + + + _httpServerReduceMemory := false + + if false { + if _error := syscall.Setrlimit (syscall.RLIMIT_DATA, & syscall.Rlimit { Max : 4 * 1024 * 1024 * 1024 }); _error != nil { + AbortError (_error, "[f661b4fe] failed to configure limits!") + } + } + + + if _processes > 1 { + + log.Printf ("[ii] [06f8c944] sub-processes starting (`%d` processes with `%d` threads each)...\n", _processes, _threads) + + _processesJoin := & sync.WaitGroup {} + + _processesPid := make ([]*os.Process, _processes) + + _processName := os.Args[0] + _processArguments := make ([]string, 0, len (os.Args)) + _processArguments = append (_processArguments, + "--bind", _bind, + ) + if _archivePath != "" { + _processArguments = append (_processArguments, "--archive", _archivePath) + } + if _archiveInmem { + _processArguments = append (_processArguments, "--archive-inmem") + } + if _archiveMmap { + _processArguments = append (_processArguments, "--archive-mmap") + } + if _archivePreload { + _processArguments = append (_processArguments, "--archive-preload") + } + if _indexPaths { + _processArguments = append (_processArguments, "--index-paths") + } + if _indexDataMeta { + _processArguments = append (_processArguments, "--index-data-meta") + } + if _indexDataContent { + _processArguments = append (_processArguments, "--index-data-content") + } + if _securityHeadersTls { + _processArguments = append (_processArguments, "--security-headers-tls") + } + if !_securityHeadersEnabled { + _processArguments = append (_processArguments, "--security-headers-disable") + } + if _timeoutDisabled { + _processArguments = append (_processArguments, "--timeout-disable") + } + if _debug { + _processArguments = append (_processArguments, "--debug") + } + if _dummy { + _processArguments = append (_processArguments, "--dummy") + } + _processArguments = append (_processArguments, "--threads", fmt.Sprintf ("%d", _threads)) + + _processAttributes := & os.ProcAttr { + Env : []string {}, + Files : []*os.File { + os.Stdin, + os.Stdout, + os.Stderr, + }, + Sys : nil, + } + + for _processIndex, _ := range _processesPid { + _processArguments := append ([]string { _processName, "--slave", fmt.Sprintf ("%d", _processIndex + 1) }, _processArguments ...) + if _processPid, _error := os.StartProcess (_processName, _processArguments, _processAttributes); _error == nil { + _processesJoin.Add (1) + _processesPid[_processIndex] = _processPid + if _debug { + log.Printf ("[ii] [63cb22f8] sub-process `%d` started (with `%d` threads);\n", _processPid.Pid, _threads) + } + go func (_index int, _processPid *os.Process) () { + if _processStatus, _error := _processPid.Wait (); _error == nil { + if _processStatus.Success () { + if _debug { + log.Printf ("[ii] [66b60b81] sub-process `%d` succeeded;\n", _processPid.Pid) + } + } else { + log.Printf ("[ww] [5d25046b] sub-process `%d` failed: `%s`; ignoring!\n", _processPid.Pid, _processStatus) + } + } else { + LogError (_error, fmt.Sprintf ("[f1bfc927] failed waiting for sub-process `%d`; ignoring!", _processPid.Pid)) + } + _processesPid[_processIndex] = nil + _processesJoin.Done () + } (_processIndex, _processPid) + } else { + LogError (_error, "[8892b34d] failed starting sub-process; ignoring!") + } + } + + { + _signals := make (chan os.Signal, 32) + signal.Notify (_signals, syscall.SIGINT, syscall.SIGTERM) + go func () () { + for { + _signal := <- _signals + if _debug { + log.Printf ("[ii] [a9243ecb] signaling sub-processes...\n") + } + for _, _processPid := range _processesPid { + if _processPid != nil { + if _error := _processPid.Signal (_signal); _error != nil { + LogError (_error, fmt.Sprintf ("[ab681164] failed signaling sub-process `%d`; ignoring!", _processPid.Pid)) + } + } + } + } + } () + } + + _processesJoin.Wait () + + if _debug { + log.Printf ("[ii] [b949bafc] sub-processes terminated;\n") + } + + return nil + } + + + if _isMaster { + log.Printf ("[ii] [6602a54a] starting (with `%d` threads)...\n", _threads) + } + + + var _cdbReader *cdb.CDB + if _archivePath != "" { + + if _debug || _isFirst { + log.Printf ("[ii] [3b788396] opening archive file `%s`...\n", _archivePath) + } + + var _cdbFile *os.File + if _cdbFile_0, _error := os.Open (_archivePath); _error == nil { + _cdbFile = _cdbFile_0 + } else { + AbortError (_error, "[9e0b5ed3] failed opening archive file!") + } + + var _cdbFileSize int + { + var _cdbFileSize_0 int64 + if _cdbFileStat, _error := _cdbFile.Stat (); _error == nil { + _cdbFileSize_0 = _cdbFileStat.Size () + } else { + AbortError (_error, "[0ccf0a3b] failed opening archive file!") + } + if _cdbFileSize_0 < 1024 { + AbortError (nil, "[6635a2a8] failed opening archive: file is too small (or empty)!") + } + if _cdbFileSize_0 >= (2 * 1024 * 1024 * 1024) { + AbortError (nil, "[545bf6ce] failed opening archive: file is too large!") + } + _cdbFileSize = int (_cdbFileSize_0) + } + + if _archivePreload { + if _debug { + log.Printf ("[ii] [13f4ebf7] preloading archive file...\n") + } + _buffer := [16 * 1024]byte {} + _loop : for { + switch _, _error := _cdbFile.Read (_buffer[:]); _error { + case io.EOF : + break _loop + case nil : + continue _loop + default : + AbortError (_error, "[a1c3b922] failed preloading archive file...\n") + } + } + } + + if _archiveInmem || _archiveMmap { + + var _cdbData []byte + + if _archiveInmem { + + if _debug { + log.Printf ("[ii] [216e584b] opening memory-loaded archive...\n") + } + + _cdbData = make ([]byte, _cdbFileSize) + if _, _error := io.ReadFull (_cdbFile, _cdbData); _error != nil { + AbortError (_error, "[73039784] failed loading archive file!") + } + + } else if _archiveMmap { + + if _debug { + log.Printf ("[ii] [f47fae8a] opening memory-mapped archive...\n") + } + + if _cdbData_0, _error := syscall.Mmap (int (_cdbFile.Fd ()), 0, int (_cdbFileSize), syscall.PROT_READ, syscall.MAP_SHARED); _error == nil { + _cdbData = _cdbData_0 + } else { + AbortError (_error, "[c0e2632c] failed mapping archive file!") + } + + if _archivePreload { + if _debug { + log.Printf ("[ii] [d96b06c9] preloading memory-loaded archive...\n") + } + _buffer := [16 * 1024]byte {} + _bufferOffset := 0 + for { + if _bufferOffset == _cdbFileSize { + break + } + _bufferOffset += copy (_buffer[:], _cdbData[_bufferOffset:]) + } + } + + } else { + panic ("e4fffcd8") + } + + if _error := _cdbFile.Close (); _error != nil { + AbortError (_error, "[5e0449c2] failed closing archive file!") + } + + if _cdbReader_0, _error := cdb.NewFromBufferWithHasher (_cdbData, nil); _error == nil { + _cdbReader = _cdbReader_0 + } else { + AbortError (_error, "[27e4813e] failed opening archive!") + } + + } else { + + if _debug || _isFirst { + log.Printf ("[ww] [dd697a66] using `read`-based archive (with significant performance impact)!\n") + } + + if _cdbReader_0, _error := cdb.NewFromReaderWithHasher (_cdbFile, nil); _error == nil { + _cdbReader = _cdbReader_0 + } else { + AbortError (_error, "[35832022] failed opening archive!") + } + + } + + if _schemaVersion, _error := _cdbReader.GetWithCdbHash ([]byte (NamespaceSchemaVersion)); _error == nil { + if _schemaVersion == nil { + AbortError (nil, "[09316866] missing archive schema version!") + } else if string (_schemaVersion) != CurrentSchemaVersion { + AbortError (nil, "[e6482cf7] invalid archive schema version!") + } + } else { + AbortError (_error, "[87cae197] failed opening archive!") + } + } + + + var _cachedFileFingerprints map[string][]byte + if _indexPaths { + _cachedFileFingerprints = make (map[string][]byte, 128 * 1024) + } + var _cachedDataMeta map[string][]byte + if _indexDataMeta { + _cachedDataMeta = make (map[string][]byte, 128 * 1024) + } + var _cachedDataContent map[string][]byte + if _indexDataContent { + _cachedDataContent = make (map[string][]byte, 128 * 1024) + } + + if _indexPaths || _indexDataMeta || _indexDataContent { + if _debug { + log.Printf ("[ii] [fa5338fd] indexing archive...\n") + } + if _filesIndex, _error := _cdbReader.GetWithCdbHash ([]byte (NamespaceFilesIndex)); _error == nil { + if _filesIndex != nil { + _keyBuffer := [1024]byte {} + for { + _offset := bytes.IndexByte (_filesIndex, '\n') + if _offset == 0 { + continue + } + if _offset == -1 { + break + } + _filePath := _filesIndex[: _offset] + _filesIndex = _filesIndex[_offset + 1 :] + var _fingerprints []byte + var _fingerprintContent []byte + var _fingerprintMeta []byte + { + _key := _keyBuffer[:0] + _key = append (_key, NamespaceFilesContent ...) + _key = append (_key, ':') + _key = append (_key, _filePath ...) + if _fingerprints_0, _error := _cdbReader.GetWithCdbHash (_key); _error == nil { + if _fingerprints_0 != nil { + _fingerprints = _fingerprints_0 + _fingerprintContent = _fingerprints[0:64] + _fingerprintMeta = _fingerprints[65:129] + } else { + AbortError (_error, "[460b3cf1] failed indexing archive!") + } + } else { + AbortError (_error, "[216f2075] failed indexing archive!") + } + } + if _indexPaths { + _cachedFileFingerprints[BytesToString (_filePath)] = _fingerprints + } + if _indexDataMeta { + if _, _wasCached := _cachedDataMeta[BytesToString (_fingerprintMeta)]; !_wasCached { + _key := _keyBuffer[:0] + _key = append (_key, NamespaceDataMetadata ...) + _key = append (_key, ':') + _key = append (_key, _fingerprintMeta ...) + if _dataMeta, _error := _cdbReader.GetWithCdbHash (_key); _error == nil { + if _dataMeta != nil { + _cachedDataMeta[BytesToString (_fingerprintMeta)] = _dataMeta + } else { + AbortError (_error, "[6df556bf] failed indexing archive!") + } + } else { + AbortError (_error, "[0d730134] failed indexing archive!") + } + } + } + if _indexDataContent { + if _, _wasCached := _cachedDataContent[BytesToString (_fingerprintContent)]; !_wasCached { + _key := _keyBuffer[:0] + _key = append (_key, NamespaceDataContent ...) + _key = append (_key, ':') + _key = append (_key, _fingerprintContent ...) + if _dataContent, _error := _cdbReader.GetWithCdbHash (_key); _error == nil { + if _dataContent != nil { + _cachedDataContent[BytesToString (_fingerprintContent)] = _dataContent + } else { + AbortError (_error, "[4e27fe46] failed indexing archive!") + } + } else { + AbortError (_error, "[532845ad] failed indexing archive!") + } + } + } + } + } else { + log.Printf ("[ww] [30314f31] missing archive files index; ignoring!\n") + } + } else { + AbortError (_error, "[82299b3d] failed indexing arcdive!") + } + } + + if _indexPaths && _indexDataMeta && _indexDataContent { + if _error := _cdbReader.Close (); _error == nil { + _cdbReader = nil + } else { + AbortError (_error, "[d7aa79e1] failed closing archive!") + } + } + + + _server := & server { + httpServer : nil, + cdbReader : _cdbReader, + cachedFileFingerprints : _cachedFileFingerprints, + cachedDataMeta : _cachedDataMeta, + cachedDataContent : _cachedDataContent, + securityHeadersTls : _securityHeadersTls, + securityHeadersEnabled : _securityHeadersEnabled, + debug : _debug, + dummy : _dummy, + } + + + if _profileCpu != "" { + log.Printf ("[ii] [70c210f3] profiling CPU to `%s`...\n", _profileCpu) + _stream, _error := os.Create (_profileCpu) + if _error != nil { + AbortError (_error, "[fd4e0009] failed opening CPU profile!") + } + _error = pprof.StartCPUProfile (_stream) + if _error != nil { + AbortError (_error, "[ac721629] failed starting CPU profile!") + } + defer pprof.StopCPUProfile () + } + if _profileMem != "" { + log.Printf ("[ii] [9196ee90] profiling MEM to `%s`...\n", _profileMem) + _stream, _error := os.Create (_profileMem) + if _error != nil { + AbortError (_error, "[907d08b5] failed opening MEM profile!") + } + _profile := pprof.Lookup ("heap") + defer func () () { + runtime.GC () + if _profile != nil { + if _error := _profile.WriteTo (_stream, 0); _error != nil { + AbortError (_error, "[4b1e5112] failed writing MEM profile!") + } + } else { + AbortError (nil, "[385dc8f0] failed loading MEM profile!") + } + _stream.Close () + } () + } + + + _httpServer := & fasthttp.Server { + + Name : "kawipiko", + Handler : _server.Serve, + GetOnly : true, + + NoDefaultServerHeader : true, + NoDefaultContentType : true, + NoDefaultDate : true, + DisableHeaderNamesNormalizing : true, + + Concurrency : 16 * 1024 + 128, + MaxRequestsPerConn : 256 * 1024, + + ReadBufferSize : 16 * 1024, + WriteBufferSize : 16 * 1024, + MaxRequestBodySize : 16 * 1024, + + ReadTimeout : 30 * time.Second, + WriteTimeout : 30 * time.Second, + IdleTimeout : 360 * time.Second, + + TCPKeepalive : true, + TCPKeepalivePeriod : 60 * time.Second, + + ReduceMemoryUsage : _httpServerReduceMemory, + + } + + if _timeoutDisabled { + _httpServer.ReadTimeout = 0 + _httpServer.WriteTimeout = 0 + _httpServer.IdleTimeout = 0 + } + + if _dummy { + _httpServer.Handler = _server.ServeDummy + } + + _server.httpServer = _httpServer + + + { + _signals := make (chan os.Signal, 32) + signal.Notify (_signals, syscall.SIGINT, syscall.SIGTERM) + go func () () { + <- _signals + if _debug { + log.Printf ("[ii] [691cb695] shutingdown...\n") + } + _server.httpServer.Shutdown () + } () + } + + + if _debug || _isFirst { + log.Printf ("[ii] [f11e4e37] listening on `http://%s/`;\n", _bind) + } + + var _httpListener net.Listener + if _httpListener_0, _error := reuseport.Listen ("tcp4", _bind); _error == nil { + _httpListener = _httpListener_0 + } else { + AbortError (_error, "[d5f51e9f] failed starting listener!") + } + + if _error := _httpServer.Serve (_httpListener); _error != nil { + AbortError (_error, "[44f45c67] failed executing server!") + } + + + if _debug { + defer log.Printf ("[ii] [a49175db] done!\n") + } + return nil +} + diff --git a/sources/lib/common/main.go b/sources/lib/common/main.go index bf2d890..764cffb 100644 --- a/sources/lib/common/main.go +++ b/sources/lib/common/main.go @@ -11,7 +11,7 @@ import "os" -func Main (_main func () (error)) () { +func Main_0 (_main func () (error)) () { log.SetFlags (0) log.SetPrefix (fmt.Sprintf ("[%8d] ", os.Getpid ()))