From 21e64cfc7911c392f462772794f3f1151c9bf44a Mon Sep 17 00:00:00 2001 From: Ciprian Dorin Craciun Date: Fri, 9 Nov 2018 02:47:14 +0200 Subject: [PATCH] [sources] Import initial version of archiver and server --- sources/cmd/archiver.go | 294 ++++++++++++++++++++++++++++++++++++++ sources/cmd/server.go | 190 ++++++++++++++++++++++++ sources/lib/common.go | 90 ++++++++++++ sources/lib/namespaces.go | 12 ++ 4 files changed, 586 insertions(+) create mode 100644 sources/cmd/archiver.go create mode 100644 sources/cmd/server.go create mode 100644 sources/lib/common.go create mode 100644 sources/lib/namespaces.go diff --git a/sources/cmd/archiver.go b/sources/cmd/archiver.go new file mode 100644 index 0000000..f46f0df --- /dev/null +++ b/sources/cmd/archiver.go @@ -0,0 +1,294 @@ + + +package main + + +import "crypto/sha256" +import "encoding/hex" +import "encoding/json" +import "flag" +import "fmt" +import "io" +import "io/ioutil" +import "log" +import "net/http" +import "path/filepath" +import "os" +import "sort" + +import "github.com/colinmarc/cdb" + +import . "../lib" + + + + +func archiveFile (_cdbWriter *cdb.Writer, _pathResolved string, _pathInArchive string, _name string, _stat os.FileInfo, _stored map[string]bool, _debug bool) (error) { + + var _data []byte + if _data_0, _error := ioutil.ReadFile (_pathResolved); _error == nil { + _data = _data_0 + } else { + return _error + } + + return archiveData (_cdbWriter, NamespaceFilesContent, _pathInArchive, _data, "", _stored, _debug) +} + + + + +func archiveFolder (_cdbWriter *cdb.Writer, _pathResolved string, _pathInArchive string, _names []string, _stats map[string]os.FileInfo, _stored map[string]bool, _debug bool) (error) { + + type Entry struct { + Name string `json:"name",omitempty` + Type string `json:"type",omitempty` + Size uint64 `json:"size",omitempty"` + } + + type Folder struct { + Entries []Entry `json:"entries"` + } + + _entries := make ([]Entry, 0, len (_names)) + for _, _name := range _names { + _entry := Entry { + Name : _name, + Type : "unknown", + } + _stat := _stats[_name] + _statMode := _stat.Mode () + if _statMode.IsRegular () { + _entry.Type = "file" + _entry.Size = uint64 (_stat.Size ()) + } else if _statMode.IsDir () { + _entry.Type = "folder" + } + _entries = append (_entries, _entry) + } + + _folder := Folder { + Entries : _entries, + } + + if _data, _error := json.Marshal (&_folder); _error == nil { + return archiveData (_cdbWriter, NamespaceFoldersMetadata, _pathInArchive, _data, "application/json; charset=utf-8", _stored, _debug) + } + + return nil +} + + + + +func archiveData (_cdbWriter *cdb.Writer, _namespace string, _pathInArchive string, _data []byte, _dataType string, _stored map[string]bool, _debug bool) (error) { + + _fingerprintRaw := sha256.Sum256 (_data) + _fingerprint := hex.EncodeToString (_fingerprintRaw[:]) + + _wasStored, _ := _stored[_fingerprint] + + if ! _wasStored { + _key := fmt.Sprintf ("%s:%s", NamespaceDataContent, _fingerprint) + if _debug { + log.Printf ("[ ] ++ %s", _key) + } + if _error := _cdbWriter.Put ([]byte (_key), _data); _error != nil { + return _error + } + } + + if ! _wasStored { + _key := fmt.Sprintf ("%s:%s", NamespaceDataContentType, _fingerprint) + if _dataType == "" { + _dataType = http.DetectContentType (_data) + } + if _debug { + log.Printf ("[ ] ++ %s %s", _key, _dataType) + } + if _error := _cdbWriter.Put ([]byte (_key), []byte (_dataType)); _error != nil { + return _error + } + } + + if _namespace != "" { + _key := fmt.Sprintf ("%s:%s", _namespace, _pathInArchive) + if _debug { + log.Printf ("[ ] ++ %s", _key) + } + if _error := _cdbWriter.Put ([]byte (_key), []byte (_fingerprint)); _error != nil { + return _error + } + } + + return nil +} + + + + +func walkPath (_cdbWriter *cdb.Writer, _path string, _prefix string, _name string, _recursed map[string]uint, _stored map[string]bool, _debug bool) (error) { + + if _recursed == nil { + _recursed = make (map[string]uint, 128) + } + if _stored == nil { + _stored = make (map[string]bool, 16 * 1024) + } + + _pathInArchive := filepath.Join (_prefix, _name) + + var _stat os.FileInfo + if _stat_0, _error := os.Lstat (_path); _error == nil { + _stat = _stat_0 + } else { + return _error + } + + _isSymlink := false + if (_stat.Mode () & os.ModeSymlink) != 0 { + _isSymlink = true + if _stat_0, _error := os.Stat (_path); _error == nil { + _stat = _stat_0 + } else { + return _error + } + } + + var _pathResolved string + if _isSymlink { + if _resolved, _error := filepath.EvalSymlinks (_path); _error == nil { + _pathResolved = _resolved + } else { + return _error + } + } else { + _pathResolved = _path + } + + if _isSymlink && _debug { + log.Printf ("[ ] ~~ %s -> %s\n", _pathInArchive, _pathResolved) + } + + if _stat.Mode () .IsRegular () { + + if _debug { + log.Printf ("[ ] ## %s\n", _pathInArchive) + } + return archiveFile (_cdbWriter, _pathResolved, _pathInArchive, _name, _stat, _stored, _debug) + + } else if _stat.Mode () .IsDir () { + + _wasRecursed, _ := _recursed[_pathResolved] + if _wasRecursed > 0 { + log.Printf ("[ww] [2e1744c9] detected directory loop for `%s` resolving to `%s`; ignoring!\n", _path, _pathResolved) + return nil + } + _recursed[_pathResolved] = _wasRecursed + 1 + + if _debug { + log.Printf ("[ ] >> %s\n", _pathInArchive) + } + + _names := make ([]string, 0, 16) + _stats := make (map[string]os.FileInfo, 16) + + if _stream, _error := os.Open (_path); _error == nil { + defer _stream.Close () + _prefix = filepath.Join (_prefix, _name) + _loop : for { + switch _buffer, _error := _stream.Readdir (128); _error { + case nil : + for _, _stat := range _buffer { + _name := _stat.Name () + _names = append (_names, _name) + _stats[_name] = _stat + if _error := walkPath (_cdbWriter, filepath.Join (_path, _name), _prefix, _name, _recursed, _stored, _debug); _error != nil { + return _error + } + } + case io.EOF : + break _loop + default : + return _error + } + } + } + + sort.Strings (_names) + + if _debug { + log.Printf ("[ ] << %s\n", _pathInArchive) + } + + if _debug { + log.Printf ("[ ] <> %s\n", _pathInArchive) + if _error := archiveFolder (_cdbWriter, _pathResolved, _pathInArchive, _names, _stats, _stored, _debug); _error != nil { + return _error + } + } + + _recursed[_pathResolved] = _wasRecursed + return nil + + } else { + return fmt.Errorf ("[d9b836d7] unexpected file type for `%s`: `%s`!", _path, _stat.Mode ()) + } +} + + + + +func main () () { + Main (main_0) +} + + +func main_0 () (error) { + + + var _sourcesFolder string + var _archiveFile string + var _debug bool + + { + _flags := flag.NewFlagSet ("cdb-archiver", flag.ContinueOnError) + + _sourcesFolder_0 := _flags.String ("sources", "", "") + _archiveFile_0 := _flags.String ("archive", "", "") + _debug_0 := _flags.Bool ("debug", false, "") + + FlagsParse (_flags, 0, 0) + + _sourcesFolder = *_sourcesFolder_0 + _archiveFile = *_archiveFile_0 + _debug = *_debug_0 + + if _sourcesFolder == "" { + AbortError (nil, "[515ee462] expected sources folder argument!") + } + if _archiveFile == "" { + AbortError (nil, "[5e8da985] expected archive file argument!") + } + } + + + var _cdbWriter *cdb.Writer + if _cdbWriter_0, _error := cdb.Create (_archiveFile); _error == nil { + _cdbWriter = _cdbWriter_0 + } else { + AbortError (_error, "[85234ba0] failed creating archive (while opening)!") + } + + if _error := walkPath (_cdbWriter, _sourcesFolder, "/", "", nil, nil, _debug); _error != nil { + AbortError (_error, "[b6a19ef4] failed walking folder!") + } + + if _error := _cdbWriter.Close (); _error != nil { + AbortError (_error, "[bbfb8478] failed creating archive (while closing)!") + } + + + return nil +} + diff --git a/sources/cmd/server.go b/sources/cmd/server.go new file mode 100644 index 0000000..8e68102 --- /dev/null +++ b/sources/cmd/server.go @@ -0,0 +1,190 @@ + + +package main + + +import "flag" +import "fmt" +import "log" +import "net/http" +import "net/url" + +import "github.com/colinmarc/cdb" + +import . "../lib" + + + +type server struct { + cdbReader *cdb.CDB + debug bool +} + + +func (_server *server) ServeHTTP (_response http.ResponseWriter, _request *http.Request) () { + + _responseHeaders := _response.Header () + + _responseHeaders.Set ("Content-Security-Policy", "upgrade-insecure-requests") + _responseHeaders.Set ("Referrer-Policy", "strict-origin-when-cross-origin") + _responseHeaders.Set ("X-Frame-Options", "SAMEORIGIN") + _responseHeaders.Set ("X-Content-Type-Options", "nosniff") + _responseHeaders.Set ("X-XSS-Protection", "1; mode=block") + + _method := _request.Method + _path := _request.URL.Path + + if _method != http.MethodGet { + log.Printf ("[ww] [bce7a75b] invalid method `%s` for `%s`!", _method, _path) + _server.ServeError (_response, http.StatusMethodNotAllowed, nil) + return + } + + var _fingerprint string + { + _path_0 := _path + if (_path != "/") && (_path[len (_path) - 1] == '/') { + _path_0 = _path[: len (_path) - 1] + } + for _, _namespace := range []string {NamespaceFilesContent, NamespaceFoldersContent, NamespaceFoldersMetadata} { + _key := fmt.Sprintf ("%s:%s", _namespace, _path_0) + if _value, _error := _server.cdbReader.Get ([]byte (_key)); _error == nil { + if _value != nil { + _fingerprint = string (_value) + if ((_namespace == NamespaceFoldersContent) || (_namespace == NamespaceFoldersMetadata)) && (_path == _path_0) { + _server.ServeRedirect (_response, http.StatusTemporaryRedirect, _path + "/") + return + } + break + } + } else { + _server.ServeError (_response, http.StatusInternalServerError, _error) + return + } + } + } + if _fingerprint == "" { + log.Printf ("[ww] [7416f61d] not found for `%s`!", _path) + _server.ServeError (_response, http.StatusNotFound, nil) + return + } + + var _data []byte + { + _key := fmt.Sprintf ("%s:%s", NamespaceDataContent, _fingerprint) + if _value, _error := _server.cdbReader.Get ([]byte (_key)); _error == nil { + if _value != nil { + _data = _value + } + } else { + _server.ServeError (_response, http.StatusInternalServerError, _error) + return + } + } + if _data == nil { + _server.ServeError (_response, http.StatusNotFound, nil) + return + } + + { + _key := fmt.Sprintf ("%s:%s", NamespaceDataContentType, _fingerprint) + if _value, _error := _server.cdbReader.Get ([]byte (_key)); _error == nil { + if _value != nil { + _responseHeaders.Set ("Content-Type", string (_value)) + } + } else { + _server.ServeError (_response, http.StatusInternalServerError, _error) + return + } + } + + if _server.debug { + log.Printf ("[dd] [b15f3cad] serving for `%s`...\n", _path) + } + + _response.WriteHeader (http.StatusOK) + _response.Write (_data) +} + + + + +func (_server *server) ServeRedirect (_response http.ResponseWriter, _status uint, _urlRaw string) () { + var _url string + if _url_0, _error := url.Parse (_urlRaw); _error == nil { + _url = _url_0.String () + } else { + _server.ServeError (_response, http.StatusInternalServerError, _error) + return + } + _response.Header () .Set ("Content-Type", "text/plain; charset=utf-8") + _response.Header () .Set ("Location", _url) + _response.WriteHeader (int (_status)) + _response.Write ([]byte (fmt.Sprintf ("[%d] %s", _status, _url))) +} + + +func (_server *server) ServeError (_response http.ResponseWriter, _status uint, _error error) () { + _response.Header () .Set ("Content-Type", "text/plain; charset=utf-8") + _response.WriteHeader (int (_status)) + _response.Write ([]byte (fmt.Sprintf ("[%d]", _status))) + LogError (_error, "") +} + + + + +func main () () { + Main (main_0) +} + + +func main_0 () (error) { + + + var _bind string + var _archive string + var _debug bool + + { + _flags := flag.NewFlagSet ("cdb-server", flag.ContinueOnError) + + _bind_0 := _flags.String ("bind", "", ":") + _archive_0 := _flags.String ("archive", "", "") + _debug_0 := _flags.Bool ("debug", false, "") + + FlagsParse (_flags, 0, 0) + + _bind = *_bind_0 + _archive = *_archive_0 + _debug = *_debug_0 + + if _bind == "" { + AbortError (nil, "[eefe1a38] expected bind address argument!") + } + if _archive == "" { + AbortError (nil, "[eefe1a38] expected archive file argument!") + } + } + + + var _cdbReader *cdb.CDB + if _cdbReader_0, _error := cdb.Open (_archive); _error == nil { + _cdbReader = _cdbReader_0 + } else { + AbortError (_error, "[85234ba0] failed opening archive!") + } + + + _server := & server { + cdbReader : _cdbReader, + debug : _debug, + } + + if _error := http.ListenAndServe (_bind, _server); _error != nil { + AbortError (_error, "[44f45c67] failed starting server!") + } + + return nil +} + diff --git a/sources/lib/common.go b/sources/lib/common.go new file mode 100644 index 0000000..f316233 --- /dev/null +++ b/sources/lib/common.go @@ -0,0 +1,90 @@ + + +package lib + + +import "flag" +import "fmt" +import "log" +import "os" +import "regexp" + + + + +func Main (_main func () (error)) () { + + log.SetFlags (0) + + if _error := _main (); _error == nil { + os.Exit (0) + } else { + AbortError (_error, "#") + } +} + + + + +func FlagsParse (_flags *flag.FlagSet, _argumentsMin uint, _argumentsMax uint) ([]string) { + + _arguments := os.Args[1:] + + if _error := _flags.Parse (_arguments); _error != nil { + AbortError (_error, fmt.Sprintf ("[8fae7a93] failed parsing arguments: `%v`!", _arguments)) + } + + _flagsNArg := uint (_flags.NArg ()) + if _argumentsMin == _argumentsMax { + if _flagsNArg != _argumentsMin { + AbortError (nil, fmt.Sprintf ("[c451f1f8] expected exactly `%d` positional arguments!", _argumentsMin)) + } + } else { + if _flagsNArg < _argumentsMin { + AbortError (nil, fmt.Sprintf ("[c451f1f8] expected at least `%d` positional arguments!", _argumentsMin)) + } + if _flagsNArg > _argumentsMax { + AbortError (nil, fmt.Sprintf ("[fa0a8c22] expected at most `%d` positional arguments!", _argumentsMax)) + } + } + + return _flags.Args () +} + + + + +func LogError (_error error, _message string) () { + + if _message != "#" { + if (_message == "") && (_error != nil) { + _message = "[70d7e7c6] unexpected error encountered!"; + } + if _message != "" { + log.Printf ("[ee] %s\n", _message) + } + } + + if _error != nil { + _errorString := _error.Error () + if _matches, _matchesError := regexp.MatchString (`^\[[0-9a-f]{8}\] [^\n]+$`, _errorString); _matchesError == nil { + if _matches { + log.Printf ("[ee] %s\n", _errorString) + } else { + log.Printf ("[ee] [c776ae31] %q\n", _errorString) + log.Printf ("[ee] [ddd6baae] %#v\n", _error) + } + } else { + log.Printf ("[ee] [609a0410] %q\n", _errorString) + log.Printf ("[ee] [2ddce4bf] %#v\n", _error) + } + } +} + + +func AbortError (_error error, _message string) () { + LogError (_error, _message) + log.Printf ("[!!] [89251d36] aborting!\n") + os.Exit (1) +} + diff --git a/sources/lib/namespaces.go b/sources/lib/namespaces.go new file mode 100644 index 0000000..e4fca47 --- /dev/null +++ b/sources/lib/namespaces.go @@ -0,0 +1,12 @@ + + +package lib + + +const NamespaceFilesContent = "files:content" +const NamespaceFoldersContent = "folders:content" +const NamespaceFoldersMetadata = "folders:metadata" + +const NamespaceDataContent = "data:content" +const NamespaceDataContentType = "data:content-type" +