[sources] Import initial version of archiver and server

sources/cmd/archiver.go

@@ -0,0 +1,294 @@
+
+
+package main
+
+
+import "crypto/sha256"
+import "encoding/hex"
+import "encoding/json"
+import "flag"
+import "fmt"
+import "io"
+import "io/ioutil"
+import "log"
+import "net/http"
+import "path/filepath"
+import "os"
+import "sort"
+
+import "github.com/colinmarc/cdb"
+
+import . "../lib"
+
+
+
+
+func archiveFile (_cdbWriter *cdb.Writer, _pathResolved string, _pathInArchive string, _name string, _stat os.FileInfo, _stored map[string]bool, _debug bool) (error) {
+	
+	var _data []byte
+	if _data_0, _error := ioutil.ReadFile (_pathResolved); _error == nil {
+		_data = _data_0
+	} else {
+		return _error
+	}
+	
+	return archiveData (_cdbWriter, NamespaceFilesContent, _pathInArchive, _data, "", _stored, _debug)
+}
+
+
+
+
+func archiveFolder (_cdbWriter *cdb.Writer, _pathResolved string, _pathInArchive string, _names []string, _stats map[string]os.FileInfo, _stored map[string]bool, _debug bool) (error) {
+	
+	type Entry struct {
+		Name string `json:"name",omitempty`
+		Type string `json:"type",omitempty`
+		Size uint64 `json:"size",omitempty"`
+	}
+	
+	type Folder struct {
+		Entries []Entry `json:"entries"`
+	}
+	
+	_entries := make ([]Entry, 0, len (_names))
+	for _, _name := range _names {
+		_entry := Entry {
+				Name : _name,
+				Type : "unknown",
+			}
+		_stat := _stats[_name]
+		_statMode := _stat.Mode ()
+		if _statMode.IsRegular () {
+			_entry.Type = "file"
+			_entry.Size = uint64 (_stat.Size ())
+		} else if _statMode.IsDir () {
+			_entry.Type = "folder"
+		}
+		_entries = append (_entries, _entry)
+	}
+	
+	_folder := Folder {
+			Entries : _entries,
+		}
+	
+	if _data, _error := json.Marshal (&_folder); _error == nil {
+		return archiveData (_cdbWriter, NamespaceFoldersMetadata, _pathInArchive, _data, "application/json; charset=utf-8", _stored, _debug)
+	}
+	
+	return nil
+}
+
+
+
+
+func archiveData (_cdbWriter *cdb.Writer, _namespace string, _pathInArchive string, _data []byte, _dataType string, _stored map[string]bool, _debug bool) (error) {
+	
+	_fingerprintRaw := sha256.Sum256 (_data)
+	_fingerprint := hex.EncodeToString (_fingerprintRaw[:])
+	
+	_wasStored, _ := _stored[_fingerprint]
+	
+	if ! _wasStored {
+		_key := fmt.Sprintf ("%s:%s", NamespaceDataContent, _fingerprint)
+		if _debug {
+			log.Printf ("[  ] ++ %s", _key)
+		}
+		if _error := _cdbWriter.Put ([]byte (_key), _data); _error != nil {
+			return _error
+		}
+	}
+	
+	if ! _wasStored {
+		_key := fmt.Sprintf ("%s:%s", NamespaceDataContentType, _fingerprint)
+		if _dataType == "" {
+			_dataType = http.DetectContentType (_data)
+		}
+		if _debug {
+			log.Printf ("[  ] ++ %s %s", _key, _dataType)
+		}
+		if _error := _cdbWriter.Put ([]byte (_key), []byte (_dataType)); _error != nil {
+			return _error
+		}
+	}
+	
+	if _namespace != "" {
+		_key := fmt.Sprintf ("%s:%s", _namespace, _pathInArchive)
+		if _debug {
+			log.Printf ("[  ] ++ %s", _key)
+		}
+		if _error := _cdbWriter.Put ([]byte (_key), []byte (_fingerprint)); _error != nil {
+			return _error
+		}
+	}
+	
+	return nil
+}
+
+
+
+
+func walkPath (_cdbWriter *cdb.Writer, _path string, _prefix string, _name string, _recursed map[string]uint, _stored map[string]bool, _debug bool) (error) {
+	
+	if _recursed == nil {
+		_recursed = make (map[string]uint, 128)
+	}
+	if _stored == nil {
+		_stored = make (map[string]bool, 16 * 1024)
+	}
+	
+	_pathInArchive := filepath.Join (_prefix, _name)
+	
+	var _stat os.FileInfo
+	if _stat_0, _error := os.Lstat (_path); _error == nil {
+		_stat = _stat_0
+	} else {
+		return _error
+	}
+	
+	_isSymlink := false
+	if (_stat.Mode () & os.ModeSymlink) != 0 {
+		_isSymlink = true
+		if _stat_0, _error := os.Stat (_path); _error == nil {
+			_stat = _stat_0
+		} else {
+			return _error
+		}
+	}
+	
+	var _pathResolved string
+	if _isSymlink {
+		if _resolved, _error := filepath.EvalSymlinks (_path); _error == nil {
+			_pathResolved = _resolved
+		} else {
+			return _error
+		}
+	} else {
+		_pathResolved = _path
+	}
+	
+	if _isSymlink && _debug {
+		log.Printf ("[  ] ~~ %s -> %s\n", _pathInArchive, _pathResolved)
+	}
+	
+	if _stat.Mode () .IsRegular () {
+		
+		if _debug {
+			log.Printf ("[  ] ## %s\n", _pathInArchive)
+		}
+		return archiveFile (_cdbWriter, _pathResolved, _pathInArchive, _name, _stat, _stored, _debug)
+		
+	} else if _stat.Mode () .IsDir () {
+		
+		_wasRecursed, _ := _recursed[_pathResolved]
+		if _wasRecursed > 0 {
+			log.Printf ("[ww] [2e1744c9]  detected directory loop for `%s` resolving to `%s`;  ignoring!\n", _path, _pathResolved)
+			return nil
+		}
+		_recursed[_pathResolved] = _wasRecursed + 1
+		
+		if _debug {
+			log.Printf ("[  ] >> %s\n", _pathInArchive)
+		}
+		
+		_names := make ([]string, 0, 16)
+		_stats := make (map[string]os.FileInfo, 16)
+		
+		if _stream, _error := os.Open (_path); _error == nil {
+			defer _stream.Close ()
+			_prefix = filepath.Join (_prefix, _name)
+			_loop : for {
+				switch _buffer, _error := _stream.Readdir (128); _error {
+					case nil :
+						for _, _stat := range _buffer {
+							_name := _stat.Name ()
+							_names = append (_names, _name)
+							_stats[_name] = _stat
+							if _error := walkPath (_cdbWriter, filepath.Join (_path, _name), _prefix, _name, _recursed, _stored, _debug); _error != nil {
+								return _error
+							}
+						}
+					case io.EOF :
+						break _loop
+					default :
+						return _error
+				}
+			}
+		}
+		
+		sort.Strings (_names)
+		
+		if _debug {
+			log.Printf ("[  ] << %s\n", _pathInArchive)
+		}
+		
+		if _debug {
+			log.Printf ("[  ] <> %s\n", _pathInArchive)
+			if _error := archiveFolder (_cdbWriter, _pathResolved, _pathInArchive, _names, _stats, _stored, _debug); _error != nil {
+				return _error
+			}
+		}
+		
+		_recursed[_pathResolved] = _wasRecursed
+		return nil
+		
+	} else {
+		return fmt.Errorf ("[d9b836d7]  unexpected file type for `%s`:  `%s`!", _path, _stat.Mode ())
+	}
+}
+
+
+
+
+func main () () {
+	Main (main_0)
+}
+
+
+func main_0 () (error) {
+	
+	
+	var _sourcesFolder string
+	var _archiveFile string
+	var _debug bool
+	
+	{
+		_flags := flag.NewFlagSet ("cdb-archiver", flag.ContinueOnError)
+		
+		_sourcesFolder_0 := _flags.String ("sources", "", "<path>")
+		_archiveFile_0 := _flags.String ("archive", "", "<path>")
+		_debug_0 := _flags.Bool ("debug", false, "")
+		
+		FlagsParse (_flags, 0, 0)
+		
+		_sourcesFolder = *_sourcesFolder_0
+		_archiveFile = *_archiveFile_0
+		_debug = *_debug_0
+		
+		if _sourcesFolder == "" {
+			AbortError (nil, "[515ee462]  expected sources folder argument!")
+		}
+		if _archiveFile == "" {
+			AbortError (nil, "[5e8da985]  expected archive file argument!")
+		}
+	}
+	
+	
+	var _cdbWriter *cdb.Writer
+	if _cdbWriter_0, _error := cdb.Create (_archiveFile); _error == nil {
+		_cdbWriter = _cdbWriter_0
+	} else {
+		AbortError (_error, "[85234ba0]  failed creating archive (while opening)!")
+	}
+	
+	if _error := walkPath (_cdbWriter, _sourcesFolder, "/", "", nil, nil, _debug); _error != nil {
+		AbortError (_error, "[b6a19ef4]  failed walking folder!")
+	}
+	
+	if _error := _cdbWriter.Close (); _error != nil {
+		AbortError (_error, "[bbfb8478]  failed creating archive (while closing)!")
+	}
+	
+	
+	return nil
+}
+

sources/cmd/server.go

@@ -0,0 +1,190 @@
+
+
+package main
+
+
+import "flag"
+import "fmt"
+import "log"
+import "net/http"
+import "net/url"
+
+import "github.com/colinmarc/cdb"
+
+import . "../lib"
+
+
+
+type server struct {
+	cdbReader *cdb.CDB
+	debug bool
+}
+
+
+func (_server *server) ServeHTTP (_response http.ResponseWriter, _request *http.Request) () {
+	
+	_responseHeaders := _response.Header ()
+	
+	_responseHeaders.Set ("Content-Security-Policy", "upgrade-insecure-requests")
+	_responseHeaders.Set ("Referrer-Policy", "strict-origin-when-cross-origin")
+	_responseHeaders.Set ("X-Frame-Options", "SAMEORIGIN")
+	_responseHeaders.Set ("X-Content-Type-Options", "nosniff")
+	_responseHeaders.Set ("X-XSS-Protection", "1; mode=block")
+	
+	_method := _request.Method
+	_path := _request.URL.Path
+	
+	if _method != http.MethodGet {
+		log.Printf ("[ww] [bce7a75b] invalid method `%s` for `%s`!", _method, _path)
+		_server.ServeError (_response, http.StatusMethodNotAllowed, nil)
+		return
+	}
+	
+	var _fingerprint string
+	{
+		_path_0 := _path
+		if (_path != "/") && (_path[len (_path) - 1] == '/') {
+			_path_0 = _path[: len (_path) - 1]
+		}
+		for _, _namespace := range []string {NamespaceFilesContent, NamespaceFoldersContent, NamespaceFoldersMetadata} {
+			_key := fmt.Sprintf ("%s:%s", _namespace, _path_0)
+			if _value, _error := _server.cdbReader.Get ([]byte (_key)); _error == nil {
+				if _value != nil {
+					_fingerprint = string (_value)
+					if ((_namespace == NamespaceFoldersContent) || (_namespace == NamespaceFoldersMetadata)) && (_path == _path_0) {
+						_server.ServeRedirect (_response, http.StatusTemporaryRedirect, _path + "/")
+						return
+					}
+					break
+				}
+			} else {
+				_server.ServeError (_response, http.StatusInternalServerError, _error)
+				return
+			}
+		}
+	}
+	if _fingerprint == "" {
+		log.Printf ("[ww] [7416f61d]  not found for `%s`!", _path)
+		_server.ServeError (_response, http.StatusNotFound, nil)
+		return
+	}
+	
+	var _data []byte
+	{
+		_key := fmt.Sprintf ("%s:%s", NamespaceDataContent, _fingerprint)
+		if _value, _error := _server.cdbReader.Get ([]byte (_key)); _error == nil {
+			if _value != nil {
+				_data = _value
+			}
+		} else {
+			_server.ServeError (_response, http.StatusInternalServerError, _error)
+			return
+		}
+	}
+	if _data == nil {
+		_server.ServeError (_response, http.StatusNotFound, nil)
+		return
+	}
+	
+	{
+		_key := fmt.Sprintf ("%s:%s", NamespaceDataContentType, _fingerprint)
+		if _value, _error := _server.cdbReader.Get ([]byte (_key)); _error == nil {
+			if _value != nil {
+				_responseHeaders.Set ("Content-Type", string (_value))
+			}
+		} else {
+			_server.ServeError (_response, http.StatusInternalServerError, _error)
+			return
+		}
+	}
+	
+	if _server.debug {
+		log.Printf ("[dd] [b15f3cad]  serving for `%s`...\n", _path)
+	}
+	
+	_response.WriteHeader (http.StatusOK)
+	_response.Write (_data)
+}
+
+
+
+
+func (_server *server) ServeRedirect (_response http.ResponseWriter, _status uint, _urlRaw string) () {
+	var _url string
+	if _url_0, _error := url.Parse (_urlRaw); _error == nil {
+		_url = _url_0.String ()
+	} else {
+		_server.ServeError (_response, http.StatusInternalServerError, _error)
+		return
+	}
+	_response.Header () .Set ("Content-Type", "text/plain; charset=utf-8")
+	_response.Header () .Set ("Location", _url)
+	_response.WriteHeader (int (_status))
+	_response.Write ([]byte (fmt.Sprintf ("[%d] %s", _status, _url)))
+}
+
+
+func (_server *server) ServeError (_response http.ResponseWriter, _status uint, _error error) () {
+	_response.Header () .Set ("Content-Type", "text/plain; charset=utf-8")
+	_response.WriteHeader (int (_status))
+	_response.Write ([]byte (fmt.Sprintf ("[%d]", _status)))
+	LogError (_error, "")
+}
+
+
+
+
+func main () () {
+	Main (main_0)
+}
+
+
+func main_0 () (error) {
+	
+	
+	var _bind string
+	var _archive string
+	var _debug bool
+	
+	{
+		_flags := flag.NewFlagSet ("cdb-server", flag.ContinueOnError)
+		
+		_bind_0 := _flags.String ("bind", "", "<ip>:<port>")
+		_archive_0 := _flags.String ("archive", "", "<path>")
+		_debug_0 := _flags.Bool ("debug", false, "")
+		
+		FlagsParse (_flags, 0, 0)
+		
+		_bind = *_bind_0
+		_archive = *_archive_0
+		_debug = *_debug_0
+		
+		if _bind == "" {
+			AbortError (nil, "[eefe1a38]  expected bind address argument!")
+		}
+		if _archive == "" {
+			AbortError (nil, "[eefe1a38]  expected archive file argument!")
+		}
+	}
+	
+	
+	var _cdbReader *cdb.CDB
+	if _cdbReader_0, _error := cdb.Open (_archive); _error == nil {
+		_cdbReader = _cdbReader_0
+	} else {
+		AbortError (_error, "[85234ba0]  failed opening archive!")
+	}
+	
+	
+	_server := & server {
+			cdbReader : _cdbReader,
+			debug : _debug,
+		}
+	
+	if _error := http.ListenAndServe (_bind, _server); _error != nil {
+		AbortError (_error, "[44f45c67]  failed starting server!")
+	}
+	
+	return nil
+}
+

sources/lib/common.go

@@ -0,0 +1,90 @@
+
+
+package lib
+
+
+import "flag"
+import "fmt"
+import "log"
+import "os"
+import "regexp"
+
+
+
+
+func Main (_main func () (error)) () {
+	
+	log.SetFlags (0)
+	
+	if _error := _main (); _error == nil {
+		os.Exit (0)
+	} else {
+		AbortError (_error, "#")
+	}
+}
+
+
+
+
+func FlagsParse (_flags *flag.FlagSet, _argumentsMin uint, _argumentsMax uint) ([]string) {
+	
+	_arguments := os.Args[1:]
+	
+	if _error := _flags.Parse (_arguments); _error != nil {
+		AbortError (_error, fmt.Sprintf ("[8fae7a93]  failed parsing arguments:  `%v`!", _arguments))
+	}
+	
+	_flagsNArg := uint (_flags.NArg ())
+	if _argumentsMin == _argumentsMax {
+		if _flagsNArg != _argumentsMin {
+			AbortError (nil, fmt.Sprintf ("[c451f1f8]  expected exactly `%d` positional arguments!", _argumentsMin))
+		}
+	} else {
+		if _flagsNArg < _argumentsMin {
+			AbortError (nil, fmt.Sprintf ("[c451f1f8]  expected at least `%d` positional arguments!", _argumentsMin))
+		}
+		if _flagsNArg > _argumentsMax {
+			AbortError (nil, fmt.Sprintf ("[fa0a8c22]  expected at most `%d` positional arguments!", _argumentsMax))
+		}
+	}
+	
+	return _flags.Args ()
+}
+
+
+
+
+func LogError (_error error, _message string) () {
+	
+	if _message != "#" {
+		if (_message == "") && (_error != nil) {
+			_message = "[70d7e7c6]  unexpected error encountered!";
+		}
+		if _message != "" {
+			log.Printf ("[ee] %s\n", _message)
+		}
+	}
+	
+	if _error != nil {
+		_errorString := _error.Error ()
+		if _matches, _matchesError := regexp.MatchString (`^\[[0-9a-f]{8}\] [^\n]+$`, _errorString); _matchesError == nil {
+			if _matches {
+				log.Printf ("[ee] %s\n", _errorString)
+			} else {
+				log.Printf ("[ee] [c776ae31]  %q\n", _errorString)
+				log.Printf ("[ee] [ddd6baae]  %#v\n", _error)
+			}
+		} else {
+			log.Printf ("[ee] [609a0410]  %q\n", _errorString)
+			log.Printf ("[ee] [2ddce4bf]  %#v\n", _error)
+		}
+	}
+}
+
+
+func AbortError (_error error, _message string) () {
+	LogError (_error, _message)
+	log.Printf ("[!!] [89251d36]  aborting!\n")
+	os.Exit (1)
+}
+

sources/lib/namespaces.go

@@ -0,0 +1,12 @@
+
+
+package lib
+
+
+const NamespaceFilesContent = "files:content"
+const NamespaceFoldersContent = "folders:content"
+const NamespaceFoldersMetadata = "folders:metadata"
+
+const NamespaceDataContent = "data:content"
+const NamespaceDataContentType = "data:content-type"
+