[sources] Import initial version of archiver and server
This commit is contained in:
Ciprian Dorin Craciun
parent
60ebab55c8
commit
21e64cfc79
4 changed files with 586 additions and 0 deletions
294
sources/cmd/archiver.go
Normal file
294
sources/cmd/archiver.go
Normal file
|
|
@ -0,0 +1,294 @@
|
|||
|
||||
|
||||
package main
|
||||
|
||||
|
||||
import "crypto/sha256"
|
||||
import "encoding/hex"
|
||||
import "encoding/json"
|
||||
import "flag"
|
||||
import "fmt"
|
||||
import "io"
|
||||
import "io/ioutil"
|
||||
import "log"
|
||||
import "net/http"
|
||||
import "path/filepath"
|
||||
import "os"
|
||||
import "sort"
|
||||
|
||||
import "github.com/colinmarc/cdb"
|
||||
|
||||
import . "../lib"
|
||||
|
||||
|
||||
|
||||
|
||||
func archiveFile (_cdbWriter *cdb.Writer, _pathResolved string, _pathInArchive string, _name string, _stat os.FileInfo, _stored map[string]bool, _debug bool) (error) {
|
||||
|
||||
var _data []byte
|
||||
if _data_0, _error := ioutil.ReadFile (_pathResolved); _error == nil {
|
||||
_data = _data_0
|
||||
} else {
|
||||
return _error
|
||||
}
|
||||
|
||||
return archiveData (_cdbWriter, NamespaceFilesContent, _pathInArchive, _data, "", _stored, _debug)
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
func archiveFolder (_cdbWriter *cdb.Writer, _pathResolved string, _pathInArchive string, _names []string, _stats map[string]os.FileInfo, _stored map[string]bool, _debug bool) (error) {
|
||||
|
||||
type Entry struct {
|
||||
Name string `json:"name",omitempty`
|
||||
Type string `json:"type",omitempty`
|
||||
Size uint64 `json:"size",omitempty"`
|
||||
}
|
||||
|
||||
type Folder struct {
|
||||
Entries []Entry `json:"entries"`
|
||||
}
|
||||
|
||||
_entries := make ([]Entry, 0, len (_names))
|
||||
for _, _name := range _names {
|
||||
_entry := Entry {
|
||||
Name : _name,
|
||||
Type : "unknown",
|
||||
}
|
||||
_stat := _stats[_name]
|
||||
_statMode := _stat.Mode ()
|
||||
if _statMode.IsRegular () {
|
||||
_entry.Type = "file"
|
||||
_entry.Size = uint64 (_stat.Size ())
|
||||
} else if _statMode.IsDir () {
|
||||
_entry.Type = "folder"
|
||||
}
|
||||
_entries = append (_entries, _entry)
|
||||
}
|
||||
|
||||
_folder := Folder {
|
||||
Entries : _entries,
|
||||
}
|
||||
|
||||
if _data, _error := json.Marshal (&_folder); _error == nil {
|
||||
return archiveData (_cdbWriter, NamespaceFoldersMetadata, _pathInArchive, _data, "application/json; charset=utf-8", _stored, _debug)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
func archiveData (_cdbWriter *cdb.Writer, _namespace string, _pathInArchive string, _data []byte, _dataType string, _stored map[string]bool, _debug bool) (error) {
|
||||
|
||||
_fingerprintRaw := sha256.Sum256 (_data)
|
||||
_fingerprint := hex.EncodeToString (_fingerprintRaw[:])
|
||||
|
||||
_wasStored, _ := _stored[_fingerprint]
|
||||
|
||||
if ! _wasStored {
|
||||
_key := fmt.Sprintf ("%s:%s", NamespaceDataContent, _fingerprint)
|
||||
if _debug {
|
||||
log.Printf ("[ ] ++ %s", _key)
|
||||
}
|
||||
if _error := _cdbWriter.Put ([]byte (_key), _data); _error != nil {
|
||||
return _error
|
||||
}
|
||||
}
|
||||
|
||||
if ! _wasStored {
|
||||
_key := fmt.Sprintf ("%s:%s", NamespaceDataContentType, _fingerprint)
|
||||
if _dataType == "" {
|
||||
_dataType = http.DetectContentType (_data)
|
||||
}
|
||||
if _debug {
|
||||
log.Printf ("[ ] ++ %s %s", _key, _dataType)
|
||||
}
|
||||
if _error := _cdbWriter.Put ([]byte (_key), []byte (_dataType)); _error != nil {
|
||||
return _error
|
||||
}
|
||||
}
|
||||
|
||||
if _namespace != "" {
|
||||
_key := fmt.Sprintf ("%s:%s", _namespace, _pathInArchive)
|
||||
if _debug {
|
||||
log.Printf ("[ ] ++ %s", _key)
|
||||
}
|
||||
if _error := _cdbWriter.Put ([]byte (_key), []byte (_fingerprint)); _error != nil {
|
||||
return _error
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
func walkPath (_cdbWriter *cdb.Writer, _path string, _prefix string, _name string, _recursed map[string]uint, _stored map[string]bool, _debug bool) (error) {
|
||||
|
||||
if _recursed == nil {
|
||||
_recursed = make (map[string]uint, 128)
|
||||
}
|
||||
if _stored == nil {
|
||||
_stored = make (map[string]bool, 16 * 1024)
|
||||
}
|
||||
|
||||
_pathInArchive := filepath.Join (_prefix, _name)
|
||||
|
||||
var _stat os.FileInfo
|
||||
if _stat_0, _error := os.Lstat (_path); _error == nil {
|
||||
_stat = _stat_0
|
||||
} else {
|
||||
return _error
|
||||
}
|
||||
|
||||
_isSymlink := false
|
||||
if (_stat.Mode () & os.ModeSymlink) != 0 {
|
||||
_isSymlink = true
|
||||
if _stat_0, _error := os.Stat (_path); _error == nil {
|
||||
_stat = _stat_0
|
||||
} else {
|
||||
return _error
|
||||
}
|
||||
}
|
||||
|
||||
var _pathResolved string
|
||||
if _isSymlink {
|
||||
if _resolved, _error := filepath.EvalSymlinks (_path); _error == nil {
|
||||
_pathResolved = _resolved
|
||||
} else {
|
||||
return _error
|
||||
}
|
||||
} else {
|
||||
_pathResolved = _path
|
||||
}
|
||||
|
||||
if _isSymlink && _debug {
|
||||
log.Printf ("[ ] ~~ %s -> %s\n", _pathInArchive, _pathResolved)
|
||||
}
|
||||
|
||||
if _stat.Mode () .IsRegular () {
|
||||
|
||||
if _debug {
|
||||
log.Printf ("[ ] ## %s\n", _pathInArchive)
|
||||
}
|
||||
return archiveFile (_cdbWriter, _pathResolved, _pathInArchive, _name, _stat, _stored, _debug)
|
||||
|
||||
} else if _stat.Mode () .IsDir () {
|
||||
|
||||
_wasRecursed, _ := _recursed[_pathResolved]
|
||||
if _wasRecursed > 0 {
|
||||
log.Printf ("[ww] [2e1744c9] detected directory loop for `%s` resolving to `%s`; ignoring!\n", _path, _pathResolved)
|
||||
return nil
|
||||
}
|
||||
_recursed[_pathResolved] = _wasRecursed + 1
|
||||
|
||||
if _debug {
|
||||
log.Printf ("[ ] >> %s\n", _pathInArchive)
|
||||
}
|
||||
|
||||
_names := make ([]string, 0, 16)
|
||||
_stats := make (map[string]os.FileInfo, 16)
|
||||
|
||||
if _stream, _error := os.Open (_path); _error == nil {
|
||||
defer _stream.Close ()
|
||||
_prefix = filepath.Join (_prefix, _name)
|
||||
_loop : for {
|
||||
switch _buffer, _error := _stream.Readdir (128); _error {
|
||||
case nil :
|
||||
for _, _stat := range _buffer {
|
||||
_name := _stat.Name ()
|
||||
_names = append (_names, _name)
|
||||
_stats[_name] = _stat
|
||||
if _error := walkPath (_cdbWriter, filepath.Join (_path, _name), _prefix, _name, _recursed, _stored, _debug); _error != nil {
|
||||
return _error
|
||||
}
|
||||
}
|
||||
case io.EOF :
|
||||
break _loop
|
||||
default :
|
||||
return _error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sort.Strings (_names)
|
||||
|
||||
if _debug {
|
||||
log.Printf ("[ ] << %s\n", _pathInArchive)
|
||||
}
|
||||
|
||||
if _debug {
|
||||
log.Printf ("[ ] <> %s\n", _pathInArchive)
|
||||
if _error := archiveFolder (_cdbWriter, _pathResolved, _pathInArchive, _names, _stats, _stored, _debug); _error != nil {
|
||||
return _error
|
||||
}
|
||||
}
|
||||
|
||||
_recursed[_pathResolved] = _wasRecursed
|
||||
return nil
|
||||
|
||||
} else {
|
||||
return fmt.Errorf ("[d9b836d7] unexpected file type for `%s`: `%s`!", _path, _stat.Mode ())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
func main () () {
|
||||
Main (main_0)
|
||||
}
|
||||
|
||||
|
||||
func main_0 () (error) {
|
||||
|
||||
|
||||
var _sourcesFolder string
|
||||
var _archiveFile string
|
||||
var _debug bool
|
||||
|
||||
{
|
||||
_flags := flag.NewFlagSet ("cdb-archiver", flag.ContinueOnError)
|
||||
|
||||
_sourcesFolder_0 := _flags.String ("sources", "", "<path>")
|
||||
_archiveFile_0 := _flags.String ("archive", "", "<path>")
|
||||
_debug_0 := _flags.Bool ("debug", false, "")
|
||||
|
||||
FlagsParse (_flags, 0, 0)
|
||||
|
||||
_sourcesFolder = *_sourcesFolder_0
|
||||
_archiveFile = *_archiveFile_0
|
||||
_debug = *_debug_0
|
||||
|
||||
if _sourcesFolder == "" {
|
||||
AbortError (nil, "[515ee462] expected sources folder argument!")
|
||||
}
|
||||
if _archiveFile == "" {
|
||||
AbortError (nil, "[5e8da985] expected archive file argument!")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
var _cdbWriter *cdb.Writer
|
||||
if _cdbWriter_0, _error := cdb.Create (_archiveFile); _error == nil {
|
||||
_cdbWriter = _cdbWriter_0
|
||||
} else {
|
||||
AbortError (_error, "[85234ba0] failed creating archive (while opening)!")
|
||||
}
|
||||
|
||||
if _error := walkPath (_cdbWriter, _sourcesFolder, "/", "", nil, nil, _debug); _error != nil {
|
||||
AbortError (_error, "[b6a19ef4] failed walking folder!")
|
||||
}
|
||||
|
||||
if _error := _cdbWriter.Close (); _error != nil {
|
||||
AbortError (_error, "[bbfb8478] failed creating archive (while closing)!")
|
||||
}
|
||||
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
190
sources/cmd/server.go
Normal file
190
sources/cmd/server.go
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
|
||||
|
||||
package main
|
||||
|
||||
|
||||
import "flag"
|
||||
import "fmt"
|
||||
import "log"
|
||||
import "net/http"
|
||||
import "net/url"
|
||||
|
||||
import "github.com/colinmarc/cdb"
|
||||
|
||||
import . "../lib"
|
||||
|
||||
|
||||
|
||||
type server struct {
|
||||
cdbReader *cdb.CDB
|
||||
debug bool
|
||||
}
|
||||
|
||||
|
||||
func (_server *server) ServeHTTP (_response http.ResponseWriter, _request *http.Request) () {
|
||||
|
||||
_responseHeaders := _response.Header ()
|
||||
|
||||
_responseHeaders.Set ("Content-Security-Policy", "upgrade-insecure-requests")
|
||||
_responseHeaders.Set ("Referrer-Policy", "strict-origin-when-cross-origin")
|
||||
_responseHeaders.Set ("X-Frame-Options", "SAMEORIGIN")
|
||||
_responseHeaders.Set ("X-Content-Type-Options", "nosniff")
|
||||
_responseHeaders.Set ("X-XSS-Protection", "1; mode=block")
|
||||
|
||||
_method := _request.Method
|
||||
_path := _request.URL.Path
|
||||
|
||||
if _method != http.MethodGet {
|
||||
log.Printf ("[ww] [bce7a75b] invalid method `%s` for `%s`!", _method, _path)
|
||||
_server.ServeError (_response, http.StatusMethodNotAllowed, nil)
|
||||
return
|
||||
}
|
||||
|
||||
var _fingerprint string
|
||||
{
|
||||
_path_0 := _path
|
||||
if (_path != "/") && (_path[len (_path) - 1] == '/') {
|
||||
_path_0 = _path[: len (_path) - 1]
|
||||
}
|
||||
for _, _namespace := range []string {NamespaceFilesContent, NamespaceFoldersContent, NamespaceFoldersMetadata} {
|
||||
_key := fmt.Sprintf ("%s:%s", _namespace, _path_0)
|
||||
if _value, _error := _server.cdbReader.Get ([]byte (_key)); _error == nil {
|
||||
if _value != nil {
|
||||
_fingerprint = string (_value)
|
||||
if ((_namespace == NamespaceFoldersContent) || (_namespace == NamespaceFoldersMetadata)) && (_path == _path_0) {
|
||||
_server.ServeRedirect (_response, http.StatusTemporaryRedirect, _path + "/")
|
||||
return
|
||||
}
|
||||
break
|
||||
}
|
||||
} else {
|
||||
_server.ServeError (_response, http.StatusInternalServerError, _error)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
if _fingerprint == "" {
|
||||
log.Printf ("[ww] [7416f61d] not found for `%s`!", _path)
|
||||
_server.ServeError (_response, http.StatusNotFound, nil)
|
||||
return
|
||||
}
|
||||
|
||||
var _data []byte
|
||||
{
|
||||
_key := fmt.Sprintf ("%s:%s", NamespaceDataContent, _fingerprint)
|
||||
if _value, _error := _server.cdbReader.Get ([]byte (_key)); _error == nil {
|
||||
if _value != nil {
|
||||
_data = _value
|
||||
}
|
||||
} else {
|
||||
_server.ServeError (_response, http.StatusInternalServerError, _error)
|
||||
return
|
||||
}
|
||||
}
|
||||
if _data == nil {
|
||||
_server.ServeError (_response, http.StatusNotFound, nil)
|
||||
return
|
||||
}
|
||||
|
||||
{
|
||||
_key := fmt.Sprintf ("%s:%s", NamespaceDataContentType, _fingerprint)
|
||||
if _value, _error := _server.cdbReader.Get ([]byte (_key)); _error == nil {
|
||||
if _value != nil {
|
||||
_responseHeaders.Set ("Content-Type", string (_value))
|
||||
}
|
||||
} else {
|
||||
_server.ServeError (_response, http.StatusInternalServerError, _error)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if _server.debug {
|
||||
log.Printf ("[dd] [b15f3cad] serving for `%s`...\n", _path)
|
||||
}
|
||||
|
||||
_response.WriteHeader (http.StatusOK)
|
||||
_response.Write (_data)
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
func (_server *server) ServeRedirect (_response http.ResponseWriter, _status uint, _urlRaw string) () {
|
||||
var _url string
|
||||
if _url_0, _error := url.Parse (_urlRaw); _error == nil {
|
||||
_url = _url_0.String ()
|
||||
} else {
|
||||
_server.ServeError (_response, http.StatusInternalServerError, _error)
|
||||
return
|
||||
}
|
||||
_response.Header () .Set ("Content-Type", "text/plain; charset=utf-8")
|
||||
_response.Header () .Set ("Location", _url)
|
||||
_response.WriteHeader (int (_status))
|
||||
_response.Write ([]byte (fmt.Sprintf ("[%d] %s", _status, _url)))
|
||||
}
|
||||
|
||||
|
||||
func (_server *server) ServeError (_response http.ResponseWriter, _status uint, _error error) () {
|
||||
_response.Header () .Set ("Content-Type", "text/plain; charset=utf-8")
|
||||
_response.WriteHeader (int (_status))
|
||||
_response.Write ([]byte (fmt.Sprintf ("[%d]", _status)))
|
||||
LogError (_error, "")
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
func main () () {
|
||||
Main (main_0)
|
||||
}
|
||||
|
||||
|
||||
func main_0 () (error) {
|
||||
|
||||
|
||||
var _bind string
|
||||
var _archive string
|
||||
var _debug bool
|
||||
|
||||
{
|
||||
_flags := flag.NewFlagSet ("cdb-server", flag.ContinueOnError)
|
||||
|
||||
_bind_0 := _flags.String ("bind", "", "<ip>:<port>")
|
||||
_archive_0 := _flags.String ("archive", "", "<path>")
|
||||
_debug_0 := _flags.Bool ("debug", false, "")
|
||||
|
||||
FlagsParse (_flags, 0, 0)
|
||||
|
||||
_bind = *_bind_0
|
||||
_archive = *_archive_0
|
||||
_debug = *_debug_0
|
||||
|
||||
if _bind == "" {
|
||||
AbortError (nil, "[eefe1a38] expected bind address argument!")
|
||||
}
|
||||
if _archive == "" {
|
||||
AbortError (nil, "[eefe1a38] expected archive file argument!")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
var _cdbReader *cdb.CDB
|
||||
if _cdbReader_0, _error := cdb.Open (_archive); _error == nil {
|
||||
_cdbReader = _cdbReader_0
|
||||
} else {
|
||||
AbortError (_error, "[85234ba0] failed opening archive!")
|
||||
}
|
||||
|
||||
|
||||
_server := & server {
|
||||
cdbReader : _cdbReader,
|
||||
debug : _debug,
|
||||
}
|
||||
|
||||
if _error := http.ListenAndServe (_bind, _server); _error != nil {
|
||||
AbortError (_error, "[44f45c67] failed starting server!")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
90
sources/lib/common.go
Normal file
90
sources/lib/common.go
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
|
||||
|
||||
package lib
|
||||
|
||||
|
||||
import "flag"
|
||||
import "fmt"
|
||||
import "log"
|
||||
import "os"
|
||||
import "regexp"
|
||||
|
||||
|
||||
|
||||
|
||||
func Main (_main func () (error)) () {
|
||||
|
||||
log.SetFlags (0)
|
||||
|
||||
if _error := _main (); _error == nil {
|
||||
os.Exit (0)
|
||||
} else {
|
||||
AbortError (_error, "#")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
func FlagsParse (_flags *flag.FlagSet, _argumentsMin uint, _argumentsMax uint) ([]string) {
|
||||
|
||||
_arguments := os.Args[1:]
|
||||
|
||||
if _error := _flags.Parse (_arguments); _error != nil {
|
||||
AbortError (_error, fmt.Sprintf ("[8fae7a93] failed parsing arguments: `%v`!", _arguments))
|
||||
}
|
||||
|
||||
_flagsNArg := uint (_flags.NArg ())
|
||||
if _argumentsMin == _argumentsMax {
|
||||
if _flagsNArg != _argumentsMin {
|
||||
AbortError (nil, fmt.Sprintf ("[c451f1f8] expected exactly `%d` positional arguments!", _argumentsMin))
|
||||
}
|
||||
} else {
|
||||
if _flagsNArg < _argumentsMin {
|
||||
AbortError (nil, fmt.Sprintf ("[c451f1f8] expected at least `%d` positional arguments!", _argumentsMin))
|
||||
}
|
||||
if _flagsNArg > _argumentsMax {
|
||||
AbortError (nil, fmt.Sprintf ("[fa0a8c22] expected at most `%d` positional arguments!", _argumentsMax))
|
||||
}
|
||||
}
|
||||
|
||||
return _flags.Args ()
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
func LogError (_error error, _message string) () {
|
||||
|
||||
if _message != "#" {
|
||||
if (_message == "") && (_error != nil) {
|
||||
_message = "[70d7e7c6] unexpected error encountered!";
|
||||
}
|
||||
if _message != "" {
|
||||
log.Printf ("[ee] %s\n", _message)
|
||||
}
|
||||
}
|
||||
|
||||
if _error != nil {
|
||||
_errorString := _error.Error ()
|
||||
if _matches, _matchesError := regexp.MatchString (`^\[[0-9a-f]{8}\] [^\n]+$`, _errorString); _matchesError == nil {
|
||||
if _matches {
|
||||
log.Printf ("[ee] %s\n", _errorString)
|
||||
} else {
|
||||
log.Printf ("[ee] [c776ae31] %q\n", _errorString)
|
||||
log.Printf ("[ee] [ddd6baae] %#v\n", _error)
|
||||
}
|
||||
} else {
|
||||
log.Printf ("[ee] [609a0410] %q\n", _errorString)
|
||||
log.Printf ("[ee] [2ddce4bf] %#v\n", _error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
func AbortError (_error error, _message string) () {
|
||||
LogError (_error, _message)
|
||||
log.Printf ("[!!] [89251d36] aborting!\n")
|
||||
os.Exit (1)
|
||||
}
|
||||
|
||||
12
sources/lib/namespaces.go
Normal file
12
sources/lib/namespaces.go
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
|
||||
|
||||
package lib
|
||||
|
||||
|
||||
const NamespaceFilesContent = "files:content"
|
||||
const NamespaceFoldersContent = "folders:content"
|
||||
const NamespaceFoldersMetadata = "folders:metadata"
|
||||
|
||||
const NamespaceDataContent = "data:content"
|
||||
const NamespaceDataContentType = "data:content-type"
|
||||
|
||||
Loading…
Reference in a new issue