[archiver] Add with/without slash redirects as part of the archive.

examples/hello-world.cdb-dump

@@ -1,7 +1,7 @@
 
 +6,14
 schema
-kawipiko-2021c
+kawipiko-2021d
 
 +3,16
 f:/
@@ -32,6 +32,66 @@ d0000004
 f:/*
 m0000001d0000002
 
++14,16
+r:/redirect-01
+m0000005d0000006
+
++8,120
+m0000005
+h0000001Zaad301h0000018H0000001h000002cZaco472276124f76fe598ac3b5b1dcc415ba68ad5b547e7854ba95bba2426cf9ff30h000003fZaab/
+
++8,0
+d0000006
+
+
++14,16
+r:/redirect-02
+m0000007d0000006
+
++8,142
+m0000007
+h0000001Zaad301h0000018H0000001h000002cZaco738710189bc00ff901ab60eecc623301aaf38f8c30da516c23352dd3bbadb599h000003fZaaxhttps://www.google.com/
+
++21,16
+r:://host/redirect-03
+m0000008d0000006
+
++8,128
+m0000008
+h0000001Zaad301h0000018H0000001h000002cZaco2cf7a3041ba8c61ed7255cd4b40b73bbc67f460a7ace01e545a4692d2755381eh000003fZaaj/hello-03
+
++18,16
+r:://*/redirect-04
+m0000009d0000006
+
++8,128
+m0000009
+h0000001Zaad301h0000018H0000001h000002cZaco4747b146ca4ba0c1c225aa5e2a76fd8aa2b0ee6ea3814fbf1e73d90bbfcb6c38h000003fZaaj/hello-04
+
++14,16
+r:/redirect-05
+m000000ad0000006
+
++8,128
+m000000a
+h0000001Zaad301h0000018H0000001h000002cZaco014b8c78de8c2f0104abc363ffd9b608c548748eee84fc5a6eb1abc4094e585bh000003fZaaj/hello-05
+
++16,16
+r:/redirect-06/*
+m000000bd0000006
+
++8,128
+m000000b
+h0000001Zaad301h0000018H0000001h000002cZaco7a4ba64159e3148e07347c1b31e322eabf9bb1b4d2d7d13f0d4d10f06b5e6340h000003fZaaj/hello-06
+
++16,16
+r:/redirect-07/*
+m000000cd0000006
+
++8,128
+m000000c
+h0000001Zaad301h0000018H0000001h000002cZaco4d4c8494af06c18d7c0ff210c832b6f76596631dda1ab4cef4aeae14ed2cdc82h000003fZaaj/hello-07
+
 +8,16
 f:/index
 m0000001d0000002
@@ -40,60 +100,216 @@ m0000001d0000002
 f:/index.txt
 m0000001d0000002
 
-+8,16
++9,16
-l:/tests
+l:/tests/
-m0000005d0000006
+m000000dd000000e
 
 +8,140
-m0000005
+m000000d
-h0000001H000000dh0000018H0000001h000001dH0000003h0000026H0000014h000002cZaco9bdc558765f25a06d86e90f2036e2bb3d743c6c2aba4038bccd1ddd90fdbb074
+h0000001H000000dh0000018H0000001h000001dH0000003h0000026H0000014h000002cZacoa4e718fdd63f51c8688b106aa8ba26fa435972af1d2ec5a8eda8fd0980afc737
 
-+8,89
++8,126
-d0000006
+d000000e
-{"entries":[{"name":"status-200","type":"folder"},{"name":"status-404","type":"folder"}]}
+{"entries":[{"name":"redirects","type":"folder"},{"name":"status-200","type":"folder"},{"name":"status-404","type":"folder"}]}
 
 +19,16
-l:/tests/status-200
+l:/tests/redirects/
-m0000007d0000008
+m000000fd0000010
 
 +8,140
-m0000007
+m000000f
 h0000001H000000dh0000018H0000001h000001dH0000003h0000026H0000014h000002cZaco6e46dd10defc9b56c29a6ec56b508c21f54c08192194e4df25bf36f0c9c3c279
 
 +8,2
-d0000008
+d0000010
 {}
 
++21,16
+r:/tests/redirects/91
+m0000011d0000006
+
++8,128
+m0000011
+h0000001Zaad307h0000018H0000001h000002cZaco58dd91c8086c180b8d4b828554618e0bf7c546b99d14630240e20b2b9296e913h000003fZaaj/hello-91
+
++22,16
+r:/tests/redirects/92/
+m0000012d0000006
+
++8,128
+m0000012
+h0000001Zaad307h0000018H0000001h000002cZaco4c79e594135b8124f473c37a8c98280dedb49776d879c0defccbae1725252b7ah000003fZaaj/hello-92
+
++23,16
+r:/tests/redirects/93/*
+m0000013d0000006
+
++8,128
+m0000013
+h0000001Zaad307h0000018H0000001h000002cZaco9879530863f118f9329952c85763c52cb649a3b808079eb2969ab651d8d6c2f8h000003fZaaj/hello-93
+
++20,16
+l:/tests/status-200/
+m000000fd0000010
+
 +21,16
 f:/tests/status-200/*
-m0000009d000000a
+m0000014d0000015
 
 +8,140
-m0000009
+m0000014
 h0000001H000000dh0000018H0000001h000001dH0000003h0000026H000000fh000002cZaco289220028fc1b56bc1f8afb2b26c9c87b40898409a5c519defc6c15bd4242bcb
 
 +8,17
-d000000a
+d0000015
 Anything is 200!
 
 
-+19,16
++20,16
-l:/tests/status-404
+l:/tests/status-404/
-m0000007d0000008
+m000000fd0000010
 
 +21,16
 f:/tests/status-404/*
-m000000bd000000c
+m0000016d0000017
 
 +8,140
-m000000b
+m0000016
 h0000001H000000eh0000018H0000001h000001dH0000003h0000026H000000fh000002cZaco09c18884511967b78751fd4614620efa112760d039edecf044a42233701cbb7d
 
 +8,17
-d000000c
+d0000017
 Anything is 404!
 
 
++9,16
+r:/index/
+m0000018d0000006
+
++8,49
+m0000018
+h0000001Zaad308h0000018H0000001h000003fZaag/index
+
++13,16
+r:/index.txt/
+m0000019d0000006
+
++8,53
+m0000019
+h0000001Zaad308h0000018H0000001h000003fZaak/index.txt
+
++15,16
+r:/redirect-01/
+m000001ad0000006
+
++8,55
+m000001a
+h0000001Zaad308h0000018H0000001h000003fZaam/redirect-01
+
++15,16
+r:/redirect-02/
+m000001bd0000006
+
++8,55
+m000001b
+h0000001Zaad308h0000018H0000001h000003fZaam/redirect-02
+
++15,16
+r:/redirect-05/
+m000001cd0000006
+
++8,55
+m000001c
+h0000001Zaad308h0000018H0000001h000003fZaam/redirect-05
+
++14,16
+r:/redirect-06
+m000001dd0000006
+
++8,56
+m000001d
+h0000001Zaad308h0000018H0000001h000003fZaan/redirect-06/
+
++14,16
+r:/redirect-07
+m000001ed0000006
+
++8,56
+m000001e
+h0000001Zaad308h0000018H0000001h000003fZaan/redirect-07/
+
++8,16
+r:/tests
+m000001fd0000006
+
++8,50
+m000001f
+h0000001Zaad308h0000018H0000001h000003fZaah/tests/
+
++18,16
+r:/tests/redirects
+m0000020d0000006
+
++8,60
+m0000020
+h0000001Zaad308h0000018H0000001h000003fZaar/tests/redirects/
+
++22,16
+r:/tests/redirects/91/
+m0000021d0000006
+
++8,62
+m0000021
+h0000001Zaad308h0000018H0000001h000003fZaat/tests/redirects/91
+
++21,16
+r:/tests/redirects/92
+m0000022d0000006
+
++8,63
+m0000022
+h0000001Zaad308h0000018H0000001h000003fZaau/tests/redirects/92/
+
++21,16
+r:/tests/redirects/93
+m0000023d0000006
+
++8,63
+m0000023
+h0000001Zaad308h0000018H0000001h000003fZaau/tests/redirects/93/
+
++19,16
+r:/tests/status-200
+m0000024d0000006
+
++8,61
+m0000024
+h0000001Zaad308h0000018H0000001h000003fZaas/tests/status-200/
+
++19,16
+r:/tests/status-404
+m0000025d0000006
+
++8,61
+m0000025
+h0000001Zaad308h0000018H0000001h000003fZaas/tests/status-404/
+
++19,16
+r:://*/redirect-04/
+m0000026d0000006
+
++8,55
+m0000026
+h0000001Zaad308h0000018H0000001h000003fZaam/redirect-04
+
++22,16
+r:://host/redirect-03/
+m0000027d0000006
+
++8,55
+m0000027
+h0000001Zaad308h0000018H0000001h000003fZaam/redirect-03
+
 +8,63
 F0000001
 /
@@ -104,11 +320,42 @@ F0000001
 /tests/status-404/*
 
 
-+8,45
++8,66
 L0000001
 /
+/tests/
+/tests/redirects/
+/tests/status-200/
+/tests/status-404/
+
+
++8,417
+R0000001
+/index.txt/
+/index/
+/redirect-01
+/redirect-01/
+/redirect-02
+/redirect-02/
+/redirect-05
+/redirect-05/
+/redirect-06
+/redirect-06/*
+/redirect-07
+/redirect-07/*
 /tests
+/tests/redirects
+/tests/redirects/91
+/tests/redirects/91/
+/tests/redirects/92
+/tests/redirects/92/
+/tests/redirects/93
+/tests/redirects/93/*
 /tests/status-200
 /tests/status-404
+://*/redirect-04
+://*/redirect-04/
+://host/redirect-03
+://host/redirect-03/
 
 

examples/hello-world/_redirects.txt

@@ -0,0 +1,12 @@
+
+/redirect-01 / 301
+/redirect-02 https://www.google.com/ 301
+
+://host/redirect-03 /hello-03 301
+://*/redirect-04 /hello-04 301
+
+./redirect-05 /hello-05 301
+
+/redirect-06/* /hello-06 301
+./redirect-07/* /hello-07 301
+

examples/hello-world/tests/redirects/_redirects.txt

@@ -0,0 +1,5 @@
+
+./91    /hello-91 307
+./92/   /hello-92 307
+./93/*  /hello-93 307
+

scripts/examples.z-run

@@ -46,6 +46,8 @@
 			--processes 1 \
 			--threads 1 \
 			\
+			--security-headers-disable \
+			\
 			--debug \
 			\
 			"${@}" \
@@ -88,7 +90,7 @@
 			--compress-cache ./.outputs/examples/huge-compress.cache \
 			--sources-cache ./.outputs/examples/huge-sources.cache \
 			--exclude-strip \
-			--exclude-file-listing \
+			--exclude-paths-index \
 			--progress \
 			"${@}" \
 	#
@@ -101,6 +103,7 @@
 			--archive-mmap \
 			--processes 1 \
 			--threads 1 \
+			--security-headers-disable \
 			--debug \
 			"${@}" \
 	#

sources/cmd/archiver/archiver.go

@@ -59,6 +59,7 @@ type context struct {
 	dataCompressedSize int
 	includeIndex bool
 	includeStripped bool
+	includeSlashRedirects bool
 	includeCache bool
 	includeEtag bool
 	includeFolderListing bool
@@ -1281,6 +1282,7 @@ func main_0 () (error) {
 	var _compressCache string
 	var _includeIndex bool
 	var _includeStripped bool
+	var _includeSlashRedirects bool
 	var _includeCache bool
 	var _includeEtag bool
 	var _includeFolderListing bool
@@ -1303,6 +1305,7 @@ func main_0 () (error) {
 		_compressCache_0 := _flags.String ("compress-cache", "", "")
 		_excludeIndex_0 := _flags.Bool ("exclude-index", false, "")
 		_excludeStripped_0 := _flags.Bool ("exclude-strip", false, "")
+		_excludeSlashRedirects_0 := _flags.Bool ("exclude-slash-redirects", false, "")
 		_excludeCache_0 := _flags.Bool ("exclude-cache", false, "")
 		_includeEtag_0 := _flags.Bool ("include-etag", false, "")
 		_includeFolderListing_0 := _flags.Bool ("include-folder-listing", false, "")
@@ -1320,6 +1323,7 @@ func main_0 () (error) {
 		_compressCache = *_compressCache_0
 		_includeIndex = ! *_excludeIndex_0
 		_includeStripped = ! *_excludeStripped_0
+		_includeSlashRedirects = ! *_excludeSlashRedirects_0
 		_includeCache = ! *_excludeCache_0
 		_includeEtag = *_includeEtag_0
 		_includeFolderListing = *_includeFolderListing_0
@@ -1390,6 +1394,7 @@ func main_0 () (error) {
 			sourcesCache : _sourcesCacheDb,
 			includeIndex : _includeIndex,
 			includeStripped : _includeStripped,
+			includeSlashRedirects : _includeSlashRedirects,
 			includeCache : _includeCache,
 			includeEtag : _includeEtag,
 			includeFolderListing : _includeFolderListing,
@@ -1411,6 +1416,97 @@ func main_0 () (error) {
 		AbortError (_error, "[b6a19ef4]  failed walking folder!")
 	}
 	
+	sort.Strings (_context.storedFilePaths)
+	sort.Strings (_context.storedFolderPaths)
+	sort.Strings (_context.storedRedirectPaths)
+	
+	if _includeSlashRedirects {
+		_paths := make ([]string, 0, 16 * 1024)
+		_paths = append (_paths, _context.storedFilePaths ...)
+		_paths = append (_paths, _context.storedFolderPaths ...)
+		_paths = append (_paths, _context.storedRedirectPaths ...)
+		sort.Strings (_paths)
+		_pathsExisting := make (map[string]bool, len (_paths))
+		_pathsProcessed := make (map[string]bool, len (_paths))
+		for _, _path := range _paths {
+			_pathsExisting[_path] = true
+		}
+		for _, _path := range _paths {
+			if _processed, _ := _pathsProcessed[_path]; !_processed {
+				_pathsProcessed[_path] = true
+			} else {
+				continue
+			}
+			_pathHost := ""
+			if strings.HasPrefix (_path, "://") {
+				_path = _path[3:]
+				_slashOffset := strings.Index (_path, "/")
+				_pathHost = "://" + _path[: _slashOffset]
+				_path = _path[_slashOffset :]
+			}
+			_alternate := ""
+			_target := ""
+			if _alternate == "" {
+				for _, _suffix := range []string { "/", "/*" } {
+					if _path == _suffix {
+						continue
+					}
+					if ! strings.HasSuffix (_path, _suffix) {
+						continue
+					}
+					_alternate_0 := strings.TrimSuffix (_path, _suffix)
+					_alternateQualified_0 := _pathHost + _alternate_0
+					if _exists, _ := _pathsExisting[_alternateQualified_0]; !_exists {
+						_alternate = _alternate_0
+						_target = _alternate + "/"
+						break
+					}
+				}
+			}
+			if _alternate == "" {
+				for _, _suffix := range []string { "/" } {
+					if strings.HasSuffix (_path, "/*") {
+						continue
+					}
+					if strings.HasSuffix (_path, _suffix) {
+						continue
+					}
+					_alternate_0 := _path + _suffix
+					_alternateQualified_0 := _pathHost + _alternate_0
+					if _exists, _ := _pathsExisting[_alternateQualified_0]; !_exists {
+						_alternate = _alternate_0
+						_target = _path
+						break
+					}
+				}
+			}
+			if _alternate == "" {
+				continue
+			}
+			_alternateQualified := _pathHost + _alternate
+			_pathsExisting[_alternateQualified] = true
+			if _context.debug {
+				log.Printf ("[dd] [f6970144]  alternate    -- `%s` -> `%s`\n", _alternateQualified, _target)
+			}
+			{
+				_dataMeta := map[string]string {
+						"!Status" : fmt.Sprintf ("%d", 308),
+						"Location" : _target,
+					}
+				if _context.includeCache {
+					_dataMeta["Cache-Control"] = "public, immutable, max-age=3600"
+				}
+				if _, _, _error := archiveReferenceAndDataWithMeta (_context, NamespaceRedirectsContent, _alternateQualified, []byte (""), _dataMeta); _error != nil {
+					AbortError (_error, "[27a5dabe]  failed writing archive!")
+				}
+			}
+		}
+	}
+	
+	sort.Strings (_context.storedFilePaths)
+	sort.Strings (_context.storedFolderPaths)
+	sort.Strings (_context.storedRedirectPaths)
+	
 	if _includePathsIndex {
 		_buffer := make ([]byte, 0, 1024 * 1024)
 		for _, _path := range _context.storedFilePaths {