Nulo/kawipiko
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

[archiver] Add support for compressing with zopfli.

Browse source
This commit is contained in:
Ciprian Dorin Craciun 2021-11-17 19:50:30 +02:00
parent 4f8d45d2b4
commit a53d4be6e7
5 changed files with 44 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
documentation/readme.rst
View file

@ -15,7 +15,7 @@ However "simple" doesn't imply "dumb" or "limited", instead it implies "efficien
As such ``kawipiko`` basically supports only ``GET`` (and ``HEAD``) requests and does not provide features like dynamic content, authentication, reverse proxying, etc.
However, ``kawipiko`` does provide something unique, that no other HTTP server offers: the static website content is served from a CDB_ database with almost zero latency.
Moreover, the static website content can be compressed (with either ``gzip`` or ``brotli``) ahead of time, thus reducing not only CPU but also bandwidth and latency.
Moreover, the static website content can be compressed (with either ``gzip``, ``zopfli` or ``brotli``) ahead of time, thus reducing not only CPU but also bandwidth and latency.
CDB_ databases are binary files that provide efficient read-only key-value lookup tables, initially used in some DNS and SMTP servers, mainly for their low overhead lookup operations, zero locking in multi-threaded / multi-process scenarios, and "atomic" multi-record updates.
This also makes them suitable for low-latency static website content serving over HTTP, which this project provides.
@ -262,7 +262,7 @@ Flags
--sources <path>
--archive <path>
--compress <gzip | brotli | identity>
--compress <gzip | zopfli | brotli | identity>
--exclude-index
--exclude-strip
@ -380,6 +380,15 @@ Examples
--debug \
#
* create the CDB archive (with ``zopfli`` compression): ::
kawipiko-archiver \
--archive ./python-3.7.3-docs-html-zopfli.cdb \
--sources ./python-3.7.3-docs-html \
--compress zopfli \
--debug \
#
* create the CDB archive (with ``brotli`` compression): ::
kawipiko-archiver \
@ -406,6 +415,7 @@ Examples
\
./python-3.7.3-docs-html-nozip.cdb \
./python-3.7.3-docs-html-gzip.cdb \
./python-3.7.3-docs-html-zopfli.cdb \
./python-3.7.3-docs-html-brotli.cdb \
\
./python-3.7.3-docs-html \
@ -414,6 +424,7 @@ Examples
45M ./python-3.7.3-docs-html-nozip.cdb
9.7M ./python-3.7.3-docs-html-gzip.cdb
??? ./python-3.7.3-docs-html-zopfli.cdb
7.9M ./python-3.7.3-docs-html-brotli.cdb
46M ./python-3.7.3-docs-html

2
sources/cmd/archiver/archiver.go
View file

@ -665,7 +665,7 @@ func main_0 () (error) {
--sources <path>
--archive <path>
--compress <gzip | brotli | identity>
--compress <gzip | zopfli | brotli | identity>
--exclude-index
--exclude-strip

1
sources/go.mod
View file

@ -6,6 +6,7 @@ require (
github.com/Pallinder/go-randomdata v1.2.0 // indirect
github.com/colinmarc/cdb v0.0.0-00000000000000-000000000000
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/foobaz/go-zopfli v0.0.0-20140122214029-7432051485e2 // indirect
github.com/google/brotli v1.0.7
github.com/klauspost/compress v1.7.6 // indirect
github.com/klauspost/cpuid v1.2.1 // indirect

2
sources/go.sum
View file

@ -8,6 +8,8 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/foobaz/go-zopfli v0.0.0-20140122214029-7432051485e2 h1:VA6jElpcJ+wkwEBufbnVkSBCA2TEnxdRppjRT5Kvh0A=
github.com/foobaz/go-zopfli v0.0.0-20140122214029-7432051485e2/go.mod h1:Yi95+RbwKz7uGndSuUhoq7LJKh8qH8DT9fnL4ewU30k=
github.com/google/brotli v1.0.7 h1:fxwwohNEPaVS6qvtnjwgzRR62Upa70pkw0f9qarjrQs=
github.com/google/brotli v1.0.7/go.mod h1:XpGqLY1HgMKTQI5TU8iAKE/okaKqS9h1e6KRlRztlOU=
github.com/klauspost/compress v1.4.0 h1:8nsMz3tWa9SWWPL60G1V6CUsf4lLjWLTNEtibhe8gh8=

27
sources/lib/archiver/compress.go
View file

@ -8,12 +8,17 @@ import "compress/gzip"
import "fmt"
import "github.com/foobaz/go-zopfli/zopfli"
func Compress (_data []byte, _algorithm string) ([]byte, string, error) {
switch _algorithm {
case "gz", "gzip" :
return CompressGzip (_data)
case "zopfli" :
return CompressZopfli (_data)
case "br", "brotli" :
return CompressBrotli (_data)
case "", "none", "identity" :
@ -48,3 +53,25 @@ func CompressGzip (_data []byte) ([]byte, string, error) {
return _data, "gzip", nil
}
func CompressZopfli (_data []byte) ([]byte, string, error) {
_buffer := & bytes.Buffer {}
_options := zopfli.DefaultOptions ()
_options.NumIterations = 15
_options.BlockSplitting = true
_options.BlockSplittingLast = true
_options.BlockSplittingMax = 0
_options.BlockType = zopfli.DYNAMIC_BLOCK
if _error := zopfli.GzipCompress (&_options, _data, _buffer); _error != nil {
return nil, "", _error
}
_data = _buffer.Bytes ()
return _data, "gzip", nil
}