From a53d4be6e7ca22bed151d8603373b75518ea9782 Mon Sep 17 00:00:00 2001 From: Ciprian Dorin Craciun Date: Wed, 17 Nov 2021 19:50:30 +0200 Subject: [PATCH] [archiver] Add support for compressing with `zopfli`. --- documentation/readme.rst | 15 +++++++++++++-- sources/cmd/archiver/archiver.go | 2 +- sources/go.mod | 1 + sources/go.sum | 2 ++ sources/lib/archiver/compress.go | 27 +++++++++++++++++++++++++++ 5 files changed, 44 insertions(+), 3 deletions(-) diff --git a/documentation/readme.rst b/documentation/readme.rst index 35656c9..128b2de 100644 --- a/documentation/readme.rst +++ b/documentation/readme.rst @@ -15,7 +15,7 @@ However "simple" doesn't imply "dumb" or "limited", instead it implies "efficien As such ``kawipiko`` basically supports only ``GET`` (and ``HEAD``) requests and does not provide features like dynamic content, authentication, reverse proxying, etc. However, ``kawipiko`` does provide something unique, that no other HTTP server offers: the static website content is served from a CDB_ database with almost zero latency. -Moreover, the static website content can be compressed (with either ``gzip`` or ``brotli``) ahead of time, thus reducing not only CPU but also bandwidth and latency. +Moreover, the static website content can be compressed (with either ``gzip``, ``zopfli` or ``brotli``) ahead of time, thus reducing not only CPU but also bandwidth and latency. CDB_ databases are binary files that provide efficient read-only key-value lookup tables, initially used in some DNS and SMTP servers, mainly for their low overhead lookup operations, zero locking in multi-threaded / multi-process scenarios, and "atomic" multi-record updates. This also makes them suitable for low-latency static website content serving over HTTP, which this project provides. @@ -262,7 +262,7 @@ Flags --sources --archive - --compress + --compress --exclude-index --exclude-strip @@ -380,6 +380,15 @@ Examples --debug \ # +* create the CDB archive (with ``zopfli`` compression): :: + + kawipiko-archiver \ + --archive ./python-3.7.3-docs-html-zopfli.cdb \ + --sources ./python-3.7.3-docs-html \ + --compress zopfli \ + --debug \ + # + * create the CDB archive (with ``brotli`` compression): :: kawipiko-archiver \ @@ -406,6 +415,7 @@ Examples \ ./python-3.7.3-docs-html-nozip.cdb \ ./python-3.7.3-docs-html-gzip.cdb \ + ./python-3.7.3-docs-html-zopfli.cdb \ ./python-3.7.3-docs-html-brotli.cdb \ \ ./python-3.7.3-docs-html \ @@ -414,6 +424,7 @@ Examples 45M ./python-3.7.3-docs-html-nozip.cdb 9.7M ./python-3.7.3-docs-html-gzip.cdb + ??? ./python-3.7.3-docs-html-zopfli.cdb 7.9M ./python-3.7.3-docs-html-brotli.cdb 46M ./python-3.7.3-docs-html diff --git a/sources/cmd/archiver/archiver.go b/sources/cmd/archiver/archiver.go index b643de8..730e5a1 100644 --- a/sources/cmd/archiver/archiver.go +++ b/sources/cmd/archiver/archiver.go @@ -665,7 +665,7 @@ func main_0 () (error) { --sources --archive - --compress + --compress --exclude-index --exclude-strip diff --git a/sources/go.mod b/sources/go.mod index a2002a8..bb1090d 100644 --- a/sources/go.mod +++ b/sources/go.mod @@ -6,6 +6,7 @@ require ( github.com/Pallinder/go-randomdata v1.2.0 // indirect github.com/colinmarc/cdb v0.0.0-00000000000000-000000000000 github.com/davecgh/go-spew v1.1.1 // indirect + github.com/foobaz/go-zopfli v0.0.0-20140122214029-7432051485e2 // indirect github.com/google/brotli v1.0.7 github.com/klauspost/compress v1.7.6 // indirect github.com/klauspost/cpuid v1.2.1 // indirect diff --git a/sources/go.sum b/sources/go.sum index ecae5ed..563888d 100644 --- a/sources/go.sum +++ b/sources/go.sum @@ -8,6 +8,8 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/foobaz/go-zopfli v0.0.0-20140122214029-7432051485e2 h1:VA6jElpcJ+wkwEBufbnVkSBCA2TEnxdRppjRT5Kvh0A= +github.com/foobaz/go-zopfli v0.0.0-20140122214029-7432051485e2/go.mod h1:Yi95+RbwKz7uGndSuUhoq7LJKh8qH8DT9fnL4ewU30k= github.com/google/brotli v1.0.7 h1:fxwwohNEPaVS6qvtnjwgzRR62Upa70pkw0f9qarjrQs= github.com/google/brotli v1.0.7/go.mod h1:XpGqLY1HgMKTQI5TU8iAKE/okaKqS9h1e6KRlRztlOU= github.com/klauspost/compress v1.4.0 h1:8nsMz3tWa9SWWPL60G1V6CUsf4lLjWLTNEtibhe8gh8= diff --git a/sources/lib/archiver/compress.go b/sources/lib/archiver/compress.go index d94cbe8..09ef3c9 100644 --- a/sources/lib/archiver/compress.go +++ b/sources/lib/archiver/compress.go @@ -8,12 +8,17 @@ import "compress/gzip" import "fmt" +import "github.com/foobaz/go-zopfli/zopfli" + + func Compress (_data []byte, _algorithm string) ([]byte, string, error) { switch _algorithm { case "gz", "gzip" : return CompressGzip (_data) + case "zopfli" : + return CompressZopfli (_data) case "br", "brotli" : return CompressBrotli (_data) case "", "none", "identity" : @@ -48,3 +53,25 @@ func CompressGzip (_data []byte) ([]byte, string, error) { return _data, "gzip", nil } + + + +func CompressZopfli (_data []byte) ([]byte, string, error) { + + _buffer := & bytes.Buffer {} + + _options := zopfli.DefaultOptions () + _options.NumIterations = 15 + _options.BlockSplitting = true + _options.BlockSplittingLast = true + _options.BlockSplittingMax = 0 + _options.BlockType = zopfli.DYNAMIC_BLOCK + + if _error := zopfli.GzipCompress (&_options, _data, _buffer); _error != nil { + return nil, "", _error + } + + _data = _buffer.Bytes () + return _data, "gzip", nil +} +