From 08397f69c160427898639efb72f34fbb0c1e868b Mon Sep 17 00:00:00 2001 From: Ciprian Dorin Craciun Date: Thu, 16 Dec 2021 11:28:22 +0200 Subject: [PATCH] [scripts] Split scripts into multiple files. --- scripts/benchmark.z-run | 236 ++++++++++++++ scripts/examples.z-run | 64 ++++ scripts/execute.z-run | 41 +++ scripts/go.z-run | 291 +++++++++++++++++ scripts/workspace.z-run | 130 ++++++++ scripts/z-run | 690 +--------------------------------------- 6 files changed, 774 insertions(+), 678 deletions(-) create mode 100644 scripts/benchmark.z-run create mode 100644 scripts/examples.z-run create mode 100644 scripts/execute.z-run create mode 100644 scripts/go.z-run create mode 100644 scripts/workspace.z-run diff --git a/scripts/benchmark.z-run b/scripts/benchmark.z-run new file mode 100644 index 0000000..de44b6a --- /dev/null +++ b/scripts/benchmark.z-run @@ -0,0 +1,236 @@ +#!/dev/null + + + + +:: benchmark / server / 1x1 :: exec -- "${ZRUN[@]}" ':: benchmark / server / inmem' "${@}" --processes 1 --threads 1 +:: benchmark / server / 2x1 :: exec -- "${ZRUN[@]}" ':: benchmark / server / inmem' "${@}" --processes 2 --threads 1 +:: benchmark / server / 1x2 :: exec -- "${ZRUN[@]}" ':: benchmark / server / inmem' "${@}" --processes 1 --threads 2 +:: benchmark / server / 2x2 :: exec -- "${ZRUN[@]}" ':: benchmark / server / inmem' "${@}" --processes 2 --threads 2 + + + + +<< benchmark / server + if test "${#}" -ge 1 ; then + if test "${1:0:1}" != '-' ; then + _archive="${1}" + shift -- 1 + else + _archive='' + fi + else + _archive='' + fi + if test -z "${_archive}" ; then + _archive=./examples/hello-world.cdb + fi + exec -- \ + sudo -u root -n -E -P -- \ + taskset -c 0,1 \ + nice -n -19 -- \ + ionice -c 2 -n 0 -- \ + chrt -r 10 \ + prlimit -n262144 -- \ + sudo -u "${USER}" -n -E -P -- \ + env -i -- \ + ./.outputs/binaries/release/kawipiko-server \ + --bind 127.9.185.194:8080 \ + --archive "${_archive}" \ + --security-headers-disable \ + --timeout-disable \ + "${@}" \ + # +!! + + +<< benchmark / server / mmap + exec -- "${ZRUN[@]}" ':: benchmark / server' \ + "${@}" \ + --archive-mmap \ + --archive-preload \ + # +!! + +<< benchmark / server / inmem + exec -- "${ZRUN[@]}" ':: benchmark / server' \ + "${@}" \ + --archive-inmem \ + --index-all \ + # +!! + + + + +<< benchmark / server / profile / cpu + exec -- "${ZRUN[@]}" ':: benchmark / server / mmap' \ + "${@}" \ + --profile-cpu ./.outputs/server-cpu.txt \ + # +!! + +<< benchmark / server / profile / mem + exec -- "${ZRUN[@]}" ':: benchmark / server / mmap' \ + "${@}" \ + --profile-mem ./.outputs/server-mem.txt \ + # +!! + +<< benchmark / server / profile / cpu / analyze + export -- PPROF_BINARY_PATH=./.outputs/binaries/release/kawipiko-server + exec -- go tool pprof "${@}" -- ./.outputs/server-cpu.txt +!! + +<< benchmark / server / profile / mem / analyze + export -- PPROF_BINARY_PATH=./.outputs/binaries/release/kawipiko-server + exec -- go tool pprof "${@}" ./.outputs/server-mem.txt +!! + + + + + + + + +:: benchmark / client / 16384 :: exec -- "${ZRUN[@]}" ':: benchmark / client' "${@}" --threads 2 --connections 16384 --timeout 6s +:: benchmark / client / 4096 :: exec -- "${ZRUN[@]}" ':: benchmark / client' "${@}" --threads 2 --connections 4096 +:: benchmark / client / 2048 :: exec -- "${ZRUN[@]}" ':: benchmark / client' "${@}" --threads 2 --connections 2048 +:: benchmark / client / 1024 :: exec -- "${ZRUN[@]}" ':: benchmark / client' "${@}" --threads 2 --connections 1024 +:: benchmark / client / 512 :: exec -- "${ZRUN[@]}" ':: benchmark / client' "${@}" --threads 1 --connections 512 +:: benchmark / client / 256 :: exec -- "${ZRUN[@]}" ':: benchmark / client' "${@}" --threads 1 --connections 256 +:: benchmark / client / 128 :: exec -- "${ZRUN[@]}" ':: benchmark / client' "${@}" --threads 1 --connections 128 +:: benchmark / client / 64 :: exec -- "${ZRUN[@]}" ':: benchmark / client' "${@}" --threads 1 --connections 64 +:: benchmark / client / 32 :: exec -- "${ZRUN[@]}" ':: benchmark / client' "${@}" --threads 1 --connections 32 +:: benchmark / client / 16 :: exec -- "${ZRUN[@]}" ':: benchmark / client' "${@}" --threads 1 --connections 16 +:: benchmark / client / 8 :: exec -- "${ZRUN[@]}" ':: benchmark / client' "${@}" --threads 1 --connections 8 +:: benchmark / client / 4 :: exec -- "${ZRUN[@]}" ':: benchmark / client' "${@}" --threads 1 --connections 4 +:: benchmark / client / 2 :: exec -- "${ZRUN[@]}" ':: benchmark / client' "${@}" --threads 1 --connections 2 +:: benchmark / client / 1 :: exec -- "${ZRUN[@]}" ':: benchmark / client' "${@}" --threads 1 --connections 1 + + + + +<< benchmark / client + if test "${#}" -ge 1 ; then + if test "${1:0:1}" != '-' ; then + _path="${1}" + _path="${_path#/}" + shift -- 1 + else + _path='' + fi + else + _path='' + fi + exec -- \ + sudo -u root -n -E -P -- \ + taskset -c 2,3 \ + nice -n -19 -- \ + ionice -c 2 -n 0 -- \ + chrt -r 10 \ + prlimit -n262144 -- \ + sudo -u "${USER}" -n -E -P -- \ + env -i -- \ + ./.bin/wrk \ + --threads 1 \ + --connections 1 \ + --conn-reqs 1024 \ + --timeout 1s \ + --duration 30s \ + --progress \ + --latency \ + "${@}" \ + -- "http://127.9.185.194:8080/${_path}" \ + # +!! + + +<< benchmark / client / paths + test "${#}" -ge 1 + _paths="${1}" + shift -- 1 + exec -- \ + sudo -u root -n -E -P -- \ + taskset -c 2,3 \ + nice -n -19 -- \ + ionice -c 2 -n 0 -- \ + chrt -r 10 \ + prlimit -n262144 -- \ + sudo -u "${USER}" -n -E -P -- \ + env -i -- \ + ./.bin/wrk \ + --threads 1 \ + --connections 1 \ + --conn-reqs 1024 \ + --timeout 1s \ + --duration 30s \ + --progress \ + --latency \ + --script ../kawipiko-examples/sources/wrk-paths-from-file.lua \ + "${@}" \ + -- "http://127.9.185.194:8080/" \ + "${_paths}" \ + # +!! + + + + +<< benchmark / curl + if test "${#}" -ge 1 ; then + if test "${1:0:1}" != '-' ; then + _path="${1}" + _path="${_path#/}" + shift -- 1 + else + _path='' + fi + else + _path='' + fi + exec -- \ + env -i -- \ + "$( type -P -- curl )" \ + --silent \ + --compressed \ + --output /dev/stdout \ + --dump-header /dev/stdout \ + "${@}" \ + -- \ + "http://127.9.185.194:8080/${_path}" \ + # +!! + + + + +<< benchmark / dummy + test "${#}" -eq 0 + _outputs="$( exec -- readlink -e -- ./.outputs )" + mkdir -p -- "${_outputs}/binaries/release" + rm -f -- "${_outputs}/binaries/release/kawipiko-server-dummy" + "${ZRUN[@]}" ':: go / tool' \ + build \ + -mod readonly \ + -tags netgo \ + -ldflags 'all=-s -extld=gcc -extldflags=-static' \ + -gcflags 'all=-l=4' \ + -o "${_outputs}/binaries/release/kawipiko-server-dummy" \ + -- ./cmd/server-dummy.go \ + # + exec -- \ + sudo -u root -n -E -P -- \ + taskset -c 0,1 \ + nice -n -19 -- \ + ionice -c 2 -n 0 -- \ + chrt -r 10 \ + prlimit -n262144 -- \ + sudo -u "${USER}" -n -E -P -- \ + env -i -- \ + ./.outputs/binaries/release/kawipiko-server-dummy \ + "${@}" \ + # +!! + diff --git a/scripts/examples.z-run b/scripts/examples.z-run new file mode 100644 index 0000000..e0c80ab --- /dev/null +++ b/scripts/examples.z-run @@ -0,0 +1,64 @@ +#!/dev/null + + + + +<< examples / hello-world / archive + "${ZRUN[@]}" ':: execute / archiver / debug' \ + --sources ./examples/hello-world \ + --archive ./examples/hello-world.cdb \ + --progress \ + --debug \ + "${@}" \ + # + cdb -d -- ./examples/hello-world.cdb \ + | tr -d '\r' \ + | sed -r -e 's#^(\+[0-9]+,[0-9]+):(.+?)->#\n\1\n\2\n#' \ + >| ./examples/hello-world.cdb-dump \ + # +!! + + +<< examples / hello-world / serve + exec -- "${ZRUN[@]}" ':: execute / server / debug' \ + --bind 127.153.215.30:8080 \ + --archive ./examples/hello-world.cdb \ + --archive-inmem \ + --index-all \ + --processes 1 \ + --threads 1 \ + --debug \ + "${@}" \ + # +!! + + + + +<< examples / huge / archive + exec -- "${ZRUN[@]}" ':: execute / archiver / debug' \ + --sources ./.outputs/examples/huge \ + --archive ./.outputs/examples/huge.cdb \ + --compress gzip \ + --compress-cache ./.outputs/examples/huge-compress.cache \ + --sources-cache ./.outputs/examples/huge-sources.cache \ + --exclude-strip \ + --exclude-file-listing \ + --progress \ + "${@}" \ + # +!! + + +<< examples / huge / serve + exec -- "${ZRUN[@]}" ':: execute / server / debug' \ + --bind 127.153.215.30:8080 \ + --archive ./.outputs/examples/huge.cdb \ + --archive-mmap \ + --processes 1 \ + --threads 1 \ + --debug \ + "${@}" \ + # +!! + diff --git a/scripts/execute.z-run b/scripts/execute.z-run new file mode 100644 index 0000000..cb47afa --- /dev/null +++ b/scripts/execute.z-run @@ -0,0 +1,41 @@ +#!/dev/null + + + + +++<< execute / server / debug + "${ZRUN[@]}" ':: go / build / server / debug' + exec -- env -i -- ./.outputs/binaries/debug/kawipiko-server "${@}" +!! + +<< execute / server / release + "${ZRUN[@]}" ':: go / build / server / release' + exec -- env -i -- ./.outputs/binaries/release/kawipiko-server "${@}" +!! + + + + +++<< execute / archiver / debug + "${ZRUN[@]}" ':: go / build / archiver / debug' + exec -- env -i -- ./.outputs/binaries/debug/kawipiko-archiver "${@}" +!! + +<< execute / archiver / release + "${ZRUN[@]}" ':: go / build / archiver / release' + exec -- env -i -- ./.outputs/binaries/release/kawipiko-archiver "${@}" +!! + + + + +<< execute / wrapper / debug + "${ZRUN[@]}" ':: go / build / wrapper / debug' + exec -- env -i -- ./.outputs/binaries/debug/kawipiko-wrapper "${@}" +!! + +<< execute / wrapper / release + "${ZRUN[@]}" ':: go / build / wrapper / release' + exec -- env -i -- ./.outputs/binaries/release/kawipiko-wrapper "${@}" +!! + diff --git a/scripts/go.z-run b/scripts/go.z-run new file mode 100644 index 0000000..ab4f6f8 --- /dev/null +++ b/scripts/go.z-run @@ -0,0 +1,291 @@ +#!/dev/null + + + + +<< go / build / server / debug + test "${#}" -eq 0 + _outputs="$( exec -- readlink -e -- ./.outputs )" + mkdir -p -- "${_outputs}/binaries/debug" + if test -e "${_outputs}/binaries/debug/kawipiko-server" ; then + rm -- "${_outputs}/binaries/debug/kawipiko-server" + fi + exec -- \ + nodaemon \ + nice -n 19 \ + "${ZRUN[@]}" ':: go / tool' \ + build \ + -mod readonly \ + -o "${_outputs}/binaries/debug/kawipiko-server" \ + -- ./cmd/server.go \ + # +!! + + +<< go / build / server / release + test "${#}" -eq 0 + _outputs="$( exec -- readlink -e -- ./.outputs )" + mkdir -p -- "${_outputs}/binaries/release" + if test -e "${_outputs}/binaries/release/kawipiko-server" ; then + rm -- "${_outputs}/binaries/release/kawipiko-server" + fi + exec -- \ + nodaemon \ + nice -n 19 -- \ + "${ZRUN[@]}" ':: go / tool' \ + build \ + -mod readonly \ + -tags netgo \ + -a \ + -gcflags 'all=-l=4' \ + -ldflags 'all=-s' \ + -buildmode exe \ + -trimpath \ + -o "${_outputs}/binaries/release/kawipiko-server" \ + -- ./cmd/server.go \ + # +!! + + + + +<< go / build / archiver / debug + test "${#}" -eq 0 + _outputs="$( exec -- readlink -e -- ./.outputs )" + mkdir -p -- "${_outputs}/binaries/debug" + if test -e "${_outputs}/binaries/debug/kawipiko-archiver" ; then + rm -- ${_outputs}/binaries/debug/kawipiko-archiver + fi + exec -- \ + nodaemon \ + nice -n 19 -- \ + "${ZRUN[@]}" ':: go / tool' \ + build \ + -mod readonly \ + -o "${_outputs}/binaries/debug/kawipiko-archiver" \ + -- ./cmd/archiver.go \ + # +!! + +<< go / build / archiver / release + test "${#}" -eq 0 + _outputs="$( exec -- readlink -e -- ./.outputs )" + mkdir -p -- "${_outputs}/binaries/release" + if test -e "${_outputs}/binaries/release/kawipiko-archiver" ; then + rm -- "${_outputs}/binaries/release/kawipiko-archiver" + fi + exec -- \ + nodaemon \ + nice -n 19 -- \ + "${ZRUN[@]}" ':: go / tool' \ + build \ + -mod readonly \ + -tags netgo \ + -a \ + -gcflags 'all=-l=4' \ + -ldflags 'all=-s' \ + -buildmode exe \ + -trimpath \ + -o "${_outputs}/binaries/release/kawipiko-archiver" \ + -- ./cmd/archiver.go \ + # +!! + + + + +<< go / build / wrapper / debug + test "${#}" -eq 0 + _outputs="$( exec -- readlink -e -- ./.outputs )" + mkdir -p -- "${_outputs}/binaries/debug" + if test -e "${_outputs}/binaries/debug/kawipiko-wrapper" ; then + rm -- "${_outputs}/binaries/debug/kawipiko-wrapper" + fi + exec -- \ + nodaemon \ + nice -n 19 \ + "${ZRUN[@]}" ':: go / tool' \ + build \ + -mod readonly \ + -o "${_outputs}/binaries/debug/kawipiko-wrapper" \ + -- ./cmd/wrapper.go \ + # +!! + + +<< go / build / wrapper / release + test "${#}" -eq 0 + _outputs="$( exec -- readlink -e -- ./.outputs )" + mkdir -p -- "${_outputs}/binaries/release" + if test -e "${_outputs}/binaries/release/kawipiko-wrapper" ; then + rm -- "${_outputs}/binaries/release/kawipiko-wrapper" + fi + exec -- \ + nodaemon \ + nice -n 19 -- \ + "${ZRUN[@]}" ':: go / tool' \ + build \ + -mod readonly \ + -tags netgo \ + -a \ + -gcflags 'all=-l=4' \ + -ldflags 'all=-s' \ + -buildmode exe \ + -trimpath \ + -o "${_outputs}/binaries/release/kawipiko-wrapper" \ + -- ./cmd/wrapper.go \ + # +!! + + + + +<< go / build / all / release + test "${#}" -eq 0 + _outputs="$( exec -- readlink -e -- ./.outputs )" + mkdir -p -- "${_outputs}/binaries/release" + for _platform in linux darwin openbsd freebsd ; do + for _component in archiver server wrapper ; do + if test -e "${_outputs}/binaries/release/kawipiko-${_component}--${_platform}" ; then + rm -f -- "${_outputs}/binaries/release/kawipiko-${_component}--${_platform}" + fi + done + done + for _platform in linux darwin openbsd freebsd ; do + for _component in archiver server wrapper ; do + printf -- '[ii] building `%s` for `%s`...\n' "${_component}" "${_platform}" >&2 + env GOOS="${_platform}" \ + nodaemon \ + nice -n 19 -- \ + "${ZRUN[@]}" ':: go / tool' \ + build \ + -mod readonly \ + -tags netgo \ + -a \ + -gcflags 'all=-l=4' \ + -ldflags 'all=-s' \ + -buildmode exe \ + -trimpath \ + -o "${_outputs}/binaries/release/kawipiko-${_component}--${_platform}" \ + -- "./cmd/${_component}.go" \ + # + done + done +!! + + + + +<< go / build / all / publish + test "${#}" -eq 0 + if test ! -e ./.outputs/publish ; then + mkdir -- ./.outputs/publish + fi + if test ! -e ./.outputs/publish/binaries ; then + mkdir -- ./.outputs/publish/binaries + fi + for _platform in linux darwin openbsd freebsd ; do + for _component in archiver server wrapper ; do + cp -T -- \ + "./.outputs/binaries/release/kawipiko-${_component}--${_platform}" \ + "./.outputs/publish/binaries/kawipiko-${_component}--${_platform}" \ + # + done + done +!! + + + + +<< go / build / server / analyze + test "${#}" -eq 0 + _outputs="$( exec -- readlink -e -- ./.outputs )" + mkdir -p -- "${_outputs}/binaries/debug" + exec -- "${ZRUN[@]}" ':: go / tool' \ + build \ + -mod readonly \ + -gcflags '-m' \ + -o "${_outputs}/binaries/debug/kawipiko-server-analyze" \ + -- ./cmd/server.go \ + # +!! + + +<< go / build / server / analyze / escapes + "${ZRUN[@]}" ':: go / build / server / analyze' "${@}" \ + 2>&1 \ + | grep -F -e ' escapes ' \ + | LC_ALL=C sort -n -k 2,2 -t ':' +!! + + + + +<< go / dependencies / update + test "${#}" -eq 0 + exec -- "${ZRUN[@]}" ':: go / tool' \ + get \ + -v \ + -d \ + -u \ + -- \ + all \ + # +!! + + +<< go / dependencies / list + test "${#}" -eq 0 + exec -- "${ZRUN[@]}" ':: go / tool' \ + list \ + -m \ + -- \ + all \ + # +!! + + +<< go / dependencies / tidy + test "${#}" -eq 0 + exec -- "${ZRUN[@]}" ':: go / tool' \ + mod tidy \ + # +!! + + +<< go / dependencies / verify + test "${#}" -eq 0 + exec -- "${ZRUN[@]}" ':: go / tool' \ + mod verify \ + # +!! + + +<< go / dependencies / upgrade + test "${#}" -eq 0 + "${ZRUN[@]}" ':: go / dependencies / update' + "${ZRUN[@]}" ':: go / dependencies / tidy' + "${ZRUN[@]}" ':: go / dependencies / verify' + "${ZRUN[@]}" ':: go / dependencies / list' +!! + + + + +<< go / tool + test "${#}" -ge 1 + + _outputs="$( exec -- readlink -e -- ./.outputs )" + _sources="$( exec -- readlink -e -- ./sources )" + + export -- GOPATH="${_outputs}/go" + export -- GOBIN="${_outputs}/go/bin" + export -- GOTMPDIR="${_outputs}/go/tmp" + export -- GOCACHE="${_outputs}/go/cache" + export -- GO111MODULE=on + + cd -- "${_sources}" + + exec -- go "${@}" +!! + diff --git a/scripts/workspace.z-run b/scripts/workspace.z-run new file mode 100644 index 0000000..f710018 --- /dev/null +++ b/scripts/workspace.z-run @@ -0,0 +1,130 @@ +#!/dev/null + + + + +<< workspace / publish + test "${#}" -eq 0 + printf -- '[ii] publishing to `https://data.volution.ro/ciprian/ad5264afc6e27e46b5d71837552cb3fd`...\n' >&2 + exec -- rsync -a -v -- ./.outputs/publish/ ./.publish/ +!! + + + + +<< workspace / initialize + "${ZRUN[@]}" ':: workspace / initialize / outputs' + "${ZRUN[@]}" ':: workspace / initialize / go' +!! + + + + +<< workspace / initialize / outputs + test "${#}" -eq 0 + if test ! -d ./.outputs ; then + _outputs_store="${TMPDIR:-/tmp}/go--${UID}--${RANDOM}-${RANDOM}-${RANDOM}-${RANDOM}" + mkdir -- "${_outputs_store}" + _outputs_store="$( exec -- readlink -e -- "${_outputs_store}" )" + ln -s -f -T -- "${_outputs_store}" ./.outputs + fi +!! + + + + +<< workspace / initialize / go + test "${#}" -eq 0 + _outputs="$( exec -- readlink -e -- ./.outputs )" + _sources="$( exec -- readlink -e -- ./sources )" + + GOPATH="${_outputs}/go" + GOBIN="${_outputs}/go/bin" + GOTMPDIR="${_outputs}/go/tmp" + GOCACHE="${_outputs}/go/cache" + _gosrc="${_outputs}/go/src" + _gopkg="${_outputs}/go/pkg" + + for _folder in "${GOPATH}" "${GOBIN}" "${GOTMPDIR}" "${GOTMPDIR}" "${GOCACHE}" "${_gosrc}" "${_gopkg}" ; do + if test ! -e "${_folder}" ; then + mkdir -- "${_folder}" + fi + done +!! + + + + +<< workspace / initialize / python + test "${#}" -eq 0 + if test -d ./.python ; then + exit -- 0 + fi + + test -d ./.outputs + mkdir -- ./.outputs/python + + virtualenv-2.7 \ + --python python2.7 \ + --clear \ + --no-site-packages \ + --no-setuptools \ + --no-wheel \ + --no-download \ + --unzip-setuptools \ + -- \ + ./.outputs/python \ + # + + ./.outputs/python/bin/pip install --upgrade --compile -- pip + ./.outputs/python/bin/pip install --upgrade --compile -- wheel + ./.outputs/python/bin/pip install --upgrade --compile -- setuptools + + ./.outputs/python/bin/pip install --compile -- docutils + ./.outputs/python/bin/pip install --compile -- restview + + ln -s -f -T -- ./.outputs/python ./.python +!! + + + + +<< workspace / sources / codes / duplicates + test "${#}" -eq 0 + exec -- xargs \ + -r -d '\n' -I {} \ + -a <( + grep \ + -o \ + -P \ + -e '(?<=\[)[0-9a-f]{8}(?=\])|(?<=0x)[0-9a-f]{8}(?=[^0-9a-zA-Z]|$)' \ + -h \ + -r ./sources \ + --include '*.go' \ + | sort \ + | uniq -d + ) \ + -- \ + grep \ + -P \ + -e '(?<=\[){}(?=\])|(?<=0x){}(?=[^0-9a-zA-Z]|$)' \ + -r ./sources \ + -n \ + --include '*.go' \ + --color \ + # +!! + + + + +<< documentation / readme / open + test "${#}" -eq 0 + exec -- x-www open http://127.33.237.174:8080/ +!! + +<< documentation / readme / server + test "${#}" -eq 0 + exec -- env -i -- ./.python/bin/restview --no-browser --listen 127.33.237.174:8080 --allowed-hosts '*' -- ./readme.rst +!! + diff --git a/scripts/z-run b/scripts/z-run index 75f9334..443f7f2 100644 --- a/scripts/z-run +++ b/scripts/z-run @@ -1,683 +1,17 @@ #!/dev/null - - -::// * - -::// execute / ... -::// benchmark / ... -::// examples / ... -::// documentation / ... ::// workspace / ... - -::// go / build / ... -::// go / dependencies / ... - - - - -++<< execute / server / debug - "${ZRUN[@]}" ':: go / build / server / debug' - exec -- env -i -- ./.outputs/binaries/debug/kawipiko-server "${@}" -!! - -<< execute / server / release - "${ZRUN[@]}" ':: go / build / server / release' - exec -- env -i -- ./.outputs/binaries/release/kawipiko-server "${@}" -!! - - -<< go / build / server / debug - test "${#}" -eq 0 - _outputs="$( exec -- readlink -e -- ./.outputs )" - mkdir -p -- "${_outputs}/binaries/debug" - rm -f -- "${_outputs}/binaries/debug/kawipiko-server" - exec -- "${ZRUN[@]}" ':: go / tool' \ - build \ - -mod readonly \ - -o "${_outputs}/binaries/debug/kawipiko-server" \ - -- ./cmd/server.go \ - # -!! - -<< go / build / server / release - test "${#}" -eq 0 - _outputs="$( exec -- readlink -e -- ./.outputs )" - mkdir -p -- "${_outputs}/binaries/release" - rm -f -- "${_outputs}/binaries/release/kawipiko-server" - exec -- \ - nice -n 19 -- \ - "${ZRUN[@]}" ':: go / tool' \ - build \ - -mod readonly \ - -tags netgo \ - -a \ - -gcflags 'all=-l=4' \ - -ldflags 'all=-s' \ - -buildmode exe \ - -trimpath \ - -o "${_outputs}/binaries/release/kawipiko-server" \ - -- ./cmd/server.go \ - # -!! - - - - -++<< execute / archiver / debug - "${ZRUN[@]}" ':: go / build / archiver / debug' - exec -- env -i -- ./.outputs/binaries/debug/kawipiko-archiver "${@}" -!! - -<< execute / archiver / release - "${ZRUN[@]}" ':: go / build / archiver / release' - exec -- env -i -- ./.outputs/binaries/release/kawipiko-archiver "${@}" -!! - - -<< go / build / archiver / debug - test "${#}" -eq 0 - _outputs="$( exec -- readlink -e -- ./.outputs )" - mkdir -p -- "${_outputs}/binaries/debug" - rm -f -- "${_outputs}/binaries/debug/kawipiko-archiver" - exec -- "${ZRUN[@]}" ':: go / tool' \ - build \ - -mod readonly \ - -o "${_outputs}/binaries/debug/kawipiko-archiver" \ - -- ./cmd/archiver.go \ - # -!! - -<< go / build / archiver / release - test "${#}" -eq 0 - _outputs="$( exec -- readlink -e -- ./.outputs )" - mkdir -p -- "${_outputs}/binaries/release" - rm -f -- "${_outputs}/binaries/release/kawipiko-archiver" - exec -- \ - nice -n 19 -- \ - "${ZRUN[@]}" ':: go / tool' \ - build \ - -mod readonly \ - -tags netgo \ - -a \ - -gcflags 'all=-l=4' \ - -ldflags 'all=-s' \ - -buildmode exe \ - -trimpath \ - -o "${_outputs}/binaries/release/kawipiko-archiver" \ - -- ./cmd/archiver.go \ - # -!! - - - - -<< execute / wrapper / release - "${ZRUN[@]}" ':: go / build / wrapper / release' - exec -- env -i -- ./.outputs/binaries/release/kawipiko "${@}" -!! - - -<< go / build / wrapper / release - test "${#}" -eq 0 - _outputs="$( exec -- readlink -e -- ./.outputs )" - mkdir -p -- "${_outputs}/binaries/release" - rm -f -- "${_outputs}/binaries/release/kawipiko" - exec -- \ - nice -n 19 -- \ - "${ZRUN[@]}" ':: go / tool' \ - build \ - -mod readonly \ - -tags netgo \ - -a \ - -gcflags 'all=-l=4' \ - -ldflags 'all=-s' \ - -buildmode exe \ - -trimpath \ - -o "${_outputs}/binaries/release/kawipiko" \ - -- ./cmd/wrapper.go \ - # -!! - - - - -<< go / build / all / release - test "${#}" -eq 0 - _outputs="$( exec -- readlink -e -- ./.outputs )" - mkdir -p -- "${_outputs}/binaries/release" - for _platform in linux darwin openbsd freebsd ; do - for _component in archiver server wrapper ; do - printf -- '[ii] building `%s` for `%s`...\n' "${_component}" "${_platform}" >&2 - rm -f -- "${_outputs}/binaries/release/kawipiko-${_component}--${_platform}" - nodaemon \ - env GOOS="${_platform}" \ - nice -n 19 -- \ - "${ZRUN[@]}" ':: go / tool' \ - build \ - -mod readonly \ - -tags netgo \ - -a \ - -gcflags 'all=-l=4' \ - -ldflags 'all=-s' \ - -buildmode exe \ - -trimpath \ - -o "${_outputs}/binaries/release/kawipiko-${_component}--${_platform}" \ - -- "./cmd/${_component}.go" \ - # - done - done -!! - - -<< go / build / all / publish - test "${#}" -eq 0 - if test ! -e ./.outputs/publish ; then - mkdir -- ./.outputs/publish - fi - if test ! -e ./.outputs/publish/binaries ; then - mkdir -- ./.outputs/publish/binaries - fi - for _platform in linux darwin openbsd freebsd ; do - for _component in archiver server wrapper ; do - cp -T -- \ - "./.outputs/binaries/release/kawipiko-${_component}--${_platform}" \ - "./.outputs/publish/binaries/kawipiko-${_component}--${_platform}" \ - # - done - done -!! - - - - -<< workspace / publish - test "${#}" -eq 0 - printf -- '[ii] publishing to `https://data.volution.ro/ciprian/ad5264afc6e27e46b5d71837552cb3fd`...\n' >&2 - exec -- rsync -a -v -- ./.outputs/publish/ ./.publish/ -!! - - - - -<< go / build / server / analyze - test "${#}" -eq 0 - _outputs="$( exec -- readlink -e -- ./.outputs )" - mkdir -p -- "${_outputs}/binaries/debug" - rm -f -- "${_outputs}/binaries/debug/kawipiko-server" - exec -- "${ZRUN[@]}" ':: go / tool' \ - build \ - -mod readonly \ - -gcflags '-m' \ - -o "${_outputs}/binaries/debug/kawipiko-server" \ - -- ./cmd/server.go \ - # -!! - -<< go / build / server / analyze / escapes - "${ZRUN[@]}" ':: go / build / server / analyze' "${@}" \ - 2>&1 \ - | grep -F -e ' escapes ' \ - | LC_ALL=C sort -n -k 2,2 -t ':' -!! - - - - -<< go / dependencies / update - test "${#}" -eq 0 - exec -- "${ZRUN[@]}" ':: go / tool' \ - get \ - -v \ - -d \ - -u \ - -- \ - all \ - # -!! - - -<< go / dependencies / list - test "${#}" -eq 0 - exec -- "${ZRUN[@]}" ':: go / tool' \ - list \ - -m \ - -mod readonly \ - -- \ - all \ - # -!! - - -<< go / dependencies / tidy - test "${#}" -eq 0 - exec -- "${ZRUN[@]}" ':: go / tool' \ - mod tidy \ - # -!! - - -<< go / dependencies / verify - test "${#}" -eq 0 - exec -- "${ZRUN[@]}" ':: go / tool' \ - mod verify \ - # -!! - - -<< go / dependencies / upgrade - test "${#}" -eq 0 - "${ZRUN[@]}" ':: go / dependencies / update' - "${ZRUN[@]}" ':: go / dependencies / tidy' - "${ZRUN[@]}" ':: go / dependencies / verify' - "${ZRUN[@]}" ':: go / dependencies / list' -!! - - - - -<< go / tool - test "${#}" -ge 1 - _outputs="$( exec -- readlink -e -- ./.outputs )" - _sources="$( exec -- readlink -e -- ./sources )" - - export -- GOPATH="${_outputs}/go" - export -- GOBIN="${_outputs}/go/bin" - export -- GOTMPDIR="${_outputs}/go/tmp" - export -- GOCACHE="${_outputs}/go/cache" - export -- GO111MODULE=on - - cd -- "${_sources}" - - exec -- go "${@}" -!! - - - - -<< workspace / initialize / outputs - test "${#}" -eq 0 - if test ! -d ./.outputs ; then - _outputs_store="${TMPDIR:-/tmp}/go--${UID}--${RANDOM}-${RANDOM}-${RANDOM}-${RANDOM}" - mkdir -- "${_outputs_store}" - _outputs_store="$( exec -- readlink -e -- "${_outputs_store}" )" - ln -s -f -T -- "${_outputs_store}" ./.outputs - fi -!! - - -<< workspace / initialize / go - test "${#}" -eq 0 - _outputs="$( exec -- readlink -e -- ./.outputs )" - _sources="$( exec -- readlink -e -- ./sources )" - - GOPATH="${_outputs}/go" - GOBIN="${_outputs}/go/bin" - GOTMPDIR="${_outputs}/go/tmp" - GOCACHE="${_outputs}/go/cache" - _gosrc="${_outputs}/go/src" - _gopkg="${_outputs}/go/pkg" - - for _folder in "${GOPATH}" "${GOBIN}" "${GOTMPDIR}" "${GOTMPDIR}" "${GOCACHE}" "${_gosrc}" "${_gopkg}" ; do - if test ! -e "${_folder}" ; then - mkdir -- "${_folder}" - fi - done -!! - - -<< workspace / initialize / python - test "${#}" -eq 0 - if test -d ./.python ; then - exit -- 0 - fi - - test -d ./.outputs - mkdir -- ./.outputs/python - - virtualenv-2.7 \ - --python python2.7 \ - --clear \ - --no-site-packages \ - --no-setuptools \ - --no-wheel \ - --no-download \ - --unzip-setuptools \ - -- \ - ./.outputs/python \ - # - - ./.outputs/python/bin/pip install --upgrade --compile -- pip - ./.outputs/python/bin/pip install --upgrade --compile -- wheel - ./.outputs/python/bin/pip install --upgrade --compile -- setuptools - - ./.outputs/python/bin/pip install --compile -- docutils - ./.outputs/python/bin/pip install --compile -- restview - - ln -s -f -T -- ./.outputs/python ./.python -!! - - - - -<< workspace / sources / codes / duplicates - test "${#}" -eq 0 - exec -- xargs \ - -r -d '\n' -I {} \ - -a <( - grep \ - -o \ - -P \ - -e '(?<=\[)[0-9a-f]{8}(?=\])|(?<=0x)[0-9a-f]{8}(?=[^0-9a-zA-Z]|$)' \ - -h \ - -r ./sources \ - --include '*.go' \ - | sort \ - | uniq -d - ) \ - -- \ - grep \ - -P \ - -e '(?<=\[){}(?=\])|(?<=0x){}(?=[^0-9a-zA-Z]|$)' \ - -r ./sources \ - -n \ - --include '*.go' \ - --color \ - # -!! - - - - -<< documentation / readme / open - test "${#}" -eq 0 - exec -- x-www open http://127.33.237.174:8080/ -!! - -<< documentation / readme / server - test "${#}" -eq 0 - exec -- env -i -- ./.python/bin/restview --no-browser --listen 127.33.237.174:8080 --allowed-hosts '*' -- ./readme.rst -!! - - - - -:: benchmark / server / 1x1 :: exec -- "${ZRUN[@]}" ':: benchmark / server / inmem' "${@}" --processes 1 --threads 1 -:: benchmark / server / 2x1 :: exec -- "${ZRUN[@]}" ':: benchmark / server / inmem' "${@}" --processes 2 --threads 1 -++:: benchmark / server / 1x2 :: exec -- "${ZRUN[@]}" ':: benchmark / server / inmem' "${@}" --processes 1 --threads 2 -:: benchmark / server / 2x2 :: exec -- "${ZRUN[@]}" ':: benchmark / server / inmem' "${@}" --processes 2 --threads 2 - - -<< benchmark / server - if test "${#}" -ge 1 ; then - if test "${1:0:1}" != '-' ; then - _archive="${1}" - shift -- 1 - else - _archive='' - fi - else - _archive='' - fi - if test -z "${_archive}" ; then - _archive=./examples/hello-world.cdb - fi - exec -- \ - sudo -u root -n -E -P -- \ - taskset -c 0,1 \ - nice -n -19 -- \ - ionice -c 2 -n 0 -- \ - chrt -r 10 \ - prlimit -n262144 -- \ - sudo -u "${USER}" -n -E -P -- \ - env -i -- \ - ./.outputs/binaries/release/kawipiko-server \ - --bind 127.9.185.194:8080 \ - --archive "${_archive}" \ - --security-headers-disable \ - --timeout-disable \ - "${@}" \ - # -!! - -<< benchmark / server / mmap - exec -- "${ZRUN[@]}" ':: benchmark / server' \ - "${@}" \ - --archive-mmap \ - --archive-preload \ - # -!! - -<< benchmark / server / inmem - exec -- "${ZRUN[@]}" ':: benchmark / server' \ - "${@}" \ - --archive-inmem \ - --index-all \ - # -!! - - -<< benchmark / server / profile / cpu - exec -- "${ZRUN[@]}" ':: benchmark / server / mmap' \ - "${@}" \ - --profile-cpu ./.outputs/server-cpu.txt \ - # -!! - -<< benchmark / server / profile / mem - exec -- "${ZRUN[@]}" ':: benchmark / server / mmap' \ - "${@}" \ - --profile-mem ./.outputs/server-mem.txt \ - # -!! - -<< benchmark / server / profile / cpu / analyze - export -- PPROF_BINARY_PATH=./.outputs/binaries/release/kawipiko-server - exec -- go tool pprof "${@}" -- ./.outputs/server-cpu.txt -!! - -<< benchmark / server / profile / mem / analyze - export -- PPROF_BINARY_PATH=./.outputs/binaries/release/kawipiko-server - exec -- go tool pprof "${@}" ./.outputs/server-mem.txt -!! - - - - -:: benchmark / client / 16384 :: exec -- "${ZRUN[@]}" ':: benchmark / client' "${@}" --threads 2 --connections 16384 --timeout 6s -:: benchmark / client / 4096 :: exec -- "${ZRUN[@]}" ':: benchmark / client' "${@}" --threads 2 --connections 4096 -:: benchmark / client / 2048 :: exec -- "${ZRUN[@]}" ':: benchmark / client' "${@}" --threads 2 --connections 2048 -:: benchmark / client / 1024 :: exec -- "${ZRUN[@]}" ':: benchmark / client' "${@}" --threads 2 --connections 1024 -:: benchmark / client / 512 :: exec -- "${ZRUN[@]}" ':: benchmark / client' "${@}" --threads 1 --connections 512 -:: benchmark / client / 256 :: exec -- "${ZRUN[@]}" ':: benchmark / client' "${@}" --threads 1 --connections 256 -++:: benchmark / client / 128 :: exec -- "${ZRUN[@]}" ':: benchmark / client' "${@}" --threads 1 --connections 128 -:: benchmark / client / 64 :: exec -- "${ZRUN[@]}" ':: benchmark / client' "${@}" --threads 1 --connections 64 -:: benchmark / client / 32 :: exec -- "${ZRUN[@]}" ':: benchmark / client' "${@}" --threads 1 --connections 32 -:: benchmark / client / 16 :: exec -- "${ZRUN[@]}" ':: benchmark / client' "${@}" --threads 1 --connections 16 -:: benchmark / client / 8 :: exec -- "${ZRUN[@]}" ':: benchmark / client' "${@}" --threads 1 --connections 8 -:: benchmark / client / 4 :: exec -- "${ZRUN[@]}" ':: benchmark / client' "${@}" --threads 1 --connections 4 -:: benchmark / client / 2 :: exec -- "${ZRUN[@]}" ':: benchmark / client' "${@}" --threads 1 --connections 2 -:: benchmark / client / 1 :: exec -- "${ZRUN[@]}" ':: benchmark / client' "${@}" --threads 1 --connections 1 - - -<< benchmark / client - if test "${#}" -ge 1 ; then - if test "${1:0:1}" != '-' ; then - _path="${1}" - _path="${_path#/}" - shift -- 1 - else - _path='' - fi - else - _path='' - fi - exec -- \ - sudo -u root -n -E -P -- \ - taskset -c 2,3 \ - nice -n -19 -- \ - ionice -c 2 -n 0 -- \ - chrt -r 10 \ - prlimit -n262144 -- \ - sudo -u "${USER}" -n -E -P -- \ - env -i -- \ - ./.bin/wrk \ - --threads 1 \ - --connections 1 \ - --conn-reqs 1024 \ - --timeout 1s \ - --duration 30s \ - --progress \ - --latency \ - "${@}" \ - -- "http://127.9.185.194:8080/${_path}" \ - # -!! - - -<< benchmark / client / paths - test "${#}" -ge 1 - _paths="${1}" - shift -- 1 - exec -- \ - sudo -u root -n -E -P -- \ - taskset -c 2,3 \ - nice -n -19 -- \ - ionice -c 2 -n 0 -- \ - chrt -r 10 \ - prlimit -n262144 -- \ - sudo -u "${USER}" -n -E -P -- \ - env -i -- \ - ./.bin/wrk \ - --threads 1 \ - --connections 1 \ - --conn-reqs 1024 \ - --timeout 1s \ - --duration 30s \ - --progress \ - --latency \ - --script ../kawipiko-examples/sources/wrk-paths-from-file.lua \ - "${@}" \ - -- "http://127.9.185.194:8080/" \ - "${_paths}" \ - # -!! - - -<< benchmark / curl - if test "${#}" -ge 1 ; then - if test "${1:0:1}" != '-' ; then - _path="${1}" - _path="${_path#/}" - shift -- 1 - else - _path='' - fi - else - _path='' - fi - exec -- \ - env -i -- \ - "$( type -P -- curl )" \ - --silent \ - --compressed \ - --output /dev/stdout \ - --dump-header /dev/stdout \ - "${@}" \ - -- \ - "http://127.9.185.194:8080/${_path}" \ - # -!! - - - - -<< examples / hello-world / archive - "${ZRUN[@]}" ':: execute / archiver / debug' \ - --sources ./examples/hello-world \ - --archive ./examples/hello-world.cdb \ - --progress \ - --debug \ - "${@}" \ - # - cdb -d -- ./examples/hello-world.cdb \ - | tr -d '\r' \ - | sed -r -e 's#^(\+[0-9]+,[0-9]+):(.+?)->#\n\1\n\2\n#' \ - >| ./examples/hello-world.cdb-dump \ - # -!! - -<< examples / hello-world / serve - exec -- "${ZRUN[@]}" ':: execute / server / debug' \ - --bind 127.153.215.30:8080 \ - --archive ./examples/hello-world.cdb \ - --archive-inmem \ - --index-all \ - --processes 1 \ - --threads 1 \ - --debug \ - "${@}" \ - # -!! - - - - -<< examples / huge / archive - exec -- "${ZRUN[@]}" ':: execute / archiver / debug' \ - --sources ./.outputs/examples/huge \ - --archive ./.outputs/examples/huge.cdb \ - --compress gzip \ - --compress-cache ./.outputs/examples/huge-compress.cache \ - --sources-cache ./.outputs/examples/huge-sources.cache \ - --exclude-strip \ - --exclude-file-listing \ - --progress \ - "${@}" \ - # -!! - -<< examples / huge / serve - exec -- "${ZRUN[@]}" ':: execute / server / debug' \ - --bind 127.153.215.30:8080 \ - --archive ./.outputs/examples/huge.cdb \ - --archive-mmap \ - --processes 1 \ - --threads 1 \ - --debug \ - "${@}" \ - # -!! - - - - -<< benchmark / dummy - test "${#}" -eq 0 - _outputs="$( exec -- readlink -e -- ./.outputs )" - mkdir -p -- "${_outputs}/binaries/release" - rm -f -- "${_outputs}/binaries/release/kawipiko-server-dummy" - "${ZRUN[@]}" ':: go / tool' \ - build \ - -mod readonly \ - -tags netgo \ - -ldflags 'all=-s -extld=gcc -extldflags=-static' \ - -gcflags 'all=-l=4' \ - -o "${_outputs}/binaries/release/kawipiko-server-dummy" \ - -- ./cmd/server-dummy.go \ - # - exec -- \ - sudo -u root -n -E -P -- \ - taskset -c 0,1 \ - nice -n -19 -- \ - ionice -c 2 -n 0 -- \ - chrt -r 10 \ - prlimit -n262144 -- \ - sudo -u "${USER}" -n -E -P -- \ - env -i -- \ - ./.outputs/binaries/release/kawipiko-server-dummy \ - "${@}" \ - # -!! +::// execute / ... +::// examples / ... +::// benchmark / ... +::// go / ... +::// documentation / ... + + +&& _/go.z-run +&& _/execute.z-run +&& _/examples.z-run +&& _/benchmark.z-run +&& _/workspace.z-run