diff --git a/scripts/_z-run b/scripts/_z-run index 32186d8..bbea90a 100644 --- a/scripts/_z-run +++ b/scripts/_z-run @@ -1,7 +1,6 @@ #!/dev/null -&&== env-fallback __BUILD_VERSION 0.1.0 &&== env-fallback __BUILD_QUICK false @@ -23,6 +22,8 @@ && _/examples.z-run && _/benchmark.z-run && _/workspace.z-run +&& _/sources.z-run +&& _/documentation.z-run && _/tools.z-run && _/tls.z-run diff --git a/scripts/deploy.z-run b/scripts/deploy.z-run index 7f2996f..f66ebec 100644 --- a/scripts/deploy.z-run +++ b/scripts/deploy.z-run @@ -49,6 +49,10 @@ test -d ./.publish/ + "${ZRUN[@]}" ':: documentation / all / render / forced' + "${ZRUN[@]}" ':: sources / embedded / bundle' + + if test "${__DEPLOY_NOGITHUB:-}" == true ; then _github_publish=false else @@ -56,7 +60,7 @@ fi - _build_version="${__BUILD_VERSION}" + _build_version="$( exec -- cat -- ./sources/embedded/build/version.txt )" case "${_build_version}" in ( 0.0.0 | *.0 ) _build_release=false diff --git a/scripts/documentation.z-run b/scripts/documentation.z-run new file mode 100644 index 0000000..896a537 --- /dev/null +++ b/scripts/documentation.z-run @@ -0,0 +1,73 @@ +#!/dev/null + + + + +<< documentation / all / render + test "${#}" -eq 0 + "${ZRUN[@]}" ':: documentation / manuals / render' + "${ZRUN[@]}" ':: sources / embedded / documentation' +!! + +<< documentation / all / render / forced + test "${#}" -eq 0 + "${ZRUN[@]}" ':: documentation / manuals / render' + "${ZRUN[@]}" ':: sources / embedded / documentation' +!! + + + + +<< documentation / readme / open + test "${#}" -eq 0 + exec -- x-www 'guest:*' open http://127.33.237.174:8080/ +!! + +<< documentation / readme / server + test "${#}" -eq 0 + exec -- env -i -- ./.outputs/python/bin/restview --no-browser --listen 127.33.237.174:8080 --allowed-hosts '*' -- ./readme.rst +!! + + +<< documentation / manuals / render + test "${#}" -eq 0 + + _date="$( exec -- date -- '+%Y-%m-%d' )" + + for _manual in archiver server ; do + + printf -- '[ii] rendering `%s`...\n' "${_manual}" >&2 + + rst2man --strict \ + < "./documentation/manuals/${_manual}.rst" \ + | sed -r \ + -e 's#^\.TH .*#.TH "KAWIPIKO\-'"${_manual^^}"'" "1" "'"${_date}"'" "volution.ro" "kawipiko"#' \ + >| "./documentation/manuals/${_manual}.1.man.tmp" \ + # + + mv -T -- \ + "./documentation/manuals/${_manual}.1.man.tmp" \ + "./documentation/manuals/${_manual}.1.man" \ + # + + mandoc \ + -c \ + -man \ + -T utf8 \ + -I os=POSIX \ + -O indent=2 \ + -O width=78 \ + -W error \ + < "./documentation/manuals/${_manual}.1.man" \ + | col -b -x \ + >| "./documentation/manuals/${_manual}.txt.tmp" \ + # + + mv -T -- \ + "./documentation/manuals/${_manual}.txt.tmp" \ + "./documentation/manuals/${_manual}.txt" \ + # + + done +!! + diff --git a/scripts/execute.z-run b/scripts/execute.z-run index 42cb4e9..22108d4 100644 --- a/scripts/execute.z-run +++ b/scripts/execute.z-run @@ -13,6 +13,11 @@ exec -- env -i -- ./.outputs/binaries/release/kawipiko-server "${@}" !! +<< execute / server / release / quick + "${ZRUN[@]}" ':: go / build / server / release / quick' + exec -- env -i -- ./.outputs/binaries/release/kawipiko-server "${@}" +!! + @@ -26,6 +31,11 @@ exec -- env -i -- ./.outputs/binaries/release/kawipiko-archiver "${@}" !! +<< execute / archiver / release / quick + "${ZRUN[@]}" ':: go / build / archiver / release / quick' + exec -- env -i -- ./.outputs/binaries/release/kawipiko-archiver "${@}" +!! + @@ -39,6 +49,11 @@ exec -- env -i -- ./.outputs/binaries/release/kawipiko-wrapper "${@}" !! +<< execute / wrapper / release / quick + "${ZRUN[@]}" ':: go / build / wrapper / release / quick' + exec -- env -i -- ./.outputs/binaries/release/kawipiko-wrapper "${@}" +!! + @@ -52,3 +67,8 @@ exec -- env -i -- ./.outputs/binaries/release/kawipiko-server-dummy "${@}" !! +<< execute / server-dummy / release / quick + "${ZRUN[@]}" ':: go / build / server-dummy / release / quick' + exec -- env -i -- ./.outputs/binaries/release/kawipiko-server-dummy "${@}" +!! + diff --git a/scripts/go.z-run b/scripts/go.z-run index 0fa0e5c..8d9f913 100644 --- a/scripts/go.z-run +++ b/scripts/go.z-run @@ -70,18 +70,27 @@ if test -e "${_outputs}/binaries/release/kawipiko-server" ; then rm -- "${_outputs}/binaries/release/kawipiko-server" fi + _tags=( + netgo + ) _flags=( #? -buildmode exe - -tags netgo + ) + _gcflags=() + _ldflags=( + -X "github.com/volution/kawipiko/embedded.BUILD_TARGET=release" + -X "github.com/volution/kawipiko/embedded.BUILD_GIT_HASH=$( exec -- cat -- ./.outputs/markers/build-git-hash.txt )" ) if test "${__BUILD_QUICK:-false}" != true ; then - _flags+=( - -gcflags 'all=-l=4' - -ldflags 'all=-s' - -trimpath - -a - ) + _flags+=( -trimpath -a ) + _gcflags+=( 'all=-l=4' ) + _ldflags+=( 'all=-s' ) fi + _flags+=( + -tags "$( printf -- '%s,' "${_tags[@]}" )" + -gcflags "$( printf -- '%s ' "${_gcflags[@]}" )" + -ldflags "$( printf -- '%s ' "${_ldflags[@]}" )" + ) exec -- \ nodaemon \ nice -n 19 -- \ @@ -125,18 +134,27 @@ if test -e "${_outputs}/binaries/release/kawipiko-archiver" ; then rm -- "${_outputs}/binaries/release/kawipiko-archiver" fi + _tags=( + netgo + ) _flags=( #? -buildmode exe - -tags netgo + ) + _gcflags=() + _ldflags=( + -X "github.com/volution/kawipiko/embedded.BUILD_TARGET=release" + -X "github.com/volution/kawipiko/embedded.BUILD_GIT_HASH=$( exec -- cat -- ./.outputs/markers/build-git-hash.txt )" ) if test "${__BUILD_QUICK:-false}" != true ; then - _flags+=( - -gcflags 'all=-l=4' - -ldflags 'all=-s' - -trimpath - -a - ) + _flags+=( -trimpath -a ) + _gcflags+=( 'all=-l=4' ) + _ldflags+=( 'all=-s' ) fi + _flags+=( + -tags "$( printf -- '%s,' "${_tags[@]}" )" + -gcflags "$( printf -- '%s ' "${_gcflags[@]}" )" + -ldflags "$( printf -- '%s ' "${_ldflags[@]}" )" + ) exec -- \ nodaemon \ nice -n 19 -- \ @@ -180,18 +198,27 @@ if test -e "${_outputs}/binaries/release/kawipiko-wrapper" ; then rm -- "${_outputs}/binaries/release/kawipiko-wrapper" fi + _tags=( + netgo + ) _flags=( #? -buildmode exe - -tags netgo + ) + _gcflags=() + _ldflags=( + -X "github.com/volution/kawipiko/embedded.BUILD_TARGET=release" + -X "github.com/volution/kawipiko/embedded.BUILD_GIT_HASH=$( exec -- cat -- ./.outputs/markers/build-git-hash.txt )" ) if test "${__BUILD_QUICK:-false}" != true ; then - _flags+=( - -gcflags 'all=-l=4' - -ldflags 'all=-s' - -trimpath - -a - ) + _flags+=( -trimpath -a ) + _gcflags+=( 'all=-l=4' ) + _ldflags+=( 'all=-s' ) fi + _flags+=( + -tags "$( printf -- '%s,' "${_tags[@]}" )" + -gcflags "$( printf -- '%s ' "${_gcflags[@]}" )" + -ldflags "$( printf -- '%s ' "${_ldflags[@]}" )" + ) exec -- \ nodaemon \ nice -n 19 -- \ @@ -235,18 +262,27 @@ if test -e "${_outputs}/binaries/release/kawipiko-server-dummy" ; then rm -- "${_outputs}/binaries/release/kawipiko-server-dummy" fi + _tags=( + netgo + ) _flags=( #? -buildmode exe - -tags netgo + ) + _gcflags=() + _ldflags=( + -X "github.com/volution/kawipiko/embedded.BUILD_TARGET=release" + -X "github.com/volution/kawipiko/embedded.BUILD_GIT_HASH=$( exec -- cat -- ./.outputs/markers/build-git-hash.txt )" ) if test "${__BUILD_QUICK:-false}" != true ; then - _flags+=( - -gcflags 'all=-l=4' - -ldflags 'all=-s' - -trimpath - -a - ) + _flags+=( -trimpath -a ) + _gcflags+=( 'all=-l=4' ) + _ldflags+=( 'all=-s' ) fi + _flags+=( + -tags "$( printf -- '%s,' "${_tags[@]}" )" + -gcflags "$( printf -- '%s ' "${_gcflags[@]}" )" + -ldflags "$( printf -- '%s ' "${_ldflags[@]}" )" + ) exec -- \ nodaemon \ nice -n 19 -- \ @@ -275,6 +311,27 @@ _component_0=all fi test "${#}" -eq 0 + _tags=( + netgo + ) + _flags=( + #? -buildmode exe + ) + _gcflags=() + _ldflags=( + -X "github.com/volution/kawipiko/embedded.BUILD_TARGET=release" + -X "github.com/volution/kawipiko/embedded.BUILD_GIT_HASH=$( exec -- cat -- ./.outputs/markers/build-git-hash.txt )" + ) + if test "${__BUILD_QUICK:-false}" != true ; then + _flags+=( -trimpath -a ) + _gcflags+=( 'all=-l=4' ) + _ldflags+=( 'all=-s' ) + fi + _flags+=( + -tags "$( printf -- '%s,' "${_tags[@]}" )" + -gcflags "$( printf -- '%s ' "${_gcflags[@]}" )" + -ldflags "$( printf -- '%s ' "${_ldflags[@]}" )" + ) _outputs="$( exec -- readlink -e -- ./.outputs )" for _platform in linux--x86_64 darwin--x86_64 darwin--aarch64 openbsd--x86_64 freebsd--x86_64 android--aarch64 ; do if test "${_platform_0}" != all -a "${_platform_0}" != "${_platform}" ; then @@ -299,18 +356,6 @@ fi test ! -e "${_outputs}/binaries/publish/kawipiko-${_component}--${_platform}" printf -- '[ii] building `%s` for `%s`...\n' "${_component}" "${_platform}" >&2 - _flags=( - #? -buildmode exe - -tags netgo - ) - if test "${__BUILD_QUICK:-false}" != true ; then - _flags+=( - -gcflags 'all=-l=4' - -ldflags 'all=-s' - -trimpath - -a - ) - fi case "${_platform}" in ( linux--x86_64 ) export -- GOOS=linux diff --git a/scripts/sources.z-run b/scripts/sources.z-run new file mode 100644 index 0000000..753e87f --- /dev/null +++ b/scripts/sources.z-run @@ -0,0 +1,371 @@ +#!/dev/null + + + + +::// sources / ... + + + + +<< sources / codes / duplicates + test "${#}" -eq 0 + exec -- xargs \ + -r -d '\n' -I {} \ + -a <( + grep \ + -o \ + -P \ + -e '(?<=\[)[0-9a-f]{8}(?=\])|(?<=0x)[0-9a-f]{8}(?=[^0-9a-zA-Z]|$)' \ + -h \ + -r ./sources \ + --include '*.go' \ + | sort \ + | uniq -d + ) \ + -- \ + grep \ + -P \ + -e '(?<=\[){}(?=\])|(?<=0x){}(?=[^0-9a-zA-Z]|$)' \ + -r ./sources \ + -n \ + --include '*.go' \ + --color \ + # +!! + + + + +<< sources / embedded / prepare + + test "${#}" -eq 0 + + "${ZRUN[@]}" ':: sources / embedded / documentation' + "${ZRUN[@]}" ':: sources / embedded / sbom' + "${ZRUN[@]}" ':: sources / embedded / hash' +!! + + + + +<< sources / embedded / documentation + + test "${#}" -eq 0 + + if ! cmp -s -- ./documentation/manuals/server.txt ./sources/cmd/server/manual.txt ; then + cp -T -- ./documentation/manuals/server.txt ./sources/cmd/server/manual.txt + fi + + if ! cmp -s -- ./documentation/manuals/archiver.txt ./sources/cmd/archiver/manual.txt ; then + cp -T -- ./documentation/manuals/archiver.txt ./sources/cmd/archiver/manual.txt + fi +!! + + + + +<< sources / embedded / sbom + + test "${#}" -eq 0 + + if ! cmp -s -- ./documentation/sbom/sbom.txt ./sources/embedded/documentation/sbom.txt ; then + cp -T -- ./documentation/sbom/sbom.txt ./sources/embedded/documentation/sbom.txt + fi + + if ! cmp -s -- ./documentation/sbom/sbom.html ./sources/embedded/documentation/sbom.html ; then + cp -T -- ./documentation/sbom/sbom.html ./sources/embedded/documentation/sbom.html + fi + + if ! cmp -s -- ./documentation/sbom/cyclonedx.json ./sources/embedded/documentation/sbom.json ; then + cp -T -- ./documentation/sbom/cyclonedx.json ./sources/embedded/documentation/sbom.json + fi +!! + + + + +<< sources / embedded / hash + + test "${#}" -eq 0 + + if test -e ./.git ; then + _git_hash="$( exec -- git rev-parse HEAD )" + if ! git diff --quiet || ! git diff --quiet --cached ; then + _git_hash+="-uncommitted" + fi + else + _git_hash='unknown' + fi + + if test -s ./sources/embedded/build/sources.cpio.gz ; then + rm -- ./sources/embedded/build/sources.cpio.gz + touch -- ./sources/embedded/build/sources.cpio.gz + fi + + execline-cd ./sources \ + find . \ + -type f \ + -not -path './embedded/build/*' \ + -printf '%P\0' \ + | LC_ALL=C sort -z \ + >| ./.outputs/markers/build-sources.list.tmp \ + # + + execline-cd ./sources \ + xargs -0 -I {} -- md5sum -b -z -- {} \ + < ./.outputs/markers/build-sources.list.tmp \ + | tr '\0' '\n' \ + >| ./.outputs/markers/build-sources.md5.tmp \ + # + + md5sum -b \ + < ./.outputs/markers/build-sources.md5.tmp \ + | cut -f 1 -d ' ' \ + >| ./.outputs/markers/build-sources.hash.tmp \ + # + + if ! cmp -s -- ./.outputs/markers/build-sources.hash.tmp ./sources/embedded/build/sources.hash ; then + + _build_number="$( exec -- cat -- ./sources/embedded/build/number.txt )" + _build_number="$(( _build_number + 1 ))" + _build_timestamp="$( exec -- date -- '+%Y-%m-%d-%H-%M-%S' )" + + printf -- '%d\n' "${_build_number}" >| ./sources/embedded/build/number.txt.tmp + printf -- '%s\n' "${_build_timestamp}" >| ./sources/embedded/build/timestamp.txt.tmp + + mv -T -- ./sources/embedded/build/number.txt.tmp ./sources/embedded/build/number.txt + mv -T -- ./sources/embedded/build/timestamp.txt.tmp ./sources/embedded/build/timestamp.txt + + mv -T -- ./.outputs/markers/build-sources.list.tmp ./.outputs/markers/build-sources.list + + mv -T -- ./.outputs/markers/build-sources.md5.tmp ./sources/embedded/build/sources.md5 + mv -T -- ./.outputs/markers/build-sources.hash.tmp ./sources/embedded/build/sources.hash + + else + + rm -- ./.outputs/markers/build-sources.list.tmp + rm -- ./.outputs/markers/build-sources.md5.tmp + rm -- ./.outputs/markers/build-sources.hash.tmp + fi + + printf -- '%s\n' "${_git_hash}" >| ./.outputs/markers/build-git-hash.txt.tmp + mv -T -- ./.outputs/markers/build-git-hash.txt.tmp ./.outputs/markers/build-git-hash.txt +!! + + + + +<< sources / embedded / bundle + + test "${#}" -eq 0 + + "${ZRUN[@]}" ':: sources / embedded / prepare' + + if test -s ./sources/embedded/build/sources.cpio.gz ; then + rm -- ./sources/embedded/build/sources.cpio.gz + fi + touch --reference ./sources/embedded/build/timestamp.txt -- ./sources/embedded/build/sources.cpio.gz + + execline-cd ./sources \ + find . \ + -type f \ + -print0 \ + | LC_ALL=C sort -z \ + | execline-cd ./sources \ + cpio \ + --create \ + --format newc \ + --no-absolute-filenames \ + --quiet \ + --warning all \ + --owner +65534:+65534 \ + --ignore-devno \ + --renumber-inodes \ + --block-size 1 \ + --null \ + | gzip \ + --best \ + --rsyncable \ + --no-name \ + >| ./.outputs/sources.cpio.gz.tmp \ + # + + mv -T -- ./.outputs/sources.cpio.gz.tmp ./sources/embedded/build/sources.cpio.gz +!! + + + + +<< sources / watch + + test "${#}" -ge 1 + _command="${1}" + shift -- 1 + + _delegate=() + case "${_command}" in + ( '::'* ) + _delegate+=( "${ZRUN[@]}" "${_command}" ) + ;; + ( /* ) + _delegate+=( "${_command}" ) + ;; + ( * ) + printf -- '[ee] invalid delegate `%s`; aborting!\n' "${_delegate}" + exit -- 1 + ;; + esac + + _sources="$( exec -- readlink -e -- ./sources )" + + exec -- watchexec \ + \ + --watch="${_sources}" \ + \ + --filter="${_sources}/go.*" \ + --filter="${_sources}/cmd/**" \ + --filter="${_sources}/lib/**" \ + --filter="${_sources}/embedded/*.go" \ + --filter="${_sources}/embedded/documentation/**" \ + \ + --restart \ + \ + --no-shell \ + \ + --no-default-ignore \ + --no-vcs-ignore \ + --no-ignore \ + \ + -- \ + \ + "${ZRUN[@]}" ':: sources / watch / delegate' \ + "${_delegate[@]}" \ + "${@}" \ + # +!! + + +--<< sources / watch / delegate + test "${#}" -ge 1 + _timestamp="$( exec -- date -- '+%H:%M:%S' )" + printf -- '\n[--] --------[ %s ]------------------------------------------------------\n\n' "${_timestamp}" >&2 + exec -- "${@}" +!! + + + + +<< sources / generate / sbom + test "${#}" -eq 0 + + execline-cd ./sources \ + cyclonedx-gomod mod -json -licenses \ + >| ./documentation/sbom/cyclonedx.json \ + # + + jq \ + ' + .components[] + | .licenses = [ + (.evidence // {}) + | (.licenses // []) + | .[] + | .license.id + ] + | .externalReferences = ( + (.externalReferences // []) + | group_by (.type) + | map ({ + key : .[0].type, + value : . | map (del (.type)) + }) + | from_entries + ) + | .vcs_url = ( + .externalReferences.vcs + | (if . != null then .[0].url else "" end) + ) + ' \ + < ./documentation/sbom/cyclonedx.json \ + | jq \ + ' + . as $component + | { + name : .name, + version : .version, + purl : .purl, + license : + .licenses + | join (" | ") + | ( + ({ + }[$component.name]) + // ({ + "" : "{unknown}", + }[.]) + // . + ), + vcs : + .vcs_url + | ( + ({ + "go.starlark.net" : "https://github.com/google/starlark-go", + "golang.org/x/sync" : "https://github.com/golang/sync", + "golang.org/x/sys" : "https://github.com/golang/sys", + "golang.org/x/term" : "https://github.com/golang/term", + "golang.org/x/text" : "https://github.com/golang/text", + }[$component.name]) + // ({ + "" : "{unknown}", + "" : "{unknown}", + }[.]) + // . + ), + } + ' \ + | jq --slurp . \ + >| ./documentation/sbom/sbom.json \ + # + + gomplate \ + -c .=stdin:///.json \ + --in \ + ' + ## SBOM (Software Bill of Materials) + + | Dependency | Version | License (SPDX) | VCS URL | + | --- | --- | --- | --- | + {{ range $_, $_component := . -}} + | `{{ $_component.name }}` | `{{ $_component.version }}` | `{{ $_component.license }}` | <{{ $_component.vcs }}> | + {{ end }} + ' \ + < ./documentation/sbom/sbom.json \ + >| ./documentation/sbom/sbom.md \ + # + + cmark-gfm \ + --to html \ + --extension autolink \ + --extension table \ + --validate-utf8 \ + --unsafe \ + < ./documentation/sbom/sbom.md \ + >| ./documentation/sbom/sbom.html \ + # + + w3m \ + -dump \ + -T text/html \ + -cols 160 \ + -no-graph \ + -I utf-8 \ + -O utf-8 \ + -o tabstop=4 \ + -o indent_incr=4 \ + -o display_link_number=false \ + < ./documentation/sbom/sbom.html \ + >| ./documentation/sbom/sbom.txt \ + # +!! + diff --git a/scripts/workspace.z-run b/scripts/workspace.z-run index 511c86d..de6caa8 100644 --- a/scripts/workspace.z-run +++ b/scripts/workspace.z-run @@ -37,6 +37,7 @@ binaries/publish binaries/analyze examples + markers ) for _folder in "${_folders[@]}" ; do @@ -63,6 +64,8 @@ !! + + << workspace / initialize / go test "${#}" -eq 0 @@ -124,93 +127,3 @@ ./.outputs/python/bin/pip install --compile -- restview !! - - - -<< workspace / sources / codes / duplicates - test "${#}" -eq 0 - exec -- xargs \ - -r -d '\n' -I {} \ - -a <( - grep \ - -o \ - -P \ - -e '(?<=\[)[0-9a-f]{8}(?=\])|(?<=0x)[0-9a-f]{8}(?=[^0-9a-zA-Z]|$)' \ - -h \ - -r ./sources \ - --include '*.go' \ - | sort \ - | uniq -d - ) \ - -- \ - grep \ - -P \ - -e '(?<=\[){}(?=\])|(?<=0x){}(?=[^0-9a-zA-Z]|$)' \ - -r ./sources \ - -n \ - --include '*.go' \ - --color \ - # -!! - - - - -<< documentation / readme / open - test "${#}" -eq 0 - exec -- x-www 'guest:*' open http://127.33.237.174:8080/ -!! - -<< documentation / readme / server - test "${#}" -eq 0 - exec -- env -i -- ./.outputs/python/bin/restview --no-browser --listen 127.33.237.174:8080 --allowed-hosts '*' -- ./readme.rst -!! - - -<< documentation / manuals / render - test "${#}" -eq 0 - - _date="$( exec -- date -- '+%Y-%m-%d' )" - - for _manual in archiver server ; do - - printf -- '[ii] rendering `%s`...\n' "${_manual}" >&2 - - rst2man --strict \ - < "./documentation/manuals/${_manual}.rst" \ - | sed -r \ - -e 's#^\.TH .*#.TH "KAWIPIKO\-'"${_manual^^}"'" "1" "'"${_date}"'" "volution.ro" "kawipiko"#' \ - >| "./documentation/manuals/${_manual}.1.man.tmp" \ - # - - mv -T -- \ - "./documentation/manuals/${_manual}.1.man.tmp" \ - "./documentation/manuals/${_manual}.1.man" \ - # - - mandoc \ - -c \ - -man \ - -T utf8 \ - -I os=POSIX \ - -O indent=2 \ - -O width=78 \ - -W error \ - < "./documentation/manuals/${_manual}.1.man" \ - | col -b -x \ - >| "./documentation/manuals/${_manual}.txt.tmp" \ - # - - mv -T -- \ - "./documentation/manuals/${_manual}.txt.tmp" \ - "./documentation/manuals/${_manual}.txt" \ - # - - cp -T -- \ - "./documentation/manuals/${_manual}.txt" \ - "./sources/cmd/${_manual}/manual.txt" \ - # - - done -!! -