[scripts] Port support for embedded sources and various build-metadata from z-run.

This commit is contained in:
Ciprian Dorin Craciun 2022-09-02 14:33:10 +03:00
parent bbca9d7096
commit ce6fc73524
7 changed files with 559 additions and 132 deletions

View file

@ -1,7 +1,6 @@
#!/dev/null
&&== env-fallback __BUILD_VERSION 0.1.0
&&== env-fallback __BUILD_QUICK false
@ -23,6 +22,8 @@
&& _/examples.z-run
&& _/benchmark.z-run
&& _/workspace.z-run
&& _/sources.z-run
&& _/documentation.z-run
&& _/tools.z-run
&& _/tls.z-run

View file

@ -49,6 +49,10 @@
test -d ./.publish/
"${ZRUN[@]}" ':: documentation / all / render / forced'
"${ZRUN[@]}" ':: sources / embedded / bundle'
if test "${__DEPLOY_NOGITHUB:-}" == true ; then
_github_publish=false
else
@ -56,7 +60,7 @@
fi
_build_version="${__BUILD_VERSION}"
_build_version="$( exec -- cat -- ./sources/embedded/build/version.txt )"
case "${_build_version}" in
( 0.0.0 | *.0 )
_build_release=false

View file

@ -0,0 +1,73 @@
#!/dev/null
<< documentation / all / render
test "${#}" -eq 0
"${ZRUN[@]}" ':: documentation / manuals / render'
"${ZRUN[@]}" ':: sources / embedded / documentation'
!!
<< documentation / all / render / forced
test "${#}" -eq 0
"${ZRUN[@]}" ':: documentation / manuals / render'
"${ZRUN[@]}" ':: sources / embedded / documentation'
!!
<< documentation / readme / open
test "${#}" -eq 0
exec -- x-www 'guest:*' open http://127.33.237.174:8080/
!!
<< documentation / readme / server
test "${#}" -eq 0
exec -- env -i -- ./.outputs/python/bin/restview --no-browser --listen 127.33.237.174:8080 --allowed-hosts '*' -- ./readme.rst
!!
<< documentation / manuals / render
test "${#}" -eq 0
_date="$( exec -- date -- '+%Y-%m-%d' )"
for _manual in archiver server ; do
printf -- '[ii] rendering `%s`...\n' "${_manual}" >&2
rst2man --strict \
< "./documentation/manuals/${_manual}.rst" \
| sed -r \
-e 's#^\.TH .*#.TH "KAWIPIKO\-'"${_manual^^}"'" "1" "'"${_date}"'" "volution.ro" "kawipiko"#' \
>| "./documentation/manuals/${_manual}.1.man.tmp" \
#
mv -T -- \
"./documentation/manuals/${_manual}.1.man.tmp" \
"./documentation/manuals/${_manual}.1.man" \
#
mandoc \
-c \
-man \
-T utf8 \
-I os=POSIX \
-O indent=2 \
-O width=78 \
-W error \
< "./documentation/manuals/${_manual}.1.man" \
| col -b -x \
>| "./documentation/manuals/${_manual}.txt.tmp" \
#
mv -T -- \
"./documentation/manuals/${_manual}.txt.tmp" \
"./documentation/manuals/${_manual}.txt" \
#
done
!!

View file

@ -13,6 +13,11 @@
exec -- env -i -- ./.outputs/binaries/release/kawipiko-server "${@}"
!!
<< execute / server / release / quick
"${ZRUN[@]}" ':: go / build / server / release / quick'
exec -- env -i -- ./.outputs/binaries/release/kawipiko-server "${@}"
!!
@ -26,6 +31,11 @@
exec -- env -i -- ./.outputs/binaries/release/kawipiko-archiver "${@}"
!!
<< execute / archiver / release / quick
"${ZRUN[@]}" ':: go / build / archiver / release / quick'
exec -- env -i -- ./.outputs/binaries/release/kawipiko-archiver "${@}"
!!
@ -39,6 +49,11 @@
exec -- env -i -- ./.outputs/binaries/release/kawipiko-wrapper "${@}"
!!
<< execute / wrapper / release / quick
"${ZRUN[@]}" ':: go / build / wrapper / release / quick'
exec -- env -i -- ./.outputs/binaries/release/kawipiko-wrapper "${@}"
!!
@ -52,3 +67,8 @@
exec -- env -i -- ./.outputs/binaries/release/kawipiko-server-dummy "${@}"
!!
<< execute / server-dummy / release / quick
"${ZRUN[@]}" ':: go / build / server-dummy / release / quick'
exec -- env -i -- ./.outputs/binaries/release/kawipiko-server-dummy "${@}"
!!

View file

@ -70,18 +70,27 @@
if test -e "${_outputs}/binaries/release/kawipiko-server" ; then
rm -- "${_outputs}/binaries/release/kawipiko-server"
fi
_tags=(
netgo
)
_flags=(
#? -buildmode exe
-tags netgo
)
_gcflags=()
_ldflags=(
-X "github.com/volution/kawipiko/embedded.BUILD_TARGET=release"
-X "github.com/volution/kawipiko/embedded.BUILD_GIT_HASH=$( exec -- cat -- ./.outputs/markers/build-git-hash.txt )"
)
if test "${__BUILD_QUICK:-false}" != true ; then
_flags+=(
-gcflags 'all=-l=4'
-ldflags 'all=-s'
-trimpath
-a
)
_flags+=( -trimpath -a )
_gcflags+=( 'all=-l=4' )
_ldflags+=( 'all=-s' )
fi
_flags+=(
-tags "$( printf -- '%s,' "${_tags[@]}" )"
-gcflags "$( printf -- '%s ' "${_gcflags[@]}" )"
-ldflags "$( printf -- '%s ' "${_ldflags[@]}" )"
)
exec -- \
nodaemon \
nice -n 19 -- \
@ -125,18 +134,27 @@
if test -e "${_outputs}/binaries/release/kawipiko-archiver" ; then
rm -- "${_outputs}/binaries/release/kawipiko-archiver"
fi
_tags=(
netgo
)
_flags=(
#? -buildmode exe
-tags netgo
)
_gcflags=()
_ldflags=(
-X "github.com/volution/kawipiko/embedded.BUILD_TARGET=release"
-X "github.com/volution/kawipiko/embedded.BUILD_GIT_HASH=$( exec -- cat -- ./.outputs/markers/build-git-hash.txt )"
)
if test "${__BUILD_QUICK:-false}" != true ; then
_flags+=(
-gcflags 'all=-l=4'
-ldflags 'all=-s'
-trimpath
-a
)
_flags+=( -trimpath -a )
_gcflags+=( 'all=-l=4' )
_ldflags+=( 'all=-s' )
fi
_flags+=(
-tags "$( printf -- '%s,' "${_tags[@]}" )"
-gcflags "$( printf -- '%s ' "${_gcflags[@]}" )"
-ldflags "$( printf -- '%s ' "${_ldflags[@]}" )"
)
exec -- \
nodaemon \
nice -n 19 -- \
@ -180,18 +198,27 @@
if test -e "${_outputs}/binaries/release/kawipiko-wrapper" ; then
rm -- "${_outputs}/binaries/release/kawipiko-wrapper"
fi
_tags=(
netgo
)
_flags=(
#? -buildmode exe
-tags netgo
)
_gcflags=()
_ldflags=(
-X "github.com/volution/kawipiko/embedded.BUILD_TARGET=release"
-X "github.com/volution/kawipiko/embedded.BUILD_GIT_HASH=$( exec -- cat -- ./.outputs/markers/build-git-hash.txt )"
)
if test "${__BUILD_QUICK:-false}" != true ; then
_flags+=(
-gcflags 'all=-l=4'
-ldflags 'all=-s'
-trimpath
-a
)
_flags+=( -trimpath -a )
_gcflags+=( 'all=-l=4' )
_ldflags+=( 'all=-s' )
fi
_flags+=(
-tags "$( printf -- '%s,' "${_tags[@]}" )"
-gcflags "$( printf -- '%s ' "${_gcflags[@]}" )"
-ldflags "$( printf -- '%s ' "${_ldflags[@]}" )"
)
exec -- \
nodaemon \
nice -n 19 -- \
@ -235,18 +262,27 @@
if test -e "${_outputs}/binaries/release/kawipiko-server-dummy" ; then
rm -- "${_outputs}/binaries/release/kawipiko-server-dummy"
fi
_tags=(
netgo
)
_flags=(
#? -buildmode exe
-tags netgo
)
_gcflags=()
_ldflags=(
-X "github.com/volution/kawipiko/embedded.BUILD_TARGET=release"
-X "github.com/volution/kawipiko/embedded.BUILD_GIT_HASH=$( exec -- cat -- ./.outputs/markers/build-git-hash.txt )"
)
if test "${__BUILD_QUICK:-false}" != true ; then
_flags+=(
-gcflags 'all=-l=4'
-ldflags 'all=-s'
-trimpath
-a
)
_flags+=( -trimpath -a )
_gcflags+=( 'all=-l=4' )
_ldflags+=( 'all=-s' )
fi
_flags+=(
-tags "$( printf -- '%s,' "${_tags[@]}" )"
-gcflags "$( printf -- '%s ' "${_gcflags[@]}" )"
-ldflags "$( printf -- '%s ' "${_ldflags[@]}" )"
)
exec -- \
nodaemon \
nice -n 19 -- \
@ -275,6 +311,27 @@
_component_0=all
fi
test "${#}" -eq 0
_tags=(
netgo
)
_flags=(
#? -buildmode exe
)
_gcflags=()
_ldflags=(
-X "github.com/volution/kawipiko/embedded.BUILD_TARGET=release"
-X "github.com/volution/kawipiko/embedded.BUILD_GIT_HASH=$( exec -- cat -- ./.outputs/markers/build-git-hash.txt )"
)
if test "${__BUILD_QUICK:-false}" != true ; then
_flags+=( -trimpath -a )
_gcflags+=( 'all=-l=4' )
_ldflags+=( 'all=-s' )
fi
_flags+=(
-tags "$( printf -- '%s,' "${_tags[@]}" )"
-gcflags "$( printf -- '%s ' "${_gcflags[@]}" )"
-ldflags "$( printf -- '%s ' "${_ldflags[@]}" )"
)
_outputs="$( exec -- readlink -e -- ./.outputs )"
for _platform in linux--x86_64 darwin--x86_64 darwin--aarch64 openbsd--x86_64 freebsd--x86_64 android--aarch64 ; do
if test "${_platform_0}" != all -a "${_platform_0}" != "${_platform}" ; then
@ -299,18 +356,6 @@
fi
test ! -e "${_outputs}/binaries/publish/kawipiko-${_component}--${_platform}"
printf -- '[ii] building `%s` for `%s`...\n' "${_component}" "${_platform}" >&2
_flags=(
#? -buildmode exe
-tags netgo
)
if test "${__BUILD_QUICK:-false}" != true ; then
_flags+=(
-gcflags 'all=-l=4'
-ldflags 'all=-s'
-trimpath
-a
)
fi
case "${_platform}" in
( linux--x86_64 )
export -- GOOS=linux

371
scripts/sources.z-run Normal file
View file

@ -0,0 +1,371 @@
#!/dev/null
::// sources / ...
<< sources / codes / duplicates
test "${#}" -eq 0
exec -- xargs \
-r -d '\n' -I {} \
-a <(
grep \
-o \
-P \
-e '(?<=\[)[0-9a-f]{8}(?=\])|(?<=0x)[0-9a-f]{8}(?=[^0-9a-zA-Z]|$)' \
-h \
-r ./sources \
--include '*.go' \
| sort \
| uniq -d
) \
-- \
grep \
-P \
-e '(?<=\[){}(?=\])|(?<=0x){}(?=[^0-9a-zA-Z]|$)' \
-r ./sources \
-n \
--include '*.go' \
--color \
#
!!
<< sources / embedded / prepare
test "${#}" -eq 0
"${ZRUN[@]}" ':: sources / embedded / documentation'
"${ZRUN[@]}" ':: sources / embedded / sbom'
"${ZRUN[@]}" ':: sources / embedded / hash'
!!
<< sources / embedded / documentation
test "${#}" -eq 0
if ! cmp -s -- ./documentation/manuals/server.txt ./sources/cmd/server/manual.txt ; then
cp -T -- ./documentation/manuals/server.txt ./sources/cmd/server/manual.txt
fi
if ! cmp -s -- ./documentation/manuals/archiver.txt ./sources/cmd/archiver/manual.txt ; then
cp -T -- ./documentation/manuals/archiver.txt ./sources/cmd/archiver/manual.txt
fi
!!
<< sources / embedded / sbom
test "${#}" -eq 0
if ! cmp -s -- ./documentation/sbom/sbom.txt ./sources/embedded/documentation/sbom.txt ; then
cp -T -- ./documentation/sbom/sbom.txt ./sources/embedded/documentation/sbom.txt
fi
if ! cmp -s -- ./documentation/sbom/sbom.html ./sources/embedded/documentation/sbom.html ; then
cp -T -- ./documentation/sbom/sbom.html ./sources/embedded/documentation/sbom.html
fi
if ! cmp -s -- ./documentation/sbom/cyclonedx.json ./sources/embedded/documentation/sbom.json ; then
cp -T -- ./documentation/sbom/cyclonedx.json ./sources/embedded/documentation/sbom.json
fi
!!
<< sources / embedded / hash
test "${#}" -eq 0
if test -e ./.git ; then
_git_hash="$( exec -- git rev-parse HEAD )"
if ! git diff --quiet || ! git diff --quiet --cached ; then
_git_hash+="-uncommitted"
fi
else
_git_hash='unknown'
fi
if test -s ./sources/embedded/build/sources.cpio.gz ; then
rm -- ./sources/embedded/build/sources.cpio.gz
touch -- ./sources/embedded/build/sources.cpio.gz
fi
execline-cd ./sources \
find . \
-type f \
-not -path './embedded/build/*' \
-printf '%P\0' \
| LC_ALL=C sort -z \
>| ./.outputs/markers/build-sources.list.tmp \
#
execline-cd ./sources \
xargs -0 -I {} -- md5sum -b -z -- {} \
< ./.outputs/markers/build-sources.list.tmp \
| tr '\0' '\n' \
>| ./.outputs/markers/build-sources.md5.tmp \
#
md5sum -b \
< ./.outputs/markers/build-sources.md5.tmp \
| cut -f 1 -d ' ' \
>| ./.outputs/markers/build-sources.hash.tmp \
#
if ! cmp -s -- ./.outputs/markers/build-sources.hash.tmp ./sources/embedded/build/sources.hash ; then
_build_number="$( exec -- cat -- ./sources/embedded/build/number.txt )"
_build_number="$(( _build_number + 1 ))"
_build_timestamp="$( exec -- date -- '+%Y-%m-%d-%H-%M-%S' )"
printf -- '%d\n' "${_build_number}" >| ./sources/embedded/build/number.txt.tmp
printf -- '%s\n' "${_build_timestamp}" >| ./sources/embedded/build/timestamp.txt.tmp
mv -T -- ./sources/embedded/build/number.txt.tmp ./sources/embedded/build/number.txt
mv -T -- ./sources/embedded/build/timestamp.txt.tmp ./sources/embedded/build/timestamp.txt
mv -T -- ./.outputs/markers/build-sources.list.tmp ./.outputs/markers/build-sources.list
mv -T -- ./.outputs/markers/build-sources.md5.tmp ./sources/embedded/build/sources.md5
mv -T -- ./.outputs/markers/build-sources.hash.tmp ./sources/embedded/build/sources.hash
else
rm -- ./.outputs/markers/build-sources.list.tmp
rm -- ./.outputs/markers/build-sources.md5.tmp
rm -- ./.outputs/markers/build-sources.hash.tmp
fi
printf -- '%s\n' "${_git_hash}" >| ./.outputs/markers/build-git-hash.txt.tmp
mv -T -- ./.outputs/markers/build-git-hash.txt.tmp ./.outputs/markers/build-git-hash.txt
!!
<< sources / embedded / bundle
test "${#}" -eq 0
"${ZRUN[@]}" ':: sources / embedded / prepare'
if test -s ./sources/embedded/build/sources.cpio.gz ; then
rm -- ./sources/embedded/build/sources.cpio.gz
fi
touch --reference ./sources/embedded/build/timestamp.txt -- ./sources/embedded/build/sources.cpio.gz
execline-cd ./sources \
find . \
-type f \
-print0 \
| LC_ALL=C sort -z \
| execline-cd ./sources \
cpio \
--create \
--format newc \
--no-absolute-filenames \
--quiet \
--warning all \
--owner +65534:+65534 \
--ignore-devno \
--renumber-inodes \
--block-size 1 \
--null \
| gzip \
--best \
--rsyncable \
--no-name \
>| ./.outputs/sources.cpio.gz.tmp \
#
mv -T -- ./.outputs/sources.cpio.gz.tmp ./sources/embedded/build/sources.cpio.gz
!!
<< sources / watch
test "${#}" -ge 1
_command="${1}"
shift -- 1
_delegate=()
case "${_command}" in
( '::'* )
_delegate+=( "${ZRUN[@]}" "${_command}" )
;;
( /* )
_delegate+=( "${_command}" )
;;
( * )
printf -- '[ee] invalid delegate `%s`; aborting!\n' "${_delegate}"
exit -- 1
;;
esac
_sources="$( exec -- readlink -e -- ./sources )"
exec -- watchexec \
\
--watch="${_sources}" \
\
--filter="${_sources}/go.*" \
--filter="${_sources}/cmd/**" \
--filter="${_sources}/lib/**" \
--filter="${_sources}/embedded/*.go" \
--filter="${_sources}/embedded/documentation/**" \
\
--restart \
\
--no-shell \
\
--no-default-ignore \
--no-vcs-ignore \
--no-ignore \
\
-- \
\
"${ZRUN[@]}" ':: sources / watch / delegate' \
"${_delegate[@]}" \
"${@}" \
#
!!
--<< sources / watch / delegate
test "${#}" -ge 1
_timestamp="$( exec -- date -- '+%H:%M:%S' )"
printf -- '\n[--] --------[ %s ]------------------------------------------------------\n\n' "${_timestamp}" >&2
exec -- "${@}"
!!
<< sources / generate / sbom
test "${#}" -eq 0
execline-cd ./sources \
cyclonedx-gomod mod -json -licenses \
>| ./documentation/sbom/cyclonedx.json \
#
jq \
'
.components[]
| .licenses = [
(.evidence // {})
| (.licenses // [])
| .[]
| .license.id
]
| .externalReferences = (
(.externalReferences // [])
| group_by (.type)
| map ({
key : .[0].type,
value : . | map (del (.type))
})
| from_entries
)
| .vcs_url = (
.externalReferences.vcs
| (if . != null then .[0].url else "" end)
)
' \
< ./documentation/sbom/cyclonedx.json \
| jq \
'
. as $component
| {
name : .name,
version : .version,
purl : .purl,
license :
.licenses
| join (" | ")
| (
({
}[$component.name])
// ({
"" : "{unknown}",
}[.])
// .
),
vcs :
.vcs_url
| (
({
"go.starlark.net" : "https://github.com/google/starlark-go",
"golang.org/x/sync" : "https://github.com/golang/sync",
"golang.org/x/sys" : "https://github.com/golang/sys",
"golang.org/x/term" : "https://github.com/golang/term",
"golang.org/x/text" : "https://github.com/golang/text",
}[$component.name])
// ({
"" : "{unknown}",
"<no value>" : "{unknown}",
}[.])
// .
),
}
' \
| jq --slurp . \
>| ./documentation/sbom/sbom.json \
#
gomplate \
-c .=stdin:///.json \
--in \
'
## SBOM (Software Bill of Materials)
| Dependency | Version | License (SPDX) | VCS URL |
| --- | --- | --- | --- |
{{ range $_, $_component := . -}}
| `{{ $_component.name }}` | `{{ $_component.version }}` | `{{ $_component.license }}` | <{{ $_component.vcs }}> |
{{ end }}
' \
< ./documentation/sbom/sbom.json \
>| ./documentation/sbom/sbom.md \
#
cmark-gfm \
--to html \
--extension autolink \
--extension table \
--validate-utf8 \
--unsafe \
< ./documentation/sbom/sbom.md \
>| ./documentation/sbom/sbom.html \
#
w3m \
-dump \
-T text/html \
-cols 160 \
-no-graph \
-I utf-8 \
-O utf-8 \
-o tabstop=4 \
-o indent_incr=4 \
-o display_link_number=false \
< ./documentation/sbom/sbom.html \
>| ./documentation/sbom/sbom.txt \
#
!!

View file

@ -37,6 +37,7 @@
binaries/publish
binaries/analyze
examples
markers
)
for _folder in "${_folders[@]}" ; do
@ -63,6 +64,8 @@
!!
<< workspace / initialize / go
test "${#}" -eq 0
@ -124,93 +127,3 @@
./.outputs/python/bin/pip install --compile -- restview
!!
<< workspace / sources / codes / duplicates
test "${#}" -eq 0
exec -- xargs \
-r -d '\n' -I {} \
-a <(
grep \
-o \
-P \
-e '(?<=\[)[0-9a-f]{8}(?=\])|(?<=0x)[0-9a-f]{8}(?=[^0-9a-zA-Z]|$)' \
-h \
-r ./sources \
--include '*.go' \
| sort \
| uniq -d
) \
-- \
grep \
-P \
-e '(?<=\[){}(?=\])|(?<=0x){}(?=[^0-9a-zA-Z]|$)' \
-r ./sources \
-n \
--include '*.go' \
--color \
#
!!
<< documentation / readme / open
test "${#}" -eq 0
exec -- x-www 'guest:*' open http://127.33.237.174:8080/
!!
<< documentation / readme / server
test "${#}" -eq 0
exec -- env -i -- ./.outputs/python/bin/restview --no-browser --listen 127.33.237.174:8080 --allowed-hosts '*' -- ./readme.rst
!!
<< documentation / manuals / render
test "${#}" -eq 0
_date="$( exec -- date -- '+%Y-%m-%d' )"
for _manual in archiver server ; do
printf -- '[ii] rendering `%s`...\n' "${_manual}" >&2
rst2man --strict \
< "./documentation/manuals/${_manual}.rst" \
| sed -r \
-e 's#^\.TH .*#.TH "KAWIPIKO\-'"${_manual^^}"'" "1" "'"${_date}"'" "volution.ro" "kawipiko"#' \
>| "./documentation/manuals/${_manual}.1.man.tmp" \
#
mv -T -- \
"./documentation/manuals/${_manual}.1.man.tmp" \
"./documentation/manuals/${_manual}.1.man" \
#
mandoc \
-c \
-man \
-T utf8 \
-I os=POSIX \
-O indent=2 \
-O width=78 \
-W error \
< "./documentation/manuals/${_manual}.1.man" \
| col -b -x \
>| "./documentation/manuals/${_manual}.txt.tmp" \
#
mv -T -- \
"./documentation/manuals/${_manual}.txt.tmp" \
"./documentation/manuals/${_manual}.txt" \
#
cp -T -- \
"./documentation/manuals/${_manual}.txt" \
"./sources/cmd/${_manual}/manual.txt" \
#
done
!!