From d382eeeb9d3b795c16533c5f64ab1455e774f925 Mon Sep 17 00:00:00 2001 From: Joel Challis Date: Tue, 5 Nov 2019 00:18:57 +0000 Subject: [PATCH] CI - Fix change detection (#7038) * Fix travis change detection * Fix travis change detection - add back python ignore --- .travis.yml | 1 + util/travis_build.sh | 79 +++++++++++++++--------------------- util/travis_compiled_push.sh | 1 + util/travis_docs.sh | 1 + util/travis_push.sh | 5 --- util/travis_test.sh | 27 +++--------- util/travis_utils.sh | 24 +++++++++++ 7 files changed, 64 insertions(+), 74 deletions(-) create mode 100755 util/travis_utils.sh diff --git a/.travis.yml b/.travis.yml index 84fa63faf3..bfac998c11 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,7 @@ install: - npm install -g moxygen script: - git rev-parse --short HEAD + - git diff --name-only HEAD $TRAVIS_BRANCH - bash util/travis_test.sh - bash util/travis_build.sh - bash util/travis_docs.sh diff --git a/util/travis_build.sh b/util/travis_build.sh index 225c8548f5..6d5dbed684 100755 --- a/util/travis_build.sh +++ b/util/travis_build.sh @@ -1,57 +1,42 @@ #!/bin/bash -# if docker is installed - patch calls to within the qmk docker image -if command -v docker >/dev/null; then - function make() { - docker run --rm -e MAKEFLAGS="$MAKEFLAGS" -w /qmk_firmware/ -v "$PWD":/qmk_firmware --user $(id -u):$(id -g) qmkfm/base_container make "$@" - } +source util/travis_utils.sh + +NUM_CORE_CHANGES=$(echo "$QMK_CHANGES" | grep -Ecv -e '^(docs/)' -e '^(keyboards/)' -e '^(layouts/)' -e '^(util/)' -e '^(lib/python/)' -e '^(bin/qmk)' -e '^(requirements.txt)' -e '(.travis.yml)') + +if [[ "$TRAVIS_COMMIT_MESSAGE" == *"[skip build]"* ]]; then + echo "Skipping due to commit message" + exit 0 fi -# test force push -#TRAVIS_COMMIT_RANGE="c287f1bfc5c8...81f62atc4c1d" +if [ "$LOCAL_BRANCH" == "master" ] || [ "$NUM_CORE_CHANGES" != "0" ]; then + echo "Making default keymaps for all keyboards" + make all:default + exit $? +fi -TRAVIS_COMMIT_MESSAGE="${TRAVIS_COMMIT_MESSAGE:-none}" -TRAVIS_COMMIT_RANGE="${TRAVIS_COMMIT_RANGE:-HEAD~1..HEAD}" -MAKE_ALL="make all:default" +exit_code=0 -if [[ "$TRAVIS_COMMIT_MESSAGE" != *"[skip build]"* ]] ; then - exit_code=0 - git diff --name-only -n 1 ${TRAVIS_COMMIT_RANGE} - if [ $? -eq 128 ]; then - # We don't know what changed so just build the default keymaps - echo "Making default keymaps for all keyboards (fallback)" - eval $MAKE_ALL +for KB in $(make list-keyboards); do + KEYBOARD_CHANGES=$(echo "$QMK_CHANGES" | grep -E '^(keyboards/'${KB}'/)') + if [[ -z "$KEYBOARD_CHANGES" ]]; then + # skip as no changes for this keyboard + continue + fi + + KEYMAP_ONLY=$(echo "$KEYBOARD_CHANGES" | grep -cv /keymaps/) + if [[ $KEYMAP_ONLY -gt 0 ]]; then + echo "Making all keymaps for $KB" + make ${KB}:all : $((exit_code = $exit_code + $?)) else - NEFM=$(git diff --name-only -n 1 ${TRAVIS_COMMIT_RANGE} | grep -Ev '^(keyboards/)' | grep -Ev '^(docs/)' | grep -Ev '^(lib/python/)' | grep -Ev '^(bin/qmk)' | grep -Ev '^(requirements.txt)' | grep -Ev '^(util/)' | wc -l) - BRANCH=$(git rev-parse --abbrev-ref HEAD) - # is this branch master or a "non docs, non keyboards" change - if [ $NEFM -gt 0 -o "$BRANCH" = "master" ]; then - echo "Making default keymaps for all keyboards" - eval $MAKE_ALL + CHANGED_KEYMAPS=$(echo "$KEYBOARD_CHANGES" | grep -oP '(?<=keyboards/'${KB}'/keymaps/)([a-zA-Z0-9_-]+)(?=\/)') + for KM in $CHANGED_KEYMAPS ; do + echo "Making $KM for $KB" + make ${KB}:${KM} : $((exit_code = $exit_code + $?)) - else - # keyboards project format - # /keyboards/board1/rev/keymaps/ - # /keyboards/board2/keymaps/ - # ensure we strip everything off after and including the keymaps folder to get board and/or revision - MKB=$(git diff --name-only -n 1 ${TRAVIS_COMMIT_RANGE} | grep -oP '(?<=keyboards\/)([a-zA-Z0-9_\/]+)(?=\/)' | sed 's^/keymaps/.*^^' | sort -u) - for KB in $MKB ; do - KEYMAP_ONLY=$(git diff --name-only -n 1 ${TRAVIS_COMMIT_RANGE} | grep -Ev '^(keyboards/'${KB}'/keymaps/)' | wc -l) - if [[ $KEYMAP_ONLY -gt 0 ]]; then - echo "Making all keymaps for $KB" - make ${KB}:all - : $((exit_code = $exit_code + $?)) - else - MKM=$(git diff --name-only -n 1 ${TRAVIS_COMMIT_RANGE} | grep -oP '(?<=keyboards/'${KB}'/keymaps/)([a-zA-Z0-9_]+)(?=\/)' | sort -u) - for KM in $MKM ; do - echo "Making $KM for $KB" - make ${KB}:${KM} - : $((exit_code = $exit_code + $?)) - done - fi - done - fi + done fi - exit $exit_code -fi +done + +exit $exit_code diff --git a/util/travis_compiled_push.sh b/util/travis_compiled_push.sh index 8ca65f21a0..e0490cd702 100755 --- a/util/travis_compiled_push.sh +++ b/util/travis_compiled_push.sh @@ -1,5 +1,6 @@ #!/bin/bash +source util/travis_utils.sh source util/travis_push.sh set -o errexit -o nounset diff --git a/util/travis_docs.sh b/util/travis_docs.sh index 5cfd63466d..93c2d71867 100755 --- a/util/travis_docs.sh +++ b/util/travis_docs.sh @@ -1,5 +1,6 @@ #!/bin/bash +source util/travis_utils.sh source util/travis_push.sh if [[ "$TRAVIS_COMMIT_MESSAGE" != *"[skip docs]"* ]] ; then diff --git a/util/travis_push.sh b/util/travis_push.sh index d100f9d48f..7f7d4c2c64 100755 --- a/util/travis_push.sh +++ b/util/travis_push.sh @@ -1,10 +1,5 @@ # Use this by sourcing it in your script. -TRAVIS_BRANCH="${TRAVIS_BRANCH:master}" -TRAVIS_PULL_REQUEST="${TRAVIS_PULL_REQUEST:false}" -TRAVIS_COMMIT_MESSAGE="${TRAVIS_COMMIT_MESSAGE:-none}" -TRAVIS_COMMIT_RANGE="${TRAVIS_COMMIT_RANGE:-HEAD~1..HEAD}" - if [[ "$TRAVIS_BRANCH" == "master" && "$TRAVIS_PULL_REQUEST" == "false" ]] ; then git config --global user.name "QMK Bot" git config --global user.email "hello@qmk.fm" diff --git a/util/travis_test.sh b/util/travis_test.sh index 9b7402c282..95991907ef 100644 --- a/util/travis_test.sh +++ b/util/travis_test.sh @@ -1,26 +1,9 @@ #!/bin/bash -# if docker is installed - patch calls to within the qmk docker image -if command -v docker >/dev/null; then - function make() { - docker run --rm -e MAKEFLAGS="$MAKEFLAGS" -w /qmk_firmware/ -v "$PWD":/qmk_firmware --user $(id -u):$(id -g) qmkfm/base_container make "$@" - } - function qmk() { - docker run --rm -w /qmk_firmware/ -v "$PWD":/qmk_firmware --user $(id -u):$(id -g) qmkfm/base_container bin/qmk "$@" - } -fi +source util/travis_utils.sh -TRAVIS_COMMIT_MESSAGE="${TRAVIS_COMMIT_MESSAGE:-none}" -TRAVIS_COMMIT_RANGE="${TRAVIS_COMMIT_RANGE:-HEAD~1..HEAD}" - -# test force push -#TRAVIS_COMMIT_RANGE="c287f1bfc5c8...81f62atc4c1d" - -BRANCH=$(git rev-parse --abbrev-ref HEAD) -CHANGES=$(git diff --name-only -n 1 ${TRAVIS_COMMIT_RANGE}) - -NUM_CORE_CHANGES=$(echo "$CHANGES" | grep -Ecv -e '^(docs/)' -e '^(keyboards/)' -e '^(layouts/)') -NUM_PY_CHANGES=$(echo "$CHANGES" | grep -Ec -e '^(lib/python/)' -e '^(bin/qmk)') +NUM_CORE_CHANGES=$(echo "$QMK_CHANGES" | grep -Ecv -e '^(docs/)' -e '^(keyboards/)' -e '^(layouts/)' -e '^(util/)' -e '^(lib/python/)' -e '^(bin/qmk)' -e '^(requirements.txt)' -e '(.travis.yml)') +NUM_PY_CHANGES=$(echo "$QMK_CHANGES" | grep -Ec -e '^(lib/python/)' -e '^(bin/qmk)') if [[ "$TRAVIS_COMMIT_MESSAGE" == *"[skip test]"* ]]; then echo "Skipping due to commit message" @@ -29,14 +12,14 @@ fi exit_code=0 -if [ "$BRANCH" == "master" ] || [ "$NUM_CORE_CHANGES" != "0" ]; then +if [ "$LOCAL_BRANCH" == "master" ] || [ "$NUM_CORE_CHANGES" != "0" ]; then echo "Running tests." make test:all : $((exit_code = $exit_code + $?)) fi -if [ "$BRANCH" == "master" ] || [ "$NUM_PY_CHANGES" != "0" ]; then +if [ "$LOCAL_BRANCH" == "master" ] || [ "$NUM_PY_CHANGES" != "0" ]; then echo "Running python tests." qmk pytest : $((exit_code = $exit_code + $?)) diff --git a/util/travis_utils.sh b/util/travis_utils.sh new file mode 100755 index 0000000000..031bcda1a8 --- /dev/null +++ b/util/travis_utils.sh @@ -0,0 +1,24 @@ +# Use this by sourcing it in your script. + +# Provide sane defaults for travis environment +TRAVIS_BRANCH="${TRAVIS_BRANCH:master}" +TRAVIS_PULL_REQUEST="${TRAVIS_PULL_REQUEST:false}" +TRAVIS_COMMIT_MESSAGE="${TRAVIS_COMMIT_MESSAGE:-none}" +TRAVIS_COMMIT_RANGE="${TRAVIS_COMMIT_RANGE:-HEAD~1..HEAD}" + +# test force push +#TRAVIS_COMMIT_RANGE="c287f1bfc5c8...81f62atc4c1d" + +# Extra variables +LOCAL_BRANCH=${TRAVIS_PULL_REQUEST_BRANCH:-${TRAVIS_BRANCH}} +QMK_CHANGES=$(git diff --name-only HEAD ${TRAVIS_BRANCH}) + +# if docker is installed - patch calls to within the qmk docker image +if command -v docker >/dev/null; then + function make() { + docker run --rm -e MAKEFLAGS="$MAKEFLAGS" -w /qmk_firmware/ -v "$PWD":/qmk_firmware --user $(id -u):$(id -g) qmkfm/base_container make "$@" + } + function qmk() { + docker run --rm -w /qmk_firmware/ -v "$PWD":/qmk_firmware --user $(id -u):$(id -g) qmkfm/base_container bin/qmk "$@" + } +fi