Refactor and expand the CI workflows
- For each OS, builds of the default compiler plus the latest-supported compilers are run. When multiple operating systems are supported (such as Ubuntu 16.04 and latest), a build on the oldest OS using its default compiler is also performed. - Debug builds are used because they often are more thorough at detecting coding issues (debug warning counts are higher). - Runtime dynamic sanitizers are added and serialized per-compiler. Their build and runtime log-files are xz-compressed, and then GitHub's asset upload Zips the log directory. - Each workflow now holds the maximum allowed compiler warnings per-build, so we can have tighter control of when new warnings are introduced (that would otherwise pass if still below the maximum) - Use of github's new 'cache' feature has been leveraged to restore the brew, macports, and msys2 environments to eliminate the lenghthy setup times for those environments. If a new cache is needed, then we simply increment the cache `key:` value and the next CI run will archive new caches. (Note that GitHub has a 400MB limit on cache size however they have already said they are raising it - so we might be able to cache out longest running job which is MSYS+Clang) - Where it makes sense, multi-line workflow statements have been broken out into .github/scripts as files to make the workflow YAML leaner and more readable, while giving us a richer environment in the scripts.
This commit is contained in:
parent
0f11ab8ecb
commit
30b5246f9f
16 changed files with 720 additions and 202 deletions
58
.github/scripts/build-and-run-sanitizers.sh
vendored
Executable file
58
.github/scripts/build-and-run-sanitizers.sh
vendored
Executable file
|
@ -0,0 +1,58 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright (c) 2019 Kevin R Croft <krcroft@gmail.com>
|
||||
# SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
# This simple script is used to build, test, and archive the log output from
|
||||
# various sanitizer-builds.
|
||||
#
|
||||
# Usage:
|
||||
# ./build-and-run-sanitizers.sh COMPILER VERSION SANITIZER [SANITIZER [...]]"
|
||||
#
|
||||
# Examples:
|
||||
# ./build-and-run-sanitizers.sh clang 8 msan usan
|
||||
# ./build-and-run-sanitizers.sh gcc 9 asan uasan usan tsan
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
# Check the arguments
|
||||
if [[ "$#" -lt 3 ]]; then
|
||||
echo "Usage: $0 COMPILER VERSION SANITIZER [SANITIZER [...]]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Defaults and arguments
|
||||
compiler="${1}"
|
||||
compiler_version="${2}"
|
||||
logs="${compiler}-logs"
|
||||
shift 2
|
||||
sanitizers=("$@")
|
||||
|
||||
# Move to the top of our source directory
|
||||
cd "$(dirname "${0}")/../.."
|
||||
|
||||
# Make a directory to hold our build and run output
|
||||
mkdir -p "${logs}"
|
||||
|
||||
for sanitizer in "${sanitizers[@]}"; do
|
||||
|
||||
# Build DOSBox for each sanitizer
|
||||
time ./scripts/build.sh \
|
||||
--compiler "${compiler}" \
|
||||
--version-postfix "${compiler_version}" \
|
||||
--build-type "${sanitizer}" \
|
||||
&> "${logs}/${compiler}-${sanitizer}-compile.log"
|
||||
|
||||
# Exercise the testcase(s) for each sanitizer
|
||||
# Sanitizers return non-zero if one or more issues were found,
|
||||
# so we or-to-true to ensure our script doesn't end here.
|
||||
time xvfb-run ./src/dosbox -c exit \
|
||||
&> "${logs}/${compiler}-${sanitizer}-EnterExit.log" || true
|
||||
|
||||
done
|
||||
|
||||
# Compress the logs
|
||||
(
|
||||
cd "${logs}"
|
||||
xz -T0 -e ./*.log
|
||||
)
|
20
.github/scripts/download-and-build-macports.sh
vendored
Executable file
20
.github/scripts/download-and-build-macports.sh
vendored
Executable file
|
@ -0,0 +1,20 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright (c) 2019 Kevin R Croft <krcroft@gmail.com>
|
||||
# SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
# This script downloads and builds MacPorts from source.
|
||||
# Usage: ./download-and-build-macports.sh
|
||||
#
|
||||
set -xeuo pipefail
|
||||
|
||||
# Move to the top of our source directory
|
||||
cd "$(dirname "${0}")/../.."
|
||||
|
||||
# Download and install MacPorts
|
||||
git clone --quiet --depth=1 https://github.com/macports/macports-base.git
|
||||
(
|
||||
cd macports-base
|
||||
./configure
|
||||
make -j"$(sysctl -n hw.physicalcpu || echo 4)"
|
||||
)
|
27
.github/scripts/install-and-update-macports.sh
vendored
Executable file
27
.github/scripts/install-and-update-macports.sh
vendored
Executable file
|
@ -0,0 +1,27 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright (c) 2019 Kevin R Croft <krcroft@gmail.com>
|
||||
# SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
# This script installs and updates MacPorts from an already-compiled
|
||||
# source tree assumed to be located n a directory called
|
||||
# 'macports-base' off the root of our source-tree.
|
||||
#
|
||||
# Usage: ./install-and-update-macports.sh
|
||||
#
|
||||
set -xeuo pipefail
|
||||
|
||||
# Ensure we have sudo rights to install MacPorts
|
||||
if [[ $(id -u) -ne 0 ]] ; then echo "Please run as root" ; exit 1 ; fi
|
||||
|
||||
# Move to the top of our source directory
|
||||
cd "$(dirname "${0}")/../.."
|
||||
(
|
||||
cd macports-base
|
||||
make install
|
||||
)
|
||||
# Purge the now-unecessary source
|
||||
rm -rf macports-base
|
||||
|
||||
# Update our ports collection
|
||||
/opt/local/bin/port -q selfupdate || true
|
22
.github/scripts/prepare-for-macports-cachehit.sh
vendored
Executable file
22
.github/scripts/prepare-for-macports-cachehit.sh
vendored
Executable file
|
@ -0,0 +1,22 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright (c) 2019 Kevin R Croft <krcroft@gmail.com>
|
||||
# SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
# This script creates a new writable /opt/local directory with
|
||||
# sane ownership, permissions, and attributes on the directory.
|
||||
# Usage: ./prepare-for-macports-cachehit.sh
|
||||
#
|
||||
set -xeuo pipefail
|
||||
|
||||
# Ensure we have sudo rights to create the directory
|
||||
if [[ $(id -u) -ne 0 ]] ; then echo "Please run as root" ; exit 1 ; fi
|
||||
user="${SUDO_USER}"
|
||||
group="$(id -g "${user}")"
|
||||
|
||||
mkdir -p /opt/local
|
||||
cd /opt/local
|
||||
chflags nouchg .
|
||||
xattr -rc .
|
||||
chmod 770 .
|
||||
chown "${user}:${group}" .
|
34
.github/scripts/reset-brew.sh
vendored
Executable file
34
.github/scripts/reset-brew.sh
vendored
Executable file
|
@ -0,0 +1,34 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright (c) 2019 Kevin R Croft <krcroft@gmail.com>
|
||||
# SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
# This script removes all existing brew packages, and then
|
||||
# re-installs some important basic packages (listed below).
|
||||
#
|
||||
# Usage: ./reset-brew.sh
|
||||
#
|
||||
set -xuo pipefail
|
||||
set +e
|
||||
|
||||
# Pre-cleanup size
|
||||
sudo du -sch /usr/local 2> /dev/null
|
||||
|
||||
# Uninstall all packages
|
||||
# shellcheck disable=SC2046
|
||||
brew remove --force $(brew list) --ignore-dependencies
|
||||
# shellcheck disable=SC2046
|
||||
brew cask remove --force $(brew cask list)
|
||||
|
||||
# Reinstall important packages
|
||||
brew install git git-lfs python curl wget jq binutils zstd gnu-tar
|
||||
|
||||
# Clean the brew cache
|
||||
rm -rf "$(brew --cache)"
|
||||
|
||||
# Post-clean up size
|
||||
sudo du -sch /usr/local 2> /dev/null
|
||||
|
||||
# This script is best-effort, so always return success
|
||||
exit 0
|
||||
|
52
.github/scripts/shrink-brew.sh
vendored
Executable file
52
.github/scripts/shrink-brew.sh
vendored
Executable file
|
@ -0,0 +1,52 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright (c) 2019 Kevin R Croft <krcroft@gmail.com>
|
||||
# SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
# This script reduces the size of a brew-managed installation on macOS.
|
||||
# Its main use is to shrink this area to fit within GitHub's cache
|
||||
# limits however it can also be used by end-users wanting to save space.
|
||||
#
|
||||
# Note that this script remove alot of content deemed unecessary for our
|
||||
# purposes such as ruby, go, grade, azure, etc..) so using on your
|
||||
# home system is probably not what you want :-)
|
||||
#
|
||||
# Usage: ./shrink-brew.sh
|
||||
#
|
||||
set -xuo pipefail
|
||||
set +e
|
||||
|
||||
# Ensure we have sudo rights
|
||||
if [[ $(id -u) -ne 0 ]] ; then echo "Please run as root" ; exit 1 ; fi
|
||||
user="${SUDO_USER}"
|
||||
group="$(id -g "${user}")"
|
||||
|
||||
# If we don't have /usr/local then brew hasn't installed anything yet
|
||||
cd /usr/local || exit 0
|
||||
|
||||
# Purge unecessary bloat
|
||||
for dir in lib/ruby Cellar/go Cellar/gradle Cellar/azure-cli lib/node_modules \
|
||||
share/powershell Caskroom/fastlane Cellar/ruby opt/AGPM Caskroom Cellar/node \
|
||||
miniconda Cellar/python@2 Cellar/git lib/python2.7 Cellar/git-lfs Cellar/subversion \
|
||||
Cellar/maven Cellar/aria2 Homebrew/Library/Homebrew/os/mac/pkgconfig/fuse \
|
||||
.com.apple.installer.keep microsoft; do
|
||||
rm -rf "${dir}" || true
|
||||
done
|
||||
|
||||
# Cleanup permissions and attributes
|
||||
chflags nouchg .
|
||||
find . -type d -exec xattr -c {} +
|
||||
find . -type f -exec xattr -c {} +
|
||||
chown -R "${user}:${group}" ./*
|
||||
find . ! -path . -type d -exec chmod 770 {} +
|
||||
|
||||
# Binary-stripping is *extremely* verbose, so we send these to the ether
|
||||
find Cellar -name '*' -a ! -iname 'strip' -type f -perm +111 -exec strip {} + &> /dev/null
|
||||
find Cellar -name '*.a' -type f -exec strip {} + &> /dev/null
|
||||
|
||||
# Post-cleanup size check
|
||||
du -sch . 2> /dev/null
|
||||
|
||||
# This entire script is best-effort, so always return success
|
||||
exit 0
|
||||
|
36
.github/scripts/shrink-macports.sh
vendored
Executable file
36
.github/scripts/shrink-macports.sh
vendored
Executable file
|
@ -0,0 +1,36 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright (c) 2019 Kevin R Croft <krcroft@gmail.com>
|
||||
# SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
# This script reduces the size of a MacPorts installation
|
||||
# under macOS. Its main use is to shrink this area to fit within
|
||||
# GitHub's cache limits; however it can also be used by end-users
|
||||
# wanting to save space.
|
||||
#
|
||||
# Usage: ./shrink-macports.sh
|
||||
#
|
||||
set -xuo pipefail
|
||||
set +e
|
||||
|
||||
# If MacPorts doesn't exist then our job here is done!
|
||||
cd /opt/local || exit 0
|
||||
|
||||
# Ensure we have sudo rights
|
||||
if [[ $(id -u) -ne 0 ]] ; then echo "Please run as root" ; exit 1 ; fi
|
||||
|
||||
# Cleanup permissions and attributes
|
||||
chflags nouchg .
|
||||
find . -type d -exec xattr -c {} +
|
||||
find . -type f -exec xattr -c {} +
|
||||
|
||||
for dir in var/macports libexec/macports share/man; do
|
||||
rm -rf "${dir}" &> /dev/null || true
|
||||
done
|
||||
|
||||
# Binary-stripping is extremely verbose, so send these to the ether
|
||||
find . -name '*' -a ! -iname 'strip' -type f -perm +111 -exec ./bin/strip {} + &> /dev/null
|
||||
find . -name '*.a' -type f -exec ./bin/strip {} + &> /dev/null
|
||||
|
||||
# This entire script is best-effort, so always return success
|
||||
exit 0
|
44
.github/scripts/shrink-msys2.sh
vendored
Executable file
44
.github/scripts/shrink-msys2.sh
vendored
Executable file
|
@ -0,0 +1,44 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright (c) 2019 Kevin R Croft <krcroft@gmail.com>
|
||||
# SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
# This script reduces the size of an MSYS2 and MingGW installation
|
||||
# under Windows. Its main use is to shrink these areas to fit within
|
||||
# GitHub's cache limits; however it can also be used by end-users wanting
|
||||
# to save space.
|
||||
#
|
||||
# Usage: ./shrink-msys2.sh
|
||||
#
|
||||
set -xuo pipefail
|
||||
set +e
|
||||
|
||||
# Clean all the package archives from pacman
|
||||
pacman -Scc --noconfirm
|
||||
|
||||
# Strip binaries using their associated striping tool
|
||||
for dir in /usr /mingw32 /mingw64; do
|
||||
|
||||
# Enter our directory if we have it
|
||||
cd "${dir}" || continue
|
||||
|
||||
# Check if we have an associated stripping tool
|
||||
if [[ ! -f "bin/strip.exe" ]]; then continue; fi
|
||||
|
||||
# Start stripping
|
||||
find . -type f \
|
||||
\( -iname '*.exe' \
|
||||
-or -iname '*.a' \
|
||||
-or -iname '*.dll' \
|
||||
-or -iname '*.so' \
|
||||
\) \
|
||||
-a ! -iname 'strip.exe' \
|
||||
-print0 \
|
||||
| xargs -0 ./bin/strip.exe --strip-unneeded
|
||||
done
|
||||
|
||||
# Delete MinGW's share directories
|
||||
rm -rf /mingw*/share
|
||||
|
||||
# This entire script is best-effort, so always return success
|
||||
exit 0
|
19
.github/scripts/tar
vendored
Executable file
19
.github/scripts/tar
vendored
Executable file
|
@ -0,0 +1,19 @@
|
|||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
rcode=0
|
||||
# If we're running outside of GitHub then run tar normally
|
||||
if [[ -z "${GITHUB_WORKFLOW:-}" ]]; then
|
||||
"/usr/local/bin/gtar" "$@"
|
||||
rcode="$?"
|
||||
# Otherwise use zstd and ignore tar's return-code
|
||||
else
|
||||
first_arg="${1/z/}"
|
||||
shift
|
||||
[[ "${first_arg}" == "-x" ]] && decomp="-d" || decomp=""
|
||||
set +e -x
|
||||
"/usr/local/bin/gtar" \
|
||||
--warning=none \
|
||||
--use-compress-program="zstd -T0 -19 --long=31 --no-progress -v ${decomp}" \
|
||||
"${first_arg}" "$@"
|
||||
fi
|
||||
exit "${rcode}"
|
Loading…
Add table
Add a link
Reference in a new issue