From 30b5246f9fdd0eb4ae3321e48e1e061bf529e307 Mon Sep 17 00:00:00 2001 From: krcroft Date: Sat, 23 Nov 2019 19:54:58 -0800 Subject: [PATCH] Refactor and expand the CI workflows - For each OS, builds of the default compiler plus the latest-supported compilers are run. When multiple operating systems are supported (such as Ubuntu 16.04 and latest), a build on the oldest OS using its default compiler is also performed. - Debug builds are used because they often are more thorough at detecting coding issues (debug warning counts are higher). - Runtime dynamic sanitizers are added and serialized per-compiler. Their build and runtime log-files are xz-compressed, and then GitHub's asset upload Zips the log directory. - Each workflow now holds the maximum allowed compiler warnings per-build, so we can have tighter control of when new warnings are introduced (that would otherwise pass if still below the maximum) - Use of github's new 'cache' feature has been leveraged to restore the brew, macports, and msys2 environments to eliminate the lenghthy setup times for those environments. If a new cache is needed, then we simply increment the cache `key:` value and the next CI run will archive new caches. (Note that GitHub has a 400MB limit on cache size however they have already said they are raising it - so we might be able to cache out longest running job which is MSYS+Clang) - Where it makes sense, multi-line workflow statements have been broken out into .github/scripts as files to make the workflow YAML leaner and more readable, while giving us a richer environment in the scripts. --- .github/scripts/build-and-run-sanitizers.sh | 58 ++++++ .../scripts/download-and-build-macports.sh | 20 +++ .../scripts/install-and-update-macports.sh | 27 +++ .../scripts/prepare-for-macports-cachehit.sh | 22 +++ .github/scripts/reset-brew.sh | 34 ++++ .github/scripts/shrink-brew.sh | 52 ++++++ .github/scripts/shrink-macports.sh | 36 ++++ .github/scripts/shrink-msys2.sh | 44 +++++ .github/scripts/tar | 19 ++ .github/workflows/analysis.yml | 55 +++++- .github/workflows/dormant/linux.yml | 46 +++++ .github/workflows/dormant/macos-gcc.yml | 76 ++++++++ .github/workflows/dormant/windows.yml | 84 +++++++++ .github/workflows/linux.yml | 93 ++++++---- .github/workflows/macos.yml | 89 +++++----- .github/workflows/windows.yml | 167 ++++++------------ 16 files changed, 720 insertions(+), 202 deletions(-) create mode 100755 .github/scripts/build-and-run-sanitizers.sh create mode 100755 .github/scripts/download-and-build-macports.sh create mode 100755 .github/scripts/install-and-update-macports.sh create mode 100755 .github/scripts/prepare-for-macports-cachehit.sh create mode 100755 .github/scripts/reset-brew.sh create mode 100755 .github/scripts/shrink-brew.sh create mode 100755 .github/scripts/shrink-macports.sh create mode 100755 .github/scripts/shrink-msys2.sh create mode 100755 .github/scripts/tar create mode 100644 .github/workflows/dormant/linux.yml create mode 100644 .github/workflows/dormant/macos-gcc.yml create mode 100644 .github/workflows/dormant/windows.yml diff --git a/.github/scripts/build-and-run-sanitizers.sh b/.github/scripts/build-and-run-sanitizers.sh new file mode 100755 index 00000000..cdb4f4c3 --- /dev/null +++ b/.github/scripts/build-and-run-sanitizers.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +# Copyright (c) 2019 Kevin R Croft +# SPDX-License-Identifier: GPL-2.0-or-later + +# This simple script is used to build, test, and archive the log output from +# various sanitizer-builds. +# +# Usage: +# ./build-and-run-sanitizers.sh COMPILER VERSION SANITIZER [SANITIZER [...]]" +# +# Examples: +# ./build-and-run-sanitizers.sh clang 8 msan usan +# ./build-and-run-sanitizers.sh gcc 9 asan uasan usan tsan +# +set -euo pipefail + +# Check the arguments +if [[ "$#" -lt 3 ]]; then + echo "Usage: $0 COMPILER VERSION SANITIZER [SANITIZER [...]]" + exit 1 +fi + +# Defaults and arguments +compiler="${1}" +compiler_version="${2}" +logs="${compiler}-logs" +shift 2 +sanitizers=("$@") + +# Move to the top of our source directory +cd "$(dirname "${0}")/../.." + +# Make a directory to hold our build and run output +mkdir -p "${logs}" + +for sanitizer in "${sanitizers[@]}"; do + + # Build DOSBox for each sanitizer + time ./scripts/build.sh \ + --compiler "${compiler}" \ + --version-postfix "${compiler_version}" \ + --build-type "${sanitizer}" \ + &> "${logs}/${compiler}-${sanitizer}-compile.log" + + # Exercise the testcase(s) for each sanitizer + # Sanitizers return non-zero if one or more issues were found, + # so we or-to-true to ensure our script doesn't end here. + time xvfb-run ./src/dosbox -c exit \ + &> "${logs}/${compiler}-${sanitizer}-EnterExit.log" || true + +done + +# Compress the logs +( + cd "${logs}" + xz -T0 -e ./*.log +) diff --git a/.github/scripts/download-and-build-macports.sh b/.github/scripts/download-and-build-macports.sh new file mode 100755 index 00000000..3331c58b --- /dev/null +++ b/.github/scripts/download-and-build-macports.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright (c) 2019 Kevin R Croft +# SPDX-License-Identifier: GPL-2.0-or-later + +# This script downloads and builds MacPorts from source. +# Usage: ./download-and-build-macports.sh +# +set -xeuo pipefail + +# Move to the top of our source directory +cd "$(dirname "${0}")/../.." + +# Download and install MacPorts +git clone --quiet --depth=1 https://github.com/macports/macports-base.git +( + cd macports-base + ./configure + make -j"$(sysctl -n hw.physicalcpu || echo 4)" +) diff --git a/.github/scripts/install-and-update-macports.sh b/.github/scripts/install-and-update-macports.sh new file mode 100755 index 00000000..145a0436 --- /dev/null +++ b/.github/scripts/install-and-update-macports.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Copyright (c) 2019 Kevin R Croft +# SPDX-License-Identifier: GPL-2.0-or-later + +# This script installs and updates MacPorts from an already-compiled +# source tree assumed to be located n a directory called +# 'macports-base' off the root of our source-tree. +# +# Usage: ./install-and-update-macports.sh +# +set -xeuo pipefail + +# Ensure we have sudo rights to install MacPorts +if [[ $(id -u) -ne 0 ]] ; then echo "Please run as root" ; exit 1 ; fi + +# Move to the top of our source directory +cd "$(dirname "${0}")/../.." +( + cd macports-base + make install +) +# Purge the now-unecessary source +rm -rf macports-base + +# Update our ports collection +/opt/local/bin/port -q selfupdate || true diff --git a/.github/scripts/prepare-for-macports-cachehit.sh b/.github/scripts/prepare-for-macports-cachehit.sh new file mode 100755 index 00000000..bb27c5f3 --- /dev/null +++ b/.github/scripts/prepare-for-macports-cachehit.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Copyright (c) 2019 Kevin R Croft +# SPDX-License-Identifier: GPL-2.0-or-later + +# This script creates a new writable /opt/local directory with +# sane ownership, permissions, and attributes on the directory. +# Usage: ./prepare-for-macports-cachehit.sh +# +set -xeuo pipefail + +# Ensure we have sudo rights to create the directory +if [[ $(id -u) -ne 0 ]] ; then echo "Please run as root" ; exit 1 ; fi +user="${SUDO_USER}" +group="$(id -g "${user}")" + +mkdir -p /opt/local +cd /opt/local +chflags nouchg . +xattr -rc . +chmod 770 . +chown "${user}:${group}" . diff --git a/.github/scripts/reset-brew.sh b/.github/scripts/reset-brew.sh new file mode 100755 index 00000000..e00a0c72 --- /dev/null +++ b/.github/scripts/reset-brew.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Copyright (c) 2019 Kevin R Croft +# SPDX-License-Identifier: GPL-2.0-or-later + +# This script removes all existing brew packages, and then +# re-installs some important basic packages (listed below). +# +# Usage: ./reset-brew.sh +# +set -xuo pipefail +set +e + +# Pre-cleanup size +sudo du -sch /usr/local 2> /dev/null + +# Uninstall all packages +# shellcheck disable=SC2046 +brew remove --force $(brew list) --ignore-dependencies +# shellcheck disable=SC2046 +brew cask remove --force $(brew cask list) + +# Reinstall important packages +brew install git git-lfs python curl wget jq binutils zstd gnu-tar + +# Clean the brew cache +rm -rf "$(brew --cache)" + +# Post-clean up size +sudo du -sch /usr/local 2> /dev/null + +# This script is best-effort, so always return success +exit 0 + diff --git a/.github/scripts/shrink-brew.sh b/.github/scripts/shrink-brew.sh new file mode 100755 index 00000000..2210955c --- /dev/null +++ b/.github/scripts/shrink-brew.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +# Copyright (c) 2019 Kevin R Croft +# SPDX-License-Identifier: GPL-2.0-or-later + +# This script reduces the size of a brew-managed installation on macOS. +# Its main use is to shrink this area to fit within GitHub's cache +# limits however it can also be used by end-users wanting to save space. +# +# Note that this script remove alot of content deemed unecessary for our +# purposes such as ruby, go, grade, azure, etc..) so using on your +# home system is probably not what you want :-) +# +# Usage: ./shrink-brew.sh +# +set -xuo pipefail +set +e + +# Ensure we have sudo rights +if [[ $(id -u) -ne 0 ]] ; then echo "Please run as root" ; exit 1 ; fi +user="${SUDO_USER}" +group="$(id -g "${user}")" + +# If we don't have /usr/local then brew hasn't installed anything yet +cd /usr/local || exit 0 + +# Purge unecessary bloat +for dir in lib/ruby Cellar/go Cellar/gradle Cellar/azure-cli lib/node_modules \ +share/powershell Caskroom/fastlane Cellar/ruby opt/AGPM Caskroom Cellar/node \ +miniconda Cellar/python@2 Cellar/git lib/python2.7 Cellar/git-lfs Cellar/subversion \ +Cellar/maven Cellar/aria2 Homebrew/Library/Homebrew/os/mac/pkgconfig/fuse \ +.com.apple.installer.keep microsoft; do + rm -rf "${dir}" || true +done + +# Cleanup permissions and attributes +chflags nouchg . +find . -type d -exec xattr -c {} + +find . -type f -exec xattr -c {} + +chown -R "${user}:${group}" ./* +find . ! -path . -type d -exec chmod 770 {} + + +# Binary-stripping is *extremely* verbose, so we send these to the ether +find Cellar -name '*' -a ! -iname 'strip' -type f -perm +111 -exec strip {} + &> /dev/null +find Cellar -name '*.a' -type f -exec strip {} + &> /dev/null + +# Post-cleanup size check +du -sch . 2> /dev/null + +# This entire script is best-effort, so always return success +exit 0 + diff --git a/.github/scripts/shrink-macports.sh b/.github/scripts/shrink-macports.sh new file mode 100755 index 00000000..c94f1ebd --- /dev/null +++ b/.github/scripts/shrink-macports.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Copyright (c) 2019 Kevin R Croft +# SPDX-License-Identifier: GPL-2.0-or-later + +# This script reduces the size of a MacPorts installation +# under macOS. Its main use is to shrink this area to fit within +# GitHub's cache limits; however it can also be used by end-users +# wanting to save space. +# +# Usage: ./shrink-macports.sh +# +set -xuo pipefail +set +e + +# If MacPorts doesn't exist then our job here is done! +cd /opt/local || exit 0 + +# Ensure we have sudo rights +if [[ $(id -u) -ne 0 ]] ; then echo "Please run as root" ; exit 1 ; fi + +# Cleanup permissions and attributes +chflags nouchg . +find . -type d -exec xattr -c {} + +find . -type f -exec xattr -c {} + + +for dir in var/macports libexec/macports share/man; do + rm -rf "${dir}" &> /dev/null || true +done + +# Binary-stripping is extremely verbose, so send these to the ether +find . -name '*' -a ! -iname 'strip' -type f -perm +111 -exec ./bin/strip {} + &> /dev/null +find . -name '*.a' -type f -exec ./bin/strip {} + &> /dev/null + +# This entire script is best-effort, so always return success +exit 0 diff --git a/.github/scripts/shrink-msys2.sh b/.github/scripts/shrink-msys2.sh new file mode 100755 index 00000000..18ebe2a4 --- /dev/null +++ b/.github/scripts/shrink-msys2.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# Copyright (c) 2019 Kevin R Croft +# SPDX-License-Identifier: GPL-2.0-or-later + +# This script reduces the size of an MSYS2 and MingGW installation +# under Windows. Its main use is to shrink these areas to fit within +# GitHub's cache limits; however it can also be used by end-users wanting +# to save space. +# +# Usage: ./shrink-msys2.sh +# +set -xuo pipefail +set +e + +# Clean all the package archives from pacman +pacman -Scc --noconfirm + +# Strip binaries using their associated striping tool +for dir in /usr /mingw32 /mingw64; do + + # Enter our directory if we have it + cd "${dir}" || continue + + # Check if we have an associated stripping tool + if [[ ! -f "bin/strip.exe" ]]; then continue; fi + + # Start stripping + find . -type f \ + \( -iname '*.exe' \ + -or -iname '*.a' \ + -or -iname '*.dll' \ + -or -iname '*.so' \ + \) \ + -a ! -iname 'strip.exe' \ + -print0 \ + | xargs -0 ./bin/strip.exe --strip-unneeded +done + +# Delete MinGW's share directories +rm -rf /mingw*/share + +# This entire script is best-effort, so always return success +exit 0 diff --git a/.github/scripts/tar b/.github/scripts/tar new file mode 100755 index 00000000..e16aca21 --- /dev/null +++ b/.github/scripts/tar @@ -0,0 +1,19 @@ +#!/bin/bash +set -euo pipefail +rcode=0 +# If we're running outside of GitHub then run tar normally +if [[ -z "${GITHUB_WORKFLOW:-}" ]]; then + "/usr/local/bin/gtar" "$@" + rcode="$?" +# Otherwise use zstd and ignore tar's return-code +else + first_arg="${1/z/}" + shift + [[ "${first_arg}" == "-x" ]] && decomp="-d" || decomp="" + set +e -x + "/usr/local/bin/gtar" \ + --warning=none \ + --use-compress-program="zstd -T0 -19 --long=31 --no-progress -v ${decomp}" \ + "${first_arg}" "$@" +fi +exit "${rcode}" diff --git a/.github/workflows/analysis.yml b/.github/workflows/analysis.yml index 01a8eecb..71ae41a1 100644 --- a/.github/workflows/analysis.yml +++ b/.github/workflows/analysis.yml @@ -1,4 +1,4 @@ -name: Static code analysis +name: Code analysis on: push jobs: @@ -9,7 +9,7 @@ jobs: - uses: actions/checkout@v1 - run: sudo apt-get update - name: Install pylint - run: | + run: | sudo apt-get install python3-setuptools sudo pip3 install pylint beautifulsoup4 html5lib - name: Run pylint @@ -23,15 +23,15 @@ jobs: needs: run_linters steps: - uses: actions/checkout@v1 - - name: Log environment - run: ./scripts/log-env.sh - run: sudo apt-get update - - name: Install dependencies - run: sudo apt-get install python3-setuptools $(./scripts/list-build-dependencies.sh -p apt) + - name: Install C++ compiler and libraries + run: sudo apt-get install python3-setuptools $(./scripts/list-build-dependencies.sh -m apt -c gcc) - name: Install scan-build (Python version) run: sudo pip3 install scan-build beautifulsoup4 html5lib + - name: Log environment + run: ./scripts/log-env.sh - name: Build - run: | + run: | # build steps set -x g++ --version @@ -40,7 +40,8 @@ jobs: intercept-build make -j "$(nproc)" - name: Analyze run: analyze-build -v -o report --html-title="dosbox-staging (${GITHUB_SHA:0:8})" - - uses: actions/upload-artifact@master + - name: Upload report + uses: actions/upload-artifact@master with: name: report path: report @@ -49,4 +50,40 @@ jobs: # summary echo "Full report is included in build Artifacts" echo - ./scripts/count-bugs.py report/*/index.html + ./scripts/count-bugs.py -m 92 report/*/index.html + + dynamic_matrix: + name: ${{ matrix.compiler }} dynamic sanitizers + needs: run_linters + runs-on: ubuntu-latest + strategy: + matrix: + compiler: [Clang, GCC] + steps: + - uses: actions/checkout@v1 + - run: sudo apt-get update + - name: Install C++ compiler and libraries + env: + VERSION_GCC: 9 + VERSION_Clang: 8 + run: > + sudo apt-get install -y $(./scripts/list-build-dependencies.sh -m apt + -c ${{ matrix.compiler }} -v $VERSION_${{ matrix.compiler }}) + - name: Log environment + run: ./scripts/log-env.sh + - name: Build and run sanitizers + env: + VERSION_GCC: 9 + VERSION_Clang: 8 + SANITIZERS_GCC: UASAN USAN TSAN + SANITIZERS_Clang: USAN MSAN + run: | + ./.github/scripts/build-and-run-sanitizers.sh \ + ${{ matrix.compiler }} \ + $VERSION_${{ matrix.compiler }} \ + $SANITIZERS_${{ matrix.compiler }} + - name: Upload logs + uses: actions/upload-artifact@master + with: + name: ${{ matrix.compiler }}-logs + path: ${{ matrix.compiler }}-logs diff --git a/.github/workflows/dormant/linux.yml b/.github/workflows/dormant/linux.yml new file mode 100644 index 00000000..b997e543 --- /dev/null +++ b/.github/workflows/dormant/linux.yml @@ -0,0 +1,46 @@ +name: Linux builds +on: push +env: + MAX_WARNINGS_GCC_6_Debug: 378 + MAX_WARNINGS_GCC_6_Release: 661 + MAX_WARNINGS_GCC_7_Debug: 380 + MAX_WARNINGS_GCC_7_Release: 329 + MAX_WARNINGS_GCC_8_Debug: 380 + MAX_WARNINGS_GCC_8_Release: 330 + MAX_WARNINGS_GCC_9_Debug: 403 + MAX_WARNINGS_GCC_9_Release: 358 + MAX_WARNINGS_Clang_7_Debug: 193 + MAX_WARNINGS_Clang_7_Release: 193 + MAX_WARNINGS_Clang_8_Debug: 193 + MAX_WARNINGS_Clang_8_Release: 193 + +jobs: + + build_matrix: + name: ${{ matrix.compiler }}-${{ matrix.compiler_version }} + runs-on: ubuntu-latest + strategy: + matrix: + compiler: [Clang, GCC] + compiler_version: [6, 7, 8, 9] + # build_type: [Debug, Release] + exclude: + - compiler: Clang + compiler_version: 6 + - compiler: Clang + compiler_version: 9 + steps: + - uses: actions/checkout@v1 + - run: sudo apt-get update + - name: Install C++ compiler and libraries + run: sudo apt-get install -y $(./scripts/list-build-dependencies.sh -m apt -c ${{ matrix.compiler }} -v ${{ matrix.compiler_version }}) + - name: Log environment + run: ./scripts/log-env.sh + - name: Release build + run: ./scripts/build.sh --compiler ${{ matrix.compiler }} --version-postfix ${{ matrix.compiler_version }} --build-type Release + - name: Release warnings + run: ./scripts/count-warnings.py -m $MAX_WARNINGS_${{ matrix.compiler }}_${{ matrix.compiler_version }}_Release build.log + - name: Debug build + run: ./scripts/build.sh --compiler ${{ matrix.compiler }} --version-postfix ${{ matrix.compiler_version }} --build-type Debug + - name: Debug warnings + run: ./scripts/count-warnings.py -m $MAX_WARNINGS_${{ matrix.compiler }}_${{ matrix.compiler_version }}_Debug build.log diff --git a/.github/workflows/dormant/macos-gcc.yml b/.github/workflows/dormant/macos-gcc.yml new file mode 100644 index 00000000..8f5d061d --- /dev/null +++ b/.github/workflows/dormant/macos-gcc.yml @@ -0,0 +1,76 @@ +name: macOS builds +on: push +env: + MAX_WARNINGS_GCC_9_Debug: 361 + MAX_WARNINGS_GCC_9_Release: 321 + PATH: "${GITHUB_WORKSPACE}/.github/scripts:${PATH}" + +jobs: + build_macos_gcc_brew: + name: GCC-${{ matrix.compiler_version }} (HomeBrew) + runs-on: macos-latest + strategy: + matrix: + compiler_version: [9] + steps: + - uses: actions/checkout@v1 + - name: Prepare tar-zstd for the cache + run: | + brew install zstd gnu-tar + cp .github/scripts/tar /usr/local/bin/tar + - uses: actions/cache@v1 + id: cache-brew + with: {path: /usr/local, key: brew-gcc-2019-v18} + - name: Install C++ compiler and libraries + if: steps.cache-brew.outputs.cache-hit != 'true' + run: | + ./.github/scripts/reset-brew.sh + brew install $(./scripts/list-build-dependencies.sh -m brew -c gcc -v ${{ matrix.compiler_version }}) + sudo ./.github/scripts/shrink-brew.sh + - name: Log environment + run: ./scripts/log-env.sh + - name: Release build + run: ./scripts/build.sh --compiler gcc --version-postfix ${CC_PREFIX}${{ matrix.compiler_version }} --build-type Release + - name: Release warnings + run: python3 ./scripts/count-warnings.py -m $MAX_WARNINGS_GCC_${{ matrix.compiler_version }}_Release build.log + - name: Debug build + run: ./scripts/build.sh --compiler gcc --version-postfix ${CC_PREFIX}${{ matrix.compiler_version }} --build-type Debug + - name: Debug warnings + run: python3 ./scripts/count-warnings.py -m $MAX_WARNINGS_GCC_${{ matrix.compiler_version }}_Debug build.log + + build_macos_gcc_macports: + name: GCC-${{ matrix.compiler_version }} (MacPorts) + runs-on: macos-latest + strategy: + matrix: + compiler_version: [9] + steps: + - uses: actions/checkout@v1 + - name: Prepare tar-zstd and /opt/local for the cache + run: | + brew install zstd gnu-tar + cp .github/scripts/tar /usr/local/bin/tar + sudo ./.github/scripts/prepare-for-macports-cachehit.sh + - uses: actions/cache@v1 + id: cache-macports + with: {path: /opt/local, key: macports-gcc-2019-v14} + - name: Install MacPorts, C++ compiler, and libraries + if: steps.cache-macports.outputs.cache-hit != 'true' + run: | + ./.github/scripts/download-and-build-macports.sh + sudo ./.github/scripts/install-and-update-macports.sh + sudo /opt/local/bin/port -q -f install $(./scripts/list-build-dependencies.sh -m macports -c gcc -v ${{ matrix.compiler_version }}) + - name: Log environment + run: ./scripts/log-env.sh + - name: Release build + run: ./scripts/build.sh --compiler gcc --version-postfix mp-${{ matrix.compiler_version }} --build-type Release --bin-path /opt/local/bin + - name: Release warnings + run: python3 ./scripts/count-warnings.py $MAX_WARNINGS_GCC_${{ matrix.compiler_version }}_Release build.log + - name: Debug build + run: ./scripts/build.sh --compiler gcc --version-postfix mp-${{ matrix.compiler_version }} --build-type Debug --bin-path /opt/local/bin + - name: Debug warnings + run: python3 ./scripts/count-warnings.py $MAX_WARNINGS_GCC_${{ matrix.compiler_version }}_Debug build.log + + - name: Shrink MacPorts for cache + if: steps.cache-macports.outputs.cache-hit != 'true' + run: sudo ./.github/scripts/shrink-macports.sh diff --git a/.github/workflows/dormant/windows.yml b/.github/workflows/dormant/windows.yml new file mode 100644 index 00000000..3b74cc43 --- /dev/null +++ b/.github/workflows/dormant/windows.yml @@ -0,0 +1,84 @@ +name: Windows builds +on: push +env: + MAX_WARNINGS_GCC_32bit_Debug: 320 + MAX_WARNINGS_GCC_32bit_Release: 284 + MAX_WARNINGS_GCC_64bit_Debug: 405 + MAX_WARNINGS_GCC_64bit_Release: 369 + MAX_WARNINGS_Clang_32bit_Debug: 129 + MAX_WARNINGS_Clang_32bit_Release: 129 + MAX_WARNINGS_Clang_64bit_Debug: 190 + MAX_WARNINGS_Clang_64bit_Release: 190 + +jobs: + + build_msvc_matrix: + name: MSVC 32-bit + runs-on: windows-latest + steps: + - uses: actions/checkout@v1 + - name: Install packages + shell: pwsh + run: | + vcpkg install libpng sdl1 sdl1-net opusfile + vcpkg integrate install + - name: Log environment + shell: pwsh + run: .\scripts\log-env.ps1 + - name: Release build + shell: pwsh + env: + PATH: '${env:PATH};C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\amd64' + run: | + cd visualc_net + MSBuild -m dosbox.sln -p:Configuration=Release + - name: Debug build + shell: pwsh + env: + PATH: '${env:PATH};C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\amd64' + run: | + cd visualc_net + MSBuild -m dosbox.sln -p:Configuration=Debug + + + build_msys2_matrix: + name: ${{ matrix.compiler }} ${{ matrix.bits }}-bit + runs-on: windows-latest + strategy: + matrix: + compiler: [GCC, Clang] + bits: [32, 64] + env: + CHERE_INVOKING: yes + steps: + - uses: actions/checkout@v1 + - uses: actions/cache@v1 + id: cache-msys2 + if: matrix.compiler == 'GCC' + with: + path: 'C:/tools/msys64' + key: msys2-${{ matrix.compiler }}-${{ matrix.bits }}-2019-v11} + - name: Install MSYS2 + if: steps.cache-msys2.outputs.cache-hit != 'true' + run: choco install msys2 --no-progress + - name: Install C++ compiler and libraries + if: steps.cache-msys2.outputs.cache-hit != 'true' + shell: python scripts\msys-bash.py {0} + run: ./scripts/list-build-dependencies.sh -m msys2 -c ${{ matrix.compiler }} -b ${{ matrix.bits }} | xargs pacman -S --noconfirm + - name: Shrink MSYS2 for cache + if: matrix.compiler == 'GCC' && steps.cache-msys2.outputs.cache-hit != 'true' + shell: python scripts\msys-bash.py {0} + run: ./.github/scripts/shrink-msys2.sh + - name: Log environment + shell: python scripts\msys-bash.py {0} + run: ./scripts/log-env.sh + - name: Release build + shell: python scripts\msys-bash.py {0} + run: ./scripts/build.sh --compiler ${{ matrix.compiler }} --build-type Release --bin-path /mingw${{ matrix.bits }}/bin + - name: Release warnings + run: '.\scripts\count-warnings.py -m $env:MAX_WARNINGS_${{ matrix.compiler }}_${{ matrix.bits }}bit_Release build.log' + - name: Debug build + shell: python scripts\msys-bash.py {0} + run: ./scripts/build.sh --compiler ${{ matrix.compiler }} --build-type Debug --bin-path /mingw${{ matrix.bits }}/bin + - name: Debug warnings + run: '.\scripts\count-warnings.py -m $env:MAX_WARNINGS_${{ matrix.compiler }}_${{ matrix.bits }}bit_Debug build.log' diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 9fa3998e..243cde72 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -1,51 +1,80 @@ name: Linux builds on: push +env: + MAX_WARNINGS_GCC_5_Debug: 373 + MAX_WARNINGS_GCC_5_Release: -1 + MAX_WARNINGS_GCC_6_Debug: 378 + MAX_WARNINGS_GCC_6_Release: 661 + MAX_WARNINGS_GCC_7_Debug: 380 + MAX_WARNINGS_GCC_7_Release: 329 + MAX_WARNINGS_GCC_8_Debug: 380 + MAX_WARNINGS_GCC_8_Release: 330 + MAX_WARNINGS_GCC_9_Debug: 403 + MAX_WARNINGS_GCC_9_Release: 358 + MAX_WARNINGS_Clang_7_Debug: 193 + MAX_WARNINGS_Clang_7_Release: 193 + MAX_WARNINGS_Clang_8_Debug: 193 + MAX_WARNINGS_Clang_8_Release: 193 + jobs: - build_ubuntu_clang_8: - name: Clang 8 (ubuntu-18.04) - runs-on: ubuntu-18.04 + gcc-5_1604: + name: GCC-5 (Ubuntu 16.04) + runs-on: ubuntu-16.04 steps: - uses: actions/checkout@v1 + - run: sudo apt-get update + - name: Install C++ compiler and libraries + run: sudo apt-get install -y $(./scripts/list-build-dependencies.sh -m apt -c gcc) - name: Log environment run: ./scripts/log-env.sh - - run: sudo apt-get update - - name: Install dependencies - run: sudo apt-get install -y $(./scripts/list-build-dependencies.sh -p apt --compiler clang --compiler-version 8) - - name: Build - run: ./scripts/build.sh --compiler clang --compiler-version 8 --lto - - name: Summarize warnings - run: ./scripts/count-warnings.py build.log + - name: Debug build + run: ./scripts/build.sh --compiler gcc --build-type Debug + - name: Debug warnings + run: ./scripts/count-warnings.py -m $MAX_WARNINGS_GCC_5_Debug build.log - build_ubuntu_matrix_gcc: - name: GCC - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-18.04, ubuntu-16.04] + gcc-7_latest: + name: GCC-7 (Ubuntu-latest) + runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 + - run: sudo apt-get update + - name: Install C++ compiler and libraries + run: sudo apt-get install -y $(./scripts/list-build-dependencies.sh -m apt -c gcc) - name: Log environment run: ./scripts/log-env.sh - - run: sudo apt-get update - - name: Install dependencies - run: sudo apt-get install -y $(./scripts/list-build-dependencies.sh -p apt) - - name: Build - run: ./scripts/build.sh - - name: Summarize warnings - run: ./scripts/count-warnings.py build.log + - name: Debug build + run: ./scripts/build.sh --compiler gcc --build-type Debug + - name: Debug warnings + run: ./scripts/count-warnings.py -m $MAX_WARNINGS_GCC_7_Debug build.log - build_ubuntu_gcc9: - name: GCC 9 (ubuntu-18.04) - runs-on: ubuntu-18.04 + gcc-9_latest: + name: GCC-9 (Ubuntu-latest) + runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 + - run: sudo apt-get update + - name: Install C++ compiler and libraries + run: sudo apt-get install -y $(./scripts/list-build-dependencies.sh -m apt -c gcc -v 9) - name: Log environment run: ./scripts/log-env.sh + - name: Debug build + run: ./scripts/build.sh --compiler gcc --version-postfix 9 --build-type Debug + - name: Debug warnings + run: ./scripts/count-warnings.py -m $MAX_WARNINGS_GCC_9_Debug build.log + + clang-8_latest: + name: Clang-8 (Ubuntu-latest) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 - run: sudo apt-get update - - name: Install dependencies - run: sudo apt-get install -y $(./scripts/list-build-dependencies.sh -p apt --compiler-version 9) - - name: Build - run: ./scripts/build.sh --compiler-version 9 --lto - - name: Summarize warnings - run: ./scripts/count-warnings.py build.log + - name: Install C++ compiler and libraries + run: sudo apt-get install -y $(./scripts/list-build-dependencies.sh -m apt -c clang -v 8) + - name: Log environment + run: ./scripts/log-env.sh + - name: Debug build + run: ./scripts/build.sh --compiler clang --version-postfix 8 --build-type Debug + - name: Debug warnings + run: ./scripts/count-warnings.py -m $MAX_WARNINGS_Clang_8_Debug build.log + diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 0487576f..64dfe3a5 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -1,60 +1,51 @@ name: macOS builds on: push -jobs: +env: + MAX_WARNINGS_Clang_Debug: 245 + MAX_WARNINGS_Clang_Release: 245 + MAX_WARNINGS_GCC_9_Debug: 361 + MAX_WARNINGS_GCC_9_Release: 321 - build_macos_xcode_clang: +jobs: + clang_xcode: name: Clang (Xcode) runs-on: macos-latest steps: - uses: actions/checkout@v1 + - name: Install C++ compiler and libraries + run: brew install $(./scripts/list-build-dependencies.sh -m brew -c clang) - name: Log environment run: ./scripts/log-env.sh - - name: Install dependencies - run: brew install $(./scripts/list-build-dependencies.sh -p brew --compiler clang) - - name: Build - run: ./scripts/build.sh --compiler clang --lto - - name: Summarize warnings - run: python3 ./scripts/count-warnings.py build.log + - name: Debug build + run: ./scripts/build.sh --compiler clang --build-type Debug + - name: Debug warnings + run: python3 ./scripts/count-warnings.py -m $MAX_WARNINGS_Clang_Debug build.log - -# build_macos_brew_gcc9: -# name: GCC 9 (Homebrew) -# runs-on: macos-latest -# steps: -# - uses: actions/checkout@v1 -# - name: Log environment -# run: ./scripts/log-env.sh -# - name: Install dependencies -# run: brew install $(./scripts/list-build-dependencies.sh -p brew --compiler-version 9) -# - name: Build -# run: echo ./scripts/build.sh --compiler-version 9 --lto -# - name: Summarize warnings -# run: echo python3 ./scripts/count-warnings.py build.log - - -# build_macos_macports_gcc9: -# name: GCC 9 (MacPorts) -# runs-on: macos-latest -# steps: -# - uses: actions/checkout@v1 -# - name: Install MacPorts -# run: | -# # xcode-select --install || true -# # sudo xcodebuild -license || true -# git clone --quiet --depth=1 https://github.com/macports/macports-base.git -# cd macports-base -# ./configure -# make -j"$(sysctl -n hw.physicalcpu || echo 4)" -# sudo make install -# -# - name: Install dependencies -# run: | -# PATH="/opt/local/sbin:/opt/local/bin:${PATH}" -# sudo port -q selfupdate || true -# sudo port -q install $(./scripts/list-build-dependencies.sh -p macports --compiler-version 9) -# -# - name: Build -# run: echo ./scripts/build.sh --compiler-version mp-9 --bin-path /opt/local/bin -# - name: Summarize warnings -# run: echo python3 ./scripts/count-warnings.py build.log + gcc_brew: + name: GCC-${{ matrix.compiler_version }} (HomeBrew) + runs-on: macos-latest + strategy: + matrix: + compiler_version: [9] + steps: + - uses: actions/checkout@v1 + - name: Prepare tar-zstd for the cache + run: | + brew install zstd gnu-tar + cp .github/scripts/tar /usr/local/bin/tar + - uses: actions/cache@v1 + id: cache-brew + with: {path: /usr/local, key: brew-gcc-2019-v18} + - name: Install C++ compiler and libraries + if: steps.cache-brew.outputs.cache-hit != 'true' + run: | + ./.github/scripts/reset-brew.sh + brew install $(./scripts/list-build-dependencies.sh -m brew -c gcc -v ${{ matrix.compiler_version }}) + sudo ./.github/scripts/shrink-brew.sh + - name: Log environment + run: ./scripts/log-env.sh + - name: Debug build + run: ./scripts/build.sh --compiler gcc --version-postfix ${CC_PREFIX}${{ matrix.compiler_version }} --build-type Debug + - name: Debug warnings + run: python3 ./scripts/count-warnings.py -m $MAX_WARNINGS_GCC_${{ matrix.compiler_version }}_Debug build.log diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index b504bc0d..a9dc068d 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -1,132 +1,75 @@ name: Windows builds on: push +env: + MAX_WARNINGS_GCC_32bit_Debug: 320 + MAX_WARNINGS_GCC_32bit_Release: 284 + MAX_WARNINGS_GCC_64bit_Debug: 405 + MAX_WARNINGS_GCC_64bit_Release: 369 + MAX_WARNINGS_Clang_32bit_Debug: 129 + MAX_WARNINGS_Clang_32bit_Release: 129 + MAX_WARNINGS_Clang_64bit_Debug: 190 + MAX_WARNINGS_Clang_64bit_Release: 190 + jobs: - build_msys2_clang_32: - name: Clang 8 i686 (MSYS2) + build_msvc_matrix: + name: MSVC 32-bit (${{ matrix.type }}) runs-on: windows-latest - env: - CHERE_INVOKING: yes + strategy: + matrix: + type: [Release, Debug] steps: - uses: actions/checkout@v1 - - name: Install msys2 environment - run: choco install msys2 --no-progress - - name: Log environment - shell: python scripts\msys-bash.py {0} - run: ./scripts/log-env.sh - - name: Install dependencies - shell: python scripts\msys-bash.py {0} - run: ./scripts/list-build-dependencies.sh -p msys2 --compiler clang --bit-depth 32 | xargs pacman -S --noconfirm - - name: Build - shell: python scripts\msys-bash.py {0} - run: ./scripts/build.sh --compiler clang --bit-depth 32 --bin-path /mingw32/bin - - name: Summarize warnings - run: .\scripts\count-warnings.py build.log - - build_msys2_clang_64: - name: Clang 8 x86_64 (MSYS2) - runs-on: windows-latest - env: - CHERE_INVOKING: yes - steps: - - uses: actions/checkout@v1 - - name: Install msys2 environment - run: choco install msys2 --no-progress - - name: Log environment - shell: python scripts\msys-bash.py {0} - run: ./scripts/log-env.sh - - name: Install dependencies - shell: python scripts\msys-bash.py {0} - run: ./scripts/list-build-dependencies.sh -p msys2 --compiler clang | xargs pacman -S --noconfirm - - name: Build - shell: python scripts\msys-bash.py {0} - run: ./scripts/build.sh --compiler clang --bin-path /mingw64/bin - - name: Summarize warnings - run: .\scripts\count-warnings.py build.log - - build_msys2_gcc_32: - name: GCC 9 i686 (MSYS2) - runs-on: windows-latest - env: - CHERE_INVOKING: yes - steps: - - uses: actions/checkout@v1 - - name: Install msys2 environment - run: choco install msys2 --no-progress - - name: Log environment - shell: python scripts\msys-bash.py {0} - run: ./scripts/log-env.sh - - name: Install dependencies - shell: python scripts\msys-bash.py {0} - run: ./scripts/list-build-dependencies.sh -p msys2 --bit-depth 32 | xargs pacman -S --noconfirm - - name: Build - shell: python scripts\msys-bash.py {0} - run: ./scripts/build.sh --bit-depth 32 --bin-path /mingw32/bin - - name: Summarize warnings - run: .\scripts\count-warnings.py build.log - - - build_msys2_gcc_64: - name: GCC 9 x86_64 (MSYS2) - runs-on: windows-latest - env: - CHERE_INVOKING: yes - steps: - - uses: actions/checkout@v1 - - name: Install msys2 environment - run: choco install msys2 --no-progress - - name: Log environment - shell: python scripts\msys-bash.py {0} - run: ./scripts/log-env.sh - - name: Install dependencies - shell: python scripts\msys-bash.py {0} - run: ./scripts/list-build-dependencies.sh -p msys2 | xargs pacman -S --noconfirm - - name: Build - shell: python scripts\msys-bash.py {0} - run: ./scripts/build.sh --bin-path /mingw64/bin - - name: Summarize warnings - run: .\scripts\count-warnings.py build.log - - build_msvc_debug: - name: MSVC Debug x86 - runs-on: windows-latest - steps: - - uses: actions/checkout@v1 - - name: Log environment - shell: pwsh - run: .\scripts\log-env.ps1 - name: Install packages shell: pwsh run: | vcpkg install libpng sdl1 sdl1-net opusfile vcpkg integrate install - - name: Build - shell: pwsh - env: - PATH: '${env:PATH};C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\amd64' - run: | - # build steps - cd visualc_net - MSBuild -m dosbox.sln -p:Configuration=Debug - - build_msvc_release: - name: MSVC Release x86 - runs-on: windows-latest - steps: - - uses: actions/checkout@v1 - name: Log environment shell: pwsh run: .\scripts\log-env.ps1 - - name: Install packages - shell: pwsh - run: | - vcpkg install libpng sdl1 sdl1-net opusfile - vcpkg integrate install - - name: Build + - name: ${{ matrix.type }} build shell: pwsh env: PATH: '${env:PATH};C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\amd64' run: | - # build steps cd visualc_net - MSBuild -m dosbox.sln -p:Configuration=Release + MSBuild -m dosbox.sln -p:Configuration=${{ matrix.type }} + + + build_msys2_matrix: + name: ${{ matrix.compiler }} ${{ matrix.bits }}-bit + runs-on: windows-latest + strategy: + matrix: + compiler: [GCC, Clang] + bits: [32, 64] + env: + CHERE_INVOKING: yes + steps: + - uses: actions/checkout@v1 + - uses: actions/cache@v1 + id: cache-msys2 + if: matrix.compiler == 'GCC' + with: + path: 'C:/tools/msys64' + key: msys2-${{ matrix.compiler }}-${{ matrix.bits }}-2019-v11} + - name: Install MSYS2 + if: steps.cache-msys2.outputs.cache-hit != 'true' + run: choco install msys2 --no-progress + - name: Install C++ compiler and libraries + if: steps.cache-msys2.outputs.cache-hit != 'true' + shell: python scripts\msys-bash.py {0} + run: ./scripts/list-build-dependencies.sh -m msys2 -c ${{ matrix.compiler }} -b ${{ matrix.bits }} | xargs pacman -S --noconfirm + - name: Shrink MSYS2 for cache + if: matrix.compiler == 'GCC' && steps.cache-msys2.outputs.cache-hit != 'true' + shell: python scripts\msys-bash.py {0} + run: ./.github/scripts/shrink-msys2.sh + - name: Log environment + shell: python scripts\msys-bash.py {0} + run: ./scripts/log-env.sh + - name: Debug Build + shell: python scripts\msys-bash.py {0} + run: ./scripts/build.sh --compiler ${{ matrix.compiler }} --build-type Debug --bin-path /mingw${{ matrix.bits }}/bin + - name: Debug Warnings + run: '.\scripts\count-warnings.py -m $env:MAX_WARNINGS_${{ matrix.compiler }}_${{ matrix.bits }}bit_Debug build.log'