diff --git a/.github/workflows/analysis.yml b/.github/workflows/analysis.yml index 5bc0407d..43c2dde1 100644 --- a/.github/workflows/analysis.yml +++ b/.github/workflows/analysis.yml @@ -1,8 +1,13 @@ name: Code analysis + on: [push, pull_request] -jobs: +env: + CCACHE_DIR: "/dev/shm/.ccache" + CCACHE_MAXSIZE: "64M" + CCACHE_COMPRESS: "true" +jobs: run_linters: name: Script linters runs-on: ubuntu-18.04 @@ -15,13 +20,13 @@ jobs: - name: Run shellcheck run: ./scripts/verify-bash.sh - name: Install pylint - run: | + run: | sudo apt-get install python3-setuptools sudo pip3 install pylint beautifulsoup4 html5lib - name: Run pylint run: ./scripts/verify-python.sh - name: Install markdownlint - run: | + run: | sudo apt-get install ruby sudo gem install mdl sudo chmod go-w /usr/share/rust/.cargo/bin @@ -41,15 +46,31 @@ jobs: run: sudo apt-get install python3-setuptools $(./scripts/list-build-dependencies.sh -m apt -c gcc) - name: Install scan-build (Python version) run: sudo pip3 install scan-build beautifulsoup4 html5lib + + - name: Prepare compiler cache + id: prep-ccache + shell: bash + run: | + mkdir -p "${CCACHE_DIR}" + echo "::set-output name=dir::$CCACHE_DIR" + echo "::set-output name=today::$(date +%Y%M%d)" + echo "::set-output name=yesterday::$(date --date=yesterday +%Y%M%d)" + - uses: actions/cache@v1 + id: cache-ccache + with: + path: ${{ steps.prep-ccache.outputs.dir }} + key: ccache-static-clang-${{ steps.prep-ccache.outputs.today }} + restore-keys: | + ccache-static-clang-${{ steps.prep-ccache.outputs.yesterday }} - name: Log environment run: ./scripts/log-env.sh - name: Build - run: | + run: | # build steps set -x g++ --version ./autogen.sh - ./configure + ./configure CC="ccache gcc" CXX="ccache g++" intercept-build make -j "$(nproc)" - name: Analyze run: analyze-build -v -o report --html-title="dosbox-staging (${GITHUB_SHA:0:8})" @@ -79,7 +100,7 @@ jobs: fetch-depth: 1 - run: sudo apt-get update - name: Log and setup environment - run: | + run: | ./scripts/log-env.sh mkdir -p pvs-package - uses: actions/cache@v1 @@ -91,20 +112,20 @@ jobs: if: steps.cache-pvs.outputs.cache-hit != 'true' run: wget "https://files.viva64.com/${debfile}" -O "pvs-package/pvs.deb" - name: Install packages - run: | + run: | set -xeu sudo apt-get install strace $(./scripts/list-build-dependencies.sh -m apt -c gcc) sudo dpkg -i "pvs-package/pvs.deb" pvs-studio-analyzer credentials "${{ secrets.PvsStudioName }}" "${{ secrets.PvsStudioKey }}" - name: Build - run: | + run: | set -xeu ./autogen.sh export FLAGS="-Og" ./configure CFLAGS="${FLAGS}" CXXFLAGS="${FLAGS}" pvs-studio-analyzer trace -- make - name: Analyze - run: | + run: | set -xeu log="pvs-analysis.log" mirsa_criteria="MISRA:1,2" @@ -165,6 +186,21 @@ jobs: run: > sudo apt-get install -y $(./scripts/list-build-dependencies.sh -m apt -c ${{ matrix.conf.name }} -v $VERSION_${{ matrix.conf.name }}) + - name: Prepare compiler cache + id: prep-ccache + shell: bash + run: | + mkdir -p "${CCACHE_DIR}" + echo "::set-output name=dir::$CCACHE_DIR" + echo "::set-output name=today::$(date +%Y%M%d)" + echo "::set-output name=yesterday::$(date --date=yesterday +%Y%M%d)" + - uses: actions/cache@v1 + id: cache-ccache + with: + path: ${{ steps.prep-ccache.outputs.dir }} + key: ccache-sanitizers-${{ matrix.conf.name }}-${{ steps.prep-ccache.outputs.today }} + restore-keys: | + ccache-sanitizers-${{ matrix.conf.name }}-${{ steps.prep-ccache.outputs.yesterday }} - name: Log environment run: ./scripts/log-env.sh - name: Build and run sanitizers diff --git a/.github/workflows/config.yml b/.github/workflows/config.yml index 58a8fe76..f90670e5 100644 --- a/.github/workflows/config.yml +++ b/.github/workflows/config.yml @@ -62,7 +62,7 @@ jobs: steps.repo-meta.outputs.has-commits == 'true' with: path: 'C:/tools/msys64' - key: msys2-64-${{ steps.get-date.outputs.date }}-2 + key: msys2-64-${{ steps.get-date.outputs.date }}-3 - name: Install MSYS2 (Windows) if: > @@ -77,7 +77,7 @@ jobs: steps.cache-msys2.outputs.cache-hit != 'true' && steps.repo-meta.outputs.has-commits == 'true' shell: python scripts\msys-bash.py {0} - run: | + run: | scripts/list-build-dependencies.sh -m msys2 -c clang ${{ matrix.conf.without_packages }} | xargs pacman -S --noconfirm .github/scripts/shrink-msys2.sh @@ -85,7 +85,7 @@ jobs: if: > matrix.system.name == 'Linux' && steps.repo-meta.outputs.has-commits == 'true' - run: | + run: | sudo apt-get -y update sudo apt-get install -y $(./scripts/list-build-dependencies.sh -m apt -c gcc ${{ matrix.conf.without_packages }}) @@ -106,7 +106,7 @@ jobs: if: > matrix.system.name == 'macOS' && steps.repo-meta.outputs.has-commits == 'true' - run: | + run: | export PKG_CONFIG_PATH="/usr/local/opt/ncurses/lib/pkgconfig:$PKG_CONFIG_PATH" ./scripts/build.sh -c ${{ matrix.system.compiler }} -t debug ${{ matrix.conf.configure_flags }} diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index 342b7685..504b8440 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -22,19 +22,19 @@ jobs: run: ./scripts/log-env.sh - name: Install C++ compiler and dependencies - run: | + run: | sudo apt-get install zstd python3-setuptools $(./scripts/list-build-dependencies.sh -m apt -c gcc) sudo pip3 install gdown - name: Fetch the Coverity instrumenteur - run: | + run: | set -xeuo pipefail gdown --id "${TARBALL_GDRIVE_ID}" -O - \ | tee >(tar -I zstd -C /dev/shm -x) \ | sha256sum -c <(echo "${TARBALL_SHA256} -" ) - name: Build and instrument the project - run: | + run: | set -xeu g++ --version ./autogen.sh @@ -46,7 +46,7 @@ jobs: tar -cvaf package.tar.gz cov-int - name: Upload the package to Coverity - run: | + run: | curl \ --form token="${COVERITY_TOKEN}" \ --form email="${COVERITY_EMAIL}" \ diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index a021082e..f1be4dac 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -1,8 +1,13 @@ name: Linux builds + on: [push, pull_request] -jobs: +env: + CCACHE_DIR: "/dev/shm/.ccache" + CCACHE_MAXSIZE: "64M" + CCACHE_COMPRESS: "true" +jobs: build_ubuntu: name: ${{ matrix.conf.name }} runs-on: ${{ matrix.conf.os }} @@ -33,6 +38,21 @@ jobs: - run: sudo apt-get update - name: Install C++ compiler and libraries run: sudo apt-get install -y $(./scripts/list-build-dependencies.sh -m apt ${{ matrix.conf.flags }}) + - name: Prepare compiler cache + id: prep-ccache + shell: bash + run: | + mkdir -p "${CCACHE_DIR}" + echo "::set-output name=dir::$CCACHE_DIR" + echo "::set-output name=today::$(date +%Y%M%d)" + echo "::set-output name=yesterday::$(date --date=yesterday +%Y%M%d)" + - uses: actions/cache@v1 + id: cache-ccache + with: + path: ${{ steps.prep-ccache.outputs.dir }} + key: ccache-${{ matrix.conf.name }}-${{ steps.prep-ccache.outputs.today }} + restore-keys: | + ccache-${{ matrix.conf.name }}-${{ steps.prep-ccache.outputs.yesterday }} - name: Log environment run: ./scripts/log-env.sh - name: Build @@ -51,6 +71,21 @@ jobs: - run: sudo apt-get update - name: Install C++ compiler and libraries run: sudo apt-get install -y tree libpng16-dev $(./scripts/list-build-dependencies.sh -m apt -c gcc) + - name: Prepare compiler cache + id: prep-ccache + shell: bash + run: | + mkdir -p "${CCACHE_DIR}" + echo "::set-output name=dir::$CCACHE_DIR" + echo "::set-output name=today::$(date +%Y%M%d)" + echo "::set-output name=yesterday::$(date --date=yesterday +%Y%M%d)" + - uses: actions/cache@v1 + id: cache-ccache + with: + path: ${{ steps.prep-ccache.outputs.dir }} + key: ccache-linux-release-${{ steps.prep-ccache.outputs.today }} + restore-keys: | + ccache-linux-release-${{ steps.prep-ccache.outputs.yesterday }} - name: Log environment run: ./scripts/log-env.sh - name: Inject version string @@ -65,7 +100,7 @@ jobs: run: | set -x ./autogen.sh - ./configure CFLAGS="$FLAGS" CXXFLAGS="$FLAGS" --disable-screenshots + ./configure CC="ccache gcc" CXX="ccache g++" CFLAGS="$FLAGS" CXXFLAGS="$FLAGS" --disable-screenshots make -j "$(nproc)" strip src/dosbox - name: Package diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 7d0c5b99..b1325d5d 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -1,8 +1,13 @@ name: macOS builds + on: [push, pull_request] -jobs: +env: + CCACHE_DIR: "${{ github.workspace }}/.ccache" + CCACHE_MAXSIZE: "64M" + CCACHE_COMPRESS: "true" +jobs: build_macos: name: ${{ matrix.conf.name }} runs-on: macos-latest @@ -22,6 +27,21 @@ jobs: fetch-depth: 1 - name: Install C++ compiler and libraries run: brew install $(./scripts/list-build-dependencies.sh -m brew ${{ matrix.conf.flags}}) + - name: Prepare compiler cache + id: prep-ccache + shell: bash + run: | + mkdir -p "${CCACHE_DIR}" + echo "::set-output name=dir::$CCACHE_DIR" + echo "::set-output name=today::$(date +%Y%M%d)" + echo "::set-output name=yesterday::$(date --date=yesterday +%Y%M%d)" + - uses: actions/cache@v1 + id: cache-ccache + with: + path: ${{ steps.prep-ccache.outputs.dir }} + key: ccache-macos-${{ matrix.conf.name }}-${{ steps.prep-ccache.outputs.today }} + restore-keys: | + ccache-macos-${{ matrix.conf.name }}-${{ steps.prep-ccache.outputs.yesterday }} - name: Log environment run: ./scripts/log-env.sh - name: Build @@ -35,16 +55,37 @@ jobs: name: Release build runs-on: macos-latest if: github.event_name != 'pull_request' || contains('dreamer,krcroft,ant-222', github.actor) == false + env: + CC: ccache clang + CXX: ccache clang++ + FLAGS: -O3 -DNDEBUG -pipe -march=nehalem steps: - uses: actions/checkout@v1 - name: Install C++ compiler and libraries run: brew install librsvg $(./scripts/list-build-dependencies.sh -m brew -c clang) + - name: Prepare compiler cache + id: prep-ccache + shell: bash + run: | + mkdir -p "${CCACHE_DIR}" + echo "::set-output name=dir::$CCACHE_DIR" + echo "::set-output name=today::$(date +%Y%M%d)" + echo "::set-output name=yesterday::$(date --date=yesterday +%Y%M%d)" + - uses: actions/cache@v1 + id: cache-ccache + with: + path: ${{ steps.prep-ccache.outputs.dir }} + key: ccache-macos-release-${{ steps.prep-ccache.outputs.today }} + restore-keys: | + ccache-macos-release-${{ steps.prep-ccache.outputs.yesterday }} - name: Log environment run: ./scripts/log-env.sh - name: Build Opus libraries run: | set -x cd contrib/static-opus + export CFLAGS="$FLAGS" + export CXXFLAGS="$FLAGS" gmake -j "$(sysctl -n hw.physicalcpu)" - name: Inject version string run: | @@ -54,7 +95,6 @@ jobs: echo ::set-env name=VERSION::$VERSION - name: Build env: - FLAGS: -O3 -DNDEBUG -pipe -march=nehalem OPUSFILE_CFLAGS: -I${{ github.workspace }}/contrib/static-opus/include -I${{ github.workspace }}/contrib/static-opus/include/opus OPUSFILE_LIBS: ${{ github.workspace }}/contrib/static-opus/lib/libopusfile.a ${{ github.workspace }}/contrib/static-opus/lib/libogg.a ${{ github.workspace }}/contrib/static-opus/lib/libopus.a -lm run: | @@ -69,7 +109,6 @@ jobs: gmake -j "$(sysctl -n hw.physicalcpu)" strip src/dosbox otool -L src/dosbox - - name: Package run: | set -x diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index d41a1901..816dcd96 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -1,8 +1,8 @@ name: Windows builds + on: [push, pull_request] jobs: - build_windows_msys2: name: ${{ matrix.conf.compiler }} ${{ matrix.conf.bits }}-bit runs-on: windows-latest @@ -31,7 +31,10 @@ jobs: arch: x86_64 max_warnings: 82 env: - CHERE_INVOKING: yes + CHERE_INVOKING: yes + CCACHE_DIR: "${{ github.workspace }}/.ccache" + CCACHE_MAXSIZE: "64M" + CCACHE_COMPRESS: "true" steps: - uses: actions/checkout@v1 with: @@ -39,12 +42,12 @@ jobs: - name: Get Date id: get-date shell: bash - run: echo ::set-output name=date::$(date +%Y-%W) + run: echo "::set-output name=date::$(date +%Y-%W)" - uses: actions/cache@v1 id: cache-msys2 with: path: 'C:/tools/msys64' - key: msys2-${{ matrix.conf.bits }}-${{ steps.get-date.outputs.date }}-2 + key: msys2-${{ matrix.conf.bits }}-${{ steps.get-date.outputs.date }}-3 - name: Install MSYS2 if: steps.cache-msys2.outputs.cache-hit != 'true' run: choco install msys2 --no-progress @@ -56,6 +59,21 @@ jobs: if: steps.cache-msys2.outputs.cache-hit != 'true' shell: python scripts\msys-bash.py {0} run: ./.github/scripts/shrink-msys2.sh + - name: Prepare compiler cache + id: prep-ccache + shell: bash + run: | + mkdir -p "${CCACHE_DIR}" + echo "::set-output name=dir::$CCACHE_DIR" + echo "::set-output name=today::$(date +%Y%M%d)" + echo "::set-output name=yesterday::$(date --date=yesterday +%Y%M%d)" + - uses: actions/cache@v1 + id: cache-ccache + with: + path: ${{ steps.prep-ccache.outputs.dir }} + key: ccache-msys2-${{ matrix.conf.compiler }}-${{ matrix.conf.bits }}-${{ steps.prep-ccache.outputs.today }} + restore-keys: | + ccache-msys2-${{ matrix.conf.compiler }}-${{ matrix.conf.bits }}-${{ steps.prep-ccache.outputs.yesterday }} - name: Log environment shell: python scripts\msys-bash.py {0} run: ./scripts/log-env.sh diff --git a/scripts/automator/build/clang-linux_armv7l b/scripts/automator/build/clang-linux_armv7l new file mode 100644 index 00000000..e69de29b diff --git a/scripts/automator/build/gcc-linux_armv7l b/scripts/automator/build/gcc-linux_armv7l new file mode 100644 index 00000000..e69de29b diff --git a/scripts/automator/build/machine-armv7l b/scripts/automator/build/machine-armv7l new file mode 100644 index 00000000..e69de29b diff --git a/scripts/automator/build/os-defaults b/scripts/automator/build/os-defaults index 92896a59..5c003879 100644 --- a/scripts/automator/build/os-defaults +++ b/scripts/automator/build/os-defaults @@ -1,6 +1,6 @@ # Steps in-common to all OSes, plus some helper variables and functions. -STEPS=(pre_build clean autogen configure make_binary strip_binary dependencies post_build) +STEPS=(pre_build clean autogen configure make_binary strip_binary dependencies ccache_stats post_build) executable="src/dosbox" dependencies=(ldd "${executable}") @@ -35,6 +35,13 @@ function configure() { fi } +function ccache_stats() { + if [[ -n "${ccache}" ]]; then + underline "Cache statistics" "-" + ccache -s | grep --color=never 'hit\|size' + fi +} + function strip_binary() { if [[ "${selected_type}" == "release" ]]; then strip "${executable}" diff --git a/scripts/automator/packages/manager-apt b/scripts/automator/packages/manager-apt index c0f63623..f87318cb 100644 --- a/scripts/automator/packages/manager-apt +++ b/scripts/automator/packages/manager-apt @@ -1,2 +1,2 @@ # Package repo: https://packages.ubuntu.com/ -packages+=(xvfb libtool build-essential libsdl2-dev libsdl2-net-dev libncurses-dev libopusfile-dev) +packages+=(ccache xvfb libtool build-essential libsdl2-dev libsdl2-net-dev libncurses-dev libopusfile-dev) diff --git a/scripts/automator/packages/manager-brew b/scripts/automator/packages/manager-brew index 45ffb4fd..44b4c736 100644 --- a/scripts/automator/packages/manager-brew +++ b/scripts/automator/packages/manager-brew @@ -1,3 +1,3 @@ # Package repo: https://formulae.brew.sh/ delim="@" -packages+=(coreutils autogen autoconf automake make pkg-config libpng ncurses sdl2 sdl2_net opusfile) +packages+=(ccache coreutils autogen autoconf automake make pkg-config libpng ncurses sdl2 sdl2_net opusfile) diff --git a/scripts/automator/packages/manager-dnf b/scripts/automator/packages/manager-dnf index 7c202b6d..dead6f29 100644 --- a/scripts/automator/packages/manager-dnf +++ b/scripts/automator/packages/manager-dnf @@ -1,2 +1,2 @@ # Package repo: https://apps.fedoraproject.org/packages/ -packages+=(xvfb libtool ncurses-devel SDL2-devel SDL2_net-devel opusfile-devel) +packages+=(ccache xvfb libtool ncurses-devel SDL2-devel SDL2_net-devel opusfile-devel) diff --git a/scripts/automator/packages/manager-macports b/scripts/automator/packages/manager-macports index 77657a0c..445a896a 100644 --- a/scripts/automator/packages/manager-macports +++ b/scripts/automator/packages/manager-macports @@ -1,3 +1,3 @@ # Package repo: https://www.macports.org/ports.php?by=name delim="" -packages+=(coreutils autogen autoconf automake pkgconfig libpng ncurses libsdl2 libsdl2_net opusfile) +packages+=(ccache coreutils autogen autoconf automake pkgconfig libpng ncurses libsdl2 libsdl2_net opusfile) diff --git a/scripts/automator/packages/manager-msys2 b/scripts/automator/packages/manager-msys2 index 1d79c558..1c7f1fd6 100644 --- a/scripts/automator/packages/manager-msys2 +++ b/scripts/automator/packages/manager-msys2 @@ -2,6 +2,6 @@ # MSYS2 only supports the current latest releases of Clang and GCC, so we disable version customization packages+=(autogen autoconf base-devel automake-wrapper binutils) pkg_type=$([[ "${bits}" == "64" ]] && echo "x86_64" || echo "i686") -for pkg in pkg-config libtool libpng zlib ncurses pdcurses SDL2 SDL2_net opusfile; do +for pkg in ccache pkg-config libtool libpng zlib ncurses pdcurses SDL2 SDL2_net opusfile; do packages+=("mingw-w64-${pkg_type}-${pkg}") done diff --git a/scripts/automator/packages/manager-pacman b/scripts/automator/packages/manager-pacman index b3e9a6b8..ca55866d 100644 --- a/scripts/automator/packages/manager-pacman +++ b/scripts/automator/packages/manager-pacman @@ -1,4 +1,4 @@ # Package repo: https://www.archlinux.org/packages/ # Arch offers 32-bit versions of SDL (but not others) -packages+=(xvfb libtool ncurses sdl2_net opusfile) +packages+=(ccache xvfb libtool ncurses sdl2_net opusfile) [[ "${bits}" == "32" ]] && packages+=(lib32-sdl2) || packages+=(sdl2) diff --git a/scripts/automator/packages/manager-zypper b/scripts/automator/packages/manager-zypper index 9344106c..73eedb99 100644 --- a/scripts/automator/packages/manager-zypper +++ b/scripts/automator/packages/manager-zypper @@ -1,6 +1,6 @@ # Package repo: https://pkgs.org/ # openSUSE offers 32-bit versions of SDL and SDL_net (but not others) -packages+=(devel_basis xvfb libtool opusfile) +packages+=(ccache devel_basis xvfb libtool opusfile) if [[ "${bits}" == "32" ]]; then packages+=(ncurses-devel-32bit libSDL2-devel-32bit libSDL2_net-devel-32bit)