Refactor the build and list-packages scripts

Until now the build and package scripts have supported
several architectures, compilers, build types, package-managers,
and bit-depth targets.

The code might be maintainable if left as-such, however we have plans
to continue expanding the number of architectures (ARM, PPC, ... ),
operating systems (Android, BSDs, ...), and build variations
amung those.

The scripts (regardless of language) would only grow in complexity
as more variations are added. Thus, we needed a solution that can
scale without adding complexity.

To achieve this, the scripts were refactored as follows:
 - all "data" was moved out of code into configuration files
 - A back-end "Automator" engine was written to parse the
   data based on generic variables fed to it by a front-end
   script
 - build.sh and list-packages.sh were re-authored as thin front-end
   scripts that drive the automator
 - Their CLI's were retained so there has been very little change
   needed to the CI invocation lines.  The only changes have been to
   clarify the existing arguments improved based on feedback, ie:
   --build-type release, --build-type debug instead of fast, small

scripts/automator/build/clang-darwin_x86_64

@@ -0,0 +1,8 @@
+ld="ld"
+
+TYPES+=("msan" "usan")
+cflags_msan=("${cflags_debug[@]}" "-fsanitize-recover=all" "-fsanitize=memory" "-fno-omit-frame-pointer")
+cflags_usan=("${cflags_debug[@]}" "-fsanitize-recover=all" "-fsanitize=undefined")
+
+MODIFIERS+=("lto")
+cflags_lto=("-flto=thin")

scripts/automator/build/clang-defaults

@@ -0,0 +1,16 @@
+# Tool overrides
+cc="clang${postfix}"
+cxx="clang++${postfix}"
+
+# Flag additions
+TYPES+=("debug" "profile")
+cflags+=("-fcolor-diagnostics")
+cflags_release=("${cflags[@]}" "-Os")
+cflags_debug=("${cflags[@]}" "-g" "-Og" "-fno-omit-frame-pointer")
+cflags_profile=("${cflags_debug[@]}" "-fprofile-instr-generate" "-fcoverage-mapping")
+
+
+# Modifier additions
+MODIFIERS=("fdo")
+ldflags_fdo=("-fprofile-instr-generate")
+cflags_fdo=("-fprofile-sample-use=${FDO_FILE:-}")

scripts/automator/build/clang-linux_x86_64

@@ -0,0 +1,13 @@
+# Tool additions
+ar="llvm-ar${postfix}"
+ld="llvm-link${postfix}"
+ranlib="llvm-ranlib${postfix}"
+
+# Build additions
+TYPES+=("msan" "usan")
+cflags_msan=("${cflags_debug[@]}" "-fsanitize-recover=all" "-fsanitize=memory" "-fno-omit-frame-pointer")
+cflags_usan=("${cflags_debug[@]}" "-fsanitize-recover=all" "-fsanitize=undefined")
+
+# Modifier additions
+MODIFIERS+=("lto")
+cflags_lto=("-O2" "-flto=thin")

scripts/automator/build/clang-msys_nt_x86_64

@@ -0,0 +1,5 @@
+# Flag additions
+ar="llvm-ar${postfix}"
+ld="llvm-link${postfix}"
+ranlib="llvm-ranlib${postfix}"
+ldflags+=("-static-libgcc" "-static-libstdc++")

scripts/automator/build/compiler-defaults

@@ -0,0 +1,14 @@
+# Tools and flags for all compilers
+VARIABLES=("ar" "cc" "cxx" "ld" "ranlib" "cflags" "ldflags" "libs")
+ar=""
+cc=""
+cxx=""
+ld=""
+ranlib=""
+cflags=("-Wall" "-pipe")
+ldflags=("")
+libs=("")
+
+# Builds for all compilers
+TYPES=("release")
+cflags_release=("${cflags[@]}")

scripts/automator/build/gcc-darwin_x86_64

@@ -0,0 +1,15 @@
+# Tool overrides
+ar="ar"
+ranlib="ranlib"
+
+# Build additions
+TYPES+=("asan" "uasan" "usan" "tsan")
+cflags_asan=("${cflags_debug[@]}" "-fsanitize=address")
+cflags_uasan=("${cflags_debug[@]}" "-fsanitize=address,undefined" "-fsanitize-recover=signed-integer-overflow")
+cflags_usan=("${cflags_debug[@]}" "-fsanitize=undefined" "-fsanitize-recover=signed-integer-overflow")
+cflags_tsan=("${cflags_debug[@]}" "-fsanitize=thread")
+
+# Modifier additions
+MODIFIERS+=("lto")
+cflags_lto=("-flto")
+ldflags_lto=("${cflags[@]}" "-flto")

scripts/automator/build/gcc-defaults

@@ -0,0 +1,17 @@
+# Tool overrides
+ar="gcc-ar${postfix}"
+cc="gcc${postfix}"
+cxx="g++${postfix}"
+ld="gcc${postfix}"
+ranlib="gcc-ranlib${postfix}"
+
+# Flag additions
+TYPES+=("debug" "profile")
+cflags+=("-fstack-protector" "-fdiagnostics-color=always")
+cflags_release=("${cflags[@]}" "-Ofast" "-ffunction-sections" "-fdata-sections")
+cflags_debug=("${cflags[@]}" "-g" "-Og" "-fno-omit-frame-pointer")
+cflags_profile=("${cflags_debug[@]}" "-pg")
+
+# Modifier additions
+MODIFIERS=("fdo")
+cflags_fdo=("-fauto-profile=${FDO_FILE:-}")

scripts/automator/build/gcc-linux_x86_64

@@ -0,0 +1,14 @@
+# Tool additions
+ldflags+=("-Wl,--as-needed")
+
+# Build additions
+TYPES+=("asan" "uasan" "usan" "tsan")
+cflags_asan=("${cflags_debug[@]}" "-fsanitize=address")
+cflags_uasan=("${cflags_debug[@]}" "-fsanitize=address,undefined" "-fsanitize-recover=signed-integer-overflow")
+cflags_usan=("${cflags_debug[@]}" "-fsanitize=undefined" "-fsanitize-recover=signed-integer-overflow")
+cflags_tsan=("${cflags_debug[@]}" "-fsanitize=thread")
+
+# Modifier additions
+MODIFIERS+=("lto")
+cflags_lto=("-flto")
+ldflags_lto=("${cflags[@]}" "-flto=$(( $(nproc) + 2 ))")

scripts/automator/build/gcc-msys_nt_x86_64

@@ -0,0 +1,3 @@
+# Flag additions
+ldflags+=("-Wl,--as-needed" "-static-libgcc" "-static-libstdc++")
+

scripts/automator/build/machine-x86_64

@@ -0,0 +1,3 @@
+# Modifier additions universal for gcc and clang, but specific to x86_64:
+MODIFIERS+=("native")
+cflags_native=("-march=native")

scripts/automator/build/os-darwin

@@ -0,0 +1,5 @@
+# Tool additions and overrides for Darwin, regardless of compiler
+ar="ar"
+ranlib="ranlib"
+function make_binary() { make -j$(sysctl -n hw.physicalcpu) 2>&1 | tee build.log; }
+dependencies=("otool" "-L" "${executable}")

scripts/automator/build/os-defaults

@@ -0,0 +1,57 @@
+# Steps in-common to all OSes, plus some helper variables and functions.
+
+STEPS=("pre_build" "clean" "autogen" "configure" "make_binary" "strip_binary" "dependencies" "post_build")
+
+executable="src/dosbox"
+dependencies=("ldd" "${executable}")
+
+function pre_build() {
+	cd ../..
+	underline "Environment"
+	echo "[$("${CC}" --version | head -1)]"
+	for v in "${VARIABLES[@]}"; do
+		vupper=$(upper "${v}")
+		echo "${vupper}=\"$(eval echo \$"${vupper}")\""
+	done
+	underline "Launching"
+}
+
+function stamp() {
+	echo "${compiler}-${selected_type}-${modifiers[*]}"
+}
+
+function autogen() {
+	if [[ ! -f configure || configure.ac -nt configure ]]; then
+		./autogen.sh
+	fi
+}
+
+function configure() {
+	if [[ ! -f Makefile || ! -f .previous_build || configure -nt Makefile ]]; then
+		export CXXFLAGS="${CFLAGS}"
+		./configure --enable-core-inline
+	fi
+}
+
+function strip_binary() {
+	if [[ "${selected_type}" == "release" ]]; then
+		strip "${executable}"
+	fi
+}
+
+function post_build() {
+	underline "Binary" "-"
+	ls -1lh "${executable}"
+	stamp > .previous_build
+}
+
+function clean() {
+	stamp > .current_build
+	if [[ -f Makefile ]]; then
+		if [[ ! -f .previous_build ]] || ! diff -q .previous_build .current_build &> /dev/null; then
+			rm -f .previous_build
+			echo "cleaning, this is a different build type"
+			make clean &> clean.log || cat clean.log
+		fi
+	fi
+}

scripts/automator/build/os-linux

@@ -0,0 +1,2 @@
+function make_binary() { make -j$(nproc) 2>&1 | tee build.log; }
+

scripts/automator/build/os-msys_nt

@@ -0,0 +1,3 @@
+function make_binary() { make -j$(nproc) 2>&1 | tee build.log; }
+executable="src/dosbox.exe"
+

scripts/automator/main.sh

@@ -0,0 +1,120 @@
+#!/bin/bash
+
+# Copyright (c) 2019 Kevin R Croft <krcroft@gmail.com>
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+# Automator is a generic automation tool that helps separate data
+# from logical code.  It uses one or more "variables" files placed
+# in a single sub-directory, that are sourced in logical succession,
+# such as variables defined in earler files are either added-to or
+# overridden in subsequent files.
+#
+# For a thorough example, see scripts/build.sh and its data files
+# in scripts/automator/build.
+#
+# TODO: This script manages variables using eval meta-programming.
+#       Although the syntax is quite readable under bash 4.x,
+#       we have limitted our syntax to the uglier and heavier bash 3.x
+#       because Apple is still only shipping bash 3.x, even on their
+#       latest operating system (Nov-2019).
+#
+#       Future work might involve switching this to python and using YAML
+#       syntax in the variables files. Python's dictionaries can be
+#       merged or have their values overridden, and key names can be readily
+#       manipulated.
+#
+set -euo pipefail
+
+function underline() {
+	echo ""
+	echo "$1"
+	echo "${1//?/${2:--}}"
+}
+
+function lower() {
+	echo "${1}" | tr '[:upper:]' '[:lower:]'
+}
+
+function upper() {
+	echo "${1}" | tr '[:lower:]' '[:upper:]'
+}
+
+function arg_error() {
+	local PURPLE='\033[0;34m'
+	local BLACK='\033[0;30m'
+	local ON_WHITE='\033[47m'
+	local NO_COLOR='\033[0m'
+	>&2 echo -e "Please specify the ${BLACK}${ON_WHITE}${1}${NO_COLOR} argument with one of: ${PURPLE}${2}${NO_COLOR}"
+	exit 1
+}
+
+function import() { # arguments: category instance
+	category=$(lower "${1}")
+	instance=$(lower "${2}")
+	if [[ -z "$instance" ]]; then
+		instances=( "$(find "${data_dir}" -name "${category}-*" -a ! -name '*defaults' | sed 's/.*-//' | xargs)" )
+		arg_error "--${category}" "${instances[*]}"
+	else for instance in defaults "${instance}"; do
+		varfile="${data_dir}/${category}-${instance}"
+		# shellcheck disable=SC1090,SC1091
+		if [[ -f "$varfile" ]]; then source "$varfile"
+		else >&2 echo "${varfile} does not exist, skipping"; fi; done
+	fi
+}
+
+function construct_environment() {
+	# underline "Environment"
+	for base_var in "${VARIABLES[@]}"; do
+		# First check if we have a build-specific variable
+		# shellcheck disable=SC2154
+		if [[ -n "$(eval 'echo ${'"${base_var}_${selected_type}"'[*]:-}')" ]]; then
+			var_name="${base_var}_${selected_type}"
+		# Otherwise fallback to the default variable
+		else var_name="${base_var}"; fi
+
+		# Aggregate any modifiers, which are stand-alone arguments
+		mod_values=""
+		# shellcheck disable=SC2154
+		for mod_upper in "${modifiers[@]}"; do
+			mod=$(lower "${mod_upper}")
+			# Mods can only add options to existing VARs, so we check for those
+			if [[ -n "$(eval 'echo ${'"${base_var}_${mod}"'[*]:-}')" ]]; then
+				var_with_mod="${base_var}_${mod}"
+				# shellcheck disable=SC2034
+				mod_values=$(eval 'echo ${mod_values} ${'"${var_with_mod}"'[*]}')
+			fi
+		done
+
+		# Expand our variable array (or scalar) and combine it with our aggregated
+		# modification string. Echo takes care of spacing out our variables without
+		# explicit padding, because it either drops empty variables or adds spaces
+		# between them if they exist.
+		# shellcheck disable=SC2034
+		combined=$(eval 'echo ${'"${var_name}"'[*]} ${mod_values}')
+		env_var=$(upper "${base_var}")
+		eval 'export '"${env_var}"'="${combined}"'
+		# echo "${env_var}=\"$(eval echo \$"${env_var}")\""
+	done
+}
+
+function perform_steps() {
+	# underline "Launching"
+	for step in "${STEPS[@]}"; do
+		if type -t "$step" | grep -q function; then "$step"
+		else
+			# eval 'echo ${'"${step}"'[*]}'
+			eval '${'"${step}"'[*]}'
+		fi
+	done
+}
+
+function main() {
+	cd "$(dirname "$0")/automator"
+	if [[ -z "${data_dir:-}" ]]; then data_dir="$(basename "$0" '.sh')"; fi
+	parse_args "$@"
+	construct_environment
+	perform_steps
+}
+
+main "$@"
+

scripts/automator/packages/clang-brew

@@ -0,0 +1,3 @@
+# Brew does not supply "clang", so exclude it here
+compiler=""
+

scripts/automator/packages/clang-defaults

@@ -0,0 +1,2 @@
+compiler=(clang${postfix})
+

scripts/automator/packages/clang-macports

@@ -0,0 +1,2 @@
+# macports doesn't supply Clang, so knock it out here
+compiler=""

scripts/automator/packages/clang-msys2

@@ -0,0 +1,4 @@
+# Under MSYS2, only the 'default' version is available, so we don't postfix the package
+compiler="mingw-w64-${pkg_type}-${selected_type}"
+
+

scripts/automator/packages/clang-vcpkg

@@ -0,0 +1,4 @@
+# vcpkg doesn't provide clang or gcc, because it assumes you're using MS VisualStudio.
+# So we knock out the compiler.
+compiler=""
+

scripts/automator/packages/gcc-apt

@@ -0,0 +1,2 @@
+compiler=(g++${postfix})
+

scripts/automator/packages/gcc-defaults

@@ -0,0 +1 @@
+compiler=(gcc${postfix})

scripts/automator/packages/gcc-msys2

@@ -0,0 +1,3 @@
+# Under MSYS2 only the 'default' version of gcc is available, so we don't postfix it with the version
+compiler="mingw-w64-${pkg_type}-${selected_type}"
+

scripts/automator/packages/gcc-vcpkg

@@ -0,0 +1,4 @@
+# vcpkg doesn't provide clang or gcc, because it assumes you're using MS VisualStudio.
+# So we knock out the compiler.               
+compiler=""
+

scripts/automator/packages/manager-apt

@@ -0,0 +1,2 @@
+# Package repo: https://packages.ubuntu.com/
+packages+=(xvfb libtool build-essential libsdl1.2-dev libsdl-net1.2-dev libopusfile-dev)

scripts/automator/packages/manager-brew

@@ -0,0 +1,3 @@
+# Package repo: https://formulae.brew.sh/
+delim="@"
+packages+=(coreutils autogen autoconf automake pkg-config libpng sdl sdl_net opusfile)

scripts/automator/packages/manager-defaults

@@ -0,0 +1,11 @@
+VARIABLES=(packages bits delim compiler)
+TYPES=(gcc clang)
+
+packages=(autoconf-archive zstd)
+delim="-"
+compiler=""
+
+STEPS=(print)
+function print() {
+	echo "${PACKAGES[*]}" "${COMPILER}"
+}

scripts/automator/packages/manager-dnf

@@ -0,0 +1,2 @@
+# Package repo: https://apps.fedoraproject.org/packages/
+packages+=(xvfb libtool SDL SDL_net-devel opusfile-devel)

scripts/automator/packages/manager-macports

@@ -0,0 +1,3 @@
+# Package repo: https://www.macports.org/ports.php?by=name
+delim=""
+packages+=(coreutils autogen autoconf automake pkgconfig libpng libsdl libsdl_net opusfile)

scripts/automator/packages/manager-msys2

@@ -0,0 +1,7 @@
+# Package repo: https://packages.msys2.org/base
+# MSYS2 only supports the current latest releases of Clang and GCC, so we disable version customization
+packages+=(autogen autoconf base-devel automake-wrapper binutils)
+pkg_type=$([[ "${bits}" == "64" ]] && echo "x86_64" || echo "i686")
+for pkg in pkg-config libtool libpng zlib SDL SDL_net opusfile; do
+	packages+=("mingw-w64-${pkg_type}-${pkg}")
+done

scripts/automator/packages/manager-pacman

@@ -0,0 +1,4 @@
+# Package repo: https://www.archlinux.org/packages/
+# Arch offers 32-bit versions of SDL (but not others)
+packages+=(xvfb libtool sdl_net opusfile)
+[[ "${bits}" == "32" ]] && packages+=(lib32-sdl) || packages+=(sdl)

scripts/automator/packages/manager-vcpkg

@@ -0,0 +1,2 @@
+# Package repo: https://repology.org/projects/?inrepo=vcpkg
+packages+=(libpng sdl1 sdl1-net opusfile)

scripts/automator/packages/manager-zypper

@@ -0,0 +1,10 @@
+# Package repo: https://pkgs.org/
+# openSUSE offers 32-bit versions of SDL and SDL_net (but not others)
+packages+=(devel_basis xvfb libtool opusfile)
+
+if [[ "${bits}" == "32" ]]; then
+	packages+=(libSDL-devel-32bit libSDL_net-devel-32bit)
+else
+	packages+=(SDL SDL_net)
+fi
+