CI: build in ci, more streamlined toolchain

Reviewed-on: #2
Co-authored-by: Stepan Usatiuk <stepan@usatiuk.com>
Co-committed-by: Stepan Usatiuk <stepan@usatiuk.com>

.github/workflows/ficus.yml

@@ -0,0 +1,82 @@
+name: Ficus build
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+env:
+  BUILD_TYPE: Debug
+  BUILD_PARALLEL: 3
+  BUILD_PARALLEL_GCC: 3
+
+jobs:
+  build-ficus-toolchain:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+
+      - name: Install sudo for ACT runner
+        run: apt-get update && apt-get install -y sudo
+        if: env.ACT=='true'
+
+      - name: Install dependencies
+        run: sudo apt-get update && sudo apt-get install -y cmake build-essential gcc g++ nasm bison bzip2 flex mtools texinfo libgmp-dev libmpfr-dev libmpc-dev xorriso
+
+      - name: Unpack prebuilt toolchain
+        run: cd ${{github.workspace}} && tar xf toolchain_scripts/toolchain-s1-linux-aarch64.tar.xz
+
+      - name: Build ficus toolchain
+        run: source ${{github.workspace}}/env.sh && ${{github.workspace}}/ficus-toolchain/build-all.sh s2only
+
+      - name: Tar the toolchain
+        run: cd ${{github.workspace}} && tar -czvf toolchain-ficus.tar.xz toolchain sysroot
+
+      - name: Upload ficus toolchain
+        uses: actions/upload-artifact@v3
+        with:
+          name: toolchain-ficus
+          path: ${{github.workspace}}/toolchain-ficus.tar.xz
+          retention-days: 5
+
+  iso:
+    needs: build-ficus-toolchain
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+
+      - name: Install sudo for ACT runner
+        run: apt-get update && apt-get install -y sudo
+        if: env.ACT=='true'
+
+      - name: Install dependencies
+        run: sudo apt-get update && sudo apt-get install -y cmake build-essential gcc g++ nasm bison bzip2 flex mtools texinfo libgmp-dev libmpfr-dev libmpc-dev xorriso
+
+      - name: Download toolchain
+        uses: actions/download-artifact@v3
+        with:
+          name: toolchain-ficus
+          path: ${{github.workspace}}
+
+      - name: Untar the toolchain
+        run: cd ${{github.workspace}} && tar xf toolchain-ficus.tar.xz
+
+      - name: Configure CMake
+        # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
+        # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
+        run: source ${{github.workspace}}/env.sh && cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DFICUS_ROOT=${{github.workspace}} -DCMAKE_TOOLCHAIN_FILE=${{github.workspace}}/CMake-x86_64-ficus-toolchain.cmake
+
+      - name: Build iso
+        # Build your program with the given configuration
+        run: source ${{github.workspace}}/env.sh && cmake --build ${{github.workspace}}/build --target iso --config ${{env.BUILD_TYPE}}  --parallel ${{env.BUILD_PARALLEL}}
+
+      - name: Upload isos
+        uses: actions/upload-artifact@v3
+        with:
+          name: iso
+          path: ${{github.workspace}}/build/**/*.iso
+          retention-days: 5

.github/workflows/unit.yml

@@ -10,6 +10,7 @@ env:
   # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
   BUILD_TYPE: Debug
   TEST_MODE: UNIT
+  BUILD_PARALLEL: 3
 
 jobs:
   build:
@@ -34,7 +35,7 @@ jobs:
 
       - name: Build
         # Build your program with the given configuration
-        run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}  --parallel $(nproc)
+        run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}  --parallel ${{env.BUILD_PARALLEL}}
 
       - name: Test
         # Execute tests defined by the CMake configuration.

ficus-toolchain/build-all.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+set -euxo pipefail
+
+if [ -z "$FICUS_ROOT" ]; then
+    echo "$FICUS_ROOT" is blank
+fi
+export SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
+
+export PREFIX="$FICUS_ROOT/toolchain/gcc-x86_64-ficus-prefix/"
+export TARGET=x86_64-ficus
+export PATH="$PREFIX/bin:$PATH"
+
+cd "$SCRIPT_DIR"
+
+if [ $# -eq 0 ]
+  then
+    echo "No arguments supplied"
+    exit 1
+fi
+
+if [ -z "${BUILD_PARALLEL}" ]
+then
+  export BUILD_PARALLEL=$(nproc)
+fi
+if [ -z "${BUILD_PARALLEL_GCC}" ]
+then
+  export BUILD_PARALLEL_GCC=$BUILD_PARALLEL
+fi
+
+if [[ "$1" != "s2only" ]]
+then
+  "$SCRIPT_DIR"/build-s1.sh
+  cd "$SCRIPT_DIR"
+fi
+
+if [[ "$1" = "s1only" ]]
+then exit 0
+fi
+
+mkdir -p "$FICUS_ROOT/sysroot"
+
+"$SCRIPT_DIR"/build-newlib.sh
+cd "$SCRIPT_DIR"
+"$SCRIPT_DIR"/build-s2.sh
+cd "$SCRIPT_DIR"

ficus-toolchain/build-newlib.sh

@@ -14,7 +14,7 @@ cd newlib
 # rm -rf build
 mkdir -p build
 cd build
-#../newlib-4.4.0.20231231/configure --enable-newlib-supplied-syscalls --prefix=/usr --target=$TARGET
-make -j$(nproc) all
+../newlib-4.4.0.20231231/configure --enable-newlib-supplied-syscalls --prefix=/usr --target=$TARGET
+make -j$BUILD_PARALLEL all
 make DESTDIR="$FICUS_ROOT/sysroot" install
 cp -r "$FICUS_ROOT/sysroot/usr"/x86_64-ficus/* "$FICUS_ROOT/sysroot/usr"

ficus-toolchain/build-s1.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+set -euxo pipefail
+
+if [ -z "$FICUS_ROOT" ]; then
+    echo "$FICUS_ROOT" is blank
+    exit 1
+fi
+
+if [ -z "$PREFIX" ]; then
+    echo "PREFIX" is blank
+    exit 1
+fi
+
+if [ -z "$SCRIPT_DIR" ]; then
+    echo "SCRIPT_DIR" is blank
+    exit 1
+fi
+
+cd "$SCRIPT_DIR"
+
+mkdir -p "$FICUS_ROOT/sysroot"
+
+cd binutils-x86_64-ficus
+mkdir -p build
+cd build
+../binutils-2.41/configure --target=$TARGET --prefix="$PREFIX" --with-sysroot="$PREFIX/../../sysroot" --disable-nls --disable-werror
+make -j$(nproc)
+make install
+
+cd ../../
+
+cd gcc-x86_64-ficus
+rm -rf build
+mkdir -p build
+cd build
+
+pushd ../gcc-13.2.0/
+./contrib/download_prerequisites
+popd
+
+../gcc-13.2.0/configure --target=$TARGET --prefix="$PREFIX" --with-sysroot="$PREFIX/../../sysroot" --disable-nls --enable-languages=c,c++ --with-newlib --without-headers --disable-fixincludes --enable-version-specific-runtime-libs
+make -j$(nproc) all-gcc
+make -j$(nproc) all-target-libgcc
+make install-gcc
+make install-target-libgcc
+find "$PREFIX" -exec strip {} \;
+cd ../../
+

ficus-toolchain/build-s2.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+set -euxo pipefail
+
+if [ -z "$FICUS_ROOT" ]; then
+    echo "$FICUS_ROOT" is blank
+    exit 1
+fi
+
+if [ -z "$PREFIX" ]; then
+    echo "PREFIX" is blank
+    exit 1
+fi
+
+if [ -z "$SCRIPT_DIR" ]; then
+    echo "SCRIPT_DIR" is blank
+    exit 1
+fi
+
+cd "$SCRIPT_DIR"
+
+cd gcc-x86_64-ficus
+rm -rf build
+mkdir -p build
+cd build
+
+pushd ../gcc-13.2.0/
+./contrib/download_prerequisites
+popd
+
+../gcc-13.2.0/configure --target=$TARGET --prefix="$PREFIX" --with-sysroot="$PREFIX/../../sysroot" --disable-nls \
+    --enable-languages=c,c++ --with-newlib --disable-fixincludes --disable-libstdcxx-threads \
+    --enable-version-specific-runtime-libs
+make -j$BUILD_PARALLEL_GCC all-gcc
+make -j$BUILD_PARALLEL all-target-libgcc
+make -j$BUILD_PARALLEL all-target-libstdc++-v3
+make install-gcc
+find "$PREFIX" -exec strip {} \;
+make install-target-libgcc
+make install-target-libstdc++-v3

ficus-toolchain/build.sh

@@ -1,51 +0,0 @@
-#!/bin/bash
-set -euxo pipefail
-
-if [ -z "$FICUS_ROOT" ]; then
-    echo "$FICUS_ROOT" is blank
-fi
-SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
-
-export PREFIX="$FICUS_ROOT/toolchain/gcc-x86_64-ficus-prefix/"
-export TARGET=x86_64-ficus
-export PATH="$PREFIX/bin:$PATH"
-
-cd "$SCRIPT_DIR"
-
-mkdir -p "$FICUS_ROOT/sysroot"
-
-cd binutils-x86_64-ficus
-mkdir -p build
-cd build
-../binutils-2.41/configure --target=$TARGET --prefix="$PREFIX" --with-sysroot="$FICUS_ROOT/sysroot" --disable-nls --disable-werror
-make -j$(nproc)
-make install
-
-cd ../../
-
-cd gcc-x86_64-ficus
-rm -rf build
-mkdir -p build
-cd build
-../gcc-13.2.0/configure --target=$TARGET --prefix="$PREFIX" --with-sysroot="$FICUS_ROOT/sysroot" --disable-nls --enable-languages=c,c++ --with-newlib --without-headers --enable-version-specific-runtime-libs --with-gmp=/opt/homebrew --with-mpc=/opt/homebrew --with-mpfr=/opt/homebrew
-make -j$(nproc) all-gcc
-make -j$(nproc) all-target-libgcc
-make install-gcc
-make install-target-libgcc
-cd ../../
-
-"$SCRIPT_DIR"/build-newlib.sh
-
-cd gcc-x86_64-ficus
-rm -rf build
-mkdir -p build
-cd build
-../gcc-13.2.0/configure --target=$TARGET --prefix="$PREFIX" --with-sysroot="$FICUS_ROOT/sysroot" --disable-nls \
-    --enable-languages=c,c++ --with-newlib --disable-libstdcxx-threads \
-    --enable-version-specific-runtime-libs --with-gmp=/opt/homebrew --with-mpc=/opt/homebrew --with-mpfr=/opt/homebrew
-make -j$(nproc) all-gcc
-make -j$(nproc) all-target-libgcc
-make -j$(nproc) all-target-libstdc++-v3
-make install-gcc
-make install-target-libgcc
-make install-target-libstdc++-v3

src/iso/limine.cfg

@@ -11,7 +11,6 @@ TIMEOUT=1
 
     MODULE_PATH=boot:///init
     MODULE_PATH=boot:///hello2
-    MODULE_PATH=boot:///psil
 
 # Same thing, but without KASLR.
 :ficus (KASLR off)
@@ -23,4 +22,3 @@ TIMEOUT=1
     KERNEL_PATH=boot:///ficus.elf
     MODULE_PATH=boot:///init
     MODULE_PATH=boot:///hello2
-    MODULE_PATH=boot:///psil

toolchain_scripts/binutils-i686-elf.sh

@@ -15,7 +15,7 @@ pushd binutils-i686-elf
 
 if [ ! -d "binutils-2.41" ]; then
     wget https://ftp.gnu.org/gnu/binutils/binutils-2.41.tar.xz
-    tar xvf binutils-2.41.tar.xz
+    tar xf binutils-2.41.tar.xz
     rm binutils-2.41.tar.xz
 fi
 

toolchain_scripts/binutils-x86_64-elf.sh

@@ -15,7 +15,7 @@ pushd binutils-x86_64-elf
 
 if [ ! -d "binutils-2.41" ]; then
     wget https://ftp.gnu.org/gnu/binutils/binutils-2.41.tar.xz
-    tar xvf binutils-2.41.tar.xz
+    tar xf binutils-2.41.tar.xz
     rm binutils-2.41.tar.xz
 fi
 

toolchain_scripts/build-all.sh

@@ -16,3 +16,10 @@ SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
 
 "$SCRIPT_DIR"/grub.sh
 "$SCRIPT_DIR"/limine.sh
+
+rm -rf "$FICUS_ROOT/toolchain/binutils-i686-elf"
+rm -rf "$FICUS_ROOT/toolchain/gcc-i686-elf"
+rm -rf "$FICUS_ROOT/toolchain/binutils-x86_64-elf"
+rm -rf "$FICUS_ROOT/toolchain/gcc-x86_64-elf"
+
+find "$FICUS_ROOT/toolchain" -exec strip {} \;

toolchain_scripts/gcc-i686-elf.sh

@@ -15,7 +15,7 @@ pushd gcc-i686-elf
 
 if [ ! -d gcc-13.2.0 ]; then
     wget https://ftp.gnu.org/gnu/gcc/gcc-13.2.0/gcc-13.2.0.tar.xz
-    tar xvf gcc-13.2.0.tar.xz
+    tar xf gcc-13.2.0.tar.xz
     rm gcc-13.2.0.tar.xz
 fi
 
@@ -47,6 +47,9 @@ if [ ! -z ${USE_BREW_LIBS+x} ]; then
     ADDONS="$HOMEBREW_LIBS $ADDONS"
 fi
 
+pushd ../gcc-13.2.0/
+./contrib/download_prerequisites
+popd
 ../gcc-13.2.0/configure --target=$TARGET --prefix="$PREFIX" --disable-nls --enable-languages=c,c++ --with-newlib --without-headers --enable-version-specific-runtime-libs $ADDONS
 make -j$(nproc) all-gcc
 make -j$(nproc) all-target-libgcc

toolchain_scripts/gcc-x86_64-elf.sh

@@ -15,7 +15,7 @@ pushd gcc-x86_64-elf
 
 if [ ! -d gcc-13.2.0 ]; then
     wget https://ftp.gnu.org/gnu/gcc/gcc-13.2.0/gcc-13.2.0.tar.xz
-    tar xvf gcc-13.2.0.tar.xz
+    tar xf gcc-13.2.0.tar.xz
     rm gcc-13.2.0.tar.xz
 fi
 
@@ -56,6 +56,9 @@ EOF
 
 sed -i.bak 's/x86_64-\*-elf\*)/x86_64-\*-elf\*)\n\ttmake_file="\${tmake_file} i386\/t-x86_64-elf"/g' "../gcc-13.2.0/gcc/config.gcc"
 
+pushd ../gcc-13.2.0/
+./contrib/download_prerequisites
+popd
 ../gcc-13.2.0/configure --target=$TARGET --prefix="$PREFIX" --disable-nls --enable-languages=c,c++ --with-newlib --without-headers --enable-version-specific-runtime-libs $ADDONS
 make -j$(nproc) all-gcc
 make -j$(nproc) all-target-libgcc

toolchain_scripts/grub.sh

@@ -17,7 +17,7 @@ mkdir -p prefix
 
 if [ ! -d "grub-2.06" ]; then
     wget https://ftp.gnu.org/gnu/grub/grub-2.06.tar.xz
-    tar xvf grub-2.06.tar.xz
+    tar xf grub-2.06.tar.xz
     rm grub-2.06.tar.xz
 fi
 
@@ -43,4 +43,8 @@ export PATH="$PREFIX/bin:$PATH"
 make -j$(nproc)
 make install
 
+cd ..
+rm -rf build
+rm -rf grub-2.06
+
 touch -m ../done

toolchain_scripts/limine.sh

@@ -17,7 +17,7 @@ mkdir -p prefix
 
 if [ ! -d "limine-5.20230830.0" ]; then
     wget https://github.com/limine-bootloader/limine/releases/download/v5.20230830.0/limine-5.20230830.0.tar.xz
-    tar xvf limine-5.20230830.0.tar.xz
+    tar xf limine-5.20230830.0.tar.xz
     rm limine-5.20230830.0.tar.xz
 fi
 
@@ -44,4 +44,8 @@ grep -rl "define DEFAULT_VAR =" ../limine-5.20230830.0 | xargs sed -i.bak -e 's/
 
 make -j$(nproc) install
 
+cd ..
+rm -rf build
+rm -rf limine-5.20230830.0
+
 touch -m ../done