From 461ff423eec633024ed0051cdd4f670a3ec28522 Mon Sep 17 00:00:00 2001 From: Jeff King Date: Thu, 12 Sep 2024 05:43:36 -0400 Subject: [PATCH 1/5] ci: drop run-docker scripts We haven't used these scripts since 4a6e4b9602 (CI: remove Travis CI support, 2021-11-23), as the GitHub Actions config has support for directly running jobs within docker containers. It's possible we might want to resurrect something like this in order to be more agnostic to the CI platform. But it's not clear exactly what it would look like. And in the meantime, it's just a maintenance burden as we make changes to CI config, and is subject to bitrot. In fact it's already broken; it references ci/install-docker-dependencies.sh, which went away in 9cdeb34b96 (ci: merge scripts which install dependencies, 2024-04-12). Signed-off-by: Jeff King --- ci/run-docker-build.sh | 66 ------------------------------------------ ci/run-docker.sh | 47 ------------------------------ 2 files changed, 113 deletions(-) delete mode 100755 ci/run-docker-build.sh delete mode 100755 ci/run-docker.sh diff --git a/ci/run-docker-build.sh b/ci/run-docker-build.sh deleted file mode 100755 index 6cd832efb9c..00000000000 --- a/ci/run-docker-build.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/sh -# -# Build and test Git inside container -# -# Usage: -# run-docker-build.sh -# - -set -ex - -if test $# -ne 1 || test -z "$1" -then - echo >&2 "usage: run-docker-build.sh " - exit 1 -fi - -case "$jobname" in -linux32) - switch_cmd="linux32 --32bit i386" - ;; -linux-musl) - switch_cmd= - useradd () { adduser -D "$@"; } - ;; -*) - exit 1 - ;; -esac - -"${0%/*}/install-docker-dependencies.sh" - -# If this script runs inside a docker container, then all commands are -# usually executed as root. Consequently, the host user might not be -# able to access the test output files. -# If a non 0 host user id is given, then create a user "ci" with that -# user id to make everything accessible to the host user. -HOST_UID=$1 -if test $HOST_UID -eq 0 -then - # Just in case someone does want to run the test suite as root. - CI_USER=root -else - CI_USER=ci - if test "$(id -u $CI_USER 2>/dev/null)" = $HOST_UID - then - echo "user '$CI_USER' already exists with the requested ID $HOST_UID" - else - useradd -u $HOST_UID $CI_USER - fi -fi - -# Build and test -command $switch_cmd su -m -l $CI_USER -c " - set -ex - export DEVELOPER='$DEVELOPER' - export DEFAULT_TEST_TARGET='$DEFAULT_TEST_TARGET' - export GIT_PROVE_OPTS='$GIT_PROVE_OPTS' - export GIT_TEST_OPTS='$GIT_TEST_OPTS' - export GIT_TEST_CLONE_2GB='$GIT_TEST_CLONE_2GB' - export MAKEFLAGS='$MAKEFLAGS' - export cache_dir='$cache_dir' - cd /usr/src/git - test -n '$cache_dir' && ln -s '$cache_dir/.prove' t/.prove - make - make test -" diff --git a/ci/run-docker.sh b/ci/run-docker.sh deleted file mode 100755 index af89d1624a4..00000000000 --- a/ci/run-docker.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/sh -# -# Download and run Docker image to build and test Git -# - -. ${0%/*}/lib.sh - -case "$jobname" in -linux32) - CI_CONTAINER="daald/ubuntu32:xenial" - ;; -linux-musl) - CI_CONTAINER=alpine - ;; -*) - exit 1 - ;; -esac - -docker pull "$CI_CONTAINER" - -# Use the following command to debug the docker build locally: -# must be 0 if podman is used as drop-in replacement for docker -# $ docker run -itv "${PWD}:/usr/src/git" --entrypoint /bin/sh "$CI_CONTAINER" -# root@container:/# export jobname= -# root@container:/# /usr/src/git/ci/run-docker-build.sh - -container_cache_dir=/tmp/container-cache - -docker run \ - --interactive \ - --env DEVELOPER \ - --env DEFAULT_TEST_TARGET \ - --env GIT_PROVE_OPTS \ - --env GIT_TEST_OPTS \ - --env GIT_TEST_CLONE_2GB \ - --env MAKEFLAGS \ - --env jobname \ - --env cache_dir="$container_cache_dir" \ - --volume "${PWD}:/usr/src/git" \ - --volume "$cache_dir:$container_cache_dir" \ - "$CI_CONTAINER" \ - /usr/src/git/ci/run-docker-build.sh $(id -u $USER) - -check_unignored_build_artifacts - -save_good_tree -- GitLab From c075e878945c4dcd2a23e432f1a5f7e15aa83d87 Mon Sep 17 00:00:00 2001 From: Jeff King Date: Thu, 12 Sep 2024 05:45:37 -0400 Subject: [PATCH 2/5] ci: unify ubuntu and ubuntu32 dependencies The script to install dependencies has two separate entries for 32-bit and 64-bit Ubuntu systems. This increases the maintenance burden since both should need roughly the same packages. That hasn't been too bad so far because we've stayed on the same 32-bit image since 2017. Trying to move to a newer image revealed several problems with the linux32 job: - newer images complain about using "linux32 --32bit i386", due to seccomp restrictions. We can loosen these with a docker option, but I don't think running it is even doing anything. We use it only for pretending to "apt" that we're on a 32-bit machine, but inside the container image apt is already configured as a 32-bit system (even though the kernel outside the container is obviously 64-bit). Using the same apt invocation for both architectures just gets rid of this call entirely. - we set DEBIAN_FRONTEND to avoid hanging on packages that ask the user questions. This wasn't a problem on the old image, but it is on newer ones. The 64-bit stanza handles this already. As a bonus, the 64-bit stanza uses "apt -q" instead of redirecting output to /dev/null. This would have saved me a lot of debugging time trying to figure out why it was hanging. :) - the old image seems to have zlib-dev installed by default, but newer ones do not. In addition, there were probably many tests being skipped on the 32-bit build because we didn't have support packages installed (e.g., gpg). Now we'll run them. We do need to keep some parts split off just for 64-bit systems: our p4 and lfs installs reference x86_64/amd64 binaries. The downloaded jgit should work in theory, since it's just a jar file embedded in a shell script that relies on the system java. But the system java in our image is too old, so I've left it as 64-bit only for now. Signed-off-by: Jeff King --- ci/install-dependencies.sh | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index 4781cd20bb0..00cb7df67ad 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -33,7 +33,7 @@ fedora-*) dnf -yq update >/dev/null && dnf -yq install make gcc findutils diffutils perl python3 gettext zlib-devel expat-devel openssl-devel curl-devel pcre2-devel >/dev/null ;; -ubuntu-*) +ubuntu-*|ubuntu32-*) # Required so that apt doesn't wait for user input on certain packages. export DEBIAN_FRONTEND=noninteractive @@ -45,25 +45,23 @@ ubuntu-*) libemail-valid-perl libio-pty-perl libio-socket-ssl-perl libnet-smtp-ssl-perl libdbd-sqlite3-perl libcgi-pm-perl \ ${CC_PACKAGE:-${CC:-gcc}} $PYTHON_PACKAGE - mkdir --parents "$CUSTOM_PATH" - wget --quiet --directory-prefix="$CUSTOM_PATH" \ - "$P4WHENCE/bin.linux26x86_64/p4d" "$P4WHENCE/bin.linux26x86_64/p4" - chmod a+x "$CUSTOM_PATH/p4d" "$CUSTOM_PATH/p4" + case "$distro" in + ubuntu-*) + mkdir --parents "$CUSTOM_PATH" - wget --quiet "$LFSWHENCE/git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" - tar -xzf "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" \ - -C "$CUSTOM_PATH" --strip-components=1 "git-lfs-$LINUX_GIT_LFS_VERSION/git-lfs" - rm "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" + wget --quiet --directory-prefix="$CUSTOM_PATH" \ + "$P4WHENCE/bin.linux26x86_64/p4d" "$P4WHENCE/bin.linux26x86_64/p4" + chmod a+x "$CUSTOM_PATH/p4d" "$CUSTOM_PATH/p4" - wget --quiet "$JGITWHENCE" --output-document="$CUSTOM_PATH/jgit" - chmod a+x "$CUSTOM_PATH/jgit" - ;; -ubuntu32-*) - sudo linux32 --32bit i386 sh -c ' - apt update >/dev/null && - apt install -y build-essential libcurl4-openssl-dev \ - libssl-dev libexpat-dev gettext python >/dev/null - ' + wget --quiet "$LFSWHENCE/git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" + tar -xzf "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" \ + -C "$CUSTOM_PATH" --strip-components=1 "git-lfs-$LINUX_GIT_LFS_VERSION/git-lfs" + rm "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" + + wget --quiet "$JGITWHENCE" --output-document="$CUSTOM_PATH/jgit" + chmod a+x "$CUSTOM_PATH/jgit" + ;; + esac ;; macos-*) export HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1 -- GitLab From 7096e54ad54dbc05c3dc91eb965eed6c1f7531e7 Mon Sep 17 00:00:00 2001 From: Jeff King Date: Thu, 12 Sep 2024 05:47:30 -0400 Subject: [PATCH 3/5] ci: use more recent linux32 image The Xenial image we're using was released more than 8 years ago. This is a problem for using some recent GitHub Actions scripts, as they require Node.js 20, and all of the binaries they ship need glibc 2.28 or later. We're not using them yet, but moving forward prepares us for a future patch which will. Xenial was actually the last official 32-bit Ubuntu release, but you can still find i386 images for more recent releases. This patch uses Focal, which was released in 2020 (and is the oldest one with glibc 2.28). There are two small downsides here: - while Xenial is pretty old, it is still in LTS support until April 2026. So there's probably some value in testing with such an old system, and we're losing that. - there are no i386 subversion packages in the Focal repository. So we won't be able to test that (OTOH, we had never tested it until the previous patch which unified the 32/64-bit dependency code). Signed-off-by: Jeff King --- .github/workflows/main.yml | 4 ++-- ci/install-dependencies.sh | 11 ++++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1ee0433acc1..ec09be6ef6e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -339,8 +339,8 @@ jobs: image: alpine distro: alpine-latest - jobname: linux32 - image: daald/ubuntu32:xenial - distro: ubuntu32-16.04 + image: i386/ubuntu:focal + distro: ubuntu32-20.04 - jobname: pedantic image: fedora distro: fedora-latest diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index 00cb7df67ad..735ee6f4639 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -37,9 +37,18 @@ ubuntu-*|ubuntu32-*) # Required so that apt doesn't wait for user input on certain packages. export DEBIAN_FRONTEND=noninteractive + case "$distro" in + ubuntu-*) + SVN='libsvn-perl subversion' + ;; + *) + SVN= + ;; + esac + sudo apt-get -q update sudo apt-get -q -y install \ - language-pack-is libsvn-perl apache2 cvs cvsps git gnupg subversion \ + language-pack-is apache2 cvs cvsps git gnupg $SVN \ make libssl-dev libcurl4-openssl-dev libexpat-dev wget sudo default-jre \ tcl tk gettext zlib1g-dev perl-modules liberror-perl libauthen-sasl-perl \ libemail-valid-perl libio-pty-perl libio-socket-ssl-perl libnet-smtp-ssl-perl libdbd-sqlite3-perl libcgi-pm-perl \ -- GitLab From f33406dbcc16e699be71aa4982133c325838ae1b Mon Sep 17 00:00:00 2001 From: Jeff King Date: Thu, 12 Sep 2024 05:48:41 -0400 Subject: [PATCH 4/5] ci: use regular action versions for linux32 job The linux32 job runs inside a docker container with a 32-bit libc, etc. This breaks any GitHub Actions scripts that are implemented in javascript, because they ship with their own 64-bit version of Node.js that's dynamically linked. They'll fail with a message like: exec /__e/node20/bin/node: no such file or directory because they can't find the runtime linker. This hasn't been a problem until recently because we special-case older, non-javascript versions of these actions for the linux32 job. But it recently became an issue when our old version of actions/upload-artifact was deprecated, causing the job to fail. We worked around that in 90f2c7240c (ci: remove 'Upload failed tests' directories' step from linux32 jobs, 2024-09-09), but it meant a loss of functionality for that job. And we may eventually run into the same deprecation problem with actions/checkout, which can't just be removed. We can solve the linking issue by installing the 64-bit libc and stdc++ packages before doing anything else. Coupled with the switch to a more recent image in the previous patch, that lets us remove the special-casing of the action scripts entirely. Signed-off-by: Jeff King --- .github/workflows/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ec09be6ef6e..1e8fd7d183e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -350,17 +350,17 @@ jobs: runs-on: ubuntu-latest container: ${{matrix.vector.image}} steps: - - uses: actions/checkout@v4 - if: matrix.vector.jobname != 'linux32' - - uses: actions/checkout@v1 # cannot be upgraded because Node.js Actions aren't supported in this container + - name: prepare libc6 for actions if: matrix.vector.jobname == 'linux32' + run: apt -q update && apt -q -y install libc6-amd64 lib64stdc++6 + - uses: actions/checkout@v4 - run: ci/install-dependencies.sh - run: ci/run-build-and-tests.sh - name: print test failures if: failure() && env.FAILED_TEST_ARTIFACTS != '' run: ci/print-test-failures.sh - name: Upload failed tests' directories - if: failure() && env.FAILED_TEST_ARTIFACTS != '' && matrix.vector.jobname != 'linux32' + if: failure() && env.FAILED_TEST_ARTIFACTS != '' uses: actions/upload-artifact@v4 with: name: failed-tests-${{matrix.vector.jobname}} -- GitLab From cfcff505c7f7cf884619ff8afb78d0aa2f58401e Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 12 Sep 2024 13:41:27 +0200 Subject: [PATCH 5/5] ci: add Ubuntu 16.04 job to GitLab CI In the preceding commits we had to convert the linux32 job to be based on Ubuntu 20.04 instead of Ubuntu 16.04 due to a limitation in GitHub Workflows. This was the only job left that still tested against this old but supported Ubuntu version, and we have no other jobs that test with a comparatively old Linux distribution. Add a new job to GitLab CI that tests with Ubuntu 16.04 to cover the resulting test gap. GitLab doesn't modify Docker images in the same way GitHub does and thus doesn't fall pray to the same issue. There are two compatibility issues uncovered by this: - Ubuntu 16.04 does not support HTTP/2 in Apache. We thus cannot set `GIT_TEST_HTTPD=true`, which would otherwise cause us to fail when Apache fails to start. - Ubuntu 16.04 cannot use recent JGit versions as they depend on a more recent Java runtime than we have available. We thus disable installing any kind of optional dependencies that do not come from the package manager. These two restrictions are fine though, as we only really care about whether Git compiles and runs on such old distributions in the first place. Signed-off-by: Patrick Steinhardt --- .gitlab-ci.yml | 3 +++ ci/install-dependencies.sh | 5 +++++ ci/lib.sh | 9 ++++++++- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2589098eff7..80b1668ebeb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,6 +25,9 @@ test:linux: fi parallel: matrix: + - jobname: linux-old + image: ubuntu:16.04 + CC: gcc - jobname: linux-sha256 image: ubuntu:latest CC: clang diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index 735ee6f4639..08656a15308 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -55,6 +55,11 @@ ubuntu-*|ubuntu32-*) ${CC_PACKAGE:-${CC:-gcc}} $PYTHON_PACKAGE case "$distro" in + ubuntu-16.04) + # Does not support JGit, but we also don't really care about + # the others. We rather care whether Git still compiles and + # runs fine overall. + ;; ubuntu-*) mkdir --parents "$CUSTOM_PATH" diff --git a/ci/lib.sh b/ci/lib.sh index 51f8f59a296..74b430be238 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -336,7 +336,14 @@ ubuntu-*) fi MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=/usr/bin/$PYTHON_PACKAGE" - export GIT_TEST_HTTPD=true + case "$distro" in + ubuntu-16.04) + # Apache is too old for HTTP/2. + ;; + *) + export GIT_TEST_HTTPD=true + ;; + esac # The Linux build installs the defined dependency versions below. # The OS X build installs much more recent versions, whichever -- GitLab