Skip to content

Commit

Permalink
[test] Checking if criu cgroup v1 kludges help
Browse files Browse the repository at this point in the history
Testing criu PR 2545.

Makes TestCheckpoint and TestUsernsCheckpoint run 150 times.

For gha ci, use both precompiled criu and compile the one with
freezer-kludges. For cirrus/almalinux, only compile criu with
freezer-kludges.

Disabled some CI runs:
 - gha: ubuntu 24.04, actuated-arm (cgroup v2);
 - cirrus: fedora (cgroup v2);
 - both gha and cirrus: bats installation, integration tests.

Signed-off-by: Kir Kolyshkin <[email protected]>
  • Loading branch information
kolyshkin committed Dec 17, 2024
1 parent 3ddaa91 commit 73e70c3
Show file tree
Hide file tree
Showing 5 changed files with 145 additions and 563 deletions.
103 changes: 7 additions & 96 deletions .cirrus.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,70 +7,6 @@
# NOTE Cirrus execution environments lack a terminal, needed for
# some integration tests. So we use `ssh -tt` command to fake a terminal.

task:
timeout_in: 30m

env:
DEBIAN_FRONTEND: noninteractive
HOME: /root
# yamllint disable rule:key-duplicates
matrix:
DISTRO: fedora

name: vagrant DISTRO:$DISTRO

compute_engine_instance:
image_project: cirrus-images
image: family/docker-kvm
platform: linux
nested_virtualization: true
# CPU limit: `16 / NTASK`: see https://cirrus-ci.org/faq/#are-there-any-limits
cpu: 4
# Memory limit: `4GB * NCPU`
memory: 16G

host_info_script: |
uname -a
# -----
cat /etc/os-release
# -----
df -T
# -----
cat /proc/cpuinfo
install_libvirt_vagrant_script: |
curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
sudo sed -i 's/^# deb-src/deb-src/' /etc/apt/sources.list
apt-get update
apt-get install -y libvirt-daemon libvirt-daemon-system vagrant
systemctl enable --now libvirtd
apt-get build-dep -y vagrant ruby-libvirt
apt-get install -y --no-install-recommends libxslt-dev libxml2-dev libvirt-dev ruby-bundler ruby-dev zlib1g-dev
vagrant plugin install vagrant-libvirt
vagrant_cache:
fingerprint_script: cat Vagrantfile.$DISTRO
folder: /root/.vagrant.d/boxes
vagrant_up_script: |
ln -sf Vagrantfile.$DISTRO Vagrantfile
# Retry if it fails (download.fedoraproject.org returns 404 sometimes)
vagrant up --no-tty || vagrant up --no-tty
mkdir -p -m 0700 /root/.ssh
vagrant ssh-config >> /root/.ssh/config
guest_info_script: |
ssh default 'sh -exc "uname -a && systemctl --version && df -T && cat /etc/os-release && go version && sestatus && rpm -q container-selinux"'
check_config_script: |
ssh default /vagrant/script/check-config.sh
unit_tests_script: |
ssh default 'sudo -i make -C /vagrant localunittest'
integration_systemd_script: |
ssh -tt default "sudo -i make -C /vagrant localintegration RUNC_USE_SYSTEMD=yes"
integration_fs_script: |
ssh -tt default "sudo -i make -C /vagrant localintegration"
integration_systemd_rootless_script: |
ssh -tt default "sudo -i make -C /vagrant localrootlessintegration RUNC_USE_SYSTEMD=yes"
integration_fs_rootless_script: |
ssh -tt default "sudo -i make -C /vagrant localrootlessintegration"
task:
timeout_in: 30m

Expand All @@ -79,7 +15,7 @@ task:
CIRRUS_WORKING_DIR: /home/runc
GO_VERSION: "1.23"
BATS_VERSION: "v1.9.0"
RPMS: gcc git iptables jq glibc-static libseccomp-devel make criu fuse-sshfs container-selinux
RPMS: gcc git glibc-static libseccomp-devel make fuse-sshfs container-selinux gnutls-devel libaio-devel libasan libcap-devel libnet-devel libnl3-devel libselinux-devel protobuf-c-devel protobuf-devel libdrm-devel
# yamllint disable rule:key-duplicates
matrix:
DISTRO: almalinux-8
Expand All @@ -100,6 +36,7 @@ task:
yum config-manager --set-enabled powertools # for glibc-static
;;
*-9)
RPMS="$RPMS nftables-devel libbsd-devel"
dnf config-manager --set-enabled crb # for glibc-static
dnf -y install epel-release # for fuse-sshfs
# Delegate all cgroup v2 controllers to rootless user via --systemd-cgroup.
Expand All @@ -121,21 +58,9 @@ task:
# Find out the latest minor release URL.
filename=$(curl -fsSL "${PREFIX}?mode=json&include=all" | jq -r --arg Ver "go$GO_VERSION." '. | map(select(.version | contains($Ver))) | first | .files[] | select(.os == "linux" and .arch == "amd64" and .kind == "archive") | .filename')
curl -fsSL "$PREFIX$filename" | tar Cxz /usr/local
# install bats
cd /tmp
git clone https://github.com/bats-core/bats-core
cd bats-core
git checkout $BATS_VERSION
./install.sh /usr/local
cd -
# Add a user for rootless tests
useradd -u2000 -m -d/home/rootless -s/bin/bash rootless
# Allow root and rootless itself to execute `ssh rootless@localhost` in tests/rootless.sh
ssh-keygen -t ecdsa -N "" -f /root/rootless.key
mkdir -m 0700 -p /home/rootless/.ssh
cp /root/rootless.key /home/rootless/.ssh/id_ecdsa
cat /root/rootless.key.pub >> /home/rootless/.ssh/authorized_keys
chown -R rootless.rootless /home/rootless
# Testing https://github.com/checkpoint-restore/criu/pull/2545
git clone https://github.com/kolyshkin/criu.git ~/criu
(cd ~/criu && git checkout freeze-kludges && sudo make install-criu)
# set PATH
echo 'export PATH=/usr/local/go/bin:/usr/local/bin:$PATH' >> /root/.bashrc
# Setup ssh localhost for terminal emulation (script -e did not work)
Expand All @@ -147,6 +72,8 @@ task:
sed -e "s,PermitRootLogin.*,PermitRootLogin prohibit-password,g" -i /etc/ssh/sshd_config
systemctl restart sshd
host_info_script: |
criu --version
# -----
uname -a
# -----
/usr/local/go/bin/go version
Expand All @@ -160,21 +87,5 @@ task:
sestatus
# -----
cat /proc/cpuinfo
check_config_script: |
/home/runc/script/check-config.sh
unit_tests_script: |
ssh -tt localhost "make -C /home/runc localunittest"
integration_systemd_script: |
ssh -tt localhost "make -C /home/runc localintegration RUNC_USE_SYSTEMD=yes"
integration_fs_script: |
ssh -tt localhost "make -C /home/runc localintegration"
integration_systemd_rootless_script: |
case $DISTRO in
*-8)
echo "SKIP: integration_systemd_rootless_script requires cgroup v2"
;;
*)
ssh -tt localhost "make -C /home/runc localrootlessintegration RUNC_USE_SYSTEMD=yes"
esac
integration_fs_rootless_script: |
ssh -tt localhost "make -C /home/runc localrootlessintegration"
89 changes: 7 additions & 82 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,49 +23,14 @@ jobs:
strategy:
fail-fast: false
matrix:
os: [ubuntu-20.04, ubuntu-24.04, actuated-arm64-6cpu-8gb]
os: [ubuntu-20.04]
go-version: [1.22.x, 1.23.x]
rootless: ["rootless", ""]
rootless: [""]
race: ["-race", ""]
criu: ["", "criu-dev"]
exclude:
# Disable most of criu-dev jobs, as they are expensive
# (need to compile criu) and don't add much value/coverage.
- criu: criu-dev
go-version: 1.22.x
- criu: criu-dev
rootless: rootless
- criu: criu-dev
race: -race
- go-version: 1.22.x
os: actuated-arm64-6cpu-8gb
- race: "-race"
os: actuated-arm64-6cpu-8gb
- criu: criu-dev
os: actuated-arm64-6cpu-8gb

runs-on: ${{ matrix.os }}

steps:
# https://gist.github.com/alexellis/1f33e581c75e11e161fe613c46180771#file-metering-gha-md
# vmmeter start
- name: Prepare arkade
uses: alexellis/arkade-get@master
if: matrix.os == 'actuated-arm64-6cpu-8gb'
with:
crane: latest
print-summary: false

- name: Install vmmeter
if: matrix.os == 'actuated-arm64-6cpu-8gb'
run: |
crane export --platform linux/arm64 ghcr.io/openfaasltd/vmmeter:latest | sudo tar -xvf - -C /usr/local/bin
- name: Run vmmeter
uses: self-actuated/vmmeter-action@master
if: matrix.os == 'actuated-arm64-6cpu-8gb'
# vmmeter end

- name: checkout
uses: actions/checkout@v4

Expand All @@ -74,6 +39,8 @@ jobs:
set -x
# Sync `set -x` outputs with command ouputs
exec 2>&1
# CRIU
criu --version
# Version
uname -a
cat /etc/os-release
Expand Down Expand Up @@ -124,65 +91,23 @@ jobs:
sudo apt -qy install \
libcap-dev libnet1-dev libnl-3-dev \
libprotobuf-c-dev libprotobuf-dev protobuf-c-compiler protobuf-compiler
git clone https://github.com/checkpoint-restore/criu.git ~/criu
(cd ~/criu && git checkout ${{ matrix.criu }} && sudo make install-criu)
# Testing https://github.com/checkpoint-restore/criu/pull/2545
git clone https://github.com/kolyshkin/criu.git ~/criu
(cd ~/criu && git checkout freeze-kludges && sudo make install-criu)
rm -rf ~/criu
- name: install go ${{ matrix.go-version }}
uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
check-latest: true

- name: build
run: sudo -E PATH="$PATH" make EXTRA_FLAGS="${{ matrix.race }}" all

- name: Setup Bats and bats libs
uses: bats-core/[email protected]
with:
bats-version: 1.9.0
support-install: false
assert-install: false
detik-install: false
file-install: false

- name: Allow userns for runc
# https://discourse.ubuntu.com/t/ubuntu-24-04-lts-noble-numbat-release-notes/39890#unprivileged-user-namespace-restrictions-15
if: matrix.os == 'ubuntu-24.04'
run: |
sed "s;^profile runc /usr/sbin/;profile runc-test $PWD/;" < /etc/apparmor.d/runc | sudo apparmor_parser
- name: unit test
if: matrix.rootless != 'rootless'
run: sudo -E PATH="$PATH" -- make TESTFLAGS="${{ matrix.race }}" localunittest

- name: add rootless user
if: matrix.rootless == 'rootless'
run: |
sudo useradd -u2000 -m -d/home/rootless -s/bin/bash rootless
# Allow root and rootless itself to execute `ssh rootless@localhost` in tests/rootless.sh
ssh-keygen -t ecdsa -N "" -f $HOME/rootless.key
sudo mkdir -m 0700 -p /home/rootless/.ssh
sudo cp $HOME/rootless.key /home/rootless/.ssh/id_ecdsa
sudo cp $HOME/rootless.key.pub /home/rootless/.ssh/authorized_keys
sudo chown -R rootless.rootless /home/rootless
sudo chmod a+X $HOME # for Ubuntu 22.04 and later
- name: integration test (fs driver)
run: sudo -E PATH="$PATH" script -e -c 'make local${{ matrix.rootless }}integration'

- name: integration test (systemd driver)
# Skip rootless+systemd for ubuntu 20.04 because of cgroup v1.
if: ${{ !(matrix.os == 'ubuntu-20.04' && matrix.rootless == 'rootless') }}
run: |
# Delegate all cgroup v2 controllers to rootless user via --systemd-cgroup.
# The default (since systemd v252) is "pids memory cpu".
sudo mkdir -p /etc/systemd/system/[email protected]
printf "[Service]\nDelegate=yes\n" | sudo tee /etc/systemd/system/[email protected]/delegate.conf
sudo systemctl daemon-reload
# Run the tests.
sudo -E PATH="$PATH" script -e -c 'make RUNC_USE_SYSTEMD=yes local${{ matrix.rootless }}integration'
# We need to continue support for 32-bit ARM.
# However, we do not have 32-bit ARM CI, so we use i386 for testing 32bit stuff.
# We are not interested in providing official support for i386.
Expand Down
Loading

0 comments on commit 73e70c3

Please sign in to comment.