[skip-ci] TMT: system tests

This commit introduces TMT test jobs triggered via packit to run system
tests on testing-farm infrastructure. Tests are run for
Fedora 41, 42 and rawhide on x86_64. The same
test plan will be reused by Fedora for bodhi, zuul and fedora-ci gating
tests. Packit will handle syncing of test plan and sources from upstream
to downstream.

Packit failure notification has also been updated to be less noisy and
let people know they are free to ignore any failures.

TODO:
1. Enable jobs for CentOS Stream and aarch64 envs.
2. Enable separate set of jobs for release branches as they need to be
   tested with official distro packages, not with bleeding-edge
   packages.

Signed-off-by: Lokesh Mandvekar <lsm5@fedoraproject.org>

.packit.yaml

@@ -5,6 +5,27 @@
 downstream_package_name: podman
 upstream_tag_template: v{version}
 
+# These files get synced from upstream to downstream (Fedora / CentOS Stream) on every
+# propose-downstream job. This is done so tests maintained upstream can be run
+# downstream in Zuul CI and Bodhi.
+# Ref: https://packit.dev/docs/configuration#files_to_sync
+files_to_sync:
+  - src: rpm/gating.yaml
+    dest: gating.yaml
+    delete: true
+  - src: plans/
+    dest: plans/
+    delete: true
+    mkpath: true
+  - src: test/tmt/
+    dest: test/tmt/
+    delete: true
+    mkpath: true
+  - src: .fmf/
+    dest: .fmf/
+    delete: true
+  - .packit.yaml
+
 packages:
   podman-fedora:
     pkg_tool: fedpkg
@@ -80,6 +101,22 @@ jobs:
     project: podman-next
     enable_net: true
 
+  # Tests on Fedora
+  - job: tests
+    trigger: pull_request
+    packages: [podman-fedora]
+    notifications: *packit_generic_failure_notification
+    targets:
+      - fedora-rawhide
+      - fedora-42
+      - fedora-41
+    tmt_plan: "/plans/system/*"
+    tf_extra_params:
+      environments:
+        - artifacts:
+          - type: repository-file
+            id: https://copr.fedorainfracloud.org/coprs/rhcontainerbot/podman-next/repo/fedora-$releasever/rhcontainerbot-podman-next-fedora-$releasever.repo
+
   - job: tests
     identifier: cockpit-revdeps
     trigger: pull_request

plans/system.fmf

@@ -0,0 +1,59 @@
+discover:
+    how: fmf
+
+execute:
+    how: tmt
+
+prepare:
+    - how: shell
+      script: modprobe null_blk nr_devices=1
+      order: 5
+    - when: distro == centos-stream or distro == rhel
+      how: shell
+      script: |
+        dnf -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-$(rpm --eval '%{?rhel}').noarch.rpm
+        dnf -y config-manager --set-enabled epel
+      order: 10
+    - when: initiator == packit
+      how: shell
+      script: |
+        COPR_REPO_FILE="/etc/yum.repos.d/*podman-next*.repo"
+        if compgen -G $COPR_REPO_FILE > /dev/null; then
+            sed -i -n '/^priority=/!p;$apriority=1' $COPR_REPO_FILE
+        fi
+        dnf -y upgrade --allowerasing
+      order: 20
+
+adjust+:
+    - enabled: false
+      when: revdeps == yes
+
+provision:
+    how: artemis
+    hardware:
+        memory: ">= 16 GB"
+        cpu:
+            cores: ">= 4"
+            threads: ">=8"
+        disk:
+            - size: ">= 512 GB"
+
+/local-root:
+    summary: Local rootful tests
+    discover+:
+        filter: 'tag:local & tag:root'
+
+/local-rootless:
+    summary: Local rootless tests
+    discover+:
+        filter: 'tag:local & tag:rootless'
+
+/remote-root:
+    summary: Remote rootful tests
+    discover+:
+        filter: 'tag:remote & tag:root'
+
+/remote-rootless:
+    summary: Remote rootless tests
+    discover+:
+        filter: 'tag:remote & tag:rootless'

plans/tmt.fmf

@@ -0,0 +1,21 @@
+summary: Run tmt container provision test (downstream only)
+
+enabled: false
+adjust+:
+    - enabled: true
+      when: initiator != packit and distro != rhel
+
+discover:
+    how: fmf
+    filter: 'tag:tmt & tag:downstream'
+
+execute:
+    how: tmt
+
+prepare:
+    - when: distro == centos-stream or distro == rhel
+      how: shell
+      script: |
+        dnf -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-$(rpm --eval '%{?rhel}').noarch.rpm
+        dnf -y config-manager --set-enabled epel
+      order: 10

plans/toolbox.fmf

@@ -0,0 +1,29 @@
+summary: Run toolbox tests (downstream only)
+
+enabled: false
+adjust+:
+    - enabled: true
+      when: initiator != packit and distro == fedora
+
+provision:
+    how: artemis
+    hardware:
+        memory: ">= 16 GB"
+        cpu:
+            cores: ">= 4"
+            threads: ">=8"
+        disk:
+            - size: ">= 512 GB"
+
+prepare:
+    - name: packages
+      how: install
+      package: [toolbox-tests]
+
+discover:
+    how: fmf
+    url: https://src.fedoraproject.org/rpms/toolbox
+    ref: "rawhide"
+
+execute:
+    how: tmt

rpm/gating.yaml

@@ -0,0 +1,17 @@
+--- !Policy
+product_versions:
+  - fedora-*
+decision_contexts:
+  - bodhi_update_push_stable
+  - bodhi_update_push_testing
+subject_type: koji_build
+rules:
+  - !PassingTestCaseRule {test_case_name: fedora-ci.koji-build.tier0.functional}
+
+# recipients: jnovy, lsm5, santiago
+--- !Policy
+product_versions:
+  - rhel-*
+decision_context: osci_compose_gate
+rules:
+  - !PassingTestCaseRule {test_case_name: osci.brew-build.tier0.functional}

rpm/podman.spec

@@ -13,6 +13,8 @@
 %define build_with_btrfs 1
 # qemu-system* isn't packageed for CentOS Stream / RHEL
 %define qemu 1
+# bats is included in the default repos (No epel/copr etc.)
+%define distro_bats 1
 %endif
 
 %if %{defined copr_username}
@@ -139,7 +141,7 @@ pages and %{name}.
 Summary: Tests for %{name}
 
 Requires: %{name} = %{epoch}:%{version}-%{release}
-%if %{defined fedora}
+%if %{defined distro_bats}
 Requires: bats
 %endif
 Requires: attr
@@ -157,7 +159,8 @@ Requires: xfsprogs
 %description tests
 %{summary}
 
-This package contains system tests for %{name}
+This package contains system tests for %{name}. Only intended to be used for
+gating tests. Not supported for end users / customers.
 
 %package remote
 Summary: (Experimental) Remote client for managing %{name} containers

test/tmt/system.fmf

@@ -0,0 +1,48 @@
+require:
+    - podman-tests
+    - psmisc
+
+environment:
+    # PODMAN_TESTING envvar is set in system.sh
+    PODMAN: /usr/bin/podman
+    QUADLET: /usr/libexec/podman/quadlet
+    ROOTLESS_USER: "fedora"
+adjust+:
+    - when: distro == centos-stream
+      environment+:
+        ROOTLESS_USER: "ec2-user"
+    - when: distro == rhel
+      environment+:
+        ROOTLESS_USER: "cloud-user"
+
+/local-root:
+    tag: [ local, root ]
+    summary: local rootful test
+    test: bash ./system.sh
+    duration: 30m
+    
+/local-rootless:
+    tag: [ local, rootless ]
+    summary: rootless test
+    test: bash ./system.sh rootless
+    duration: 30m
+
+/remote-root:
+    tag: [ remote, root ]
+    summary: remote rootful test
+    test: bash ./system.sh
+    duration: 30m
+    environment+:
+        PODMAN: /usr/bin/podman-remote
+    require+:
+        - podman-remote
+
+/remote-rootless:
+    tag: [ remote, rootless ]
+    summary: remote rootless test
+    test: bash ./system.sh rootless
+    duration: 30m
+    environment+:
+        PODMAN: /usr/bin/podman-remote
+    require+:
+        - podman-remote

test/tmt/system.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+set -exo pipefail
+
+uname -r
+
+loginctl enable-linger "$ROOTLESS_USER"
+
+rpm -q \
+    aardvark-dns \
+    buildah \
+    conmon \
+    container-selinux \
+    containers-common \
+    criu \
+    crun \
+    netavark \
+    passt \
+    podman \
+    podman-tests \
+    skopeo \
+    slirp4netns \
+    systemd
+
+export system_service_cmd="/usr/bin/podman system service --timeout=0 &"
+export test_cmd="whoami && cd /usr/share/podman/test/system && PODMAN_TESTING=/usr/bin/podman-testing bats ."
+
+if [[ -z $1 ]]; then
+    if [[ $PODMAN == "/usr/bin/podman-remote" ]]; then
+        eval "$system_service_cmd"
+    fi
+    eval "$test_cmd"
+elif [[ $1 == "rootless" ]]; then
+    if [[ $PODMAN == "/usr/bin/podman-remote" ]]; then
+        su - "$ROOTLESS_USER" -c "eval $system_service_cmd"
+    fi
+    su - "$ROOTLESS_USER" -c "eval $test_cmd"
+fi
+
+# Kill all podman processes for remote tests
+if [[ $PODMAN == "/usr/bin/podman-remote" ]]; then
+    killall -q podman
+fi
+exit 0

test/tmt/tmt.fmf

@@ -0,0 +1,13 @@
+enabled: false
+adjust:
+    enabled: true
+    when: initiator != packit && distro != rhel
+summary: Make sure that TMT container provision works
+tag: [downstream]
+require:
+  - tmt+provision-container
+test:
+    tmt run --verbose --remove
+        provision --how container --image fedora
+        login --command 'cat /etc/os-release'
+        finish