ent-13763: various jenkins ci related fixes

ci/cfengine-build-host-setup.cf

@@ -17,6 +17,10 @@ bundle agent cfengine_build_host_setup
     debian.(!debian_13.!debian_12.!ubuntu_22.!ubuntu_24.!ubuntu_25)::
       "python" comment => "debian>=12 and ubuntu>=22 only has python3";
 
+    debian.(!debian_9.!debian_10.!debian_11.!ubuntu_20.!ubuntu_18.!ubuntu_16)::
+      "python3";
+      "python-is-python3" comment => "pipeline hosts need plain old python for buildscripts/build-scripts/get_labels_expr.py";
+
     debian_13|ubuntu_25::
       "ntpsec";
 
@@ -210,14 +214,15 @@ bundle agent cfengine_build_host_setup
       "have_tmp_mount" expression => returnszero("mount | grep '/tmp'", "useshell");
       "have_coredumpctl" expression => returnszero("command -v coredumpctl", "useshell");
       "missing_java" expression => not(fileexists("/usr/bin/java"));
+      "missing_groovy" expression => not(fileexists("/usr/bin/groovy"));
     (redhat|centos).!(redhat_6|centos_6|redhat_7|centos_7)::
       "yum_conf_ok" expression => returnszero("grep best=False /etc/yum.conf >/dev/null", "useshell");
     redhat_6|centos_6|redhat_7|centos_7::
       "yum_conf_ok" expression => "any"; # rhel/centos-6 and 7 do not support --nobest or best property in yum.conf
     redhat_8|centos_8::
       "have_fakeroot" expression => returnszero("command -v fakeroot >/dev/null", "useshell");
     (redhat|centos)::
-      "epel_release_ok" expression => returnszero("yum info --installed epel-release", "useshell");
+      "epel_release_ok" expression => returnszero("yum info installed epel-release", "useshell");
     (redhat_8|centos_8).have_perl::
       "have_data_dumper" expression => returnszero("cpan -l | grep Data::Dumper", "useshell");
       "have_data_dumper_names" expression => returnszero("cpan -l | grep Data::Dumper::Names", "useshell");
@@ -246,6 +251,8 @@ bundle agent cfengine_build_host_setup
       "sysctl kernel.core_pattern='|/lib/systemd/systemd-coredump %p %u %g %s %t %e'" -> { "ENT-12669" }
         comment => "Ensure that core_pattern is proper for systemd-coredump if coredumpctl is present.",
         contain => in_shell;
+    containers_host.missing_groovy::
+      "sh $(this.promise_dirname)/linux-install-groovy.sh" contain => in_shell;
     missing_java::
       "sh $(this.promise_dirname)/linux-install-jdk21.sh" contain => in_shell;
     (redhat_7|centos_7|redhat_8|centos_8|redhat_9|redhat_10).(!have_development_tools).(yum_dnf_conf_ok)::
@@ -288,6 +295,11 @@ bundle agent cfengine_build_host_setup
       "have_$(suse_users_and_groups)_user" expression => returnszero("grep '^$(suse_users_and_groups):' /etc/passwd >/dev/null", "useshell");
 
   files:
+    redhat::
+      "/etc/environment"
+        edit_line => contains_literal_string("LC_ALL=C"),
+        comment => "It seems that centos-7 at least needs LC_ALL=C now, maybe due to different way we access: ProxyJump ssh commands from jenkins instead of java ssh client. 2026-04-03";
+
     linux::
       "/home/jenkins/.ssh/known_hosts"
         create => "true",
@@ -358,7 +370,7 @@ findtime = 600",
 %sudo ALL=NOPASSWD: /usr/bin/podman
 %sudo ALL=NOPASSWD: /usr/sbin/lvs
 %sudo ALL=NOPASSWD: /usr/bin/journalctl
-jenkins_builds ALL=NOPASSWD: /usr/bin/podman
+jenkins ALL=NOPASSWD: /usr/bin/podman
 ",
         create => "true",
         perms => mog("400", "root", "root");

ci/linux-install-groovy.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+groovy_version=5.0.3
+groovy_zip="apache-groovy-binary-$groovy_version.zip"
+cd /opt
+rm -rf apache-groovy-binary*zip
+wget https://groovy.jfrog.io/artifactory/dist-release-local/groovy-zips/"$groovy_zip"
+echo "9d711dcb1dea94df9119558365beb6ac2909a22e30b58ae31de8bcb0dcf33698" "$groovy_zip" > "$groovy_zip".sha256
+sha256sum -c "$groovy_zip".sha256
+unzip -o "$groovy_zip"
+ln -sf /opt/groovy-"$groovy_version"/bin/groovy /usr/bin/
+echo "export JAVA_HOME=/usr" >> /etc/profile
+source /etc/profile
+groovy -v

ci/setup-cfengine-build-host.sh

@@ -1,5 +1,8 @@
 #!/usr/bin/env bash
 shopt -s expand_aliases
+set -ex
+# debug cf-remote troubles
+export CFBACKTRACE=1
 thisdir="$(dirname "$0")"
 
 # install needed packages and software for a build host
@@ -197,7 +200,7 @@ if ! /var/cfengine/bin/cf-agent -V 2>/dev/null; then
   else
     _VERSION=""
   fi
-  cf-remote --log-level info $_VERSION install --clients localhost || true
+  cf-remote --log-level debug $_VERSION install --clients localhost || true
 fi
 
 if [ ! -x /var/cfengine/bin/cf-agent ]; then