Fixed hub upgrade PG migration failing with Permission denied on old binaries

Ticket: ENT-14899
Changelog: Title
(cherry picked from commit 5f80295)

Author: nickanderson

packaging/common/cfengine-hub/postinstall.sh

@@ -594,7 +594,9 @@ migrate_db_using_dump_file() {
     safe_mv "$PREFIX/share" postgresql "$PREFIX/share.new"
     # restore old backup
     safe_cp "$BACKUP_DIR" data "$PREFIX/state/pg"
-    safe_cp "$BACKUP_DIR" bin "$PREFIX"
+    safe_cp_nolink "$BACKUP_DIR" bin "$PREFIX"
+    # Ensure restored old binaries are executable by cfpostgres.
+    chmod 755 "$PREFIX/bin"/* 2>/dev/null || true
     on_files cp "$BACKUP_DIR/lib" "$PREFIX/lib"
     safe_cp "$BACKUP_DIR/lib" postgresql "$PREFIX/lib"
     safe_cp "$BACKUP_DIR/share" postgresql "$PREFIX/share"
@@ -708,6 +710,14 @@ do_migration() {
     else
       DEBUG=0
     fi
+
+    # cfpostgres runs the old binaries during the migration (ENT-14899)
+    if ! su cfpostgres -c "test -x $BACKUP_DIR/bin/pg_ctl"; then
+      cf_console echo "Warning: cfpostgres can't execute the old PostgreSQL binaries, setting them to 755:"
+      cf_console ls -l "$BACKUP_DIR/bin"
+      chmod 755 "$BACKUP_DIR/bin"/*
+    fi
+
     cd /tmp
     cf_console echo "Migrating database using pg_upgrade utility..."
     cf_console echo
@@ -1109,6 +1119,7 @@ if ! [ -f "$PREFIX/UPGRADED_FROM.txt" ] || egrep '3\.([0-6]\.|7\.0)' "$PREFIX/UP
   # Versions <= 3.7.0 are unreliable in their daemon killing. Kill them one
   # more time now that we have upgraded.
   cf_console platform_service cfengine3 stop
+  terminate_cf_agents
 fi
 
 # Let's make sure all files and directories created above have correct SELinux

packaging/common/cfengine-hub/preinstall.sh

@@ -141,6 +141,7 @@ fi
 #stop the remaining services on upgrade
 if is_upgrade; then
   cf_console platform_service cfengine3 stop
+  terminate_cf_agents
   # CFE-2278: Migrate to split units
   if use_systemd && [ -e /usr/lib/systemd/system/cfengine3-web.service ]; then
     # When using systemd, the services are split in two, and although both will
@@ -240,7 +241,13 @@ if migrating_postgres; then
     # the postgresql.conf file was modified, we should try to use it after migration
     cp -a "$BACKUP_DIR/data/postgresql.conf" "$BACKUP_DIR/data/postgresql.conf.modified"
   fi
-  safe_cp "$PREFIX" bin "$BACKUP_DIR"
+  # Copy, don't hardlink: policy managing perms under $PREFIX/state/pg (e.g.
+  # CFE-951 in masterfiles) would otherwise change the live binaries too.
+  safe_cp_nolink "$PREFIX" bin "$BACKUP_DIR"
+  # Normalize executable permissions so cfpostgres can run old PG binaries
+  # during migration, whatever mode damage the source already had.
+  chmod 755 "$BACKUP_DIR/bin"/* 2>/dev/null || true
+  chmod 711 "$BACKUP_DIR/bin" 2>/dev/null || true
   on_files cp "$PREFIX/lib" "$BACKUP_DIR/lib"
   safe_cp "$PREFIX/lib" postgresql "$BACKUP_DIR/lib"
   safe_cp "$PREFIX/share" postgresql "$BACKUP_DIR/share"

packaging/common/cfengine-hub/preremove.sh

@@ -1,4 +1,5 @@
 cf_console platform_service cfengine3 stop
+terminate_cf_agents
 if use_systemd && [ -e /usr/lib/systemd/system/cfengine3-web.service ]; then
   # When using systemd, the services are split in two, and although both will
   # stop due to the command above, the web part may only do so after some

packaging/common/script-templates/script-common.sh

@@ -104,6 +104,64 @@ wait_for_cf_postgres_down() {
     cd /tmp && ! su cfpostgres -c "$PREFIX/bin/psql -l" >/dev/null 2>&1
 }
 
+cf_agent_pids() {
+    # Prints the PIDs of running cf-agent processes, except our own ancestors
+    # (cf-agent may be running this package upgrade) and, where /proc tells
+    # us, processes in other mount namespaces (containers).
+    _ancestors=" "
+    _pid=$$
+    while [ -n "$_pid" ] && [ "$_pid" -gt 1 ]; do
+        _ancestors="$_ancestors$_pid "
+        _pid=$(ps -o ppid= -p "$_pid" 2>/dev/null | tr -d ' ')
+    done
+    _ns=$(readlink /proc/self/ns/mnt 2>/dev/null || true)
+    for _pid in $(pgrep -x cf-agent); do
+        case "$_ancestors" in
+            *" $_pid "*) continue ;;
+        esac
+        if [ -z "$_ns" ] || [ "$(readlink /proc/$_pid/ns/mnt 2>/dev/null)" = "$_ns" ]; then
+            echo "$_pid"
+        fi
+    done
+}
+
+terminate_cf_agents() {
+    # Call after stopping cf-execd. Stopping cf-execd doesn't stop an agent it
+    # started (systemd KillMode=process), and that agent can keep changing
+    # things, like permissions under state/pg (ENT-14899). Gives running agents
+    # 1.5 times the average agent run time from promise_summary.log (150s if
+    # unknown) to finish, then kills them. Does nothing without pgrep.
+    command -v pgrep >/dev/null || return 0
+    _pids=$(cf_agent_pids)
+    if [ -z "$_pids" ]; then
+        return 0
+    fi
+    _timeout=$(tail -n 20 "$PREFIX/promise_summary.log" 2>/dev/null |
+        awk -F'[,:]' '$2 >= $1 { s += $2 - $1; n++ } END { if (n) printf "%d", s / n * 1.5 }')
+    if [ "${_timeout:-0}" -le 0 ]; then
+        _timeout=150
+    fi
+    cf_console echo "Waiting up to ${_timeout}s for cf-agent to finish:"
+    cf_console ps -o pid,etime,args -p "$(echo $_pids | tr ' ' ',')" || true
+    _waited=0
+    while [ -n "$(cf_agent_pids)" ] && [ "$_waited" -lt "$_timeout" ]; do
+        sleep 5
+        _waited=$((_waited + 5))
+    done
+    # Killing an agent can let the shell cf-execd started it from run the next
+    # one (update.cf, then promises.cf), so check a few times.
+    for _try in 1 2 3; do
+        _pids=$(cf_agent_pids)
+        if [ -z "$_pids" ]; then
+            return 0
+        fi
+        cf_console echo "Warning: killing cf-agent still running after ${_timeout}s:"
+        cf_console ps -o pid,etime,args -p "$(echo $_pids | tr ' ' ',')" || true
+        kill -KILL $_pids 2>/dev/null || true
+        sleep 1
+    done
+}
+
 safe_cp() {
     # "safe" alternative to `cp`. Tries `cp -al` first, and if it fails - `cp -a`.
     # Deletes partially-copied files if copy operation fails.
@@ -136,6 +194,21 @@ safe_cp() {
     return 1
 }
 
+safe_cp_nolink() {
+    # Like `safe_cp`, but never creates hardlinks, so the copy does not share
+    # mode and owner with the source. Args are same as in safe_cp.
+    test "$#" -eq 3 || return 2
+    from="$1"
+    name="$2"
+    to="$3"
+    if cp -a "$from/$name" "$to"; then
+        return 0
+    fi
+    echo "Copy failed, so removing partially-copied data and aborting"
+    rm -rf "$to/$name"
+    return 1
+}
+
 safe_mv() {
     # "safe" alternative to `mv`. Executes `safe_cp` and deletes source dir if
     # that succeeds.