imp: exec: wait for job cgroup to be empty before exiting

src/imp/cgroup.c

@@ -241,5 +241,28 @@ int cgroup_kill (struct cgroup_info *cgroup, int sig)
     return count;
 }
 
+int cgroup_wait_for_empty (struct cgroup_info *cgroup)
+{
+    int n;
+
+    /*  Only wait for empty cgroup if cgroup kill is enabled.
+     */
+    if (!cgroup->use_cgroup_kill)
+        return 0;
+
+    while ((n = cgroup_kill (cgroup, 0)) > 0) {
+        /*  Note: inotify/poll() do not work on the cgroup.procs virtual
+         *  file. Therefore, wait at most 1s and check to see if the cgroup
+         *  is empty again. If the job execution system requests a signal to
+         *  be delivered then the sleep will be interrupted, in which case a
+         *  a small delay is added in hopes that any terminated processes
+         *  will have been removed from cgroup.procs by then.
+         */
+        if (usleep (1e6) < 0 && errno == EINTR)
+            usleep (2000);
+    }
+    return 0;
+}
+
 /* vi: ts=4 sw=4 expandtab
  */

src/imp/cgroup.h

@@ -27,4 +27,8 @@ void cgroup_info_destroy (struct cgroup_info *cgroup);
  */
 int cgroup_kill (struct cgroup_info *cgroup, int sig);
 
+/*  Wait for all processes in cgroup (except this one) to exit.
+ */
+int cgroup_wait_for_empty (struct cgroup_info *cgroup);
+
 #endif /* !HAVE_IMP_CGROUP_H */

src/imp/exec/exec.c

@@ -293,6 +293,9 @@ int imp_exec_privileged (struct imp_state *imp, struct kv *kv)
             imp_die (1, "waitpid: %s", strerror (errno));
     }
 
+    if (cgroup_wait_for_empty (exec->imp->cgroup) < 0)
+        imp_warn ("error waiting for processes in job cgroup");
+
 #if HAVE_PAM
     /* Call privliged IMP plugins/containment finalization */
     if (imp_supports_pam (exec))

src/imp/signals.c

@@ -93,7 +93,7 @@ void imp_setup_signal_forwarding (struct imp_state *imp)
 
     memset(&sa, 0, sizeof(sa));
     sa.sa_handler = fwd_signal;
-    sa.sa_flags = SA_RESTART;
+    sa.sa_flags = 0;
     sigemptyset(&sa.sa_mask);
 
     sigfillset (&mask);

t/t2000-imp-exec.t

@@ -329,6 +329,22 @@ test_expect_success SUDO,CGROUPFS,NO_CHAIN_LINT \
 	test_must_be_empty ${CGROUP_PATH}/cgroup.procs
 '
 
+test_expect_success SUDO,CGROUPFS,NO_CHAIN_LINT \
+	'flux-imp exec: SIGUSR1 waits for cgroup to be empty' '
+	fake_input_sign_none | \
+		$SUDO FLUX_IMP_CONFIG_PATTERN=sign-none.toml \
+			./run-in-cgroup.sh "$CGROUP_PATH" \
+				$flux_imp exec $(pwd)/sleeper.sh 15 &
+	imp_pid=$! &&
+	test_when_finished "rm -f sleeper.pid" &&
+	wait_for_file sleeper.pid &&
+	kill -TERM $(cat sleeper.pid) &&
+	sleep .5 &&
+	kill -USR1 $(cat sleeper.pid) &&
+	test_expect_code 143 wait $imp_pid &&
+	test_must_be_empty ${CGROUP_PATH}/cgroup.procs
+'
+
 $flux_imp version | grep -q pam || test_set_prereq NO_PAM
 test_expect_success NO_PAM,SUDO 'flux-imp exec: fails if not built with PAM but pam-support=true' '
 	( export FLUX_IMP_CONFIG_PATTERN=pam-test.toml &&