From b7251ce3bf65bcbec7ecbb98a0eb0b3c9abde507 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Fri, 13 Sep 2024 09:35:34 -0400
Subject: Move guard_ macros to core_mem.h

* Move core_mem.h below config.h
---
 src/CMakeLists.txt    |   1 +
 src/multiprocessing.c | 373 ++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 374 insertions(+)
 create mode 100644 src/multiprocessing.c

(limited to 'src')

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 2399dc5..5d1c8ff 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -25,6 +25,7 @@ add_library(stasis_core STATIC
         github.c
         template_func_proto.c
         envctl.c
+        multiprocessing.c
 )
 
 add_executable(stasis
diff --git a/src/multiprocessing.c b/src/multiprocessing.c
new file mode 100644
index 0000000..09190d3
--- /dev/null
+++ b/src/multiprocessing.c
@@ -0,0 +1,373 @@
+#include "core_mem.h"
+#include "multiprocessing.h"
+#include "utils.h"
+
+size_t mp_global_task_count = 0;
+static struct MultiProcessingTask *mp_pool_next_available(struct MultiProcessingPool *pool) {
+    return &pool->task[pool->num_used];
+}
+
+struct MultiProcessingTask *mp_task(struct MultiProcessingPool *pool, const char *ident, char *cmd) {
+    struct MultiProcessingTask *slot = mp_pool_next_available(pool);
+    //struct MultiProcessingTask *slot = mp_pool_any_available(pool);
+    pool->num_used++;
+
+    memset(slot->ident, 0, sizeof(slot->ident));
+    strncpy(slot->ident, ident, sizeof(slot->ident) - 1);
+
+    strcat(slot->log_file, pool->log_root);
+    strcat(slot->log_file, "/");
+
+    FILE *tp = NULL;
+    char *t_name;
+    t_name = xmkstemp(&tp, "w");
+    if (!t_name || !tp) {
+        return NULL;
+    }
+
+    // Set the script's permissions so that only the calling user can use it
+    // This should help prevent eavesdropping if keys are applied in plain-text
+    // somewhere.
+    chmod(t_name, 0700);
+
+    memset(slot->parent_script, 0, sizeof(slot->parent_script));
+    strncpy(slot->parent_script, t_name, PATH_MAX - 1);
+
+    fprintf(tp, "#!/bin/bash\n%s\n", cmd);
+    fflush(tp);
+    fclose(tp);
+
+    pid_t pid = fork();
+    int child_status = 0;
+    if (pid == -1) {
+        return NULL;
+    } else if (pid == 0) { // child process
+        char *cwd = NULL;
+        FILE *fp_log = NULL;
+
+        // Synchronize sub-process startup
+        // Stop here until summoned by mp_pool_join()
+        if (sem_wait(slot->gate) < 0) {
+            perror("sem_wait failed");
+            exit(1);
+        }
+
+
+        // Redirect stdout and stderr to the log file
+        fflush(stdout);
+        fflush(stderr);
+        printf("[%s:%s] Task started (pid: %d)\n", pool->ident, slot->ident, slot->parent_pid);
+        fp_log = freopen(slot->log_file, "w+", stdout);
+        if (!fp_log) {
+            fprintf(stderr, "unable to open '%s' for writing: %s\n", slot->log_file, strerror(errno));
+            return NULL;
+        }
+        dup2(fileno(stdout), fileno(stderr));
+
+        // Generate timestamp for log header
+        time_t t = time(NULL);
+        char *timebuf = ctime(&t);
+        if (timebuf) {
+            // strip line feed from timestamp
+            timebuf[strlen(timebuf) ? strlen(timebuf) - 1 : 0] = 0;
+        }
+
+        cwd = getcwd(NULL, PATH_MAX);
+        if (!cwd) {
+            perror(cwd);
+            exit(1);
+        }
+
+        // Generate log header
+        fprintf(fp_log, "# STARTED: %s\n", timebuf ? timebuf : "unknown");
+        fprintf(fp_log, "# PID: %d\n", slot->parent_pid);
+        fprintf(fp_log, "# WORKDIR: %s\n", cwd);
+        fprintf(fp_log, "# COMMAND:\n%s\n", cmd);
+        fprintf(fp_log, "# OUTPUT:\n");
+        // Commit header to log file / clean up
+        fflush(fp_log);
+        guard_free(cwd);
+
+        // Execute task
+        fflush(stdout);
+        fflush(stderr);
+        char *args[] = {"bash", "--norc", t_name, (char *) NULL};
+        slot->status = execvp("/bin/bash", args);
+    } else { // parent process
+        printf("[%s:%s] Task queued (pid: %d)\n", pool->ident, slot->ident, pid);
+
+        // Give the child process access to our PID value
+        slot->pid = pid;
+        slot->parent_pid = pid;
+
+        // Set log file name
+        sprintf(slot->log_file + strlen(slot->log_file), "task-%zu-%d.log", mp_global_task_count, slot->parent_pid);
+        mp_global_task_count++;
+
+        // Check child's status
+        pid_t code = waitpid(pid, &child_status, WUNTRACED | WCONTINUED | WNOHANG);
+        if (code < 0) {
+            perror("waitpid failed");
+            return NULL;
+        }
+        return slot;
+    }
+    return NULL;
+}
+
+static int show_log_contents(FILE *stream, struct MultiProcessingTask *task) {
+    FILE *fp = fopen(task->log_file, "r");
+    if (!fp) {
+        return -1;
+    }
+    char buf[BUFSIZ] = {0};
+    while ((fgets(buf, sizeof(buf) - 1, fp)) != NULL) {
+        fprintf(stream, "%s", buf);
+        memset(buf, 0, sizeof(buf));
+    }
+    fprintf(stream, "\n");
+    return 0;
+}
+
+int mp_pool_kill(struct MultiProcessingPool *pool, int signum) {
+    printf("Sending signal %d to pool '%s'\n", signum, pool->ident);
+    //for (size_t i = 0; i < pool->num_alloc; i++) {
+    for (size_t i = 0; i < pool->num_used; i++) {
+        struct MultiProcessingTask *slot = &pool->task[i];
+        if (!slot) {
+            return -1;
+        }
+        if (slot->pid > 0) {
+            int status;
+            printf("Sending signal %d to task '%s' (pid: %d)\n", signum, slot->ident, slot->pid);
+            status = kill(slot->pid, signum);
+            if (status && errno != ESRCH) {
+                fprintf(stderr, "Task '%s' (pid: %d) did not respond: %s\n", slot->ident, slot->pid, strerror(errno));
+            }
+        }
+        if (!access(slot->log_file, F_OK)) {
+            remove(slot->log_file);
+        }
+        if (!access(slot->parent_script, F_OK)) {
+            remove(slot->parent_script);
+        }
+    }
+    return 0;
+}
+
+int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) {
+    int status = 0;
+    int watcher = 0;
+    int failures = 0;
+    size_t tasks_complete = 0;
+    size_t lower_i = 0;
+    size_t upper_i = jobs;
+
+    do {
+        //if (upper_i >= pool->num_alloc) {
+        if (upper_i >= pool->num_used) {
+            size_t x = upper_i - pool->num_used;
+            //size_t x = upper_i - pool->num_alloc;
+            upper_i -= (size_t) x;
+        }
+        for (size_t i = lower_i; i < upper_i; i++) {
+            struct MultiProcessingTask *slot = &pool->task[i];
+            // Unlock the semaphore to allow the queued processes to start up
+            if (sem_post(slot->gate) < 0) {
+                perror("sem_post failed");
+                exit(1);
+            }
+
+            if (slot->pid == MP_POOL_PID_UNUSED) {
+                // PID is already used up, skip it
+                continue;
+            }
+
+            // Is the process finished?
+            pid_t pid = waitpid(slot->pid, &status, WNOHANG | WUNTRACED | WCONTINUED);
+
+            // Update status
+            slot->status = status;
+
+            char progress[1024] = {0};
+            if (pid > 0) {
+                double percent = ((double) (tasks_complete) / (double) pool->num_used) * 100;
+                //double percent = ((double) (tasks_complete) / (double) pool->num_alloc) * 100;
+                sprintf(progress, "[%s:%s] [%3.1f%%]", pool->ident, slot->ident, percent);
+
+                // The process ended in one the following ways
+                // Note: SIGSTOP nor SIGCONT will not increment the tasks_complete counter
+                if (WIFEXITED(status)) {
+                    printf("%s Task finished (status: %d)\n", progress, WEXITSTATUS(status));
+                    tasks_complete++;
+                } else if (WIFSIGNALED(status)) {
+                    printf("%s Task ended by signal %d (%s)\n", progress, WTERMSIG(status), strsignal(WTERMSIG(status)));
+                    tasks_complete++;
+                } else if (WIFSTOPPED(status)) {
+                    printf("%s Task was suspended (%d)\n", progress, WSTOPSIG(status));
+                    continue;
+                } else if (WIFCONTINUED(status)) {
+                    printf("%s Task was resumed\n", progress);
+                    continue;
+                } else {
+                    fprintf(stderr, "%s Task state is unknown (0x%04X)\n", progress, status);
+                }
+
+                // Show the log (always)
+                if (show_log_contents(stdout, slot)) {
+                    perror(slot->log_file);
+                }
+                if (status >> 8 != 0 || (status & 0xff) != 0) {
+                    fprintf(stderr, "%s Task failed\n", progress);
+                    if (flags & MP_POOL_FAIL_FAST) {
+                        mp_pool_kill(pool, SIGTERM);
+                        return -2;
+                    }
+                } else {
+                    printf("%s Task finished\n", progress);
+                }
+
+                // Clean up logs and scripts left behind by the task
+                if (remove(slot->log_file)) {
+                    fprintf(stderr, "%s Unable to remove log file: '%s': %s\n", progress, slot->parent_script, strerror(errno));
+                }
+                if (remove(slot->parent_script)) {
+                    fprintf(stderr, "%s Unable to remove temporary script '%s': %s\n", progress, slot->parent_script, strerror(errno));
+                }
+
+                // Update progress and tell the poller to ignore the PID. The process is gone.
+                slot->pid = MP_POOL_PID_UNUSED;
+                failures += status;
+            } else if (pid < 0) {
+                fprintf(stderr, "waitpid failed: %s\n", strerror(errno));
+                return -1;
+            } else {
+                if (watcher > 9) {
+                    printf("[%s:%s] Task is running (pid: %d)\n", pool->ident, slot->ident, slot->parent_pid);
+                    watcher = 0;
+                } else {
+                    watcher++;
+                }
+            }
+        }
+
+        if (tasks_complete == pool->num_used) {
+            break;
+        }
+
+        if (tasks_complete == upper_i) {
+            lower_i += jobs;
+            upper_i += jobs;
+        }
+
+        // Poll again after a short delay
+        sleep(1);
+    } while (1);
+
+    puts("");
+    return failures;
+}
+
+
+struct MultiProcessingPool *mp_pool_init(const char *ident, const char *log_root) {
+    struct MultiProcessingPool *pool;
+
+    if (!log_root) {
+        // log_root must be set
+        return NULL;
+    }
+
+    //pool = malloc(1 * sizeof(*pool));
+    pool = mmap(NULL, sizeof(*pool), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS | MAP_POPULATE, -1, 0);
+    pool->ident = ident;
+    pool->log_root = log_root;
+    pool->num_used = 0;
+    pool->num_alloc = MP_POOL_TASK_MAX;
+
+    if (mkdirs(log_root, 0700) < 0) {
+        if (errno != EEXIST) {
+            perror(log_root);
+            mp_pool_free(&pool);
+            return NULL;
+        }
+    }
+
+    //pool->task = calloc(pool->num_alloc + 1, sizeof(*pool->task));
+    pool->task = mmap(NULL, (pool->num_alloc + 1) * sizeof(*pool->task), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS | MAP_POPULATE, -1, 0);
+    if (pool->task == MAP_FAILED) {
+        perror("mmap");
+        mp_pool_free(&pool);
+        return NULL;
+    }
+
+    for (size_t i = 0; i < pool->num_alloc; i++) {
+        struct MultiProcessingTask *slot = &pool->task[i];
+        slot->gate = mmap(NULL, sizeof(*slot->gate), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS | MAP_POPULATE, -1, 0);
+        if (slot->gate == MAP_FAILED) {
+            perror("mmap failed");
+            exit(1);
+        }
+
+        if (sem_init(slot->gate, 1, 0) < 0) {
+            perror("sem_init failed");
+            exit(1);
+        }
+    }
+
+
+    return pool;
+}
+
+void mp_pool_free(struct MultiProcessingPool **pool) {
+    //for (size_t i = 0; i < (*pool)->num_used; i++) {
+    for (size_t i = 0; i < (*pool)->num_alloc; i++) {
+        if ((*pool)->task[i].gate) {
+            if (sem_destroy((*pool)->task[i].gate) < 0) {
+                perror("sem_destroy failed");
+                exit(1);
+            }
+            if (munmap((*pool)->task[i].gate, sizeof(*(*pool)->task[i].gate)) < 0) {
+                perror("munmap");
+                exit(1);
+            }
+        }
+    }
+    if ((*pool)->task) {
+        if (munmap((*pool)->task, sizeof(*(*pool)->task) * (*pool)->num_alloc) < 0) {
+            perror("munmap");
+        }
+    }
+    if ((*pool)) {
+        if (munmap((*pool), sizeof(*(*pool))) < 0) {
+            perror("munmap");
+        }
+        (*pool) = NULL;
+    }
+}
+
+int exmain(int argc, char *argv[]) {
+    size_t i;
+    struct MultiProcessingPool *pool = mp_pool_init("generic", "logs");
+    if (!pool) {
+        perror("pool init");
+        exit(1);
+    }
+
+    char *commands[] = {
+        "sleep 2; dd if=/dev/zero of=file.dat bs=1M count=1",
+        "/bin/echo hello world; sleep 5",
+        "python -c 'print(1+1)'",
+        "(for x in {1..10}; do echo $x; sleep 0.5; done)",
+        "echo stdout >&1; echo stderr >&2; exit 1"
+    };
+
+    for (i = 0; i < sizeof(commands) / sizeof(*commands); i++) {
+        if (mp_task(pool, "commands array", commands[i]) == NULL) {
+            printf("Too many tasks queued (max: %d)\n", MP_POOL_TASK_MAX);
+            break;
+        }
+    }
+    mp_pool_join(pool, get_cpu_count() - 1, MP_POOL_FAIL_FAST);
+    mp_pool_free(&pool);
+    return 0;
+}
-- 
cgit 


From 8f17199d16bcdb29516d34514f95d1a117f6bd26 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Fri, 13 Sep 2024 09:58:17 -0400
Subject: Implement multiprocessing pool(s)

* Adds --cpu-limit and --parallel-fail-fast arguments
* Adds disable, parallel, and setup_script keys to [test] blocks
---
 src/delivery.c            | 187 ++++++++++++++++++++++++++++++++++------------
 src/globals.c             |  28 +++----
 src/stasis_main.c         |  27 ++++++-
 src/template_func_proto.c |  37 +++++++++
 4 files changed, 216 insertions(+), 63 deletions(-)

(limited to 'src')

diff --git a/src/delivery.c b/src/delivery.c
index 3e99aad..2718c08 100644
--- a/src/delivery.c
+++ b/src/delivery.c
@@ -2,6 +2,7 @@
 
 #include <fnmatch.h>
 #include "core.h"
+#include "multiprocessing.h"
 
 extern struct STASIS_GLOBAL globals;
 
@@ -560,7 +561,13 @@ static int populate_delivery_ini(struct Delivery *ctx, int render_mode) {
 
             test->version = ini_getval_str(ini, section_name, "version", render_mode, &err);
             test->repository = ini_getval_str(ini, section_name, "repository", render_mode, &err);
+            test->script_setup = ini_getval_str(ini, section_name, "script_setup", INI_READ_RAW, &err);
             test->script = ini_getval_str(ini, section_name, "script", INI_READ_RAW, &err);
+            test->disable = ini_getval_bool(ini, section_name, "disable", render_mode, &err);
+            test->parallel = ini_getval_bool(ini, section_name, "parallel", render_mode, &err);
+            if (err) {
+                test->parallel = true;
+            }
             test->repository_remove_tags = ini_getval_strlist(ini, section_name, "repository_remove_tags", LINE_SEP, render_mode, &err);
             test->build_recipe = ini_getval_str(ini, section_name, "build_recipe", render_mode, &err);
             test->runtime.environ = ini_getval_strlist(ini, section_name, "runtime", LINE_SEP, render_mode, &err);
@@ -1702,6 +1709,9 @@ int delivery_index_conda_artifacts(struct Delivery *ctx) {
 }
 
 void delivery_tests_run(struct Delivery *ctx) {
+    struct MultiProcessingPool *pool_parallel;
+    struct MultiProcessingPool *pool_serial;
+    struct MultiProcessingPool *pool_setup;
     struct Process proc;
     memset(&proc, 0, sizeof(proc));
 
@@ -1715,20 +1725,40 @@ void delivery_tests_run(struct Delivery *ctx) {
     if (!ctx->tests[0].name) {
         msg(STASIS_MSG_WARN | STASIS_MSG_L2, "no tests are defined!\n");
     } else {
+        pool_parallel = mp_pool_init("parallel", ctx->storage.tmpdir);
+        if (!pool_parallel) {
+            perror("mp_pool_init/parallel");
+            exit(1);
+        }
+        
+        pool_serial = mp_pool_init("serial", ctx->storage.tmpdir);
+        if (!pool_serial) {
+            perror("mp_pool_init/serial");
+            exit(1);
+        }
+
+        pool_setup = mp_pool_init("setup", ctx->storage.tmpdir);
+        if (!pool_setup) {
+            perror("mp_pool_init/setup");
+            exit(1);
+        }
+
+        const char *runner_cmd_fmt = "set -e -x\n%s\n";
         for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) {
-            if (!ctx->tests[i].name && !ctx->tests[i].repository && !ctx->tests[i].script) {
+            struct Test *test = &ctx->tests[i];
+            if (!test->name && !test->repository && !test->script) {
                 // skip unused test records
                 continue;
             }
-            msg(STASIS_MSG_L2, "Executing tests for %s %s\n", ctx->tests[i].name, ctx->tests[i].version);
-            if (!ctx->tests[i].script || !strlen(ctx->tests[i].script)) {
+            msg(STASIS_MSG_L2, "Executing tests for %s %s\n", test->name, test->version);
+            if (!test->script || !strlen(test->script)) {
                 msg(STASIS_MSG_WARN | STASIS_MSG_L3, "Nothing to do. To fix, declare a 'script' in section: [test:%s]\n",
-                    ctx->tests[i].name);
+                    test->name);
                 continue;
             }
 
             char destdir[PATH_MAX];
-            sprintf(destdir, "%s/%s", ctx->storage.build_sources_dir, path_basename(ctx->tests[i].repository));
+            sprintf(destdir, "%s/%s", ctx->storage.build_sources_dir, path_basename(test->repository));
 
             if (!access(destdir, F_OK)) {
                 msg(STASIS_MSG_L3, "Purging repository %s\n", destdir);
@@ -1736,44 +1766,31 @@ void delivery_tests_run(struct Delivery *ctx) {
                     COE_CHECK_ABORT(1, "Unable to remove repository\n");
                 }
             }
-            msg(STASIS_MSG_L3, "Cloning repository %s\n", ctx->tests[i].repository);
-            if (!git_clone(&proc, ctx->tests[i].repository, destdir, ctx->tests[i].version)) {
-                ctx->tests[i].repository_info_tag = strdup(git_describe(destdir));
-                ctx->tests[i].repository_info_ref = strdup(git_rev_parse(destdir, "HEAD"));
+            msg(STASIS_MSG_L3, "Cloning repository %s\n", test->repository);
+            if (!git_clone(&proc, test->repository, destdir, test->version)) {
+                test->repository_info_tag = strdup(git_describe(destdir));
+                test->repository_info_ref = strdup(git_rev_parse(destdir, "HEAD"));
             } else {
                 COE_CHECK_ABORT(1, "Unable to clone repository\n");
             }
 
-            if (ctx->tests[i].repository_remove_tags && strlist_count(ctx->tests[i].repository_remove_tags)) {
-                filter_repo_tags(destdir, ctx->tests[i].repository_remove_tags);
+            if (test->repository_remove_tags && strlist_count(test->repository_remove_tags)) {
+                filter_repo_tags(destdir, test->repository_remove_tags);
             }
 
             if (pushd(destdir)) {
                 COE_CHECK_ABORT(1, "Unable to enter repository directory\n");
             } else {
-#if 1
-                int status;
-                char *cmd = calloc(strlen(ctx->tests[i].script) + STASIS_BUFSIZ, sizeof(*cmd));
+                char *cmd = calloc(strlen(test->script) + STASIS_BUFSIZ, sizeof(*cmd));
+                if (!cmd) {
+                    SYSERROR("Unable to allocate test script buffer: %s", strerror(errno));
+                    exit(1);
+                }
 
-                msg(STASIS_MSG_L3, "Testing %s\n", ctx->tests[i].name);
+                msg(STASIS_MSG_L3, "Testing %s\n", test->name);
                 memset(&proc, 0, sizeof(proc));
 
-                // Apply workaround for tox positional arguments
-                char *toxconf = NULL;
-                if (!access("tox.ini", F_OK)) {
-                    if (!fix_tox_conf("tox.ini", &toxconf)) {
-                        msg(STASIS_MSG_L3, "Fixing tox positional arguments\n");
-                        if (!globals.workaround.tox_posargs) {
-                            globals.workaround.tox_posargs = calloc(PATH_MAX, sizeof(*globals.workaround.tox_posargs));
-                        } else {
-                            memset(globals.workaround.tox_posargs, 0, PATH_MAX);
-                        }
-                        snprintf(globals.workaround.tox_posargs, PATH_MAX - 1, "-c %s --root .", toxconf);
-                    }
-                }
-
-                // enable trace mode before executing each test script
-                strcpy(cmd, ctx->tests[i].script);
+                strcpy(cmd, test->script);
                 char *cmd_rendered = tpl_render(cmd);
                 if (cmd_rendered) {
                     if (strcmp(cmd_rendered, cmd) != 0) {
@@ -1786,36 +1803,110 @@ void delivery_tests_run(struct Delivery *ctx) {
                     exit(1);
                 }
 
-                puts(cmd);
+                if (test->disable) {
+                    msg(STASIS_MSG_L2, "Script execution disabled by configuration\n", test->name);
+                    guard_free(cmd);
+                    continue;
+                }
+
                 char runner_cmd[0xFFFF] = {0};
-                sprintf(runner_cmd, "set +x\nsource %s/etc/profile.d/conda.sh\nsource %s/etc/profile.d/mamba.sh\n\nmamba activate ${CONDA_DEFAULT_ENV}\n\n%s\n",
-                    ctx->storage.conda_install_prefix,
-		    ctx->storage.conda_install_prefix,
-		    cmd);
-                status = shell(&proc, runner_cmd);
-                if (status) {
-                    msg(STASIS_MSG_ERROR, "Script failure: %s\n%s\n\nExit code: %d\n", ctx->tests[i].name, ctx->tests[i].script, status);
+                char pool_name[100] = "parallel";
+                struct MultiProcessingTask *task = NULL;
+                struct MultiProcessingPool *pool = pool_parallel;
+                if (!test->parallel) {
+                    pool = pool_serial;
+                    memset(pool_name, 0, sizeof(pool_name));
+                    strcpy(pool_name, "serial");
+                }
+
+                sprintf(runner_cmd, runner_cmd_fmt, cmd);
+                task = mp_task(pool, test->name, runner_cmd);
+                if (!task) {
+                    SYSERROR("Failed to add task to %s pool: %s", pool_name, runner_cmd);
                     popd();
-                    guard_free(cmd);
                     if (!globals.continue_on_error) {
                         tpl_free();
                         delivery_free(ctx);
                         globals_free();
                     }
-                    COE_CHECK_ABORT(1, "Test failure");
+                    exit(1);
                 }
                 guard_free(cmd);
+                popd();
+
+            }
+        }
 
-                if (toxconf) {
-                    remove(toxconf);
-                    guard_free(toxconf);
+        // Configure "script_setup" tasks
+        // Directories should exist now, so no need to go through initializing everything all over again.
+        for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) {
+            struct Test *test = &ctx->tests[i];
+            if (test->script_setup) {
+                char destdir[PATH_MAX];
+                sprintf(destdir, "%s/%s", ctx->storage.build_sources_dir, path_basename(test->repository));
+                if (access(destdir, F_OK)) {
+                    SYSERROR("%s: %s", destdir, strerror(errno));
+                    exit(1);
+                }
+                if (!pushd(destdir)) {
+                    char *cmd = calloc(strlen(test->script_setup) + STASIS_BUFSIZ, sizeof(*cmd));
+                    if (!cmd) {
+                        SYSERROR("Unable to allocate test script_setup buffer: %s", strerror(errno));
+                        exit(1);
+                    }
+
+                    strcpy(cmd, test->script_setup);
+                    char *cmd_rendered = tpl_render(cmd);
+                    if (cmd_rendered) {
+                        if (strcmp(cmd_rendered, cmd) != 0) {
+                            strcpy(cmd, cmd_rendered);
+                            cmd[strlen(cmd_rendered) ? strlen(cmd_rendered) - 1 : 0] = 0;
+                        }
+                        guard_free(cmd_rendered);
+                    } else {
+                        SYSERROR("An error occurred while rendering the following:\n%s", cmd);
+                        exit(1);
+                    }
+
+                    struct MultiProcessingPool *pool = pool_setup;
+                    struct MultiProcessingTask *task = NULL;
+                    char runner_cmd[0xFFFF] = {0};
+                    sprintf(runner_cmd, runner_cmd_fmt, cmd);
+
+                    task = mp_task(pool, test->name, runner_cmd);
+                    if (!task) {
+                        SYSERROR("Failed to add task %s to setup pool: %s", test->name, runner_cmd);
+                        popd();
+                        if (!globals.continue_on_error) {
+                            tpl_free();
+                            delivery_free(ctx);
+                            globals_free();
+                        }
+                        exit(1);
+                    }
+                    guard_free(cmd);
+                    popd();
                 }
-                popd();
-#else
-                msg(STASIS_MSG_WARNING | STASIS_MSG_L3, "TESTING DISABLED BY CODE!\n");
-#endif
             }
         }
+
+        size_t opt_flags = 0;
+        opt_flags |= globals.parallel_fail_fast;
+
+        if (pool_setup->num_used) {
+            COE_CHECK_ABORT(mp_pool_join(pool_setup, 1, opt_flags) != 0, "Failure in setup task pool");
+            mp_pool_free(&pool_setup);
+        }
+
+        if (pool_parallel->num_used) {
+            COE_CHECK_ABORT(mp_pool_join(pool_parallel, globals.cpu_limit, opt_flags) != 0, "Failure in parallel task pool");
+            mp_pool_free(&pool_parallel);
+        }
+
+        if (pool_serial->num_used) {
+            COE_CHECK_ABORT(mp_pool_join(pool_serial, 1, opt_flags) != 0, "Failure in serial task pool");
+            mp_pool_free(&pool_serial);
+        }
     }
 }
 
diff --git a/src/globals.c b/src/globals.c
index 1e27959..667809b 100644
--- a/src/globals.c
+++ b/src/globals.c
@@ -25,19 +25,20 @@ const char *BANNER =
         "Association of Universities for Research in Astronomy (AURA)\n";
 
 struct STASIS_GLOBAL globals = {
-        .verbose = false,
-        .continue_on_error = false,
-        .always_update_base_environment = false,
-        .conda_fresh_start = true,
-        .conda_install_prefix = NULL,
-        .conda_packages = NULL,
-        .pip_packages = NULL,
-        .tmpdir = NULL,
-        .enable_docker = true,
-        .enable_artifactory = true,
-        .enable_artifactory_build_info = true,
-        .enable_testing = true,
-        .enable_rewrite_spec_stage_2 = true,
+        .verbose = false, ///< Toggle verbose mode
+        .continue_on_error = false, ///< Do not stop program on error
+        .always_update_base_environment = false, ///< Run "conda update --all" after installing Conda
+        .conda_fresh_start = true, ///< Remove/reinstall Conda at startup
+        .conda_install_prefix = NULL, ///< Path to install Conda
+        .conda_packages = NULL, ///< Conda packages to install
+        .pip_packages = NULL, ///< Python packages to install
+        .tmpdir = NULL, ///< Path to store temporary data
+        .enable_docker = true, ///< Toggle docker usage
+        .enable_artifactory = true, ///< Toggle artifactory server usage
+        .enable_artifactory_build_info = true, ///< Toggle build-info uploads
+        .enable_testing = true, ///< Toggle [test] block "script" execution. "script_setup" always executes.
+        .enable_rewrite_spec_stage_2 = true, ///< Leave template stings in output files
+        .parallel_fail_fast = false, ///< Kill ALL multiprocessing tasks immediately on error
 };
 
 void globals_free() {
@@ -55,7 +56,6 @@ void globals_free() {
     guard_free(globals.jfrog.jfrog_artifactory_base_url);
     guard_free(globals.jfrog.jfrog_artifactory_product);
     guard_free(globals.jfrog.remote_filename);
-    guard_free(globals.workaround.tox_posargs);
     guard_free(globals.workaround.conda_reactivate);
     if (globals.envctl) {
         envctl_free(&globals.envctl);
diff --git a/src/stasis_main.c b/src/stasis_main.c
index 7ea465c..164a9ca 100644
--- a/src/stasis_main.c
+++ b/src/stasis_main.c
@@ -12,15 +12,18 @@
 #define OPT_NO_TESTING 1004
 #define OPT_OVERWRITE 1005
 #define OPT_NO_REWRITE_SPEC_STAGE_2 1006
+#define OPT_PARALLEL_FAIL_FAST 1007
 static struct option long_options[] = {
         {"help", no_argument, 0, 'h'},
         {"version", no_argument, 0, 'V'},
         {"continue-on-error", no_argument, 0, 'C'},
         {"config", required_argument, 0, 'c'},
+        {"cpu-limit", required_argument, 0, 'l'},
         {"python", required_argument, 0, 'p'},
         {"verbose", no_argument, 0, 'v'},
         {"unbuffered", no_argument, 0, 'U'},
         {"update-base", no_argument, 0, OPT_ALWAYS_UPDATE_BASE},
+        {"parallel-fail-fast", no_argument, 0, OPT_PARALLEL_FAIL_FAST},
         {"overwrite", no_argument, 0, OPT_OVERWRITE},
         {"no-docker", no_argument, 0, OPT_NO_DOCKER},
         {"no-artifactory", no_argument, 0, OPT_NO_ARTIFACTORY},
@@ -35,10 +38,12 @@ const char *long_options_help[] = {
         "Display program version",
         "Allow tests to fail",
         "Read configuration file",
+        "Number of processes to spawn concurrently (default: cpus - 1)",
         "Override version of Python in configuration",
         "Increase output verbosity",
         "Disable line buffering",
         "Update conda installation prior to STASIS environment creation",
+        "On test error, terminate all concurrent tasks",
         "Overwrite an existing release",
         "Do not build docker images",
         "Do not upload artifacts to Artifactory",
@@ -201,6 +206,13 @@ static void check_requirements(struct Delivery *ctx) {
 }
 
 int main(int argc, char *argv[]) {
+    /*
+    extern int exmain(int argc, char *argv[]);
+    exmain(argc, argv);
+    printf("ending program\n");
+    exit(0);
+     */
+
     struct Delivery ctx;
     struct Process proc = {
             .f_stdout = "",
@@ -214,6 +226,10 @@ int main(int argc, char *argv[]) {
     char installer_url[PATH_MAX];
     char python_override_version[STASIS_NAME_MAX];
     int user_disabled_docker = false;
+    globals.cpu_limit = get_cpu_count();
+    if (globals.cpu_limit > 1) {
+        globals.cpu_limit--;
+    }
 
     memset(env_name, 0, sizeof(env_name));
     memset(env_name_testing, 0, sizeof(env_name_testing));
@@ -241,9 +257,18 @@ int main(int argc, char *argv[]) {
             case 'p':
                 strcpy(python_override_version, optarg);
                 break;
+            case 'l':
+                globals.cpu_limit = strtol(optarg, NULL, 10);
+                if (globals.cpu_limit < 1) {
+                    globals.cpu_limit = 1;
+                }
+                break;
             case OPT_ALWAYS_UPDATE_BASE:
                 globals.always_update_base_environment = true;
                 break;
+            case OPT_PARALLEL_FAIL_FAST:
+                globals.parallel_fail_fast = true;
+                break;
             case 'U':
                 setenv("PYTHONUNBUFFERED", "1", 1);
                 fflush(stdout);
@@ -327,7 +352,6 @@ int main(int argc, char *argv[]) {
     tpl_register("deploy.jfrog.repo", &globals.jfrog.repo);
     tpl_register("deploy.jfrog.url", &globals.jfrog.url);
     tpl_register("deploy.docker.registry", &ctx.deploy.docker.registry);
-    tpl_register("workaround.tox_posargs", &globals.workaround.tox_posargs);
     tpl_register("workaround.conda_reactivate", &globals.workaround.conda_reactivate);
 
     // Expose function(s) to the template engine
@@ -336,6 +360,7 @@ int main(int argc, char *argv[]) {
     tpl_register_func("get_github_release_notes_auto", &get_github_release_notes_auto_tplfunc_entrypoint, 1, &ctx);
     tpl_register_func("junitxml_file", &get_junitxml_file_entrypoint, 1, &ctx);
     tpl_register_func("basetemp_dir", &get_basetemp_dir_entrypoint, 1, &ctx);
+    tpl_register_func("tox_run", &tox_run_entrypoint, 2, &ctx);
 
     // Set up PREFIX/etc directory information
     // The user may manipulate the base directory path with STASIS_SYSCONFDIR
diff --git a/src/template_func_proto.c b/src/template_func_proto.c
index 3cf66e4..ebb595e 100644
--- a/src/template_func_proto.c
+++ b/src/template_func_proto.c
@@ -109,4 +109,41 @@ int get_basetemp_dir_entrypoint(void *frame, void *data_out) {
     sprintf(*output, "%s/truth-%s-%s", ctx->storage.tmpdir, name, ctx->info.release_name);
 
     return result;
+}
+
+int tox_run_entrypoint(void *frame, void *data_out) {
+    char **output = (char **) data_out;
+    struct tplfunc_frame *f = (struct tplfunc_frame *) frame;
+    const struct Delivery *ctx = (const struct Delivery *) f->data_in;
+
+    // Apply workaround for tox positional arguments
+    char *toxconf = NULL;
+    if (!access("tox.ini", F_OK)) {
+        if (!fix_tox_conf("tox.ini", &toxconf)) {
+            msg(STASIS_MSG_L3, "Fixing tox positional arguments\n");
+            *output = calloc(STASIS_BUFSIZ, sizeof(**output));
+            if (!*output) {
+                return -1;
+            }
+            char *basetemp_path = NULL;
+            if (get_basetemp_dir_entrypoint(f, &basetemp_path)) {
+                return -2;
+            }
+            char *jxml_path = NULL;
+            if (get_junitxml_file_entrypoint(f, &jxml_path)) {
+                return -3;
+            }
+            const char *tox_target = f->argv[0].t_char_ptr;
+            const char *pytest_args = f->argv[1].t_char_ptr;
+            if (isempty(toxconf) || !strcmp(toxconf, "/")) {
+                SYSERROR("Unsafe toxconf path: '%s'", toxconf);
+                return -4;
+            }
+            snprintf(*output, STASIS_BUFSIZ - 1, "\npip install tox && (tox -e py%s%s -c %s --root . -- --basetemp=\"%s\" --junitxml=\"%s\" %s ; rm -f '%s')\n", ctx->meta.python_compact, tox_target, toxconf, basetemp_path, jxml_path, pytest_args ? pytest_args : "", toxconf);
+
+            guard_free(jxml_path);
+            guard_free(basetemp_path);
+        }
+    }
+    return 0;
 }
\ No newline at end of file
-- 
cgit 


From 87971633eea3306a0b85a55d7d581841b9dbf905 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Fri, 13 Sep 2024 10:10:21 -0400
Subject: Fixing headers

---
 src/multiprocessing.c | 2 --
 1 file changed, 2 deletions(-)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index 09190d3..6793f2d 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -1,6 +1,4 @@
-#include "core_mem.h"
 #include "multiprocessing.h"
-#include "utils.h"
 
 size_t mp_global_task_count = 0;
 static struct MultiProcessingTask *mp_pool_next_available(struct MultiProcessingPool *pool) {
-- 
cgit 


From 1fe385d782ae117d2a68266e14777d890eddf4e0 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Fri, 13 Sep 2024 12:16:41 -0400
Subject: Darwin portability: Use sem_open and sem_close instead of sem_init
 and sem_destroy

---
 src/multiprocessing.c | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index 6793f2d..0a91a56 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -306,7 +306,11 @@ struct MultiProcessingPool *mp_pool_init(const char *ident, const char *log_root
             exit(1);
         }
 
-        if (sem_init(slot->gate, 1, 0) < 0) {
+        char sema_name[PATH_MAX] = {0};
+        sprintf(sema_name, "/sem-%zu-%s-%s", mp_global_task_count, pool->ident, slot->ident);
+        sem_unlink(sema_name);
+        slot->gate = sem_open(sema_name, O_CREAT, 0600, sizeof(*slot->gate));
+        if (slot->gate == SEM_FAILED) {
             perror("sem_init failed");
             exit(1);
         }
@@ -317,11 +321,10 @@ struct MultiProcessingPool *mp_pool_init(const char *ident, const char *log_root
 }
 
 void mp_pool_free(struct MultiProcessingPool **pool) {
-    //for (size_t i = 0; i < (*pool)->num_used; i++) {
     for (size_t i = 0; i < (*pool)->num_alloc; i++) {
         if ((*pool)->task[i].gate) {
-            if (sem_destroy((*pool)->task[i].gate) < 0) {
-                perror("sem_destroy failed");
+            if (sem_close((*pool)->task[i].gate) < 0) {
+                perror("sem_close failed");
                 exit(1);
             }
             if (munmap((*pool)->task[i].gate, sizeof(*(*pool)->task[i].gate)) < 0) {
-- 
cgit 


From 4e0e40bf54f68a98b2cfbb419c8d1cbabf7986ba Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Fri, 13 Sep 2024 12:22:43 -0400
Subject: Darwin: Remove mmap MAP_POPULATE flag

---
 src/multiprocessing.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index 0a91a56..76bd241 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -276,7 +276,7 @@ struct MultiProcessingPool *mp_pool_init(const char *ident, const char *log_root
     }
 
     //pool = malloc(1 * sizeof(*pool));
-    pool = mmap(NULL, sizeof(*pool), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS | MAP_POPULATE, -1, 0);
+    pool = mmap(NULL, sizeof(*pool), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
     pool->ident = ident;
     pool->log_root = log_root;
     pool->num_used = 0;
@@ -291,7 +291,7 @@ struct MultiProcessingPool *mp_pool_init(const char *ident, const char *log_root
     }
 
     //pool->task = calloc(pool->num_alloc + 1, sizeof(*pool->task));
-    pool->task = mmap(NULL, (pool->num_alloc + 1) * sizeof(*pool->task), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS | MAP_POPULATE, -1, 0);
+    pool->task = mmap(NULL, (pool->num_alloc + 1) * sizeof(*pool->task), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
     if (pool->task == MAP_FAILED) {
         perror("mmap");
         mp_pool_free(&pool);
@@ -300,7 +300,7 @@ struct MultiProcessingPool *mp_pool_init(const char *ident, const char *log_root
 
     for (size_t i = 0; i < pool->num_alloc; i++) {
         struct MultiProcessingTask *slot = &pool->task[i];
-        slot->gate = mmap(NULL, sizeof(*slot->gate), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS | MAP_POPULATE, -1, 0);
+        slot->gate = mmap(NULL, sizeof(*slot->gate), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
         if (slot->gate == MAP_FAILED) {
             perror("mmap failed");
             exit(1);
-- 
cgit 


From 17d3d0517123f5f07b4ac6bb9f1dec73c1c8ce4c Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Fri, 13 Sep 2024 13:36:09 -0400
Subject: Fix sem_open initial state

* Move slot->gate assignment to mp_pool_task()
* Remove mmap() to slot->gate.
* Change type of ident and log_root variables for the sake of easy (fewer maps)
---
 src/multiprocessing.c | 41 ++++++++++++++---------------------------
 1 file changed, 14 insertions(+), 27 deletions(-)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index 76bd241..ad6dec5 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -35,6 +35,15 @@ struct MultiProcessingTask *mp_task(struct MultiProcessingPool *pool, const char
     fflush(tp);
     fclose(tp);
 
+    char sema_name[PATH_MAX] = {0};
+    sprintf(sema_name, "/sem-%zu-%s-%s", mp_global_task_count, pool->ident, slot->ident);
+    sem_unlink(sema_name);
+    slot->gate = sem_open(sema_name, O_CREAT, 0644, 0);
+    if (slot->gate == SEM_FAILED) {
+        perror("sem_open failed");
+        exit(1);
+    }
+
     pid_t pid = fork();
     int child_status = 0;
     if (pid == -1) {
@@ -275,10 +284,12 @@ struct MultiProcessingPool *mp_pool_init(const char *ident, const char *log_root
         return NULL;
     }
 
-    //pool = malloc(1 * sizeof(*pool));
     pool = mmap(NULL, sizeof(*pool), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
-    pool->ident = ident;
-    pool->log_root = log_root;
+    memset(pool->ident, 0, sizeof(pool->ident));
+    strncpy(pool->ident, ident, sizeof(pool->ident) - 1);
+
+    memset(pool->log_root, 0, sizeof(pool->log_root));
+    strncpy(pool->log_root, log_root, sizeof(pool->log_root) - 1);
     pool->num_used = 0;
     pool->num_alloc = MP_POOL_TASK_MAX;
 
@@ -290,7 +301,6 @@ struct MultiProcessingPool *mp_pool_init(const char *ident, const char *log_root
         }
     }
 
-    //pool->task = calloc(pool->num_alloc + 1, sizeof(*pool->task));
     pool->task = mmap(NULL, (pool->num_alloc + 1) * sizeof(*pool->task), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
     if (pool->task == MAP_FAILED) {
         perror("mmap");
@@ -298,25 +308,6 @@ struct MultiProcessingPool *mp_pool_init(const char *ident, const char *log_root
         return NULL;
     }
 
-    for (size_t i = 0; i < pool->num_alloc; i++) {
-        struct MultiProcessingTask *slot = &pool->task[i];
-        slot->gate = mmap(NULL, sizeof(*slot->gate), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
-        if (slot->gate == MAP_FAILED) {
-            perror("mmap failed");
-            exit(1);
-        }
-
-        char sema_name[PATH_MAX] = {0};
-        sprintf(sema_name, "/sem-%zu-%s-%s", mp_global_task_count, pool->ident, slot->ident);
-        sem_unlink(sema_name);
-        slot->gate = sem_open(sema_name, O_CREAT, 0600, sizeof(*slot->gate));
-        if (slot->gate == SEM_FAILED) {
-            perror("sem_init failed");
-            exit(1);
-        }
-    }
-
-
     return pool;
 }
 
@@ -327,10 +318,6 @@ void mp_pool_free(struct MultiProcessingPool **pool) {
                 perror("sem_close failed");
                 exit(1);
             }
-            if (munmap((*pool)->task[i].gate, sizeof(*(*pool)->task[i].gate)) < 0) {
-                perror("munmap");
-                exit(1);
-            }
         }
     }
     if ((*pool)->task) {
-- 
cgit 


From f2a5bc62bf1820842d20bb692fda9634e0743006 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Fri, 13 Sep 2024 14:50:26 -0400
Subject: Fix opt_flags assignment.

---
 src/delivery.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/delivery.c b/src/delivery.c
index 2718c08..1113119 100644
--- a/src/delivery.c
+++ b/src/delivery.c
@@ -1891,7 +1891,9 @@ void delivery_tests_run(struct Delivery *ctx) {
         }
 
         size_t opt_flags = 0;
-        opt_flags |= globals.parallel_fail_fast;
+        if (globals.parallel_fail_fast) {
+            opt_flags |= MP_POOL_FAIL_FAST;
+        }
 
         if (pool_setup->num_used) {
             COE_CHECK_ABORT(mp_pool_join(pool_setup, 1, opt_flags) != 0, "Failure in setup task pool");
-- 
cgit 


From 63cf3148cc3f60edf5c604a562ec95b40b8cb010 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Fri, 13 Sep 2024 15:01:29 -0400
Subject: Add multiprocessing.h to core.h

* Remove multiprocessing.h from other files
---
 src/delivery.c        | 1 -
 src/multiprocessing.c | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

(limited to 'src')

diff --git a/src/delivery.c b/src/delivery.c
index 1113119..7759c3e 100644
--- a/src/delivery.c
+++ b/src/delivery.c
@@ -2,7 +2,6 @@
 
 #include <fnmatch.h>
 #include "core.h"
-#include "multiprocessing.h"
 
 extern struct STASIS_GLOBAL globals;
 
diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index ad6dec5..bdd74ba 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -1,4 +1,4 @@
-#include "multiprocessing.h"
+#include "core.h"
 
 size_t mp_global_task_count = 0;
 static struct MultiProcessingTask *mp_pool_next_available(struct MultiProcessingPool *pool) {
-- 
cgit 


From 7c0b2a96a8c2cff5ddf57186f583125f5a02668b Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Fri, 13 Sep 2024 17:37:20 -0400
Subject: Guard against overrun

---
 src/multiprocessing.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index bdd74ba..70f86be 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -7,8 +7,12 @@ static struct MultiProcessingTask *mp_pool_next_available(struct MultiProcessing
 
 struct MultiProcessingTask *mp_task(struct MultiProcessingPool *pool, const char *ident, char *cmd) {
     struct MultiProcessingTask *slot = mp_pool_next_available(pool);
-    //struct MultiProcessingTask *slot = mp_pool_any_available(pool);
-    pool->num_used++;
+    if (pool->num_used != pool->num_alloc) {
+        pool->num_used++;
+    } else {
+        fprintf(stderr, "Maximum number of tasks reached\n");
+        return NULL;
+    }
 
     memset(slot->ident, 0, sizeof(slot->ident));
     strncpy(slot->ident, ident, sizeof(slot->ident) - 1);
-- 
cgit 


From ead0756a0df1de9d2011359393234a605b6f0313 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Sun, 15 Sep 2024 16:12:02 -0400
Subject: Split mp_task into to functions

---
 src/multiprocessing.c | 137 +++++++++++++++++++++++++++-----------------------
 1 file changed, 74 insertions(+), 63 deletions(-)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index 70f86be..7b3f997 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -5,73 +5,30 @@ static struct MultiProcessingTask *mp_pool_next_available(struct MultiProcessing
     return &pool->task[pool->num_used];
 }
 
-struct MultiProcessingTask *mp_task(struct MultiProcessingPool *pool, const char *ident, char *cmd) {
-    struct MultiProcessingTask *slot = mp_pool_next_available(pool);
-    if (pool->num_used != pool->num_alloc) {
-        pool->num_used++;
-    } else {
-        fprintf(stderr, "Maximum number of tasks reached\n");
-        return NULL;
-    }
-
-    memset(slot->ident, 0, sizeof(slot->ident));
-    strncpy(slot->ident, ident, sizeof(slot->ident) - 1);
-
-    strcat(slot->log_file, pool->log_root);
-    strcat(slot->log_file, "/");
-
-    FILE *tp = NULL;
-    char *t_name;
-    t_name = xmkstemp(&tp, "w");
-    if (!t_name || !tp) {
-        return NULL;
-    }
-
-    // Set the script's permissions so that only the calling user can use it
-    // This should help prevent eavesdropping if keys are applied in plain-text
-    // somewhere.
-    chmod(t_name, 0700);
-
-    memset(slot->parent_script, 0, sizeof(slot->parent_script));
-    strncpy(slot->parent_script, t_name, PATH_MAX - 1);
-
-    fprintf(tp, "#!/bin/bash\n%s\n", cmd);
-    fflush(tp);
-    fclose(tp);
-
-    char sema_name[PATH_MAX] = {0};
-    sprintf(sema_name, "/sem-%zu-%s-%s", mp_global_task_count, pool->ident, slot->ident);
-    sem_unlink(sema_name);
-    slot->gate = sem_open(sema_name, O_CREAT, 0644, 0);
-    if (slot->gate == SEM_FAILED) {
-        perror("sem_open failed");
-        exit(1);
-    }
-
+static int mp_task_fork(struct MultiProcessingPool *pool, struct MultiProcessingTask *task, const char *cmd) {
     pid_t pid = fork();
     int child_status = 0;
     if (pid == -1) {
-        return NULL;
+        return -1;
     } else if (pid == 0) { // child process
         char *cwd = NULL;
         FILE *fp_log = NULL;
 
         // Synchronize sub-process startup
         // Stop here until summoned by mp_pool_join()
-        if (sem_wait(slot->gate) < 0) {
+        if (sem_wait(task->gate) < 0) {
             perror("sem_wait failed");
             exit(1);
         }
 
-
         // Redirect stdout and stderr to the log file
         fflush(stdout);
         fflush(stderr);
-        printf("[%s:%s] Task started (pid: %d)\n", pool->ident, slot->ident, slot->parent_pid);
-        fp_log = freopen(slot->log_file, "w+", stdout);
+        printf("[%s:%s] Task started (pid: %d)\n", pool->ident, task->ident, task->parent_pid);
+        fp_log = freopen(task->log_file, "w+", stdout);
         if (!fp_log) {
-            fprintf(stderr, "unable to open '%s' for writing: %s\n", slot->log_file, strerror(errno));
-            return NULL;
+            fprintf(stderr, "unable to open '%s' for writing: %s\n", task->log_file, strerror(errno));
+            return -1;
         }
         dup2(fileno(stdout), fileno(stderr));
 
@@ -91,7 +48,7 @@ struct MultiProcessingTask *mp_task(struct MultiProcessingPool *pool, const char
 
         // Generate log header
         fprintf(fp_log, "# STARTED: %s\n", timebuf ? timebuf : "unknown");
-        fprintf(fp_log, "# PID: %d\n", slot->parent_pid);
+        fprintf(fp_log, "# PID: %d\n", task->parent_pid);
         fprintf(fp_log, "# WORKDIR: %s\n", cwd);
         fprintf(fp_log, "# COMMAND:\n%s\n", cmd);
         fprintf(fp_log, "# OUTPUT:\n");
@@ -102,28 +59,77 @@ struct MultiProcessingTask *mp_task(struct MultiProcessingPool *pool, const char
         // Execute task
         fflush(stdout);
         fflush(stderr);
-        char *args[] = {"bash", "--norc", t_name, (char *) NULL};
-        slot->status = execvp("/bin/bash", args);
+        char *args[] = {"bash", "--norc", task->parent_script, (char *) NULL};
+        task->status = execvp("/bin/bash", args);
     } else { // parent process
-        printf("[%s:%s] Task queued (pid: %d)\n", pool->ident, slot->ident, pid);
+        printf("[%s:%s] Task queued (pid: %d)\n", pool->ident, task->ident, pid);
 
         // Give the child process access to our PID value
-        slot->pid = pid;
-        slot->parent_pid = pid;
+        task->pid = pid;
+        task->parent_pid = pid;
 
         // Set log file name
-        sprintf(slot->log_file + strlen(slot->log_file), "task-%zu-%d.log", mp_global_task_count, slot->parent_pid);
+        sprintf(task->log_file + strlen(task->log_file), "task-%zu-%d.log", mp_global_task_count, task->parent_pid);
         mp_global_task_count++;
 
         // Check child's status
         pid_t code = waitpid(pid, &child_status, WUNTRACED | WCONTINUED | WNOHANG);
         if (code < 0) {
             perror("waitpid failed");
-            return NULL;
+            return -1;
         }
-        return slot;
     }
-    return NULL;
+    return 0;
+}
+
+struct MultiProcessingTask *mp_task(struct MultiProcessingPool *pool, const char *ident, char *cmd) {
+    struct MultiProcessingTask *slot = mp_pool_next_available(pool);
+    if (pool->num_used != pool->num_alloc) {
+        pool->num_used++;
+    } else {
+        fprintf(stderr, "Maximum number of tasks reached\n");
+        return NULL;
+    }
+
+    memset(slot->ident, 0, sizeof(slot->ident));
+    strncpy(slot->ident, ident, sizeof(slot->ident) - 1);
+
+    strcat(slot->log_file, pool->log_root);
+    strcat(slot->log_file, "/");
+
+    FILE *tp = NULL;
+    char *t_name;
+    t_name = xmkstemp(&tp, "w");
+    if (!t_name || !tp) {
+        return NULL;
+    }
+
+    // Set the script's permissions so that only the calling user can use it
+    // This should help prevent eavesdropping if keys are applied in plain-text
+    // somewhere.
+    chmod(t_name, 0700);
+
+    memset(slot->parent_script, 0, sizeof(slot->parent_script));
+    strncpy(slot->parent_script, t_name, PATH_MAX - 1);
+    guard_free(t_name);
+
+    fprintf(tp, "#!/bin/bash\n%s\n", cmd);
+    fflush(tp);
+    fclose(tp);
+
+    char sema_name[PATH_MAX] = {0};
+    sprintf(sema_name, "/sem-%zu-%s-%s", mp_global_task_count, pool->ident, slot->ident);
+    sem_unlink(sema_name);
+    slot->gate = sem_open(sema_name, O_CREAT, 0644, 0);
+    if (slot->gate == SEM_FAILED) {
+        perror("sem_open failed");
+        exit(1);
+    }
+
+    if (mp_task_fork(pool, slot, cmd)) {
+        return NULL;
+    }
+    return slot;
 }
 
 static int show_log_contents(FILE *stream, struct MultiProcessingTask *task) {
@@ -173,12 +179,10 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) {
     size_t tasks_complete = 0;
     size_t lower_i = 0;
     size_t upper_i = jobs;
-
     do {
-        //if (upper_i >= pool->num_alloc) {
+        size_t hang_check = 0;
         if (upper_i >= pool->num_used) {
             size_t x = upper_i - pool->num_used;
-            //size_t x = upper_i - pool->num_alloc;
             upper_i -= (size_t) x;
         }
         for (size_t i = lower_i; i < upper_i; i++) {
@@ -191,6 +195,12 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) {
 
             if (slot->pid == MP_POOL_PID_UNUSED) {
                 // PID is already used up, skip it
+                hang_check++;
+                if (hang_check >= pool->num_used) {
+                    fprintf(stderr, "%s is deadlocked\n", pool->ident);
+                    failures++;
+                    goto pool_deadlocked;
+                }
                 continue;
             }
 
@@ -275,6 +285,7 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) {
         sleep(1);
     } while (1);
 
+    pool_deadlocked:
     puts("");
     return failures;
 }
-- 
cgit 


From 6921cb44f5464b59a2a169167cc63c66bcc95a9f Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Tue, 17 Sep 2024 09:30:16 -0400
Subject: Break parent/child calls into static functions

---
 src/multiprocessing.c | 157 +++++++++++++++++++++++++++-----------------------
 1 file changed, 86 insertions(+), 71 deletions(-)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index 7b3f997..a1d961e 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -5,83 +5,99 @@ static struct MultiProcessingTask *mp_pool_next_available(struct MultiProcessing
     return &pool->task[pool->num_used];
 }
 
-static int mp_task_fork(struct MultiProcessingPool *pool, struct MultiProcessingTask *task, const char *cmd) {
-    pid_t pid = fork();
-    int child_status = 0;
-    if (pid == -1) {
+int child(struct MultiProcessingPool *pool, struct MultiProcessingTask *task, const char *cmd) {
+    char *cwd = NULL;
+    FILE *fp_log = NULL;
+
+    // Synchronize sub-process startup
+    // Stop here until summoned by mp_pool_join()
+    if (sem_wait(task->gate) < 0) {
+        perror("sem_wait failed");
+        exit(1);
+    }
+
+    // Record the task start time
+    if (clock_gettime(CLOCK_REALTIME, &task->time_data.t_start) < 0) {
+        perror("clock_gettime");
+        exit(1);
+    }
+
+    // Redirect stdout and stderr to the log file
+    fflush(stdout);
+    fflush(stderr);
+    printf("[%s:%s] Task started (pid: %d)\n", pool->ident, task->ident, task->parent_pid);
+    fp_log = freopen(task->log_file, "w+", stdout);
+    if (!fp_log) {
+        fprintf(stderr, "unable to open '%s' for writing: %s\n", task->log_file, strerror(errno));
         return -1;
-    } else if (pid == 0) { // child process
-        char *cwd = NULL;
-        FILE *fp_log = NULL;
-
-        // Synchronize sub-process startup
-        // Stop here until summoned by mp_pool_join()
-        if (sem_wait(task->gate) < 0) {
-            perror("sem_wait failed");
-            exit(1);
-        }
+    }
+    dup2(fileno(stdout), fileno(stderr));
+
+    // Generate timestamp for log header
+    time_t t = time(NULL);
+    char *timebuf = ctime(&t);
+    if (timebuf) {
+        // strip line feed from timestamp
+        timebuf[strlen(timebuf) ? strlen(timebuf) - 1 : 0] = 0;
+    }
 
-        // Redirect stdout and stderr to the log file
-        fflush(stdout);
-        fflush(stderr);
-        printf("[%s:%s] Task started (pid: %d)\n", pool->ident, task->ident, task->parent_pid);
-        fp_log = freopen(task->log_file, "w+", stdout);
-        if (!fp_log) {
-            fprintf(stderr, "unable to open '%s' for writing: %s\n", task->log_file, strerror(errno));
-            return -1;
-        }
-        dup2(fileno(stdout), fileno(stderr));
-
-        // Generate timestamp for log header
-        time_t t = time(NULL);
-        char *timebuf = ctime(&t);
-        if (timebuf) {
-            // strip line feed from timestamp
-            timebuf[strlen(timebuf) ? strlen(timebuf) - 1 : 0] = 0;
-        }
+    // Get the path to the current working directory
+    // Used by the log header. Informative only.
+    cwd = getcwd(NULL, PATH_MAX);
+    if (!cwd) {
+        perror(cwd);
+        exit(1);
+    }
 
-        cwd = getcwd(NULL, PATH_MAX);
-        if (!cwd) {
-            perror(cwd);
-            exit(1);
-        }
+    // Generate log header
+    fprintf(fp_log, "# STARTED: %s\n", timebuf ? timebuf : "unknown");
+    fprintf(fp_log, "# PID: %d\n", task->parent_pid);
+    fprintf(fp_log, "# WORKDIR: %s\n", cwd);
+    fprintf(fp_log, "# COMMAND:\n%s\n", cmd);
+    fprintf(fp_log, "# OUTPUT:\n");
+    // Commit header to log file / clean up
+    fflush(fp_log);
+    guard_free(cwd);
+
+    // Execute task
+    fflush(stdout);
+    fflush(stderr);
+    char *args[] = {"bash", "--norc", task->parent_script, (char *) NULL};
+    task->status = execvp("/bin/bash", args);
+    return 0; // NOP return to satisfy the compiler
+}
 
-        // Generate log header
-        fprintf(fp_log, "# STARTED: %s\n", timebuf ? timebuf : "unknown");
-        fprintf(fp_log, "# PID: %d\n", task->parent_pid);
-        fprintf(fp_log, "# WORKDIR: %s\n", cwd);
-        fprintf(fp_log, "# COMMAND:\n%s\n", cmd);
-        fprintf(fp_log, "# OUTPUT:\n");
-        // Commit header to log file / clean up
-        fflush(fp_log);
-        guard_free(cwd);
-
-        // Execute task
-        fflush(stdout);
-        fflush(stderr);
-        char *args[] = {"bash", "--norc", task->parent_script, (char *) NULL};
-        task->status = execvp("/bin/bash", args);
-    } else { // parent process
-        printf("[%s:%s] Task queued (pid: %d)\n", pool->ident, task->ident, pid);
-
-        // Give the child process access to our PID value
-        task->pid = pid;
-        task->parent_pid = pid;
-
-        // Set log file name
-        sprintf(task->log_file + strlen(task->log_file), "task-%zu-%d.log", mp_global_task_count, task->parent_pid);
-        mp_global_task_count++;
-
-        // Check child's status
-        pid_t code = waitpid(pid, &child_status, WUNTRACED | WCONTINUED | WNOHANG);
-        if (code < 0) {
-            perror("waitpid failed");
-            return -1;
-        }
+int parent(struct MultiProcessingPool *pool, struct MultiProcessingTask *task, pid_t pid, int *child_status) {
+    printf("[%s:%s] Task queued (pid: %d)\n", pool->ident, task->ident, pid);
+
+    // Give the child process access to our PID value
+    task->pid = pid;
+    task->parent_pid = pid;
+
+    // Set log file name
+    sprintf(task->log_file + strlen(task->log_file), "task-%zu-%d.log", mp_global_task_count, task->parent_pid);
+    mp_global_task_count++;
+
+    // Check child's status
+    pid_t code = waitpid(pid, child_status, WUNTRACED | WCONTINUED | WNOHANG);
+    if (code < 0) {
+        perror("waitpid failed");
+        return -1;
     }
     return 0;
 }
 
+static int mp_task_fork(struct MultiProcessingPool *pool, struct MultiProcessingTask *task, const char *cmd) {
+    pid_t pid = fork();
+    int child_status = 0;
+    if (pid == -1) {
+        return -1;
+    } else if (pid == 0) {
+        child(pool, task, cmd);
+    }
+    return parent(pool, task, pid, &child_status);
+}
+
 struct MultiProcessingTask *mp_task(struct MultiProcessingPool *pool, const char *ident, char *cmd) {
     struct MultiProcessingTask *slot = mp_pool_next_available(pool);
     if (pool->num_used != pool->num_alloc) {
@@ -212,8 +228,7 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) {
 
             char progress[1024] = {0};
             if (pid > 0) {
-                double percent = ((double) (tasks_complete) / (double) pool->num_used) * 100;
-                //double percent = ((double) (tasks_complete) / (double) pool->num_alloc) * 100;
+                double percent = ((double) (tasks_complete + 1) / (double) pool->num_used) * 100;
                 sprintf(progress, "[%s:%s] [%3.1f%%]", pool->ident, slot->ident, percent);
 
                 // The process ended in one the following ways
-- 
cgit 


From daf8c81c35f590f2ac145801933792a070ccbee2 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Tue, 17 Sep 2024 09:32:12 -0400
Subject: Add comments, remove dead code

---
 src/multiprocessing.c | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index a1d961e..25b667c 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -1,6 +1,8 @@
 #include "core.h"
 
+/// The sum of all tasks queued by mp_task()
 size_t mp_global_task_count = 0;
+
 static struct MultiProcessingTask *mp_pool_next_available(struct MultiProcessingPool *pool) {
     return &pool->task[pool->num_used];
 }
@@ -107,12 +109,15 @@ struct MultiProcessingTask *mp_task(struct MultiProcessingPool *pool, const char
         return NULL;
     }
 
+    // Set task identifier string
     memset(slot->ident, 0, sizeof(slot->ident));
     strncpy(slot->ident, ident, sizeof(slot->ident) - 1);
 
+    // Set log file path
     strcat(slot->log_file, pool->log_root);
     strcat(slot->log_file, "/");
 
+    // Create a temporary file to act as our intermediate command script
     FILE *tp = NULL;
     char *t_name;
     t_name = xmkstemp(&tp, "w");
@@ -125,14 +130,18 @@ struct MultiProcessingTask *mp_task(struct MultiProcessingPool *pool, const char
     // somewhere.
     chmod(t_name, 0700);
 
+    // Record the script path
     memset(slot->parent_script, 0, sizeof(slot->parent_script));
     strncpy(slot->parent_script, t_name, PATH_MAX - 1);
     guard_free(t_name);
 
+    // Populate the script
     fprintf(tp, "#!/bin/bash\n%s\n", cmd);
     fflush(tp);
     fclose(tp);
 
+    // Create a uniquely named semaphore.
+    // This is used by the child process to prevent task execution until mp_pool_join is called
     char sema_name[PATH_MAX] = {0};
     sprintf(sema_name, "/sem-%zu-%s-%s", mp_global_task_count, pool->ident, slot->ident);
     sem_unlink(sema_name);
@@ -142,6 +151,7 @@ struct MultiProcessingTask *mp_task(struct MultiProcessingPool *pool, const char
         exit(1);
     }
 
+    // Execute task
     if (mp_task_fork(pool, slot, cmd)) {
         return NULL;
     }
@@ -164,12 +174,12 @@ static int show_log_contents(FILE *stream, struct MultiProcessingTask *task) {
 
 int mp_pool_kill(struct MultiProcessingPool *pool, int signum) {
     printf("Sending signal %d to pool '%s'\n", signum, pool->ident);
-    //for (size_t i = 0; i < pool->num_alloc; i++) {
     for (size_t i = 0; i < pool->num_used; i++) {
         struct MultiProcessingTask *slot = &pool->task[i];
         if (!slot) {
             return -1;
         }
+        // Kill tasks in progress
         if (slot->pid > 0) {
             int status;
             printf("Sending signal %d to task '%s' (pid: %d)\n", signum, slot->ident, slot->pid);
@@ -209,10 +219,13 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) {
                 exit(1);
             }
 
+            // Has the child been processed already?
             if (slot->pid == MP_POOL_PID_UNUSED) {
-                // PID is already used up, skip it
+                // Child is already used up, skip it
                 hang_check++;
                 if (hang_check >= pool->num_used) {
+                    // Unlikely to happen when called correctly, but if you join a pool that's already finished
+                    // it will spin forever. This protects the program from entering an infinite loop.
                     fprintf(stderr, "%s is deadlocked\n", pool->ident);
                     failures++;
                     goto pool_deadlocked;
@@ -314,15 +327,20 @@ struct MultiProcessingPool *mp_pool_init(const char *ident, const char *log_root
         return NULL;
     }
 
+    // The pool is shared with children
     pool = mmap(NULL, sizeof(*pool), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
+
+    // Set pool identity string
     memset(pool->ident, 0, sizeof(pool->ident));
     strncpy(pool->ident, ident, sizeof(pool->ident) - 1);
 
+    // Set logging base directory
     memset(pool->log_root, 0, sizeof(pool->log_root));
     strncpy(pool->log_root, log_root, sizeof(pool->log_root) - 1);
     pool->num_used = 0;
     pool->num_alloc = MP_POOL_TASK_MAX;
 
+    // Create the log directory
     if (mkdirs(log_root, 0700) < 0) {
         if (errno != EEXIST) {
             perror(log_root);
@@ -331,6 +349,7 @@ struct MultiProcessingPool *mp_pool_init(const char *ident, const char *log_root
         }
     }
 
+    // Task array is shared with children
     pool->task = mmap(NULL, (pool->num_alloc + 1) * sizeof(*pool->task), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
     if (pool->task == MAP_FAILED) {
         perror("mmap");
@@ -343,6 +362,7 @@ struct MultiProcessingPool *mp_pool_init(const char *ident, const char *log_root
 
 void mp_pool_free(struct MultiProcessingPool **pool) {
     for (size_t i = 0; i < (*pool)->num_alloc; i++) {
+        // Close all semaphores
         if ((*pool)->task[i].gate) {
             if (sem_close((*pool)->task[i].gate) < 0) {
                 perror("sem_close failed");
@@ -350,11 +370,13 @@ void mp_pool_free(struct MultiProcessingPool **pool) {
             }
         }
     }
+    // Unmap all pool tasks
     if ((*pool)->task) {
         if (munmap((*pool)->task, sizeof(*(*pool)->task) * (*pool)->num_alloc) < 0) {
             perror("munmap");
         }
     }
+    // Unmap the pool
     if ((*pool)) {
         if (munmap((*pool), sizeof(*(*pool))) < 0) {
             perror("munmap");
-- 
cgit 


From 3c3468d0c4bba45ed8626894613cc44e5fcd51c0 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Tue, 17 Sep 2024 09:32:17 -0400
Subject: Set task status to -1 by default

---
 src/multiprocessing.c | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index 25b667c..0c993d7 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -109,6 +109,9 @@ struct MultiProcessingTask *mp_task(struct MultiProcessingPool *pool, const char
         return NULL;
     }
 
+    // Set default status to "error"
+    slot->status = -1;
+
     // Set task identifier string
     memset(slot->ident, 0, sizeof(slot->ident));
     strncpy(slot->ident, ident, sizeof(slot->ident) - 1);
-- 
cgit 


From 0f95b43e2d3853995106c5c6aa55bf92f63fb331 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Tue, 17 Sep 2024 09:35:07 -0400
Subject: Wait for signaled processes to hang up

* Only initiate a kill if we have more than one process. The current process is already failed out, no need to terminate it again.
---
 src/multiprocessing.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index 0c993d7..fe15ab6 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -189,6 +189,10 @@ int mp_pool_kill(struct MultiProcessingPool *pool, int signum) {
             status = kill(slot->pid, signum);
             if (status && errno != ESRCH) {
                 fprintf(stderr, "Task '%s' (pid: %d) did not respond: %s\n", slot->ident, slot->pid, strerror(errno));
+                // Wait for process to handle the signal, then set the status accordingly
+                if (waitpid(slot->pid, &status, 0) >= 0) {
+                    slot->signaled_by = WTERMSIG(status);
+                }
             }
         }
         if (!access(slot->log_file, F_OK)) {
@@ -271,7 +275,7 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) {
                 }
                 if (status >> 8 != 0 || (status & 0xff) != 0) {
                     fprintf(stderr, "%s Task failed\n", progress);
-                    if (flags & MP_POOL_FAIL_FAST) {
+                    if (flags & MP_POOL_FAIL_FAST && pool->num_used > 1) {
                         mp_pool_kill(pool, SIGTERM);
                         return -2;
                     }
-- 
cgit 


From db1a3056296ea3ed13c5a425cf1f11602b43a6c7 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Tue, 17 Sep 2024 09:37:14 -0400
Subject: Add pool summary and elapsed time output

* Add get_task_duration()
* Add get_pool_show_summary()
* Add signaled_by member to MultiProcessingTask
* Add time_data member to MultiProcessingTask for duration tracking
---
 src/multiprocessing.c | 48 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index fe15ab6..5e605a6 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -161,6 +161,41 @@ struct MultiProcessingTask *mp_task(struct MultiProcessingPool *pool, const char
     return slot;
 }
 
+static void get_task_duration(struct MultiProcessingTask *task, struct timespec *result) {
+    struct timespec *start = &task->time_data.t_start;
+    struct timespec *stop = &task->time_data.t_stop;
+    result->tv_sec = (stop->tv_sec - start->tv_sec);
+    result->tv_nsec = (stop->tv_nsec - start->tv_nsec);
+    if (result->tv_nsec < 0) {
+        --result->tv_sec;
+        result->tv_nsec += 1000000000L;
+    }
+}
+
+void mp_pool_show_summary(struct MultiProcessingPool *pool) {
+    print_banner("=", 79);
+    printf("Pool execution summary for \"%s\"\n", pool->ident);
+    print_banner("=", 79);
+    printf("STATUS     PID     DURATION     IDENT\n");
+    for (size_t i = 0; i < pool->num_used; i++) {
+        struct MultiProcessingTask *task = &pool->task[i];
+        char status_str[10] = {0};
+        if (!task->status && !task->signaled_by) {
+            strcpy(status_str, "DONE");
+        } else if (task->signaled_by) {
+            strcpy(status_str, "TERM");
+        } else {
+            strcpy(status_str, "FAIL");
+        }
+
+        struct timespec duration;
+        get_task_duration(task, &duration);
+        long diff = duration.tv_sec + duration.tv_nsec / 1000000000L;
+        printf("%-4s    %-10d %7lds     %-10s\n", status_str, task->parent_pid, diff, task->ident) ;
+    }
+    puts("");
+}
+
 static int show_log_contents(FILE *stream, struct MultiProcessingTask *task) {
     FILE *fp = fopen(task->log_file, "r");
     if (!fp) {
@@ -189,9 +224,15 @@ int mp_pool_kill(struct MultiProcessingPool *pool, int signum) {
             status = kill(slot->pid, signum);
             if (status && errno != ESRCH) {
                 fprintf(stderr, "Task '%s' (pid: %d) did not respond: %s\n", slot->ident, slot->pid, strerror(errno));
+            } else {
                 // Wait for process to handle the signal, then set the status accordingly
                 if (waitpid(slot->pid, &status, 0) >= 0) {
                     slot->signaled_by = WTERMSIG(status);
+                    // Record the task stop time
+                    if (clock_gettime(CLOCK_REALTIME, &slot->time_data.t_stop) < 0) {
+                        perror("clock_gettime");
+                        exit(1);
+                    }
                 }
             }
         }
@@ -273,6 +314,13 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) {
                 if (show_log_contents(stdout, slot)) {
                     perror(slot->log_file);
                 }
+
+                // Record the task stop time
+                if (clock_gettime(CLOCK_REALTIME, &slot->time_data.t_stop) < 0) {
+                    perror("clock_gettime");
+                    exit(1);
+                }
+
                 if (status >> 8 != 0 || (status & 0xff) != 0) {
                     fprintf(stderr, "%s Task failed\n", progress);
                     if (flags & MP_POOL_FAIL_FAST && pool->num_used > 1) {
-- 
cgit 


From 16597bc90b810d3457296d99bd36510321a37f24 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Tue, 17 Sep 2024 09:38:18 -0400
Subject: Implement mp_pool_show_summary

---
 src/delivery.c | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

(limited to 'src')

diff --git a/src/delivery.c b/src/delivery.c
index 7759c3e..c31a4dd 100644
--- a/src/delivery.c
+++ b/src/delivery.c
@@ -1894,18 +1894,25 @@ void delivery_tests_run(struct Delivery *ctx) {
             opt_flags |= MP_POOL_FAIL_FAST;
         }
 
+        int pool_status = -1;
         if (pool_setup->num_used) {
-            COE_CHECK_ABORT(mp_pool_join(pool_setup, 1, opt_flags) != 0, "Failure in setup task pool");
+            pool_status = mp_pool_join(pool_setup, 1, opt_flags);
+            mp_pool_show_summary(pool_setup);
+            COE_CHECK_ABORT(pool_status != 0, "Failure in setup task pool");
             mp_pool_free(&pool_setup);
         }
 
         if (pool_parallel->num_used) {
-            COE_CHECK_ABORT(mp_pool_join(pool_parallel, globals.cpu_limit, opt_flags) != 0, "Failure in parallel task pool");
+            pool_status = mp_pool_join(pool_parallel, globals.cpu_limit, opt_flags);
+            mp_pool_show_summary(pool_parallel);
+            COE_CHECK_ABORT(pool_status != 0, "Failure in parallel task pool");
             mp_pool_free(&pool_parallel);
         }
 
         if (pool_serial->num_used) {
-            COE_CHECK_ABORT(mp_pool_join(pool_serial, 1, opt_flags) != 0, "Failure in serial task pool");
+            pool_status = mp_pool_join(pool_serial, 1, opt_flags);
+            mp_pool_show_summary(pool_serial);
+            COE_CHECK_ABORT(pool_serial != 0, "Failure in serial task pool");
             mp_pool_free(&pool_serial);
         }
     }
-- 
cgit 


From 60c0a3c8607b835f73e77d89b5f67a8e6bd5c8b3 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Tue, 17 Sep 2024 10:12:02 -0400
Subject: Fix format spacing

---
 src/multiprocessing.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index 5e605a6..5eb9f81 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -191,7 +191,7 @@ void mp_pool_show_summary(struct MultiProcessingPool *pool) {
         struct timespec duration;
         get_task_duration(task, &duration);
         long diff = duration.tv_sec + duration.tv_nsec / 1000000000L;
-        printf("%-4s    %-10d %7lds     %-10s\n", status_str, task->parent_pid, diff, task->ident) ;
+        printf("%-4s   %10d  %7lds     %-10s\n", status_str, task->parent_pid, diff, task->ident) ;
     }
     puts("");
 }
-- 
cgit 


From 3ef3a2758c572dcdeed7d693f52c5049394f11f5 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Tue, 17 Sep 2024 10:29:13 -0400
Subject: Fix test status expectation

* Fix child not returning result of execvp(). task->status is for program status, not fork() status.
---
 src/multiprocessing.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index 5eb9f81..8709911 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -65,8 +65,7 @@ int child(struct MultiProcessingPool *pool, struct MultiProcessingTask *task, co
     fflush(stdout);
     fflush(stderr);
     char *args[] = {"bash", "--norc", task->parent_script, (char *) NULL};
-    task->status = execvp("/bin/bash", args);
-    return 0; // NOP return to satisfy the compiler
+    return execvp("/bin/bash", args);
 }
 
 int parent(struct MultiProcessingPool *pool, struct MultiProcessingTask *task, pid_t pid, int *child_status) {
-- 
cgit 


From 6f7cf6e1094e54e593801cc452b65ae9c7b4824a Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Tue, 17 Sep 2024 10:56:41 -0400
Subject: Remove short circuit test code

* Remove exmain() and dead comments from main()
---
 src/multiprocessing.c | 29 +----------------------------
 src/stasis_main.c     |  7 -------
 2 files changed, 1 insertion(+), 35 deletions(-)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index 8709911..8e11a93 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -437,31 +437,4 @@ void mp_pool_free(struct MultiProcessingPool **pool) {
         }
         (*pool) = NULL;
     }
-}
-
-int exmain(int argc, char *argv[]) {
-    size_t i;
-    struct MultiProcessingPool *pool = mp_pool_init("generic", "logs");
-    if (!pool) {
-        perror("pool init");
-        exit(1);
-    }
-
-    char *commands[] = {
-        "sleep 2; dd if=/dev/zero of=file.dat bs=1M count=1",
-        "/bin/echo hello world; sleep 5",
-        "python -c 'print(1+1)'",
-        "(for x in {1..10}; do echo $x; sleep 0.5; done)",
-        "echo stdout >&1; echo stderr >&2; exit 1"
-    };
-
-    for (i = 0; i < sizeof(commands) / sizeof(*commands); i++) {
-        if (mp_task(pool, "commands array", commands[i]) == NULL) {
-            printf("Too many tasks queued (max: %d)\n", MP_POOL_TASK_MAX);
-            break;
-        }
-    }
-    mp_pool_join(pool, get_cpu_count() - 1, MP_POOL_FAIL_FAST);
-    mp_pool_free(&pool);
-    return 0;
-}
+}
\ No newline at end of file
diff --git a/src/stasis_main.c b/src/stasis_main.c
index 164a9ca..72b7cdd 100644
--- a/src/stasis_main.c
+++ b/src/stasis_main.c
@@ -206,13 +206,6 @@ static void check_requirements(struct Delivery *ctx) {
 }
 
 int main(int argc, char *argv[]) {
-    /*
-    extern int exmain(int argc, char *argv[]);
-    exmain(argc, argv);
-    printf("ending program\n");
-    exit(0);
-     */
-
     struct Delivery ctx;
     struct Process proc = {
             .f_stdout = "",
-- 
cgit 


From 4efce32c6efcb98e3d4eda4886a75872aff6fff9 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Tue, 17 Sep 2024 22:33:46 -0400
Subject: Bugfix: log_show_contents() did not close FILE pointer

---
 src/multiprocessing.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index 8e11a93..fce1a80 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -206,6 +206,7 @@ static int show_log_contents(FILE *stream, struct MultiProcessingTask *task) {
         memset(buf, 0, sizeof(buf));
     }
     fprintf(stream, "\n");
+    fclose(fp);
     return 0;
 }
 
-- 
cgit 


From 8b47235f7c81e04fa5efef492974509789f40273 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Wed, 18 Sep 2024 10:04:21 -0400
Subject: Rename mp_task to mp_pool_task

---
 src/delivery.c        | 4 ++--
 src/multiprocessing.c | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

(limited to 'src')

diff --git a/src/delivery.c b/src/delivery.c
index c31a4dd..aba266b 100644
--- a/src/delivery.c
+++ b/src/delivery.c
@@ -1819,7 +1819,7 @@ void delivery_tests_run(struct Delivery *ctx) {
                 }
 
                 sprintf(runner_cmd, runner_cmd_fmt, cmd);
-                task = mp_task(pool, test->name, runner_cmd);
+                task = mp_pool_task(pool, test->name, runner_cmd);
                 if (!task) {
                     SYSERROR("Failed to add task to %s pool: %s", pool_name, runner_cmd);
                     popd();
@@ -1872,7 +1872,7 @@ void delivery_tests_run(struct Delivery *ctx) {
                     char runner_cmd[0xFFFF] = {0};
                     sprintf(runner_cmd, runner_cmd_fmt, cmd);
 
-                    task = mp_task(pool, test->name, runner_cmd);
+                    task = mp_pool_task(pool, test->name, runner_cmd);
                     if (!task) {
                         SYSERROR("Failed to add task %s to setup pool: %s", test->name, runner_cmd);
                         popd();
diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index fce1a80..c784073 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -99,7 +99,7 @@ static int mp_task_fork(struct MultiProcessingPool *pool, struct MultiProcessing
     return parent(pool, task, pid, &child_status);
 }
 
-struct MultiProcessingTask *mp_task(struct MultiProcessingPool *pool, const char *ident, char *cmd) {
+struct MultiProcessingTask *mp_pool_task(struct MultiProcessingPool *pool, const char *ident, char *cmd) {
     struct MultiProcessingTask *slot = mp_pool_next_available(pool);
     if (pool->num_used != pool->num_alloc) {
         pool->num_used++;
-- 
cgit 


From 528b3b271ce13a5094d00f98e9b204ee0e24e795 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Wed, 18 Sep 2024 10:12:06 -0400
Subject: mp_pool_kill marks PIDs as unused

---
 src/multiprocessing.c | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index c784073..5478ca5 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -233,6 +233,8 @@ int mp_pool_kill(struct MultiProcessingPool *pool, int signum) {
                         perror("clock_gettime");
                         exit(1);
                     }
+                    // We are short-circuiting the normal flow, and the process is now dead, so mark it as such
+                    slot->pid = MP_POOL_PID_UNUSED;
                 }
             }
         }
-- 
cgit 


From 94e473cf5670f5b0373c3ec0692707e2ef9618ad Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Wed, 18 Sep 2024 10:15:49 -0400
Subject: Move process status checks outside of the if-statement

* Reverse the order of checks so signals actually get reported when they occur
* Changed "Task finished" to "Task ended". "Finished" sounds more like "success" and might get confusing when an error occurs
---
 src/multiprocessing.c | 28 ++++++++++++++++++----------
 1 file changed, 18 insertions(+), 10 deletions(-)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index 5478ca5..8c7c2e8 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -285,9 +285,17 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) {
 
             // Is the process finished?
             pid_t pid = waitpid(slot->pid, &status, WNOHANG | WUNTRACED | WCONTINUED);
+            int task_ended = WIFEXITED(status);
+            int task_ended_by_signal = WIFSIGNALED(status);
+            int task_stopped = WIFSTOPPED(status);
+            int task_continued = WIFCONTINUED(status);
+            int status_exit = WEXITSTATUS(status);
+            int status_signal = WTERMSIG(status);
+            int status_stopped = WSTOPSIG(status);
 
             // Update status
-            slot->status = status;
+            slot->status = status_exit;
+            slot->signaled_by = status_signal;
 
             char progress[1024] = {0};
             if (pid > 0) {
@@ -296,18 +304,18 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) {
 
                 // The process ended in one the following ways
                 // Note: SIGSTOP nor SIGCONT will not increment the tasks_complete counter
-                if (WIFEXITED(status)) {
-                    printf("%s Task finished (status: %d)\n", progress, WEXITSTATUS(status));
-                    tasks_complete++;
-                } else if (WIFSIGNALED(status)) {
-                    printf("%s Task ended by signal %d (%s)\n", progress, WTERMSIG(status), strsignal(WTERMSIG(status)));
-                    tasks_complete++;
-                } else if (WIFSTOPPED(status)) {
-                    printf("%s Task was suspended (%d)\n", progress, WSTOPSIG(status));
+                if (task_stopped) {
+                    printf("%s Task was suspended (%d)\n", progress, status_stopped);
                     continue;
-                } else if (WIFCONTINUED(status)) {
+                } else if (task_continued) {
                     printf("%s Task was resumed\n", progress);
                     continue;
+                } else if (task_ended_by_signal) {
+                    printf("%s Task ended by signal %d (%s)\n", progress, status_signal, strsignal(status_signal));
+                    tasks_complete++;
+                } else if (task_ended) {
+                    printf("%s Task ended (status: %d)\n", progress, status_exit);
+                    tasks_complete++;
                 } else {
                     fprintf(stderr, "%s Task state is unknown (0x%04X)\n", progress, status);
                 }
-- 
cgit 


From 37f68b28cf1624d9188872bea1e8efc15141824c Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Wed, 18 Sep 2024 10:16:17 -0400
Subject: Comment get_task_duration

---
 src/multiprocessing.c | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index 8c7c2e8..4ae9b31 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -161,6 +161,8 @@ struct MultiProcessingTask *mp_pool_task(struct MultiProcessingPool *pool, const
 }
 
 static void get_task_duration(struct MultiProcessingTask *task, struct timespec *result) {
+    // based on the timersub() macro in time.h
+    // This implementation uses timespec and increases the resolution from microseconds to nanoseconds.
     struct timespec *start = &task->time_data.t_start;
     struct timespec *stop = &task->time_data.t_stop;
     result->tv_sec = (stop->tv_sec - start->tv_sec);
-- 
cgit 


From 4a4cdeb3f0255b0c9979397635c3a0c930fd153f Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Wed, 18 Sep 2024 10:18:06 -0400
Subject: Fix incorrect failure count

* No longer adds the raw status value to the failure count. Just increment it instead.
---
 src/multiprocessing.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index 4ae9b31..7452314 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -335,6 +335,8 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) {
 
                 if (status >> 8 != 0 || (status & 0xff) != 0) {
                     fprintf(stderr, "%s Task failed\n", progress);
+                    failures++;
+
                     if (flags & MP_POOL_FAIL_FAST && pool->num_used > 1) {
                         mp_pool_kill(pool, SIGTERM);
                         return -2;
@@ -353,7 +355,6 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) {
 
                 // Update progress and tell the poller to ignore the PID. The process is gone.
                 slot->pid = MP_POOL_PID_UNUSED;
-                failures += status;
             } else if (pid < 0) {
                 fprintf(stderr, "waitpid failed: %s\n", strerror(errno));
                 return -1;
-- 
cgit 


From fa7dc2defb4acd3a808bf8bf787518d2d2d522db Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Fri, 20 Sep 2024 08:29:56 -0400
Subject: Export all variables when sourcing conda's profile scripts

* This results in a fully activated environment; shell functions and all
---
 src/conda.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/conda.c b/src/conda.c
index ff55f14..37a7793 100644
--- a/src/conda.c
+++ b/src/conda.c
@@ -222,7 +222,7 @@ int conda_activate(const char *root, const char *env_name) {
 
     // Fully activate conda and record its effect on the runtime environment
     char command[PATH_MAX * 3];
-    snprintf(command, sizeof(command) - 1, "source %s; source %s; conda activate %s &>/dev/null; env -0", path_conda, path_mamba, env_name);
+    snprintf(command, sizeof(command) - 1, "set -a; source %s; source %s; conda activate %s &>/dev/null; env -0", path_conda, path_mamba, env_name);
     int retval = shell(&proc, command);
     if (retval) {
         // it didn't work; drop out for cleanup
-- 
cgit 


From 1999612a69a7947fb1b6fc45705299fe6db650ba Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Fri, 20 Sep 2024 08:36:26 -0400
Subject: Refactor structure

* Break delivery.c into smaller components
---
 src/CMakeLists.txt         |   10 +
 src/delivery.c             | 2089 +-------------------------------------------
 src/delivery_artifactory.c |  192 ++++
 src/delivery_build.c       |  180 ++++
 src/delivery_conda.c       |  110 +++
 src/delivery_docker.c      |  132 +++
 src/delivery_init.c        |  345 ++++++++
 src/delivery_install.c     |  116 +++
 src/delivery_populate.c    |  348 ++++++++
 src/delivery_postprocess.c |  263 ++++++
 src/delivery_show.c        |  117 +++
 src/delivery_test.c        |  244 ++++++
 src/stasis_main.c          |    4 +-
 13 files changed, 2096 insertions(+), 2054 deletions(-)
 create mode 100644 src/delivery_artifactory.c
 create mode 100644 src/delivery_build.c
 create mode 100644 src/delivery_conda.c
 create mode 100644 src/delivery_docker.c
 create mode 100644 src/delivery_init.c
 create mode 100644 src/delivery_install.c
 create mode 100644 src/delivery_populate.c
 create mode 100644 src/delivery_postprocess.c
 create mode 100644 src/delivery_show.c
 create mode 100644 src/delivery_test.c

(limited to 'src')

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 5d1c8ff..5c5bc6e 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -12,6 +12,16 @@ add_library(stasis_core STATIC
         utils.c
         system.c
         download.c
+        delivery_postprocess.c
+        delivery_conda.c
+        delivery_docker.c
+        delivery_install.c
+        delivery_artifactory.c
+        delivery_test.c
+        delivery_build.c
+        delivery_show.c
+        delivery_populate.c
+        delivery_init.c
         delivery.c
         recipe.c
         relocation.c
diff --git a/src/delivery.c b/src/delivery.c
index aba266b..a689db2 100644
--- a/src/delivery.c
+++ b/src/delivery.c
@@ -1,98 +1,5 @@
-#define _GNU_SOURCE
-
-#include <fnmatch.h>
 #include "core.h"
 
-extern struct STASIS_GLOBAL globals;
-
-static void ini_has_key_required(struct INIFILE *ini, const char *section_name, char *key) {
-    int status = ini_has_key(ini, section_name, key);
-    if (!status) {
-        SYSERROR("%s:%s key is required but not defined", section_name, key);
-        exit(1);
-    }
-}
-
-static void conv_str(char **x, union INIVal val) {
-    if (*x) {
-        guard_free(*x);
-    }
-    if (val.as_char_p) {
-        char *tplop = tpl_render(val.as_char_p);
-        if (tplop) {
-            *x = tplop;
-        } else {
-            *x = NULL;
-        }
-    } else {
-        *x = NULL;
-    }
-}
-
-int delivery_init_tmpdir(struct Delivery *ctx) {
-    char *tmpdir = NULL;
-    char *x = NULL;
-    int unusable = 0;
-    errno = 0;
-
-    x = getenv("TMPDIR");
-    if (x) {
-        guard_free(ctx->storage.tmpdir);
-        tmpdir = strdup(x);
-    } else {
-        tmpdir = ctx->storage.tmpdir;
-    }
-
-    if (!tmpdir) {
-        // memory error
-        return -1;
-    }
-
-    // If the directory doesn't exist, create it
-    if (access(tmpdir, F_OK) < 0) {
-        if (mkdirs(tmpdir, 0755) < 0) {
-            msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Unable to create temporary storage directory: %s (%s)\n", tmpdir, strerror(errno));
-            goto l_delivery_init_tmpdir_fatal;
-        }
-    }
-
-    // If we can't read, write, or execute, then die
-    if (access(tmpdir, R_OK | W_OK | X_OK) < 0) {
-        msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "%s requires at least 0755 permissions.\n");
-        goto l_delivery_init_tmpdir_fatal;
-    }
-
-    struct statvfs st;
-    if (statvfs(tmpdir, &st) < 0) {
-        goto l_delivery_init_tmpdir_fatal;
-    }
-
-#if defined(STASIS_OS_LINUX)
-    // If we can't execute programs, or write data to the file system at all, then die
-    if ((st.f_flag & ST_NOEXEC) != 0) {
-        msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "%s is mounted with noexec\n", tmpdir);
-        goto l_delivery_init_tmpdir_fatal;
-    }
-#endif
-    if ((st.f_flag & ST_RDONLY) != 0) {
-        msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "%s is mounted read-only\n", tmpdir);
-        goto l_delivery_init_tmpdir_fatal;
-    }
-
-    if (!globals.tmpdir) {
-        globals.tmpdir = strdup(tmpdir);
-    }
-
-    if (!ctx->storage.tmpdir) {
-        ctx->storage.tmpdir = strdup(globals.tmpdir);
-    }
-    return unusable;
-
-    l_delivery_init_tmpdir_fatal:
-    unusable = 1;
-    return unusable;
-}
-
 void delivery_free(struct Delivery *ctx) {
     guard_free(ctx->system.arch);
     GENERIC_ARRAY_FREE(ctx->system.platform);
@@ -193,557 +100,6 @@ void delivery_free(struct Delivery *ctx) {
     guard_free(ctx->_stasis_ini_fp.mission_path);
 }
 
-void delivery_init_dirs_stage2(struct Delivery *ctx) {
-    path_store(&ctx->storage.build_recipes_dir, PATH_MAX, ctx->storage.build_dir, "recipes");
-    path_store(&ctx->storage.build_sources_dir, PATH_MAX, ctx->storage.build_dir, "sources");
-    path_store(&ctx->storage.build_testing_dir, PATH_MAX, ctx->storage.build_dir, "testing");
-    path_store(&ctx->storage.build_docker_dir, PATH_MAX, ctx->storage.build_dir, "docker");
-
-    path_store(&ctx->storage.delivery_dir, PATH_MAX, ctx->storage.output_dir, "delivery");
-    path_store(&ctx->storage.results_dir, PATH_MAX, ctx->storage.output_dir, "results");
-    path_store(&ctx->storage.package_dir, PATH_MAX, ctx->storage.output_dir, "packages");
-    path_store(&ctx->storage.cfgdump_dir, PATH_MAX, ctx->storage.output_dir, "config");
-    path_store(&ctx->storage.meta_dir, PATH_MAX, ctx->storage.output_dir, "meta");
-
-    path_store(&ctx->storage.conda_artifact_dir, PATH_MAX, ctx->storage.package_dir, "conda");
-    path_store(&ctx->storage.wheel_artifact_dir, PATH_MAX, ctx->storage.package_dir, "wheels");
-    path_store(&ctx->storage.docker_artifact_dir, PATH_MAX, ctx->storage.package_dir, "docker");
-}
-
-void delivery_init_dirs_stage1(struct Delivery *ctx) {
-    char *rootdir = getenv("STASIS_ROOT");
-    if (rootdir) {
-        if (isempty(rootdir)) {
-            fprintf(stderr, "STASIS_ROOT is set, but empty. Please assign a file system path to this environment variable.\n");
-            exit(1);
-        }
-        path_store(&ctx->storage.root, PATH_MAX, rootdir, ctx->info.build_name);
-    } else {
-        // use "stasis" in current working directory
-        path_store(&ctx->storage.root, PATH_MAX, "stasis", ctx->info.build_name);
-    }
-    path_store(&ctx->storage.tools_dir, PATH_MAX, ctx->storage.root, "tools");
-    path_store(&ctx->storage.tmpdir, PATH_MAX, ctx->storage.root, "tmp");
-    if (delivery_init_tmpdir(ctx)) {
-        msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Set $TMPDIR to a location other than %s\n", globals.tmpdir);
-        if (globals.tmpdir)
-            guard_free(globals.tmpdir);
-        exit(1);
-    }
-
-    path_store(&ctx->storage.build_dir, PATH_MAX, ctx->storage.root, "build");
-    path_store(&ctx->storage.output_dir, PATH_MAX, ctx->storage.root, "output");
-
-    if (!ctx->storage.mission_dir) {
-        path_store(&ctx->storage.mission_dir, PATH_MAX, globals.sysconfdir, "mission");
-    }
-
-    if (access(ctx->storage.mission_dir, F_OK)) {
-        msg(STASIS_MSG_L1, "%s: %s\n", ctx->storage.mission_dir, strerror(errno));
-        exit(1);
-    }
-
-    // Override installation prefix using global configuration key
-    if (globals.conda_install_prefix && strlen(globals.conda_install_prefix)) {
-        // user wants a specific path
-        globals.conda_fresh_start = false;
-        /*
-        if (mkdirs(globals.conda_install_prefix, 0755)) {
-            msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Unable to create directory: %s: %s\n",
-                strerror(errno), globals.conda_install_prefix);
-            exit(1);
-        }
-         */
-        /*
-        ctx->storage.conda_install_prefix = realpath(globals.conda_install_prefix, NULL);
-        if (!ctx->storage.conda_install_prefix) {
-            msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "realpath(): Conda installation prefix reassignment failed\n");
-            exit(1);
-        }
-        ctx->storage.conda_install_prefix = strdup(globals.conda_install_prefix);
-         */
-        path_store(&ctx->storage.conda_install_prefix, PATH_MAX, globals.conda_install_prefix, "conda");
-    } else {
-        // install conda under the STASIS tree
-        path_store(&ctx->storage.conda_install_prefix, PATH_MAX, ctx->storage.tools_dir, "conda");
-    }
-}
-
-int delivery_init_platform(struct Delivery *ctx) {
-    msg(STASIS_MSG_L2, "Setting architecture\n");
-    char archsuffix[20];
-    struct utsname uts;
-    if (uname(&uts)) {
-        msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "uname() failed: %s\n", strerror(errno));
-        return -1;
-    }
-
-    ctx->system.platform = calloc(DELIVERY_PLATFORM_MAX + 1, sizeof(*ctx->system.platform));
-    if (!ctx->system.platform) {
-        SYSERROR("Unable to allocate %d records for platform array\n", DELIVERY_PLATFORM_MAX);
-        return -1;
-    }
-    for (size_t i = 0; i < DELIVERY_PLATFORM_MAX; i++) {
-        ctx->system.platform[i] = calloc(DELIVERY_PLATFORM_MAXLEN, sizeof(*ctx->system.platform[0]));
-    }
-
-    ctx->system.arch = strdup(uts.machine);
-    if (!ctx->system.arch) {
-        // memory error
-        return -1;
-    }
-
-    if (!strcmp(ctx->system.arch, "x86_64")) {
-        strcpy(archsuffix, "64");
-    } else {
-        strcpy(archsuffix, ctx->system.arch);
-    }
-
-    msg(STASIS_MSG_L2, "Setting platform\n");
-    strcpy(ctx->system.platform[DELIVERY_PLATFORM], uts.sysname);
-    if (!strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Darwin")) {
-        sprintf(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], "osx-%s", archsuffix);
-        strcpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], "MacOSX");
-        strcpy(ctx->system.platform[DELIVERY_PLATFORM_RELEASE], "macos");
-    } else if (!strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Linux")) {
-        sprintf(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], "linux-%s", archsuffix);
-        strcpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], "Linux");
-        strcpy(ctx->system.platform[DELIVERY_PLATFORM_RELEASE], "linux");
-    } else {
-        // Not explicitly supported systems
-        strcpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], ctx->system.platform[DELIVERY_PLATFORM]);
-        strcpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], ctx->system.platform[DELIVERY_PLATFORM]);
-        strcpy(ctx->system.platform[DELIVERY_PLATFORM_RELEASE], ctx->system.platform[DELIVERY_PLATFORM]);
-        tolower_s(ctx->system.platform[DELIVERY_PLATFORM_RELEASE]);
-    }
-
-    long cpu_count = get_cpu_count();
-    if (!cpu_count) {
-        fprintf(stderr, "Unable to determine CPU count. Falling back to 1.\n");
-        cpu_count = 1;
-    }
-    char ncpus[100] = {0};
-    sprintf(ncpus, "%ld", cpu_count);
-
-    // Declare some important bits as environment variables
-    setenv("CPU_COUNT", ncpus, 1);
-    setenv("STASIS_CPU_COUNT", ncpus, 1);
-    setenv("STASIS_ARCH", ctx->system.arch, 1);
-    setenv("STASIS_PLATFORM", ctx->system.platform[DELIVERY_PLATFORM], 1);
-    setenv("STASIS_CONDA_ARCH", ctx->system.arch, 1);
-    setenv("STASIS_CONDA_PLATFORM", ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], 1);
-    setenv("STASIS_CONDA_PLATFORM_SUBDIR", ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], 1);
-
-    // Register template variables
-    // These were moved out of main() because we can't take the address of system.platform[x]
-    // _before_ the array has been initialized.
-    tpl_register("system.arch", &ctx->system.arch);
-    tpl_register("system.platform", &ctx->system.platform[DELIVERY_PLATFORM_RELEASE]);
-
-    return 0;
-}
-
-static int populate_mission_ini(struct Delivery **ctx, int render_mode) {
-    int err = 0;
-    struct INIFILE *ini;
-
-    if ((*ctx)->_stasis_ini_fp.mission) {
-        return 0;
-    }
-
-    // Now populate the rules
-    char missionfile[PATH_MAX] = {0};
-    if (getenv("STASIS_SYSCONFDIR")) {
-        sprintf(missionfile, "%s/%s/%s/%s.ini",
-                getenv("STASIS_SYSCONFDIR"), "mission", (*ctx)->meta.mission, (*ctx)->meta.mission);
-    } else {
-        sprintf(missionfile, "%s/%s/%s/%s.ini",
-                globals.sysconfdir, "mission", (*ctx)->meta.mission, (*ctx)->meta.mission);
-    }
-
-    msg(STASIS_MSG_L2, "Reading mission configuration: %s\n", missionfile);
-    (*ctx)->_stasis_ini_fp.mission = ini_open(missionfile);
-    ini = (*ctx)->_stasis_ini_fp.mission;
-    if (!ini) {
-        msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read misson configuration: %s, %s\n", missionfile, strerror(errno));
-        exit(1);
-    }
-    (*ctx)->_stasis_ini_fp.mission_path = strdup(missionfile);
-
-    (*ctx)->rules.release_fmt = ini_getval_str(ini, "meta", "release_fmt", render_mode, &err);
-
-    // Used for setting artifactory build info
-    (*ctx)->rules.build_name_fmt = ini_getval_str(ini, "meta", "build_name_fmt", render_mode, &err);
-
-    // Used for setting artifactory build info
-    (*ctx)->rules.build_number_fmt = ini_getval_str(ini, "meta", "build_number_fmt", render_mode, &err);
-    return 0;
-}
-
-void validate_delivery_ini(struct INIFILE *ini) {
-    if (!ini) {
-        SYSERROR("%s", "INIFILE is NULL!");
-        exit(1);
-    }
-    if (ini_section_search(&ini, INI_SEARCH_EXACT, "meta")) {
-        ini_has_key_required(ini, "meta", "name");
-        ini_has_key_required(ini, "meta", "version");
-        ini_has_key_required(ini, "meta", "rc");
-        ini_has_key_required(ini, "meta", "mission");
-        ini_has_key_required(ini, "meta", "python");
-    } else {
-        SYSERROR("%s", "[meta] configuration section is required");
-        exit(1);
-    }
-
-    if (ini_section_search(&ini, INI_SEARCH_EXACT, "conda")) {
-        ini_has_key_required(ini, "conda", "installer_name");
-        ini_has_key_required(ini, "conda", "installer_version");
-        ini_has_key_required(ini, "conda", "installer_platform");
-        ini_has_key_required(ini, "conda", "installer_arch");
-    } else {
-        SYSERROR("%s", "[conda] configuration section is required");
-        exit(1);
-    }
-
-    for (size_t i = 0; i < ini->section_count; i++) {
-        struct INISection *section = ini->section[i];
-        if (section && startswith(section->key, "test:")) {
-            char *name = strstr(section->key, ":");
-            if (name && strlen(name) > 1) {
-                name = &name[1];
-            }
-            //ini_has_key_required(ini, section->key, "version");
-            //ini_has_key_required(ini, section->key, "repository");
-            if (globals.enable_testing) {
-                ini_has_key_required(ini, section->key, "script");
-            }
-        }
-    }
-
-    if (ini_section_search(&ini, INI_SEARCH_EXACT, "deploy:docker")) {
-        // yeah?
-    }
-
-    for (size_t i = 0; i < ini->section_count; i++) {
-        struct INISection *section = ini->section[i];
-        if (section && startswith(section->key, "deploy:artifactory")) {
-            ini_has_key_required(ini, section->key, "files");
-            ini_has_key_required(ini, section->key, "dest");
-        }
-    }
-}
-
-static int populate_delivery_ini(struct Delivery *ctx, int render_mode) {
-    union INIVal val;
-    struct INIFILE *ini = ctx->_stasis_ini_fp.delivery;
-    struct INIData *rtdata;
-    RuntimeEnv *rt;
-
-    validate_delivery_ini(ini);
-    // Populate runtime variables first they may be interpreted by other
-    // keys in the configuration
-    rt = runtime_copy(__environ);
-    while ((rtdata = ini_getall(ini, "runtime")) != NULL) {
-        char rec[STASIS_BUFSIZ];
-        sprintf(rec, "%s=%s", lstrip(strip(rtdata->key)), lstrip(strip(rtdata->value)));
-        runtime_set(rt, rtdata->key, rtdata->value);
-    }
-    runtime_apply(rt);
-    ctx->runtime.environ = rt;
-
-    int err = 0;
-    ctx->meta.mission = ini_getval_str(ini, "meta", "mission", render_mode, &err);
-
-    if (!strcasecmp(ctx->meta.mission, "hst")) {
-        ctx->meta.codename = ini_getval_str(ini, "meta", "codename", render_mode, &err);
-    } else {
-        ctx->meta.codename = NULL;
-    }
-
-    ctx->meta.version = ini_getval_str(ini, "meta", "version", render_mode, &err);
-    ctx->meta.name = ini_getval_str(ini, "meta", "name", render_mode, &err);
-    ctx->meta.rc = ini_getval_int(ini, "meta", "rc", render_mode, &err);
-    ctx->meta.final = ini_getval_bool(ini, "meta", "final", render_mode, &err);
-    ctx->meta.based_on = ini_getval_str(ini, "meta", "based_on", render_mode, &err);
-
-    if (!ctx->meta.python) {
-        ctx->meta.python = ini_getval_str(ini, "meta", "python", render_mode, &err);
-        guard_free(ctx->meta.python_compact);
-        ctx->meta.python_compact = to_short_version(ctx->meta.python);
-    } else {
-        ini_setval(&ini, INI_SETVAL_REPLACE, "meta", "python", ctx->meta.python);
-    }
-
-    ctx->conda.installer_name = ini_getval_str(ini, "conda", "installer_name", render_mode, &err);
-    ctx->conda.installer_version = ini_getval_str(ini, "conda", "installer_version", render_mode, &err);
-    ctx->conda.installer_platform = ini_getval_str(ini, "conda", "installer_platform", render_mode, &err);
-    ctx->conda.installer_arch = ini_getval_str(ini, "conda", "installer_arch", render_mode, &err);
-    ctx->conda.installer_baseurl = ini_getval_str(ini, "conda", "installer_baseurl", render_mode, &err);
-    ctx->conda.conda_packages = ini_getval_strlist(ini, "conda", "conda_packages", " "LINE_SEP, render_mode, &err);
-
-    if (ctx->conda.conda_packages->data && ctx->conda.conda_packages->data[0] && strpbrk(ctx->conda.conda_packages->data[0], " \t")) {
-        normalize_space(ctx->conda.conda_packages->data[0]);
-        replace_text(ctx->conda.conda_packages->data[0], " ", LINE_SEP, 0);
-        char *pip_packages_replacement = join(ctx->conda.conda_packages->data, LINE_SEP);
-        ini_setval(&ini, INI_SETVAL_REPLACE, "conda", "conda_packages", pip_packages_replacement);
-        guard_free(pip_packages_replacement);
-        guard_strlist_free(&ctx->conda.conda_packages);
-        ctx->conda.conda_packages = ini_getval_strlist(ini, "conda", "conda_packages", LINE_SEP, render_mode, &err);
-    }
-
-    for (size_t i = 0; i < strlist_count(ctx->conda.conda_packages); i++) {
-        char *pkg = strlist_item(ctx->conda.conda_packages, i);
-        if (strpbrk(pkg, ";#") || isempty(pkg)) {
-            strlist_remove(ctx->conda.conda_packages, i);
-        }
-    }
-
-    ctx->conda.pip_packages = ini_getval_strlist(ini, "conda", "pip_packages", LINE_SEP, render_mode, &err);
-    if (ctx->conda.pip_packages->data && ctx->conda.pip_packages->data[0] && strpbrk(ctx->conda.pip_packages->data[0], " \t")) {
-        normalize_space(ctx->conda.pip_packages->data[0]);
-        replace_text(ctx->conda.pip_packages->data[0], " ", LINE_SEP, 0);
-        char *pip_packages_replacement = join(ctx->conda.pip_packages->data, LINE_SEP);
-        ini_setval(&ini, INI_SETVAL_REPLACE, "conda", "pip_packages", pip_packages_replacement);
-        guard_free(pip_packages_replacement);
-        guard_strlist_free(&ctx->conda.pip_packages);
-        ctx->conda.pip_packages = ini_getval_strlist(ini, "conda", "pip_packages", LINE_SEP, render_mode, &err);
-    }
-
-    for (size_t i = 0; i < strlist_count(ctx->conda.pip_packages); i++) {
-        char *pkg = strlist_item(ctx->conda.pip_packages, i);
-        if (strpbrk(pkg, ";#") || isempty(pkg)) {
-            strlist_remove(ctx->conda.pip_packages, i);
-        }
-    }
-
-    // Delivery metadata consumed
-    populate_mission_ini(&ctx, render_mode);
-
-    if (ctx->info.release_name) {
-        guard_free(ctx->info.release_name);
-        guard_free(ctx->info.build_name);
-        guard_free(ctx->info.build_number);
-    }
-
-    if (delivery_format_str(ctx, &ctx->info.release_name, ctx->rules.release_fmt)) {
-        fprintf(stderr, "Failed to generate release name. Format used: %s\n", ctx->rules.release_fmt);
-        return -1;
-    }
-
-    if (!ctx->info.build_name) {
-        delivery_format_str(ctx, &ctx->info.build_name, ctx->rules.build_name_fmt);
-    }
-    if (!ctx->info.build_number) {
-        delivery_format_str(ctx, &ctx->info.build_number, ctx->rules.build_number_fmt);
-    }
-
-    // Best I can do to make output directories unique. Annoying.
-    delivery_init_dirs_stage2(ctx);
-
-    if (!ctx->conda.conda_packages_defer) {
-        ctx->conda.conda_packages_defer = strlist_init();
-    }
-    if (!ctx->conda.pip_packages_defer) {
-        ctx->conda.pip_packages_defer = strlist_init();
-    }
-
-    for (size_t z = 0, i = 0; i < ini->section_count; i++) {
-        char *section_name = ini->section[i]->key;
-        if (startswith(section_name, "test:")) {
-            struct Test *test = &ctx->tests[z];
-            val.as_char_p = strchr(ini->section[i]->key, ':') + 1;
-            if (val.as_char_p && isempty(val.as_char_p)) {
-                return 1;
-            }
-            conv_str(&test->name, val);
-
-            test->version = ini_getval_str(ini, section_name, "version", render_mode, &err);
-            test->repository = ini_getval_str(ini, section_name, "repository", render_mode, &err);
-            test->script_setup = ini_getval_str(ini, section_name, "script_setup", INI_READ_RAW, &err);
-            test->script = ini_getval_str(ini, section_name, "script", INI_READ_RAW, &err);
-            test->disable = ini_getval_bool(ini, section_name, "disable", render_mode, &err);
-            test->parallel = ini_getval_bool(ini, section_name, "parallel", render_mode, &err);
-            if (err) {
-                test->parallel = true;
-            }
-            test->repository_remove_tags = ini_getval_strlist(ini, section_name, "repository_remove_tags", LINE_SEP, render_mode, &err);
-            test->build_recipe = ini_getval_str(ini, section_name, "build_recipe", render_mode, &err);
-            test->runtime.environ = ini_getval_strlist(ini, section_name, "runtime", LINE_SEP, render_mode, &err);
-            z++;
-        }
-    }
-
-    for (size_t z = 0, i = 0; i < ini->section_count; i++) {
-        char *section_name = ini->section[i]->key;
-        struct Deploy *deploy = &ctx->deploy;
-        if (startswith(section_name, "deploy:artifactory")) {
-            struct JFrog *jfrog = &deploy->jfrog[z];
-            // Artifactory base configuration
-
-            jfrog->upload_ctx.workaround_parent_only = ini_getval_bool(ini, section_name, "workaround_parent_only", render_mode, &err);
-            jfrog->upload_ctx.exclusions = ini_getval_str(ini, section_name, "exclusions", render_mode, &err);
-            jfrog->upload_ctx.explode = ini_getval_bool(ini, section_name, "explode", render_mode, &err);
-            jfrog->upload_ctx.recursive = ini_getval_bool(ini, section_name, "recursive", render_mode, &err);
-            jfrog->upload_ctx.retries = ini_getval_int(ini, section_name, "retries", render_mode, &err);
-            jfrog->upload_ctx.retry_wait_time = ini_getval_int(ini, section_name, "retry_wait_time", render_mode, &err);
-            jfrog->upload_ctx.detailed_summary = ini_getval_bool(ini, section_name, "detailed_summary", render_mode, &err);
-            jfrog->upload_ctx.quiet = ini_getval_bool(ini, section_name, "quiet", render_mode, &err);
-            jfrog->upload_ctx.regexp = ini_getval_bool(ini, section_name, "regexp", render_mode, &err);
-            jfrog->upload_ctx.spec = ini_getval_str(ini, section_name, "spec", render_mode, &err);
-            jfrog->upload_ctx.flat = ini_getval_bool(ini, section_name, "flat", render_mode, &err);
-            jfrog->repo = ini_getval_str(ini, section_name, "repo", render_mode, &err);
-            jfrog->dest = ini_getval_str(ini, section_name, "dest", render_mode, &err);
-            jfrog->files = ini_getval_strlist(ini, section_name, "files", LINE_SEP, render_mode, &err);
-            z++;
-        }
-    }
-
-    for (size_t i = 0; i < ini->section_count; i++) {
-        char *section_name = ini->section[i]->key;
-        struct Deploy *deploy = &ctx->deploy;
-        if (startswith(ini->section[i]->key, "deploy:docker")) {
-            struct Docker *docker = &deploy->docker;
-
-            docker->registry = ini_getval_str(ini, section_name, "registry", render_mode, &err);
-            docker->image_compression = ini_getval_str(ini, section_name, "image_compression", render_mode, &err);
-            docker->test_script = ini_getval_str(ini, section_name, "test_script", render_mode, &err);
-            docker->build_args = ini_getval_strlist(ini, section_name, "build_args", LINE_SEP, render_mode, &err);
-            docker->tags = ini_getval_strlist(ini, section_name, "tags", LINE_SEP, render_mode, &err);
-        }
-    }
-    return 0;
-}
-
-int populate_delivery_cfg(struct Delivery *ctx, int render_mode) {
-    struct INIFILE *cfg = ctx->_stasis_ini_fp.cfg;
-    if (!cfg) {
-        return -1;
-    }
-    int err = 0;
-    ctx->storage.conda_staging_dir = ini_getval_str(cfg, "default", "conda_staging_dir", render_mode, &err);
-    ctx->storage.conda_staging_url = ini_getval_str(cfg, "default", "conda_staging_url", render_mode, &err);
-    ctx->storage.wheel_staging_dir = ini_getval_str(cfg, "default", "wheel_staging_dir", render_mode, &err);
-    ctx->storage.wheel_staging_url = ini_getval_str(cfg, "default", "wheel_staging_url", render_mode, &err);
-    globals.conda_fresh_start = ini_getval_bool(cfg, "default", "conda_fresh_start", render_mode, &err);
-    if (!globals.continue_on_error) {
-        globals.continue_on_error = ini_getval_bool(cfg, "default", "continue_on_error", render_mode, &err);
-    }
-    if (!globals.always_update_base_environment) {
-        globals.always_update_base_environment = ini_getval_bool(cfg, "default", "always_update_base_environment", render_mode, &err);
-    }
-    globals.conda_install_prefix = ini_getval_str(cfg, "default", "conda_install_prefix", render_mode, &err);
-    globals.conda_packages = ini_getval_strlist(cfg, "default", "conda_packages", LINE_SEP, render_mode, &err);
-    globals.pip_packages = ini_getval_strlist(cfg, "default", "pip_packages", LINE_SEP, render_mode, &err);
-
-    globals.jfrog.jfrog_artifactory_base_url = ini_getval_str(cfg, "jfrog_cli_download", "url", render_mode, &err);
-    globals.jfrog.jfrog_artifactory_product = ini_getval_str(cfg, "jfrog_cli_download", "product", render_mode, &err);
-    globals.jfrog.cli_major_ver = ini_getval_str(cfg, "jfrog_cli_download", "version_series", render_mode, &err);
-    globals.jfrog.version = ini_getval_str(cfg, "jfrog_cli_download", "version", render_mode, &err);
-    globals.jfrog.remote_filename = ini_getval_str(cfg, "jfrog_cli_download", "filename", render_mode, &err);
-    globals.jfrog.url = ini_getval_str(cfg, "deploy:artifactory", "url", render_mode, &err);
-    globals.jfrog.repo = ini_getval_str(cfg, "deploy:artifactory", "repo", render_mode, &err);
-
-    return 0;
-}
-
-static int populate_info(struct Delivery *ctx) {
-    if (!ctx->info.time_str_epoch) {
-        // Record timestamp used for release
-        time(&ctx->info.time_now);
-        ctx->info.time_info = localtime(&ctx->info.time_now);
-
-        ctx->info.time_str_epoch = calloc(STASIS_TIME_STR_MAX, sizeof(*ctx->info.time_str_epoch));
-        if (!ctx->info.time_str_epoch) {
-            msg(STASIS_MSG_ERROR, "Unable to allocate memory for Unix epoch string\n");
-            return -1;
-        }
-        snprintf(ctx->info.time_str_epoch, STASIS_TIME_STR_MAX - 1, "%li", ctx->info.time_now);
-    }
-    return 0;
-}
-
-int *bootstrap_build_info(struct Delivery *ctx) {
-    struct Delivery local;
-    memset(&local, 0, sizeof(local));
-    local._stasis_ini_fp.cfg = ini_open(ctx->_stasis_ini_fp.cfg_path);
-    local._stasis_ini_fp.delivery = ini_open(ctx->_stasis_ini_fp.delivery_path);
-    delivery_init_platform(&local);
-    populate_delivery_cfg(&local, INI_READ_RENDER);
-    populate_delivery_ini(&local, INI_READ_RENDER);
-    populate_info(&local);
-    ctx->info.build_name = strdup(local.info.build_name);
-    ctx->info.build_number = strdup(local.info.build_number);
-    ctx->info.release_name = strdup(local.info.release_name);
-    memcpy(&ctx->info.time_info, &local.info.time_info, sizeof(ctx->info.time_info));
-    ctx->info.time_now = local.info.time_now;
-    ctx->info.time_str_epoch = strdup(local.info.time_str_epoch);
-    delivery_free(&local);
-    return 0;
-}
-
-int delivery_init(struct Delivery *ctx, int render_mode) {
-    populate_info(ctx);
-    populate_delivery_cfg(ctx, INI_READ_RENDER);
-
-    // Set artifactory URL via environment variable if possible
-    char *jfurl = getenv("STASIS_JF_ARTIFACTORY_URL");
-    if (jfurl) {
-        if (globals.jfrog.url) {
-            guard_free(globals.jfrog.url);
-        }
-        globals.jfrog.url = strdup(jfurl);
-    }
-
-    // Set artifactory repository via environment if possible
-    char *jfrepo = getenv("STASIS_JF_REPO");
-    if (jfrepo) {
-        if (globals.jfrog.repo) {
-            guard_free(globals.jfrog.repo);
-        }
-        globals.jfrog.repo = strdup(jfrepo);
-    }
-
-    // Configure architecture and platform information
-    delivery_init_platform(ctx);
-
-    // Create STASIS directory structure
-    delivery_init_dirs_stage1(ctx);
-
-    char config_local[PATH_MAX];
-    sprintf(config_local, "%s/%s", ctx->storage.tmpdir, "config");
-    setenv("XDG_CONFIG_HOME", config_local, 1);
-
-    char cache_local[PATH_MAX];
-    sprintf(cache_local, "%s/%s", ctx->storage.tmpdir, "cache");
-    setenv("XDG_CACHE_HOME", ctx->storage.tmpdir, 1);
-
-    // add tools to PATH
-    char pathvar_tmp[STASIS_BUFSIZ];
-    sprintf(pathvar_tmp, "%s/bin:%s", ctx->storage.tools_dir, getenv("PATH"));
-    setenv("PATH", pathvar_tmp, 1);
-
-    // Prevent git from paginating output
-    setenv("GIT_PAGER", "", 1);
-
-    populate_delivery_ini(ctx, render_mode);
-
-    if (ctx->deploy.docker.tags) {
-        for (size_t i = 0; i < strlist_count(ctx->deploy.docker.tags); i++) {
-            char *item = strlist_item(ctx->deploy.docker.tags, i);
-            tolower_s(item);
-        }
-    }
-
-    if (ctx->deploy.docker.image_compression) {
-        if (docker_validate_compression_program(ctx->deploy.docker.image_compression)) {
-            SYSERROR("[deploy:docker].image_compression - invalid command / program is not installed: %s", ctx->deploy.docker.image_compression);
-            return -1;
-        }
-    }
-    return 0;
-}
-
 int delivery_format_str(struct Delivery *ctx, char **dest, const char *fmt) {
     size_t fmt_len = strlen(fmt);
 
@@ -805,672 +161,53 @@ int delivery_format_str(struct Delivery *ctx, char **dest, const char *fmt) {
     return 0;
 }
 
-void delivery_debug_show(struct Delivery *ctx) {
-    printf("\n====DEBUG====\n");
-    printf("%-20s %-10s\n", "System configuration directory:", globals.sysconfdir);
-    printf("%-20s %-10s\n", "Mission directory:", ctx->storage.mission_dir);
-    printf("%-20s %-10s\n", "Testing enabled:", globals.enable_testing ? "Yes" : "No");
-    printf("%-20s %-10s\n", "Docker image builds enabled:", globals.enable_docker ? "Yes" : "No");
-    printf("%-20s %-10s\n", "Artifact uploading enabled:", globals.enable_artifactory ? "Yes" : "No");
-}
+void delivery_defer_packages(struct Delivery *ctx, int type) {
+    struct StrList *dataptr = NULL;
+    struct StrList *deferred = NULL;
+    char *name = NULL;
+    char cmd[PATH_MAX];
 
-void delivery_meta_show(struct Delivery *ctx) {
-    if (globals.verbose) {
-        delivery_debug_show(ctx);
-    }
+    memset(cmd, 0, sizeof(cmd));
 
-    printf("\n====DELIVERY====\n");
-    printf("%-20s %-10s\n", "Target Python:", ctx->meta.python);
-    printf("%-20s %-10s\n", "Name:", ctx->meta.name);
-    printf("%-20s %-10s\n", "Mission:", ctx->meta.mission);
-    if (ctx->meta.codename) {
-        printf("%-20s %-10s\n", "Codename:", ctx->meta.codename);
-    }
-    if (ctx->meta.version) {
-        printf("%-20s %-10s\n", "Version", ctx->meta.version);
-    }
-    if (!ctx->meta.final) {
-        printf("%-20s %-10d\n", "RC Level:", ctx->meta.rc);
+    char mode[10];
+    if (DEFER_CONDA == type) {
+        dataptr = ctx->conda.conda_packages;
+        deferred = ctx->conda.conda_packages_defer;
+        strcpy(mode, "conda");
+    } else if (DEFER_PIP == type) {
+        dataptr = ctx->conda.pip_packages;
+        deferred = ctx->conda.pip_packages_defer;
+        strcpy(mode, "pip");
     }
-    printf("%-20s %-10s\n", "Final Release:", ctx->meta.final ? "Yes" : "No");
-    printf("%-20s %-10s\n", "Based On:", ctx->meta.based_on ? ctx->meta.based_on : "New");
-}
-
-void delivery_conda_show(struct Delivery *ctx) {
-    printf("\n====CONDA====\n");
-    printf("%-20s %-10s\n", "Prefix:", ctx->storage.conda_install_prefix);
+    msg(STASIS_MSG_L2, "Filtering %s packages by test definition...\n", mode);
 
-    puts("Native Packages:");
-    if (strlist_count(ctx->conda.conda_packages) || strlist_count(ctx->conda.conda_packages_defer)) {
-        struct StrList *list_conda = strlist_init();
-        if (strlist_count(ctx->conda.conda_packages)) {
-            strlist_append_strlist(list_conda, ctx->conda.conda_packages);
-        }
-        if (strlist_count(ctx->conda.conda_packages_defer)) {
-            strlist_append_strlist(list_conda, ctx->conda.conda_packages_defer);
-        }
-        strlist_sort(list_conda, STASIS_SORT_ALPHA);
-        
-        for (size_t i = 0; i < strlist_count(list_conda); i++) {
-            char *token = strlist_item(list_conda, i);
-            if (isempty(token) || isblank(*token) || startswith(token, "-")) {
-                continue;
-            }
-            printf("%21s%s\n", "", token);
-        }
-        guard_strlist_free(&list_conda);
-    } else {
-       printf("%21s%s\n", "", "N/A");
-    }
+    struct StrList *filtered = NULL;
+    filtered = strlist_init();
+    for (size_t i = 0; i < strlist_count(dataptr); i++) {
+        int ignore_pkg = 0;
 
-    puts("Python Packages:");
-    if (strlist_count(ctx->conda.pip_packages) || strlist_count(ctx->conda.pip_packages_defer)) {
-        struct StrList *list_python = strlist_init();
-        if (strlist_count(ctx->conda.pip_packages)) {
-            strlist_append_strlist(list_python, ctx->conda.pip_packages);
-        }
-        if (strlist_count(ctx->conda.pip_packages_defer)) {
-            strlist_append_strlist(list_python, ctx->conda.pip_packages_defer);
+        name = strlist_item(dataptr, i);
+        if (!strlen(name) || isblank(*name) || isspace(*name)) {
+            // no data
+            continue;
         }
-        strlist_sort(list_python, STASIS_SORT_ALPHA);
+        msg(STASIS_MSG_L3, "package '%s': ", name);
 
-        for (size_t i = 0; i < strlist_count(list_python); i++) {
-            char *token = strlist_item(list_python, i);
-            if (isempty(token) || isblank(*token) || startswith(token, "-")) {
-                continue;
+        // Compile a list of packages that are *also* to be tested.
+        char *version;
+        char *spec_begin = strpbrk(name, "@~=<>!");
+        char *spec_end = spec_begin;
+        if (spec_end) {
+            // A version is present in the package name. Jump past operator(s).
+            while (*spec_end != '\0' && !isalnum(*spec_end)) {
+                spec_end++;
             }
-            printf("%21s%s\n", "", token);
         }
-        guard_strlist_free(&list_python);
-    } else {
-        printf("%21s%s\n", "", "N/A");
-    }
-}
 
-void delivery_tests_show(struct Delivery *ctx) {
-    printf("\n====TESTS====\n");
-    for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) {
-        if (!ctx->tests[i].name) {
-            continue;
-        }
-        printf("%-20s %-20s %s\n", ctx->tests[i].name,
-               ctx->tests[i].version,
-               ctx->tests[i].repository);
-    }
-}
-
-void delivery_runtime_show(struct Delivery *ctx) {
-    printf("\n====RUNTIME====\n");
-    struct StrList *rt = NULL;
-    rt = strlist_copy(ctx->runtime.environ);
-    if (!rt) {
-        // no data
-        return;
-    }
-    strlist_sort(rt, STASIS_SORT_ALPHA);
-    size_t total = strlist_count(rt);
-    for (size_t i = 0; i < total; i++) {
-        char *item = strlist_item(rt, i);
-        if (!item) {
-            // not supposed to occur
-            msg(STASIS_MSG_WARN | STASIS_MSG_L1, "Encountered unexpected NULL at record %zu of %zu of runtime array.\n", i);
-            return;
-        }
-        printf("%s\n", item);
-    }
-}
-
-int delivery_build_recipes(struct Delivery *ctx) {
-    for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) {
-        char *recipe_dir = NULL;
-        if (ctx->tests[i].build_recipe) { // build a conda recipe
-            int recipe_type;
-            int status;
-            if (recipe_clone(ctx->storage.build_recipes_dir, ctx->tests[i].build_recipe, NULL, &recipe_dir)) {
-                fprintf(stderr, "Encountered an issue while cloning recipe for: %s\n", ctx->tests[i].name);
-                return -1;
-            }
-            recipe_type = recipe_get_type(recipe_dir);
-            pushd(recipe_dir);
-            {
-                if (RECIPE_TYPE_ASTROCONDA == recipe_type) {
-                    pushd(path_basename(ctx->tests[i].repository));
-                } else if (RECIPE_TYPE_CONDA_FORGE == recipe_type) {
-                    pushd("recipe");
-                }
-
-                char recipe_version[100];
-                char recipe_buildno[100];
-                char recipe_git_url[PATH_MAX];
-                char recipe_git_rev[PATH_MAX];
-
-                //sprintf(recipe_version, "{%% set version = GIT_DESCRIBE_TAG ~ \".dev\" ~ GIT_DESCRIBE_NUMBER ~ \"+\" ~ GIT_DESCRIBE_HASH %%}");
-                //sprintf(recipe_git_url, "  git_url: %s", ctx->tests[i].repository);
-                //sprintf(recipe_git_rev, "  git_rev: %s", ctx->tests[i].version);
-                // TODO: Conditionally download archives if github.com is the origin. Else, use raw git_* keys ^^^
-                sprintf(recipe_version, "{%% set version = \"%s\" %%}", ctx->tests[i].repository_info_tag ? ctx->tests[i].repository_info_tag : ctx->tests[i].version);
-                sprintf(recipe_git_url, "  url: %s/archive/refs/tags/{{ version }}.tar.gz", ctx->tests[i].repository);
-                strcpy(recipe_git_rev, "");
-                sprintf(recipe_buildno, "  number: 0");
-
-                unsigned flags = REPLACE_TRUNCATE_AFTER_MATCH;
-                //file_replace_text("meta.yaml", "{% set version = ", recipe_version);
-                if (ctx->meta.final) {
-                    sprintf(recipe_version, "{%% set version = \"%s\" %%}", ctx->tests[i].version);
-                    // TODO: replace sha256 of tagged archive
-                    // TODO: leave the recipe unchanged otherwise. in theory this should produce the same conda package hash as conda forge.
-                    // For now, remove the sha256 requirement
-                    file_replace_text("meta.yaml", "sha256:", "\n", flags);
-                } else {
-                    file_replace_text("meta.yaml", "{% set version = ", recipe_version, flags);
-                    file_replace_text("meta.yaml", "  url:", recipe_git_url, flags);
-                    //file_replace_text("meta.yaml", "sha256:", recipe_git_rev);
-                    file_replace_text("meta.yaml", "  sha256:", "\n", flags);
-                    file_replace_text("meta.yaml", "  number:", recipe_buildno, flags);
-                }
-
-                char command[PATH_MAX];
-                if (RECIPE_TYPE_CONDA_FORGE == recipe_type) {
-                    char arch[STASIS_NAME_MAX] = {0};
-                    char platform[STASIS_NAME_MAX] = {0};
-
-                    strcpy(platform, ctx->system.platform[DELIVERY_PLATFORM]);
-                    if (strstr(platform, "Darwin")) {
-                        memset(platform, 0, sizeof(platform));
-                        strcpy(platform, "osx");
-                    }
-                    tolower_s(platform);
-                    if (strstr(ctx->system.arch, "arm64")) {
-                        strcpy(arch, "arm64");
-                    } else if (strstr(ctx->system.arch, "64")) {
-                        strcpy(arch, "64");
-                    } else {
-                        strcat(arch, "32"); // blind guess
-                    }
-                    tolower_s(arch);
-
-                    sprintf(command, "mambabuild --python=%s -m ../.ci_support/%s_%s_.yaml .",
-                        ctx->meta.python, platform, arch);
-                    } else {
-                        sprintf(command, "mambabuild --python=%s .", ctx->meta.python);
-                    }
-                status = conda_exec(command);
-                if (status) {
-                    return -1;
-                }
-
-                if (RECIPE_TYPE_GENERIC != recipe_type) {
-                    popd();
-                }
-                popd();
-            }
-        }
-        if (recipe_dir) {
-            guard_free(recipe_dir);
-        }
-    }
-    return 0;
-}
-
-static int filter_repo_tags(char *repo, struct StrList *patterns) {
-    int result = 0;
-
-    if (!pushd(repo)) {
-        int list_status = 0;
-        char *tags_raw = shell_output("git tag -l", &list_status);
-        struct StrList *tags = strlist_init();
-        strlist_append_tokenize(tags, tags_raw, LINE_SEP);
-
-        for (size_t i = 0; tags && i < strlist_count(tags); i++) {
-            char *tag = strlist_item(tags, i);
-            for (size_t p = 0; p < strlist_count(patterns); p++) {
-                char *pattern = strlist_item(patterns, p);
-                int match = fnmatch(pattern, tag, 0);
-                if (!match) {
-                    char cmd[PATH_MAX] = {0};
-                    sprintf(cmd, "git tag -d %s", tag);
-                    result += system(cmd);
-                    break;
-                }
-            }
-        }
-        guard_strlist_free(&tags);
-        guard_free(tags_raw);
-        popd();
-    } else {
-        result = -1;
-    }
-    return result;
-}
-
-struct StrList *delivery_build_wheels(struct Delivery *ctx) {
-    struct StrList *result = NULL;
-    struct Process proc;
-    memset(&proc, 0, sizeof(proc));
-
-    result = strlist_init();
-    if (!result) {
-        perror("unable to allocate memory for string list");
-        return NULL;
-    }
-
-    for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) {
-        if (!ctx->tests[i].build_recipe && ctx->tests[i].repository) { // build from source
-            char srcdir[PATH_MAX];
-            char wheeldir[PATH_MAX];
-            memset(srcdir, 0, sizeof(srcdir));
-            memset(wheeldir, 0, sizeof(wheeldir));
-
-            sprintf(srcdir, "%s/%s", ctx->storage.build_sources_dir, ctx->tests[i].name);
-            git_clone(&proc, ctx->tests[i].repository, srcdir, ctx->tests[i].version);
-
-            if (ctx->tests[i].repository_remove_tags && strlist_count(ctx->tests[i].repository_remove_tags)) {
-                filter_repo_tags(srcdir, ctx->tests[i].repository_remove_tags);
-            }
-
-            pushd(srcdir);
-            {
-                char dname[NAME_MAX];
-                char outdir[PATH_MAX];
-                char cmd[PATH_MAX * 2];
-                memset(dname, 0, sizeof(dname));
-                memset(outdir, 0, sizeof(outdir));
-                memset(cmd, 0, sizeof(outdir));
-
-                strcpy(dname, ctx->tests[i].name);
-                tolower_s(dname);
-                sprintf(outdir, "%s/%s", ctx->storage.wheel_artifact_dir, dname);
-                if (mkdirs(outdir, 0755)) {
-                    fprintf(stderr, "failed to create output directory: %s\n", outdir);
-                }
-
-                sprintf(cmd, "-m build -w -o %s", outdir);
-                if (python_exec(cmd)) {
-                    fprintf(stderr, "failed to generate wheel package for %s-%s\n", ctx->tests[i].name, ctx->tests[i].version);
-                    strlist_free(&result);
-                    return NULL;
-                }
-                popd();
-            }
-        }
-    }
-    return result;
-}
-
-static const struct Test *requirement_from_test(struct Delivery *ctx, const char *name) {
-    struct Test *result;
-
-    result = NULL;
-    for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) {
-        if (ctx->tests[i].name && strstr(name, ctx->tests[i].name)) {
-            result = &ctx->tests[i];
-            break;
-        }
-    }
-    return result;
-}
-
-int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, char *env_name, int type, struct StrList **manifest) {
-    char cmd[PATH_MAX];
-    char pkgs[STASIS_BUFSIZ];
-    char *env_current = getenv("CONDA_DEFAULT_ENV");
-
-    if (env_current) {
-        // The requested environment is not the current environment
-        if (strcmp(env_current, env_name) != 0) {
-            // Activate the requested environment
-            printf("Activating: %s\n", env_name);
-            conda_activate(conda_install_dir, env_name);
-            runtime_replace(&ctx->runtime.environ, __environ);
-        }
-    }
-
-    memset(cmd, 0, sizeof(cmd));
-    memset(pkgs, 0, sizeof(pkgs));
-    strcat(cmd, "install");
-
-    typedef int (*Runner)(const char *);
-    Runner runner = NULL;
-    if (INSTALL_PKG_CONDA & type) {
-        runner = conda_exec;
-    } else if (INSTALL_PKG_PIP & type) {
-        runner = pip_exec;
-    }
-
-    if (INSTALL_PKG_CONDA_DEFERRED & type) {
-        strcat(cmd, " --use-local");
-    } else if (INSTALL_PKG_PIP_DEFERRED & type) {
-        // Don't change the baseline package set unless we're working with a
-        // new build. Release candidates will need to keep packages as stable
-        // as possible between releases.
-        if (!ctx->meta.based_on) {
-            strcat(cmd, " --upgrade");
-        }
-        sprintf(cmd + strlen(cmd), " --extra-index-url 'file://%s'", ctx->storage.wheel_artifact_dir);
-    }
-
-    for (size_t x = 0; manifest[x] != NULL; x++) {
-        char *name = NULL;
-        for (size_t p = 0; p < strlist_count(manifest[x]); p++) {
-            name = strlist_item(manifest[x], p);
-            strip(name);
-            if (!strlen(name)) {
-                continue;
-            }
-            if (INSTALL_PKG_PIP_DEFERRED & type) {
-                struct Test *info = (struct Test *) requirement_from_test(ctx, name);
-                if (info) {
-                    if (!strcmp(info->version, "HEAD")) {
-                        struct StrList *tag_data = strlist_init();
-                        if (!tag_data) {
-                            SYSERROR("%s", "Unable to allocate memory for tag data\n");
-                            return -1;
-                        }
-                        strlist_append_tokenize(tag_data, info->repository_info_tag, "-");
-
-                        struct Wheel *whl = NULL;
-                        char *post_commit = NULL;
-                        char *hash = NULL;
-                        if (strlist_count(tag_data) > 1) {
-                            post_commit = strlist_item(tag_data, 1);
-                            hash = strlist_item(tag_data, 2);
-                        }
-
-                        // We can't match on version here (index 0). The wheel's version is not guaranteed to be
-                        // equal to the tag; setuptools_scm auto-increments the value, the user can change it manually,
-                        // etc.
-                        whl = get_wheel_file(ctx->storage.wheel_artifact_dir, info->name,
-                                             (char *[]) {ctx->meta.python_compact, ctx->system.arch,
-                                                         "none", "any",
-                                                         post_commit, hash,
-                                                         NULL}, WHEEL_MATCH_ANY);
-
-                        guard_strlist_free(&tag_data);
-                        info->version = whl->version;
-                        sprintf(cmd + strlen(cmd), " '%s==%s'", info->name, whl->version);
-                    } else {
-                        sprintf(cmd + strlen(cmd), " '%s==%s'", info->name, info->version);
-                    }
-                } else {
-                    fprintf(stderr, "Deferred package '%s' is not present in the tested package list!\n", name);
-                    return -1;
-                }
-            } else {
-                if (startswith(name, "--") || startswith(name, "-")) {
-                    sprintf(cmd + strlen(cmd), " %s", name);
-                } else {
-                    sprintf(cmd + strlen(cmd), " '%s'", name);
-                }
-            }
-        }
-        int status = runner(cmd);
-        if (status) {
-            return status;
-        }
-    }
-    return 0;
-}
-
-void delivery_get_installer_url(struct Delivery *ctx, char *result) {
-    if (ctx->conda.installer_version) {
-        // Use version specified by configuration file
-        sprintf(result, "%s/%s-%s-%s-%s.sh", ctx->conda.installer_baseurl,
-                ctx->conda.installer_name,
-                ctx->conda.installer_version,
-                ctx->conda.installer_platform,
-                ctx->conda.installer_arch);
-    } else {
-        // Use latest installer
-        sprintf(result, "%s/%s-%s-%s.sh", ctx->conda.installer_baseurl,
-                ctx->conda.installer_name,
-                ctx->conda.installer_platform,
-                ctx->conda.installer_arch);
-    }
-
-}
-
-int delivery_get_installer(struct Delivery *ctx, char *installer_url) {
-    char script_path[PATH_MAX];
-    char *installer = path_basename(installer_url);
-
-    memset(script_path, 0, sizeof(script_path));
-    sprintf(script_path, "%s/%s", ctx->storage.tmpdir, installer);
-    if (access(script_path, F_OK)) {
-        // Script doesn't exist
-        long fetch_status = download(installer_url, script_path, NULL);
-        if (HTTP_ERROR(fetch_status) || fetch_status < 0) {
-            // download failed
-            return -1;
-        }
-    } else {
-        msg(STASIS_MSG_RESTRICT | STASIS_MSG_L3, "Skipped, installer already exists\n", script_path);
-    }
-
-    ctx->conda.installer_path = strdup(script_path);
-    if (!ctx->conda.installer_path) {
-        SYSERROR("Unable to duplicate script_path: '%s'", script_path);
-        return -1;
-    }
-
-    return 0;
-}
-
-int delivery_copy_conda_artifacts(struct Delivery *ctx) {
-    char cmd[STASIS_BUFSIZ];
-    char conda_build_dir[PATH_MAX];
-    char subdir[PATH_MAX];
-    memset(cmd, 0, sizeof(cmd));
-    memset(conda_build_dir, 0, sizeof(conda_build_dir));
-    memset(subdir, 0, sizeof(subdir));
-
-    sprintf(conda_build_dir, "%s/%s", ctx->storage.conda_install_prefix, "conda-bld");
-    // One must run conda build at least once to create the "conda-bld" directory.
-    // When this directory is missing there can be no build artifacts.
-    if (access(conda_build_dir, F_OK) < 0) {
-        msg(STASIS_MSG_RESTRICT | STASIS_MSG_WARN | STASIS_MSG_L3,
-            "Skipped: 'conda build' has never been executed.\n");
-        return 0;
-    }
-
-    snprintf(cmd, sizeof(cmd) - 1, "rsync -avi --progress %s/%s %s",
-             conda_build_dir,
-             ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR],
-             ctx->storage.conda_artifact_dir);
-
-    return system(cmd);
-}
-
-int delivery_copy_wheel_artifacts(struct Delivery *ctx) {
-    char cmd[PATH_MAX];
-    memset(cmd, 0, sizeof(cmd));
-    snprintf(cmd, sizeof(cmd) - 1, "rsync -avi --progress %s/*/dist/*.whl %s",
-             ctx->storage.build_sources_dir,
-             ctx->storage.wheel_artifact_dir);
-    return system(cmd);
-}
-
-int delivery_index_wheel_artifacts(struct Delivery *ctx) {
-    struct dirent *rec;
-    DIR *dp;
-    FILE *top_fp;
-
-    dp = opendir(ctx->storage.wheel_artifact_dir);
-    if (!dp) {
-        return -1;
-    }
-
-    // Generate a "dumb" local pypi index that is compatible with:
-    // pip install --extra-index-url
-    char top_index[PATH_MAX];
-    memset(top_index, 0, sizeof(top_index));
-    sprintf(top_index, "%s/index.html", ctx->storage.wheel_artifact_dir);
-    top_fp = fopen(top_index, "w+");
-    if (!top_fp) {
-        return -2;
-    }
-
-    while ((rec = readdir(dp)) != NULL) {
-        // skip directories
-        if (DT_REG == rec->d_type || !strcmp(rec->d_name, "..") || !strcmp(rec->d_name, ".")) {
-            continue;
-        }
-
-        FILE *bottom_fp;
-        char bottom_index[PATH_MAX * 2];
-        memset(bottom_index, 0, sizeof(bottom_index));
-        sprintf(bottom_index, "%s/%s/index.html", ctx->storage.wheel_artifact_dir, rec->d_name);
-        bottom_fp = fopen(bottom_index, "w+");
-        if (!bottom_fp) {
-            return -3;
-        }
-
-        if (globals.verbose) {
-            printf("+ %s\n", rec->d_name);
-        }
-        // Add record to top level index
-        fprintf(top_fp, "<a href=\"%s/\">%s</a><br/>\n", rec->d_name, rec->d_name);
-
-        char dpath[PATH_MAX * 2];
-        memset(dpath, 0, sizeof(dpath));
-        sprintf(dpath, "%s/%s", ctx->storage.wheel_artifact_dir, rec->d_name);
-        struct StrList *packages = listdir(dpath);
-        if (!packages) {
-            fclose(top_fp);
-            fclose(bottom_fp);
-            return -4;
-        }
-
-        for (size_t i = 0; i < strlist_count(packages); i++) {
-            char *package = strlist_item(packages, i);
-            if (!endswith(package, ".whl")) {
-                continue;
-            }
-            if (globals.verbose) {
-                printf("`- %s\n", package);
-            }
-            // Write record to bottom level index
-            fprintf(bottom_fp, "<a href=\"%s\">%s</a><br/>\n", package, package);
-        }
-        fclose(bottom_fp);
-
-        guard_strlist_free(&packages);
-    }
-    closedir(dp);
-    fclose(top_fp);
-    return 0;
-}
-
-void delivery_install_conda(char *install_script, char *conda_install_dir) {
-    struct Process proc;
-    memset(&proc, 0, sizeof(proc));
-
-    if (globals.conda_fresh_start) {
-        if (!access(conda_install_dir, F_OK)) {
-            // directory exists so remove it
-            if (rmtree(conda_install_dir)) {
-                perror("unable to remove previous installation");
-                exit(1);
-            }
-
-            // Proceed with the installation
-            // -b = batch mode (non-interactive)
-            char cmd[PATH_MAX] = {0};
-            snprintf(cmd, sizeof(cmd) - 1, "%s %s -b -p %s",
-                     find_program("bash"),
-                     install_script,
-                     conda_install_dir);
-            if (shell_safe(&proc, cmd)) {
-                fprintf(stderr, "conda installation failed\n");
-                exit(1);
-            }
-        } else {
-            // Proceed with the installation
-            // -b = batch mode (non-interactive)
-            char cmd[PATH_MAX] = {0};
-            snprintf(cmd, sizeof(cmd) - 1, "%s %s -b -p %s",
-                     find_program("bash"),
-                     install_script,
-                     conda_install_dir);
-            if (shell_safe(&proc, cmd)) {
-                fprintf(stderr, "conda installation failed\n");
-                exit(1);
-            }
-        }
-    } else {
-        msg(STASIS_MSG_L3, "Conda removal disabled by configuration\n");
-    }
-}
-
-void delivery_conda_enable(struct Delivery *ctx, char *conda_install_dir) {
-    if (conda_activate(conda_install_dir, "base")) {
-        fprintf(stderr, "conda activation failed\n");
-        exit(1);
-    }
-
-    // Setting the CONDARC environment variable appears to be the only consistent
-    // way to make sure the file is used. Not setting this variable leads to strange
-    // behavior, especially if a conda environment is already active when STASIS is loaded.
-    char rcpath[PATH_MAX];
-    sprintf(rcpath, "%s/%s", conda_install_dir, ".condarc");
-    setenv("CONDARC", rcpath, 1);
-    if (runtime_replace(&ctx->runtime.environ, __environ)) {
-        perror("unable to replace runtime environment after activating conda");
-        exit(1);
-    }
-
-    if (conda_setup_headless()) {
-        // no COE check. this call must succeed.
-        exit(1);
-    }
-}
-
-void delivery_defer_packages(struct Delivery *ctx, int type) {
-    struct StrList *dataptr = NULL;
-    struct StrList *deferred = NULL;
-    char *name = NULL;
-    char cmd[PATH_MAX];
-
-    memset(cmd, 0, sizeof(cmd));
-
-    char mode[10];
-    if (DEFER_CONDA == type) {
-        dataptr = ctx->conda.conda_packages;
-        deferred = ctx->conda.conda_packages_defer;
-        strcpy(mode, "conda");
-    } else if (DEFER_PIP == type) {
-        dataptr = ctx->conda.pip_packages;
-        deferred = ctx->conda.pip_packages_defer;
-        strcpy(mode, "pip");
-    }
-    msg(STASIS_MSG_L2, "Filtering %s packages by test definition...\n", mode);
-
-    struct StrList *filtered = NULL;
-    filtered = strlist_init();
-    for (size_t i = 0; i < strlist_count(dataptr); i++) {
-        int ignore_pkg = 0;
-
-        name = strlist_item(dataptr, i);
-        if (!strlen(name) || isblank(*name) || isspace(*name)) {
-            // no data
-            continue;
-        }
-        msg(STASIS_MSG_L3, "package '%s': ", name);
-
-        // Compile a list of packages that are *also* to be tested.
-        char *version;
-        char *spec_begin = strpbrk(name, "@~=<>!");
-        char *spec_end = spec_begin;
-        if (spec_end) {
-            // A version is present in the package name. Jump past operator(s).
-            while (*spec_end != '\0' && !isalnum(*spec_end)) {
-                spec_end++;
-            }
-        }
-
-        // When spec is present in name, set tests->version to the version detected in the name
-        for (size_t x = 0; x < sizeof(ctx->tests) / sizeof(ctx->tests[0]) && ctx->tests[x].name != NULL; x++) {
-            struct Test *test = &ctx->tests[x];
-            version = NULL;
+        // When spec is present in name, set tests->version to the version detected in the name
+        for (size_t x = 0; x < sizeof(ctx->tests) / sizeof(ctx->tests[0]) && ctx->tests[x].name != NULL; x++) {
+            struct Test *test = &ctx->tests[x];
+            version = NULL;
 
             char nametmp[1024] = {0};
             if (spec_end != NULL && spec_begin != NULL) {
@@ -1551,373 +288,6 @@ void delivery_defer_packages(struct Delivery *ctx, int type) {
     }
 }
 
-const char *release_header = "# delivery_name: %s\n"
-                             "# delivery_fmt: %s\n"
-                     "# creation_time: %s\n"
-                     "# conda_ident: %s\n"
-                     "# conda_build_ident: %s\n";
-
-char *delivery_get_release_header(struct Delivery *ctx) {
-    char output[STASIS_BUFSIZ];
-    char stamp[100];
-    strftime(stamp, sizeof(stamp) - 1, "%c", ctx->info.time_info);
-    sprintf(output, release_header,
-            ctx->info.release_name,
-            ctx->rules.release_fmt,
-            stamp,
-            ctx->conda.tool_version,
-            ctx->conda.tool_build_version);
-    return strdup(output);
-}
-
-int delivery_dump_metadata(struct Delivery *ctx) {
-    FILE *fp;
-    char filename[PATH_MAX];
-    sprintf(filename, "%s/meta-%s.stasis", ctx->storage.meta_dir, ctx->info.release_name);
-    fp = fopen(filename, "w+");
-    if (!fp) {
-        return -1;
-    }
-    if (globals.verbose) {
-        printf("%s\n", filename);
-    }
-    fprintf(fp, "name %s\n", ctx->meta.name);
-    fprintf(fp, "version %s\n", ctx->meta.version);
-    fprintf(fp, "rc %d\n", ctx->meta.rc);
-    fprintf(fp, "python %s\n", ctx->meta.python);
-    fprintf(fp, "python_compact %s\n", ctx->meta.python_compact);
-    fprintf(fp, "mission %s\n", ctx->meta.mission);
-    fprintf(fp, "codename %s\n", ctx->meta.codename ? ctx->meta.codename : "");
-    fprintf(fp, "platform %s %s %s %s\n",
-            ctx->system.platform[DELIVERY_PLATFORM],
-            ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR],
-            ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER],
-            ctx->system.platform[DELIVERY_PLATFORM_RELEASE]);
-    fprintf(fp, "arch %s\n", ctx->system.arch);
-    fprintf(fp, "time %s\n", ctx->info.time_str_epoch);
-    fprintf(fp, "release_fmt %s\n", ctx->rules.release_fmt);
-    fprintf(fp, "release_name %s\n", ctx->info.release_name);
-    fprintf(fp, "build_name_fmt %s\n", ctx->rules.build_name_fmt);
-    fprintf(fp, "build_name %s\n", ctx->info.build_name);
-    fprintf(fp, "build_number_fmt %s\n", ctx->rules.build_number_fmt);
-    fprintf(fp, "build_number %s\n", ctx->info.build_number);
-    fprintf(fp, "conda_installer_baseurl %s\n", ctx->conda.installer_baseurl);
-    fprintf(fp, "conda_installer_name %s\n", ctx->conda.installer_name);
-    fprintf(fp, "conda_installer_version %s\n", ctx->conda.installer_version);
-    fprintf(fp, "conda_installer_platform %s\n", ctx->conda.installer_platform);
-    fprintf(fp, "conda_installer_arch %s\n", ctx->conda.installer_arch);
-
-    fclose(fp);
-    return 0;
-}
-
-void delivery_rewrite_spec(struct Delivery *ctx, char *filename, unsigned stage) {
-    char output[PATH_MAX];
-    char *header = NULL;
-    char *tempfile = NULL;
-    FILE *tp = NULL;
-
-    if (stage == DELIVERY_REWRITE_SPEC_STAGE_1) {
-        header = delivery_get_release_header(ctx);
-        if (!header) {
-            msg(STASIS_MSG_ERROR, "failed to generate release header string\n", filename);
-            exit(1);
-        }
-        tempfile = xmkstemp(&tp, "w+");
-        if (!tempfile || !tp) {
-            msg(STASIS_MSG_ERROR, "%s: unable to create temporary file\n", strerror(errno));
-            exit(1);
-        }
-        fprintf(tp, "%s", header);
-
-        // Read the original file
-        char **contents = file_readlines(filename, 0, 0, NULL);
-        if (!contents) {
-            msg(STASIS_MSG_ERROR, "%s: unable to read %s", filename);
-            exit(1);
-        }
-
-        // Write temporary data
-        for (size_t i = 0; contents[i] != NULL; i++) {
-            if (startswith(contents[i], "channels:")) {
-                // Allow for additional conda channel injection
-                if (ctx->conda.conda_packages_defer && strlist_count(ctx->conda.conda_packages_defer)) {
-                    fprintf(tp, "%s  - @CONDA_CHANNEL@\n", contents[i]);
-                    continue;
-                }
-            } else if (strstr(contents[i], "- pip:")) {
-                if (ctx->conda.pip_packages_defer && strlist_count(ctx->conda.pip_packages_defer)) {
-                    // Allow for additional pip argument injection
-                    fprintf(tp, "%s      - @PIP_ARGUMENTS@\n", contents[i]);
-                    continue;
-                }
-            } else if (startswith(contents[i], "prefix:")) {
-                // Remove the prefix key
-                if (strstr(contents[i], "/") || strstr(contents[i], "\\")) {
-                    // path is on the same line as the key
-                    continue;
-                } else {
-                    // path is on the next line?
-                    if (contents[i + 1] && (strstr(contents[i + 1], "/") || strstr(contents[i + 1], "\\"))) {
-                        i++;
-                    }
-                    continue;
-                }
-            }
-            fprintf(tp, "%s", contents[i]);
-        }
-        GENERIC_ARRAY_FREE(contents);
-        guard_free(header);
-        fflush(tp);
-        fclose(tp);
-
-        // Replace the original file with our temporary data
-        if (copy2(tempfile, filename, CT_PERM) < 0) {
-            fprintf(stderr, "%s: could not rename '%s' to '%s'\n", strerror(errno), tempfile, filename);
-            exit(1);
-        }
-        remove(tempfile);
-        guard_free(tempfile);
-    } else if (globals.enable_rewrite_spec_stage_2 && stage == DELIVERY_REWRITE_SPEC_STAGE_2) {
-        // Replace "local" channel with the staging URL
-        if (ctx->storage.conda_staging_url) {
-            file_replace_text(filename, "@CONDA_CHANNEL@", ctx->storage.conda_staging_url, 0);
-        } else if (globals.jfrog.repo) {
-            sprintf(output, "%s/%s/%s/%s/packages/conda", globals.jfrog.url, globals.jfrog.repo, ctx->meta.mission, ctx->info.build_name);
-            file_replace_text(filename, "@CONDA_CHANNEL@", output, 0);
-        } else {
-            msg(STASIS_MSG_WARN, "conda_staging_dir is not configured. Using fallback: '%s'\n", ctx->storage.conda_artifact_dir);
-            file_replace_text(filename, "@CONDA_CHANNEL@", ctx->storage.conda_artifact_dir, 0);
-        }
-
-        if (ctx->storage.wheel_staging_url) {
-            file_replace_text(filename, "@PIP_ARGUMENTS@", ctx->storage.wheel_staging_url, 0);
-        } else if (globals.enable_artifactory && globals.jfrog.url && globals.jfrog.repo) {
-            sprintf(output, "--extra-index-url %s/%s/%s/%s/packages/wheels", globals.jfrog.url, globals.jfrog.repo, ctx->meta.mission, ctx->info.build_name);
-            file_replace_text(filename, "@PIP_ARGUMENTS@", output, 0);
-        } else {
-            msg(STASIS_MSG_WARN, "wheel_staging_dir is not configured. Using fallback: '%s'\n", ctx->storage.wheel_artifact_dir);
-            sprintf(output, "--extra-index-url file://%s", ctx->storage.wheel_artifact_dir);
-            file_replace_text(filename, "@PIP_ARGUMENTS@", output, 0);
-        }
-    }
-}
-
-int delivery_index_conda_artifacts(struct Delivery *ctx) {
-    return conda_index(ctx->storage.conda_artifact_dir);
-}
-
-void delivery_tests_run(struct Delivery *ctx) {
-    struct MultiProcessingPool *pool_parallel;
-    struct MultiProcessingPool *pool_serial;
-    struct MultiProcessingPool *pool_setup;
-    struct Process proc;
-    memset(&proc, 0, sizeof(proc));
-
-    if (!globals.workaround.conda_reactivate) {
-        globals.workaround.conda_reactivate = calloc(PATH_MAX, sizeof(*globals.workaround.conda_reactivate));
-    } else {
-        memset(globals.workaround.conda_reactivate, 0, PATH_MAX);
-    }
-    snprintf(globals.workaround.conda_reactivate, PATH_MAX - 1, "\nmamba activate ${CONDA_DEFAULT_ENV}\n");
-
-    if (!ctx->tests[0].name) {
-        msg(STASIS_MSG_WARN | STASIS_MSG_L2, "no tests are defined!\n");
-    } else {
-        pool_parallel = mp_pool_init("parallel", ctx->storage.tmpdir);
-        if (!pool_parallel) {
-            perror("mp_pool_init/parallel");
-            exit(1);
-        }
-        
-        pool_serial = mp_pool_init("serial", ctx->storage.tmpdir);
-        if (!pool_serial) {
-            perror("mp_pool_init/serial");
-            exit(1);
-        }
-
-        pool_setup = mp_pool_init("setup", ctx->storage.tmpdir);
-        if (!pool_setup) {
-            perror("mp_pool_init/setup");
-            exit(1);
-        }
-
-        const char *runner_cmd_fmt = "set -e -x\n%s\n";
-        for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) {
-            struct Test *test = &ctx->tests[i];
-            if (!test->name && !test->repository && !test->script) {
-                // skip unused test records
-                continue;
-            }
-            msg(STASIS_MSG_L2, "Executing tests for %s %s\n", test->name, test->version);
-            if (!test->script || !strlen(test->script)) {
-                msg(STASIS_MSG_WARN | STASIS_MSG_L3, "Nothing to do. To fix, declare a 'script' in section: [test:%s]\n",
-                    test->name);
-                continue;
-            }
-
-            char destdir[PATH_MAX];
-            sprintf(destdir, "%s/%s", ctx->storage.build_sources_dir, path_basename(test->repository));
-
-            if (!access(destdir, F_OK)) {
-                msg(STASIS_MSG_L3, "Purging repository %s\n", destdir);
-                if (rmtree(destdir)) {
-                    COE_CHECK_ABORT(1, "Unable to remove repository\n");
-                }
-            }
-            msg(STASIS_MSG_L3, "Cloning repository %s\n", test->repository);
-            if (!git_clone(&proc, test->repository, destdir, test->version)) {
-                test->repository_info_tag = strdup(git_describe(destdir));
-                test->repository_info_ref = strdup(git_rev_parse(destdir, "HEAD"));
-            } else {
-                COE_CHECK_ABORT(1, "Unable to clone repository\n");
-            }
-
-            if (test->repository_remove_tags && strlist_count(test->repository_remove_tags)) {
-                filter_repo_tags(destdir, test->repository_remove_tags);
-            }
-
-            if (pushd(destdir)) {
-                COE_CHECK_ABORT(1, "Unable to enter repository directory\n");
-            } else {
-                char *cmd = calloc(strlen(test->script) + STASIS_BUFSIZ, sizeof(*cmd));
-                if (!cmd) {
-                    SYSERROR("Unable to allocate test script buffer: %s", strerror(errno));
-                    exit(1);
-                }
-
-                msg(STASIS_MSG_L3, "Testing %s\n", test->name);
-                memset(&proc, 0, sizeof(proc));
-
-                strcpy(cmd, test->script);
-                char *cmd_rendered = tpl_render(cmd);
-                if (cmd_rendered) {
-                    if (strcmp(cmd_rendered, cmd) != 0) {
-                        strcpy(cmd, cmd_rendered);
-                        cmd[strlen(cmd_rendered) ? strlen(cmd_rendered) - 1 : 0] = 0;
-                    }
-                    guard_free(cmd_rendered);
-                } else {
-                    SYSERROR("An error occurred while rendering the following:\n%s", cmd);
-                    exit(1);
-                }
-
-                if (test->disable) {
-                    msg(STASIS_MSG_L2, "Script execution disabled by configuration\n", test->name);
-                    guard_free(cmd);
-                    continue;
-                }
-
-                char runner_cmd[0xFFFF] = {0};
-                char pool_name[100] = "parallel";
-                struct MultiProcessingTask *task = NULL;
-                struct MultiProcessingPool *pool = pool_parallel;
-                if (!test->parallel) {
-                    pool = pool_serial;
-                    memset(pool_name, 0, sizeof(pool_name));
-                    strcpy(pool_name, "serial");
-                }
-
-                sprintf(runner_cmd, runner_cmd_fmt, cmd);
-                task = mp_pool_task(pool, test->name, runner_cmd);
-                if (!task) {
-                    SYSERROR("Failed to add task to %s pool: %s", pool_name, runner_cmd);
-                    popd();
-                    if (!globals.continue_on_error) {
-                        tpl_free();
-                        delivery_free(ctx);
-                        globals_free();
-                    }
-                    exit(1);
-                }
-                guard_free(cmd);
-                popd();
-
-            }
-        }
-
-        // Configure "script_setup" tasks
-        // Directories should exist now, so no need to go through initializing everything all over again.
-        for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) {
-            struct Test *test = &ctx->tests[i];
-            if (test->script_setup) {
-                char destdir[PATH_MAX];
-                sprintf(destdir, "%s/%s", ctx->storage.build_sources_dir, path_basename(test->repository));
-                if (access(destdir, F_OK)) {
-                    SYSERROR("%s: %s", destdir, strerror(errno));
-                    exit(1);
-                }
-                if (!pushd(destdir)) {
-                    char *cmd = calloc(strlen(test->script_setup) + STASIS_BUFSIZ, sizeof(*cmd));
-                    if (!cmd) {
-                        SYSERROR("Unable to allocate test script_setup buffer: %s", strerror(errno));
-                        exit(1);
-                    }
-
-                    strcpy(cmd, test->script_setup);
-                    char *cmd_rendered = tpl_render(cmd);
-                    if (cmd_rendered) {
-                        if (strcmp(cmd_rendered, cmd) != 0) {
-                            strcpy(cmd, cmd_rendered);
-                            cmd[strlen(cmd_rendered) ? strlen(cmd_rendered) - 1 : 0] = 0;
-                        }
-                        guard_free(cmd_rendered);
-                    } else {
-                        SYSERROR("An error occurred while rendering the following:\n%s", cmd);
-                        exit(1);
-                    }
-
-                    struct MultiProcessingPool *pool = pool_setup;
-                    struct MultiProcessingTask *task = NULL;
-                    char runner_cmd[0xFFFF] = {0};
-                    sprintf(runner_cmd, runner_cmd_fmt, cmd);
-
-                    task = mp_pool_task(pool, test->name, runner_cmd);
-                    if (!task) {
-                        SYSERROR("Failed to add task %s to setup pool: %s", test->name, runner_cmd);
-                        popd();
-                        if (!globals.continue_on_error) {
-                            tpl_free();
-                            delivery_free(ctx);
-                            globals_free();
-                        }
-                        exit(1);
-                    }
-                    guard_free(cmd);
-                    popd();
-                }
-            }
-        }
-
-        size_t opt_flags = 0;
-        if (globals.parallel_fail_fast) {
-            opt_flags |= MP_POOL_FAIL_FAST;
-        }
-
-        int pool_status = -1;
-        if (pool_setup->num_used) {
-            pool_status = mp_pool_join(pool_setup, 1, opt_flags);
-            mp_pool_show_summary(pool_setup);
-            COE_CHECK_ABORT(pool_status != 0, "Failure in setup task pool");
-            mp_pool_free(&pool_setup);
-        }
-
-        if (pool_parallel->num_used) {
-            pool_status = mp_pool_join(pool_parallel, globals.cpu_limit, opt_flags);
-            mp_pool_show_summary(pool_parallel);
-            COE_CHECK_ABORT(pool_status != 0, "Failure in parallel task pool");
-            mp_pool_free(&pool_parallel);
-        }
-
-        if (pool_serial->num_used) {
-            pool_status = mp_pool_join(pool_serial, 1, opt_flags);
-            mp_pool_show_summary(pool_serial);
-            COE_CHECK_ABORT(pool_serial != 0, "Failure in serial task pool");
-            mp_pool_free(&pool_serial);
-        }
-    }
-}
-
 void delivery_gather_tool_versions(struct Delivery *ctx) {
     int status = 0;
 
@@ -1931,388 +301,3 @@ void delivery_gather_tool_versions(struct Delivery *ctx) {
         strip(ctx->conda.tool_version);
 }
 
-int delivery_init_artifactory(struct Delivery *ctx) {
-    int status = 0;
-    char dest[PATH_MAX] = {0};
-    char filepath[PATH_MAX] = {0};
-    snprintf(dest, sizeof(dest) - 1, "%s/bin", ctx->storage.tools_dir);
-    snprintf(filepath, sizeof(dest) - 1, "%s/bin/jf", ctx->storage.tools_dir);
-
-    if (!access(filepath, F_OK)) {
-        // already have it
-        msg(STASIS_MSG_L3, "Skipped download, %s already exists\n", filepath);
-        goto delivery_init_artifactory_envsetup;
-    }
-
-    char *platform = ctx->system.platform[DELIVERY_PLATFORM];
-    msg(STASIS_MSG_L3, "Downloading %s for %s %s\n", globals.jfrog.remote_filename, platform, ctx->system.arch);
-    if ((status = artifactory_download_cli(dest,
-            globals.jfrog.jfrog_artifactory_base_url,
-            globals.jfrog.jfrog_artifactory_product,
-            globals.jfrog.cli_major_ver,
-            globals.jfrog.version,
-            platform,
-            ctx->system.arch,
-            globals.jfrog.remote_filename))) {
-        remove(filepath);
-    }
-
-    delivery_init_artifactory_envsetup:
-    // CI (ridiculously generic, why?) disables interactive prompts and progress bar output
-    setenv("CI", "1", 1);
-
-    // JFROG_CLI_HOME_DIR is where .jfrog is stored
-    char path[PATH_MAX] = {0};
-    snprintf(path, sizeof(path) - 1, "%s/.jfrog", ctx->storage.build_dir);
-    setenv("JFROG_CLI_HOME_DIR", path, 1);
-
-    // JFROG_CLI_TEMP_DIR is where the obvious is stored
-    setenv("JFROG_CLI_TEMP_DIR", ctx->storage.tmpdir, 1);
-    return status;
-}
-
-int delivery_artifact_upload(struct Delivery *ctx) {
-    int status = 0;
-
-    if (jfrt_auth_init(&ctx->deploy.jfrog_auth)) {
-        fprintf(stderr, "Failed to initialize Artifactory authentication context\n");
-        return -1;
-    }
-
-    for (size_t i = 0; i < sizeof(ctx->deploy.jfrog) / sizeof(*ctx->deploy.jfrog); i++) {
-        if (!ctx->deploy.jfrog[i].files || !ctx->deploy.jfrog[i].dest) {
-            break;
-        }
-        jfrt_upload_init(&ctx->deploy.jfrog[i].upload_ctx);
-
-        if (!globals.jfrog.repo) {
-            msg(STASIS_MSG_WARN, "Artifactory repository path is not configured!\n");
-            fprintf(stderr, "set STASIS_JF_REPO environment variable...\nOr append to configuration file:\n\n");
-            fprintf(stderr, "[deploy:artifactory]\nrepo = example/generic/repo/path\n\n");
-            status++;
-            break;
-        } else if (!ctx->deploy.jfrog[i].repo) {
-            ctx->deploy.jfrog[i].repo = strdup(globals.jfrog.repo);
-        }
-
-        if (!ctx->deploy.jfrog[i].repo || isempty(ctx->deploy.jfrog[i].repo) || !strlen(ctx->deploy.jfrog[i].repo)) {
-            // Unlikely to trigger if the config parser is working correctly
-            msg(STASIS_MSG_ERROR, "Artifactory repository path is empty. Cannot continue.\n");
-            status++;
-            break;
-        }
-
-        ctx->deploy.jfrog[i].upload_ctx.workaround_parent_only = true;
-        ctx->deploy.jfrog[i].upload_ctx.build_name = ctx->info.build_name;
-        ctx->deploy.jfrog[i].upload_ctx.build_number = ctx->info.build_number;
-
-        char files[PATH_MAX];
-        char dest[PATH_MAX];  // repo + remote dir
-
-        if (jfrog_cli_rt_ping(&ctx->deploy.jfrog_auth)) {
-            msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Unable to contact artifactory server: %s\n", ctx->deploy.jfrog_auth.url);
-            return -1;
-        }
-
-        if (strlist_count(ctx->deploy.jfrog[i].files)) {
-            for (size_t f = 0; f < strlist_count(ctx->deploy.jfrog[i].files); f++) {
-                memset(dest, 0, sizeof(dest));
-                memset(files, 0, sizeof(files));
-                snprintf(dest, sizeof(dest) - 1, "%s/%s", ctx->deploy.jfrog[i].repo, ctx->deploy.jfrog[i].dest);
-                snprintf(files, sizeof(files) - 1, "%s", strlist_item(ctx->deploy.jfrog[i].files, f));
-                status += jfrog_cli_rt_upload(&ctx->deploy.jfrog_auth, &ctx->deploy.jfrog[i].upload_ctx, files, dest);
-            }
-        }
-    }
-
-    if (globals.enable_artifactory_build_info) {
-        if (!status && ctx->deploy.jfrog[0].files && ctx->deploy.jfrog[0].dest) {
-            jfrog_cli_rt_build_collect_env(&ctx->deploy.jfrog_auth, ctx->deploy.jfrog[0].upload_ctx.build_name,
-                                           ctx->deploy.jfrog[0].upload_ctx.build_number);
-            jfrog_cli_rt_build_publish(&ctx->deploy.jfrog_auth, ctx->deploy.jfrog[0].upload_ctx.build_name,
-                                       ctx->deploy.jfrog[0].upload_ctx.build_number);
-        }
-    } else {
-        msg(STASIS_MSG_WARN | STASIS_MSG_L2, "Artifactory build info upload is disabled by CLI argument\n");
-    }
-
-    return status;
-}
-
-int delivery_mission_render_files(struct Delivery *ctx) {
-    if (!ctx->storage.mission_dir) {
-        fprintf(stderr, "Mission directory is not configured. Context not initialized?\n");
-        return -1;
-    }
-    struct Data {
-        char *src;
-        char *dest;
-    } data;
-    struct INIFILE *cfg = ctx->_stasis_ini_fp.mission;
-    union INIVal val;
-
-    memset(&data, 0, sizeof(data));
-    data.src = calloc(PATH_MAX, sizeof(*data.src));
-    if (!data.src) {
-        perror("data.src");
-        return -1;
-    }
-
-    for (size_t i = 0; i < cfg->section_count; i++) {
-        char *section_name = cfg->section[i]->key;
-        if (!startswith(section_name, "template:")) {
-            continue;
-        }
-        val.as_char_p = strchr(section_name, ':') + 1;
-        if (val.as_char_p && isempty(val.as_char_p)) {
-            guard_free(data.src);
-            return 1;
-        }
-        sprintf(data.src, "%s/%s/%s", ctx->storage.mission_dir, ctx->meta.mission, val.as_char_p);
-        msg(STASIS_MSG_L2, "%s\n", data.src);
-
-        int err = 0;
-        data.dest = ini_getval_str(cfg, section_name, "destination", INI_READ_RENDER, &err);
-
-        char *contents;
-        struct stat st;
-        if (lstat(data.src, &st)) {
-            perror(data.src);
-            guard_free(data.dest);
-            continue;
-        }
-
-        contents = calloc(st.st_size + 1, sizeof(*contents));
-        if (!contents) {
-            perror("template file contents");
-            guard_free(data.dest);
-            continue;
-        }
-
-        FILE *fp;
-        fp = fopen(data.src, "rb");
-        if (!fp) {
-            perror(data.src);
-            guard_free(contents);
-            guard_free(data.dest);
-            continue;
-        }
-
-        if (fread(contents, st.st_size, sizeof(*contents), fp) < 1) {
-            perror("while reading template file");
-            guard_free(contents);
-            guard_free(data.dest);
-            fclose(fp);
-            continue;
-        }
-        fclose(fp);
-
-        msg(STASIS_MSG_L3, "Writing %s\n", data.dest);
-        if (tpl_render_to_file(contents, data.dest)) {
-            guard_free(contents);
-            guard_free(data.dest);
-            continue;
-        }
-        guard_free(contents);
-        guard_free(data.dest);
-    }
-
-    guard_free(data.src);
-    return 0;
-}
-
-int delivery_docker(struct Delivery *ctx) {
-    if (!docker_capable(&ctx->deploy.docker.capabilities)) {
-        return -1;
-    }
-    char tag[STASIS_NAME_MAX];
-    char args[PATH_MAX];
-    int has_registry = ctx->deploy.docker.registry != NULL;
-    size_t total_tags = strlist_count(ctx->deploy.docker.tags);
-    size_t total_build_args = strlist_count(ctx->deploy.docker.build_args);
-
-    if (!has_registry) {
-        msg(STASIS_MSG_WARN | STASIS_MSG_L2, "No docker registry defined. You will need to manually retag the resulting image.\n");
-    }
-
-    if (!total_tags) {
-        char default_tag[PATH_MAX];
-        msg(STASIS_MSG_WARN | STASIS_MSG_L2, "No docker tags defined by configuration. Generating default tag(s).\n");
-        // generate local tag
-        memset(default_tag, 0, sizeof(default_tag));
-        sprintf(default_tag, "%s:%s-py%s", ctx->meta.name, ctx->info.build_name, ctx->meta.python_compact);
-        tolower_s(default_tag);
-
-        // Add tag
-        ctx->deploy.docker.tags = strlist_init();
-        strlist_append(&ctx->deploy.docker.tags, default_tag);
-
-        if (has_registry) {
-            // generate tag for target registry
-            memset(default_tag, 0, sizeof(default_tag));
-            sprintf(default_tag, "%s/%s:%s-py%s", ctx->deploy.docker.registry, ctx->meta.name, ctx->info.build_number, ctx->meta.python_compact);
-            tolower_s(default_tag);
-
-            // Add tag
-            strlist_append(&ctx->deploy.docker.tags, default_tag);
-        }
-        // regenerate total tag available
-        total_tags = strlist_count(ctx->deploy.docker.tags);
-    }
-
-    memset(args, 0, sizeof(args));
-
-    // Append image tags to command
-    for (size_t i = 0; i < total_tags; i++) {
-        char *tag_orig = strlist_item(ctx->deploy.docker.tags, i);
-        strcpy(tag, tag_orig);
-        docker_sanitize_tag(tag);
-        sprintf(args + strlen(args), " -t \"%s\" ", tag);
-    }
-
-    // Append build arguments to command (i.e. --build-arg "key=value"
-    for (size_t i = 0; i < total_build_args; i++) {
-        char *build_arg = strlist_item(ctx->deploy.docker.build_args, i);
-        if (!build_arg) {
-            break;
-        }
-        sprintf(args + strlen(args), " --build-arg \"%s\" ", build_arg);
-    }
-
-    // Build the image
-    char delivery_file[PATH_MAX];
-    char dest[PATH_MAX];
-    char rsync_cmd[PATH_MAX * 2];
-    memset(delivery_file, 0, sizeof(delivery_file));
-    memset(dest, 0, sizeof(dest));
-
-    sprintf(delivery_file, "%s/%s.yml", ctx->storage.delivery_dir, ctx->info.release_name);
-    if (access(delivery_file, F_OK) < 0) {
-        fprintf(stderr, "docker build cannot proceed without delivery file: %s\n", delivery_file);
-        return -1;
-    }
-
-    sprintf(dest, "%s/%s.yml", ctx->storage.build_docker_dir, ctx->info.release_name);
-    if (copy2(delivery_file, dest, CT_PERM)) {
-        fprintf(stderr, "Failed to copy delivery file to %s: %s\n", dest, strerror(errno));
-        return -1;
-    }
-
-    memset(dest, 0, sizeof(dest));
-    sprintf(dest, "%s/packages", ctx->storage.build_docker_dir);
-
-    msg(STASIS_MSG_L2, "Copying conda packages\n");
-    memset(rsync_cmd, 0, sizeof(rsync_cmd));
-    sprintf(rsync_cmd, "rsync -avi --progress '%s' '%s'", ctx->storage.conda_artifact_dir, dest);
-    if (system(rsync_cmd)) {
-        fprintf(stderr, "Failed to copy conda artifacts to docker build directory\n");
-        return -1;
-    }
-
-    msg(STASIS_MSG_L2, "Copying wheel packages\n");
-    memset(rsync_cmd, 0, sizeof(rsync_cmd));
-    sprintf(rsync_cmd, "rsync -avi --progress '%s' '%s'", ctx->storage.wheel_artifact_dir, dest);
-    if (system(rsync_cmd)) {
-        fprintf(stderr, "Failed to copy wheel artifactory to docker build directory\n");
-    }
-
-    if (docker_build(ctx->storage.build_docker_dir, args, ctx->deploy.docker.capabilities.build)) {
-        return -1;
-    }
-
-    // Test the image
-    // All tags point back to the same image so test the first one we see
-    // regardless of how many are defined
-    strcpy(tag, strlist_item(ctx->deploy.docker.tags, 0));
-    docker_sanitize_tag(tag);
-
-    msg(STASIS_MSG_L2, "Executing image test script for %s\n", tag);
-    if (ctx->deploy.docker.test_script) {
-        if (isempty(ctx->deploy.docker.test_script)) {
-            msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "Image test script has no content\n");
-        } else {
-            int state;
-            if ((state = docker_script(tag, ctx->deploy.docker.test_script, 0))) {
-                msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "Non-zero exit (%d) from test script. %s image archive will not be generated.\n", state >> 8, tag);
-                // test failed -- don't save the image
-                return -1;
-            }
-        }
-    } else {
-        msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "No image test script defined\n");
-    }
-
-    // Test successful, save image
-    if (docker_save(path_basename(tag), ctx->storage.docker_artifact_dir, ctx->deploy.docker.image_compression)) {
-        // save failed
-        return -1;
-    }
-
-    return 0;
-}
-
-int delivery_fixup_test_results(struct Delivery *ctx) {
-    struct dirent *rec;
-    DIR *dp;
-
-    dp = opendir(ctx->storage.results_dir);
-    if (!dp) {
-        perror(ctx->storage.results_dir);
-        return -1;
-    }
-
-    while ((rec = readdir(dp)) != NULL) {
-        char path[PATH_MAX];
-        memset(path, 0, sizeof(path));
-
-        if (!strcmp(rec->d_name, ".") || !strcmp(rec->d_name, "..")) {
-            continue;
-        } else if (!endswith(rec->d_name, ".xml")) {
-            continue;
-        }
-
-        sprintf(path, "%s/%s", ctx->storage.results_dir, rec->d_name);
-        msg(STASIS_MSG_L3, "%s\n", rec->d_name);
-        if (xml_pretty_print_in_place(path, STASIS_XML_PRETTY_PRINT_PROG, STASIS_XML_PRETTY_PRINT_ARGS)) {
-            msg(STASIS_MSG_L3 | STASIS_MSG_WARN, "Failed to rewrite file '%s'\n", rec->d_name);
-        }
-    }
-
-    closedir(dp);
-    return 0;
-}
-
-int delivery_exists(struct Delivery *ctx) {
-    int release_exists = 0;
-    char release_pattern[PATH_MAX] = {0};
-    sprintf(release_pattern, "*%s*", ctx->info.release_name);
-
-    if (globals.enable_artifactory) {
-        if (jfrt_auth_init(&ctx->deploy.jfrog_auth)) {
-            fprintf(stderr, "Failed to initialize Artifactory authentication context\n");
-            return -1;  // error
-        }
-
-        struct JFRT_Search search = {.fail_no_op = true};
-        release_exists = jfrog_cli_rt_search(&ctx->deploy.jfrog_auth, &search, globals.jfrog.repo, release_pattern);
-        if (release_exists != 2) {
-            if (!globals.enable_overwrite && !release_exists) {
-                // --fail_no_op returns 2 on failure
-                // without: it returns an empty list "[]" and exit code 0
-                return 1;  // found
-            }
-        }
-    } else {
-        struct StrList *files = listdir(ctx->storage.delivery_dir);
-        for (size_t i = 0; i < strlist_count(files); i++) {
-            char *filename = strlist_item(files, i);
-            release_exists = fnmatch(release_pattern, filename, FNM_PATHNAME);
-            if (!globals.enable_overwrite && !release_exists) {
-                guard_strlist_free(&files);
-                return 1;  // found
-            }
-        }
-        guard_strlist_free(&files);
-    }
-    return 0;  // not found
-}
diff --git a/src/delivery_artifactory.c b/src/delivery_artifactory.c
new file mode 100644
index 0000000..27f4823
--- /dev/null
+++ b/src/delivery_artifactory.c
@@ -0,0 +1,192 @@
+#include "delivery.h"
+
+int delivery_init_artifactory(struct Delivery *ctx) {
+    int status = 0;
+    char dest[PATH_MAX] = {0};
+    char filepath[PATH_MAX] = {0};
+    snprintf(dest, sizeof(dest) - 1, "%s/bin", ctx->storage.tools_dir);
+    snprintf(filepath, sizeof(dest) - 1, "%s/bin/jf", ctx->storage.tools_dir);
+
+    if (!access(filepath, F_OK)) {
+        // already have it
+        msg(STASIS_MSG_L3, "Skipped download, %s already exists\n", filepath);
+        goto delivery_init_artifactory_envsetup;
+    }
+
+    char *platform = ctx->system.platform[DELIVERY_PLATFORM];
+    msg(STASIS_MSG_L3, "Downloading %s for %s %s\n", globals.jfrog.remote_filename, platform, ctx->system.arch);
+    if ((status = artifactory_download_cli(dest,
+                                           globals.jfrog.jfrog_artifactory_base_url,
+                                           globals.jfrog.jfrog_artifactory_product,
+                                           globals.jfrog.cli_major_ver,
+                                           globals.jfrog.version,
+                                           platform,
+                                           ctx->system.arch,
+                                           globals.jfrog.remote_filename))) {
+        remove(filepath);
+    }
+
+    delivery_init_artifactory_envsetup:
+    // CI (ridiculously generic, why?) disables interactive prompts and progress bar output
+    setenv("CI", "1", 1);
+
+    // JFROG_CLI_HOME_DIR is where .jfrog is stored
+    char path[PATH_MAX] = {0};
+    snprintf(path, sizeof(path) - 1, "%s/.jfrog", ctx->storage.build_dir);
+    setenv("JFROG_CLI_HOME_DIR", path, 1);
+
+    // JFROG_CLI_TEMP_DIR is where the obvious is stored
+    setenv("JFROG_CLI_TEMP_DIR", ctx->storage.tmpdir, 1);
+    return status;
+}
+
+int delivery_artifact_upload(struct Delivery *ctx) {
+    int status = 0;
+
+    if (jfrt_auth_init(&ctx->deploy.jfrog_auth)) {
+        fprintf(stderr, "Failed to initialize Artifactory authentication context\n");
+        return -1;
+    }
+
+    for (size_t i = 0; i < sizeof(ctx->deploy.jfrog) / sizeof(*ctx->deploy.jfrog); i++) {
+        if (!ctx->deploy.jfrog[i].files || !ctx->deploy.jfrog[i].dest) {
+            break;
+        }
+        jfrt_upload_init(&ctx->deploy.jfrog[i].upload_ctx);
+
+        if (!globals.jfrog.repo) {
+            msg(STASIS_MSG_WARN, "Artifactory repository path is not configured!\n");
+            fprintf(stderr, "set STASIS_JF_REPO environment variable...\nOr append to configuration file:\n\n");
+            fprintf(stderr, "[deploy:artifactory]\nrepo = example/generic/repo/path\n\n");
+            status++;
+            break;
+        } else if (!ctx->deploy.jfrog[i].repo) {
+            ctx->deploy.jfrog[i].repo = strdup(globals.jfrog.repo);
+        }
+
+        if (!ctx->deploy.jfrog[i].repo || isempty(ctx->deploy.jfrog[i].repo) || !strlen(ctx->deploy.jfrog[i].repo)) {
+            // Unlikely to trigger if the config parser is working correctly
+            msg(STASIS_MSG_ERROR, "Artifactory repository path is empty. Cannot continue.\n");
+            status++;
+            break;
+        }
+
+        ctx->deploy.jfrog[i].upload_ctx.workaround_parent_only = true;
+        ctx->deploy.jfrog[i].upload_ctx.build_name = ctx->info.build_name;
+        ctx->deploy.jfrog[i].upload_ctx.build_number = ctx->info.build_number;
+
+        char files[PATH_MAX];
+        char dest[PATH_MAX];  // repo + remote dir
+
+        if (jfrog_cli_rt_ping(&ctx->deploy.jfrog_auth)) {
+            msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Unable to contact artifactory server: %s\n", ctx->deploy.jfrog_auth.url);
+            return -1;
+        }
+
+        if (strlist_count(ctx->deploy.jfrog[i].files)) {
+            for (size_t f = 0; f < strlist_count(ctx->deploy.jfrog[i].files); f++) {
+                memset(dest, 0, sizeof(dest));
+                memset(files, 0, sizeof(files));
+                snprintf(dest, sizeof(dest) - 1, "%s/%s", ctx->deploy.jfrog[i].repo, ctx->deploy.jfrog[i].dest);
+                snprintf(files, sizeof(files) - 1, "%s", strlist_item(ctx->deploy.jfrog[i].files, f));
+                status += jfrog_cli_rt_upload(&ctx->deploy.jfrog_auth, &ctx->deploy.jfrog[i].upload_ctx, files, dest);
+            }
+        }
+    }
+
+    if (globals.enable_artifactory_build_info) {
+        if (!status && ctx->deploy.jfrog[0].files && ctx->deploy.jfrog[0].dest) {
+            jfrog_cli_rt_build_collect_env(&ctx->deploy.jfrog_auth, ctx->deploy.jfrog[0].upload_ctx.build_name,
+                                           ctx->deploy.jfrog[0].upload_ctx.build_number);
+            jfrog_cli_rt_build_publish(&ctx->deploy.jfrog_auth, ctx->deploy.jfrog[0].upload_ctx.build_name,
+                                       ctx->deploy.jfrog[0].upload_ctx.build_number);
+        }
+    } else {
+        msg(STASIS_MSG_WARN | STASIS_MSG_L2, "Artifactory build info upload is disabled by CLI argument\n");
+    }
+
+    return status;
+}
+
+int delivery_mission_render_files(struct Delivery *ctx) {
+    if (!ctx->storage.mission_dir) {
+        fprintf(stderr, "Mission directory is not configured. Context not initialized?\n");
+        return -1;
+    }
+    struct Data {
+        char *src;
+        char *dest;
+    } data;
+    struct INIFILE *cfg = ctx->_stasis_ini_fp.mission;
+    union INIVal val;
+
+    memset(&data, 0, sizeof(data));
+    data.src = calloc(PATH_MAX, sizeof(*data.src));
+    if (!data.src) {
+        perror("data.src");
+        return -1;
+    }
+
+    for (size_t i = 0; i < cfg->section_count; i++) {
+        char *section_name = cfg->section[i]->key;
+        if (!startswith(section_name, "template:")) {
+            continue;
+        }
+        val.as_char_p = strchr(section_name, ':') + 1;
+        if (val.as_char_p && isempty(val.as_char_p)) {
+            guard_free(data.src);
+            return 1;
+        }
+        sprintf(data.src, "%s/%s/%s", ctx->storage.mission_dir, ctx->meta.mission, val.as_char_p);
+        msg(STASIS_MSG_L2, "%s\n", data.src);
+
+        int err = 0;
+        data.dest = ini_getval_str(cfg, section_name, "destination", INI_READ_RENDER, &err);
+
+        char *contents;
+        struct stat st;
+        if (lstat(data.src, &st)) {
+            perror(data.src);
+            guard_free(data.dest);
+            continue;
+        }
+
+        contents = calloc(st.st_size + 1, sizeof(*contents));
+        if (!contents) {
+            perror("template file contents");
+            guard_free(data.dest);
+            continue;
+        }
+
+        FILE *fp;
+        fp = fopen(data.src, "rb");
+        if (!fp) {
+            perror(data.src);
+            guard_free(contents);
+            guard_free(data.dest);
+            continue;
+        }
+
+        if (fread(contents, st.st_size, sizeof(*contents), fp) < 1) {
+            perror("while reading template file");
+            guard_free(contents);
+            guard_free(data.dest);
+            fclose(fp);
+            continue;
+        }
+        fclose(fp);
+
+        msg(STASIS_MSG_L3, "Writing %s\n", data.dest);
+        if (tpl_render_to_file(contents, data.dest)) {
+            guard_free(contents);
+            guard_free(data.dest);
+            continue;
+        }
+        guard_free(contents);
+        guard_free(data.dest);
+    }
+
+    guard_free(data.src);
+    return 0;
+}
+
diff --git a/src/delivery_build.c b/src/delivery_build.c
new file mode 100644
index 0000000..6a8d371
--- /dev/null
+++ b/src/delivery_build.c
@@ -0,0 +1,180 @@
+#include <fnmatch.h>
+#include "delivery.h"
+
+int delivery_build_recipes(struct Delivery *ctx) {
+    for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) {
+        char *recipe_dir = NULL;
+        if (ctx->tests[i].build_recipe) { // build a conda recipe
+            int recipe_type;
+            int status;
+            if (recipe_clone(ctx->storage.build_recipes_dir, ctx->tests[i].build_recipe, NULL, &recipe_dir)) {
+                fprintf(stderr, "Encountered an issue while cloning recipe for: %s\n", ctx->tests[i].name);
+                return -1;
+            }
+            recipe_type = recipe_get_type(recipe_dir);
+            pushd(recipe_dir);
+            {
+                if (RECIPE_TYPE_ASTROCONDA == recipe_type) {
+                    pushd(path_basename(ctx->tests[i].repository));
+                } else if (RECIPE_TYPE_CONDA_FORGE == recipe_type) {
+                    pushd("recipe");
+                }
+
+                char recipe_version[100];
+                char recipe_buildno[100];
+                char recipe_git_url[PATH_MAX];
+                char recipe_git_rev[PATH_MAX];
+
+                //sprintf(recipe_version, "{%% set version = GIT_DESCRIBE_TAG ~ \".dev\" ~ GIT_DESCRIBE_NUMBER ~ \"+\" ~ GIT_DESCRIBE_HASH %%}");
+                //sprintf(recipe_git_url, "  git_url: %s", ctx->tests[i].repository);
+                //sprintf(recipe_git_rev, "  git_rev: %s", ctx->tests[i].version);
+                // TODO: Conditionally download archives if github.com is the origin. Else, use raw git_* keys ^^^
+                sprintf(recipe_version, "{%% set version = \"%s\" %%}", ctx->tests[i].repository_info_tag ? ctx->tests[i].repository_info_tag : ctx->tests[i].version);
+                sprintf(recipe_git_url, "  url: %s/archive/refs/tags/{{ version }}.tar.gz", ctx->tests[i].repository);
+                strcpy(recipe_git_rev, "");
+                sprintf(recipe_buildno, "  number: 0");
+
+                unsigned flags = REPLACE_TRUNCATE_AFTER_MATCH;
+                //file_replace_text("meta.yaml", "{% set version = ", recipe_version);
+                if (ctx->meta.final) {
+                    sprintf(recipe_version, "{%% set version = \"%s\" %%}", ctx->tests[i].version);
+                    // TODO: replace sha256 of tagged archive
+                    // TODO: leave the recipe unchanged otherwise. in theory this should produce the same conda package hash as conda forge.
+                    // For now, remove the sha256 requirement
+                    file_replace_text("meta.yaml", "sha256:", "\n", flags);
+                } else {
+                    file_replace_text("meta.yaml", "{% set version = ", recipe_version, flags);
+                    file_replace_text("meta.yaml", "  url:", recipe_git_url, flags);
+                    //file_replace_text("meta.yaml", "sha256:", recipe_git_rev);
+                    file_replace_text("meta.yaml", "  sha256:", "\n", flags);
+                    file_replace_text("meta.yaml", "  number:", recipe_buildno, flags);
+                }
+
+                char command[PATH_MAX];
+                if (RECIPE_TYPE_CONDA_FORGE == recipe_type) {
+                    char arch[STASIS_NAME_MAX] = {0};
+                    char platform[STASIS_NAME_MAX] = {0};
+
+                    strcpy(platform, ctx->system.platform[DELIVERY_PLATFORM]);
+                    if (strstr(platform, "Darwin")) {
+                        memset(platform, 0, sizeof(platform));
+                        strcpy(platform, "osx");
+                    }
+                    tolower_s(platform);
+                    if (strstr(ctx->system.arch, "arm64")) {
+                        strcpy(arch, "arm64");
+                    } else if (strstr(ctx->system.arch, "64")) {
+                        strcpy(arch, "64");
+                    } else {
+                        strcat(arch, "32"); // blind guess
+                    }
+                    tolower_s(arch);
+
+                    sprintf(command, "mambabuild --python=%s -m ../.ci_support/%s_%s_.yaml .",
+                            ctx->meta.python, platform, arch);
+                } else {
+                    sprintf(command, "mambabuild --python=%s .", ctx->meta.python);
+                }
+                status = conda_exec(command);
+                if (status) {
+                    return -1;
+                }
+
+                if (RECIPE_TYPE_GENERIC != recipe_type) {
+                    popd();
+                }
+                popd();
+            }
+        }
+        if (recipe_dir) {
+            guard_free(recipe_dir);
+        }
+    }
+    return 0;
+}
+
+int filter_repo_tags(char *repo, struct StrList *patterns) {
+    int result = 0;
+
+    if (!pushd(repo)) {
+        int list_status = 0;
+        char *tags_raw = shell_output("git tag -l", &list_status);
+        struct StrList *tags = strlist_init();
+        strlist_append_tokenize(tags, tags_raw, LINE_SEP);
+
+        for (size_t i = 0; tags && i < strlist_count(tags); i++) {
+            char *tag = strlist_item(tags, i);
+            for (size_t p = 0; p < strlist_count(patterns); p++) {
+                char *pattern = strlist_item(patterns, p);
+                int match = fnmatch(pattern, tag, 0);
+                if (!match) {
+                    char cmd[PATH_MAX] = {0};
+                    sprintf(cmd, "git tag -d %s", tag);
+                    result += system(cmd);
+                    break;
+                }
+            }
+        }
+        guard_strlist_free(&tags);
+        guard_free(tags_raw);
+        popd();
+    } else {
+        result = -1;
+    }
+    return result;
+}
+
+struct StrList *delivery_build_wheels(struct Delivery *ctx) {
+    struct StrList *result = NULL;
+    struct Process proc;
+    memset(&proc, 0, sizeof(proc));
+
+    result = strlist_init();
+    if (!result) {
+        perror("unable to allocate memory for string list");
+        return NULL;
+    }
+
+    for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) {
+        if (!ctx->tests[i].build_recipe && ctx->tests[i].repository) { // build from source
+            char srcdir[PATH_MAX];
+            char wheeldir[PATH_MAX];
+            memset(srcdir, 0, sizeof(srcdir));
+            memset(wheeldir, 0, sizeof(wheeldir));
+
+            sprintf(srcdir, "%s/%s", ctx->storage.build_sources_dir, ctx->tests[i].name);
+            git_clone(&proc, ctx->tests[i].repository, srcdir, ctx->tests[i].version);
+
+            if (ctx->tests[i].repository_remove_tags && strlist_count(ctx->tests[i].repository_remove_tags)) {
+                filter_repo_tags(srcdir, ctx->tests[i].repository_remove_tags);
+            }
+
+            pushd(srcdir);
+            {
+                char dname[NAME_MAX];
+                char outdir[PATH_MAX];
+                char cmd[PATH_MAX * 2];
+                memset(dname, 0, sizeof(dname));
+                memset(outdir, 0, sizeof(outdir));
+                memset(cmd, 0, sizeof(outdir));
+
+                strcpy(dname, ctx->tests[i].name);
+                tolower_s(dname);
+                sprintf(outdir, "%s/%s", ctx->storage.wheel_artifact_dir, dname);
+                if (mkdirs(outdir, 0755)) {
+                    fprintf(stderr, "failed to create output directory: %s\n", outdir);
+                }
+
+                sprintf(cmd, "-m build -w -o %s", outdir);
+                if (python_exec(cmd)) {
+                    fprintf(stderr, "failed to generate wheel package for %s-%s\n", ctx->tests[i].name, ctx->tests[i].version);
+                    strlist_free(&result);
+                    return NULL;
+                }
+                popd();
+            }
+        }
+    }
+    return result;
+}
+
diff --git a/src/delivery_conda.c b/src/delivery_conda.c
new file mode 100644
index 0000000..93a06fc
--- /dev/null
+++ b/src/delivery_conda.c
@@ -0,0 +1,110 @@
+#include "delivery.h"
+
+void delivery_get_conda_installer_url(struct Delivery *ctx, char *result) {
+    if (ctx->conda.installer_version) {
+        // Use version specified by configuration file
+        sprintf(result, "%s/%s-%s-%s-%s.sh", ctx->conda.installer_baseurl,
+                ctx->conda.installer_name,
+                ctx->conda.installer_version,
+                ctx->conda.installer_platform,
+                ctx->conda.installer_arch);
+    } else {
+        // Use latest installer
+        sprintf(result, "%s/%s-%s-%s.sh", ctx->conda.installer_baseurl,
+                ctx->conda.installer_name,
+                ctx->conda.installer_platform,
+                ctx->conda.installer_arch);
+    }
+
+}
+
+int delivery_get_conda_installer(struct Delivery *ctx, char *installer_url) {
+    char script_path[PATH_MAX];
+    char *installer = path_basename(installer_url);
+
+    memset(script_path, 0, sizeof(script_path));
+    sprintf(script_path, "%s/%s", ctx->storage.tmpdir, installer);
+    if (access(script_path, F_OK)) {
+        // Script doesn't exist
+        long fetch_status = download(installer_url, script_path, NULL);
+        if (HTTP_ERROR(fetch_status) || fetch_status < 0) {
+            // download failed
+            return -1;
+        }
+    } else {
+        msg(STASIS_MSG_RESTRICT | STASIS_MSG_L3, "Skipped, installer already exists\n", script_path);
+    }
+
+    ctx->conda.installer_path = strdup(script_path);
+    if (!ctx->conda.installer_path) {
+        SYSERROR("Unable to duplicate script_path: '%s'", script_path);
+        return -1;
+    }
+
+    return 0;
+}
+
+void delivery_install_conda(char *install_script, char *conda_install_dir) {
+    struct Process proc;
+    memset(&proc, 0, sizeof(proc));
+
+    if (globals.conda_fresh_start) {
+        if (!access(conda_install_dir, F_OK)) {
+            // directory exists so remove it
+            if (rmtree(conda_install_dir)) {
+                perror("unable to remove previous installation");
+                exit(1);
+            }
+
+            // Proceed with the installation
+            // -b = batch mode (non-interactive)
+            char cmd[PATH_MAX] = {0};
+            snprintf(cmd, sizeof(cmd) - 1, "%s %s -b -p %s",
+                     find_program("bash"),
+                     install_script,
+                     conda_install_dir);
+            if (shell_safe(&proc, cmd)) {
+                fprintf(stderr, "conda installation failed\n");
+                exit(1);
+            }
+        } else {
+            // Proceed with the installation
+            // -b = batch mode (non-interactive)
+            char cmd[PATH_MAX] = {0};
+            snprintf(cmd, sizeof(cmd) - 1, "%s %s -b -p %s",
+                     find_program("bash"),
+                     install_script,
+                     conda_install_dir);
+            if (shell_safe(&proc, cmd)) {
+                fprintf(stderr, "conda installation failed\n");
+                exit(1);
+            }
+        }
+    } else {
+        msg(STASIS_MSG_L3, "Conda removal disabled by configuration\n");
+    }
+}
+
+void delivery_conda_enable(struct Delivery *ctx, char *conda_install_dir) {
+    if (conda_activate(conda_install_dir, "base")) {
+        fprintf(stderr, "conda activation failed\n");
+        exit(1);
+    }
+
+    // Setting the CONDARC environment variable appears to be the only consistent
+    // way to make sure the file is used. Not setting this variable leads to strange
+    // behavior, especially if a conda environment is already active when STASIS is loaded.
+    char rcpath[PATH_MAX];
+    sprintf(rcpath, "%s/%s", conda_install_dir, ".condarc");
+    setenv("CONDARC", rcpath, 1);
+    if (runtime_replace(&ctx->runtime.environ, __environ)) {
+        perror("unable to replace runtime environment after activating conda");
+        exit(1);
+    }
+
+    if (conda_setup_headless()) {
+        // no COE check. this call must succeed.
+        exit(1);
+    }
+}
+
diff --git a/src/delivery_docker.c b/src/delivery_docker.c
new file mode 100644
index 0000000..e1d7f60
--- /dev/null
+++ b/src/delivery_docker.c
@@ -0,0 +1,132 @@
+#include "delivery.h"
+
+int delivery_docker(struct Delivery *ctx) {
+    if (!docker_capable(&ctx->deploy.docker.capabilities)) {
+        return -1;
+    }
+    char tag[STASIS_NAME_MAX];
+    char args[PATH_MAX];
+    int has_registry = ctx->deploy.docker.registry != NULL;
+    size_t total_tags = strlist_count(ctx->deploy.docker.tags);
+    size_t total_build_args = strlist_count(ctx->deploy.docker.build_args);
+
+    if (!has_registry) {
+        msg(STASIS_MSG_WARN | STASIS_MSG_L2, "No docker registry defined. You will need to manually retag the resulting image.\n");
+    }
+
+    if (!total_tags) {
+        char default_tag[PATH_MAX];
+        msg(STASIS_MSG_WARN | STASIS_MSG_L2, "No docker tags defined by configuration. Generating default tag(s).\n");
+        // generate local tag
+        memset(default_tag, 0, sizeof(default_tag));
+        sprintf(default_tag, "%s:%s-py%s", ctx->meta.name, ctx->info.build_name, ctx->meta.python_compact);
+        tolower_s(default_tag);
+
+        // Add tag
+        ctx->deploy.docker.tags = strlist_init();
+        strlist_append(&ctx->deploy.docker.tags, default_tag);
+
+        if (has_registry) {
+            // generate tag for target registry
+            memset(default_tag, 0, sizeof(default_tag));
+            sprintf(default_tag, "%s/%s:%s-py%s", ctx->deploy.docker.registry, ctx->meta.name, ctx->info.build_number, ctx->meta.python_compact);
+            tolower_s(default_tag);
+
+            // Add tag
+            strlist_append(&ctx->deploy.docker.tags, default_tag);
+        }
+        // regenerate total tag available
+        total_tags = strlist_count(ctx->deploy.docker.tags);
+    }
+
+    memset(args, 0, sizeof(args));
+
+    // Append image tags to command
+    for (size_t i = 0; i < total_tags; i++) {
+        char *tag_orig = strlist_item(ctx->deploy.docker.tags, i);
+        strcpy(tag, tag_orig);
+        docker_sanitize_tag(tag);
+        sprintf(args + strlen(args), " -t \"%s\" ", tag);
+    }
+
+    // Append build arguments to command (i.e. --build-arg "key=value"
+    for (size_t i = 0; i < total_build_args; i++) {
+        char *build_arg = strlist_item(ctx->deploy.docker.build_args, i);
+        if (!build_arg) {
+            break;
+        }
+        sprintf(args + strlen(args), " --build-arg \"%s\" ", build_arg);
+    }
+
+    // Build the image
+    char delivery_file[PATH_MAX];
+    char dest[PATH_MAX];
+    char rsync_cmd[PATH_MAX * 2];
+    memset(delivery_file, 0, sizeof(delivery_file));
+    memset(dest, 0, sizeof(dest));
+
+    sprintf(delivery_file, "%s/%s.yml", ctx->storage.delivery_dir, ctx->info.release_name);
+    if (access(delivery_file, F_OK) < 0) {
+        fprintf(stderr, "docker build cannot proceed without delivery file: %s\n", delivery_file);
+        return -1;
+    }
+
+    sprintf(dest, "%s/%s.yml", ctx->storage.build_docker_dir, ctx->info.release_name);
+    if (copy2(delivery_file, dest, CT_PERM)) {
+        fprintf(stderr, "Failed to copy delivery file to %s: %s\n", dest, strerror(errno));
+        return -1;
+    }
+
+    memset(dest, 0, sizeof(dest));
+    sprintf(dest, "%s/packages", ctx->storage.build_docker_dir);
+
+    msg(STASIS_MSG_L2, "Copying conda packages\n");
+    memset(rsync_cmd, 0, sizeof(rsync_cmd));
+    sprintf(rsync_cmd, "rsync -avi --progress '%s' '%s'", ctx->storage.conda_artifact_dir, dest);
+    if (system(rsync_cmd)) {
+        fprintf(stderr, "Failed to copy conda artifacts to docker build directory\n");
+        return -1;
+    }
+
+    msg(STASIS_MSG_L2, "Copying wheel packages\n");
+    memset(rsync_cmd, 0, sizeof(rsync_cmd));
+    sprintf(rsync_cmd, "rsync -avi --progress '%s' '%s'", ctx->storage.wheel_artifact_dir, dest);
+    if (system(rsync_cmd)) {
+        fprintf(stderr, "Failed to copy wheel artifactory to docker build directory\n");
+    }
+
+    if (docker_build(ctx->storage.build_docker_dir, args, ctx->deploy.docker.capabilities.build)) {
+        return -1;
+    }
+
+    // Test the image
+    // All tags point back to the same image so test the first one we see
+    // regardless of how many are defined
+    strcpy(tag, strlist_item(ctx->deploy.docker.tags, 0));
+    docker_sanitize_tag(tag);
+
+    msg(STASIS_MSG_L2, "Executing image test script for %s\n", tag);
+    if (ctx->deploy.docker.test_script) {
+        if (isempty(ctx->deploy.docker.test_script)) {
+            msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "Image test script has no content\n");
+        } else {
+            int state;
+            if ((state = docker_script(tag, ctx->deploy.docker.test_script, 0))) {
+                msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "Non-zero exit (%d) from test script. %s image archive will not be generated.\n", state >> 8, tag);
+                // test failed -- don't save the image
+                return -1;
+            }
+        }
+    } else {
+        msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "No image test script defined\n");
+    }
+
+    // Test successful, save image
+    if (docker_save(path_basename(tag), ctx->storage.docker_artifact_dir, ctx->deploy.docker.image_compression)) {
+        // save failed
+        return -1;
+    }
+
+    return 0;
+}
+
diff --git a/src/delivery_init.c b/src/delivery_init.c
new file mode 100644
index 0000000..e914f99
--- /dev/null
+++ b/src/delivery_init.c
@@ -0,0 +1,345 @@
+#include "delivery.h"
+
+int has_mount_flags(const char *mount_point, const unsigned long flags) {
+    struct statvfs st;
+    if (statvfs(mount_point, &st)) {
+        SYSERROR("Unable to determine mount-point flags: %s", strerror(errno));
+        return -1;
+    }
+    return (st.f_flag & flags) != 0;
+}
+
+int delivery_init_tmpdir(struct Delivery *ctx) {
+    char *tmpdir = NULL;
+    char *x = NULL;
+    int unusable = 0;
+    errno = 0;
+
+    x = getenv("TMPDIR");
+    if (x) {
+        guard_free(ctx->storage.tmpdir);
+        tmpdir = strdup(x);
+    } else {
+        tmpdir = ctx->storage.tmpdir;
+    }
+
+    if (!tmpdir) {
+        // memory error
+        return -1;
+    }
+
+    // If the directory doesn't exist, create it
+    if (access(tmpdir, F_OK) < 0) {
+        if (mkdirs(tmpdir, 0755) < 0) {
+            msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Unable to create temporary storage directory: %s (%s)\n", tmpdir, strerror(errno));
+            goto l_delivery_init_tmpdir_fatal;
+        }
+    }
+
+    // If we can't read, write, or execute, then die
+    if (access(tmpdir, R_OK | W_OK | X_OK) < 0) {
+        msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "%s requires at least 0755 permissions.\n");
+        goto l_delivery_init_tmpdir_fatal;
+    }
+
+    struct statvfs st;
+    if (statvfs(tmpdir, &st) < 0) {
+        goto l_delivery_init_tmpdir_fatal;
+    }
+
+#if defined(STASIS_OS_LINUX)
+    // If we can't execute programs, or write data to the file system at all, then die
+    if ((st.f_flag & ST_NOEXEC) != 0) {
+        msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "%s is mounted with noexec\n", tmpdir);
+        goto l_delivery_init_tmpdir_fatal;
+    }
+#endif
+    if ((st.f_flag & ST_RDONLY) != 0) {
+        msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "%s is mounted read-only\n", tmpdir);
+        goto l_delivery_init_tmpdir_fatal;
+    }
+
+    if (!globals.tmpdir) {
+        globals.tmpdir = strdup(tmpdir);
+    }
+
+    if (!ctx->storage.tmpdir) {
+        ctx->storage.tmpdir = strdup(globals.tmpdir);
+    }
+    return unusable;
+
+    l_delivery_init_tmpdir_fatal:
+    unusable = 1;
+    return unusable;
+}
+
+void delivery_init_dirs_stage2(struct Delivery *ctx) {
+    path_store(&ctx->storage.build_recipes_dir, PATH_MAX, ctx->storage.build_dir, "recipes");
+    path_store(&ctx->storage.build_sources_dir, PATH_MAX, ctx->storage.build_dir, "sources");
+    path_store(&ctx->storage.build_testing_dir, PATH_MAX, ctx->storage.build_dir, "testing");
+    path_store(&ctx->storage.build_docker_dir, PATH_MAX, ctx->storage.build_dir, "docker");
+
+    path_store(&ctx->storage.delivery_dir, PATH_MAX, ctx->storage.output_dir, "delivery");
+    path_store(&ctx->storage.results_dir, PATH_MAX, ctx->storage.output_dir, "results");
+    path_store(&ctx->storage.package_dir, PATH_MAX, ctx->storage.output_dir, "packages");
+    path_store(&ctx->storage.cfgdump_dir, PATH_MAX, ctx->storage.output_dir, "config");
+    path_store(&ctx->storage.meta_dir, PATH_MAX, ctx->storage.output_dir, "meta");
+
+    path_store(&ctx->storage.conda_artifact_dir, PATH_MAX, ctx->storage.package_dir, "conda");
+    path_store(&ctx->storage.wheel_artifact_dir, PATH_MAX, ctx->storage.package_dir, "wheels");
+    path_store(&ctx->storage.docker_artifact_dir, PATH_MAX, ctx->storage.package_dir, "docker");
+}
+
+void delivery_init_dirs_stage1(struct Delivery *ctx) {
+    char *rootdir = getenv("STASIS_ROOT");
+    if (rootdir) {
+        if (isempty(rootdir)) {
+            fprintf(stderr, "STASIS_ROOT is set, but empty. Please assign a file system path to this environment variable.\n");
+            exit(1);
+        }
+        path_store(&ctx->storage.root, PATH_MAX, rootdir, ctx->info.build_name);
+    } else {
+        // use "stasis" in current working directory
+        path_store(&ctx->storage.root, PATH_MAX, "stasis", ctx->info.build_name);
+    }
+    path_store(&ctx->storage.tools_dir, PATH_MAX, ctx->storage.root, "tools");
+    path_store(&ctx->storage.tmpdir, PATH_MAX, ctx->storage.root, "tmp");
+    if (delivery_init_tmpdir(ctx)) {
+        msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Set $TMPDIR to a location other than %s\n", globals.tmpdir);
+        if (globals.tmpdir)
+            guard_free(globals.tmpdir);
+        exit(1);
+    }
+
+    path_store(&ctx->storage.build_dir, PATH_MAX, ctx->storage.root, "build");
+    path_store(&ctx->storage.output_dir, PATH_MAX, ctx->storage.root, "output");
+
+    if (!ctx->storage.mission_dir) {
+        path_store(&ctx->storage.mission_dir, PATH_MAX, globals.sysconfdir, "mission");
+    }
+
+    if (access(ctx->storage.mission_dir, F_OK)) {
+        msg(STASIS_MSG_L1, "%s: %s\n", ctx->storage.mission_dir, strerror(errno));
+        exit(1);
+    }
+
+    // Override installation prefix using global configuration key
+    if (globals.conda_install_prefix && strlen(globals.conda_install_prefix)) {
+        // user wants a specific path
+        globals.conda_fresh_start = false;
+        /*
+        if (mkdirs(globals.conda_install_prefix, 0755)) {
+            msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Unable to create directory: %s: %s\n",
+                strerror(errno), globals.conda_install_prefix);
+            exit(1);
+        }
+         */
+        /*
+        ctx->storage.conda_install_prefix = realpath(globals.conda_install_prefix, NULL);
+        if (!ctx->storage.conda_install_prefix) {
+            msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "realpath(): Conda installation prefix reassignment failed\n");
+            exit(1);
+        }
+        ctx->storage.conda_install_prefix = strdup(globals.conda_install_prefix);
+         */
+        path_store(&ctx->storage.conda_install_prefix, PATH_MAX, globals.conda_install_prefix, "conda");
+    } else {
+        // install conda under the STASIS tree
+        path_store(&ctx->storage.conda_install_prefix, PATH_MAX, ctx->storage.tools_dir, "conda");
+    }
+}
+
+int delivery_init_platform(struct Delivery *ctx) {
+    msg(STASIS_MSG_L2, "Setting architecture\n");
+    char archsuffix[20];
+    struct utsname uts;
+    if (uname(&uts)) {
+        msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "uname() failed: %s\n", strerror(errno));
+        return -1;
+    }
+
+    ctx->system.platform = calloc(DELIVERY_PLATFORM_MAX + 1, sizeof(*ctx->system.platform));
+    if (!ctx->system.platform) {
+        SYSERROR("Unable to allocate %d records for platform array\n", DELIVERY_PLATFORM_MAX);
+        return -1;
+    }
+    for (size_t i = 0; i < DELIVERY_PLATFORM_MAX; i++) {
+        ctx->system.platform[i] = calloc(DELIVERY_PLATFORM_MAXLEN, sizeof(*ctx->system.platform[0]));
+    }
+
+    ctx->system.arch = strdup(uts.machine);
+    if (!ctx->system.arch) {
+        // memory error
+        return -1;
+    }
+
+    if (!strcmp(ctx->system.arch, "x86_64")) {
+        strcpy(archsuffix, "64");
+    } else {
+        strcpy(archsuffix, ctx->system.arch);
+    }
+
+    msg(STASIS_MSG_L2, "Setting platform\n");
+    strcpy(ctx->system.platform[DELIVERY_PLATFORM], uts.sysname);
+    if (!strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Darwin")) {
+        sprintf(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], "osx-%s", archsuffix);
+        strcpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], "MacOSX");
+        strcpy(ctx->system.platform[DELIVERY_PLATFORM_RELEASE], "macos");
+    } else if (!strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Linux")) {
+        sprintf(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], "linux-%s", archsuffix);
+        strcpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], "Linux");
+        strcpy(ctx->system.platform[DELIVERY_PLATFORM_RELEASE], "linux");
+    } else {
+        // Not explicitly supported systems
+        strcpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], ctx->system.platform[DELIVERY_PLATFORM]);
+        strcpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], ctx->system.platform[DELIVERY_PLATFORM]);
+        strcpy(ctx->system.platform[DELIVERY_PLATFORM_RELEASE], ctx->system.platform[DELIVERY_PLATFORM]);
+        tolower_s(ctx->system.platform[DELIVERY_PLATFORM_RELEASE]);
+    }
+
+    long cpu_count = get_cpu_count();
+    if (!cpu_count) {
+        fprintf(stderr, "Unable to determine CPU count. Falling back to 1.\n");
+        cpu_count = 1;
+    }
+    char ncpus[100] = {0};
+    sprintf(ncpus, "%ld", cpu_count);
+
+    // Declare some important bits as environment variables
+    setenv("CPU_COUNT", ncpus, 1);
+    setenv("STASIS_CPU_COUNT", ncpus, 1);
+    setenv("STASIS_ARCH", ctx->system.arch, 1);
+    setenv("STASIS_PLATFORM", ctx->system.platform[DELIVERY_PLATFORM], 1);
+    setenv("STASIS_CONDA_ARCH", ctx->system.arch, 1);
+    setenv("STASIS_CONDA_PLATFORM", ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], 1);
+    setenv("STASIS_CONDA_PLATFORM_SUBDIR", ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], 1);
+
+    // Register template variables
+    // These were moved out of main() because we can't take the address of system.platform[x]
+    // _before_ the array has been initialized.
+    tpl_register("system.arch", &ctx->system.arch);
+    tpl_register("system.platform", &ctx->system.platform[DELIVERY_PLATFORM_RELEASE]);
+
+    return 0;
+}
+
+int delivery_init(struct Delivery *ctx, int render_mode) {
+    populate_info(ctx);
+    populate_delivery_cfg(ctx, INI_READ_RENDER);
+
+    // Set artifactory URL via environment variable if possible
+    char *jfurl = getenv("STASIS_JF_ARTIFACTORY_URL");
+    if (jfurl) {
+        if (globals.jfrog.url) {
+            guard_free(globals.jfrog.url);
+        }
+        globals.jfrog.url = strdup(jfurl);
+    }
+
+    // Set artifactory repository via environment if possible
+    char *jfrepo = getenv("STASIS_JF_REPO");
+    if (jfrepo) {
+        if (globals.jfrog.repo) {
+            guard_free(globals.jfrog.repo);
+        }
+        globals.jfrog.repo = strdup(jfrepo);
+    }
+
+    // Configure architecture and platform information
+    delivery_init_platform(ctx);
+
+    // Create STASIS directory structure
+    delivery_init_dirs_stage1(ctx);
+
+    char config_local[PATH_MAX];
+    sprintf(config_local, "%s/%s", ctx->storage.tmpdir, "config");
+    setenv("XDG_CONFIG_HOME", config_local, 1);
+
+    char cache_local[PATH_MAX];
+    sprintf(cache_local, "%s/%s", ctx->storage.tmpdir, "cache");
+    setenv("XDG_CACHE_HOME", ctx->storage.tmpdir, 1);
+
+    // add tools to PATH
+    char pathvar_tmp[STASIS_BUFSIZ];
+    sprintf(pathvar_tmp, "%s/bin:%s", ctx->storage.tools_dir, getenv("PATH"));
+    setenv("PATH", pathvar_tmp, 1);
+
+    // Prevent git from paginating output
+    setenv("GIT_PAGER", "", 1);
+
+    populate_delivery_ini(ctx, render_mode);
+
+    if (ctx->deploy.docker.tags) {
+        for (size_t i = 0; i < strlist_count(ctx->deploy.docker.tags); i++) {
+            char *item = strlist_item(ctx->deploy.docker.tags, i);
+            tolower_s(item);
+        }
+    }
+
+    if (ctx->deploy.docker.image_compression) {
+        if (docker_validate_compression_program(ctx->deploy.docker.image_compression)) {
+            SYSERROR("[deploy:docker].image_compression - invalid command / program is not installed: %s", ctx->deploy.docker.image_compression);
+            return -1;
+        }
+    }
+    return 0;
+}
+
+int bootstrap_build_info(struct Delivery *ctx) {
+    struct Delivery local;
+    memset(&local, 0, sizeof(local));
+    local._stasis_ini_fp.cfg = ini_open(ctx->_stasis_ini_fp.cfg_path);
+    local._stasis_ini_fp.delivery = ini_open(ctx->_stasis_ini_fp.delivery_path);
+    delivery_init_platform(&local);
+    populate_delivery_cfg(&local, INI_READ_RENDER);
+    populate_delivery_ini(&local, INI_READ_RENDER);
+    populate_info(&local);
+    ctx->info.build_name = strdup(local.info.build_name);
+    ctx->info.build_number = strdup(local.info.build_number);
+    ctx->info.release_name = strdup(local.info.release_name);
+    ctx->info.time_info = malloc(sizeof(*ctx->info.time_info));
+    if (!ctx->info.time_info) {
+        SYSERROR("Unable to allocate %zu bytes for tm struct: %s", sizeof(*local.info.time_info), strerror(errno));
+        return -1;
+    }
+    memcpy(ctx->info.time_info, local.info.time_info, sizeof(*local.info.time_info));
+    ctx->info.time_now = local.info.time_now;
+    ctx->info.time_str_epoch = strdup(local.info.time_str_epoch);
+    delivery_free(&local);
+    return 0;
+}
+
+int delivery_exists(struct Delivery *ctx) {
+    int release_exists = 0;
+    char release_pattern[PATH_MAX] = {0};
+    sprintf(release_pattern, "*%s*", ctx->info.release_name);
+
+    if (globals.enable_artifactory) {
+        if (jfrt_auth_init(&ctx->deploy.jfrog_auth)) {
+            fprintf(stderr, "Failed to initialize Artifactory authentication context\n");
+            return -1;  // error
+        }
+
+        struct JFRT_Search search = {.fail_no_op = true};
+        release_exists = jfrog_cli_rt_search(&ctx->deploy.jfrog_auth, &search, globals.jfrog.repo, release_pattern);
+        if (release_exists != 2) {
+            if (!globals.enable_overwrite && !release_exists) {
+                // --fail_no_op returns 2 on failure
+                // without: it returns an empty list "[]" and exit code 0
+                return 1;  // found
+            }
+        }
+    } else {
+        struct StrList *files = listdir(ctx->storage.delivery_dir);
+        for (size_t i = 0; i < strlist_count(files); i++) {
+            char *filename = strlist_item(files, i);
+            release_exists = fnmatch(release_pattern, filename, FNM_PATHNAME);
+            if (!globals.enable_overwrite && !release_exists) {
+                guard_strlist_free(&files);
+                return 1;  // found
+            }
+        }
+        guard_strlist_free(&files);
+    }
+    return 0;  // not found
+}
diff --git a/src/delivery_install.c b/src/delivery_install.c
new file mode 100644
index 0000000..72f2663
--- /dev/null
+++ b/src/delivery_install.c
@@ -0,0 +1,116 @@
+#include "delivery.h"
+
+static const struct Test *requirement_from_test(struct Delivery *ctx, const char *name) {
+    struct Test *result;
+
+    result = NULL;
+    for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) {
+        if (ctx->tests[i].name && strstr(name, ctx->tests[i].name)) {
+            result = &ctx->tests[i];
+            break;
+        }
+    }
+    return result;
+}
+
+int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, char *env_name, int type, struct StrList **manifest) {
+    char cmd[PATH_MAX];
+    char pkgs[STASIS_BUFSIZ];
+    char *env_current = getenv("CONDA_DEFAULT_ENV");
+
+    if (env_current) {
+        // The requested environment is not the current environment
+        if (strcmp(env_current, env_name) != 0) {
+            // Activate the requested environment
+            printf("Activating: %s\n", env_name);
+            conda_activate(conda_install_dir, env_name);
+            runtime_replace(&ctx->runtime.environ, __environ);
+        }
+    }
+
+    memset(cmd, 0, sizeof(cmd));
+    memset(pkgs, 0, sizeof(pkgs));
+    strcat(cmd, "install");
+
+    typedef int (*Runner)(const char *);
+    Runner runner = NULL;
+    if (INSTALL_PKG_CONDA & type) {
+        runner = conda_exec;
+    } else if (INSTALL_PKG_PIP & type) {
+        runner = pip_exec;
+    }
+
+    if (INSTALL_PKG_CONDA_DEFERRED & type) {
+        strcat(cmd, " --use-local");
+    } else if (INSTALL_PKG_PIP_DEFERRED & type) {
+        // Don't change the baseline package set unless we're working with a
+        // new build. Release candidates will need to keep packages as stable
+        // as possible between releases.
+        if (!ctx->meta.based_on) {
+            strcat(cmd, " --upgrade");
+        }
+        sprintf(cmd + strlen(cmd), " --extra-index-url 'file://%s'", ctx->storage.wheel_artifact_dir);
+    }
+
+    for (size_t x = 0; manifest[x] != NULL; x++) {
+        char *name = NULL;
+        for (size_t p = 0; p < strlist_count(manifest[x]); p++) {
+            name = strlist_item(manifest[x], p);
+            strip(name);
+            if (!strlen(name)) {
+                continue;
+            }
+            if (INSTALL_PKG_PIP_DEFERRED & type) {
+                struct Test *info = (struct Test *) requirement_from_test(ctx, name);
+                if (info) {
+                    if (!strcmp(info->version, "HEAD")) {
+                        struct StrList *tag_data = strlist_init();
+                        if (!tag_data) {
+                            SYSERROR("%s", "Unable to allocate memory for tag data\n");
+                            return -1;
+                        }
+                        strlist_append_tokenize(tag_data, info->repository_info_tag, "-");
+
+                        struct Wheel *whl = NULL;
+                        char *post_commit = NULL;
+                        char *hash = NULL;
+                        if (strlist_count(tag_data) > 1) {
+                            post_commit = strlist_item(tag_data, 1);
+                            hash = strlist_item(tag_data, 2);
+                        }
+
+                        // We can't match on version here (index 0). The wheel's version is not guaranteed to be
+                        // equal to the tag; setuptools_scm auto-increments the value, the user can change it manually,
+                        // etc.
+                        whl = get_wheel_file(ctx->storage.wheel_artifact_dir, info->name,
+                                             (char *[]) {ctx->meta.python_compact, ctx->system.arch,
+                                                         "none", "any",
+                                                         post_commit, hash,
+                                                         NULL}, WHEEL_MATCH_ANY);
+
+                        guard_strlist_free(&tag_data);
+                        info->version = whl->version;
+                        sprintf(cmd + strlen(cmd), " '%s==%s'", info->name, whl->version);
+                    } else {
+                        sprintf(cmd + strlen(cmd), " '%s==%s'", info->name, info->version);
+                    }
+                } else {
+                    fprintf(stderr, "Deferred package '%s' is not present in the tested package list!\n", name);
+                    return -1;
+                }
+            } else {
+                if (startswith(name, "--") || startswith(name, "-")) {
+                    sprintf(cmd + strlen(cmd), " %s", name);
+                } else {
+                    sprintf(cmd + strlen(cmd), " '%s'", name);
+                }
+            }
+        }
+        int status = runner(cmd);
+        if (status) {
+            return status;
+        }
+    }
+    return 0;
+}
+
diff --git a/src/delivery_populate.c b/src/delivery_populate.c
new file mode 100644
index 0000000..02ce257
--- /dev/null
+++ b/src/delivery_populate.c
@@ -0,0 +1,348 @@
+#include "delivery.h"
+
+static void ini_has_key_required(struct INIFILE *ini, const char *section_name, char *key) {
+    int status = ini_has_key(ini, section_name, key);
+    if (!status) {
+        SYSERROR("%s:%s key is required but not defined", section_name, key);
+        exit(1);
+    }
+}
+
+static void conv_str(char **x, union INIVal val) {
+    if (*x) {
+        guard_free(*x);
+    }
+    if (val.as_char_p) {
+        char *tplop = tpl_render(val.as_char_p);
+        if (tplop) {
+            *x = tplop;
+        } else {
+            *x = NULL;
+        }
+    } else {
+        *x = NULL;
+    }
+}
+
+
+
+int populate_info(struct Delivery *ctx) {
+    if (!ctx->info.time_str_epoch) {
+        // Record timestamp used for release
+        time(&ctx->info.time_now);
+        ctx->info.time_info = localtime(&ctx->info.time_now);
+
+        ctx->info.time_str_epoch = calloc(STASIS_TIME_STR_MAX, sizeof(*ctx->info.time_str_epoch));
+        if (!ctx->info.time_str_epoch) {
+            msg(STASIS_MSG_ERROR, "Unable to allocate memory for Unix epoch string\n");
+            return -1;
+        }
+        snprintf(ctx->info.time_str_epoch, STASIS_TIME_STR_MAX - 1, "%li", ctx->info.time_now);
+    }
+    return 0;
+}
+
+int populate_delivery_cfg(struct Delivery *ctx, int render_mode) {
+    struct INIFILE *cfg = ctx->_stasis_ini_fp.cfg;
+    if (!cfg) {
+        return -1;
+    }
+    int err = 0;
+    ctx->storage.conda_staging_dir = ini_getval_str(cfg, "default", "conda_staging_dir", render_mode, &err);
+    ctx->storage.conda_staging_url = ini_getval_str(cfg, "default", "conda_staging_url", render_mode, &err);
+    ctx->storage.wheel_staging_dir = ini_getval_str(cfg, "default", "wheel_staging_dir", render_mode, &err);
+    ctx->storage.wheel_staging_url = ini_getval_str(cfg, "default", "wheel_staging_url", render_mode, &err);
+    globals.conda_fresh_start = ini_getval_bool(cfg, "default", "conda_fresh_start", render_mode, &err);
+    if (!globals.continue_on_error) {
+        globals.continue_on_error = ini_getval_bool(cfg, "default", "continue_on_error", render_mode, &err);
+    }
+    if (!globals.always_update_base_environment) {
+        globals.always_update_base_environment = ini_getval_bool(cfg, "default", "always_update_base_environment", render_mode, &err);
+    }
+    globals.conda_install_prefix = ini_getval_str(cfg, "default", "conda_install_prefix", render_mode, &err);
+    globals.conda_packages = ini_getval_strlist(cfg, "default", "conda_packages", LINE_SEP, render_mode, &err);
+    globals.pip_packages = ini_getval_strlist(cfg, "default", "pip_packages", LINE_SEP, render_mode, &err);
+
+    globals.jfrog.jfrog_artifactory_base_url = ini_getval_str(cfg, "jfrog_cli_download", "url", render_mode, &err);
+    globals.jfrog.jfrog_artifactory_product = ini_getval_str(cfg, "jfrog_cli_download", "product", render_mode, &err);
+    globals.jfrog.cli_major_ver = ini_getval_str(cfg, "jfrog_cli_download", "version_series", render_mode, &err);
+    globals.jfrog.version = ini_getval_str(cfg, "jfrog_cli_download", "version", render_mode, &err);
+    globals.jfrog.remote_filename = ini_getval_str(cfg, "jfrog_cli_download", "filename", render_mode, &err);
+    globals.jfrog.url = ini_getval_str(cfg, "deploy:artifactory", "url", render_mode, &err);
+    globals.jfrog.repo = ini_getval_str(cfg, "deploy:artifactory", "repo", render_mode, &err);
+
+    return 0;
+}
+
+int populate_delivery_ini(struct Delivery *ctx, int render_mode) {
+    union INIVal val;
+    struct INIFILE *ini = ctx->_stasis_ini_fp.delivery;
+    struct INIData *rtdata;
+    RuntimeEnv *rt;
+
+    validate_delivery_ini(ini);
+    // Populate runtime variables first they may be interpreted by other
+    // keys in the configuration
+    rt = runtime_copy(__environ);
+    while ((rtdata = ini_getall(ini, "runtime")) != NULL) {
+        char rec[STASIS_BUFSIZ];
+        sprintf(rec, "%s=%s", lstrip(strip(rtdata->key)), lstrip(strip(rtdata->value)));
+        runtime_set(rt, rtdata->key, rtdata->value);
+    }
+    runtime_apply(rt);
+    ctx->runtime.environ = rt;
+
+    int err = 0;
+    ctx->meta.mission = ini_getval_str(ini, "meta", "mission", render_mode, &err);
+
+    if (!strcasecmp(ctx->meta.mission, "hst")) {
+        ctx->meta.codename = ini_getval_str(ini, "meta", "codename", render_mode, &err);
+    } else {
+        ctx->meta.codename = NULL;
+    }
+
+    ctx->meta.version = ini_getval_str(ini, "meta", "version", render_mode, &err);
+    ctx->meta.name = ini_getval_str(ini, "meta", "name", render_mode, &err);
+    ctx->meta.rc = ini_getval_int(ini, "meta", "rc", render_mode, &err);
+    ctx->meta.final = ini_getval_bool(ini, "meta", "final", render_mode, &err);
+    ctx->meta.based_on = ini_getval_str(ini, "meta", "based_on", render_mode, &err);
+
+    if (!ctx->meta.python) {
+        ctx->meta.python = ini_getval_str(ini, "meta", "python", render_mode, &err);
+        guard_free(ctx->meta.python_compact);
+        ctx->meta.python_compact = to_short_version(ctx->meta.python);
+    } else {
+        ini_setval(&ini, INI_SETVAL_REPLACE, "meta", "python", ctx->meta.python);
+    }
+
+    ctx->conda.installer_name = ini_getval_str(ini, "conda", "installer_name", render_mode, &err);
+    ctx->conda.installer_version = ini_getval_str(ini, "conda", "installer_version", render_mode, &err);
+    ctx->conda.installer_platform = ini_getval_str(ini, "conda", "installer_platform", render_mode, &err);
+    ctx->conda.installer_arch = ini_getval_str(ini, "conda", "installer_arch", render_mode, &err);
+    ctx->conda.installer_baseurl = ini_getval_str(ini, "conda", "installer_baseurl", render_mode, &err);
+    ctx->conda.conda_packages = ini_getval_strlist(ini, "conda", "conda_packages", " "LINE_SEP, render_mode, &err);
+
+    if (ctx->conda.conda_packages->data && ctx->conda.conda_packages->data[0] && strpbrk(ctx->conda.conda_packages->data[0], " \t")) {
+        normalize_space(ctx->conda.conda_packages->data[0]);
+        replace_text(ctx->conda.conda_packages->data[0], " ", LINE_SEP, 0);
+        char *pip_packages_replacement = join(ctx->conda.conda_packages->data, LINE_SEP);
+        ini_setval(&ini, INI_SETVAL_REPLACE, "conda", "conda_packages", pip_packages_replacement);
+        guard_free(pip_packages_replacement);
+        guard_strlist_free(&ctx->conda.conda_packages);
+        ctx->conda.conda_packages = ini_getval_strlist(ini, "conda", "conda_packages", LINE_SEP, render_mode, &err);
+    }
+
+    for (size_t i = 0; i < strlist_count(ctx->conda.conda_packages); i++) {
+        char *pkg = strlist_item(ctx->conda.conda_packages, i);
+        if (strpbrk(pkg, ";#") || isempty(pkg)) {
+            strlist_remove(ctx->conda.conda_packages, i);
+        }
+    }
+
+    ctx->conda.pip_packages = ini_getval_strlist(ini, "conda", "pip_packages", LINE_SEP, render_mode, &err);
+    if (ctx->conda.pip_packages->data && ctx->conda.pip_packages->data[0] && strpbrk(ctx->conda.pip_packages->data[0], " \t")) {
+        normalize_space(ctx->conda.pip_packages->data[0]);
+        replace_text(ctx->conda.pip_packages->data[0], " ", LINE_SEP, 0);
+        char *pip_packages_replacement = join(ctx->conda.pip_packages->data, LINE_SEP);
+        ini_setval(&ini, INI_SETVAL_REPLACE, "conda", "pip_packages", pip_packages_replacement);
+        guard_free(pip_packages_replacement);
+        guard_strlist_free(&ctx->conda.pip_packages);
+        ctx->conda.pip_packages = ini_getval_strlist(ini, "conda", "pip_packages", LINE_SEP, render_mode, &err);
+    }
+
+    for (size_t i = 0; i < strlist_count(ctx->conda.pip_packages); i++) {
+        char *pkg = strlist_item(ctx->conda.pip_packages, i);
+        if (strpbrk(pkg, ";#") || isempty(pkg)) {
+            strlist_remove(ctx->conda.pip_packages, i);
+        }
+    }
+
+    // Delivery metadata consumed
+    populate_mission_ini(&ctx, render_mode);
+
+    if (ctx->info.release_name) {
+        guard_free(ctx->info.release_name);
+        guard_free(ctx->info.build_name);
+        guard_free(ctx->info.build_number);
+    }
+
+    if (delivery_format_str(ctx, &ctx->info.release_name, ctx->rules.release_fmt)) {
+        fprintf(stderr, "Failed to generate release name. Format used: %s\n", ctx->rules.release_fmt);
+        return -1;
+    }
+
+    if (!ctx->info.build_name) {
+        delivery_format_str(ctx, &ctx->info.build_name, ctx->rules.build_name_fmt);
+    }
+    if (!ctx->info.build_number) {
+        delivery_format_str(ctx, &ctx->info.build_number, ctx->rules.build_number_fmt);
+    }
+
+    // Best I can do to make output directories unique. Annoying.
+    delivery_init_dirs_stage2(ctx);
+
+    if (!ctx->conda.conda_packages_defer) {
+        ctx->conda.conda_packages_defer = strlist_init();
+    }
+    if (!ctx->conda.pip_packages_defer) {
+        ctx->conda.pip_packages_defer = strlist_init();
+    }
+
+    for (size_t z = 0, i = 0; i < ini->section_count; i++) {
+        char *section_name = ini->section[i]->key;
+        if (startswith(section_name, "test:")) {
+            struct Test *test = &ctx->tests[z];
+            val.as_char_p = strchr(ini->section[i]->key, ':') + 1;
+            if (val.as_char_p && isempty(val.as_char_p)) {
+                return 1;
+            }
+            conv_str(&test->name, val);
+
+            test->version = ini_getval_str(ini, section_name, "version", render_mode, &err);
+            test->repository = ini_getval_str(ini, section_name, "repository", render_mode, &err);
+            test->script_setup = ini_getval_str(ini, section_name, "script_setup", INI_READ_RAW, &err);
+            test->script = ini_getval_str(ini, section_name, "script", INI_READ_RAW, &err);
+            test->disable = ini_getval_bool(ini, section_name, "disable", render_mode, &err);
+            test->parallel = ini_getval_bool(ini, section_name, "parallel", render_mode, &err);
+            if (err) {
+                test->parallel = true;
+            }
+            test->repository_remove_tags = ini_getval_strlist(ini, section_name, "repository_remove_tags", LINE_SEP, render_mode, &err);
+            test->build_recipe = ini_getval_str(ini, section_name, "build_recipe", render_mode, &err);
+            test->runtime.environ = ini_getval_strlist(ini, section_name, "runtime", LINE_SEP, render_mode, &err);
+            z++;
+        }
+    }
+
+    for (size_t z = 0, i = 0; i < ini->section_count; i++) {
+        char *section_name = ini->section[i]->key;
+        struct Deploy *deploy = &ctx->deploy;
+        if (startswith(section_name, "deploy:artifactory")) {
+            struct JFrog *jfrog = &deploy->jfrog[z];
+            // Artifactory base configuration
+
+            jfrog->upload_ctx.workaround_parent_only = ini_getval_bool(ini, section_name, "workaround_parent_only", render_mode, &err);
+            jfrog->upload_ctx.exclusions = ini_getval_str(ini, section_name, "exclusions", render_mode, &err);
+            jfrog->upload_ctx.explode = ini_getval_bool(ini, section_name, "explode", render_mode, &err);
+            jfrog->upload_ctx.recursive = ini_getval_bool(ini, section_name, "recursive", render_mode, &err);
+            jfrog->upload_ctx.retries = ini_getval_int(ini, section_name, "retries", render_mode, &err);
+            jfrog->upload_ctx.retry_wait_time = ini_getval_int(ini, section_name, "retry_wait_time", render_mode, &err);
+            jfrog->upload_ctx.detailed_summary = ini_getval_bool(ini, section_name, "detailed_summary", render_mode, &err);
+            jfrog->upload_ctx.quiet = ini_getval_bool(ini, section_name, "quiet", render_mode, &err);
+            jfrog->upload_ctx.regexp = ini_getval_bool(ini, section_name, "regexp", render_mode, &err);
+            jfrog->upload_ctx.spec = ini_getval_str(ini, section_name, "spec", render_mode, &err);
+            jfrog->upload_ctx.flat = ini_getval_bool(ini, section_name, "flat", render_mode, &err);
+            jfrog->repo = ini_getval_str(ini, section_name, "repo", render_mode, &err);
+            jfrog->dest = ini_getval_str(ini, section_name, "dest", render_mode, &err);
+            jfrog->files = ini_getval_strlist(ini, section_name, "files", LINE_SEP, render_mode, &err);
+            z++;
+        }
+    }
+
+    for (size_t i = 0; i < ini->section_count; i++) {
+        char *section_name = ini->section[i]->key;
+        struct Deploy *deploy = &ctx->deploy;
+        if (startswith(ini->section[i]->key, "deploy:docker")) {
+            struct Docker *docker = &deploy->docker;
+
+            docker->registry = ini_getval_str(ini, section_name, "registry", render_mode, &err);
+            docker->image_compression = ini_getval_str(ini, section_name, "image_compression", render_mode, &err);
+            docker->test_script = ini_getval_str(ini, section_name, "test_script", render_mode, &err);
+            docker->build_args = ini_getval_strlist(ini, section_name, "build_args", LINE_SEP, render_mode, &err);
+            docker->tags = ini_getval_strlist(ini, section_name, "tags", LINE_SEP, render_mode, &err);
+        }
+    }
+    return 0;
+}
+
+int populate_mission_ini(struct Delivery **ctx, int render_mode) {
+    int err = 0;
+    struct INIFILE *ini;
+
+    if ((*ctx)->_stasis_ini_fp.mission) {
+        return 0;
+    }
+
+    // Now populate the rules
+    char missionfile[PATH_MAX] = {0};
+    if (getenv("STASIS_SYSCONFDIR")) {
+        sprintf(missionfile, "%s/%s/%s/%s.ini",
+                getenv("STASIS_SYSCONFDIR"), "mission", (*ctx)->meta.mission, (*ctx)->meta.mission);
+    } else {
+        sprintf(missionfile, "%s/%s/%s/%s.ini",
+                globals.sysconfdir, "mission", (*ctx)->meta.mission, (*ctx)->meta.mission);
+    }
+
+    msg(STASIS_MSG_L2, "Reading mission configuration: %s\n", missionfile);
+    (*ctx)->_stasis_ini_fp.mission = ini_open(missionfile);
+    ini = (*ctx)->_stasis_ini_fp.mission;
+    if (!ini) {
+        msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read misson configuration: %s, %s\n", missionfile, strerror(errno));
+        exit(1);
+    }
+    (*ctx)->_stasis_ini_fp.mission_path = strdup(missionfile);
+
+    (*ctx)->rules.release_fmt = ini_getval_str(ini, "meta", "release_fmt", render_mode, &err);
+
+    // Used for setting artifactory build info
+    (*ctx)->rules.build_name_fmt = ini_getval_str(ini, "meta", "build_name_fmt", render_mode, &err);
+
+    // Used for setting artifactory build info
+    (*ctx)->rules.build_number_fmt = ini_getval_str(ini, "meta", "build_number_fmt", render_mode, &err);
+    return 0;
+}
+
+void validate_delivery_ini(struct INIFILE *ini) {
+    if (!ini) {
+        SYSERROR("%s", "INIFILE is NULL!");
+        exit(1);
+    }
+    if (ini_section_search(&ini, INI_SEARCH_EXACT, "meta")) {
+        ini_has_key_required(ini, "meta", "name");
+        ini_has_key_required(ini, "meta", "version");
+        ini_has_key_required(ini, "meta", "rc");
+        ini_has_key_required(ini, "meta", "mission");
+        ini_has_key_required(ini, "meta", "python");
+    } else {
+        SYSERROR("%s", "[meta] configuration section is required");
+        exit(1);
+    }
+
+    if (ini_section_search(&ini, INI_SEARCH_EXACT, "conda")) {
+        ini_has_key_required(ini, "conda", "installer_name");
+        ini_has_key_required(ini, "conda", "installer_version");
+        ini_has_key_required(ini, "conda", "installer_platform");
+        ini_has_key_required(ini, "conda", "installer_arch");
+    } else {
+        SYSERROR("%s", "[conda] configuration section is required");
+        exit(1);
+    }
+
+    for (size_t i = 0; i < ini->section_count; i++) {
+        struct INISection *section = ini->section[i];
+        if (section && startswith(section->key, "test:")) {
+            char *name = strstr(section->key, ":");
+            if (name && strlen(name) > 1) {
+                name = &name[1];
+            }
+            //ini_has_key_required(ini, section->key, "version");
+            //ini_has_key_required(ini, section->key, "repository");
+            if (globals.enable_testing) {
+                ini_has_key_required(ini, section->key, "script");
+            }
+        }
+    }
+
+    if (ini_section_search(&ini, INI_SEARCH_EXACT, "deploy:docker")) {
+        // yeah?
+    }
+
+    for (size_t i = 0; i < ini->section_count; i++) {
+        struct INISection *section = ini->section[i];
+        if (section && startswith(section->key, "deploy:artifactory")) {
+            ini_has_key_required(ini, section->key, "files");
+            ini_has_key_required(ini, section->key, "dest");
+        }
+    }
+}
+
diff --git a/src/delivery_postprocess.c b/src/delivery_postprocess.c
new file mode 100644
index 0000000..43a1105
--- /dev/null
+++ b/src/delivery_postprocess.c
@@ -0,0 +1,263 @@
+#include "delivery.h"
+
+
+const char *release_header = "# delivery_name: %s\n"
+                             "# delivery_fmt: %s\n"
+                             "# creation_time: %s\n"
+                             "# conda_ident: %s\n"
+                             "# conda_build_ident: %s\n";
+
+char *delivery_get_release_header(struct Delivery *ctx) {
+    char output[STASIS_BUFSIZ];
+    char stamp[100];
+    strftime(stamp, sizeof(stamp) - 1, "%c", ctx->info.time_info);
+    sprintf(output, release_header,
+            ctx->info.release_name,
+            ctx->rules.release_fmt,
+            stamp,
+            ctx->conda.tool_version,
+            ctx->conda.tool_build_version);
+    return strdup(output);
+}
+
+int delivery_dump_metadata(struct Delivery *ctx) {
+    FILE *fp;
+    char filename[PATH_MAX];
+    sprintf(filename, "%s/meta-%s.stasis", ctx->storage.meta_dir, ctx->info.release_name);
+    fp = fopen(filename, "w+");
+    if (!fp) {
+        return -1;
+    }
+    if (globals.verbose) {
+        printf("%s\n", filename);
+    }
+    fprintf(fp, "name %s\n", ctx->meta.name);
+    fprintf(fp, "version %s\n", ctx->meta.version);
+    fprintf(fp, "rc %d\n", ctx->meta.rc);
+    fprintf(fp, "python %s\n", ctx->meta.python);
+    fprintf(fp, "python_compact %s\n", ctx->meta.python_compact);
+    fprintf(fp, "mission %s\n", ctx->meta.mission);
+    fprintf(fp, "codename %s\n", ctx->meta.codename ? ctx->meta.codename : "");
+    fprintf(fp, "platform %s %s %s %s\n",
+            ctx->system.platform[DELIVERY_PLATFORM],
+            ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR],
+            ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER],
+            ctx->system.platform[DELIVERY_PLATFORM_RELEASE]);
+    fprintf(fp, "arch %s\n", ctx->system.arch);
+    fprintf(fp, "time %s\n", ctx->info.time_str_epoch);
+    fprintf(fp, "release_fmt %s\n", ctx->rules.release_fmt);
+    fprintf(fp, "release_name %s\n", ctx->info.release_name);
+    fprintf(fp, "build_name_fmt %s\n", ctx->rules.build_name_fmt);
+    fprintf(fp, "build_name %s\n", ctx->info.build_name);
+    fprintf(fp, "build_number_fmt %s\n", ctx->rules.build_number_fmt);
+    fprintf(fp, "build_number %s\n", ctx->info.build_number);
+    fprintf(fp, "conda_installer_baseurl %s\n", ctx->conda.installer_baseurl);
+    fprintf(fp, "conda_installer_name %s\n", ctx->conda.installer_name);
+    fprintf(fp, "conda_installer_version %s\n", ctx->conda.installer_version);
+    fprintf(fp, "conda_installer_platform %s\n", ctx->conda.installer_platform);
+    fprintf(fp, "conda_installer_arch %s\n", ctx->conda.installer_arch);
+
+    fclose(fp);
+    return 0;
+}
+
+void delivery_rewrite_spec(struct Delivery *ctx, char *filename, unsigned stage) {
+    char output[PATH_MAX];
+    char *header = NULL;
+    char *tempfile = NULL;
+    FILE *tp = NULL;
+
+    if (stage == DELIVERY_REWRITE_SPEC_STAGE_1) {
+        header = delivery_get_release_header(ctx);
+        if (!header) {
+            msg(STASIS_MSG_ERROR, "failed to generate release header string\n", filename);
+            exit(1);
+        }
+        tempfile = xmkstemp(&tp, "w+");
+        if (!tempfile || !tp) {
+            msg(STASIS_MSG_ERROR, "%s: unable to create temporary file\n", strerror(errno));
+            exit(1);
+        }
+        fprintf(tp, "%s", header);
+
+        // Read the original file
+        char **contents = file_readlines(filename, 0, 0, NULL);
+        if (!contents) {
+            msg(STASIS_MSG_ERROR, "%s: unable to read %s", filename);
+            exit(1);
+        }
+
+        // Write temporary data
+        for (size_t i = 0; contents[i] != NULL; i++) {
+            if (startswith(contents[i], "channels:")) {
+                // Allow for additional conda channel injection
+                if (ctx->conda.conda_packages_defer && strlist_count(ctx->conda.conda_packages_defer)) {
+                    fprintf(tp, "%s  - @CONDA_CHANNEL@\n", contents[i]);
+                    continue;
+                }
+            } else if (strstr(contents[i], "- pip:")) {
+                if (ctx->conda.pip_packages_defer && strlist_count(ctx->conda.pip_packages_defer)) {
+                    // Allow for additional pip argument injection
+                    fprintf(tp, "%s      - @PIP_ARGUMENTS@\n", contents[i]);
+                    continue;
+                }
+            } else if (startswith(contents[i], "prefix:")) {
+                // Remove the prefix key
+                if (strstr(contents[i], "/") || strstr(contents[i], "\\")) {
+                    // path is on the same line as the key
+                    continue;
+                } else {
+                    // path is on the next line?
+                    if (contents[i + 1] && (strstr(contents[i + 1], "/") || strstr(contents[i + 1], "\\"))) {
+                        i++;
+                    }
+                    continue;
+                }
+            }
+            fprintf(tp, "%s", contents[i]);
+        }
+        GENERIC_ARRAY_FREE(contents);
+        guard_free(header);
+        fflush(tp);
+        fclose(tp);
+
+        // Replace the original file with our temporary data
+        if (copy2(tempfile, filename, CT_PERM) < 0) {
+            fprintf(stderr, "%s: could not rename '%s' to '%s'\n", strerror(errno), tempfile, filename);
+            exit(1);
+        }
+        remove(tempfile);
+        guard_free(tempfile);
+    } else if (globals.enable_rewrite_spec_stage_2 && stage == DELIVERY_REWRITE_SPEC_STAGE_2) {
+        // Replace "local" channel with the staging URL
+        if (ctx->storage.conda_staging_url) {
+            file_replace_text(filename, "@CONDA_CHANNEL@", ctx->storage.conda_staging_url, 0);
+        } else if (globals.jfrog.repo) {
+            sprintf(output, "%s/%s/%s/%s/packages/conda", globals.jfrog.url, globals.jfrog.repo, ctx->meta.mission, ctx->info.build_name);
+            file_replace_text(filename, "@CONDA_CHANNEL@", output, 0);
+        } else {
+            msg(STASIS_MSG_WARN, "conda_staging_dir is not configured. Using fallback: '%s'\n", ctx->storage.conda_artifact_dir);
+            file_replace_text(filename, "@CONDA_CHANNEL@", ctx->storage.conda_artifact_dir, 0);
+        }
+
+        if (ctx->storage.wheel_staging_url) {
+            file_replace_text(filename, "@PIP_ARGUMENTS@", ctx->storage.wheel_staging_url, 0);
+        } else if (globals.enable_artifactory && globals.jfrog.url && globals.jfrog.repo) {
+            sprintf(output, "--extra-index-url %s/%s/%s/%s/packages/wheels", globals.jfrog.url, globals.jfrog.repo, ctx->meta.mission, ctx->info.build_name);
+            file_replace_text(filename, "@PIP_ARGUMENTS@", output, 0);
+        } else {
+            msg(STASIS_MSG_WARN, "wheel_staging_dir is not configured. Using fallback: '%s'\n", ctx->storage.wheel_artifact_dir);
+            sprintf(output, "--extra-index-url file://%s", ctx->storage.wheel_artifact_dir);
+            file_replace_text(filename, "@PIP_ARGUMENTS@", output, 0);
+        }
+    }
+}
+
+int delivery_copy_conda_artifacts(struct Delivery *ctx) {
+    char cmd[STASIS_BUFSIZ];
+    char conda_build_dir[PATH_MAX];
+    char subdir[PATH_MAX];
+    memset(cmd, 0, sizeof(cmd));
+    memset(conda_build_dir, 0, sizeof(conda_build_dir));
+    memset(subdir, 0, sizeof(subdir));
+
+    sprintf(conda_build_dir, "%s/%s", ctx->storage.conda_install_prefix, "conda-bld");
+    // One must run conda build at least once to create the "conda-bld" directory.
+    // When this directory is missing there can be no build artifacts.
+    if (access(conda_build_dir, F_OK) < 0) {
+        msg(STASIS_MSG_RESTRICT | STASIS_MSG_WARN | STASIS_MSG_L3,
+            "Skipped: 'conda build' has never been executed.\n");
+        return 0;
+    }
+
+    snprintf(cmd, sizeof(cmd) - 1, "rsync -avi --progress %s/%s %s",
+             conda_build_dir,
+             ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR],
+             ctx->storage.conda_artifact_dir);
+
+    return system(cmd);
+}
+
+int delivery_index_conda_artifacts(struct Delivery *ctx) {
+    return conda_index(ctx->storage.conda_artifact_dir);
+}
+
+int delivery_copy_wheel_artifacts(struct Delivery *ctx) {
+    char cmd[PATH_MAX];
+    memset(cmd, 0, sizeof(cmd));
+    snprintf(cmd, sizeof(cmd) - 1, "rsync -avi --progress %s/*/dist/*.whl %s",
+             ctx->storage.build_sources_dir,
+             ctx->storage.wheel_artifact_dir);
+    return system(cmd);
+}
+
+int delivery_index_wheel_artifacts(struct Delivery *ctx) {
+    struct dirent *rec;
+    DIR *dp;
+    FILE *top_fp;
+
+    dp = opendir(ctx->storage.wheel_artifact_dir);
+    if (!dp) {
+        return -1;
+    }
+
+    // Generate a "dumb" local pypi index that is compatible with:
+    // pip install --extra-index-url
+    char top_index[PATH_MAX];
+    memset(top_index, 0, sizeof(top_index));
+    sprintf(top_index, "%s/index.html", ctx->storage.wheel_artifact_dir);
+    top_fp = fopen(top_index, "w+");
+    if (!top_fp) {
+        return -2;
+    }
+
+    while ((rec = readdir(dp)) != NULL) {
+        // skip directories
+        if (DT_REG == rec->d_type || !strcmp(rec->d_name, "..") || !strcmp(rec->d_name, ".")) {
+            continue;
+        }
+
+        FILE *bottom_fp;
+        char bottom_index[PATH_MAX * 2];
+        memset(bottom_index, 0, sizeof(bottom_index));
+        sprintf(bottom_index, "%s/%s/index.html", ctx->storage.wheel_artifact_dir, rec->d_name);
+        bottom_fp = fopen(bottom_index, "w+");
+        if (!bottom_fp) {
+            return -3;
+        }
+
+        if (globals.verbose) {
+            printf("+ %s\n", rec->d_name);
+        }
+        // Add record to top level index
+        fprintf(top_fp, "<a href=\"%s/\">%s</a><br/>\n", rec->d_name, rec->d_name);
+
+        char dpath[PATH_MAX * 2];
+        memset(dpath, 0, sizeof(dpath));
+        sprintf(dpath, "%s/%s", ctx->storage.wheel_artifact_dir, rec->d_name);
+        struct StrList *packages = listdir(dpath);
+        if (!packages) {
+            fclose(top_fp);
+            fclose(bottom_fp);
+            return -4;
+        }
+
+        for (size_t i = 0; i < strlist_count(packages); i++) {
+            char *package = strlist_item(packages, i);
+            if (!endswith(package, ".whl")) {
+                continue;
+            }
+            if (globals.verbose) {
+                printf("`- %s\n", package);
+            }
+            // Write record to bottom level index
+            fprintf(bottom_fp, "<a href=\"%s\">%s</a><br/>\n", package, package);
+        }
+        fclose(bottom_fp);
+
+        guard_strlist_free(&packages);
+    }
+    closedir(dp);
+    fclose(top_fp);
+    return 0;
+}
diff --git a/src/delivery_show.c b/src/delivery_show.c
new file mode 100644
index 0000000..adfa1be
--- /dev/null
+++ b/src/delivery_show.c
@@ -0,0 +1,117 @@
+#include "delivery.h"
+
+void delivery_debug_show(struct Delivery *ctx) {
+    printf("\n====DEBUG====\n");
+    printf("%-20s %-10s\n", "System configuration directory:", globals.sysconfdir);
+    printf("%-20s %-10s\n", "Mission directory:", ctx->storage.mission_dir);
+    printf("%-20s %-10s\n", "Testing enabled:", globals.enable_testing ? "Yes" : "No");
+    printf("%-20s %-10s\n", "Docker image builds enabled:", globals.enable_docker ? "Yes" : "No");
+    printf("%-20s %-10s\n", "Artifact uploading enabled:", globals.enable_artifactory ? "Yes" : "No");
+}
+
+void delivery_meta_show(struct Delivery *ctx) {
+    if (globals.verbose) {
+        delivery_debug_show(ctx);
+    }
+
+    printf("\n====DELIVERY====\n");
+    printf("%-20s %-10s\n", "Target Python:", ctx->meta.python);
+    printf("%-20s %-10s\n", "Name:", ctx->meta.name);
+    printf("%-20s %-10s\n", "Mission:", ctx->meta.mission);
+    if (ctx->meta.codename) {
+        printf("%-20s %-10s\n", "Codename:", ctx->meta.codename);
+    }
+    if (ctx->meta.version) {
+        printf("%-20s %-10s\n", "Version", ctx->meta.version);
+    }
+    if (!ctx->meta.final) {
+        printf("%-20s %-10d\n", "RC Level:", ctx->meta.rc);
+    }
+    printf("%-20s %-10s\n", "Final Release:", ctx->meta.final ? "Yes" : "No");
+    printf("%-20s %-10s\n", "Based On:", ctx->meta.based_on ? ctx->meta.based_on : "New");
+}
+
+void delivery_conda_show(struct Delivery *ctx) {
+    printf("\n====CONDA====\n");
+    printf("%-20s %-10s\n", "Prefix:", ctx->storage.conda_install_prefix);
+
+    puts("Native Packages:");
+    if (strlist_count(ctx->conda.conda_packages) || strlist_count(ctx->conda.conda_packages_defer)) {
+        struct StrList *list_conda = strlist_init();
+        if (strlist_count(ctx->conda.conda_packages)) {
+            strlist_append_strlist(list_conda, ctx->conda.conda_packages);
+        }
+        if (strlist_count(ctx->conda.conda_packages_defer)) {
+            strlist_append_strlist(list_conda, ctx->conda.conda_packages_defer);
+        }
+        strlist_sort(list_conda, STASIS_SORT_ALPHA);
+
+        for (size_t i = 0; i < strlist_count(list_conda); i++) {
+            char *token = strlist_item(list_conda, i);
+            if (isempty(token) || isblank(*token) || startswith(token, "-")) {
+                continue;
+            }
+            printf("%21s%s\n", "", token);
+        }
+        guard_strlist_free(&list_conda);
+    } else {
+        printf("%21s%s\n", "", "N/A");
+    }
+
+    puts("Python Packages:");
+    if (strlist_count(ctx->conda.pip_packages) || strlist_count(ctx->conda.pip_packages_defer)) {
+        struct StrList *list_python = strlist_init();
+        if (strlist_count(ctx->conda.pip_packages)) {
+            strlist_append_strlist(list_python, ctx->conda.pip_packages);
+        }
+        if (strlist_count(ctx->conda.pip_packages_defer)) {
+            strlist_append_strlist(list_python, ctx->conda.pip_packages_defer);
+        }
+        strlist_sort(list_python, STASIS_SORT_ALPHA);
+
+        for (size_t i = 0; i < strlist_count(list_python); i++) {
+            char *token = strlist_item(list_python, i);
+            if (isempty(token) || isblank(*token) || startswith(token, "-")) {
+                continue;
+            }
+            printf("%21s%s\n", "", token);
+        }
+        guard_strlist_free(&list_python);
+    } else {
+        printf("%21s%s\n", "", "N/A");
+    }
+}
+
+void delivery_tests_show(struct Delivery *ctx) {
+    printf("\n====TESTS====\n");
+    for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) {
+        if (!ctx->tests[i].name) {
+            continue;
+        }
+        printf("%-20s %-20s %s\n", ctx->tests[i].name,
+               ctx->tests[i].version,
+               ctx->tests[i].repository);
+    }
+}
+
+void delivery_runtime_show(struct Delivery *ctx) {
+    printf("\n====RUNTIME====\n");
+    struct StrList *rt = NULL;
+    rt = strlist_copy(ctx->runtime.environ);
+    if (!rt) {
+        // no data
+        return;
+    }
+    strlist_sort(rt, STASIS_SORT_ALPHA);
+    size_t total = strlist_count(rt);
+    for (size_t i = 0; i < total; i++) {
+        char *item = strlist_item(rt, i);
+        if (!item) {
+            // not supposed to occur
+            msg(STASIS_MSG_WARN | STASIS_MSG_L1, "Encountered unexpected NULL at record %zu of %zu of runtime array.\n", i);
+            return;
+        }
+        printf("%s\n", item);
+    }
+}
+
diff --git a/src/delivery_test.c b/src/delivery_test.c
new file mode 100644
index 0000000..beea510
--- /dev/null
+++ b/src/delivery_test.c
@@ -0,0 +1,244 @@
+#include "delivery.h"
+
+void delivery_tests_run(struct Delivery *ctx) {
+    struct MultiProcessingPool *pool_parallel;
+    struct MultiProcessingPool *pool_serial;
+    struct MultiProcessingPool *pool_setup;
+    struct Process proc;
+    memset(&proc, 0, sizeof(proc));
+
+    if (!globals.workaround.conda_reactivate) {
+        globals.workaround.conda_reactivate = calloc(PATH_MAX, sizeof(*globals.workaround.conda_reactivate));
+    } else {
+        memset(globals.workaround.conda_reactivate, 0, PATH_MAX);
+    }
+    snprintf(globals.workaround.conda_reactivate, PATH_MAX - 1, "\nmamba activate ${CONDA_DEFAULT_ENV}\n");
+
+    if (!ctx->tests[0].name) {
+        msg(STASIS_MSG_WARN | STASIS_MSG_L2, "no tests are defined!\n");
+    } else {
+        pool_parallel = mp_pool_init("parallel", ctx->storage.tmpdir);
+        if (!pool_parallel) {
+            perror("mp_pool_init/parallel");
+            exit(1);
+        }
+
+        pool_serial = mp_pool_init("serial", ctx->storage.tmpdir);
+        if (!pool_serial) {
+            perror("mp_pool_init/serial");
+            exit(1);
+        }
+
+        pool_setup = mp_pool_init("setup", ctx->storage.tmpdir);
+        if (!pool_setup) {
+            perror("mp_pool_init/setup");
+            exit(1);
+        }
+
+        const char *runner_cmd_fmt = "set -e -x\n%s\n";
+        for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) {
+            struct Test *test = &ctx->tests[i];
+            if (!test->name && !test->repository && !test->script) {
+                // skip unused test records
+                continue;
+            }
+            msg(STASIS_MSG_L2, "Executing tests for %s %s\n", test->name, test->version);
+            if (!test->script || !strlen(test->script)) {
+                msg(STASIS_MSG_WARN | STASIS_MSG_L3, "Nothing to do. To fix, declare a 'script' in section: [test:%s]\n",
+                    test->name);
+                continue;
+            }
+
+            char destdir[PATH_MAX];
+            sprintf(destdir, "%s/%s", ctx->storage.build_sources_dir, path_basename(test->repository));
+
+            if (!access(destdir, F_OK)) {
+                msg(STASIS_MSG_L3, "Purging repository %s\n", destdir);
+                if (rmtree(destdir)) {
+                    COE_CHECK_ABORT(1, "Unable to remove repository\n");
+                }
+            }
+            msg(STASIS_MSG_L3, "Cloning repository %s\n", test->repository);
+            if (!git_clone(&proc, test->repository, destdir, test->version)) {
+                test->repository_info_tag = strdup(git_describe(destdir));
+                test->repository_info_ref = strdup(git_rev_parse(destdir, "HEAD"));
+            } else {
+                COE_CHECK_ABORT(1, "Unable to clone repository\n");
+            }
+
+            if (test->repository_remove_tags && strlist_count(test->repository_remove_tags)) {
+                filter_repo_tags(destdir, test->repository_remove_tags);
+            }
+
+            if (pushd(destdir)) {
+                COE_CHECK_ABORT(1, "Unable to enter repository directory\n");
+            } else {
+                char *cmd = calloc(strlen(test->script) + STASIS_BUFSIZ, sizeof(*cmd));
+                if (!cmd) {
+                    SYSERROR("Unable to allocate test script buffer: %s", strerror(errno));
+                    exit(1);
+                }
+
+                msg(STASIS_MSG_L3, "Testing %s\n", test->name);
+                memset(&proc, 0, sizeof(proc));
+
+                strcpy(cmd, test->script);
+                char *cmd_rendered = tpl_render(cmd);
+                if (cmd_rendered) {
+                    if (strcmp(cmd_rendered, cmd) != 0) {
+                        strcpy(cmd, cmd_rendered);
+                        cmd[strlen(cmd_rendered) ? strlen(cmd_rendered) - 1 : 0] = 0;
+                    }
+                    guard_free(cmd_rendered);
+                } else {
+                    SYSERROR("An error occurred while rendering the following:\n%s", cmd);
+                    exit(1);
+                }
+
+                if (test->disable) {
+                    msg(STASIS_MSG_L2, "Script execution disabled by configuration\n", test->name);
+                    guard_free(cmd);
+                    continue;
+                }
+
+                char runner_cmd[0xFFFF] = {0};
+                char pool_name[100] = "parallel";
+                struct MultiProcessingTask *task = NULL;
+                struct MultiProcessingPool *pool = pool_parallel;
+                if (!test->parallel) {
+                    pool = pool_serial;
+                    memset(pool_name, 0, sizeof(pool_name));
+                    strcpy(pool_name, "serial");
+                }
+
+                sprintf(runner_cmd, runner_cmd_fmt, cmd);
+                task = mp_pool_task(pool, test->name, runner_cmd);
+                if (!task) {
+                    SYSERROR("Failed to add task to %s pool: %s", pool_name, runner_cmd);
+                    popd();
+                    if (!globals.continue_on_error) {
+                        tpl_free();
+                        delivery_free(ctx);
+                        globals_free();
+                    }
+                    exit(1);
+                }
+                guard_free(cmd);
+                popd();
+
+            }
+        }
+
+        // Configure "script_setup" tasks
+        // Directories should exist now, so no need to go through initializing everything all over again.
+        for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) {
+            struct Test *test = &ctx->tests[i];
+            if (test->script_setup) {
+                char destdir[PATH_MAX];
+                sprintf(destdir, "%s/%s", ctx->storage.build_sources_dir, path_basename(test->repository));
+                if (access(destdir, F_OK)) {
+                    SYSERROR("%s: %s", destdir, strerror(errno));
+                    exit(1);
+                }
+                if (!pushd(destdir)) {
+                    char *cmd = calloc(strlen(test->script_setup) + STASIS_BUFSIZ, sizeof(*cmd));
+                    if (!cmd) {
+                        SYSERROR("Unable to allocate test script_setup buffer: %s", strerror(errno));
+                        exit(1);
+                    }
+
+                    strcpy(cmd, test->script_setup);
+                    char *cmd_rendered = tpl_render(cmd);
+                    if (cmd_rendered) {
+                        if (strcmp(cmd_rendered, cmd) != 0) {
+                            strcpy(cmd, cmd_rendered);
+                            cmd[strlen(cmd_rendered) ? strlen(cmd_rendered) - 1 : 0] = 0;
+                        }
+                        guard_free(cmd_rendered);
+                    } else {
+                        SYSERROR("An error occurred while rendering the following:\n%s", cmd);
+                        exit(1);
+                    }
+
+                    struct MultiProcessingPool *pool = pool_setup;
+                    struct MultiProcessingTask *task = NULL;
+                    char runner_cmd[0xFFFF] = {0};
+                    sprintf(runner_cmd, runner_cmd_fmt, cmd);
+
+                    task = mp_pool_task(pool, test->name, runner_cmd);
+                    if (!task) {
+                        SYSERROR("Failed to add task %s to setup pool: %s", test->name, runner_cmd);
+                        popd();
+                        if (!globals.continue_on_error) {
+                            tpl_free();
+                            delivery_free(ctx);
+                            globals_free();
+                        }
+                        exit(1);
+                    }
+                    guard_free(cmd);
+                    popd();
+                }
+            }
+        }
+
+        size_t opt_flags = 0;
+        if (globals.parallel_fail_fast) {
+            opt_flags |= MP_POOL_FAIL_FAST;
+        }
+
+        int pool_status = -1;
+        if (pool_setup->num_used) {
+            pool_status = mp_pool_join(pool_setup, 1, opt_flags);
+            mp_pool_show_summary(pool_setup);
+            COE_CHECK_ABORT(pool_status != 0, "Failure in setup task pool");
+            mp_pool_free(&pool_setup);
+        }
+
+        if (pool_parallel->num_used) {
+            pool_status = mp_pool_join(pool_parallel, globals.cpu_limit, opt_flags);
+            mp_pool_show_summary(pool_parallel);
+            COE_CHECK_ABORT(pool_status != 0, "Failure in parallel task pool");
+            mp_pool_free(&pool_parallel);
+        }
+
+        if (pool_serial->num_used) {
+            pool_status = mp_pool_join(pool_serial, 1, opt_flags);
+            mp_pool_show_summary(pool_serial);
+            COE_CHECK_ABORT(pool_serial != 0, "Failure in serial task pool");
+            mp_pool_free(&pool_serial);
+        }
+    }
+}
+
+int delivery_fixup_test_results(struct Delivery *ctx) {
+    struct dirent *rec;
+    DIR *dp;
+
+    dp = opendir(ctx->storage.results_dir);
+    if (!dp) {
+        perror(ctx->storage.results_dir);
+        return -1;
+    }
+
+    while ((rec = readdir(dp)) != NULL) {
+        char path[PATH_MAX];
+        memset(path, 0, sizeof(path));
+
+        if (!strcmp(rec->d_name, ".") || !strcmp(rec->d_name, "..")) {
+            continue;
+        } else if (!endswith(rec->d_name, ".xml")) {
+            continue;
+        }
+
+        sprintf(path, "%s/%s", ctx->storage.results_dir, rec->d_name);
+        msg(STASIS_MSG_L3, "%s\n", rec->d_name);
+        if (xml_pretty_print_in_place(path, STASIS_XML_PRETTY_PRINT_PROG, STASIS_XML_PRETTY_PRINT_ARGS)) {
+            msg(STASIS_MSG_L3 | STASIS_MSG_WARN, "Failed to rewrite file '%s'\n", rec->d_name);
+        }
+    }
+
+    closedir(dp);
+    return 0;
+}
+
diff --git a/src/stasis_main.c b/src/stasis_main.c
index 72b7cdd..da050f2 100644
--- a/src/stasis_main.c
+++ b/src/stasis_main.c
@@ -441,9 +441,9 @@ int main(int argc, char *argv[]) {
     }
 
     msg(STASIS_MSG_L1, "Conda setup\n");
-    delivery_get_installer_url(&ctx, installer_url);
+    delivery_get_conda_installer_url(&ctx, installer_url);
     msg(STASIS_MSG_L2, "Downloading: %s\n", installer_url);
-    if (delivery_get_installer(&ctx, installer_url)) {
+    if (delivery_get_conda_installer(&ctx, installer_url)) {
         msg(STASIS_MSG_ERROR, "download failed: %s\n", installer_url);
         exit(1);
     }
-- 
cgit 


From 926d1f30206fb16801bfcf76661155cd18ccaab7 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Fri, 20 Sep 2024 08:40:54 -0400
Subject: Replace strncpy with a raw loops to populate the delim_* arrays

* I wasn't able to finesse the nelem argument in a way that prevents the compiler(s) from throwing warnings
---
 src/str.c | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

(limited to 'src')

diff --git a/src/str.c b/src/str.c
index 6afbf73..56ea893 100644
--- a/src/str.c
+++ b/src/str.c
@@ -223,21 +223,35 @@ char *join_ex(char *separator, ...) {
 }
 
 char *substring_between(char *sptr, const char *delims) {
+    char delim_open[255] = {0};
+    char delim_close[255] = {0};
     if (sptr == NULL || delims == NULL) {
         return NULL;
     }
 
     // Ensure we have enough delimiters to continue
     size_t delim_count = strlen(delims);
-    if (delim_count < 2 || delim_count % 2) {
+    if (delim_count < 2 || delim_count % 2 || (delim_count > (sizeof(delim_open) - 1)) != 0) {
         return NULL;
     }
+    size_t delim_take = delim_count / 2;
 
-    char delim_open[255] = {0};
-    strncpy(delim_open, delims, delim_count / 2);
+    // How else am I supposed to consume the first and last n chars of the string? Give me a break.
+    // warning: ‘__builtin___strncpy_chk’ specified bound depends on the length of the source argument
+    // ---
+    //strncpy(delim_open, delims, delim_take);
+    size_t i = 0;
+    while (i < delim_take && i < sizeof(delim_open)) {
+        delim_open[i] = delims[i];
+        i++;
+    }
 
-    char delim_close[255] = {0};
-    strcpy(delim_close, &delims[delim_count / 2]);
+    //strncpy(delim_close, &delims[delim_take], delim_take);
+    i = 0;
+    while (i < delim_take && i < sizeof(delim_close)) {
+        delim_close[i] = delims[i + delim_take];
+        i++;
+    }
 
     // Create pointers to the delimiters
     char *start = strstr(sptr, delim_open);
-- 
cgit 


From 8608e897410ca4a5ff94dd9868834a00214526f5 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Fri, 20 Sep 2024 08:42:09 -0400
Subject: value can never be NULL here, and is initialized to zero. Use
 strcpy().

---
 src/strlist.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/strlist.c b/src/strlist.c
index de76744..7a045f1 100644
--- a/src/strlist.c
+++ b/src/strlist.c
@@ -331,7 +331,7 @@ void strlist_set(struct StrList **pStrList, size_t index, char *value) {
         }
 
         memset((*pStrList)->data[index], '\0', strlen(value) + 1);
-        strncpy((*pStrList)->data[index], value, strlen(value));
+        strcpy((*pStrList)->data[index], value);
     }
 }
 
-- 
cgit 


From d76321976cfe42bb03b5e7e48ac9f6d95d97b14a Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Fri, 20 Sep 2024 08:42:33 -0400
Subject: Die if the current working directory cannot be determined

---
 src/template_func_proto.c | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

(limited to 'src')

diff --git a/src/template_func_proto.c b/src/template_func_proto.c
index ebb595e..9f1283f 100644
--- a/src/template_func_proto.c
+++ b/src/template_func_proto.c
@@ -74,7 +74,10 @@ int get_junitxml_file_entrypoint(void *frame, void *data_out) {
     const struct Delivery *ctx = (const struct Delivery *) f->data_in;
 
     char cwd[PATH_MAX] = {0};
-    getcwd(cwd, PATH_MAX - 1);
+    if (!getcwd(cwd, PATH_MAX - 1)) {
+        SYSERROR("unable to determine current working directory: %s", strerror(errno));
+        return -1;
+    }
     char nametmp[PATH_MAX] = {0};
     strcpy(nametmp, cwd);
     char *name = path_basename(nametmp);
@@ -96,7 +99,10 @@ int get_basetemp_dir_entrypoint(void *frame, void *data_out) {
     const struct Delivery *ctx = (const struct Delivery *) f->data_in;
 
     char cwd[PATH_MAX] = {0};
-    getcwd(cwd, PATH_MAX - 1);
+    if (!getcwd(cwd, PATH_MAX - 1)) {
+        SYSERROR("unable to determine current working directory: %s", strerror(errno));
+        return -1;
+    }
     char nametmp[PATH_MAX] = {0};
     strcpy(nametmp, cwd);
     char *name = path_basename(nametmp);
-- 
cgit 


From 0de2a305fc2187f62b3df36d7541e7f4fa254f61 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Fri, 20 Sep 2024 08:45:20 -0400
Subject: Fix string op warnings

* Fix unused-result warnings
---
 src/docker.c         |  5 +++--
 src/environment.c    |  4 ++--
 src/recipe.c         |  2 +-
 src/stasis_indexer.c |  4 +++-
 src/utils.c          | 13 ++++++++-----
 5 files changed, 17 insertions(+), 11 deletions(-)

(limited to 'src')

diff --git a/src/docker.c b/src/docker.c
index da7c1ce..2d5e2cc 100644
--- a/src/docker.c
+++ b/src/docker.c
@@ -44,8 +44,9 @@ int docker_script(const char *image, char *data, unsigned flags) {
 
     do {
         memset(buffer, 0, sizeof(buffer));
-        fgets(buffer, sizeof(buffer) - 1, infile);
-        fputs(buffer, outfile);
+        if (fgets(buffer, sizeof(buffer) - 1, infile) != NULL) {
+            fputs(buffer, outfile);
+        }
     } while (!feof(infile));
 
     fclose(infile);
diff --git a/src/environment.c b/src/environment.c
index 924fbf8..580062c 100644
--- a/src/environment.c
+++ b/src/environment.c
@@ -305,7 +305,7 @@ char *runtime_expand_var(RuntimeEnv *env, char *input) {
         // Handle literal statement "$$var"
         // Value becomes "$var" (unexpanded)
         if (strncmp(&input[i], delim_literal, strlen(delim_literal)) == 0) {
-            strncat(expanded, &delim, 1);
+            strncat(expanded, &delim, 2);
             i += strlen(delim_literal);
             // Ignore opening brace
             if (input[i] == '{') {
@@ -349,7 +349,7 @@ char *runtime_expand_var(RuntimeEnv *env, char *input) {
                 continue;
             }
             // Append expanded environment variable to output
-            strncat(expanded, tmp, strlen(tmp));
+            strncat(expanded, tmp, STASIS_BUFSIZ - 1);
             if (env) {
                 guard_free(tmp);
             }
diff --git a/src/recipe.c b/src/recipe.c
index e51fde6..833908c 100644
--- a/src/recipe.c
+++ b/src/recipe.c
@@ -16,7 +16,7 @@ int recipe_clone(char *recipe_dir, char *url, char *gitref, char **result) {
             return -1;
         }
     }
-    strncpy(*result, destdir, PATH_MAX - 1);
+    strncpy(*result, destdir, PATH_MAX);
 
     if (!access(destdir, F_OK)) {
         if (!strcmp(destdir, "/")) {
diff --git a/src/stasis_indexer.c b/src/stasis_indexer.c
index ef6375b..8ee575e 100644
--- a/src/stasis_indexer.c
+++ b/src/stasis_indexer.c
@@ -381,7 +381,9 @@ int indexer_make_website(struct Delivery *ctx) {
                 char link_dest[PATH_MAX] = {0};
                 strcpy(link_from, "README.html");
                 sprintf(link_dest, "%s/%s", root, "index.html");
-                symlink(link_from, link_dest);
+                if (symlink(link_from, link_dest)) {
+                    SYSERROR("Warning: symlink(%s, %s) failed: %s", link_from, link_dest, strerror(errno));
+                }
             }
         }
         guard_strlist_free(&inputs);
diff --git a/src/utils.c b/src/utils.c
index c0b3733..63c0cb7 100644
--- a/src/utils.c
+++ b/src/utils.c
@@ -34,7 +34,7 @@ int popd() {
 int rmtree(char *_path) {
     int status = 0;
     char path[PATH_MAX] = {0};
-    strncpy(path, _path, sizeof(path));
+    strncpy(path, _path, sizeof(path) - 1);
     DIR *dir;
     struct dirent *d_entity;
 
@@ -122,10 +122,10 @@ char *expandpath(const char *_path) {
     }
 
     // Construct the new path
-    strncat(result, home, PATH_MAX - 1);
+    strncat(result, home, sizeof(result) - strlen(home) + 1);
     if (sep) {
-        strncat(result, DIR_SEP, PATH_MAX - 1);
-        strncat(result, ptmp, PATH_MAX - 1);
+        strncat(result, DIR_SEP, sizeof(result) - strlen(home) + 1);
+        strncat(result, ptmp, sizeof(result) - strlen(home) + 1);
     }
 
     return strdup(result);
@@ -444,7 +444,10 @@ void msg(unsigned type, char *fmt, ...) {
 
 void debug_shell() {
     msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "ENTERING STASIS DEBUG SHELL\n" STASIS_COLOR_RESET);
-    system("/bin/bash -c 'PS1=\"(STASIS DEBUG) \\W $ \" bash --norc --noprofile'");
+    if (system("/bin/bash -c 'PS1=\"(STASIS DEBUG) \\W $ \" bash --norc --noprofile'") < 0) {
+        SYSERROR("unable to spawn debug shell: %s", strerror(errno));
+        exit(errno);
+    }
     msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "EXITING STASIS DEBUG SHELL\n" STASIS_COLOR_RESET);
     exit(255);
 }
-- 
cgit 


From fb4a7d1b212c9c6a51551e5043f6941a2371d075 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Thu, 26 Sep 2024 09:51:59 -0400
Subject: Remove usage of POSIX semaphores

---
 src/multiprocessing.c | 35 +++++++++--------------------------
 1 file changed, 9 insertions(+), 26 deletions(-)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index 7452314..433d57a 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -11,10 +11,9 @@ int child(struct MultiProcessingPool *pool, struct MultiProcessingTask *task, co
     char *cwd = NULL;
     FILE *fp_log = NULL;
 
-    // Synchronize sub-process startup
-    // Stop here until summoned by mp_pool_join()
-    if (sem_wait(task->gate) < 0) {
-        perror("sem_wait failed");
+    // The task starts inside the requested working directory
+    if (chdir(task->working_dir)) {
+        perror(task->working_dir);
         exit(1);
     }
 
@@ -142,16 +141,6 @@ struct MultiProcessingTask *mp_pool_task(struct MultiProcessingPool *pool, const
     fflush(tp);
     fclose(tp);
 
-    // Create a uniquely named semaphore.
-    // This is used by the child process to prevent task execution until mp_pool_join is called
-    char sema_name[PATH_MAX] = {0};
-    sprintf(sema_name, "/sem-%zu-%s-%s", mp_global_task_count, pool->ident, slot->ident);
-    sem_unlink(sema_name);
-    slot->gate = sem_open(sema_name, O_CREAT, 0644, 0);
-    if (slot->gate == SEM_FAILED) {
-        perror("sem_open failed");
-        exit(1);
-    }
 
     // Execute task
     if (mp_task_fork(pool, slot, cmd)) {
@@ -265,11 +254,12 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) {
         }
         for (size_t i = lower_i; i < upper_i; i++) {
             struct MultiProcessingTask *slot = &pool->task[i];
-            // Unlock the semaphore to allow the queued processes to start up
-            if (sem_post(slot->gate) < 0) {
-                perror("sem_post failed");
-                exit(1);
-            }
+            if (slot->status == -1) {
+                if (mp_task_fork(pool, slot)) {
+                    fprintf(stderr, "%s: mp_task_fork failed\n", slot->ident);
+                    exit(1);
+                }
+            } 
 
             // Has the child been processed already?
             if (slot->pid == MP_POOL_PID_UNUSED) {
@@ -430,13 +420,6 @@ struct MultiProcessingPool *mp_pool_init(const char *ident, const char *log_root
 
 void mp_pool_free(struct MultiProcessingPool **pool) {
     for (size_t i = 0; i < (*pool)->num_alloc; i++) {
-        // Close all semaphores
-        if ((*pool)->task[i].gate) {
-            if (sem_close((*pool)->task[i].gate) < 0) {
-                perror("sem_close failed");
-                exit(1);
-            }
-        }
     }
     // Unmap all pool tasks
     if ((*pool)->task) {
-- 
cgit 


From 3e610e935858995f411df7bb0a77e2efeeae3d66 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Thu, 26 Sep 2024 09:56:25 -0400
Subject: Implement cmd and working_dir

---
 src/multiprocessing.c | 29 +++++++++++++----------------
 1 file changed, 13 insertions(+), 16 deletions(-)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index 433d57a..5c66672 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -7,8 +7,7 @@ static struct MultiProcessingTask *mp_pool_next_available(struct MultiProcessing
     return &pool->task[pool->num_used];
 }
 
-int child(struct MultiProcessingPool *pool, struct MultiProcessingTask *task, const char *cmd) {
-    char *cwd = NULL;
+int child(struct MultiProcessingPool *pool, struct MultiProcessingTask *task) {
     FILE *fp_log = NULL;
 
     // The task starts inside the requested working directory
@@ -42,23 +41,14 @@ int child(struct MultiProcessingPool *pool, struct MultiProcessingTask *task, co
         timebuf[strlen(timebuf) ? strlen(timebuf) - 1 : 0] = 0;
     }
 
-    // Get the path to the current working directory
-    // Used by the log header. Informative only.
-    cwd = getcwd(NULL, PATH_MAX);
-    if (!cwd) {
-        perror(cwd);
-        exit(1);
-    }
-
     // Generate log header
     fprintf(fp_log, "# STARTED: %s\n", timebuf ? timebuf : "unknown");
     fprintf(fp_log, "# PID: %d\n", task->parent_pid);
-    fprintf(fp_log, "# WORKDIR: %s\n", cwd);
-    fprintf(fp_log, "# COMMAND:\n%s\n", cmd);
+    fprintf(fp_log, "# WORKDIR: %s\n", task->working_dir);
+    fprintf(fp_log, "# COMMAND:\n%s\n", task->cmd);
     fprintf(fp_log, "# OUTPUT:\n");
     // Commit header to log file / clean up
     fflush(fp_log);
-    guard_free(cwd);
 
     // Execute task
     fflush(stdout);
@@ -87,18 +77,18 @@ int parent(struct MultiProcessingPool *pool, struct MultiProcessingTask *task, p
     return 0;
 }
 
-static int mp_task_fork(struct MultiProcessingPool *pool, struct MultiProcessingTask *task, const char *cmd) {
+static int mp_task_fork(struct MultiProcessingPool *pool, struct MultiProcessingTask *task) {
     pid_t pid = fork();
     int child_status = 0;
     if (pid == -1) {
         return -1;
     } else if (pid == 0) {
-        child(pool, task, cmd);
+        child(pool, task);
     }
     return parent(pool, task, pid, &child_status);
 }
 
-struct MultiProcessingTask *mp_pool_task(struct MultiProcessingPool *pool, const char *ident, char *cmd) {
+struct MultiProcessingTask *mp_pool_task(struct MultiProcessingPool *pool, const char *ident, char *working_dir, char *cmd) {
     struct MultiProcessingTask *slot = mp_pool_next_available(pool);
     if (pool->num_used != pool->num_alloc) {
         pool->num_used++;
@@ -118,6 +108,13 @@ struct MultiProcessingTask *mp_pool_task(struct MultiProcessingPool *pool, const
     strcat(slot->log_file, pool->log_root);
     strcat(slot->log_file, "/");
 
+    // Set working directory
+    if (isempty(working_dir)) {
+        strcpy(slot->working_dir, ".");
+    } else {
+        strcpy(slot->working_dir, working_dir);
+    }
+
     // Create a temporary file to act as our intermediate command script
     FILE *tp = NULL;
     char *t_name;
-- 
cgit 


From 42807d1d62b85b00c7898d0434bb278a780237f9 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Thu, 26 Sep 2024 09:57:38 -0400
Subject: Code size: Consolidate pools into an array

---
 src/delivery_test.c | 76 +++++++++++++++++++++++++++++++----------------------
 1 file changed, 45 insertions(+), 31 deletions(-)

(limited to 'src')

diff --git a/src/delivery_test.c b/src/delivery_test.c
index beea510..177a9e7 100644
--- a/src/delivery_test.c
+++ b/src/delivery_test.c
@@ -1,9 +1,10 @@
 #include "delivery.h"
 
 void delivery_tests_run(struct Delivery *ctx) {
-    struct MultiProcessingPool *pool_parallel;
-    struct MultiProcessingPool *pool_serial;
-    struct MultiProcessingPool *pool_setup;
+    static const int SETUP = 0;
+    static const int PARALLEL = 1;
+    static const int SERIAL = 2;
+    struct MultiProcessingPool *pool[3];
     struct Process proc;
     memset(&proc, 0, sizeof(proc));
 
@@ -17,20 +18,20 @@ void delivery_tests_run(struct Delivery *ctx) {
     if (!ctx->tests[0].name) {
         msg(STASIS_MSG_WARN | STASIS_MSG_L2, "no tests are defined!\n");
     } else {
-        pool_parallel = mp_pool_init("parallel", ctx->storage.tmpdir);
-        if (!pool_parallel) {
+        pool[PARALLEL] = mp_pool_init("parallel", ctx->storage.tmpdir);
+        if (!pool[PARALLEL]) {
             perror("mp_pool_init/parallel");
             exit(1);
         }
 
-        pool_serial = mp_pool_init("serial", ctx->storage.tmpdir);
-        if (!pool_serial) {
+        pool[SERIAL] = mp_pool_init("serial", ctx->storage.tmpdir);
+        if (!pool[SERIAL]) {
             perror("mp_pool_init/serial");
             exit(1);
         }
 
-        pool_setup = mp_pool_init("setup", ctx->storage.tmpdir);
-        if (!pool_setup) {
+        pool[SETUP] = mp_pool_init("setup", ctx->storage.tmpdir);
+        if (!pool[SETUP]) {
             perror("mp_pool_init/setup");
             exit(1);
         }
@@ -104,15 +105,15 @@ void delivery_tests_run(struct Delivery *ctx) {
                 char runner_cmd[0xFFFF] = {0};
                 char pool_name[100] = "parallel";
                 struct MultiProcessingTask *task = NULL;
-                struct MultiProcessingPool *pool = pool_parallel;
+                int selected = PARALLEL;
                 if (!test->parallel) {
-                    pool = pool_serial;
+                    selected = SERIAL;
                     memset(pool_name, 0, sizeof(pool_name));
                     strcpy(pool_name, "serial");
                 }
 
                 sprintf(runner_cmd, runner_cmd_fmt, cmd);
-                task = mp_pool_task(pool, test->name, runner_cmd);
+                task = mp_pool_task(pool[selected], test->name, destdir, runner_cmd);
                 if (!task) {
                     SYSERROR("Failed to add task to %s pool: %s", pool_name, runner_cmd);
                     popd();
@@ -160,12 +161,11 @@ void delivery_tests_run(struct Delivery *ctx) {
                         exit(1);
                     }
 
-                    struct MultiProcessingPool *pool = pool_setup;
                     struct MultiProcessingTask *task = NULL;
                     char runner_cmd[0xFFFF] = {0};
                     sprintf(runner_cmd, runner_cmd_fmt, cmd);
 
-                    task = mp_pool_task(pool, test->name, runner_cmd);
+                    task = mp_pool_task(pool[SETUP], test->name, destdir, runner_cmd);
                     if (!task) {
                         SYSERROR("Failed to add task %s to setup pool: %s", test->name, runner_cmd);
                         popd();
@@ -178,6 +178,9 @@ void delivery_tests_run(struct Delivery *ctx) {
                     }
                     guard_free(cmd);
                     popd();
+                } else {
+                    SYSERROR("Failed to change directory: %s\n", destdir);
+                    exit(1);
                 }
             }
         }
@@ -187,26 +190,37 @@ void delivery_tests_run(struct Delivery *ctx) {
             opt_flags |= MP_POOL_FAIL_FAST;
         }
 
-        int pool_status = -1;
-        if (pool_setup->num_used) {
-            pool_status = mp_pool_join(pool_setup, 1, opt_flags);
-            mp_pool_show_summary(pool_setup);
-            COE_CHECK_ABORT(pool_status != 0, "Failure in setup task pool");
-            mp_pool_free(&pool_setup);
-        }
+        for (size_t p = 0; p < sizeof(pool) / sizeof(*pool); p++) {
+            int pool_status;
+            long jobs = globals.cpu_limit;
+
+            if (!pool[p]->num_used) {
+                // Skip empty pool
+                continue;
+            }
+
+            // Setup tasks run sequentially
+            if (p == (size_t) SETUP) {
+                jobs = 1;
+            }
 
-        if (pool_parallel->num_used) {
-            pool_status = mp_pool_join(pool_parallel, globals.cpu_limit, opt_flags);
-            mp_pool_show_summary(pool_parallel);
-            COE_CHECK_ABORT(pool_status != 0, "Failure in parallel task pool");
-            mp_pool_free(&pool_parallel);
+            // Run tasks in the pool
+            // 1. Setup (builds)
+            // 2. Parallel (fast jobs)
+            // 3. Serial (long jobs)
+            pool_status = mp_pool_join(pool[p], jobs, opt_flags);
+
+            // On error show a summary of the current pool, and die
+            if (pool_status != 0) {
+                mp_pool_show_summary(pool[p]);
+                COE_CHECK_ABORT(true, "Task failure");
+            }
         }
 
-        if (pool_serial->num_used) {
-            pool_status = mp_pool_join(pool_serial, 1, opt_flags);
-            mp_pool_show_summary(pool_serial);
-            COE_CHECK_ABORT(pool_serial != 0, "Failure in serial task pool");
-            mp_pool_free(&pool_serial);
+        // All tasks were successful
+        for (size_t p = 0; p < sizeof(pool) / sizeof(*pool); p++) {
+            mp_pool_show_summary(pool[p]);
+            mp_pool_free(&pool[p]);
         }
     }
 }
-- 
cgit 


From 34aaf0e2bbf47e712a0b55fa3f94387b466f314b Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Thu, 26 Sep 2024 09:58:12 -0400
Subject: Zero log_file to avoid garbage output

---
 src/multiprocessing.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index 5c66672..cd48c6f 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -105,6 +105,7 @@ struct MultiProcessingTask *mp_pool_task(struct MultiProcessingPool *pool, const
     strncpy(slot->ident, ident, sizeof(slot->ident) - 1);
 
     // Set log file path
+    memset(slot->log_file, 0, sizeof(*slot->log_file));
     strcat(slot->log_file, pool->log_root);
     strcat(slot->log_file, "/");
 
-- 
cgit 


From 25cb7df692786c0802dd67f5e32edf77fe344e5c Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Thu, 26 Sep 2024 09:58:47 -0400
Subject: Do not fork immediately after a task is created

---
 src/multiprocessing.c | 4 ----
 1 file changed, 4 deletions(-)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index cd48c6f..2b77059 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -140,10 +140,6 @@ struct MultiProcessingTask *mp_pool_task(struct MultiProcessingPool *pool, const
     fclose(tp);
 
 
-    // Execute task
-    if (mp_task_fork(pool, slot, cmd)) {
-        return NULL;
-    }
     return slot;
 }
 
-- 
cgit 


From c645252886ce1ff63df9ebb252acca81ffa7b8bf Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Thu, 26 Sep 2024 09:58:57 -0400
Subject: Spacing

---
 src/multiprocessing.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index 2b77059..3c3c5eb 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -246,6 +246,7 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) {
             size_t x = upper_i - pool->num_used;
             upper_i -= (size_t) x;
         }
+
         for (size_t i = lower_i; i < upper_i; i++) {
             struct MultiProcessingTask *slot = &pool->task[i];
             if (slot->status == -1) {
-- 
cgit 


From 40d1de9306d6de8a57d4398fef696ad8e1395c08 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Fri, 27 Sep 2024 09:58:43 -0400
Subject: Add comments

---
 src/delivery_test.c | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

(limited to 'src')

diff --git a/src/delivery_test.c b/src/delivery_test.c
index 177a9e7..cb8b434 100644
--- a/src/delivery_test.c
+++ b/src/delivery_test.c
@@ -36,14 +36,33 @@ void delivery_tests_run(struct Delivery *ctx) {
             exit(1);
         }
 
+        // Test block scripts shall exit non-zero on error.
+        // This will fail a test block immediately if "string" is not found in file.txt:
+        //      grep string file.txt
+        //
+        // And this is how to avoid that scenario:
+        // #1:
+        //      if ! grep string file.txt; then
+        //          # handle error
+        //      fi
+        //
+        //  #2:
+        //      grep string file.txt || handle error
+        //
+        //  #3:
+        //      # Use ':' as a NO-OP if/when the result doesn't matter
+        //      grep string file.txt || :
         const char *runner_cmd_fmt = "set -e -x\n%s\n";
+
+        // Iterate over our test records, retrieving the source code for each package, and assigning its scripted tasks
+        // to the appropriate processing pool
         for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) {
             struct Test *test = &ctx->tests[i];
             if (!test->name && !test->repository && !test->script) {
                 // skip unused test records
                 continue;
             }
-            msg(STASIS_MSG_L2, "Executing tests for %s %s\n", test->name, test->version);
+            msg(STASIS_MSG_L2, "Loading tests for %s %s\n", test->name, test->version);
             if (!test->script || !strlen(test->script)) {
                 msg(STASIS_MSG_WARN | STASIS_MSG_L3, "Nothing to do. To fix, declare a 'script' in section: [test:%s]\n",
                     test->name);
@@ -80,7 +99,7 @@ void delivery_tests_run(struct Delivery *ctx) {
                     exit(1);
                 }
 
-                msg(STASIS_MSG_L3, "Testing %s\n", test->name);
+                msg(STASIS_MSG_L3, "Queuing task for %s\n", test->name);
                 memset(&proc, 0, sizeof(proc));
 
                 strcpy(cmd, test->script);
@@ -190,6 +209,7 @@ void delivery_tests_run(struct Delivery *ctx) {
             opt_flags |= MP_POOL_FAIL_FAST;
         }
 
+        // Execute all queued tasks
         for (size_t p = 0; p < sizeof(pool) / sizeof(*pool); p++) {
             int pool_status;
             long jobs = globals.cpu_limit;
-- 
cgit 


From 1ce4673c7c0151f69c2c35984633b6c9512ce97f Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Fri, 27 Sep 2024 09:59:27 -0400
Subject: Only show a summary if a pool ran a job

---
 src/delivery_test.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/delivery_test.c b/src/delivery_test.c
index cb8b434..959a99b 100644
--- a/src/delivery_test.c
+++ b/src/delivery_test.c
@@ -239,7 +239,10 @@ void delivery_tests_run(struct Delivery *ctx) {
 
         // All tasks were successful
         for (size_t p = 0; p < sizeof(pool) / sizeof(*pool); p++) {
-            mp_pool_show_summary(pool[p]);
+            if (pool[p]->num_used) {
+                // Only show pools that actually had jobs to run
+                mp_pool_show_summary(pool[p]);
+            }
             mp_pool_free(&pool[p]);
         }
     }
-- 
cgit 


From c59871a5018927836af991799022c2831cdccb5d Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Fri, 27 Sep 2024 10:00:20 -0400
Subject: "Task started" message is redundant

---
 src/multiprocessing.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index 3c3c5eb..1bd9ec0 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -25,7 +25,8 @@ int child(struct MultiProcessingPool *pool, struct MultiProcessingTask *task) {
     // Redirect stdout and stderr to the log file
     fflush(stdout);
     fflush(stderr);
-    printf("[%s:%s] Task started (pid: %d)\n", pool->ident, task->ident, task->parent_pid);
+    // Set log file name
+    sprintf(task->log_file + strlen(task->log_file), "task-%zu-%d.log", mp_global_task_count, task->parent_pid);
     fp_log = freopen(task->log_file, "w+", stdout);
     if (!fp_log) {
         fprintf(stderr, "unable to open '%s' for writing: %s\n", task->log_file, strerror(errno));
-- 
cgit 


From b2cf2c2acbdda61a510b6cf36f07d14375163887 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Fri, 27 Sep 2024 10:00:43 -0400
Subject: Move log_file path assignment into the child

---
 src/multiprocessing.c | 2 --
 1 file changed, 2 deletions(-)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index 1bd9ec0..8677909 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -65,8 +65,6 @@ int parent(struct MultiProcessingPool *pool, struct MultiProcessingTask *task, p
     task->pid = pid;
     task->parent_pid = pid;
 
-    // Set log file name
-    sprintf(task->log_file + strlen(task->log_file), "task-%zu-%d.log", mp_global_task_count, task->parent_pid);
     mp_global_task_count++;
 
     // Check child's status
-- 
cgit 


From d657705546d8a4758ec1c65ce13ae7d2f954bc95 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Fri, 27 Sep 2024 10:01:22 -0400
Subject: Terminate everything when forking fails

---
 src/multiprocessing.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index 8677909..bd4678a 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -251,7 +251,7 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) {
             if (slot->status == -1) {
                 if (mp_task_fork(pool, slot)) {
                     fprintf(stderr, "%s: mp_task_fork failed\n", slot->ident);
-                    exit(1);
+                    kill(0, SIGTERM);
                 }
             } 
 
-- 
cgit 


From 8012b6f930210701dd62f2ffb21247e58037d80f Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Fri, 27 Sep 2024 10:04:39 -0400
Subject: Show "Task is running" every X seconds, instead of X iterations

---
 src/multiprocessing.c | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index bd4678a..6a97b3b 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -234,8 +234,8 @@ int mp_pool_kill(struct MultiProcessingPool *pool, int signum) {
 
 int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) {
     int status = 0;
-    int watcher = 0;
     int failures = 0;
+    time_t watcher = time(NULL);
     size_t tasks_complete = 0;
     size_t lower_i = 0;
     size_t upper_i = jobs;
@@ -253,7 +253,7 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) {
                     fprintf(stderr, "%s: mp_task_fork failed\n", slot->ident);
                     kill(0, SIGTERM);
                 }
-            } 
+            }
 
             // Has the child been processed already?
             if (slot->pid == MP_POOL_PID_UNUSED) {
@@ -343,11 +343,12 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) {
                 fprintf(stderr, "waitpid failed: %s\n", strerror(errno));
                 return -1;
             } else {
-                if (watcher > 9) {
+                time_t watcher_diff = time(NULL) - watcher;
+                if (watcher_diff == 0) {
                     printf("[%s:%s] Task is running (pid: %d)\n", pool->ident, slot->ident, slot->parent_pid);
-                    watcher = 0;
-                } else {
-                    watcher++;
+                }
+                if (watcher > 9) {
+                    watcher = time(NULL);
                 }
             }
         }
-- 
cgit 


From 8621754d1126b0047a4e5817b85dcf0e8749e721 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Fri, 27 Sep 2024 10:05:14 -0400
Subject: Squelch git's detached HEAD message

---
 src/utils.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/utils.c b/src/utils.c
index 63c0cb7..3a98f28 100644
--- a/src/utils.c
+++ b/src/utils.c
@@ -315,7 +315,7 @@ int git_clone(struct Process *proc, char *url, char *destdir, char *gitref) {
     }
 
     static char command[PATH_MAX];
-    sprintf(command, "%s clone --recursive %s", program, url);
+    sprintf(command, "%s clone -c advice.detachedHead=false --recursive %s", program, url);
     if (destdir && access(destdir, F_OK) < 0) {
         sprintf(command + strlen(command), " %s", destdir);
         result = shell(proc, command);
-- 
cgit 


From 108242ce16fc7d7d9e81a1a6e9783fd9bda8b60c Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Fri, 27 Sep 2024 13:23:07 -0400
Subject: mp_pool_init(): return NULL when ident argument is NULL

* reported by @kmacdonald-stsci
---
 src/multiprocessing.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index 6a97b3b..5d22901 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -375,7 +375,8 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) {
 struct MultiProcessingPool *mp_pool_init(const char *ident, const char *log_root) {
     struct MultiProcessingPool *pool;
 
-    if (!log_root) {
+    if (!ident || !log_root) {
+        // Pool must have an ident string
         // log_root must be set
         return NULL;
     }
-- 
cgit 


From 9f535ba4e016f02c6d1bca45d6adfd04a036c9c1 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Fri, 27 Sep 2024 13:24:10 -0400
Subject: Fix missing COMMAND string in the log header

---
 src/multiprocessing.c | 10 ++++++++++
 1 file changed, 10 insertions(+)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index 5d22901..1320a2a 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -138,6 +138,11 @@ struct MultiProcessingTask *mp_pool_task(struct MultiProcessingPool *pool, const
     fflush(tp);
     fclose(tp);
 
+    // Record the command(s)
+    slot->cmd_len = (strlen(cmd) * sizeof(*cmd)) + 1;
+    slot->cmd = mmap(NULL, slot->cmd_len, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
+    memset(slot->cmd, 0, slot->cmd_len);
+    strncpy(slot->cmd, cmd, slot->cmd_len);
 
     return slot;
 }
@@ -419,6 +424,11 @@ void mp_pool_free(struct MultiProcessingPool **pool) {
     }
     // Unmap all pool tasks
     if ((*pool)->task) {
+        if ((*pool)->task->cmd) {
+            if (munmap((*pool)->task->cmd, (*pool)->task->cmd_len) < 0) {
+                perror("munmap");
+            }
+        }
         if (munmap((*pool)->task, sizeof(*(*pool)->task) * (*pool)->num_alloc) < 0) {
             perror("munmap");
         }
-- 
cgit 


From a84b874a027fd8007efc10e0602396c4b5da170c Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Fri, 27 Sep 2024 16:29:03 -0400
Subject: Fix leak

* When strdup fails and the temporary file handle is open, close the handle and die.
* reported by @kmacdonald-stsci
---
 src/multiprocessing.c |  2 +-
 src/utils.c           | 11 +++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index 1320a2a..74ba981 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -117,7 +117,7 @@ struct MultiProcessingTask *mp_pool_task(struct MultiProcessingPool *pool, const
 
     // Create a temporary file to act as our intermediate command script
     FILE *tp = NULL;
-    char *t_name;
+    char *t_name = NULL;
     t_name = xmkstemp(&tp, "w");
     if (!t_name || !tp) {
         return NULL;
diff --git a/src/utils.c b/src/utils.c
index 3a98f28..e037088 100644
--- a/src/utils.c
+++ b/src/utils.c
@@ -468,12 +468,23 @@ char *xmkstemp(FILE **fp, const char *mode) {
     fd = mkstemp(t_name);
     *fp = fdopen(fd, mode);
     if (!*fp) {
+        // unable to open, die
         if (fd > 0)
             close(fd);
         *fp = NULL;
         return NULL;
     }
+
     char *path = strdup(t_name);
+    if (!path) {
+        // strdup failed, die
+        if (*fp) {
+            // close the file handle
+            fclose(*fp);
+            *fp = NULL;
+        }
+        // fall through. path is NULL.
+    }
     return path;
 }
 
-- 
cgit 


From d1642b3514f23be90e8292fce4569990c813d8eb Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Mon, 30 Sep 2024 11:54:34 -0400
Subject: Fix leaking of basetemp_path and jxml_path on error

* Reported by @kmacdonald-stsci
---
 src/template_func_proto.c | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'src')

diff --git a/src/template_func_proto.c b/src/template_func_proto.c
index 9f1283f..9c325bb 100644
--- a/src/template_func_proto.c
+++ b/src/template_func_proto.c
@@ -137,12 +137,15 @@ int tox_run_entrypoint(void *frame, void *data_out) {
             }
             char *jxml_path = NULL;
             if (get_junitxml_file_entrypoint(f, &jxml_path)) {
+                guard_free(basetemp_path);
                 return -3;
             }
             const char *tox_target = f->argv[0].t_char_ptr;
             const char *pytest_args = f->argv[1].t_char_ptr;
             if (isempty(toxconf) || !strcmp(toxconf, "/")) {
                 SYSERROR("Unsafe toxconf path: '%s'", toxconf);
+                guard_free(basetemp_path);
+                guard_free(jxml_path);
                 return -4;
             }
             snprintf(*output, STASIS_BUFSIZ - 1, "\npip install tox && (tox -e py%s%s -c %s --root . -- --basetemp=\"%s\" --junitxml=\"%s\" %s ; rm -f '%s')\n", ctx->meta.python_compact, tox_target, toxconf, basetemp_path, jxml_path, pytest_args ? pytest_args : "", toxconf);
-- 
cgit 


From 73219c6e26a4847b3429318148918b78f6452ac1 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Mon, 30 Sep 2024 12:39:09 -0400
Subject: Allocate runner_cmd using asprintf

---
 src/delivery_test.c | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

(limited to 'src')

diff --git a/src/delivery_test.c b/src/delivery_test.c
index 959a99b..804cbc9 100644
--- a/src/delivery_test.c
+++ b/src/delivery_test.c
@@ -121,7 +121,7 @@ void delivery_tests_run(struct Delivery *ctx) {
                     continue;
                 }
 
-                char runner_cmd[0xFFFF] = {0};
+                char *runner_cmd = NULL;
                 char pool_name[100] = "parallel";
                 struct MultiProcessingTask *task = NULL;
                 int selected = PARALLEL;
@@ -131,18 +131,23 @@ void delivery_tests_run(struct Delivery *ctx) {
                     strcpy(pool_name, "serial");
                 }
 
-                sprintf(runner_cmd, runner_cmd_fmt, cmd);
+                if (asprintf(&runner_cmd, runner_cmd_fmt, cmd) < 0) {
+                    SYSERROR("Unable to allocate memory for runner command: %s", strerror(errno));
+                    exit(1);
+                }
                 task = mp_pool_task(pool[selected], test->name, destdir, runner_cmd);
                 if (!task) {
                     SYSERROR("Failed to add task to %s pool: %s", pool_name, runner_cmd);
                     popd();
                     if (!globals.continue_on_error) {
+                        guard_free(runner_cmd);
                         tpl_free();
                         delivery_free(ctx);
                         globals_free();
                     }
                     exit(1);
                 }
+                guard_free(runner_cmd);
                 guard_free(cmd);
                 popd();
 
@@ -161,7 +166,8 @@ void delivery_tests_run(struct Delivery *ctx) {
                     exit(1);
                 }
                 if (!pushd(destdir)) {
-                    char *cmd = calloc(strlen(test->script_setup) + STASIS_BUFSIZ, sizeof(*cmd));
+                    const size_t cmd_len = strlen(test->script_setup) + STASIS_BUFSIZ;
+                    char *cmd = calloc(cmd_len, sizeof(*cmd));
                     if (!cmd) {
                         SYSERROR("Unable to allocate test script_setup buffer: %s", strerror(errno));
                         exit(1);
@@ -181,20 +187,25 @@ void delivery_tests_run(struct Delivery *ctx) {
                     }
 
                     struct MultiProcessingTask *task = NULL;
-                    char runner_cmd[0xFFFF] = {0};
-                    sprintf(runner_cmd, runner_cmd_fmt, cmd);
+                    char *runner_cmd = NULL;
+                    if (asprintf(&runner_cmd, runner_cmd_fmt, cmd) < 0) {
+                        SYSERROR("Unable to allocate memory for runner command: %s", strerror(errno));
+                        exit(1);
+                    }
 
                     task = mp_pool_task(pool[SETUP], test->name, destdir, runner_cmd);
                     if (!task) {
                         SYSERROR("Failed to add task %s to setup pool: %s", test->name, runner_cmd);
                         popd();
                         if (!globals.continue_on_error) {
+                            guard_free(runner_cmd);
                             tpl_free();
                             delivery_free(ctx);
                             globals_free();
                         }
                         exit(1);
                     }
+                    guard_free(runner_cmd);
                     guard_free(cmd);
                     popd();
                 } else {
-- 
cgit 


From 4e45792a63fad1ed8ff159168b44387136e23b12 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Mon, 30 Sep 2024 12:39:43 -0400
Subject: Shorten comment

---
 src/multiprocessing.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index 74ba981..039383e 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -265,8 +265,9 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) {
                 // Child is already used up, skip it
                 hang_check++;
                 if (hang_check >= pool->num_used) {
-                    // Unlikely to happen when called correctly, but if you join a pool that's already finished
-                    // it will spin forever. This protects the program from entering an infinite loop.
+                    // If you join a pool that's already finished it will spin
+                    // forever. This protects the program from entering an
+                    // infinite loop.
                     fprintf(stderr, "%s is deadlocked\n", pool->ident);
                     failures++;
                     goto pool_deadlocked;
-- 
cgit 


From 3881ec0a37f9880d6e602a8205b0fb2bd4d54b4f Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Mon, 30 Sep 2024 12:40:03 -0400
Subject: Replace sprintf with snprintf

---
 src/multiprocessing.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index 039383e..5f5451e 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -292,7 +292,7 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) {
             char progress[1024] = {0};
             if (pid > 0) {
                 double percent = ((double) (tasks_complete + 1) / (double) pool->num_used) * 100;
-                sprintf(progress, "[%s:%s] [%3.1f%%]", pool->ident, slot->ident, percent);
+                snprintf(progress, sizeof(progress) - 1, "[%s:%s] [%3.1f%%]", pool->ident, slot->ident, percent);
 
                 // The process ended in one the following ways
                 // Note: SIGSTOP nor SIGCONT will not increment the tasks_complete counter
-- 
cgit 


From 8a0b0d1977b54e0ccb2a2e30658d31e10cdd3239 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Mon, 30 Sep 2024 12:40:39 -0400
Subject: Replace strcpy with strlcpy

---
 src/delivery_test.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'src')

diff --git a/src/delivery_test.c b/src/delivery_test.c
index 804cbc9..a9153d3 100644
--- a/src/delivery_test.c
+++ b/src/delivery_test.c
@@ -173,11 +173,11 @@ void delivery_tests_run(struct Delivery *ctx) {
                         exit(1);
                     }
 
-                    strcpy(cmd, test->script_setup);
+                    strlcpy(cmd, test->script_setup, cmd_len);
                     char *cmd_rendered = tpl_render(cmd);
                     if (cmd_rendered) {
                         if (strcmp(cmd_rendered, cmd) != 0) {
-                            strcpy(cmd, cmd_rendered);
+                            strlcpy(cmd, cmd_rendered, cmd_len);
                             cmd[strlen(cmd_rendered) ? strlen(cmd_rendered) - 1 : 0] = 0;
                         }
                         guard_free(cmd_rendered);
-- 
cgit 


From d5fa0c23542746c642a00d24956f20afa639392c Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Mon, 30 Sep 2024 12:48:49 -0400
Subject: Replace strlcpy with strncpy (maybe later)

---
 src/delivery_test.c   | 4 ++--
 src/multiprocessing.c | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

(limited to 'src')

diff --git a/src/delivery_test.c b/src/delivery_test.c
index a9153d3..e2bfe97 100644
--- a/src/delivery_test.c
+++ b/src/delivery_test.c
@@ -173,11 +173,11 @@ void delivery_tests_run(struct Delivery *ctx) {
                         exit(1);
                     }
 
-                    strlcpy(cmd, test->script_setup, cmd_len);
+                    strncpy(cmd, test->script_setup, cmd_len - 1);
                     char *cmd_rendered = tpl_render(cmd);
                     if (cmd_rendered) {
                         if (strcmp(cmd_rendered, cmd) != 0) {
-                            strlcpy(cmd, cmd_rendered, cmd_len);
+                            strncpy(cmd, cmd_rendered, cmd_len - 1);
                             cmd[strlen(cmd_rendered) ? strlen(cmd_rendered) - 1 : 0] = 0;
                         }
                         guard_free(cmd_rendered);
diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index 5f5451e..38d85fc 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -112,7 +112,7 @@ struct MultiProcessingTask *mp_pool_task(struct MultiProcessingPool *pool, const
     if (isempty(working_dir)) {
         strcpy(slot->working_dir, ".");
     } else {
-        strcpy(slot->working_dir, working_dir);
+        strncpy(slot->working_dir, working_dir, PATH_MAX - 1);
     }
 
     // Create a temporary file to act as our intermediate command script
-- 
cgit 


From 9366c5660b8626baa4b25a093d19babd294cad60 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Tue, 1 Oct 2024 15:40:01 -0400
Subject: shell: exit program when stream redirection fails

---
 src/system.c | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/system.c b/src/system.c
index a564769..4e605ec 100644
--- a/src/system.c
+++ b/src/system.c
@@ -46,11 +46,19 @@ int shell(struct Process *proc, char *args) {
 
         if (strlen(proc->f_stdout)) {
             fp_out = freopen(proc->f_stdout, "w+", stdout);
+            if (!fp_out) {
+                fprintf(stderr, "Unable to redirect stdout to %s: %s\n", proc->f_stdout, strerror(errno));
+                exit(1);
+            }
         }
 
         if (strlen(proc->f_stderr)) {
             if (!proc->redirect_stderr) {
                 fp_err = freopen(proc->f_stderr, "w+", stderr);
+                if (!fp_err) {
+                    fprintf(stderr, "Unable to redirect stderr to %s: %s\n", proc->f_stdout, strerror(errno));
+                    exit(1);
+                }
             }
         }
 
@@ -59,7 +67,10 @@ int shell(struct Process *proc, char *args) {
                 fclose(fp_err);
                 fclose(stderr);
             }
-            dup2(fileno(stdout), fileno(stderr));
+            if (dup2(fileno(stdout), fileno(stderr)) < 0) {
+                fprintf(stderr, "Unable to redirect stderr to stdout: %s\n", strerror(errno));
+                exit(1);
+            }
         }
 
         return execl("/bin/bash", "bash", "--norc", t_name, (char *) NULL);
-- 
cgit 


From 1ba1fcc61a200c833f4a89849dd9d2577693bfa0 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Tue, 1 Oct 2024 15:40:46 -0400
Subject: Add comment about use of xtrace

---
 src/delivery_test.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/delivery_test.c b/src/delivery_test.c
index e2bfe97..3809768 100644
--- a/src/delivery_test.c
+++ b/src/delivery_test.c
@@ -13,7 +13,9 @@ void delivery_tests_run(struct Delivery *ctx) {
     } else {
         memset(globals.workaround.conda_reactivate, 0, PATH_MAX);
     }
-    snprintf(globals.workaround.conda_reactivate, PATH_MAX - 1, "\nmamba activate ${CONDA_DEFAULT_ENV}\n");
+    // Test blocks always run with xtrace enabled. Disable, and reenable it. Conda's wrappers produce an incredible
+    // amount of debug information.
+    snprintf(globals.workaround.conda_reactivate, PATH_MAX - 1, "\nset +x; mamba activate ${CONDA_DEFAULT_ENV}; set -x\n");
 
     if (!ctx->tests[0].name) {
         msg(STASIS_MSG_WARN | STASIS_MSG_L2, "no tests are defined!\n");
-- 
cgit 


From e462c06cae9f75526fa200531615bdcc47716eef Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Tue, 1 Oct 2024 15:42:41 -0400
Subject: Add missing space to destdir to ensure its separate from the srcdir
 string

* Fix leaks caused by css_filename path and the dirs array
---
 src/stasis_indexer.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

(limited to 'src')

diff --git a/src/stasis_indexer.c b/src/stasis_indexer.c
index 8ee575e..b38c8d0 100644
--- a/src/stasis_indexer.c
+++ b/src/stasis_indexer.c
@@ -72,9 +72,9 @@ int indexer_combine_rootdirs(const char *dest, char **rootdirs, const size_t roo
         if (!access(srcdir_with_output, F_OK)) {
             srcdir = srcdir_with_output;
         }
-        sprintf(cmd + strlen(cmd), "'%s'/ ", srcdir);
+        snprintf(cmd + strlen(cmd), sizeof(srcdir) - strlen(srcdir) + 4, "'%s'/ ", srcdir);
     }
-    sprintf(cmd + strlen(cmd), "%s/", destdir);
+    snprintf(cmd + strlen(cmd), sizeof(cmd) - strlen(destdir) + 1, " %s/", destdir);
 
     if (globals.verbose) {
         puts(cmd);
@@ -368,6 +368,8 @@ int indexer_make_website(struct Delivery *ctx) {
             // This might be negative when killed by a signal.
             // Otherwise, the return code is not critical to us.
             if (system(cmd) < 0) {
+                guard_free(css_filename);
+                guard_strlist_free(&dirs);
                 return 1;
             }
             if (file_replace_text(fullpath_dest, ".md", ".html", 0)) {
@@ -388,6 +390,7 @@ int indexer_make_website(struct Delivery *ctx) {
         }
         guard_strlist_free(&inputs);
     }
+    guard_free(css_filename);
     guard_strlist_free(&dirs);
 
     return 0;
-- 
cgit 


From 23253d98bce3c4b21b887f175c7cd7135614ea5a Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Tue, 1 Oct 2024 15:49:11 -0400
Subject: Use watcher_diff to see how many seconds have elapsed.

---
 src/multiprocessing.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index 38d85fc..1c585ff 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -352,8 +352,7 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) {
                 time_t watcher_diff = time(NULL) - watcher;
                 if (watcher_diff == 0) {
                     printf("[%s:%s] Task is running (pid: %d)\n", pool->ident, slot->ident, slot->parent_pid);
-                }
-                if (watcher > 9) {
+                } else if (watcher_diff > 9) {
                     watcher = time(NULL);
                 }
             }
-- 
cgit 


From 4b948ef52add8c3a93aef6fd38a3928ccf4e36d7 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Wed, 2 Oct 2024 14:54:45 -0400
Subject: Free resources on error in delivery_index_wheel_artifacts

---
 src/delivery_postprocess.c | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'src')

diff --git a/src/delivery_postprocess.c b/src/delivery_postprocess.c
index 43a1105..1a902e3 100644
--- a/src/delivery_postprocess.c
+++ b/src/delivery_postprocess.c
@@ -208,6 +208,7 @@ int delivery_index_wheel_artifacts(struct Delivery *ctx) {
     sprintf(top_index, "%s/index.html", ctx->storage.wheel_artifact_dir);
     top_fp = fopen(top_index, "w+");
     if (!top_fp) {
+        closedir(dp);
         return -2;
     }
 
@@ -223,6 +224,7 @@ int delivery_index_wheel_artifacts(struct Delivery *ctx) {
         sprintf(bottom_index, "%s/%s/index.html", ctx->storage.wheel_artifact_dir, rec->d_name);
         bottom_fp = fopen(bottom_index, "w+");
         if (!bottom_fp) {
+            closedir(dp);
             return -3;
         }
 
@@ -237,6 +239,7 @@ int delivery_index_wheel_artifacts(struct Delivery *ctx) {
         sprintf(dpath, "%s/%s", ctx->storage.wheel_artifact_dir, rec->d_name);
         struct StrList *packages = listdir(dpath);
         if (!packages) {
+            closedir(dp);
             fclose(top_fp);
             fclose(bottom_fp);
             return -4;
-- 
cgit 


From f6c504630ecfa38c1516123019bdf67b921f1107 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Wed, 2 Oct 2024 14:56:59 -0400
Subject: Allow user to define the time interval for "task is running" message

---
 src/delivery_test.c |  3 +++
 src/globals.c       |  1 +
 src/stasis_main.c   | 13 +++++++++++++
 3 files changed, 17 insertions(+)

(limited to 'src')

diff --git a/src/delivery_test.c b/src/delivery_test.c
index 3809768..d79e088 100644
--- a/src/delivery_test.c
+++ b/src/delivery_test.c
@@ -25,18 +25,21 @@ void delivery_tests_run(struct Delivery *ctx) {
             perror("mp_pool_init/parallel");
             exit(1);
         }
+        pool[PARALLEL]->status_interval = globals.pool_status_interval;
 
         pool[SERIAL] = mp_pool_init("serial", ctx->storage.tmpdir);
         if (!pool[SERIAL]) {
             perror("mp_pool_init/serial");
             exit(1);
         }
+        pool[SERIAL]->status_interval = globals.pool_status_interval;
 
         pool[SETUP] = mp_pool_init("setup", ctx->storage.tmpdir);
         if (!pool[SETUP]) {
             perror("mp_pool_init/setup");
             exit(1);
         }
+        pool[SETUP]->status_interval = globals.pool_status_interval;
 
         // Test block scripts shall exit non-zero on error.
         // This will fail a test block immediately if "string" is not found in file.txt:
diff --git a/src/globals.c b/src/globals.c
index 667809b..b93566a 100644
--- a/src/globals.c
+++ b/src/globals.c
@@ -39,6 +39,7 @@ struct STASIS_GLOBAL globals = {
         .enable_testing = true, ///< Toggle [test] block "script" execution. "script_setup" always executes.
         .enable_rewrite_spec_stage_2 = true, ///< Leave template stings in output files
         .parallel_fail_fast = false, ///< Kill ALL multiprocessing tasks immediately on error
+        .pool_status_interval = 30, ///< Report "Task is running"
 };
 
 void globals_free() {
diff --git a/src/stasis_main.c b/src/stasis_main.c
index da050f2..8f14e5f 100644
--- a/src/stasis_main.c
+++ b/src/stasis_main.c
@@ -13,12 +13,14 @@
 #define OPT_OVERWRITE 1005
 #define OPT_NO_REWRITE_SPEC_STAGE_2 1006
 #define OPT_PARALLEL_FAIL_FAST 1007
+#define OPT_POOL_STATUS_INTERVAL 1009
 static struct option long_options[] = {
         {"help", no_argument, 0, 'h'},
         {"version", no_argument, 0, 'V'},
         {"continue-on-error", no_argument, 0, 'C'},
         {"config", required_argument, 0, 'c'},
         {"cpu-limit", required_argument, 0, 'l'},
+        {"pool-status-interval", required_argument, 0, OPT_POOL_STATUS_INTERVAL},
         {"python", required_argument, 0, 'p'},
         {"verbose", no_argument, 0, 'v'},
         {"unbuffered", no_argument, 0, 'U'},
@@ -39,6 +41,7 @@ const char *long_options_help[] = {
         "Allow tests to fail",
         "Read configuration file",
         "Number of processes to spawn concurrently (default: cpus - 1)",
+        "Report task status every n seconds (default: 30)",
         "Override version of Python in configuration",
         "Increase output verbosity",
         "Disable line buffering",
@@ -262,6 +265,16 @@ int main(int argc, char *argv[]) {
             case OPT_PARALLEL_FAIL_FAST:
                 globals.parallel_fail_fast = true;
                 break;
+            case OPT_POOL_STATUS_INTERVAL:
+                globals.pool_status_interval = (int) strtol(optarg, NULL, 10);
+                if (globals.pool_status_interval < 1) {
+                    globals.pool_status_interval = 1;
+                } else if (globals.pool_status_interval > 60 * 10) {
+                    // Possible poor choice alert
+                    fprintf(stderr, "Caution: Excessive pausing between status updates may cause third-party CI/CD"
+                                    " jobs to fail if the stdout/stderr streams are idle for too long!\n");
+                }
+                break;
             case 'U':
                 setenv("PYTHONUNBUFFERED", "1", 1);
                 fflush(stdout);
-- 
cgit 


From 9028e5ef90c1b7f5a42c6bf969ac3c838b570a7e Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Wed, 2 Oct 2024 14:58:31 -0400
Subject: Allow user to define the time interval for "task is running" message

---
 src/multiprocessing.c | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index 1c585ff..baa6df7 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -240,10 +240,10 @@ int mp_pool_kill(struct MultiProcessingPool *pool, int signum) {
 int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) {
     int status = 0;
     int failures = 0;
-    time_t watcher = time(NULL);
     size_t tasks_complete = 0;
     size_t lower_i = 0;
     size_t upper_i = jobs;
+
     do {
         size_t hang_check = 0;
         if (upper_i >= pool->num_used) {
@@ -349,11 +349,16 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) {
                 fprintf(stderr, "waitpid failed: %s\n", strerror(errno));
                 return -1;
             } else {
-                time_t watcher_diff = time(NULL) - watcher;
-                if (watcher_diff == 0) {
+                // Track the number of seconds elapsed for each task.
+                // When a task has executed for longer than status_intervals, print a status update
+                // _seconds represents the time between intervals, not the total runtime of the task
+                slot->_seconds = time(NULL) - slot->_now;
+                if (slot->_seconds >  pool->status_interval) {
+                    slot->_now = time(NULL);
+                    slot->_seconds = 0;
+                }
+                if (slot->_seconds == 0) {
                     printf("[%s:%s] Task is running (pid: %d)\n", pool->ident, slot->ident, slot->parent_pid);
-                } else if (watcher_diff > 9) {
-                    watcher = time(NULL);
                 }
             }
         }
-- 
cgit 


From 6fe8c2572fbf73cee3936ab241fcbfbdd54fe633 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Wed, 2 Oct 2024 15:00:12 -0400
Subject: Allow user to disable parallel mode (shortcut for --cpu-limit=1)

---
 src/delivery_test.c |  2 +-
 src/globals.c       |  1 +
 src/stasis_main.c   | 12 ++++++++++--
 3 files changed, 12 insertions(+), 3 deletions(-)

(limited to 'src')

diff --git a/src/delivery_test.c b/src/delivery_test.c
index d79e088..8ff08cc 100644
--- a/src/delivery_test.c
+++ b/src/delivery_test.c
@@ -130,7 +130,7 @@ void delivery_tests_run(struct Delivery *ctx) {
                 char pool_name[100] = "parallel";
                 struct MultiProcessingTask *task = NULL;
                 int selected = PARALLEL;
-                if (!test->parallel) {
+                if (!globals.enable_parallel || !test->parallel) {
                     selected = SERIAL;
                     memset(pool_name, 0, sizeof(pool_name));
                     strcpy(pool_name, "serial");
diff --git a/src/globals.c b/src/globals.c
index b93566a..1b682cb 100644
--- a/src/globals.c
+++ b/src/globals.c
@@ -38,6 +38,7 @@ struct STASIS_GLOBAL globals = {
         .enable_artifactory_build_info = true, ///< Toggle build-info uploads
         .enable_testing = true, ///< Toggle [test] block "script" execution. "script_setup" always executes.
         .enable_rewrite_spec_stage_2 = true, ///< Leave template stings in output files
+        .enable_parallel = true, ///< Toggle testing in parallel
         .parallel_fail_fast = false, ///< Kill ALL multiprocessing tasks immediately on error
         .pool_status_interval = 30, ///< Report "Task is running"
 };
diff --git a/src/stasis_main.c b/src/stasis_main.c
index 8f14e5f..2fcfcaf 100644
--- a/src/stasis_main.c
+++ b/src/stasis_main.c
@@ -13,7 +13,9 @@
 #define OPT_OVERWRITE 1005
 #define OPT_NO_REWRITE_SPEC_STAGE_2 1006
 #define OPT_PARALLEL_FAIL_FAST 1007
+#define OPT_NO_PARALLEL 1008
 #define OPT_POOL_STATUS_INTERVAL 1009
+
 static struct option long_options[] = {
         {"help", no_argument, 0, 'h'},
         {"version", no_argument, 0, 'V'},
@@ -31,6 +33,7 @@ static struct option long_options[] = {
         {"no-artifactory", no_argument, 0, OPT_NO_ARTIFACTORY},
         {"no-artifactory-build-info", no_argument, 0, OPT_NO_ARTIFACTORY_BUILD_INFO},
         {"no-testing", no_argument, 0, OPT_NO_TESTING},
+        {"no-parallel", no_argument, 0, OPT_NO_PARALLEL},
         {"no-rewrite", no_argument, 0, OPT_NO_REWRITE_SPEC_STAGE_2},
         {0, 0, 0, 0},
 };
@@ -52,6 +55,7 @@ const char *long_options_help[] = {
         "Do not upload artifacts to Artifactory",
         "Do not upload build info objects to Artifactory",
         "Do not execute test scripts",
+        "Do not execute tests in parallel",
         "Do not rewrite paths and URLs in output files",
         NULL,
 };
@@ -224,7 +228,7 @@ int main(int argc, char *argv[]) {
     int user_disabled_docker = false;
     globals.cpu_limit = get_cpu_count();
     if (globals.cpu_limit > 1) {
-        globals.cpu_limit--;
+        globals.cpu_limit--; // max - 1
     }
 
     memset(env_name, 0, sizeof(env_name));
@@ -255,8 +259,9 @@ int main(int argc, char *argv[]) {
                 break;
             case 'l':
                 globals.cpu_limit = strtol(optarg, NULL, 10);
-                if (globals.cpu_limit < 1) {
+                if (globals.cpu_limit <= 1) {
                     globals.cpu_limit = 1;
+                    globals.enable_parallel = false; // No point
                 }
                 break;
             case OPT_ALWAYS_UPDATE_BASE:
@@ -304,6 +309,9 @@ int main(int argc, char *argv[]) {
             case OPT_NO_REWRITE_SPEC_STAGE_2:
                 globals.enable_rewrite_spec_stage_2 = false;
                 break;
+            case OPT_NO_PARALLEL:
+                globals.enable_parallel = false;
+                break;
             case '?':
             default:
                 exit(1);
-- 
cgit 


From b041e8216c05940f5b3676ef5c7e0ce2b4441bc8 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Wed, 2 Oct 2024 15:00:57 -0400
Subject: "Task started" is more accurate than "queued" when this is printed

---
 src/multiprocessing.c | 4 ++--
 src/stasis_main.c     | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

(limited to 'src')

diff --git a/src/multiprocessing.c b/src/multiprocessing.c
index baa6df7..2a1b350 100644
--- a/src/multiprocessing.c
+++ b/src/multiprocessing.c
@@ -1,6 +1,6 @@
 #include "core.h"
 
-/// The sum of all tasks queued by mp_task()
+/// The sum of all tasks started by mp_task()
 size_t mp_global_task_count = 0;
 
 static struct MultiProcessingTask *mp_pool_next_available(struct MultiProcessingPool *pool) {
@@ -59,7 +59,7 @@ int child(struct MultiProcessingPool *pool, struct MultiProcessingTask *task) {
 }
 
 int parent(struct MultiProcessingPool *pool, struct MultiProcessingTask *task, pid_t pid, int *child_status) {
-    printf("[%s:%s] Task queued (pid: %d)\n", pool->ident, task->ident, pid);
+    printf("[%s:%s] Task started (pid: %d)\n", pool->ident, task->ident, pid);
 
     // Give the child process access to our PID value
     task->pid = pid;
diff --git a/src/stasis_main.c b/src/stasis_main.c
index 2fcfcaf..47579a9 100644
--- a/src/stasis_main.c
+++ b/src/stasis_main.c
@@ -12,7 +12,7 @@
 #define OPT_NO_TESTING 1004
 #define OPT_OVERWRITE 1005
 #define OPT_NO_REWRITE_SPEC_STAGE_2 1006
-#define OPT_PARALLEL_FAIL_FAST 1007
+#define OPT_FAIL_FAST 1007
 #define OPT_NO_PARALLEL 1008
 #define OPT_POOL_STATUS_INTERVAL 1009
 
-- 
cgit 


From f0ba8cd378a460f927644e41f49be95d0e956f81 Mon Sep 17 00:00:00 2001
From: Joseph Hunkeler <jhunkeler@gmail.com>
Date: Wed, 2 Oct 2024 15:01:54 -0400
Subject: Rename argument --parallel-fail-fast to --fail-fast

* All tasks are executed by the same machinery under the hood. So have them all react the same way.
---
 src/stasis_main.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'src')

diff --git a/src/stasis_main.c b/src/stasis_main.c
index 47579a9..b5d43e3 100644
--- a/src/stasis_main.c
+++ b/src/stasis_main.c
@@ -27,7 +27,7 @@ static struct option long_options[] = {
         {"verbose", no_argument, 0, 'v'},
         {"unbuffered", no_argument, 0, 'U'},
         {"update-base", no_argument, 0, OPT_ALWAYS_UPDATE_BASE},
-        {"parallel-fail-fast", no_argument, 0, OPT_PARALLEL_FAIL_FAST},
+        {"fail-fast", no_argument, 0, OPT_FAIL_FAST},
         {"overwrite", no_argument, 0, OPT_OVERWRITE},
         {"no-docker", no_argument, 0, OPT_NO_DOCKER},
         {"no-artifactory", no_argument, 0, OPT_NO_ARTIFACTORY},
@@ -49,7 +49,7 @@ const char *long_options_help[] = {
         "Increase output verbosity",
         "Disable line buffering",
         "Update conda installation prior to STASIS environment creation",
-        "On test error, terminate all concurrent tasks",
+        "On error, immediately terminate all tasks",
         "Overwrite an existing release",
         "Do not build docker images",
         "Do not upload artifacts to Artifactory",
@@ -267,7 +267,7 @@ int main(int argc, char *argv[]) {
             case OPT_ALWAYS_UPDATE_BASE:
                 globals.always_update_base_environment = true;
                 break;
-            case OPT_PARALLEL_FAIL_FAST:
+            case OPT_FAIL_FAST:
                 globals.parallel_fail_fast = true;
                 break;
             case OPT_POOL_STATUS_INTERVAL:
-- 
cgit