Regression tests, envctl, and bug fixes (#13)

* Found too many bugs

* Implements a regression test
* Moves and completely refactors the envctl code
* Allows the user to keep @STR@ values in output files (if you want full control over where external packages comes from post-build)
* Fixes wording in a few places
* envctl redaction is not implemented yet. The original redaction code hasn't been modified.

* Use generic.ini instead of bare_minimum.ini

5 files changed, 187 insertions, 46 deletions

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index a7b06f7..2399dc5 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -24,6 +24,7 @@ add_library(stasis_core STATIC
         junitxml.c
         github.c
         template_func_proto.c
+        envctl.c
 )
 
 add_executable(stasis
diff --git a/src/delivery.c b/src/delivery.c
index b27ab08..d7b9b99 100644
--- a/src/delivery.c
+++ b/src/delivery.c
@@ -1676,7 +1676,7 @@ void delivery_rewrite_spec(struct Delivery *ctx, char *filename, unsigned stage)
         }
         remove(tempfile);
         guard_free(tempfile);
-    } else if (stage == DELIVERY_REWRITE_SPEC_STAGE_2) {
+    } else if (globals.enable_rewrite_spec_stage_2 && stage == DELIVERY_REWRITE_SPEC_STAGE_2) {
         // Replace "local" channel with the staging URL
         if (ctx->storage.conda_staging_url) {
             file_replace_text(filename, "@CONDA_CHANNEL@", ctx->storage.conda_staging_url, 0);
@@ -1684,15 +1684,19 @@ void delivery_rewrite_spec(struct Delivery *ctx, char *filename, unsigned stage)
             sprintf(output, "%s/%s/%s/%s/packages/conda", globals.jfrog.url, globals.jfrog.repo, ctx->meta.mission, ctx->info.build_name);
             file_replace_text(filename, "@CONDA_CHANNEL@", output, 0);
         } else {
-            msg(STASIS_MSG_WARN, "conda_staging_url is not configured\n", filename);
-            file_replace_text(filename, "  - @CONDA_CHANNEL@", "", 0);
+            msg(STASIS_MSG_WARN, "conda_staging_dir is not configured. Using fallback: '%s'\n", ctx->storage.conda_artifact_dir);
+            file_replace_text(filename, "@CONDA_CHANNEL@", ctx->storage.conda_artifact_dir, 0);
         }
 
         if (ctx->storage.wheel_staging_url) {
             file_replace_text(filename, "@PIP_ARGUMENTS@", ctx->storage.wheel_staging_url, 0);
-        } else if (globals.jfrog.repo) {
+        } else if (globals.jfrog.url && globals.jfrog.repo) {
             sprintf(output, "--extra-index-url %s/%s/%s/%s/packages/wheels", globals.jfrog.url, globals.jfrog.repo, ctx->meta.mission, ctx->info.build_name);
             file_replace_text(filename, "@PIP_ARGUMENTS@", output, 0);
+        } else {
+            msg(STASIS_MSG_WARN, "wheel_staging_dir is not configured. Using fallback: '%s'\n", ctx->storage.wheel_artifact_dir);
+            sprintf(output, "--extra-index-url file://%s", ctx->storage.wheel_artifact_dir);
+            file_replace_text(filename, "@PIP_ARGUMENTS@", output, 0);
         }
     }
 }
diff --git a/src/envctl.c b/src/envctl.c
new file mode 100644
index 0000000..78dd760
--- /dev/null
+++ b/src/envctl.c
@@ -0,0 +1,125 @@
+#include "envctl.h"
+#include "core.h"
+
+struct EnvCtl *envctl_init() {
+    struct EnvCtl *result;
+
+    result = calloc(1, sizeof(*result));
+    if (!result) {
+        return NULL;
+    }
+
+    result->num_alloc = STASIS_ENVCTL_DEFAULT_ALLOC;
+    result->item = calloc(result->num_alloc + 1, sizeof(result->item));
+    if (!result->item) {
+        guard_free(result);
+        return NULL;
+    }
+
+    return result;
+}
+
+static int callback_builtin_nop(const void *a, const void *b) {
+    return STASIS_ENVCTL_RET_SUCCESS;
+}
+
+int envctl_register(struct EnvCtl **envctl, unsigned flags, envctl_except_fn *callback, const char *name) {
+    if ((*envctl)->num_used == (*envctl)->num_alloc) {
+        (*envctl)->num_alloc += STASIS_ENVCTL_DEFAULT_ALLOC;
+        struct EnvCtl_Item **tmp = realloc((*envctl)->item, (*envctl)->num_alloc + 1 * sizeof((*envctl)->item));
+        if (!tmp) {
+            return 1;
+        } else {
+            (*envctl)->item = tmp;
+        }
+    }
+
+    struct EnvCtl_Item **item = (*envctl)->item;
+    item[(*envctl)->num_used] = calloc(1, sizeof(*item[0]));
+    if (!item[(*envctl)->num_used]) {
+        return 1;
+    }
+    if (!callback) {
+        callback = &callback_builtin_nop;
+    }
+    item[(*envctl)->num_used]->callback = callback;
+    item[(*envctl)->num_used]->name = name;
+    item[(*envctl)->num_used]->flags = flags;
+
+    (*envctl)->num_used++;
+    return 0;
+}
+
+size_t envctl_get_index(const struct EnvCtl *envctl, const char *name) {
+    for (size_t i = 0; i < envctl->num_used; i++) {
+        if (!strcmp(envctl->item[i]->name, name)) {
+            // pack state flag, outer (struct) index and inner (name) index
+            return 1L << 63L | i;
+        }
+    }
+    return 0;
+}
+
+void envctl_decode_index(size_t in_i, size_t *state, size_t *out_i, size_t *name_i) {
+    *state = ((in_i >> 63L) & 1);
+    *out_i = in_i & 0xffffffffL;
+}
+
+unsigned envctl_check_required(unsigned flags) {
+    return flags & STASIS_ENVCTL_REQUIRED;
+}
+
+unsigned envctl_check_redact(unsigned flags) {
+    return flags & STASIS_ENVCTL_REDACT;
+}
+
+int envctl_check_present(const struct EnvCtl_Item *item, const char *name) {
+    return ((!strcmp(item->name, name)) && getenv(name)) ? 1 : 0;
+}
+
+unsigned envctl_get_flags(const struct EnvCtl *envctl, const char *name) {
+    size_t poll_index = envctl_get_index(envctl, name);
+    size_t id = 0;
+    size_t name_id = 0;
+    size_t state = 0;
+    envctl_decode_index(poll_index, &state, &id, &name_id);
+    if (!state) {
+        return 0;
+    } else {
+        fprintf(stderr, "managed environment variable: %s\n", name);
+    }
+    return envctl->item[id]->flags;
+}
+
+void envctl_do_required(const struct EnvCtl *envctl, int verbose) {
+    for (size_t i = 0; i < envctl->num_used; i++) {
+        struct EnvCtl_Item *item = envctl->item[i];
+        const char *name = item->name;
+        envctl_except_fn *callback = item->callback;
+
+        if (verbose) {
+            msg(STASIS_MSG_L2, "Verifying %s\n", name);
+        }
+        int code = callback((const void *) item, (const void *) name);
+        if (code == STASIS_ENVCTL_RET_IGNORE || code == STASIS_ENVCTL_RET_SUCCESS) {
+            continue;
+        } else if (code == STASIS_ENVCTL_RET_FAIL) {
+            fprintf(stderr, "\n%s must be set. Exiting.\n", name);
+            exit(1);
+        } else {
+            fprintf(stderr, "\nan unknown envctl callback code occurred: %d\n", code);
+            exit(1);
+        }
+    }
+}
+
+void envctl_free(struct EnvCtl **envctl) {
+    if (!envctl) {
+        return;
+    }
+    for (size_t i = 0; i < (*envctl)->num_used; i++) {
+        guard_free((*envctl)->item[i]);
+    }
+    guard_free((*envctl)->item);
+    guard_free(*envctl);
+}
\ No newline at end of file
diff --git a/src/globals.c b/src/globals.c
index 18a32b5..5fdf05d 100644
--- a/src/globals.c
+++ b/src/globals.c
@@ -36,22 +36,7 @@ struct STASIS_GLOBAL globals = {
         .enable_docker = true,
         .enable_artifactory = true,
         .enable_testing = true,
-        .envctl = {
-            {.flags = STASIS_ENVCTL_PASSTHRU, .name = {"TMPDIR", NULL}},
-            {.flags = STASIS_ENVCTL_PASSTHRU, .name = {"STASIS_ROOT", NULL}},
-            {.flags = STASIS_ENVCTL_PASSTHRU, .name = {"STASIS_SYSCONFDIR", NULL}},
-            {.flags = STASIS_ENVCTL_PASSTHRU, .name = {"STASIS_CPU_COUNT", "CPU_COUNT", NULL}},
-            {.flags = STASIS_ENVCTL_REQUIRED | STASIS_ENVCTL_REDACT, .name={"STASIS_GH_TOKEN", "GITHUB_TOKEN", NULL}},
-            {.flags = STASIS_ENVCTL_REDACT, .name = {"STASIS_JF_ACCESS_TOKEN", NULL}},
-            {.flags = STASIS_ENVCTL_PASSTHRU, .name = {"STASIS_JF_USER", NULL}},
-            {.flags = STASIS_ENVCTL_REDACT, .name = {"STASIS_JF_PASSWORD", NULL}},
-            {.flags = STASIS_ENVCTL_REDACT, .name = {"STASIS_JF_SSH_KEY_PATH", NULL}},
-            {.flags = STASIS_ENVCTL_REDACT, .name = {"STASIS_JF_SSH_PASSPHRASE", NULL}},
-            {.flags = STASIS_ENVCTL_REDACT, .name = {"STASIS_JF_CLIENT_CERT_CERT_PATH", NULL}},
-            {.flags = STASIS_ENVCTL_REDACT, .name = {"STASIS_JF_CLIENT_CERT_KEY_PATH", NULL}},
-            {.flags = STASIS_ENVCTL_REQUIRED, .name = {"STASIS_JF_REPO", NULL}},
-            {.flags = 0, .name = {NULL}},
-        }
+        .enable_rewrite_spec_stage_2 = true,
 };
 
 void globals_free() {
@@ -71,4 +56,7 @@ void globals_free() {
     guard_free(globals.jfrog.remote_filename);
     guard_free(globals.workaround.tox_posargs);
     guard_free(globals.workaround.conda_reactivate);
+    if (globals.envctl) {
+        envctl_free(&globals.envctl);
+    }
 }
diff --git a/src/stasis_main.c b/src/stasis_main.c
index 9348140..7e2262a 100644
--- a/src/stasis_main.c
+++ b/src/stasis_main.c
@@ -10,6 +10,7 @@
 #define OPT_NO_ARTIFACTORY 1002
 #define OPT_NO_TESTING 1003
 #define OPT_OVERWRITE 1004
+#define OPT_NO_REWRITE_SPEC_STAGE_2 1005
 static struct option long_options[] = {
         {"help", no_argument, 0, 'h'},
         {"version", no_argument, 0, 'V'},
@@ -23,6 +24,7 @@ static struct option long_options[] = {
         {"no-docker", no_argument, 0, OPT_NO_DOCKER},
         {"no-artifactory", no_argument, 0, OPT_NO_ARTIFACTORY},
         {"no-testing", no_argument, 0, OPT_NO_TESTING},
+        {"no-rewrite", no_argument, 0, OPT_NO_REWRITE_SPEC_STAGE_2},
         {0, 0, 0, 0},
 };
 
@@ -39,6 +41,7 @@ const char *long_options_help[] = {
         "Do not build docker images",
         "Do not upload artifacts to Artifactory",
         "Do not execute test scripts",
+        "Do not rewrite paths and URLs in output files",
         NULL,
 };
 
@@ -98,36 +101,53 @@ static void usage(char *progname) {
     }
 }
 
-static int get_envctl_key_index_(size_t i) {
-    for (int x = 0; x < (int)(sizeof(globals.envctl[i].name) / sizeof(*globals.envctl[i].name)); x++) {
-        const char *name = globals.envctl[i].name[x];
-        if (!name) {
-            return -1;
-        }
-        const char *data = getenv(name);
-        if (data != NULL) {
-            return x;
+static int callback_except_jf(const void *a, const void *b) {
+    const struct EnvCtl_Item *item = a;
+    const char *name = b;
+
+    if (!globals.enable_artifactory) {
+        return STASIS_ENVCTL_RET_IGNORE;
+    }
+
+    if (envctl_check_required(item->flags)) {
+        const char *content = getenv(name);
+        if (!content || isempty((char *) content)) {
+            return STASIS_ENVCTL_RET_FAIL;
         }
     }
-    return -1;
+
+    return STASIS_ENVCTL_RET_SUCCESS;
+}
+
+static int callback_except_gh(const void *a, const void *b) {
+    const struct EnvCtl_Item *item = a;
+    const char *name = b;
+    //printf("GH exception check: %s\n", name);
+    if (envctl_check_required(item->flags) && envctl_check_present(item, name)) {
+        return STASIS_ENVCTL_RET_SUCCESS;
+    }
+
+    return STASIS_ENVCTL_RET_FAIL;
 }
 
 static void check_system_env_requirements() {
     msg(STASIS_MSG_L1, "Checking environment\n");
-    for (size_t i = 0; globals.envctl[i].name[0] != NULL; i++) {
-        unsigned int flags = globals.envctl[i].flags;
-        int key = get_envctl_key_index_(i);
-        if (key < 0) {
-            if (flags & STASIS_ENVCTL_REQUIRED) {
-                if (!strcmp(globals.envctl[i].name[0], "STASIS_JF_REPO") && !globals.enable_artifactory) {
-                    continue;
-                }
-                msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "Environment variable '%s' must be defined.\n",
-                    globals.envctl[i].name[0]);
-                exit(1);
-            }
-        }
-    }
+    globals.envctl = envctl_init();
+    envctl_register(&globals.envctl, STASIS_ENVCTL_PASSTHRU, NULL, "TMPDIR");
+    envctl_register(&globals.envctl, STASIS_ENVCTL_PASSTHRU, NULL, "STASIS_ROOT");
+    envctl_register(&globals.envctl, STASIS_ENVCTL_PASSTHRU, NULL, "STASIS_SYSCONFDIR");
+    envctl_register(&globals.envctl, STASIS_ENVCTL_PASSTHRU, NULL, "STASIS_CPU_COUNT");
+    envctl_register(&globals.envctl, STASIS_ENVCTL_REQUIRED | STASIS_ENVCTL_REDACT, callback_except_gh, "STASIS_GH_TOKEN");
+    envctl_register(&globals.envctl, STASIS_ENVCTL_REQUIRED, callback_except_jf, "STASIS_JF_ARTIFACTORY_URL");
+    envctl_register(&globals.envctl, STASIS_ENVCTL_REDACT, NULL, "STASIS_JF_ACCESS_TOKEN");
+    envctl_register(&globals.envctl, STASIS_ENVCTL_PASSTHRU, NULL, "STASIS_JF_USER");
+    envctl_register(&globals.envctl, STASIS_ENVCTL_REDACT, NULL, "STASIS_JF_PASSWORD");
+    envctl_register(&globals.envctl, STASIS_ENVCTL_REDACT, NULL, "STASIS_JF_SSH_KEY_PATH");
+    envctl_register(&globals.envctl, STASIS_ENVCTL_REDACT, NULL, "STASIS_JF_SSH_PASSPHRASE");
+    envctl_register(&globals.envctl, STASIS_ENVCTL_REDACT, NULL, "STASIS_JF_CLIENT_CERT_CERT_PATH");
+    envctl_register(&globals.envctl, STASIS_ENVCTL_REDACT, NULL, "STASIS_JF_CLIENT_CERT_KEY_PATH");
+    envctl_register(&globals.envctl, STASIS_ENVCTL_REQUIRED, callback_except_jf, "STASIS_JF_REPO");
+    envctl_do_required(globals.envctl, globals.verbose);
 }
 
 static void check_system_requirements(struct Delivery *ctx) {
@@ -244,6 +264,9 @@ int main(int argc, char *argv[]) {
             case OPT_NO_TESTING:
                 globals.enable_testing = false;
                 break;
+            case OPT_NO_REWRITE_SPEC_STAGE_2:
+                globals.enable_rewrite_spec_stage_2 = false;
+                break;
             case '?':
             default:
                 exit(1);
@@ -649,10 +672,10 @@ int main(int argc, char *argv[]) {
             msg(STASIS_MSG_L1, "Uploading artifacts\n");
             delivery_artifact_upload(&ctx);
         } else {
-            msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Artifact uploading is disabled\n");
+            msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Artifactory upload is disabled by CLI argument\n");
         }
     } else {
-        msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Artifact uploading is disabled. deploy:artifactory is not configured\n");
+        msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Artifactory upload is disabled. deploy:artifactory is not configured\n");
     }
 
     msg(STASIS_MSG_L1, "Cleaning up\n");