Merge pull request #64 from jhunkeler/combine-provides

Combine functions: pip_index_provides and conda_index_provides

8 files changed, 180 insertions, 108 deletions

diff --git a/include/conda.h b/include/conda.h
index 1eb42f4..f031479 100644
--- a/include/conda.h
+++ b/include/conda.h
@@ -11,6 +11,21 @@
 #define CONDA_INSTALL_PREFIX "conda"
 #define PYPI_INDEX_DEFAULT "https://pypi.org/simple"
 
+#define PKG_USE_PIP 0
+#define PKG_USE_CONDA 1
+
+#define PKG_NOT_FOUND 0
+#define PKG_FOUND 1
+
+#define PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET (-10)
+#define PKG_E_SUCCESS (PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET + 0)
+#define PKG_INDEX_PROVIDES_E_INTERNAL_MODE_UNKNOWN (PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET + 1)
+#define PKG_INDEX_PROVIDES_E_INTERNAL_LOG_HANDLE (PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET + 2)
+#define PKG_INDEX_PROVIDES_E_MANAGER_RUNTIME (PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET + 3)
+#define PKG_INDEX_PROVIDES_E_MANAGER_SIGNALED (PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET + 4)
+#define PKG_INDEX_PROVIDES_E_MANAGER_EXEC (PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET + 5)
+#define PKG_INDEX_PROVIDES_FAILED(ECODE) (ECODE <= PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET)
+
 struct MicromambaInfo {
     char *micromamba_prefix;    //!< Path to write micromamba binary
     char *conda_prefix;         //!< Path to install conda base tree
@@ -186,19 +201,31 @@ int conda_env_export(char *name, char *output_dir, char *output_filename);
 int conda_index(const char *path);
 
 /**
- * Determine whether a simple index contains a package
- * @param index_url a file system path or url pointing to a simple index
+ * Determine whether a package index contains a package
+ *
+ * ```c
+ * int result = pkg_index_provides(USE_PIP, NULL, "numpy>1.26");
+ * if (PKG_INDEX_PROVIDES_FAILED(result)) {
+ *     fprintf(stderr, "failed: %s\n", pkg_index_provides_strerror(result));
+ *     exit(1);
+ * } else if (result == PKG_NOT_FOUND) {
+ *     // package does not exist upstream
+ * } else {
+ *     // package exists upstream
+ * }
+ * ```
+ *
+ * @param mode USE_PIP
+ * @param mode USE_CONDA
+ * @param index a file system path or url pointing to a simple index or conda channel
  * @param spec a pip package specification (e.g. `name==1.2.3`)
- * @return not found = 0, found = 1, error = -1
- */
-int pip_index_provides(const char *index_url, const char *spec);
-
-/**
- * Determine whether conda can find a package in its channel list
  * @param spec a conda package specification (e.g. `name=1.2.3`)
- * @return not found = 0, found = 1, error = -1
+ * @return PKG_NOT_FOUND, if not found
+ * @return PKG_FOUND, if found
+ * @return PKG_E_INDEX_PROVIDES_{ERROR}, on error (see conda.h)
  */
-int conda_provides(const char *spec);
+int pkg_index_provides(int mode, const char *index, const char *spec);
+const char *pkg_index_provides_strerror(int code);
 
 char *conda_get_active_environment();
 
diff --git a/src/cli/stasis/system_requirements.c b/src/cli/stasis/system_requirements.c
index 4554b93..53ebbf7 100644
--- a/src/cli/stasis/system_requirements.c
+++ b/src/cli/stasis/system_requirements.c
@@ -67,7 +67,7 @@ void check_requirements(struct Delivery *ctx) {
     check_system_env_requirements();
 }
 
-char *check_pathvar(struct Delivery *ctx) {
+void check_pathvar(struct Delivery *ctx) {
     char *pathvar = NULL;
     pathvar = getenv("PATH");
     if (!pathvar) {
diff --git a/src/cli/stasis/system_requirements.h b/src/cli/stasis/system_requirements.h
index 4c2231a..3a6fa25 100644
--- a/src/cli/stasis/system_requirements.h
+++ b/src/cli/stasis/system_requirements.h
@@ -8,6 +8,6 @@
 void check_system_env_requirements();
 void check_system_requirements(struct Delivery *ctx);
 void check_requirements(struct Delivery *ctx);
-char *check_pathvar(struct Delivery *ctx);
+void check_pathvar(struct Delivery *ctx);
 
 #endif //STASIS_SYSTEM_REQUIREMENTS_H
diff --git a/src/lib/core/conda.c b/src/lib/core/conda.c
index 25069f8..5954f20 100644
--- a/src/lib/core/conda.c
+++ b/src/lib/core/conda.c
@@ -78,13 +78,27 @@ int pip_exec(const char *args) {
     return system(command);
 }
 
-int pip_index_provides(const char *index_url, const char *spec) {
+static const char *PKG_ERROR_STR[] = {
+    "success",
+    "[internal] unhandled package manager mode",
+    "[internal] unable to create temporary log for process output",
+    "package manager encountered an error at runtime",
+    "package manager received a signal",
+    "package manager failed to execute",
+};
+
+const char *pkg_index_provides_strerror(int code) {
+    code = -code - (-PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET);
+    return PKG_ERROR_STR[code];
+}
+
+int pkg_index_provides(int mode, const char *index, const char *spec) {
     char cmd[PATH_MAX] = {0};
     char spec_local[255] = {0};
 
     if (isempty((char *) spec)) {
         // NULL or zero-length; no package spec means there's nothing to do.
-        return -1;
+        return PKG_NOT_FOUND;
     }
 
     // Normalize the local spec string
@@ -98,22 +112,36 @@ int pip_index_provides(const char *index_url, const char *spec) {
     if (logfd < 0) {
         perror(logfile);
         remove(logfile);  // fail harmlessly if not present
-        return -1;
+        return PKG_INDEX_PROVIDES_E_INTERNAL_LOG_HANDLE;
     }
 
-
     int status = 0;
     struct Process proc;
     memset(&proc, 0, sizeof(proc));
     proc.redirect_stderr = 1;
     strcpy(proc.f_stdout, logfile);
 
-    // Do an installation in dry-run mode to see if the package exists in the given index.
-    snprintf(cmd, sizeof(cmd) - 1, "python -m pip install --dry-run --no-cache --no-deps --index-url=%s '%s'", index_url, spec_local);
-    status = shell(&proc, cmd);
+    if (mode == PKG_USE_PIP) {
+        // Do an installation in dry-run mode to see if the package exists in the given index.
+        strncpy(cmd, "python -m pip install --dry-run --no-cache --no-deps ", sizeof(cmd) - 1);
+        if (index) {
+            snprintf(cmd + strlen(cmd), (sizeof(cmd) - 1) - strlen(cmd), "--index-url='%s' ", index);
+        }
+        snprintf(cmd + strlen(cmd), (sizeof(cmd) - 1) - strlen(cmd), "'%s' ", spec_local);
+    } else if (mode == PKG_USE_CONDA) {
+        strncpy(cmd, "mamba search ", sizeof(cmd) - 1);
+        if (index) {
+            snprintf(cmd + strlen(cmd), (sizeof(cmd) - 1) - strlen(cmd), "--channel '%s' ", index);
+        }
+        snprintf(cmd + strlen(cmd), (sizeof(cmd) - 1) - strlen(cmd), "'%s' ", spec_local);
+    } else {
+        return PKG_INDEX_PROVIDES_E_INTERNAL_MODE_UNKNOWN;
+    }
 
     // Print errors only when shell() itself throws one
-    // If some day we want to see the errors thrown by pip too, use this condition instead:  (status != 0)
+    // If some day we want to see the errors thrown by pip too, use this
+    // condition instead:  (status != 0)
+    status = shell(&proc, cmd);
     if (status < 0) {
         FILE *fp = fdopen(logfd, "r");
         if (!fp) {
@@ -131,7 +159,25 @@ int pip_index_provides(const char *index_url, const char *spec) {
         }
     }
     remove(logfile);
-    return proc.returncode == 0;
+
+    if (WTERMSIG(proc.returncode)) {
+        // This gets its own return value because if the external program
+        // received a signal, even its status is zero, it's not reliable
+        return PKG_INDEX_PROVIDES_E_MANAGER_SIGNALED;
+    }
+
+    if (status < 0) {
+        return PKG_INDEX_PROVIDES_E_MANAGER_EXEC;
+    } else if (WEXITSTATUS(proc.returncode) > 1) {
+        // Pip and conda both return 2 on argument parsing errors
+        return PKG_INDEX_PROVIDES_E_MANAGER_RUNTIME;
+    } else if (WEXITSTATUS(proc.returncode) == 1) {
+        // Pip and conda both return 1 when a package is not found.
+        // Unfortunately this applies to botched version specs, too.
+        return PKG_NOT_FOUND;
+    } else {
+        return PKG_FOUND;
+    }
 }
 
 int conda_exec(const char *args) {
@@ -167,7 +213,6 @@ int conda_exec(const char *args) {
 }
 
 int conda_activate(const char *root, const char *env_name) {
-    int fd = -1;
     FILE *fp = NULL;
     const char *init_script_conda = "/etc/profile.d/conda.sh";
     const char *init_script_mamba = "/etc/profile.d/mamba.sh";
@@ -185,7 +230,8 @@ int conda_activate(const char *root, const char *env_name) {
     // Emulate mktemp()'s behavior. Give us a unique file name, but don't use
     // the file handle at all. We'll open it as a FILE stream soon enough.
     sprintf(logfile, "%s/%s", globals.tmpdir, "shell_XXXXXX");
-    fd = mkstemp(logfile);
+
+    int fd = mkstemp(logfile);
     if (fd < 0) {
        perror(logfile);
        return -1;
@@ -440,33 +486,6 @@ char *conda_get_active_environment() {
     return result;
 }
 
-int conda_provides(const char *spec) {
-    struct Process proc;
-    memset(&proc, 0, sizeof(proc));
-
-    // Short circuit:
-    // Running "mamba search" without an argument will print every package in
-    // all channels, then return "found". Prevent this.
-    // No data implies "not found".
-    if (isempty((char *) spec)) {
-        return 0;
-    }
-
-    strcpy(proc.f_stdout, "/dev/null");
-    strcpy(proc.f_stderr, "/dev/null");
-
-    // It's worth noting the departure from using conda_exec() here:
-    // conda_exec() expects the program output to be visible to the user.
-    // For this operation we only need the exit value.
-    char cmd[PATH_MAX] = {0};
-    snprintf(cmd, sizeof(cmd) - 1, "mamba search %s", spec);
-    if (shell(&proc, cmd) < 0) {
-        fprintf(stderr, "shell: %s", strerror(errno));
-        return -1;
-    }
-    return proc.returncode == 0; // 0=not_found, 1=found
-}
-
 int conda_index(const char *path) {
     char command[PATH_MAX];
     sprintf(command, "index %s", path);
diff --git a/src/lib/core/delivery.c b/src/lib/core/delivery.c
index 5645dcc..ada708d 100644
--- a/src/lib/core/delivery.c
+++ b/src/lib/core/delivery.c
@@ -254,20 +254,16 @@ void delivery_defer_packages(struct Delivery *ctx, int type) {
 
                 int upstream_exists = 0;
                 if (DEFER_PIP == type) {
-                    upstream_exists = pip_index_provides(PYPI_INDEX_DEFAULT, name);
+                    upstream_exists = pkg_index_provides(PKG_USE_PIP, PYPI_INDEX_DEFAULT, name);
                 } else if (DEFER_CONDA == type) {
-                    upstream_exists = conda_provides(name);
-                } else {
-                    fprintf(stderr, "\nUnknown package type: %d\n", type);
-                    exit(1);
+                    upstream_exists = pkg_index_provides(PKG_USE_CONDA, NULL, name);
                 }
 
-                if (upstream_exists < 0) {
-                    fprintf(stderr, "%s's existence command failed for '%s'\n"
-                        "(This may be due to a network/firewall issue!)\n", mode, name);
+                if (PKG_INDEX_PROVIDES_FAILED(upstream_exists)) {
+                    fprintf(stderr, "%s's existence command failed for '%s': %s\n",
+                            mode, name, pkg_index_provides_strerror(upstream_exists));
                     exit(1);
-                }
-                if (!upstream_exists) {
+                } else if (upstream_exists == PKG_NOT_FOUND) {
                     build_for_host = 1;
                 } else {
                     build_for_host = 0;
diff --git a/src/lib/core/delivery_build.c b/src/lib/core/delivery_build.c
index b4d610a..615fa76 100644
--- a/src/lib/core/delivery_build.c
+++ b/src/lib/core/delivery_build.c
@@ -140,48 +140,59 @@ struct StrList *delivery_build_wheels(struct Delivery *ctx) {
         return NULL;
     }
 
-    for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) {
-        if (!ctx->tests[i].build_recipe && ctx->tests[i].repository) { // build from source
-            char srcdir[PATH_MAX];
-            char wheeldir[PATH_MAX];
-            memset(srcdir, 0, sizeof(srcdir));
-            memset(wheeldir, 0, sizeof(wheeldir));
+    for (size_t p = 0; p < strlist_count(ctx->conda.pip_packages_defer); p++) {
+        char name[100] = {0};
+        char *fullspec = strlist_item(ctx->conda.pip_packages_defer, p);
+        strncpy(name, fullspec, sizeof(name) - 1);
+        char *spec = find_version_spec(name);
+        if (spec) {
+            *spec = '\0';
+        }
 
-            sprintf(srcdir, "%s/%s", ctx->storage.build_sources_dir, ctx->tests[i].name);
-            git_clone(&proc, ctx->tests[i].repository, srcdir, ctx->tests[i].version);
+        for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) {
+            if ((ctx->tests[i].name && !strcmp(name, ctx->tests[i].name)) && (!ctx->tests[i].build_recipe && ctx->tests[i].repository)) { // build from source
+                char srcdir[PATH_MAX];
+                char wheeldir[PATH_MAX];
+                memset(srcdir, 0, sizeof(srcdir));
+                memset(wheeldir, 0, sizeof(wheeldir));
 
-            if (ctx->tests[i].repository_remove_tags && strlist_count(ctx->tests[i].repository_remove_tags)) {
-                filter_repo_tags(srcdir, ctx->tests[i].repository_remove_tags);
-            }
+                sprintf(srcdir, "%s/%s", ctx->storage.build_sources_dir, ctx->tests[i].name);
+                git_clone(&proc, ctx->tests[i].repository, srcdir, ctx->tests[i].version);
 
-            if (!pushd(srcdir)) {
-                char dname[NAME_MAX];
-                char outdir[PATH_MAX];
-                char cmd[PATH_MAX * 2];
-                memset(dname, 0, sizeof(dname));
-                memset(outdir, 0, sizeof(outdir));
-                memset(cmd, 0, sizeof(outdir));
-
-                strcpy(dname, ctx->tests[i].name);
-                tolower_s(dname);
-                sprintf(outdir, "%s/%s", ctx->storage.wheel_artifact_dir, dname);
-                if (mkdirs(outdir, 0755)) {
-                    fprintf(stderr, "failed to create output directory: %s\n", outdir);
-                    guard_strlist_free(&result);
-                    return NULL;
+                if (ctx->tests[i].repository_remove_tags && strlist_count(ctx->tests[i].repository_remove_tags)) {
+                    filter_repo_tags(srcdir, ctx->tests[i].repository_remove_tags);
                 }
 
-                sprintf(cmd, "-m build -w -o %s", outdir);
-                if (python_exec(cmd)) {
-                    fprintf(stderr, "failed to generate wheel package for %s-%s\n", ctx->tests[i].name, ctx->tests[i].version);
+                if (!pushd(srcdir)) {
+                    char dname[NAME_MAX];
+                    char outdir[PATH_MAX];
+                    char cmd[PATH_MAX * 2];
+                    memset(dname, 0, sizeof(dname));
+                    memset(outdir, 0, sizeof(outdir));
+                    memset(cmd, 0, sizeof(outdir));
+
+                    strcpy(dname, ctx->tests[i].name);
+                    tolower_s(dname);
+                    sprintf(outdir, "%s/%s", ctx->storage.wheel_artifact_dir, dname);
+                    if (mkdirs(outdir, 0755)) {
+                        fprintf(stderr, "failed to create output directory: %s\n", outdir);
+                        guard_strlist_free(&result);
+                        return NULL;
+                    }
+
+                    sprintf(cmd, "-m build -w -o %s", outdir);
+                    if (python_exec(cmd)) {
+                        fprintf(stderr, "failed to generate wheel package for %s-%s\n", ctx->tests[i].name,
+                                ctx->tests[i].version);
+                        guard_strlist_free(&result);
+                        return NULL;
+                    }
+                    popd();
+                } else {
+                    fprintf(stderr, "Unable to enter source directory %s: %s\n", srcdir, strerror(errno));
                     guard_strlist_free(&result);
                     return NULL;
                 }
-                popd();
-            } else {
-                fprintf(stderr, "Unable to enter source directory %s: %s\n", srcdir, strerror(errno));
-                guard_strlist_free(&result);
-                return NULL;
             }
         }
     }
diff --git a/src/lib/core/delivery_install.c b/src/lib/core/delivery_install.c
index 76c3f4a..098e6f4 100644
--- a/src/lib/core/delivery_install.c
+++ b/src/lib/core/delivery_install.c
@@ -3,9 +3,21 @@
 static struct Test *requirement_from_test(struct Delivery *ctx, const char *name) {
     struct Test *result = NULL;
     for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) {
-        if (ctx->tests[i].name && !strcmp(name, ctx->tests[i].name)) {
-            result = &ctx->tests[i];
-            break;
+        char *package_name = strdup(name);
+        if (package_name) {
+            char *spec = find_version_spec(package_name);
+            if (spec) {
+                *spec = '\0';
+            }
+
+            if (ctx->tests[i].name && !strcmp(package_name, ctx->tests[i].name)) {
+                result = &ctx->tests[i];
+                break;
+            }
+            guard_free(package_name);
+        } else {
+            SYSERROR("unable to allocate memory for package name: %s", name);
+            return NULL;
         }
     }
     return result;
diff --git a/tests/test_conda.c b/tests/test_conda.c
index 9ad12c8..84f98bc 100644
--- a/tests/test_conda.c
+++ b/tests/test_conda.c
@@ -133,16 +133,23 @@ void test_pip_index_provides() {
         int expected;
     };
     struct testcase tc[] = {
-        {.pindex = PYPI_INDEX_DEFAULT, .name = "firewatch", .expected = 1},
-        {.pindex = PYPI_INDEX_DEFAULT, .name = "doesnotexistfirewatch", .expected = 0},
-        {.pindex = "bad_index", .name = "firewatch", .expected = 0},
-        {.pindex = PYPI_INDEX_DEFAULT, .name = "", .expected = -1},
-        {.pindex = "", .name = "", .expected = -1},
+        {.pindex = PYPI_INDEX_DEFAULT, .name = "firewatch", .expected = PKG_FOUND},
+        {.pindex = PYPI_INDEX_DEFAULT, .name = "doesnotexistfirewatch", .expected = PKG_NOT_FOUND},
+        {.pindex = "bad_index", .name = "firewatch", .expected = PKG_NOT_FOUND},
+        {.pindex = PYPI_INDEX_DEFAULT, .name = "", .expected = PKG_NOT_FOUND},
+        {.pindex = "", .name = "", .expected = PKG_NOT_FOUND},
     };
     for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) {
         struct testcase *test = &tc[i];
-        int result = pip_index_provides(test->pindex, test->name) ;
+        int result = pkg_index_provides(PKG_USE_PIP, test->pindex, test->name);
         STASIS_ASSERT(result == test->expected, "Unexpected result");
+        if (PKG_INDEX_PROVIDES_FAILED(result)) {
+            fprintf(stderr, "error: %s\n", pkg_index_provides_strerror(result));
+        } else if (result == PKG_NOT_FOUND) {
+            fprintf(stderr, "package not found: '%s'\n", test->name);
+        } else {
+            printf("package found: '%s'\n", test->name);
+        }
     }
 }
 
@@ -157,14 +164,14 @@ void test_conda_provides() {
         int expected;
     };
     struct testcase tc[] = {
-        {.name = "fitsverify", .expected = 1},
-        {.name = "doesnotexistfitsverify", .expected = 0},
-        {.name = "", .expected = 0},
+        {.name = "fitsverify", .expected = PKG_FOUND},
+        {.name = "doesnotexistfitsverify", .expected = PKG_NOT_FOUND},
+        {.name = "", .expected = PKG_NOT_FOUND},
     };
 
     for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) {
         struct testcase *test = &tc[i];
-        int result = conda_provides(test->name);
+        int result = pkg_index_provides(PKG_USE_CONDA, NULL, test->name);
         printf("%s returned %d, expecting %d\n", test->name, result, test->expected);
         STASIS_ASSERT(result == test->expected, "Unexpected result");
     }
@@ -185,13 +192,13 @@ int main(int argc, char *argv[]) {
         test_conda_activate,
         test_conda_setup_headless,
         test_conda_provides,
+        test_pip_index_provides,
         test_conda_get_active_environment,
         test_conda_exec,
         test_python_exec,
         test_conda_env_create_from_uri,
         test_conda_env_create_export_remove,
         test_conda_index,
-        test_pip_index_provides,
         test_delivery_gather_tool_versions,
     };