diff options
| -rw-r--r-- | include/conda.h | 47 | ||||
| -rw-r--r-- | src/cli/stasis/system_requirements.c | 2 | ||||
| -rw-r--r-- | src/cli/stasis/system_requirements.h | 2 | ||||
| -rw-r--r-- | src/lib/core/conda.c | 95 | ||||
| -rw-r--r-- | src/lib/core/delivery.c | 16 | ||||
| -rw-r--r-- | src/lib/core/delivery_build.c | 79 | ||||
| -rw-r--r-- | src/lib/core/delivery_install.c | 18 | ||||
| -rw-r--r-- | tests/test_conda.c | 29 |
8 files changed, 180 insertions, 108 deletions
diff --git a/include/conda.h b/include/conda.h index 1eb42f4..f031479 100644 --- a/include/conda.h +++ b/include/conda.h @@ -11,6 +11,21 @@ #define CONDA_INSTALL_PREFIX "conda" #define PYPI_INDEX_DEFAULT "https://pypi.org/simple" +#define PKG_USE_PIP 0 +#define PKG_USE_CONDA 1 + +#define PKG_NOT_FOUND 0 +#define PKG_FOUND 1 + +#define PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET (-10) +#define PKG_E_SUCCESS (PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET + 0) +#define PKG_INDEX_PROVIDES_E_INTERNAL_MODE_UNKNOWN (PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET + 1) +#define PKG_INDEX_PROVIDES_E_INTERNAL_LOG_HANDLE (PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET + 2) +#define PKG_INDEX_PROVIDES_E_MANAGER_RUNTIME (PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET + 3) +#define PKG_INDEX_PROVIDES_E_MANAGER_SIGNALED (PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET + 4) +#define PKG_INDEX_PROVIDES_E_MANAGER_EXEC (PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET + 5) +#define PKG_INDEX_PROVIDES_FAILED(ECODE) (ECODE <= PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET) + struct MicromambaInfo { char *micromamba_prefix; //!< Path to write micromamba binary char *conda_prefix; //!< Path to install conda base tree @@ -186,19 +201,31 @@ int conda_env_export(char *name, char *output_dir, char *output_filename); int conda_index(const char *path); /** - * Determine whether a simple index contains a package - * @param index_url a file system path or url pointing to a simple index + * Determine whether a package index contains a package + * + * ```c + * int result = pkg_index_provides(USE_PIP, NULL, "numpy>1.26"); + * if (PKG_INDEX_PROVIDES_FAILED(result)) { + * fprintf(stderr, "failed: %s\n", pkg_index_provides_strerror(result)); + * exit(1); + * } else if (result == PKG_NOT_FOUND) { + * // package does not exist upstream + * } else { + * // package exists upstream + * } + * ``` + * + * @param mode USE_PIP + * @param mode USE_CONDA + * @param index a file system path or url pointing to a simple index or conda channel * @param spec a pip package specification (e.g. `name==1.2.3`) - * @return not found = 0, found = 1, error = -1 - */ -int pip_index_provides(const char *index_url, const char *spec); - -/** - * Determine whether conda can find a package in its channel list * @param spec a conda package specification (e.g. `name=1.2.3`) - * @return not found = 0, found = 1, error = -1 + * @return PKG_NOT_FOUND, if not found + * @return PKG_FOUND, if found + * @return PKG_E_INDEX_PROVIDES_{ERROR}, on error (see conda.h) */ -int conda_provides(const char *spec); +int pkg_index_provides(int mode, const char *index, const char *spec); +const char *pkg_index_provides_strerror(int code); char *conda_get_active_environment(); diff --git a/src/cli/stasis/system_requirements.c b/src/cli/stasis/system_requirements.c index 4554b93..53ebbf7 100644 --- a/src/cli/stasis/system_requirements.c +++ b/src/cli/stasis/system_requirements.c @@ -67,7 +67,7 @@ void check_requirements(struct Delivery *ctx) { check_system_env_requirements(); } -char *check_pathvar(struct Delivery *ctx) { +void check_pathvar(struct Delivery *ctx) { char *pathvar = NULL; pathvar = getenv("PATH"); if (!pathvar) { diff --git a/src/cli/stasis/system_requirements.h b/src/cli/stasis/system_requirements.h index 4c2231a..3a6fa25 100644 --- a/src/cli/stasis/system_requirements.h +++ b/src/cli/stasis/system_requirements.h @@ -8,6 +8,6 @@ void check_system_env_requirements(); void check_system_requirements(struct Delivery *ctx); void check_requirements(struct Delivery *ctx); -char *check_pathvar(struct Delivery *ctx); +void check_pathvar(struct Delivery *ctx); #endif //STASIS_SYSTEM_REQUIREMENTS_H diff --git a/src/lib/core/conda.c b/src/lib/core/conda.c index 25069f8..5954f20 100644 --- a/src/lib/core/conda.c +++ b/src/lib/core/conda.c @@ -78,13 +78,27 @@ int pip_exec(const char *args) { return system(command); } -int pip_index_provides(const char *index_url, const char *spec) { +static const char *PKG_ERROR_STR[] = { + "success", + "[internal] unhandled package manager mode", + "[internal] unable to create temporary log for process output", + "package manager encountered an error at runtime", + "package manager received a signal", + "package manager failed to execute", +}; + +const char *pkg_index_provides_strerror(int code) { + code = -code - (-PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET); + return PKG_ERROR_STR[code]; +} + +int pkg_index_provides(int mode, const char *index, const char *spec) { char cmd[PATH_MAX] = {0}; char spec_local[255] = {0}; if (isempty((char *) spec)) { // NULL or zero-length; no package spec means there's nothing to do. - return -1; + return PKG_NOT_FOUND; } // Normalize the local spec string @@ -98,22 +112,36 @@ int pip_index_provides(const char *index_url, const char *spec) { if (logfd < 0) { perror(logfile); remove(logfile); // fail harmlessly if not present - return -1; + return PKG_INDEX_PROVIDES_E_INTERNAL_LOG_HANDLE; } - int status = 0; struct Process proc; memset(&proc, 0, sizeof(proc)); proc.redirect_stderr = 1; strcpy(proc.f_stdout, logfile); - // Do an installation in dry-run mode to see if the package exists in the given index. - snprintf(cmd, sizeof(cmd) - 1, "python -m pip install --dry-run --no-cache --no-deps --index-url=%s '%s'", index_url, spec_local); - status = shell(&proc, cmd); + if (mode == PKG_USE_PIP) { + // Do an installation in dry-run mode to see if the package exists in the given index. + strncpy(cmd, "python -m pip install --dry-run --no-cache --no-deps ", sizeof(cmd) - 1); + if (index) { + snprintf(cmd + strlen(cmd), (sizeof(cmd) - 1) - strlen(cmd), "--index-url='%s' ", index); + } + snprintf(cmd + strlen(cmd), (sizeof(cmd) - 1) - strlen(cmd), "'%s' ", spec_local); + } else if (mode == PKG_USE_CONDA) { + strncpy(cmd, "mamba search ", sizeof(cmd) - 1); + if (index) { + snprintf(cmd + strlen(cmd), (sizeof(cmd) - 1) - strlen(cmd), "--channel '%s' ", index); + } + snprintf(cmd + strlen(cmd), (sizeof(cmd) - 1) - strlen(cmd), "'%s' ", spec_local); + } else { + return PKG_INDEX_PROVIDES_E_INTERNAL_MODE_UNKNOWN; + } // Print errors only when shell() itself throws one - // If some day we want to see the errors thrown by pip too, use this condition instead: (status != 0) + // If some day we want to see the errors thrown by pip too, use this + // condition instead: (status != 0) + status = shell(&proc, cmd); if (status < 0) { FILE *fp = fdopen(logfd, "r"); if (!fp) { @@ -131,7 +159,25 @@ int pip_index_provides(const char *index_url, const char *spec) { } } remove(logfile); - return proc.returncode == 0; + + if (WTERMSIG(proc.returncode)) { + // This gets its own return value because if the external program + // received a signal, even its status is zero, it's not reliable + return PKG_INDEX_PROVIDES_E_MANAGER_SIGNALED; + } + + if (status < 0) { + return PKG_INDEX_PROVIDES_E_MANAGER_EXEC; + } else if (WEXITSTATUS(proc.returncode) > 1) { + // Pip and conda both return 2 on argument parsing errors + return PKG_INDEX_PROVIDES_E_MANAGER_RUNTIME; + } else if (WEXITSTATUS(proc.returncode) == 1) { + // Pip and conda both return 1 when a package is not found. + // Unfortunately this applies to botched version specs, too. + return PKG_NOT_FOUND; + } else { + return PKG_FOUND; + } } int conda_exec(const char *args) { @@ -167,7 +213,6 @@ int conda_exec(const char *args) { } int conda_activate(const char *root, const char *env_name) { - int fd = -1; FILE *fp = NULL; const char *init_script_conda = "/etc/profile.d/conda.sh"; const char *init_script_mamba = "/etc/profile.d/mamba.sh"; @@ -185,7 +230,8 @@ int conda_activate(const char *root, const char *env_name) { // Emulate mktemp()'s behavior. Give us a unique file name, but don't use // the file handle at all. We'll open it as a FILE stream soon enough. sprintf(logfile, "%s/%s", globals.tmpdir, "shell_XXXXXX"); - fd = mkstemp(logfile); + + int fd = mkstemp(logfile); if (fd < 0) { perror(logfile); return -1; @@ -440,33 +486,6 @@ char *conda_get_active_environment() { return result; } -int conda_provides(const char *spec) { - struct Process proc; - memset(&proc, 0, sizeof(proc)); - - // Short circuit: - // Running "mamba search" without an argument will print every package in - // all channels, then return "found". Prevent this. - // No data implies "not found". - if (isempty((char *) spec)) { - return 0; - } - - strcpy(proc.f_stdout, "/dev/null"); - strcpy(proc.f_stderr, "/dev/null"); - - // It's worth noting the departure from using conda_exec() here: - // conda_exec() expects the program output to be visible to the user. - // For this operation we only need the exit value. - char cmd[PATH_MAX] = {0}; - snprintf(cmd, sizeof(cmd) - 1, "mamba search %s", spec); - if (shell(&proc, cmd) < 0) { - fprintf(stderr, "shell: %s", strerror(errno)); - return -1; - } - return proc.returncode == 0; // 0=not_found, 1=found -} - int conda_index(const char *path) { char command[PATH_MAX]; sprintf(command, "index %s", path); diff --git a/src/lib/core/delivery.c b/src/lib/core/delivery.c index 5645dcc..ada708d 100644 --- a/src/lib/core/delivery.c +++ b/src/lib/core/delivery.c @@ -254,20 +254,16 @@ void delivery_defer_packages(struct Delivery *ctx, int type) { int upstream_exists = 0; if (DEFER_PIP == type) { - upstream_exists = pip_index_provides(PYPI_INDEX_DEFAULT, name); + upstream_exists = pkg_index_provides(PKG_USE_PIP, PYPI_INDEX_DEFAULT, name); } else if (DEFER_CONDA == type) { - upstream_exists = conda_provides(name); - } else { - fprintf(stderr, "\nUnknown package type: %d\n", type); - exit(1); + upstream_exists = pkg_index_provides(PKG_USE_CONDA, NULL, name); } - if (upstream_exists < 0) { - fprintf(stderr, "%s's existence command failed for '%s'\n" - "(This may be due to a network/firewall issue!)\n", mode, name); + if (PKG_INDEX_PROVIDES_FAILED(upstream_exists)) { + fprintf(stderr, "%s's existence command failed for '%s': %s\n", + mode, name, pkg_index_provides_strerror(upstream_exists)); exit(1); - } - if (!upstream_exists) { + } else if (upstream_exists == PKG_NOT_FOUND) { build_for_host = 1; } else { build_for_host = 0; diff --git a/src/lib/core/delivery_build.c b/src/lib/core/delivery_build.c index b4d610a..615fa76 100644 --- a/src/lib/core/delivery_build.c +++ b/src/lib/core/delivery_build.c @@ -140,48 +140,59 @@ struct StrList *delivery_build_wheels(struct Delivery *ctx) { return NULL; } - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { - if (!ctx->tests[i].build_recipe && ctx->tests[i].repository) { // build from source - char srcdir[PATH_MAX]; - char wheeldir[PATH_MAX]; - memset(srcdir, 0, sizeof(srcdir)); - memset(wheeldir, 0, sizeof(wheeldir)); + for (size_t p = 0; p < strlist_count(ctx->conda.pip_packages_defer); p++) { + char name[100] = {0}; + char *fullspec = strlist_item(ctx->conda.pip_packages_defer, p); + strncpy(name, fullspec, sizeof(name) - 1); + char *spec = find_version_spec(name); + if (spec) { + *spec = '\0'; + } - sprintf(srcdir, "%s/%s", ctx->storage.build_sources_dir, ctx->tests[i].name); - git_clone(&proc, ctx->tests[i].repository, srcdir, ctx->tests[i].version); + for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { + if ((ctx->tests[i].name && !strcmp(name, ctx->tests[i].name)) && (!ctx->tests[i].build_recipe && ctx->tests[i].repository)) { // build from source + char srcdir[PATH_MAX]; + char wheeldir[PATH_MAX]; + memset(srcdir, 0, sizeof(srcdir)); + memset(wheeldir, 0, sizeof(wheeldir)); - if (ctx->tests[i].repository_remove_tags && strlist_count(ctx->tests[i].repository_remove_tags)) { - filter_repo_tags(srcdir, ctx->tests[i].repository_remove_tags); - } + sprintf(srcdir, "%s/%s", ctx->storage.build_sources_dir, ctx->tests[i].name); + git_clone(&proc, ctx->tests[i].repository, srcdir, ctx->tests[i].version); - if (!pushd(srcdir)) { - char dname[NAME_MAX]; - char outdir[PATH_MAX]; - char cmd[PATH_MAX * 2]; - memset(dname, 0, sizeof(dname)); - memset(outdir, 0, sizeof(outdir)); - memset(cmd, 0, sizeof(outdir)); - - strcpy(dname, ctx->tests[i].name); - tolower_s(dname); - sprintf(outdir, "%s/%s", ctx->storage.wheel_artifact_dir, dname); - if (mkdirs(outdir, 0755)) { - fprintf(stderr, "failed to create output directory: %s\n", outdir); - guard_strlist_free(&result); - return NULL; + if (ctx->tests[i].repository_remove_tags && strlist_count(ctx->tests[i].repository_remove_tags)) { + filter_repo_tags(srcdir, ctx->tests[i].repository_remove_tags); } - sprintf(cmd, "-m build -w -o %s", outdir); - if (python_exec(cmd)) { - fprintf(stderr, "failed to generate wheel package for %s-%s\n", ctx->tests[i].name, ctx->tests[i].version); + if (!pushd(srcdir)) { + char dname[NAME_MAX]; + char outdir[PATH_MAX]; + char cmd[PATH_MAX * 2]; + memset(dname, 0, sizeof(dname)); + memset(outdir, 0, sizeof(outdir)); + memset(cmd, 0, sizeof(outdir)); + + strcpy(dname, ctx->tests[i].name); + tolower_s(dname); + sprintf(outdir, "%s/%s", ctx->storage.wheel_artifact_dir, dname); + if (mkdirs(outdir, 0755)) { + fprintf(stderr, "failed to create output directory: %s\n", outdir); + guard_strlist_free(&result); + return NULL; + } + + sprintf(cmd, "-m build -w -o %s", outdir); + if (python_exec(cmd)) { + fprintf(stderr, "failed to generate wheel package for %s-%s\n", ctx->tests[i].name, + ctx->tests[i].version); + guard_strlist_free(&result); + return NULL; + } + popd(); + } else { + fprintf(stderr, "Unable to enter source directory %s: %s\n", srcdir, strerror(errno)); guard_strlist_free(&result); return NULL; } - popd(); - } else { - fprintf(stderr, "Unable to enter source directory %s: %s\n", srcdir, strerror(errno)); - guard_strlist_free(&result); - return NULL; } } } diff --git a/src/lib/core/delivery_install.c b/src/lib/core/delivery_install.c index 76c3f4a..098e6f4 100644 --- a/src/lib/core/delivery_install.c +++ b/src/lib/core/delivery_install.c @@ -3,9 +3,21 @@ static struct Test *requirement_from_test(struct Delivery *ctx, const char *name) { struct Test *result = NULL; for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { - if (ctx->tests[i].name && !strcmp(name, ctx->tests[i].name)) { - result = &ctx->tests[i]; - break; + char *package_name = strdup(name); + if (package_name) { + char *spec = find_version_spec(package_name); + if (spec) { + *spec = '\0'; + } + + if (ctx->tests[i].name && !strcmp(package_name, ctx->tests[i].name)) { + result = &ctx->tests[i]; + break; + } + guard_free(package_name); + } else { + SYSERROR("unable to allocate memory for package name: %s", name); + return NULL; } } return result; diff --git a/tests/test_conda.c b/tests/test_conda.c index 9ad12c8..84f98bc 100644 --- a/tests/test_conda.c +++ b/tests/test_conda.c @@ -133,16 +133,23 @@ void test_pip_index_provides() { int expected; }; struct testcase tc[] = { - {.pindex = PYPI_INDEX_DEFAULT, .name = "firewatch", .expected = 1}, - {.pindex = PYPI_INDEX_DEFAULT, .name = "doesnotexistfirewatch", .expected = 0}, - {.pindex = "bad_index", .name = "firewatch", .expected = 0}, - {.pindex = PYPI_INDEX_DEFAULT, .name = "", .expected = -1}, - {.pindex = "", .name = "", .expected = -1}, + {.pindex = PYPI_INDEX_DEFAULT, .name = "firewatch", .expected = PKG_FOUND}, + {.pindex = PYPI_INDEX_DEFAULT, .name = "doesnotexistfirewatch", .expected = PKG_NOT_FOUND}, + {.pindex = "bad_index", .name = "firewatch", .expected = PKG_NOT_FOUND}, + {.pindex = PYPI_INDEX_DEFAULT, .name = "", .expected = PKG_NOT_FOUND}, + {.pindex = "", .name = "", .expected = PKG_NOT_FOUND}, }; for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) { struct testcase *test = &tc[i]; - int result = pip_index_provides(test->pindex, test->name) ; + int result = pkg_index_provides(PKG_USE_PIP, test->pindex, test->name); STASIS_ASSERT(result == test->expected, "Unexpected result"); + if (PKG_INDEX_PROVIDES_FAILED(result)) { + fprintf(stderr, "error: %s\n", pkg_index_provides_strerror(result)); + } else if (result == PKG_NOT_FOUND) { + fprintf(stderr, "package not found: '%s'\n", test->name); + } else { + printf("package found: '%s'\n", test->name); + } } } @@ -157,14 +164,14 @@ void test_conda_provides() { int expected; }; struct testcase tc[] = { - {.name = "fitsverify", .expected = 1}, - {.name = "doesnotexistfitsverify", .expected = 0}, - {.name = "", .expected = 0}, + {.name = "fitsverify", .expected = PKG_FOUND}, + {.name = "doesnotexistfitsverify", .expected = PKG_NOT_FOUND}, + {.name = "", .expected = PKG_NOT_FOUND}, }; for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) { struct testcase *test = &tc[i]; - int result = conda_provides(test->name); + int result = pkg_index_provides(PKG_USE_CONDA, NULL, test->name); printf("%s returned %d, expecting %d\n", test->name, result, test->expected); STASIS_ASSERT(result == test->expected, "Unexpected result"); } @@ -185,13 +192,13 @@ int main(int argc, char *argv[]) { test_conda_activate, test_conda_setup_headless, test_conda_provides, + test_pip_index_provides, test_conda_get_active_environment, test_conda_exec, test_python_exec, test_conda_env_create_from_uri, test_conda_env_create_export_remove, test_conda_index, - test_pip_index_provides, test_delivery_gather_tool_versions, }; |