diff options
| author | Joseph Hunkeler <jhunkeler@users.noreply.github.com> | 2026-06-23 11:00:55 -0400 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2026-06-23 11:00:55 -0400 |
| commit | 5e0b3fbefebaeff1fa5bfb338c760b9b9e270f02 (patch) | |
| tree | 14b22fc5b82a9db87720c820c54422c9b9c47a1c /src | |
| parent | 70c1ba3962166853fc7a1e4f2bb1d637104312b1 (diff) | |
| parent | 0f6a1c982c2f417e9f0de968bbb7ce58299a8e8d (diff) | |
| download | stasis-3c51faf2c6bcb532e9cc737c629b4307ce500de9.tar.gz | |
Merge pull request #147 from jhunkeler/conda-updates1.8.0
Support modern versions of conda/mamba
Diffstat (limited to 'src')
| -rw-r--r-- | src/cli/stasis/stasis_main.c | 2 | ||||
| -rw-r--r-- | src/lib/core/conda.c | 264 | ||||
| -rw-r--r-- | src/lib/core/include/conda.h | 62 | ||||
| -rw-r--r-- | src/lib/core/include/recipe.h | 28 | ||||
| -rw-r--r-- | src/lib/core/include/utils.h | 2 | ||||
| -rw-r--r-- | src/lib/core/recipe.c | 32 | ||||
| -rw-r--r-- | src/lib/core/system.c | 4 | ||||
| -rw-r--r-- | src/lib/core/utils.c | 80 | ||||
| -rw-r--r-- | src/lib/delivery/delivery.c | 43 | ||||
| -rw-r--r-- | src/lib/delivery/delivery_build.c | 67 | ||||
| -rw-r--r-- | src/lib/delivery/delivery_conda.c | 51 | ||||
| -rw-r--r-- | src/lib/delivery/delivery_populate.c | 48 | ||||
| -rw-r--r-- | src/lib/delivery/include/delivery.h | 5 |
13 files changed, 518 insertions, 170 deletions
diff --git a/src/cli/stasis/stasis_main.c b/src/cli/stasis/stasis_main.c index ef7bf26..c5c1f00 100644 --- a/src/cli/stasis/stasis_main.c +++ b/src/cli/stasis/stasis_main.c @@ -176,7 +176,7 @@ static void setup_conda(struct Delivery *ctx, char *installer_url, const size_t delivery_install_conda(ctx->conda.installer_path, ctx->storage.conda_install_prefix); msg(STASIS_MSG_L2, "Configuring: %s\n", ctx->storage.conda_install_prefix); - delivery_conda_enable(ctx, ctx->storage.conda_install_prefix); + delivery_conda_enable(ctx); } static void configure_conda_base(struct Delivery *ctx, char *envs[]) { diff --git a/src/lib/core/conda.c b/src/lib/core/conda.c index 5c7779f..bf27d21 100644 --- a/src/lib/core/conda.c +++ b/src/lib/core/conda.c @@ -194,6 +194,31 @@ int pip_exec(const char *args) { return result; } +char *python_importlib_metadata_version(const char *package_name) { + int status = 0; + char cmd[PATH_MAX] = {0}; + + if (strpbrk(package_name, "\\/*{}()|;&\"'\r\n")) { + SYSERROR("package name is invalid: '%s'", package_name); + return NULL; + } + snprintf(cmd, sizeof(cmd), "python3 -c 'from importlib.metadata import version; print(version(r\x22%s\x22))'", package_name); + + char *version = shell_output(cmd, &status); + if (status) { + SYSERROR("version detection failed"); + guard_free(version); + return NULL; + } + if (!version) { + SYSERROR("unable to allocate version"); + return NULL; + } + strip(version); + return version; +} + + static const char *PKG_ERROR_STR[] = { "success", "[internal] unhandled package manager mode", @@ -227,21 +252,30 @@ int pkg_index_provides(int mode, const char *index, const char *spec, const char SYSERROR("Unable to create log directory: %s", logdir ? logdir : "NULL"); return -1; } - const char logfile_template[] = "STASIS-package_exists.XXXXXX"; - char logfile[PATH_MAX] = {0}; - snprintf(logfile, sizeof(logfile), "%s/%s", logdir, logfile_template); - int logfd = mkstemp(logfile); - if (logfd < 0) { - SYSERROR("unable to create log file: %s", logfile); - remove(logfile); // fail harmlessly if not present - return PKG_INDEX_PROVIDES_E_INTERNAL_LOG_HANDLE; + const int stdout_stream = 0; + const int stderr_stream = 1; + const int stdout_st = 0; + //const int stderr_st = 1; + char logfile[2][PATH_MAX] = {0}; + int logfile_fd[2] = {-1, -1}; + struct stat logfile_st[2] = {0}; + + for (size_t i = 0; i < sizeof(logfile) / sizeof(logfile[0]); i++) { + const char logfile_template[] = "STASIS-package_exists.XXXXXX"; + snprintf(logfile[i], sizeof(logfile[i]), "%s/%s", logdir, logfile_template); + logfile_fd[i] = mkstemp(logfile[i]); + if (logfile_fd[i] < 0) { + SYSERROR("unable to create log file: %s", logfile[i]); + remove(logfile[i]); // fail harmlessly if not present + return PKG_INDEX_PROVIDES_E_INTERNAL_LOG_HANDLE; + } } int status = 0; struct Process proc = {0}; - proc.redirect_stderr = 1; - snprintf(proc.f_stdout, sizeof(proc.f_stdout), "%s", logfile); + snprintf(proc.f_stdout, sizeof(proc.f_stdout), "%s", logfile[stdout_stream]); + snprintf(proc.f_stderr, sizeof(proc.f_stderr), "%s", logfile[stderr_stream]); if (mode == PKG_USE_PIP) { // Do an installation in dry-run mode to see if the package exists in the given index. @@ -268,26 +302,40 @@ int pkg_index_provides(int mode, const char *index, const char *spec, const char SYSDEBUG("Executing: %s", cmd); status = shell(&proc, cmd); - SYSDEBUG("Log file: %s", logfile); - if (status != 0) { - FILE *fp = fdopen(logfd, "r"); - if (!fp) { - remove(logfile); + // Populate stat data for log files + for (size_t i = 0; i < sizeof(logfile) / sizeof(logfile[0]); i++) { + if (stat(logfile[i], &logfile_st[i])) { + SYSERROR("Unable to stat %s", logfile[i]); return -1; } + } + + if (status != 0) { + SYSERROR("Command exited non-zero (%d)", status); + for (size_t i = 0; i < sizeof(logfile) / sizeof(logfile[0]); i++) { + const char *stream_name = i == 0 ? "stdout" : "stderr"; + if (!logfile_st[i].st_size) { + continue; + } + SYSDEBUG("(%s): %s", stream_name, logfile[i]); + FILE *fp = fdopen(logfile_fd[i], "r"); + if (!fp) { + remove(logfile[i]); + return -1; + } - fflush(stdout); - fflush(stderr); + fflush(stdout); + fflush(stderr); - char line[STASIS_BUFSIZ] = {0}; - while (fgets(line, sizeof(line) - 1, fp) != NULL) { - SYSDEBUG("%s", strip(line)); - } + char line[STASIS_BUFSIZ] = {0}; + while (fgets(line, sizeof(line) - 1, fp) != NULL) { + SYSINFO("(%s): %s", stream_name, strip(line)); + } - fflush(stderr); - fclose(fp); + fflush(stderr); + fclose(fp); + } } - remove(logfile); if (WTERMSIG(proc.returncode)) { // This gets its own return value because if the external program @@ -295,18 +343,40 @@ int pkg_index_provides(int mode, const char *index, const char *spec, const char return PKG_INDEX_PROVIDES_E_MANAGER_SIGNALED; } + int final = PKG_FOUND; if (status < 0) { - return PKG_INDEX_PROVIDES_E_MANAGER_EXEC; + final = PKG_INDEX_PROVIDES_E_MANAGER_EXEC; } else if (WEXITSTATUS(proc.returncode) > 1) { // Pip and conda both return 2 on argument parsing errors - return PKG_INDEX_PROVIDES_E_MANAGER_RUNTIME; + final = PKG_INDEX_PROVIDES_E_MANAGER_RUNTIME; + } else if (logfile_st[stdout_st].st_size > 1 && WEXITSTATUS(proc.returncode) == 0) { + // modern mamba return zero when a package not found. + // even more ridiculous; the error messages are written to stdout, not stderr + struct StrList *output = strlist_init(); + if (!output) { + SYSERROR("unable to allocate memory for stdout log list"); + return -1; + } + if (strlist_append_file(output, logfile[stdout_stream], NULL)) { + strlist_free(&output); + SYSERROR("unable to append stdout log to list: %s", logfile[stdout_stream]); + return -1; + } + if (strlist_contains(output, "No entries", NULL)) { + final = PKG_NOT_FOUND; + } + strlist_free(&output); } else if (WEXITSTATUS(proc.returncode) == 1) { // Pip and conda both return 1 when a package is not found. // Unfortunately this applies to botched version specs, too. - return PKG_NOT_FOUND; - } else { - return PKG_FOUND; + final = PKG_NOT_FOUND; } + + // Remove log files + for (size_t i = 0; i < sizeof(logfile) / sizeof(logfile[0]); i++) { + remove(logfile[stdout_stream]); + } + return final; } int conda_exec(const char *args) { @@ -326,14 +396,22 @@ int conda_exec(const char *args) { }; char conda_as[10] = {0}; + const char *last_mamba_command = NULL; safe_strncpy(conda_as, "conda", sizeof(conda_as)); for (size_t i = 0; mamba_commands[i] != NULL; i++) { if (startswith(args, mamba_commands[i])) { + last_mamba_command = mamba_commands[i]; safe_strncpy(conda_as, "mamba", sizeof(conda_as)); break; } } + const int boa_present = find_program("boa") != NULL; + if (!boa_present) { + if (last_mamba_command && !strcmp(last_mamba_command, "build")) { + safe_strncpy(conda_as, "conda", sizeof(conda_as)); + } + } const char *command_fmt = "%s %s"; const int len = snprintf(NULL, 0, command_fmt, conda_as, args); @@ -416,15 +494,12 @@ static int env0_to_runtime(const char *logfile) { int conda_activate(const char *root, const char *env_name) { const char *init_script_conda = "/etc/profile.d/conda.sh"; - const char *init_script_mamba = "/etc/profile.d/mamba.sh"; char path_conda[PATH_MAX] = {0}; - char path_mamba[PATH_MAX] = {0}; char logfile[PATH_MAX] = {0}; struct Process proc = {0}; // Where to find conda's init scripts snprintf(path_conda, sizeof(path_conda), "%s%s", root, init_script_conda); - snprintf(path_mamba, sizeof(path_mamba), "%s%s", root, init_script_mamba); // Set the path to our stdout log // Emulate mktemp()'s behavior. Give us a unique file name, but don't use @@ -448,12 +523,6 @@ int conda_activate(const char *root, const char *env_name) { return -1; } - if (access(path_mamba, F_OK) < 0) { - SYSERROR("mamba is missing: %s, %s", path_mamba, strerror(errno)); - remove(logfile); - return -1; - } - // Fully activate conda and record its effect on the runtime environment char command[PATH_MAX * 3]; const char *conda_shlvl_str = getenv("CONDA_SHLVL"); @@ -475,20 +544,19 @@ int conda_activate(const char *root, const char *env_name) { snprintf(command, sizeof(command), "set -a\n" "source %s\n" - "__conda_exe() (\n" - " \"$CONDA_PYTHON_EXE\" \"$CONDA_EXE\" $_CE_M $_CE_CONDA \"$@\"\n" - ")\n\n" - "export -f __conda_exe\n" - "source %s\n" - "__mamba_exe() (\n" - " \\local MAMBA_CONDA_EXE_BACKUP=$CONDA_EXE\n" - " \\local MAMBA_EXE=$(\\dirname \"${CONDA_EXE}\")/mamba\n" - " \"$CONDA_PYTHON_EXE\" \"$MAMBA_EXE\" $_CE_M $_CE_CONDA \"$@\"\n" - ")\n\n" - "export -f __mamba_exe\n" + "tempfile=$(mktemp)\n" + "chmod 600 \"${tempfile}\"\n" + "%s/bin/mamba shell init --shell bash --dry-run 2>/dev/null\\\n" + "| (ignore=1; \\\n" + " while read line; do \\\n" + " if [[ \"$line\" == \"#\"* ]]; then ignore=0; fi; \\\n" + " if (( ignore == 0 )); then echo $line; fi; \\\n" + " done) 1> \"${tempfile}\"\n" + "source \"${tempfile}\"\n" + "rm -f \"${tempfile}\"\n" "%s\n" "conda activate %s 1>&2\n" - "env -0\n", path_conda, path_mamba, conda_shlvl ? "conda deactivate" : ":", env_name); + "env -0\n", path_conda, root, conda_shlvl ? "conda deactivate" : ":", env_name); int retval = shell(&proc, command); if (retval) { @@ -561,7 +629,20 @@ int conda_check_required() { return 0; } -int conda_setup_headless() { +int conda_setup_headless(struct CondaCapabilities *cc) { + mkdirs(cc->prefix, 0755); + + char rcpath[PATH_MAX]; + snprintf(rcpath, sizeof(rcpath), "%s/.condarc", cc->prefix); + touch(rcpath); + if (errno == ENOENT) { + errno = 0; + } + + setenv("CONDARC", rcpath, 1); + setenv("MAMBARC", rcpath, 1); + setenv("MAMBA_ROOT_PREFIX", cc->prefix, 1); + if (globals.verbose) { conda_exec("config --system --set quiet false"); } else { @@ -582,6 +663,18 @@ int conda_setup_headless() { size_t total = 0; const char *cmd_fmt = "'%s'"; if (globals.conda_packages && strlist_count(globals.conda_packages)) { + // Push or pop build packages based on capabilities + size_t boa_index = 0; + const int boa_requested = strlist_contains(globals.conda_packages, "boa", &boa_index); + if (boa_requested && !cc->require_boa) { + SYSWARN("Removing boa from global package list due to incompatible conda version (too new): %s", cc->conda_version); + strlist_remove(globals.conda_packages, boa_index); + } else if (!boa_requested && cc->require_boa) { + SYSWARN("Adding boa to global package list"); + strlist_append(&globals.conda_packages, "boa"); + } + + memset(cmd, 0, sizeof(cmd)); safe_strncpy(cmd, "install ", sizeof(cmd)); @@ -719,7 +812,18 @@ int conda_env_remove(char *name) { int conda_env_export(char *name, char *output_dir, char *output_filename) { char env_command[PATH_MAX]; - snprintf(env_command, sizeof(env_command), "env export -n %s -f %s/%s.yml", name, output_dir, output_filename); + int vr = 0; + const char *env_format = NULL; + char *version = shell_output("conda --version", &vr); + if (version) { + const size_t v_offset = strlen("conda "); + if (version_compare(GT, version + v_offset, "25.1.0")) { + env_format = "--format=yml"; + } + guard_free(version); + } + + snprintf(env_command, sizeof(env_command), "env export %s -n %s -f %s/%s.yml", env_format ? env_format : "", name, output_dir, output_filename); return conda_exec(env_command); } @@ -749,3 +853,59 @@ int conda_env_exists(const char *root, const char *name) { snprintf(path, sizeof(path), "%s/envs/%s", root, name); return access(path, F_OK) == 0; } + +void conda_capable_free(struct CondaCapabilities *ccap) { + guard_free(ccap->conda_version); + guard_free(ccap->mamba_version); + memset(ccap, 0, sizeof(*ccap)); +} + +int conda_capable(struct CondaCapabilities *ccap, const char *root) { + struct CondaCapabilities *cc = ccap; + memset(cc, 0, sizeof(*cc)); + + if (find_program("conda")) { + cc->available = true; + } + + if (cc->available) { + char *conda_version = python_importlib_metadata_version("conda"); + if (!conda_version) { + SYSERROR("conda version detection failed"); + return -1; + } + + char *mamba_version = python_importlib_metadata_version("libmambapy"); + if (!mamba_version) { + SYSERROR("unable to allocate mamba_version"); + guard_free(conda_version); + return -1; + } + + cc->prefix = root; + cc->conda_version = strdup(conda_version); + cc->mamba_version = strdup(mamba_version); + if (version_compare(GT | EQ, cc->conda_version, "25.3.0")) { + cc->require_explicit_export_format = true; + cc->require_boa = false; + } else { + cc->require_explicit_export_format = false; + cc->require_manual_activation_shim = true; + cc->require_boa = true; + cc->require_libmamba_solver = true; + } + + struct Process proc = {0}; + safe_strncpy(proc.f_stderr, "/dev/null", sizeof(proc.f_stderr)); + safe_strncpy(proc.f_stdout, "/dev/null", sizeof(proc.f_stdout)); + if (shell(&proc, "mamba install --use-local --dry-run conda")) { + cc->missing_use_local = true; + } + + cc->usable = true; + + guard_free(mamba_version); + guard_free(conda_version); + } + return 0; +}
\ No newline at end of file diff --git a/src/lib/core/include/conda.h b/src/lib/core/include/conda.h index a7108ec..9a6178b 100644 --- a/src/lib/core/include/conda.h +++ b/src/lib/core/include/conda.h @@ -26,6 +26,48 @@ #define PKG_INDEX_PROVIDES_E_MANAGER_EXEC (PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET + 5) #define PKG_INDEX_PROVIDES_FAILED(ECODE) ((ECODE) <= PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET) +/** + * @struct CondaCapabilities + * @brief Collection of feature flags to support older, newer, and transitional versions of conda/mamba. + * @see implementation in `conda_capable` + */ +struct CondaCapabilities { + /// Conda installation prefix + const char *prefix; + /// Currently installed version of Conda + char *conda_version; + /// Currently installed version of Mamba + char *mamba_version; + /// Is conda available in the runtime environment? + bool available; + /// Can conda execute? + bool usable; + /// Do we need to pass extra arguments to "conda env export"? + bool require_explicit_export_format; + /// Do we need to inject our own shim to make Conda available? + bool require_manual_activation_shim; + /// Does this version of Conda support building with boa? + bool require_boa; + /// Does this version of Conda need to be configured to use libmamba? + bool require_libmamba_solver; + /// Does "mamba install --use-local" work on this version of mamba? + bool missing_use_local; +}; + +/** + * Check for the existence of "Conda" and set flags based on the availability of features + * @param ccap a pointer to an empty `struct CondaCapabilities` + * @param root conda installation root directory + * @return 0 on success + */ +int conda_capable(struct CondaCapabilities *ccap, const char *root); + +/** + * Free memory allocated by `conda_capable` + * @param ccap a pointer to a populated `struct CondaCapabilities` + */ +void conda_capable_free(struct CondaCapabilities *ccap); + struct MicromambaInfo { char *micromamba_prefix; //!< Path to write micromamba binary char *conda_prefix; //!< Path to install conda base tree @@ -83,6 +125,24 @@ int python_exec(const char *args); int pip_exec(const char *args); /** + * Use importlib to resolve the version of an installed package + * + * ```c + * char *numpy_version = python_importlib_metadata_version("numpy"); + * if (!numpy_version) { + * fprintf(stderr, "failed to get numpy version\n"); + * exit(1); + * } + * + * printf("numpy version: %s\n", numpy_version); + * free(numpy_version); + * ``` + * @param package_name of installed python package + * @return + */ +char *python_importlib_metadata_version(const char *package_name); + +/** * Execute conda (or if possible, mamba) * Conda/Mamba is determined by PATH * @@ -117,7 +177,7 @@ int conda_activate(const char *root, const char *env_name); /** * Configure the active conda installation for headless operation */ -int conda_setup_headless(); +int conda_setup_headless(struct CondaCapabilities *cc); /** * Creates a Conda environment from a YAML config diff --git a/src/lib/core/include/recipe.h b/src/lib/core/include/recipe.h index 4dea248..ddcbabf 100644 --- a/src/lib/core/include/recipe.h +++ b/src/lib/core/include/recipe.h @@ -6,13 +6,20 @@ #include "utils.h" //! Unable to determine recipe repo type -#define RECIPE_TYPE_UNKNOWN 0 +#define RECIPE_STYLE_UNKNOWN 0 //! Recipe repo is from conda-forge -#define RECIPE_TYPE_CONDA_FORGE 1 +#define RECIPE_STYLE_CONDA_FORGE 1 //! Recipe repo is from astroconda -#define RECIPE_TYPE_ASTROCONDA 2 +#define RECIPE_STYLE_ASTROCONDA 2 //! Recipe repo provides the required build configurations but doesn't match conda-forge or astroconda's signature -#define RECIPE_TYPE_GENERIC 3 +#define RECIPE_STYLE_GENERIC 3 + +//! Unable to determine required build system +#define RECIPE_BUILD_UNKNOWN 0 +//! Build uses meta.yaml +#define RECIPE_BUILD_CONDA_BUILD 1 +//! Build uses recipe.yaml +#define RECIPE_BUILD_RATTLER 2 /** * Download a Conda package recipe @@ -46,18 +53,18 @@ int recipe_clone(char *recipe_dir, char *url, char *gitref, char **result); * } * * int recipe_type; - * recipe_type = recipe_get_type(recipe); + * recipe_type = recipe_get_style(recipe); * switch (recipe_type) { - * case RECIPE_TYPE_CONDA_FORGE: + * case RECIPE_STYLE_CONDA_FORGE: * // do something specific for conda-forge directory structure * break; - * case RECIPE_TYPE_ASTROCONDA: + * case RECIPE_STYLE_ASTROCONDA: * // do something specific for astroconda directory structure * break; - * case RECIPE_TYPE_GENERIC: + * case RECIPE_STYLE_GENERIC: * // do something specific for a directory containing a meta.yaml config * break; - * case RECIPE_TYPE_UNKNOWN: + * case RECIPE_STYLE_UNKNOWN: * default: * // the structure is foreign or the path doesn't contain a conda recipe * break; @@ -67,6 +74,7 @@ int recipe_clone(char *recipe_dir, char *url, char *gitref, char **result); * @param repopath path to git repository containing conda recipe(s) * @return One of RECIPE_TYPE_UNKNOWN, RECIPE_TYPE_CONDA_FORGE, RECIPE_TYPE_ASTROCONDA, RECIPE_TYPE_GENERIC */ -int recipe_get_type(char *repopath); +int recipe_get_style(char *repopath); +int recipe_get_build_system(const char *repopath, int style); #endif //STASIS_RECIPE_H diff --git a/src/lib/core/include/utils.h b/src/lib/core/include/utils.h index 3f0fe9f..e75995a 100644 --- a/src/lib/core/include/utils.h +++ b/src/lib/core/include/utils.h @@ -429,7 +429,7 @@ int gen_file_extension_str(char *filename, size_t maxlen, const char *extension) */ char *remove_extras(char *s); -void debug_hexdump(char *data, int len); +void debug_hexdump(char *data, size_t len); /** * Realloc helper diff --git a/src/lib/core/recipe.c b/src/lib/core/recipe.c index 8cc8e21..2e18492 100644 --- a/src/lib/core/recipe.c +++ b/src/lib/core/recipe.c @@ -34,8 +34,26 @@ int recipe_clone(char *recipe_dir, char *url, char *gitref, char **result) { return git_clone(&proc, url, destdir, gitref); } +int recipe_get_build_system(const char *repopath, const int style) { + char filename[PATH_MAX] = {0}; + safe_strncat(filename, repopath, sizeof(filename)); -int recipe_get_type(char *repopath) { + if (style == RECIPE_STYLE_CONDA_FORGE) { + safe_strncat(filename, "/recipe/recipe.yaml", sizeof(filename)); + if (!access(filename, F_OK)) { + return RECIPE_BUILD_RATTLER; + } + return RECIPE_BUILD_CONDA_BUILD; + } + + if (style == RECIPE_STYLE_ASTROCONDA || style == RECIPE_STYLE_GENERIC) { + return RECIPE_BUILD_CONDA_BUILD; + } + + return RECIPE_BUILD_UNKNOWN; +} + +int recipe_get_style(char *repopath) { // conda-forge is a collection of repositories // "conda-forge.yml" is guaranteed to exist const char *marker[] = { @@ -44,10 +62,10 @@ int recipe_get_type(char *repopath) { "meta.yaml", NULL }; - const int type[] = { - RECIPE_TYPE_CONDA_FORGE, - RECIPE_TYPE_ASTROCONDA, - RECIPE_TYPE_GENERIC + const int style[] = { + RECIPE_STYLE_CONDA_FORGE, + RECIPE_STYLE_ASTROCONDA, + RECIPE_STYLE_GENERIC }; for (size_t i = 0; marker[i] != NULL; i++) { @@ -55,9 +73,9 @@ int recipe_get_type(char *repopath) { snprintf(path, sizeof(path), "%s/%s", repopath, marker[i]); int result = access(path, F_OK); if (!result) { - return type[i]; + return style[i]; } } - return RECIPE_TYPE_UNKNOWN; + return RECIPE_STYLE_UNKNOWN; }
\ No newline at end of file diff --git a/src/lib/core/system.c b/src/lib/core/system.c index 4f1ece6..542f9fd 100644 --- a/src/lib/core/system.c +++ b/src/lib/core/system.c @@ -37,7 +37,9 @@ int shell(struct Process *proc, char *args) { // Set the script's permissions so that only the calling user can use it // This should help prevent eavesdropping if keys are applied in plain-text // somewhere. - chmod(t_name, 0700); + if (chmod(t_name, 0700)) { + SYSWARN("unable to change script permissions: %s, %s", t_name, strerror(errno)); + } pid_t pid = fork(); if (pid == -1) { diff --git a/src/lib/core/utils.c b/src/lib/core/utils.c index 31208ad..462604d 100644 --- a/src/lib/core/utils.c +++ b/src/lib/core/utils.c @@ -946,57 +946,43 @@ int gen_file_extension_str(char *filename, const size_t maxlen, const char *exte return replace_text(ext_orig, ext_orig, extension, 0); } -#define DEBUG_HEXDUMP_FMT_BYTES 6 -#define DEBUG_HEXDUMP_ADDR_MAXLEN 20 -#define DEBUG_HEXDUMP_BYTES_MAXLEN (16 * 3 + 2) -#define DEBUG_HEXDUMP_ASCII_MAXLEN (16 + 1) -#define DEBUG_HEXDUMP_OUTPUT_MAXLEN (DEBUG_HEXDUMP_FMT_BYTES + DEBUG_HEXDUMP_ADDR_MAXLEN + DEBUG_HEXDUMP_BYTES_MAXLEN + DEBUG_HEXDUMP_ASCII_MAXLEN + 1) - -void debug_hexdump(char *data, int len) { - int count = 0; - char addr[DEBUG_HEXDUMP_ADDR_MAXLEN] = {0}; - char bytes[DEBUG_HEXDUMP_BYTES_MAXLEN] = {0}; - char ascii[DEBUG_HEXDUMP_ASCII_MAXLEN] = {0}; - char output[DEBUG_HEXDUMP_OUTPUT_MAXLEN] = {0}; +void debug_hexdump(char *data, const size_t len) { + if (!data) { + SYSWARN("DATA NULL\n"); + return; + } + if (len <= 0) { + SYSWARN("ZERO LENGTH\n"); + return; + } + const size_t window = 16; + size_t avail = len; + const size_t chunk = avail / window ? window : avail; char *start = data; - char *end = data + len; - - char *pos = start; - while (pos != end) { - if (count == 0) { - snprintf(addr + strlen(addr), sizeof(addr) - strlen(addr), "%p", pos); + while (avail > 0) { + const size_t need = chunk > avail ? avail : chunk; + char *p = start; + printf("%p | ", p); + size_t j = 0; + for (j = 0; j < need; j++) { + printf("%02x ", p[j]); } - if (count == 8) { - safe_strncat(bytes, " ", sizeof(bytes)); + const size_t padding = window - j; + for (size_t i = 0; i < padding; i++) { + printf("00 "); } - if (count > 15) { - snprintf(output, sizeof(output), "%s | %s | %s", addr, bytes, ascii); - puts(output); - memset(output, 0, sizeof(output)); - memset(addr, 0, sizeof(addr)); - memset(bytes, 0, sizeof(bytes)); - memset(ascii, 0, sizeof(ascii)); - count = 0; - continue; + printf("| "); + for (j = 0; j < need; j++) { + char *c = p + j; + printf("%c", isprint(*c) ? *c : '.'); } - - snprintf(bytes + strlen(bytes), sizeof(bytes) - strlen(bytes), "%02X ", (unsigned char) *pos); - snprintf(ascii + strlen(ascii), sizeof(ascii) - strlen(ascii), "%c", isprint(*pos) ? *pos : '.'); - - pos++; - count++; - } - - if (count <= 8) { - // Add group padding - safe_strncat(bytes, " ", sizeof(bytes)); - } - const int padding = 16 - count; - for (int i = 0; i < padding; i++) { - safe_strncat(bytes, " ", sizeof(bytes)); + for (size_t i = 0; i < padding; i++) { + printf("."); + } + printf(" |\n"); + avail -= need; + start += need; } - snprintf(output, sizeof(output), "%s | %s | %s", addr, bytes, ascii); - puts(output); } int grow(const size_t size_new, size_t *size_orig, char **data) { @@ -1295,7 +1281,7 @@ int is_file_compressed(const char *filename) { {(unsigned char *) "PK\03\04", 3}, // zip {(unsigned char *) "PK\05\06", 3}, // zip (empty) {(unsigned char *) "PK\07\08", 3}, // zip (spanned) - {(unsigned char *) "\xfd\x2f\xb5\x28", 4} // zstd + {(unsigned char *) "\x28\xb5\x2f\xfd", 4} // zstd }; unsigned char buf[8] = {0}; // unsigned long size_t bytes_read = 0; diff --git a/src/lib/delivery/delivery.c b/src/lib/delivery/delivery.c index dc9e2ce..d256681 100644 --- a/src/lib/delivery/delivery.c +++ b/src/lib/delivery/delivery.c @@ -236,6 +236,8 @@ void delivery_free(struct Delivery *ctx) { guard_free(ctx->info.build_number); guard_free(ctx->info.release_name); guard_free(ctx->info.time_info); + + conda_capable_free(&ctx->conda.capabilities); guard_free(ctx->conda.installer_baseurl); guard_free(ctx->conda.installer_name); guard_free(ctx->conda.installer_version); @@ -308,37 +310,38 @@ int delivery_format_str(struct Delivery *ctx, char **dest, size_t maxlen, const i++; switch (fmt[i]) { case 'n': // name - strncat(*dest, ctx->meta.name, maxlen - 1); + safe_strncat(*dest, ctx->meta.name, maxlen); break; case 'c': // codename - strncat(*dest, ctx->meta.codename, maxlen - 1); + safe_strncat(*dest, ctx->meta.codename, maxlen); break; case 'm': // mission - strncat(*dest, ctx->meta.mission, maxlen - 1); + safe_strncat(*dest, ctx->meta.mission, maxlen); break; case 'r': // revision snprintf(*dest + strlen(*dest), maxlen - strlen(*dest), "%d", ctx->meta.rc); break; case 'R': // "final"-aware revision - if (ctx->meta.final) - strncat(*dest, "final", maxlen); - else + if (ctx->meta.final) { + safe_strncat(*dest, "final", maxlen); + } else { snprintf(*dest + strlen(*dest), maxlen - strlen(*dest), "%d", ctx->meta.rc); + } break; case 'v': // version - strncat(*dest, ctx->meta.version, maxlen - 1); + safe_strncat(*dest, ctx->meta.version, maxlen); break; case 'P': // python version - strncat(*dest, ctx->meta.python, maxlen - 1); + safe_strncat(*dest, ctx->meta.python, maxlen); break; case 'p': // python version major/minor - strncat(*dest, ctx->meta.python_compact, maxlen - 1); + safe_strncat(*dest, ctx->meta.python_compact, maxlen); break; case 'a': // system architecture name - strncat(*dest, ctx->system.arch, maxlen - 1); + safe_strncat(*dest, ctx->system.arch, maxlen); break; case 'o': // system platform (OS) name - strncat(*dest, ctx->system.platform[DELIVERY_PLATFORM_RELEASE], maxlen - 1); + safe_strncat(*dest, ctx->system.platform[DELIVERY_PLATFORM_RELEASE], maxlen); break; case 't': // unix epoch snprintf(*dest + strlen(*dest), maxlen - strlen(*dest), "%ld", ctx->info.time_now); @@ -363,11 +366,11 @@ void delivery_defer_packages(struct Delivery *ctx, int type) { if (DEFER_CONDA == type) { dataptr = ctx->conda.conda_packages; deferred = ctx->conda.conda_packages_defer; - strncpy(mode, "conda", sizeof(mode) - 1); + safe_strncpy(mode, "conda", sizeof(mode)); } else if (DEFER_PIP == type) { dataptr = ctx->conda.pip_packages; deferred = ctx->conda.pip_packages_defer; - strncpy(mode, "pip", sizeof(mode) - 1); + safe_strncpy(mode, "pip", sizeof(mode)); } else { SYSERROR("BUG: type %d does not map to a supported package manager!", type); exit(1); @@ -397,11 +400,10 @@ void delivery_defer_packages(struct Delivery *ctx, int type) { while (*spec_end != '\0' && !isalnum(*spec_end)) { spec_end++; } - strncpy(package_name, name, spec_begin - name); - package_name[spec_begin - name] = '\0'; + size_t spec_len = spec_begin - name; + safe_strncpy(package_name, name, spec_len ? spec_len + 1 : sizeof(package_name)); } else { - strncpy(package_name, name, sizeof(package_name) - 1); - package_name[sizeof(package_name) - 1] = '\0'; + safe_strncpy(package_name, name, sizeof(package_name)); } remove_extras(package_name); @@ -412,8 +414,7 @@ void delivery_defer_packages(struct Delivery *ctx, int type) { struct Test *test = ctx->tests->test[x]; char nametmp[STASIS_NAME_MAX] = {0}; - strncpy(nametmp, package_name, sizeof(nametmp) - 1); - nametmp[sizeof(nametmp) - 1] = '\0'; + safe_strncpy(nametmp, package_name, sizeof(nametmp)); // Is the [test:NAME] in the package name? if (!strcmp(nametmp, test->name)) { @@ -479,6 +480,10 @@ void delivery_defer_packages(struct Delivery *ctx, int type) { } } + if (getenv("STASIS_ALWAYS_BUILD_FOR_HOST")) { + build_for_host = 1; + } + if (build_for_host) { printf("BUILD FOR HOST\n"); strlist_append(&deferred, name); diff --git a/src/lib/delivery/delivery_build.c b/src/lib/delivery/delivery_build.c index d8674e0..9ef5d92 100644 --- a/src/lib/delivery/delivery_build.c +++ b/src/lib/delivery/delivery_build.c @@ -16,11 +16,13 @@ int delivery_build_recipes(struct Delivery *ctx) { SYSERROR("BUG: recipe_clone() succeeded but recipe_dir is NULL: %s", strerror(errno)); return -1; } - int recipe_type = recipe_get_type(recipe_dir); + const int recipe_style = recipe_get_style(recipe_dir); + const int recipe_build_system = recipe_get_build_system(recipe_dir, recipe_style); + if(!pushd(recipe_dir)) { - if (RECIPE_TYPE_ASTROCONDA == recipe_type) { + if (RECIPE_STYLE_ASTROCONDA == recipe_style) { pushd(path_basename(ctx->tests->test[i]->repository)); - } else if (RECIPE_TYPE_CONDA_FORGE == recipe_type) { + } else if (RECIPE_STYLE_CONDA_FORGE == recipe_style) { pushd("recipe"); } @@ -56,23 +58,28 @@ int delivery_build_recipes(struct Delivery *ctx) { snprintf(recipe_buildno, sizeof(recipe_buildno), " number: 0"); unsigned flags = REPLACE_TRUNCATE_AFTER_MATCH; - //file_replace_text("meta.yaml", "{% set version = ", recipe_version); - if (ctx->meta.final) { // remove this. i.e. statis cannot deploy a release to conda-forge - snprintf(recipe_version, sizeof(recipe_version), "{%% set version = \"%s\" %%}", ctx->tests->test[i]->version); - // TODO: replace sha256 of tagged archive - // TODO: leave the recipe unchanged otherwise. in theory this should produce the same conda package hash as conda forge. - // For now, remove the sha256 requirement - file_replace_text("meta.yaml", "sha256:", "\n", flags); - } else { - file_replace_text("meta.yaml", "{% set version = ", recipe_version, flags); - file_replace_text("meta.yaml", " url:", recipe_git_url, flags); - //file_replace_text("meta.yaml", "sha256:", recipe_git_rev); - file_replace_text("meta.yaml", " sha256:", "\n", flags); - file_replace_text("meta.yaml", " number:", recipe_buildno, flags); + if (recipe_build_system == RECIPE_BUILD_CONDA_BUILD) { + if (ctx->meta.final) { // remove this. i.e. statis cannot deploy a release to conda-forge + snprintf(recipe_version, sizeof(recipe_version), "{%% set version = \"%s\" %%}", ctx->tests->test[i]->version); + // TODO: replace sha256 of tagged archive + // TODO: leave the recipe unchanged otherwise. in theory this should produce the same conda package hash as conda forge. + // For now, remove the sha256 requirement + file_replace_text("meta.yaml", "sha256:", "\n", flags); + } else { + file_replace_text("meta.yaml", "{% set version = ", recipe_version, flags); + file_replace_text("meta.yaml", " url:", recipe_git_url, flags); + file_replace_text("meta.yaml", " sha256:", "\n", flags); + file_replace_text("meta.yaml", " number:", recipe_buildno, flags); + } + } else if (recipe_build_system == RECIPE_BUILD_RATTLER) { + file_replace_text("recipe.yaml", " version:", ctx->tests->test[i]->version, flags); + file_replace_text("recipe.yaml", " url:", recipe_git_url, flags); + file_replace_text("recipe.yaml", " sha256:", "\n", flags); + file_replace_text("recipe.yaml", " number:", recipe_buildno, flags); } char command[PATH_MAX]; - if (RECIPE_TYPE_CONDA_FORGE == recipe_type) { + if (RECIPE_STYLE_CONDA_FORGE == recipe_style) { char arch[STASIS_NAME_MAX] = {0}; char platform[STASIS_NAME_MAX] = {0}; @@ -92,18 +99,34 @@ int delivery_build_recipes(struct Delivery *ctx) { } tolower_s(arch); - snprintf(command, sizeof(command), "mambabuild --python=%s -m ../.ci_support/%s_%s_.yaml .", - ctx->meta.python, platform, arch); + // default build tool is "conda build" aka "build" + char tool[STASIS_NAME_MAX] = "build"; + if (recipe_build_system == RECIPE_BUILD_CONDA_BUILD) { + if (strlist_contains(globals.conda_packages, "boa", NULL)) { + safe_strncpy(tool, "mambabuild", sizeof(tool)); + } + } else if (recipe_build_system == RECIPE_BUILD_RATTLER) { + snprintf(tool, sizeof(tool), "rattler-build"); + } + snprintf(command, sizeof(command), "%s --python=%s -m ../.ci_support/%s_%s_.yaml .", + tool, ctx->meta.python, platform, arch); + } else { + snprintf(command, sizeof(command), "build --python=%s .", ctx->meta.python); + } + + int status = 0; + if (recipe_build_system == RECIPE_BUILD_RATTLER) { + // rattler-build is a standalone program, not a conda sub-command + status = system(command); } else { - snprintf(command, sizeof(command), "mambabuild --python=%s .", ctx->meta.python); + status = conda_exec(command); } - int status = conda_exec(command); if (status) { guard_free(recipe_dir); return -1; } - if (RECIPE_TYPE_GENERIC != recipe_type) { + if (RECIPE_STYLE_GENERIC != recipe_style) { popd(); } popd(); diff --git a/src/lib/delivery/delivery_conda.c b/src/lib/delivery/delivery_conda.c index 117e6c9..4f2920e 100644 --- a/src/lib/delivery/delivery_conda.c +++ b/src/lib/delivery/delivery_conda.c @@ -103,24 +103,61 @@ void delivery_install_conda(char *install_script, char *conda_install_dir) { } } -void delivery_conda_enable(struct Delivery *ctx, char *conda_install_dir) { - if (conda_activate(conda_install_dir, "base")) { - SYSERROR("conda activation failed"); - exit(1); - } +void delivery_conda_enable(struct Delivery *ctx) { + setenv("MAMBA_ROOT_PREFIX", ctx->storage.conda_install_prefix, 1); // Setting the CONDARC environment variable appears to be the only consistent // way to make sure the file is used. Not setting this variable leads to strange // behavior, especially if a conda environment is already active when STASIS is loaded. char rcpath[PATH_MAX]; - snprintf(rcpath, sizeof(rcpath), "%s/%s", conda_install_dir, ".condarc"); + snprintf(rcpath, sizeof(rcpath), "%s/%s", ctx->storage.conda_install_prefix, ".condarc"); setenv("CONDARC", rcpath, 1); + setenv("MAMBARC", rcpath, 1); if (runtime_replace(&ctx->runtime.environ, __environ)) { SYSERROR("unable to replace runtime environment after activating conda"); exit(1); } - if (conda_setup_headless()) { + if (conda_activate(ctx->storage.conda_install_prefix, "base")) { + SYSERROR("conda activation failed"); + exit(1); + } + + char pinned[PATH_MAX]; + snprintf(pinned, sizeof(pinned), "%s/conda-meta/pinned", ctx->storage.conda_install_prefix); + touch(pinned); + if (errno == ENOENT) { + errno = 0; + } + + char *conda_version = strdup(ctx->conda.installer_version); + if (conda_version) { + char *rev = strpbrk(conda_version, "-"); + if (rev) { + *rev = '\0'; + } + + FILE *pinned_fp = fopen(pinned, "w+"); + if (!pinned_fp) { + SYSERROR("unable to open conda-meta/pinned file for writing: %s", strerror(errno)); + exit(1); + } + fprintf(pinned_fp, "conda=%s\n", conda_version); + fclose(pinned_fp); + guard_free(conda_version); + } + + if (conda_capable(&ctx->conda.capabilities, ctx->storage.conda_install_prefix)) { + SYSERROR("Conda capability check failed"); + exit(1); + } + + if (!ctx->conda.capabilities.usable) { + SYSERROR("Conda is broken"); + exit(1); + } + + if (conda_setup_headless(&ctx->conda.capabilities)) { // no COE check. this call must succeed. exit(1); } diff --git a/src/lib/delivery/delivery_populate.c b/src/lib/delivery/delivery_populate.c index 5ce11d7..314ef46 100644 --- a/src/lib/delivery/delivery_populate.c +++ b/src/lib/delivery/delivery_populate.c @@ -199,6 +199,44 @@ static void normalize_ini_list(struct INIFILE **inip, struct StrList **listp, ch (*listp) = list; } +static int check_package_spec_list(struct StrList *list, const char *ini_section, const char *ini_key) { + if (!list) { + // empty lists are OK + return 0; + } + for (size_t i = 0; i < strlist_count(list); i++) { + const char *item = strlist_item(list, i); + if (!item) { + continue; + } + const char *invalid_chars = "@:/<>!~"; + const char *invalid_spec = strpbrk(item, invalid_chars); + if (invalid_spec) { + SYSERROR("Invalid version specification detected at %s:%s[%zu], \"%s\"", ini_section, ini_key, i, item); + SYSERROR("Package specification may not contain the operator '%c' (or any of \"%s\")", *invalid_spec, invalid_chars); + SYSERROR(""); + SYSERROR("%s:%s supports:", ini_section, ini_key); + SYSERROR(" {package}[=={version|tag|branch|ref}]"); + SYSERROR(""); + char *reported_name = strdup(item); + if (!reported_name) { + SYSERROR("unable to allocate memory for reported_name"); + return -1; + } + const char *reported_name_end = reported_name; + while (!ispunct(*reported_name_end)) { + reported_name_end++; + } + reported_name[reported_name_end - reported_name] = '\0'; + SYSERROR("Set test:%s.repository to point to a valid Git repository URL to enable [tag|branch|ref]", reported_name); + + guard_free(reported_name); + return -1; + } + } + return 0; +} + int populate_delivery_ini(struct Delivery *ctx, int render_mode) { struct INIFILE *ini = ctx->_stasis_ini_fp.delivery; struct INIData *rtdata; @@ -235,12 +273,22 @@ int populate_delivery_ini(struct Delivery *ctx, int render_mode) { ctx->conda.installer_platform = ini_getval_str(ini, "conda", "installer_platform", render_mode, &err); ctx->conda.installer_arch = ini_getval_str(ini, "conda", "installer_arch", render_mode, &err); ctx->conda.installer_baseurl = ini_getval_str(ini, "conda", "installer_baseurl", render_mode, &err); + ctx->conda.conda_packages = ini_getval_strlist(ini, "conda", "conda_packages", LINE_SEP, render_mode, &err); normalize_ini_list(&ini, &ctx->conda.conda_packages, "conda", "conda_packages", render_mode); + if (check_package_spec_list(ctx->conda.conda_packages, "conda", "conda_packages")) { + return -1; + } + ctx->conda.conda_packages_purge = ini_getval_strlist(ini, "conda", "conda_packages_purge", LINE_SEP, render_mode, &err); normalize_ini_list(&ini, &ctx->conda.conda_packages_purge, "conda", "conda_package_purge", render_mode); + ctx->conda.pip_packages = ini_getval_strlist(ini, "conda", "pip_packages", LINE_SEP, render_mode, &err); normalize_ini_list(&ini, &ctx->conda.pip_packages, "conda", "pip_packages", render_mode); + if (check_package_spec_list(ctx->conda.pip_packages, "conda", "pip_packages")) { + return -1; + } + ctx->conda.pip_packages_purge = ini_getval_strlist(ini, "conda", "pip_packages_purge", LINE_SEP, render_mode, &err); normalize_ini_list(&ini, &ctx->conda.pip_packages_purge, "conda", "pip_packages_purge", render_mode); diff --git a/src/lib/delivery/include/delivery.h b/src/lib/delivery/include/delivery.h index 3103a86..7d846ab 100644 --- a/src/lib/delivery/include/delivery.h +++ b/src/lib/delivery/include/delivery.h @@ -10,6 +10,7 @@ #include "environment.h" #include "ini.h" #include "multiprocessing.h" +#include "conda.h" #define DELIVERY_PLATFORM_MAX 4 #define DELIVERY_PLATFORM_MAXLEN 65 @@ -157,6 +158,7 @@ struct Delivery { struct StrList *pip_packages_defer; ///< Python packages to be built for delivery struct StrList *pip_packages_purge; ///< Python packages to remove from a delivery (for: based_on) struct StrList *wheels_packages; ///< Wheel packages built for delivery + struct CondaCapabilities capabilities; ///< Capability information } conda; /*! \struct Runtime @@ -354,9 +356,8 @@ void delivery_defer_packages(struct Delivery *ctx, int type); /** * Configure and activate a Conda installation based on Delivery context * @param ctx pointer to Delivery context - * @param conda_install_dir path to Conda installation */ -void delivery_conda_enable(struct Delivery *ctx, char *conda_install_dir); +void delivery_conda_enable(struct Delivery *ctx); /** * Install Conda |
