diff options
| -rw-r--r-- | include/conda.h | 2 | ||||
| -rw-r--r-- | include/utils.h | 16 | ||||
| -rw-r--r-- | src/cli/stasis/stasis_main.c | 36 | ||||
| -rw-r--r-- | src/lib/core/conda.c | 154 | ||||
| -rw-r--r-- | src/lib/core/utils.c | 42 | ||||
| -rw-r--r-- | stasis.ini | 1 | ||||
| -rw-r--r-- | tests/data/gbo.ini (renamed from tests/data/generic_based_on.ini) | 4 | ||||
| -rw-r--r-- | tests/data/gbo.yml (renamed from tests/data/generic_based_on.yml) | 0 | ||||
| -rw-r--r-- | tests/rt_generic_based_on.sh | 2 | ||||
| -rw-r--r-- | tests/setup.sh | 14 |
10 files changed, 188 insertions, 83 deletions
diff --git a/include/conda.h b/include/conda.h index f031479..b8d0caa 100644 --- a/include/conda.h +++ b/include/conda.h @@ -229,4 +229,6 @@ const char *pkg_index_provides_strerror(int code); char *conda_get_active_environment(); +int conda_env_exists(const char *root, const char *name); + #endif //STASIS_CONDA_H diff --git a/include/utils.h b/include/utils.h index 4ade817..e26b3c5 100644 --- a/include/utils.h +++ b/include/utils.h @@ -392,4 +392,20 @@ int mkdirs(const char *_path, mode_t mode); */ char *find_version_spec(char *package_name); +// mode flags for env_manipulate_pathstr +#define PM_APPEND 1 << 0 +#define PM_PREPEND 1 << 1 +#define PM_ONCE 1 << 2 + +/** +* Add paths to the head or tail of an environment variable. +* +* @param key environment variable to manipulate +* @param path to insert (does not need to exist) +* @param mode PM_APPEND `$path:$PATH` +* @param mode PM_PREPEND `$PATH:path` +* @param mode PM_ONCE do not manipulate if `path` is present in PATH variable +*/ +int env_manipulate_pathstr(const char *key, char *path, int mode); + #endif //STASIS_UTILS_H diff --git a/src/cli/stasis/stasis_main.c b/src/cli/stasis/stasis_main.c index e188b2e..093e32e 100644 --- a/src/cli/stasis/stasis_main.c +++ b/src/cli/stasis/stasis_main.c @@ -225,14 +225,6 @@ int main(int argc, char *argv[]) { exit(1); } - msg(STASIS_MSG_L1, "Conda setup\n"); - delivery_get_conda_installer_url(&ctx, installer_url); - msg(STASIS_MSG_L2, "Downloading: %s\n", installer_url); - if (delivery_get_conda_installer(&ctx, installer_url)) { - msg(STASIS_MSG_ERROR, "download failed: %s\n", installer_url); - exit(1); - } - // Unlikely to occur: this should help prevent rmtree() from destroying your entire filesystem // if path is "/" then, die // or if empty string, die @@ -241,6 +233,30 @@ int main(int argc, char *argv[]) { exit(1); } + // 2 = #! + // 5 = /bin\n + const size_t prefix_len = strlen(ctx.storage.conda_install_prefix) + 2 + 5; + const size_t prefix_len_max = 127; + msg(STASIS_MSG_L1, "Checking length of conda installation prefix\n"); + if (!strcmp(ctx.system.platform[DELIVERY_PLATFORM], "Linux") && prefix_len > prefix_len_max) { + msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, + "The shebang, '#!%s/bin/python\\n' is too long (%zu > %zu).\n", + ctx.storage.conda_install_prefix, prefix_len, prefix_len_max); + msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, + "Conda's workaround to handle long path names does not work consistently within STASIS.\n"); + msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, + "Please try again from a different, \"shorter\", directory.\n"); + exit(1); + } + + msg(STASIS_MSG_L1, "Conda setup\n"); + delivery_get_conda_installer_url(&ctx, installer_url); + msg(STASIS_MSG_L2, "Downloading: %s\n", installer_url); + if (delivery_get_conda_installer(&ctx, installer_url)) { + msg(STASIS_MSG_ERROR, "download failed: %s\n", installer_url); + exit(1); + } + msg(STASIS_MSG_L2, "Installing: %s\n", ctx.conda.installer_name); delivery_install_conda(ctx.conda.installer_path, ctx.storage.conda_install_prefix); @@ -294,7 +310,7 @@ int main(int argc, char *argv[]) { } if (!isempty(ctx.meta.based_on)) { - if (conda_env_remove(env_name)) { + if (conda_env_exists(ctx.storage.conda_install_prefix, env_name) && conda_env_remove(env_name)) { msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to remove release environment: %s\n", env_name); exit(1); } @@ -305,7 +321,7 @@ int main(int argc, char *argv[]) { exit(1); } - if (conda_env_remove(env_name_testing)) { + if (conda_env_exists(ctx.storage.conda_install_prefix, env_name_testing) && conda_env_remove(env_name_testing)) { msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to remove testing environment %s\n", env_name_testing); exit(1); } diff --git a/src/lib/core/conda.c b/src/lib/core/conda.c index 435af35..b2caa63f 100644 --- a/src/lib/core/conda.c +++ b/src/lib/core/conda.c @@ -208,25 +208,70 @@ int conda_exec(const char *args) { } static int conda_prepend_bin(const char *root) { - const char *system_path_old = getenv("PATH"); char conda_bin[PATH_MAX] = {0}; - snprintf(conda_bin, sizeof(conda_bin) - 1, "%s/bin:%s/condabin", root, root); + snprintf(conda_bin, sizeof(conda_bin) - 1, "%s/bin", root); + if (env_manipulate_pathstr("PATH", conda_bin, PM_PREPEND | PM_ONCE)) { + return -1; + } + return 0; +} - if (!strstr(system_path_old, conda_bin)) { - // conda_bin is not present in PATH. Add it to the head. - char system_path_new[STASIS_BUFSIZ]; - sprintf(system_path_new, "%s:%s", conda_bin, system_path_old); - if (setenv("PATH", system_path_new, 1) < 0) { - SYSERROR("Unable to prepend to PATH: %s", conda_bin); +static int conda_prepend_condabin(const char *root) { + char conda_condabin[PATH_MAX] = {0}; + + snprintf(conda_condabin, sizeof(conda_condabin) - 1, "%s/condabin", root); + if (env_manipulate_pathstr("PATH", conda_condabin, PM_PREPEND | PM_ONCE)) { + return -1; + } + return 0; +} + +static int env0_to_runtime(const char *logfile) { + FILE *fp = fopen(logfile, "r"); + if (!fp) { + perror(logfile); + return -1; + } + + while (!feof(fp)) { + char buf[STASIS_BUFSIZ] = {0}; + int ch = 0; + size_t z = 0; + // We are ingesting output from "env -0" and can't use fgets() + // Copy each character into the buffer until we encounter '\0' or EOF + while (z < sizeof(buf) && (ch = (int) fgetc(fp)) != 0) { + if (ch == EOF) { + break; + } + buf[z] = (char) ch; + z++; + } + buf[strlen(buf)] = 0; + + if (!strlen(buf)) { + continue; + } + + char **part = split(buf, "=", 1); + if (!part) { + perror("unable to split environment variable buffer"); return -1; } + if (!part[0]) { + msg(STASIS_MSG_WARN | STASIS_MSG_L1, "Invalid environment variable key ignored: '%s'\n", buf); + } else if (!part[1]) { + msg(STASIS_MSG_WARN | STASIS_MSG_L1, "Invalid environment variable value ignored: '%s'\n", buf); + } else { + setenv(part[0], part[1], 1); + } + GENERIC_ARRAY_FREE(part); } + fclose(fp); return 0; } int conda_activate(const char *root, const char *env_name) { - FILE *fp = NULL; const char *init_script_conda = "/etc/profile.d/conda.sh"; const char *init_script_mamba = "/etc/profile.d/mamba.sh"; char path_conda[PATH_MAX] = {0}; @@ -266,24 +311,42 @@ int conda_activate(const char *root, const char *env_name) { return -1; } - if (conda_prepend_bin(root)) { + // Fully activate conda and record its effect on the runtime environment + char command[PATH_MAX * 3]; + const char *conda_shlvl_str = getenv("CONDA_SHLVL"); + unsigned long conda_shlvl = 0; + if (conda_shlvl_str) { + conda_shlvl = strtol(conda_shlvl_str, NULL, 10); + } + + if (conda_prepend_condabin(root)) { remove(logfile); return -1; } - // Fully activate conda and record its effect on the runtime environment - char command[PATH_MAX * 3]; - const char *conda_shlvl = getenv("CONDA_SHLVL"); - if (conda_shlvl == NULL || strcmp(conda_shlvl, "0") == 0) { - // First-run initialization - snprintf(command, sizeof(command) - 1, "source %s; source %s; conda activate %s &>/dev/null; env -0", path_conda, path_mamba, env_name); - } else { - // Conda is already available and configured. - // Make calls directly to conda using conda's base interpreter. - // The shell functions generated by sourcing path_conda and path_mamba are extremely inconsistent - // in this environment. DO NOT USE THEM. - snprintf(command, sizeof(command) - 1, "$CONDA_PYTHON_EXE $CONDA_EXE activate %s &>/dev/null; env -0", env_name); + if (conda_prepend_bin(root)) { + remove(logfile); + return -1; } + + snprintf(command, sizeof(command) - 1, + "set -a\n" + "source %s\n" + "__conda_exe() (\n" + " \"$CONDA_PYTHON_EXE\" \"$CONDA_EXE\" $_CE_M $_CE_CONDA \"$@\"\n" + ")\n\n" + "export -f __conda_exe\n" + "source %s\n" + "__mamba_exe() (\n" + " \\local MAMBA_CONDA_EXE_BACKUP=$CONDA_EXE\n" + " \\local MAMBA_EXE=$(\\dirname \"${CONDA_EXE}\")/mamba\n" + " \"$CONDA_PYTHON_EXE\" \"$MAMBA_EXE\" $_CE_M $_CE_CONDA \"$@\"\n" + ")\n\n" + "export -f __mamba_exe\n" + "%s\n" + "conda activate %s 1>&2\n" + "env -0\n", path_conda, path_mamba, conda_shlvl ? "conda deactivate" : ":", env_name); + int retval = shell(&proc, command); if (retval) { // it didn't work; drop out for cleanup @@ -294,47 +357,9 @@ int conda_activate(const char *root, const char *env_name) { // Parse the log file: // 1. Extract the environment keys and values from the sub-shell // 2. Apply it to STASIS's runtime environment - // 3. Now we're ready to execute conda commands anywhere - fp = fopen(proc.f_stdout, "r"); - if (!fp) { - perror(logfile); + if (env0_to_runtime(logfile) < 0) { return -1; } - - while (!feof(fp)) { - char buf[STASIS_BUFSIZ] = {0}; - int ch = 0; - size_t z = 0; - // We are ingesting output from "env -0" and can't use fgets() - // Copy each character into the buffer until we encounter '\0' or EOF - while (z < sizeof(buf) && (ch = (int) fgetc(fp)) != 0) { - if (ch == EOF) { - break; - } - buf[z] = (char) ch; - z++; - } - buf[strlen(buf)] = 0; - - if (!strlen(buf)) { - continue; - } - - char **part = split(buf, "=", 1); - if (!part) { - perror("unable to split environment variable buffer"); - return -1; - } - if (!part[0]) { - msg(STASIS_MSG_WARN | STASIS_MSG_L1, "Invalid environment variable key ignored: '%s'\n", buf); - } else if (!part[1]) { - msg(STASIS_MSG_WARN | STASIS_MSG_L1, "Invalid environment variable value ignored: '%s'\n", buf); - } else { - setenv(part[0], part[1], 1); - } - GENERIC_ARRAY_FREE(part); - } - fclose(fp); remove(logfile); return 0; } @@ -346,7 +371,6 @@ int conda_check_required() { const char *conda_minimum_viable_tools[] = { "boa", "conda-build", - "conda-verify", NULL }; @@ -518,3 +542,9 @@ int conda_index(const char *path) { sprintf(command, "index %s", path); return conda_exec(command); } + +int conda_env_exists(const char *root, const char *name) { + char path[PATH_MAX] = {0}; + snprintf(path, sizeof(path) - 1 - 6, "%s/envs/%s", root, name); + return access(path, F_OK) == 0; +} diff --git a/src/lib/core/utils.c b/src/lib/core/utils.c index 5f0807c..18731e6 100644 --- a/src/lib/core/utils.c +++ b/src/lib/core/utils.c @@ -808,3 +808,45 @@ int mkdirs(const char *_path, mode_t mode) { char *find_version_spec(char *str) { return strpbrk(str, "@~=<>!"); } + +int env_manipulate_pathstr(const char *key, char *path, int mode) { + if (isempty(path)) { + SYSERROR("%s", "New PATH element cannot be zero-length or NULL"); + return -1; + } + + const char *system_path_old = getenv("PATH"); + if (!system_path_old) { + SYSERROR("%s", "Unable to read PATH"); + return -1; + } + + char *system_path_new = NULL; + + if (mode & PM_APPEND) { + asprintf(&system_path_new, "%s%s%s", system_path_old, PATH_SEP, path); + } else if (mode & PM_PREPEND) { + asprintf(&system_path_new, "%s%s%s", path, PATH_SEP, system_path_old); + } + + if (!system_path_new) { + SYSERROR("%s", "Unable to allocate memory to update PATH"); + return -1; + } + + if (mode & PM_ONCE) { + if (!strstr(system_path_old, path)) { + guard_free(system_path_new); + return 0; + } + } + if (setenv(key, system_path_new, 1) < 0) { + SYSERROR("Unable to %s to PATH: %s", mode & PM_APPEND ? "append" : "prepend", path); + guard_free(system_path_new); + return -1; + } + + guard_free(system_path_new); + return 0; +} + @@ -18,7 +18,6 @@ conda_fresh_start = true conda_packages = conda-build>=3.22.0 boa - conda-verify conda-libmamba-solver ; (list) Python packages to be installed/overridden in the base environment diff --git a/tests/data/generic_based_on.ini b/tests/data/gbo.ini index 1287933..609b8f3 100644 --- a/tests/data/generic_based_on.ini +++ b/tests/data/gbo.ini @@ -1,10 +1,10 @@ [meta] mission = generic -name = GENERIC_BASED_ON +name = GBO version = 1.2.3 rc = 1 final = false -based_on = {{ env:TEST_DATA }}/generic_based_on.yml +based_on = {{ env:TEST_DATA }}/gbo.yml python = 3.11 diff --git a/tests/data/generic_based_on.yml b/tests/data/gbo.yml index 3ab97a7..3ab97a7 100644 --- a/tests/data/generic_based_on.yml +++ b/tests/data/gbo.yml diff --git a/tests/rt_generic_based_on.sh b/tests/rt_generic_based_on.sh index 7d78399..08498be 100644 --- a/tests/rt_generic_based_on.sh +++ b/tests/rt_generic_based_on.sh @@ -2,7 +2,7 @@ here="$(dirname ${BASH_SOURCE[0]})" source $here/setup.sh -TEST_NAME=generic_based_on +TEST_NAME=gbo PYTHON_VERSIONS=( 3.11 ) diff --git a/tests/setup.sh b/tests/setup.sh index 50209ae..7e38cf9 100644 --- a/tests/setup.sh +++ b/tests/setup.sh @@ -32,7 +32,7 @@ popd() { command popd 1>/dev/null } -WS_DEFAULT="workspaces/rt_workspace_" +WS_DEFAULT="ws/_" setup_workspace() { if [ -z "$1" ]; then echo "setup_workspace requires a name argument" >&2 @@ -46,8 +46,8 @@ setup_workspace() { fi WORKSPACE="$(realpath $WORKSPACE)" - export PREFIX="$WORKSPACE"/local - if ! mkdir -p "$PREFIX"; then + export INSTALL_DIR="$WORKSPACE"/local + if ! mkdir -p "$INSTALL_DIR"; then echo "directory creation failed. cannot continue" >&2 return 1; fi @@ -78,7 +78,7 @@ teardown_workspace() { install_stasis() { pushd "$BUILD_DIR" - if ! cmake -DCMAKE_INSTALL_PREFIX="$PREFIX" -DCMAKE_BUILD_TYPE=Debug "${TOPDIR}"/../..; then + if ! cmake -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -DCMAKE_BUILD_TYPE=Debug "${TOPDIR}"/../..; then echo "cmake failed" >&2 return 1 fi @@ -88,7 +88,7 @@ install_stasis() { return 1 fi - export PATH="$PREFIX/bin:$PATH" + export PATH="$INSTALL_DIR/bin:$PATH" hash -r if ! type -P stasis; then echo "stasis program not on PATH" >&2 @@ -120,7 +120,7 @@ run_command() { (( STASIS_TEST_RESULT_SKIP++ )) else echo "... FAIL" - if (( $(wc -c "$logfile" | cut -d ' ' -f 1) > 1 )); then + if [[ -s "$logfile" ]]; then echo "#" echo "# Last $lines_on_error line(s) follow:" echo "#" @@ -282,4 +282,4 @@ clean_up() { else exit 0 fi -}
\ No newline at end of file +} |