From fab09092031b440c752d54f2e4fea467ed5bff2d Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Mon, 11 Nov 2024 12:50:47 -0500 Subject: Remove conda-verify requirement entirely * This package only exists for python<=3.11 and is considered optional by conda-build. --- src/lib/core/conda.c | 1 - stasis.ini | 1 - 2 files changed, 2 deletions(-) diff --git a/src/lib/core/conda.c b/src/lib/core/conda.c index 435af35..0e0161d 100644 --- a/src/lib/core/conda.c +++ b/src/lib/core/conda.c @@ -346,7 +346,6 @@ int conda_check_required() { const char *conda_minimum_viable_tools[] = { "boa", "conda-build", - "conda-verify", NULL }; diff --git a/stasis.ini b/stasis.ini index ba3331a..043fcec 100644 --- a/stasis.ini +++ b/stasis.ini @@ -18,7 +18,6 @@ conda_fresh_start = true conda_packages = conda-build>=3.22.0 boa - conda-verify conda-libmamba-solver ; (list) Python packages to be installed/overridden in the base environment -- cgit From 4c3c5a27eb591fa3fcb4267f27390133b0dd5ba0 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Mon, 11 Nov 2024 12:58:24 -0500 Subject: Revert activation procedure, with tweaks * Convert CONDA_SHLVL to an integer. If this variable is non-zero then deactivate the current environment before activating a new one. * Conda's initialization scripts don't handle shebang lines longer than 127 bytes on Linux. I'm aware they handle it in conda itself, but the "bug" is such that you cannot use conda to reap the benefits of their workaround(s). * This redefines the __conda_exe shell function so that conda is always executed by the base environment's Python interpreter via CONDA_PYTHON_EXE. --- src/lib/core/conda.c | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/lib/core/conda.c b/src/lib/core/conda.c index 0e0161d..48ed6c4 100644 --- a/src/lib/core/conda.c +++ b/src/lib/core/conda.c @@ -266,24 +266,28 @@ int conda_activate(const char *root, const char *env_name) { return -1; } + // Fully activate conda and record its effect on the runtime environment + char command[PATH_MAX * 3]; + const char *conda_shlvl_str = getenv("CONDA_SHLVL"); + unsigned long conda_shlvl = 0; + if (conda_shlvl_str) { + conda_shlvl = strtol(conda_shlvl_str, NULL, 10); + } + if (conda_prepend_bin(root)) { remove(logfile); return -1; } - // Fully activate conda and record its effect on the runtime environment - char command[PATH_MAX * 3]; - const char *conda_shlvl = getenv("CONDA_SHLVL"); - if (conda_shlvl == NULL || strcmp(conda_shlvl, "0") == 0) { - // First-run initialization - snprintf(command, sizeof(command) - 1, "source %s; source %s; conda activate %s &>/dev/null; env -0", path_conda, path_mamba, env_name); - } else { - // Conda is already available and configured. - // Make calls directly to conda using conda's base interpreter. - // The shell functions generated by sourcing path_conda and path_mamba are extremely inconsistent - // in this environment. DO NOT USE THEM. - snprintf(command, sizeof(command) - 1, "$CONDA_PYTHON_EXE $CONDA_EXE activate %s &>/dev/null; env -0", env_name); - } + snprintf(command, sizeof(command) - 1, + "set -a\n" + "source %s\n" + "function __conda_exe (\n\t\"$CONDA_PYTHON_EXE\" \"$CONDA_EXE\" $_CE_M $_CE_CONDA \"$@\"\n)\n" + "source %s\n" + "%s\n" + "conda activate %s 1>&2\n" + "env -0\n", path_conda, path_mamba, conda_shlvl ? "conda deactivate" : ":", env_name); + int retval = shell(&proc, command); if (retval) { // it didn't work; drop out for cleanup -- cgit From 23f96e574b756339911103816c3b1a582a27f69b Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Mon, 11 Nov 2024 13:08:19 -0500 Subject: Change PREFIX to INSTALL_DIR * Conda uses PREFIX (or can use it) so it's better to avoid overriding it. --- tests/setup.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/setup.sh b/tests/setup.sh index 50209ae..a56c899 100644 --- a/tests/setup.sh +++ b/tests/setup.sh @@ -46,8 +46,8 @@ setup_workspace() { fi WORKSPACE="$(realpath $WORKSPACE)" - export PREFIX="$WORKSPACE"/local - if ! mkdir -p "$PREFIX"; then + export INSTALL_DIR="$WORKSPACE"/local + if ! mkdir -p "$INSTALL_DIR"; then echo "directory creation failed. cannot continue" >&2 return 1; fi @@ -78,7 +78,7 @@ teardown_workspace() { install_stasis() { pushd "$BUILD_DIR" - if ! cmake -DCMAKE_INSTALL_PREFIX="$PREFIX" -DCMAKE_BUILD_TYPE=Debug "${TOPDIR}"/../..; then + if ! cmake -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -DCMAKE_BUILD_TYPE=Debug "${TOPDIR}"/../..; then echo "cmake failed" >&2 return 1 fi @@ -88,7 +88,7 @@ install_stasis() { return 1 fi - export PATH="$PREFIX/bin:$PATH" + export PATH="$INSTALL_DIR/bin:$PATH" hash -r if ! type -P stasis; then echo "stasis program not on PATH" >&2 @@ -282,4 +282,4 @@ clean_up() { else exit 0 fi -} \ No newline at end of file +} -- cgit From 2c0b70aa52a96156f70e895c4113620c7de12e27 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Mon, 11 Nov 2024 14:53:13 -0500 Subject: Use built in method for "if file size is greater than zero" check --- tests/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/setup.sh b/tests/setup.sh index a56c899..08e0de5 100644 --- a/tests/setup.sh +++ b/tests/setup.sh @@ -120,7 +120,7 @@ run_command() { (( STASIS_TEST_RESULT_SKIP++ )) else echo "... FAIL" - if (( $(wc -c "$logfile" | cut -d ' ' -f 1) > 1 )); then + if [[ -s "$logfile" ]]; then echo "#" echo "# Last $lines_on_error line(s) follow:" echo "#" -- cgit From 0febd2dd08d5d89435a66a41ec8336f5986cce01 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Mon, 11 Nov 2024 16:04:29 -0500 Subject: Again with the syntax issues on macos --- src/lib/core/conda.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lib/core/conda.c b/src/lib/core/conda.c index 48ed6c4..adbc4c9 100644 --- a/src/lib/core/conda.c +++ b/src/lib/core/conda.c @@ -282,7 +282,9 @@ int conda_activate(const char *root, const char *env_name) { snprintf(command, sizeof(command) - 1, "set -a\n" "source %s\n" - "function __conda_exe (\n\t\"$CONDA_PYTHON_EXE\" \"$CONDA_EXE\" $_CE_M $_CE_CONDA \"$@\"\n)\n" + "__conda_exe() (\n" + " \"$CONDA_PYTHON_EXE\" \"$CONDA_EXE\" $_CE_M $_CE_CONDA \"$@\"\n" + ")\n\n" "source %s\n" "%s\n" "conda activate %s 1>&2\n" -- cgit From f650250e8d59dc292353575d0492400435f4b82a Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Mon, 11 Nov 2024 16:29:10 -0500 Subject: Add __mamba_exe shell function as well --- src/lib/core/conda.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/lib/core/conda.c b/src/lib/core/conda.c index adbc4c9..48c114f 100644 --- a/src/lib/core/conda.c +++ b/src/lib/core/conda.c @@ -285,6 +285,11 @@ int conda_activate(const char *root, const char *env_name) { "__conda_exe() (\n" " \"$CONDA_PYTHON_EXE\" \"$CONDA_EXE\" $_CE_M $_CE_CONDA \"$@\"\n" ")\n\n" + "__mamba_exe() (\n" + " \\local MAMBA_CONDA_EXE_BACKUP=$CONDA_EXE\n" + " \\local MAMBA_EXE=$(\\dirname \"${CONDA_EXE}\")/mamba\n" + " \"$CONDA_PYTHON_EXE\" \"$MAMBA_EXE\" $_CE_M $_CE_CONDA \"$@\"\n" + ")\n\n" "source %s\n" "%s\n" "conda activate %s 1>&2\n" -- cgit From 6fe076163eaacac892efd83f9ffe967d2d0e8f52 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 12 Nov 2024 00:57:38 -0500 Subject: Add source mamba.sh before redefining the function --- src/lib/core/conda.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/core/conda.c b/src/lib/core/conda.c index 48c114f..a696744 100644 --- a/src/lib/core/conda.c +++ b/src/lib/core/conda.c @@ -285,12 +285,12 @@ int conda_activate(const char *root, const char *env_name) { "__conda_exe() (\n" " \"$CONDA_PYTHON_EXE\" \"$CONDA_EXE\" $_CE_M $_CE_CONDA \"$@\"\n" ")\n\n" + "source %s\n" "__mamba_exe() (\n" " \\local MAMBA_CONDA_EXE_BACKUP=$CONDA_EXE\n" " \\local MAMBA_EXE=$(\\dirname \"${CONDA_EXE}\")/mamba\n" " \"$CONDA_PYTHON_EXE\" \"$MAMBA_EXE\" $_CE_M $_CE_CONDA \"$@\"\n" ")\n\n" - "source %s\n" "%s\n" "conda activate %s 1>&2\n" "env -0\n", path_conda, path_mamba, conda_shlvl ? "conda deactivate" : ":", env_name); -- cgit From 5a8efc84f7a3c2264926b393aafd5585deaa7195 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 12 Nov 2024 00:59:08 -0500 Subject: Add path_manip function --- include/utils.h | 13 +++++++++++++ src/lib/core/utils.c | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/include/utils.h b/include/utils.h index 4ade817..2756347 100644 --- a/include/utils.h +++ b/include/utils.h @@ -30,6 +30,10 @@ #define STASIS_XML_PRETTY_PRINT_PROG "xmllint" #define STASIS_XML_PRETTY_PRINT_ARGS "--format" +#define PM_APPEND 1 << 0 +#define PM_PREPEND 1 << 1 +#define PM_ONCE 1 << 2 + /** * Change directory. Push path on directory stack. * @@ -392,4 +396,13 @@ int mkdirs(const char *_path, mode_t mode); */ char *find_version_spec(char *package_name); +/** +* Manipulate the PATH environment variable +* @param path to insert (does not need to exist) +* @param mode PM_APPEND `$path:$PATH` +* @param mode PM_PREPEND `$PATH:path` +* @param mode PM_ONCE do not manipulate if `path` is present in PATH variable +*/ +int path_manip(char *path, int mode); + #endif //STASIS_UTILS_H diff --git a/src/lib/core/utils.c b/src/lib/core/utils.c index 5f0807c..793b445 100644 --- a/src/lib/core/utils.c +++ b/src/lib/core/utils.c @@ -808,3 +808,46 @@ int mkdirs(const char *_path, mode_t mode) { char *find_version_spec(char *str) { return strpbrk(str, "@~=<>!"); } + +int path_manip(char *path, int mode) { + if (isempty(path)) { + SYSERROR("%s", "New PATH element cannot be zero-length or NULL"); + return -1; + } + + const char *system_path_old = getenv("PATH"); + if (!system_path_old) { + SYSERROR("%s", "Unable to read PATH"); + return -1; + } + + char *system_path_new = NULL; + + if (mode & PM_APPEND) { + asprintf(&system_path_new, "%s:%s", system_path_old, path); + } else if (mode & PM_PREPEND) { + asprintf(&system_path_new, "%s:%s", path, system_path_old); + } + + if (!system_path_new) { + SYSERROR("%s", "Unable to allocate memory to update PATH"); + return -1; + } + + if (mode & PM_ONCE) { + if (!strstr(system_path_old, path)) { + guard_free(system_path_new); + return 0; + } + } + + if (setenv("PATH", system_path_new, 1) < 0) { + SYSERROR("Unable to prepend to PATH: %s", path); + guard_free(system_path_new); + return -1; + } + + guard_free(system_path_new); + return 0; +} + -- cgit From e15448666cec18570001f2857276d74a8595d1a3 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 12 Nov 2024 00:59:47 -0500 Subject: Use path_manip function * Add conda_prepend_condabin helper --- src/lib/core/conda.c | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/src/lib/core/conda.c b/src/lib/core/conda.c index a696744..4533803 100644 --- a/src/lib/core/conda.c +++ b/src/lib/core/conda.c @@ -208,23 +208,26 @@ int conda_exec(const char *args) { } static int conda_prepend_bin(const char *root) { - const char *system_path_old = getenv("PATH"); char conda_bin[PATH_MAX] = {0}; - snprintf(conda_bin, sizeof(conda_bin) - 1, "%s/bin:%s/condabin", root, root); + snprintf(conda_bin, sizeof(conda_bin) - 1, "%s/bin", root); + if (path_manip(conda_bin, PM_PREPEND | PM_ONCE)) { + return -1; + } + return 0; +} - if (!strstr(system_path_old, conda_bin)) { - // conda_bin is not present in PATH. Add it to the head. - char system_path_new[STASIS_BUFSIZ]; - sprintf(system_path_new, "%s:%s", conda_bin, system_path_old); - if (setenv("PATH", system_path_new, 1) < 0) { - SYSERROR("Unable to prepend to PATH: %s", conda_bin); - return -1; - } +static int conda_prepend_condabin(const char *root) { + char conda_condabin[PATH_MAX] = {0}; + + snprintf(conda_condabin, sizeof(conda_condabin) - 1, "%s/condabin", root); + if (path_manip(conda_condabin, PM_PREPEND | PM_ONCE)) { + return -1; } return 0; } + int conda_activate(const char *root, const char *env_name) { FILE *fp = NULL; const char *init_script_conda = "/etc/profile.d/conda.sh"; @@ -274,6 +277,11 @@ int conda_activate(const char *root, const char *env_name) { conda_shlvl = strtol(conda_shlvl_str, NULL, 10); } + if (conda_prepend_condabin(root)) { + remove(logfile); + return -1; + } + if (conda_prepend_bin(root)) { remove(logfile); return -1; -- cgit From c377ff9c52e219622044d0ff28c4ad844dee8e8f Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 12 Nov 2024 01:10:52 -0500 Subject: Export the functions? --- src/lib/core/conda.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/core/conda.c b/src/lib/core/conda.c index 4533803..3e93873 100644 --- a/src/lib/core/conda.c +++ b/src/lib/core/conda.c @@ -293,12 +293,14 @@ int conda_activate(const char *root, const char *env_name) { "__conda_exe() (\n" " \"$CONDA_PYTHON_EXE\" \"$CONDA_EXE\" $_CE_M $_CE_CONDA \"$@\"\n" ")\n\n" + "export -f __conda_exe()\n" "source %s\n" "__mamba_exe() (\n" " \\local MAMBA_CONDA_EXE_BACKUP=$CONDA_EXE\n" " \\local MAMBA_EXE=$(\\dirname \"${CONDA_EXE}\")/mamba\n" " \"$CONDA_PYTHON_EXE\" \"$MAMBA_EXE\" $_CE_M $_CE_CONDA \"$@\"\n" ")\n\n" + "export -f __mamba_exe\n" "%s\n" "conda activate %s 1>&2\n" "env -0\n", path_conda, path_mamba, conda_shlvl ? "conda deactivate" : ":", env_name); -- cgit From bc786e086c14e3942c62a8a2c48e5a7f4b8d6bc6 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 12 Nov 2024 01:14:42 -0500 Subject: Export the functions correctly? --- src/lib/core/conda.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/core/conda.c b/src/lib/core/conda.c index 3e93873..5412353 100644 --- a/src/lib/core/conda.c +++ b/src/lib/core/conda.c @@ -293,7 +293,7 @@ int conda_activate(const char *root, const char *env_name) { "__conda_exe() (\n" " \"$CONDA_PYTHON_EXE\" \"$CONDA_EXE\" $_CE_M $_CE_CONDA \"$@\"\n" ")\n\n" - "export -f __conda_exe()\n" + "export -f __conda_exe\n" "source %s\n" "__mamba_exe() (\n" " \\local MAMBA_CONDA_EXE_BACKUP=$CONDA_EXE\n" -- cgit From af0625b73cf7b948931e7d370a894bb2ccbbd7dc Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 12 Nov 2024 12:54:55 -0500 Subject: Move 'env -0' translation routine out of conda_activate * Add static env0_to_runtime function --- src/lib/core/conda.c | 83 ++++++++++++++++++++++++++++------------------------ 1 file changed, 44 insertions(+), 39 deletions(-) diff --git a/src/lib/core/conda.c b/src/lib/core/conda.c index 5412353..4ac1530 100644 --- a/src/lib/core/conda.c +++ b/src/lib/core/conda.c @@ -227,6 +227,49 @@ static int conda_prepend_condabin(const char *root) { return 0; } +static int env0_to_runtime(const char *logfile) { + FILE *fp = fopen(logfile, "r"); + if (!fp) { + perror(logfile); + return -1; + } + + while (!feof(fp)) { + char buf[STASIS_BUFSIZ] = {0}; + int ch = 0; + size_t z = 0; + // We are ingesting output from "env -0" and can't use fgets() + // Copy each character into the buffer until we encounter '\0' or EOF + while (z < sizeof(buf) && (ch = (int) fgetc(fp)) != 0) { + if (ch == EOF) { + break; + } + buf[z] = (char) ch; + z++; + } + buf[strlen(buf)] = 0; + + if (!strlen(buf)) { + continue; + } + + char **part = split(buf, "=", 1); + if (!part) { + perror("unable to split environment variable buffer"); + return -1; + } + if (!part[0]) { + msg(STASIS_MSG_WARN | STASIS_MSG_L1, "Invalid environment variable key ignored: '%s'\n", buf); + } else if (!part[1]) { + msg(STASIS_MSG_WARN | STASIS_MSG_L1, "Invalid environment variable value ignored: '%s'\n", buf); + } else { + setenv(part[0], part[1], 1); + } + GENERIC_ARRAY_FREE(part); + } + fclose(fp); + return 0; +} int conda_activate(const char *root, const char *env_name) { FILE *fp = NULL; @@ -315,47 +358,9 @@ int conda_activate(const char *root, const char *env_name) { // Parse the log file: // 1. Extract the environment keys and values from the sub-shell // 2. Apply it to STASIS's runtime environment - // 3. Now we're ready to execute conda commands anywhere - fp = fopen(proc.f_stdout, "r"); - if (!fp) { - perror(logfile); + if (env0_to_runtime(logfile) < 0) { return -1; } - - while (!feof(fp)) { - char buf[STASIS_BUFSIZ] = {0}; - int ch = 0; - size_t z = 0; - // We are ingesting output from "env -0" and can't use fgets() - // Copy each character into the buffer until we encounter '\0' or EOF - while (z < sizeof(buf) && (ch = (int) fgetc(fp)) != 0) { - if (ch == EOF) { - break; - } - buf[z] = (char) ch; - z++; - } - buf[strlen(buf)] = 0; - - if (!strlen(buf)) { - continue; - } - - char **part = split(buf, "=", 1); - if (!part) { - perror("unable to split environment variable buffer"); - return -1; - } - if (!part[0]) { - msg(STASIS_MSG_WARN | STASIS_MSG_L1, "Invalid environment variable key ignored: '%s'\n", buf); - } else if (!part[1]) { - msg(STASIS_MSG_WARN | STASIS_MSG_L1, "Invalid environment variable value ignored: '%s'\n", buf); - } else { - setenv(part[0], part[1], 1); - } - GENERIC_ARRAY_FREE(part); - } - fclose(fp); remove(logfile); return 0; } -- cgit From 9e9dd4cde2e71410af81460f747dcbf0e27f2088 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 12 Nov 2024 12:56:33 -0500 Subject: Remove unused file stream --- src/lib/core/conda.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lib/core/conda.c b/src/lib/core/conda.c index 4ac1530..94653c4 100644 --- a/src/lib/core/conda.c +++ b/src/lib/core/conda.c @@ -272,7 +272,6 @@ static int env0_to_runtime(const char *logfile) { } int conda_activate(const char *root, const char *env_name) { - FILE *fp = NULL; const char *init_script_conda = "/etc/profile.d/conda.sh"; const char *init_script_mamba = "/etc/profile.d/mamba.sh"; char path_conda[PATH_MAX] = {0}; -- cgit From 47c74c23d5a04c4a6d22276e99d0e486a9a8f151 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Wed, 13 Nov 2024 00:52:52 -0500 Subject: Shorten workspace names and paths --- tests/data/gbo.ini | 65 +++++++++++++++++++++++++++++++++++++++++ tests/data/gbo.yml | 9 ++++++ tests/data/generic_based_on.ini | 65 ----------------------------------------- tests/data/generic_based_on.yml | 9 ------ tests/rt_generic_based_on.sh | 2 +- tests/setup.sh | 2 +- 6 files changed, 76 insertions(+), 76 deletions(-) create mode 100644 tests/data/gbo.ini create mode 100644 tests/data/gbo.yml delete mode 100644 tests/data/generic_based_on.ini delete mode 100644 tests/data/generic_based_on.yml diff --git a/tests/data/gbo.ini b/tests/data/gbo.ini new file mode 100644 index 0000000..c57ae8c --- /dev/null +++ b/tests/data/gbo.ini @@ -0,0 +1,65 @@ +[meta] +mission = generic +name = GBO +version = 1.2.3 +rc = 1 +final = false +based_on = {{ env:TEST_DATA }}/generic_based_on.yml +python = 3.11 + + +[conda] +installer_name = Miniforge3 +installer_version = 24.3.0-0 +installer_platform = {{env:STASIS_CONDA_PLATFORM}} +installer_arch = {{env:STASIS_CONDA_ARCH}} +installer_baseurl = https://github.com/conda-forge/miniforge/releases/download/{{conda.installer_version}} +;conda_packages = +pip_packages = + firewatch==0.0.4 + tweakwcs==0.8.8 + + +[runtime] +CPPFLAGS = ${CPPFLAGS} -fpermissive +PYTHONUNBUFFERED = 1 + + +[test:firewatch] +repository = https://github.com/astroconda/firewatch +script_setup = + pip install -e '.' +script = + firewatch -c conda-forge -p ${STASIS_CONDA_PLATFORM_SUBDIR} | grep -E ' python-[0-9]' + + +[test:tweakwcs] +repository = https://github.com/spacetelescope/tweakwcs +script_setup = + pip install -e '.[test]' +script = + pytest \ + -r fEsx \ + --basetemp="{{ func:basetemp_dir() }}" \ + --junitxml="{{ func:junitxml_file() }}" + + +[deploy:artifactory:delivery] +files = + {{ storage.output_dir }}/** +dest = {{ meta.mission }}/{{ info.build_name }}/ + + +[deploy:docker] +registry = bytesalad.stsci.edu +image_compression = zstd -v -9 -c +build_args = + SNAPSHOT_INPUT={{ info.release_name }}.yml + SNAPSHOT_PKGDIR=packages +tags = + {{ meta.name }}:{{ info.build_number }}-py{{ meta.python_compact }} + {{ deploy.docker.registry }}/{{ meta.name }}:{{ info.build_number }}-py{{ meta.python_compact }} +test_script = + source /etc/profile + python -m pip freeze + mamba info diff --git a/tests/data/gbo.yml b/tests/data/gbo.yml new file mode 100644 index 0000000..3ab97a7 --- /dev/null +++ b/tests/data/gbo.yml @@ -0,0 +1,9 @@ +channels: + - conda-forge +dependencies: + - pip + - python + - setuptools + - pip: + - firewatch==0.0.3 + - tweakwcs==0.8.7 \ No newline at end of file diff --git a/tests/data/generic_based_on.ini b/tests/data/generic_based_on.ini deleted file mode 100644 index 1287933..0000000 --- a/tests/data/generic_based_on.ini +++ /dev/null @@ -1,65 +0,0 @@ -[meta] -mission = generic -name = GENERIC_BASED_ON -version = 1.2.3 -rc = 1 -final = false -based_on = {{ env:TEST_DATA }}/generic_based_on.yml -python = 3.11 - - -[conda] -installer_name = Miniforge3 -installer_version = 24.3.0-0 -installer_platform = {{env:STASIS_CONDA_PLATFORM}} -installer_arch = {{env:STASIS_CONDA_ARCH}} -installer_baseurl = https://github.com/conda-forge/miniforge/releases/download/{{conda.installer_version}} -;conda_packages = -pip_packages = - firewatch==0.0.4 - tweakwcs==0.8.8 - - -[runtime] -CPPFLAGS = ${CPPFLAGS} -fpermissive -PYTHONUNBUFFERED = 1 - - -[test:firewatch] -repository = https://github.com/astroconda/firewatch -script_setup = - pip install -e '.' -script = - firewatch -c conda-forge -p ${STASIS_CONDA_PLATFORM_SUBDIR} | grep -E ' python-[0-9]' - - -[test:tweakwcs] -repository = https://github.com/spacetelescope/tweakwcs -script_setup = - pip install -e '.[test]' -script = - pytest \ - -r fEsx \ - --basetemp="{{ func:basetemp_dir() }}" \ - --junitxml="{{ func:junitxml_file() }}" - - -[deploy:artifactory:delivery] -files = - {{ storage.output_dir }}/** -dest = {{ meta.mission }}/{{ info.build_name }}/ - - -[deploy:docker] -registry = bytesalad.stsci.edu -image_compression = zstd -v -9 -c -build_args = - SNAPSHOT_INPUT={{ info.release_name }}.yml - SNAPSHOT_PKGDIR=packages -tags = - {{ meta.name }}:{{ info.build_number }}-py{{ meta.python_compact }} - {{ deploy.docker.registry }}/{{ meta.name }}:{{ info.build_number }}-py{{ meta.python_compact }} -test_script = - source /etc/profile - python -m pip freeze - mamba info diff --git a/tests/data/generic_based_on.yml b/tests/data/generic_based_on.yml deleted file mode 100644 index 3ab97a7..0000000 --- a/tests/data/generic_based_on.yml +++ /dev/null @@ -1,9 +0,0 @@ -channels: - - conda-forge -dependencies: - - pip - - python - - setuptools - - pip: - - firewatch==0.0.3 - - tweakwcs==0.8.7 \ No newline at end of file diff --git a/tests/rt_generic_based_on.sh b/tests/rt_generic_based_on.sh index 7d78399..08498be 100644 --- a/tests/rt_generic_based_on.sh +++ b/tests/rt_generic_based_on.sh @@ -2,7 +2,7 @@ here="$(dirname ${BASH_SOURCE[0]})" source $here/setup.sh -TEST_NAME=generic_based_on +TEST_NAME=gbo PYTHON_VERSIONS=( 3.11 ) diff --git a/tests/setup.sh b/tests/setup.sh index 08e0de5..7e38cf9 100644 --- a/tests/setup.sh +++ b/tests/setup.sh @@ -32,7 +32,7 @@ popd() { command popd 1>/dev/null } -WS_DEFAULT="workspaces/rt_workspace_" +WS_DEFAULT="ws/_" setup_workspace() { if [ -z "$1" ]; then echo "setup_workspace requires a name argument" >&2 -- cgit From 233e06aff49a8a4603814cd6c1c0e62167518e2b Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Wed, 13 Nov 2024 00:54:27 -0500 Subject: When conda's prefix is too long, die. --- src/cli/stasis/stasis_main.c | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/src/cli/stasis/stasis_main.c b/src/cli/stasis/stasis_main.c index e188b2e..2a2ed3b 100644 --- a/src/cli/stasis/stasis_main.c +++ b/src/cli/stasis/stasis_main.c @@ -225,14 +225,6 @@ int main(int argc, char *argv[]) { exit(1); } - msg(STASIS_MSG_L1, "Conda setup\n"); - delivery_get_conda_installer_url(&ctx, installer_url); - msg(STASIS_MSG_L2, "Downloading: %s\n", installer_url); - if (delivery_get_conda_installer(&ctx, installer_url)) { - msg(STASIS_MSG_ERROR, "download failed: %s\n", installer_url); - exit(1); - } - // Unlikely to occur: this should help prevent rmtree() from destroying your entire filesystem // if path is "/" then, die // or if empty string, die @@ -241,6 +233,30 @@ int main(int argc, char *argv[]) { exit(1); } + // 2 = #! + // 5 = /bin\n + const size_t prefix_len = strlen(ctx.storage.conda_install_prefix) + 2 + 5; + const size_t prefix_len_max = 127; + msg(STASIS_MSG_L1, "Checking length of conda installation prefix\n"); + if (!strcmp(ctx.system.platform[DELIVERY_PLATFORM], "Linux") && prefix_len > prefix_len_max) { + msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, + "The shebang, '#!%s/bin/python\\n' is too long (%zu > %zu).\n", + ctx.storage.conda_install_prefix, prefix_len, prefix_len_max); + msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, + "Conda's workaround to handle long path names does not work consistently within STASIS.\n"); + msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, + "Please try again from a different, \"shorter\", directory.\n"); + exit(1); + } + + msg(STASIS_MSG_L1, "Conda setup\n"); + delivery_get_conda_installer_url(&ctx, installer_url); + msg(STASIS_MSG_L2, "Downloading: %s\n", installer_url); + if (delivery_get_conda_installer(&ctx, installer_url)) { + msg(STASIS_MSG_ERROR, "download failed: %s\n", installer_url); + exit(1); + } + msg(STASIS_MSG_L2, "Installing: %s\n", ctx.conda.installer_name); delivery_install_conda(ctx.conda.installer_path, ctx.storage.conda_install_prefix); -- cgit From 8af575409e01bf4e539c73bd3490a7885d9f79a9 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Wed, 13 Nov 2024 01:17:15 -0500 Subject: Use shortened YAML config --- tests/data/gbo.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data/gbo.ini b/tests/data/gbo.ini index c57ae8c..609b8f3 100644 --- a/tests/data/gbo.ini +++ b/tests/data/gbo.ini @@ -4,7 +4,7 @@ name = GBO version = 1.2.3 rc = 1 final = false -based_on = {{ env:TEST_DATA }}/generic_based_on.yml +based_on = {{ env:TEST_DATA }}/gbo.yml python = 3.11 -- cgit From 35471289b619994c4f04fd2b6cb6d04a16cb1b33 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Wed, 13 Nov 2024 11:30:34 -0500 Subject: Rename path_manip() to env_manipulate_pathstr() * Add key argument to generalize the function --- include/utils.h | 15 +++++++++------ src/lib/core/conda.c | 4 ++-- src/lib/core/utils.c | 11 +++++------ 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/include/utils.h b/include/utils.h index 2756347..e26b3c5 100644 --- a/include/utils.h +++ b/include/utils.h @@ -30,10 +30,6 @@ #define STASIS_XML_PRETTY_PRINT_PROG "xmllint" #define STASIS_XML_PRETTY_PRINT_ARGS "--format" -#define PM_APPEND 1 << 0 -#define PM_PREPEND 1 << 1 -#define PM_ONCE 1 << 2 - /** * Change directory. Push path on directory stack. * @@ -396,13 +392,20 @@ int mkdirs(const char *_path, mode_t mode); */ char *find_version_spec(char *package_name); +// mode flags for env_manipulate_pathstr +#define PM_APPEND 1 << 0 +#define PM_PREPEND 1 << 1 +#define PM_ONCE 1 << 2 + /** -* Manipulate the PATH environment variable +* Add paths to the head or tail of an environment variable. +* +* @param key environment variable to manipulate * @param path to insert (does not need to exist) * @param mode PM_APPEND `$path:$PATH` * @param mode PM_PREPEND `$PATH:path` * @param mode PM_ONCE do not manipulate if `path` is present in PATH variable */ -int path_manip(char *path, int mode); +int env_manipulate_pathstr(const char *key, char *path, int mode); #endif //STASIS_UTILS_H diff --git a/src/lib/core/conda.c b/src/lib/core/conda.c index 94653c4..c98be1c 100644 --- a/src/lib/core/conda.c +++ b/src/lib/core/conda.c @@ -211,7 +211,7 @@ static int conda_prepend_bin(const char *root) { char conda_bin[PATH_MAX] = {0}; snprintf(conda_bin, sizeof(conda_bin) - 1, "%s/bin", root); - if (path_manip(conda_bin, PM_PREPEND | PM_ONCE)) { + if (env_manipulate_pathstr("PATH", conda_bin, PM_PREPEND | PM_ONCE)) { return -1; } return 0; @@ -221,7 +221,7 @@ static int conda_prepend_condabin(const char *root) { char conda_condabin[PATH_MAX] = {0}; snprintf(conda_condabin, sizeof(conda_condabin) - 1, "%s/condabin", root); - if (path_manip(conda_condabin, PM_PREPEND | PM_ONCE)) { + if (env_manipulate_pathstr("PATH", conda_condabin, PM_PREPEND | PM_ONCE)) { return -1; } return 0; diff --git a/src/lib/core/utils.c b/src/lib/core/utils.c index 793b445..18731e6 100644 --- a/src/lib/core/utils.c +++ b/src/lib/core/utils.c @@ -809,7 +809,7 @@ char *find_version_spec(char *str) { return strpbrk(str, "@~=<>!"); } -int path_manip(char *path, int mode) { +int env_manipulate_pathstr(const char *key, char *path, int mode) { if (isempty(path)) { SYSERROR("%s", "New PATH element cannot be zero-length or NULL"); return -1; @@ -824,9 +824,9 @@ int path_manip(char *path, int mode) { char *system_path_new = NULL; if (mode & PM_APPEND) { - asprintf(&system_path_new, "%s:%s", system_path_old, path); + asprintf(&system_path_new, "%s%s%s", system_path_old, PATH_SEP, path); } else if (mode & PM_PREPEND) { - asprintf(&system_path_new, "%s:%s", path, system_path_old); + asprintf(&system_path_new, "%s%s%s", path, PATH_SEP, system_path_old); } if (!system_path_new) { @@ -840,9 +840,8 @@ int path_manip(char *path, int mode) { return 0; } } - - if (setenv("PATH", system_path_new, 1) < 0) { - SYSERROR("Unable to prepend to PATH: %s", path); + if (setenv(key, system_path_new, 1) < 0) { + SYSERROR("Unable to %s to PATH: %s", mode & PM_APPEND ? "append" : "prepend", path); guard_free(system_path_new); return -1; } -- cgit From 836cc753ea22fd8a3d152f2b00dae971ee3dc943 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Wed, 13 Nov 2024 12:13:34 -0500 Subject: Attempt conda environment removal only when its present on-disk --- include/conda.h | 2 ++ src/cli/stasis/stasis_main.c | 4 ++-- src/lib/core/conda.c | 6 ++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/include/conda.h b/include/conda.h index f031479..b8d0caa 100644 --- a/include/conda.h +++ b/include/conda.h @@ -229,4 +229,6 @@ const char *pkg_index_provides_strerror(int code); char *conda_get_active_environment(); +int conda_env_exists(const char *root, const char *name); + #endif //STASIS_CONDA_H diff --git a/src/cli/stasis/stasis_main.c b/src/cli/stasis/stasis_main.c index 2a2ed3b..093e32e 100644 --- a/src/cli/stasis/stasis_main.c +++ b/src/cli/stasis/stasis_main.c @@ -310,7 +310,7 @@ int main(int argc, char *argv[]) { } if (!isempty(ctx.meta.based_on)) { - if (conda_env_remove(env_name)) { + if (conda_env_exists(ctx.storage.conda_install_prefix, env_name) && conda_env_remove(env_name)) { msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to remove release environment: %s\n", env_name); exit(1); } @@ -321,7 +321,7 @@ int main(int argc, char *argv[]) { exit(1); } - if (conda_env_remove(env_name_testing)) { + if (conda_env_exists(ctx.storage.conda_install_prefix, env_name_testing) && conda_env_remove(env_name_testing)) { msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to remove testing environment %s\n", env_name_testing); exit(1); } diff --git a/src/lib/core/conda.c b/src/lib/core/conda.c index c98be1c..b2caa63f 100644 --- a/src/lib/core/conda.c +++ b/src/lib/core/conda.c @@ -542,3 +542,9 @@ int conda_index(const char *path) { sprintf(command, "index %s", path); return conda_exec(command); } + +int conda_env_exists(const char *root, const char *name) { + char path[PATH_MAX] = {0}; + snprintf(path, sizeof(path) - 1 - 6, "%s/envs/%s", root, name); + return access(path, F_OK) == 0; +} -- cgit