diff options
29 files changed, 746 insertions, 221 deletions
@@ -207,6 +207,7 @@ stasis mydelivery.ini | STASIS_DOWNLOAD_TIMEOUT | Number of seconds before timing out a remote file download | | STASIS_DOWNLOAD_RETRY_MAX | Number of retries before giving up on a remote file download | | STASIS_DOWNLOAD_RETRY_SECONDS | Number of seconds to wait before retrying a remote file download | +| STASIS_ALWAYS_BUILD_FOR_HOST | If set, build all software from source (for debugging) | ## Main configuration (stasis.ini) diff --git a/src/cli/stasis/stasis_main.c b/src/cli/stasis/stasis_main.c index ef7bf26..c5c1f00 100644 --- a/src/cli/stasis/stasis_main.c +++ b/src/cli/stasis/stasis_main.c @@ -176,7 +176,7 @@ static void setup_conda(struct Delivery *ctx, char *installer_url, const size_t delivery_install_conda(ctx->conda.installer_path, ctx->storage.conda_install_prefix); msg(STASIS_MSG_L2, "Configuring: %s\n", ctx->storage.conda_install_prefix); - delivery_conda_enable(ctx, ctx->storage.conda_install_prefix); + delivery_conda_enable(ctx); } static void configure_conda_base(struct Delivery *ctx, char *envs[]) { diff --git a/src/lib/core/conda.c b/src/lib/core/conda.c index 5c7779f..bf27d21 100644 --- a/src/lib/core/conda.c +++ b/src/lib/core/conda.c @@ -194,6 +194,31 @@ int pip_exec(const char *args) { return result; } +char *python_importlib_metadata_version(const char *package_name) { + int status = 0; + char cmd[PATH_MAX] = {0}; + + if (strpbrk(package_name, "\\/*{}()|;&\"'\r\n")) { + SYSERROR("package name is invalid: '%s'", package_name); + return NULL; + } + snprintf(cmd, sizeof(cmd), "python3 -c 'from importlib.metadata import version; print(version(r\x22%s\x22))'", package_name); + + char *version = shell_output(cmd, &status); + if (status) { + SYSERROR("version detection failed"); + guard_free(version); + return NULL; + } + if (!version) { + SYSERROR("unable to allocate version"); + return NULL; + } + strip(version); + return version; +} + + static const char *PKG_ERROR_STR[] = { "success", "[internal] unhandled package manager mode", @@ -227,21 +252,30 @@ int pkg_index_provides(int mode, const char *index, const char *spec, const char SYSERROR("Unable to create log directory: %s", logdir ? logdir : "NULL"); return -1; } - const char logfile_template[] = "STASIS-package_exists.XXXXXX"; - char logfile[PATH_MAX] = {0}; - snprintf(logfile, sizeof(logfile), "%s/%s", logdir, logfile_template); - int logfd = mkstemp(logfile); - if (logfd < 0) { - SYSERROR("unable to create log file: %s", logfile); - remove(logfile); // fail harmlessly if not present - return PKG_INDEX_PROVIDES_E_INTERNAL_LOG_HANDLE; + const int stdout_stream = 0; + const int stderr_stream = 1; + const int stdout_st = 0; + //const int stderr_st = 1; + char logfile[2][PATH_MAX] = {0}; + int logfile_fd[2] = {-1, -1}; + struct stat logfile_st[2] = {0}; + + for (size_t i = 0; i < sizeof(logfile) / sizeof(logfile[0]); i++) { + const char logfile_template[] = "STASIS-package_exists.XXXXXX"; + snprintf(logfile[i], sizeof(logfile[i]), "%s/%s", logdir, logfile_template); + logfile_fd[i] = mkstemp(logfile[i]); + if (logfile_fd[i] < 0) { + SYSERROR("unable to create log file: %s", logfile[i]); + remove(logfile[i]); // fail harmlessly if not present + return PKG_INDEX_PROVIDES_E_INTERNAL_LOG_HANDLE; + } } int status = 0; struct Process proc = {0}; - proc.redirect_stderr = 1; - snprintf(proc.f_stdout, sizeof(proc.f_stdout), "%s", logfile); + snprintf(proc.f_stdout, sizeof(proc.f_stdout), "%s", logfile[stdout_stream]); + snprintf(proc.f_stderr, sizeof(proc.f_stderr), "%s", logfile[stderr_stream]); if (mode == PKG_USE_PIP) { // Do an installation in dry-run mode to see if the package exists in the given index. @@ -268,26 +302,40 @@ int pkg_index_provides(int mode, const char *index, const char *spec, const char SYSDEBUG("Executing: %s", cmd); status = shell(&proc, cmd); - SYSDEBUG("Log file: %s", logfile); - if (status != 0) { - FILE *fp = fdopen(logfd, "r"); - if (!fp) { - remove(logfile); + // Populate stat data for log files + for (size_t i = 0; i < sizeof(logfile) / sizeof(logfile[0]); i++) { + if (stat(logfile[i], &logfile_st[i])) { + SYSERROR("Unable to stat %s", logfile[i]); return -1; } + } + + if (status != 0) { + SYSERROR("Command exited non-zero (%d)", status); + for (size_t i = 0; i < sizeof(logfile) / sizeof(logfile[0]); i++) { + const char *stream_name = i == 0 ? "stdout" : "stderr"; + if (!logfile_st[i].st_size) { + continue; + } + SYSDEBUG("(%s): %s", stream_name, logfile[i]); + FILE *fp = fdopen(logfile_fd[i], "r"); + if (!fp) { + remove(logfile[i]); + return -1; + } - fflush(stdout); - fflush(stderr); + fflush(stdout); + fflush(stderr); - char line[STASIS_BUFSIZ] = {0}; - while (fgets(line, sizeof(line) - 1, fp) != NULL) { - SYSDEBUG("%s", strip(line)); - } + char line[STASIS_BUFSIZ] = {0}; + while (fgets(line, sizeof(line) - 1, fp) != NULL) { + SYSINFO("(%s): %s", stream_name, strip(line)); + } - fflush(stderr); - fclose(fp); + fflush(stderr); + fclose(fp); + } } - remove(logfile); if (WTERMSIG(proc.returncode)) { // This gets its own return value because if the external program @@ -295,18 +343,40 @@ int pkg_index_provides(int mode, const char *index, const char *spec, const char return PKG_INDEX_PROVIDES_E_MANAGER_SIGNALED; } + int final = PKG_FOUND; if (status < 0) { - return PKG_INDEX_PROVIDES_E_MANAGER_EXEC; + final = PKG_INDEX_PROVIDES_E_MANAGER_EXEC; } else if (WEXITSTATUS(proc.returncode) > 1) { // Pip and conda both return 2 on argument parsing errors - return PKG_INDEX_PROVIDES_E_MANAGER_RUNTIME; + final = PKG_INDEX_PROVIDES_E_MANAGER_RUNTIME; + } else if (logfile_st[stdout_st].st_size > 1 && WEXITSTATUS(proc.returncode) == 0) { + // modern mamba return zero when a package not found. + // even more ridiculous; the error messages are written to stdout, not stderr + struct StrList *output = strlist_init(); + if (!output) { + SYSERROR("unable to allocate memory for stdout log list"); + return -1; + } + if (strlist_append_file(output, logfile[stdout_stream], NULL)) { + strlist_free(&output); + SYSERROR("unable to append stdout log to list: %s", logfile[stdout_stream]); + return -1; + } + if (strlist_contains(output, "No entries", NULL)) { + final = PKG_NOT_FOUND; + } + strlist_free(&output); } else if (WEXITSTATUS(proc.returncode) == 1) { // Pip and conda both return 1 when a package is not found. // Unfortunately this applies to botched version specs, too. - return PKG_NOT_FOUND; - } else { - return PKG_FOUND; + final = PKG_NOT_FOUND; } + + // Remove log files + for (size_t i = 0; i < sizeof(logfile) / sizeof(logfile[0]); i++) { + remove(logfile[stdout_stream]); + } + return final; } int conda_exec(const char *args) { @@ -326,14 +396,22 @@ int conda_exec(const char *args) { }; char conda_as[10] = {0}; + const char *last_mamba_command = NULL; safe_strncpy(conda_as, "conda", sizeof(conda_as)); for (size_t i = 0; mamba_commands[i] != NULL; i++) { if (startswith(args, mamba_commands[i])) { + last_mamba_command = mamba_commands[i]; safe_strncpy(conda_as, "mamba", sizeof(conda_as)); break; } } + const int boa_present = find_program("boa") != NULL; + if (!boa_present) { + if (last_mamba_command && !strcmp(last_mamba_command, "build")) { + safe_strncpy(conda_as, "conda", sizeof(conda_as)); + } + } const char *command_fmt = "%s %s"; const int len = snprintf(NULL, 0, command_fmt, conda_as, args); @@ -416,15 +494,12 @@ static int env0_to_runtime(const char *logfile) { int conda_activate(const char *root, const char *env_name) { const char *init_script_conda = "/etc/profile.d/conda.sh"; - const char *init_script_mamba = "/etc/profile.d/mamba.sh"; char path_conda[PATH_MAX] = {0}; - char path_mamba[PATH_MAX] = {0}; char logfile[PATH_MAX] = {0}; struct Process proc = {0}; // Where to find conda's init scripts snprintf(path_conda, sizeof(path_conda), "%s%s", root, init_script_conda); - snprintf(path_mamba, sizeof(path_mamba), "%s%s", root, init_script_mamba); // Set the path to our stdout log // Emulate mktemp()'s behavior. Give us a unique file name, but don't use @@ -448,12 +523,6 @@ int conda_activate(const char *root, const char *env_name) { return -1; } - if (access(path_mamba, F_OK) < 0) { - SYSERROR("mamba is missing: %s, %s", path_mamba, strerror(errno)); - remove(logfile); - return -1; - } - // Fully activate conda and record its effect on the runtime environment char command[PATH_MAX * 3]; const char *conda_shlvl_str = getenv("CONDA_SHLVL"); @@ -475,20 +544,19 @@ int conda_activate(const char *root, const char *env_name) { snprintf(command, sizeof(command), "set -a\n" "source %s\n" - "__conda_exe() (\n" - " \"$CONDA_PYTHON_EXE\" \"$CONDA_EXE\" $_CE_M $_CE_CONDA \"$@\"\n" - ")\n\n" - "export -f __conda_exe\n" - "source %s\n" - "__mamba_exe() (\n" - " \\local MAMBA_CONDA_EXE_BACKUP=$CONDA_EXE\n" - " \\local MAMBA_EXE=$(\\dirname \"${CONDA_EXE}\")/mamba\n" - " \"$CONDA_PYTHON_EXE\" \"$MAMBA_EXE\" $_CE_M $_CE_CONDA \"$@\"\n" - ")\n\n" - "export -f __mamba_exe\n" + "tempfile=$(mktemp)\n" + "chmod 600 \"${tempfile}\"\n" + "%s/bin/mamba shell init --shell bash --dry-run 2>/dev/null\\\n" + "| (ignore=1; \\\n" + " while read line; do \\\n" + " if [[ \"$line\" == \"#\"* ]]; then ignore=0; fi; \\\n" + " if (( ignore == 0 )); then echo $line; fi; \\\n" + " done) 1> \"${tempfile}\"\n" + "source \"${tempfile}\"\n" + "rm -f \"${tempfile}\"\n" "%s\n" "conda activate %s 1>&2\n" - "env -0\n", path_conda, path_mamba, conda_shlvl ? "conda deactivate" : ":", env_name); + "env -0\n", path_conda, root, conda_shlvl ? "conda deactivate" : ":", env_name); int retval = shell(&proc, command); if (retval) { @@ -561,7 +629,20 @@ int conda_check_required() { return 0; } -int conda_setup_headless() { +int conda_setup_headless(struct CondaCapabilities *cc) { + mkdirs(cc->prefix, 0755); + + char rcpath[PATH_MAX]; + snprintf(rcpath, sizeof(rcpath), "%s/.condarc", cc->prefix); + touch(rcpath); + if (errno == ENOENT) { + errno = 0; + } + + setenv("CONDARC", rcpath, 1); + setenv("MAMBARC", rcpath, 1); + setenv("MAMBA_ROOT_PREFIX", cc->prefix, 1); + if (globals.verbose) { conda_exec("config --system --set quiet false"); } else { @@ -582,6 +663,18 @@ int conda_setup_headless() { size_t total = 0; const char *cmd_fmt = "'%s'"; if (globals.conda_packages && strlist_count(globals.conda_packages)) { + // Push or pop build packages based on capabilities + size_t boa_index = 0; + const int boa_requested = strlist_contains(globals.conda_packages, "boa", &boa_index); + if (boa_requested && !cc->require_boa) { + SYSWARN("Removing boa from global package list due to incompatible conda version (too new): %s", cc->conda_version); + strlist_remove(globals.conda_packages, boa_index); + } else if (!boa_requested && cc->require_boa) { + SYSWARN("Adding boa to global package list"); + strlist_append(&globals.conda_packages, "boa"); + } + + memset(cmd, 0, sizeof(cmd)); safe_strncpy(cmd, "install ", sizeof(cmd)); @@ -719,7 +812,18 @@ int conda_env_remove(char *name) { int conda_env_export(char *name, char *output_dir, char *output_filename) { char env_command[PATH_MAX]; - snprintf(env_command, sizeof(env_command), "env export -n %s -f %s/%s.yml", name, output_dir, output_filename); + int vr = 0; + const char *env_format = NULL; + char *version = shell_output("conda --version", &vr); + if (version) { + const size_t v_offset = strlen("conda "); + if (version_compare(GT, version + v_offset, "25.1.0")) { + env_format = "--format=yml"; + } + guard_free(version); + } + + snprintf(env_command, sizeof(env_command), "env export %s -n %s -f %s/%s.yml", env_format ? env_format : "", name, output_dir, output_filename); return conda_exec(env_command); } @@ -749,3 +853,59 @@ int conda_env_exists(const char *root, const char *name) { snprintf(path, sizeof(path), "%s/envs/%s", root, name); return access(path, F_OK) == 0; } + +void conda_capable_free(struct CondaCapabilities *ccap) { + guard_free(ccap->conda_version); + guard_free(ccap->mamba_version); + memset(ccap, 0, sizeof(*ccap)); +} + +int conda_capable(struct CondaCapabilities *ccap, const char *root) { + struct CondaCapabilities *cc = ccap; + memset(cc, 0, sizeof(*cc)); + + if (find_program("conda")) { + cc->available = true; + } + + if (cc->available) { + char *conda_version = python_importlib_metadata_version("conda"); + if (!conda_version) { + SYSERROR("conda version detection failed"); + return -1; + } + + char *mamba_version = python_importlib_metadata_version("libmambapy"); + if (!mamba_version) { + SYSERROR("unable to allocate mamba_version"); + guard_free(conda_version); + return -1; + } + + cc->prefix = root; + cc->conda_version = strdup(conda_version); + cc->mamba_version = strdup(mamba_version); + if (version_compare(GT | EQ, cc->conda_version, "25.3.0")) { + cc->require_explicit_export_format = true; + cc->require_boa = false; + } else { + cc->require_explicit_export_format = false; + cc->require_manual_activation_shim = true; + cc->require_boa = true; + cc->require_libmamba_solver = true; + } + + struct Process proc = {0}; + safe_strncpy(proc.f_stderr, "/dev/null", sizeof(proc.f_stderr)); + safe_strncpy(proc.f_stdout, "/dev/null", sizeof(proc.f_stdout)); + if (shell(&proc, "mamba install --use-local --dry-run conda")) { + cc->missing_use_local = true; + } + + cc->usable = true; + + guard_free(mamba_version); + guard_free(conda_version); + } + return 0; +}
\ No newline at end of file diff --git a/src/lib/core/include/conda.h b/src/lib/core/include/conda.h index a7108ec..9a6178b 100644 --- a/src/lib/core/include/conda.h +++ b/src/lib/core/include/conda.h @@ -26,6 +26,48 @@ #define PKG_INDEX_PROVIDES_E_MANAGER_EXEC (PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET + 5) #define PKG_INDEX_PROVIDES_FAILED(ECODE) ((ECODE) <= PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET) +/** + * @struct CondaCapabilities + * @brief Collection of feature flags to support older, newer, and transitional versions of conda/mamba. + * @see implementation in `conda_capable` + */ +struct CondaCapabilities { + /// Conda installation prefix + const char *prefix; + /// Currently installed version of Conda + char *conda_version; + /// Currently installed version of Mamba + char *mamba_version; + /// Is conda available in the runtime environment? + bool available; + /// Can conda execute? + bool usable; + /// Do we need to pass extra arguments to "conda env export"? + bool require_explicit_export_format; + /// Do we need to inject our own shim to make Conda available? + bool require_manual_activation_shim; + /// Does this version of Conda support building with boa? + bool require_boa; + /// Does this version of Conda need to be configured to use libmamba? + bool require_libmamba_solver; + /// Does "mamba install --use-local" work on this version of mamba? + bool missing_use_local; +}; + +/** + * Check for the existence of "Conda" and set flags based on the availability of features + * @param ccap a pointer to an empty `struct CondaCapabilities` + * @param root conda installation root directory + * @return 0 on success + */ +int conda_capable(struct CondaCapabilities *ccap, const char *root); + +/** + * Free memory allocated by `conda_capable` + * @param ccap a pointer to a populated `struct CondaCapabilities` + */ +void conda_capable_free(struct CondaCapabilities *ccap); + struct MicromambaInfo { char *micromamba_prefix; //!< Path to write micromamba binary char *conda_prefix; //!< Path to install conda base tree @@ -83,6 +125,24 @@ int python_exec(const char *args); int pip_exec(const char *args); /** + * Use importlib to resolve the version of an installed package + * + * ```c + * char *numpy_version = python_importlib_metadata_version("numpy"); + * if (!numpy_version) { + * fprintf(stderr, "failed to get numpy version\n"); + * exit(1); + * } + * + * printf("numpy version: %s\n", numpy_version); + * free(numpy_version); + * ``` + * @param package_name of installed python package + * @return + */ +char *python_importlib_metadata_version(const char *package_name); + +/** * Execute conda (or if possible, mamba) * Conda/Mamba is determined by PATH * @@ -117,7 +177,7 @@ int conda_activate(const char *root, const char *env_name); /** * Configure the active conda installation for headless operation */ -int conda_setup_headless(); +int conda_setup_headless(struct CondaCapabilities *cc); /** * Creates a Conda environment from a YAML config diff --git a/src/lib/core/include/recipe.h b/src/lib/core/include/recipe.h index 4dea248..ddcbabf 100644 --- a/src/lib/core/include/recipe.h +++ b/src/lib/core/include/recipe.h @@ -6,13 +6,20 @@ #include "utils.h" //! Unable to determine recipe repo type -#define RECIPE_TYPE_UNKNOWN 0 +#define RECIPE_STYLE_UNKNOWN 0 //! Recipe repo is from conda-forge -#define RECIPE_TYPE_CONDA_FORGE 1 +#define RECIPE_STYLE_CONDA_FORGE 1 //! Recipe repo is from astroconda -#define RECIPE_TYPE_ASTROCONDA 2 +#define RECIPE_STYLE_ASTROCONDA 2 //! Recipe repo provides the required build configurations but doesn't match conda-forge or astroconda's signature -#define RECIPE_TYPE_GENERIC 3 +#define RECIPE_STYLE_GENERIC 3 + +//! Unable to determine required build system +#define RECIPE_BUILD_UNKNOWN 0 +//! Build uses meta.yaml +#define RECIPE_BUILD_CONDA_BUILD 1 +//! Build uses recipe.yaml +#define RECIPE_BUILD_RATTLER 2 /** * Download a Conda package recipe @@ -46,18 +53,18 @@ int recipe_clone(char *recipe_dir, char *url, char *gitref, char **result); * } * * int recipe_type; - * recipe_type = recipe_get_type(recipe); + * recipe_type = recipe_get_style(recipe); * switch (recipe_type) { - * case RECIPE_TYPE_CONDA_FORGE: + * case RECIPE_STYLE_CONDA_FORGE: * // do something specific for conda-forge directory structure * break; - * case RECIPE_TYPE_ASTROCONDA: + * case RECIPE_STYLE_ASTROCONDA: * // do something specific for astroconda directory structure * break; - * case RECIPE_TYPE_GENERIC: + * case RECIPE_STYLE_GENERIC: * // do something specific for a directory containing a meta.yaml config * break; - * case RECIPE_TYPE_UNKNOWN: + * case RECIPE_STYLE_UNKNOWN: * default: * // the structure is foreign or the path doesn't contain a conda recipe * break; @@ -67,6 +74,7 @@ int recipe_clone(char *recipe_dir, char *url, char *gitref, char **result); * @param repopath path to git repository containing conda recipe(s) * @return One of RECIPE_TYPE_UNKNOWN, RECIPE_TYPE_CONDA_FORGE, RECIPE_TYPE_ASTROCONDA, RECIPE_TYPE_GENERIC */ -int recipe_get_type(char *repopath); +int recipe_get_style(char *repopath); +int recipe_get_build_system(const char *repopath, int style); #endif //STASIS_RECIPE_H diff --git a/src/lib/core/include/utils.h b/src/lib/core/include/utils.h index 3f0fe9f..e75995a 100644 --- a/src/lib/core/include/utils.h +++ b/src/lib/core/include/utils.h @@ -429,7 +429,7 @@ int gen_file_extension_str(char *filename, size_t maxlen, const char *extension) */ char *remove_extras(char *s); -void debug_hexdump(char *data, int len); +void debug_hexdump(char *data, size_t len); /** * Realloc helper diff --git a/src/lib/core/recipe.c b/src/lib/core/recipe.c index 8cc8e21..2e18492 100644 --- a/src/lib/core/recipe.c +++ b/src/lib/core/recipe.c @@ -34,8 +34,26 @@ int recipe_clone(char *recipe_dir, char *url, char *gitref, char **result) { return git_clone(&proc, url, destdir, gitref); } +int recipe_get_build_system(const char *repopath, const int style) { + char filename[PATH_MAX] = {0}; + safe_strncat(filename, repopath, sizeof(filename)); -int recipe_get_type(char *repopath) { + if (style == RECIPE_STYLE_CONDA_FORGE) { + safe_strncat(filename, "/recipe/recipe.yaml", sizeof(filename)); + if (!access(filename, F_OK)) { + return RECIPE_BUILD_RATTLER; + } + return RECIPE_BUILD_CONDA_BUILD; + } + + if (style == RECIPE_STYLE_ASTROCONDA || style == RECIPE_STYLE_GENERIC) { + return RECIPE_BUILD_CONDA_BUILD; + } + + return RECIPE_BUILD_UNKNOWN; +} + +int recipe_get_style(char *repopath) { // conda-forge is a collection of repositories // "conda-forge.yml" is guaranteed to exist const char *marker[] = { @@ -44,10 +62,10 @@ int recipe_get_type(char *repopath) { "meta.yaml", NULL }; - const int type[] = { - RECIPE_TYPE_CONDA_FORGE, - RECIPE_TYPE_ASTROCONDA, - RECIPE_TYPE_GENERIC + const int style[] = { + RECIPE_STYLE_CONDA_FORGE, + RECIPE_STYLE_ASTROCONDA, + RECIPE_STYLE_GENERIC }; for (size_t i = 0; marker[i] != NULL; i++) { @@ -55,9 +73,9 @@ int recipe_get_type(char *repopath) { snprintf(path, sizeof(path), "%s/%s", repopath, marker[i]); int result = access(path, F_OK); if (!result) { - return type[i]; + return style[i]; } } - return RECIPE_TYPE_UNKNOWN; + return RECIPE_STYLE_UNKNOWN; }
\ No newline at end of file diff --git a/src/lib/core/system.c b/src/lib/core/system.c index 4f1ece6..542f9fd 100644 --- a/src/lib/core/system.c +++ b/src/lib/core/system.c @@ -37,7 +37,9 @@ int shell(struct Process *proc, char *args) { // Set the script's permissions so that only the calling user can use it // This should help prevent eavesdropping if keys are applied in plain-text // somewhere. - chmod(t_name, 0700); + if (chmod(t_name, 0700)) { + SYSWARN("unable to change script permissions: %s, %s", t_name, strerror(errno)); + } pid_t pid = fork(); if (pid == -1) { diff --git a/src/lib/core/utils.c b/src/lib/core/utils.c index 31208ad..462604d 100644 --- a/src/lib/core/utils.c +++ b/src/lib/core/utils.c @@ -946,57 +946,43 @@ int gen_file_extension_str(char *filename, const size_t maxlen, const char *exte return replace_text(ext_orig, ext_orig, extension, 0); } -#define DEBUG_HEXDUMP_FMT_BYTES 6 -#define DEBUG_HEXDUMP_ADDR_MAXLEN 20 -#define DEBUG_HEXDUMP_BYTES_MAXLEN (16 * 3 + 2) -#define DEBUG_HEXDUMP_ASCII_MAXLEN (16 + 1) -#define DEBUG_HEXDUMP_OUTPUT_MAXLEN (DEBUG_HEXDUMP_FMT_BYTES + DEBUG_HEXDUMP_ADDR_MAXLEN + DEBUG_HEXDUMP_BYTES_MAXLEN + DEBUG_HEXDUMP_ASCII_MAXLEN + 1) - -void debug_hexdump(char *data, int len) { - int count = 0; - char addr[DEBUG_HEXDUMP_ADDR_MAXLEN] = {0}; - char bytes[DEBUG_HEXDUMP_BYTES_MAXLEN] = {0}; - char ascii[DEBUG_HEXDUMP_ASCII_MAXLEN] = {0}; - char output[DEBUG_HEXDUMP_OUTPUT_MAXLEN] = {0}; +void debug_hexdump(char *data, const size_t len) { + if (!data) { + SYSWARN("DATA NULL\n"); + return; + } + if (len <= 0) { + SYSWARN("ZERO LENGTH\n"); + return; + } + const size_t window = 16; + size_t avail = len; + const size_t chunk = avail / window ? window : avail; char *start = data; - char *end = data + len; - - char *pos = start; - while (pos != end) { - if (count == 0) { - snprintf(addr + strlen(addr), sizeof(addr) - strlen(addr), "%p", pos); + while (avail > 0) { + const size_t need = chunk > avail ? avail : chunk; + char *p = start; + printf("%p | ", p); + size_t j = 0; + for (j = 0; j < need; j++) { + printf("%02x ", p[j]); } - if (count == 8) { - safe_strncat(bytes, " ", sizeof(bytes)); + const size_t padding = window - j; + for (size_t i = 0; i < padding; i++) { + printf("00 "); } - if (count > 15) { - snprintf(output, sizeof(output), "%s | %s | %s", addr, bytes, ascii); - puts(output); - memset(output, 0, sizeof(output)); - memset(addr, 0, sizeof(addr)); - memset(bytes, 0, sizeof(bytes)); - memset(ascii, 0, sizeof(ascii)); - count = 0; - continue; + printf("| "); + for (j = 0; j < need; j++) { + char *c = p + j; + printf("%c", isprint(*c) ? *c : '.'); } - - snprintf(bytes + strlen(bytes), sizeof(bytes) - strlen(bytes), "%02X ", (unsigned char) *pos); - snprintf(ascii + strlen(ascii), sizeof(ascii) - strlen(ascii), "%c", isprint(*pos) ? *pos : '.'); - - pos++; - count++; - } - - if (count <= 8) { - // Add group padding - safe_strncat(bytes, " ", sizeof(bytes)); - } - const int padding = 16 - count; - for (int i = 0; i < padding; i++) { - safe_strncat(bytes, " ", sizeof(bytes)); + for (size_t i = 0; i < padding; i++) { + printf("."); + } + printf(" |\n"); + avail -= need; + start += need; } - snprintf(output, sizeof(output), "%s | %s | %s", addr, bytes, ascii); - puts(output); } int grow(const size_t size_new, size_t *size_orig, char **data) { @@ -1295,7 +1281,7 @@ int is_file_compressed(const char *filename) { {(unsigned char *) "PK\03\04", 3}, // zip {(unsigned char *) "PK\05\06", 3}, // zip (empty) {(unsigned char *) "PK\07\08", 3}, // zip (spanned) - {(unsigned char *) "\xfd\x2f\xb5\x28", 4} // zstd + {(unsigned char *) "\x28\xb5\x2f\xfd", 4} // zstd }; unsigned char buf[8] = {0}; // unsigned long size_t bytes_read = 0; diff --git a/src/lib/delivery/delivery.c b/src/lib/delivery/delivery.c index dc9e2ce..d256681 100644 --- a/src/lib/delivery/delivery.c +++ b/src/lib/delivery/delivery.c @@ -236,6 +236,8 @@ void delivery_free(struct Delivery *ctx) { guard_free(ctx->info.build_number); guard_free(ctx->info.release_name); guard_free(ctx->info.time_info); + + conda_capable_free(&ctx->conda.capabilities); guard_free(ctx->conda.installer_baseurl); guard_free(ctx->conda.installer_name); guard_free(ctx->conda.installer_version); @@ -308,37 +310,38 @@ int delivery_format_str(struct Delivery *ctx, char **dest, size_t maxlen, const i++; switch (fmt[i]) { case 'n': // name - strncat(*dest, ctx->meta.name, maxlen - 1); + safe_strncat(*dest, ctx->meta.name, maxlen); break; case 'c': // codename - strncat(*dest, ctx->meta.codename, maxlen - 1); + safe_strncat(*dest, ctx->meta.codename, maxlen); break; case 'm': // mission - strncat(*dest, ctx->meta.mission, maxlen - 1); + safe_strncat(*dest, ctx->meta.mission, maxlen); break; case 'r': // revision snprintf(*dest + strlen(*dest), maxlen - strlen(*dest), "%d", ctx->meta.rc); break; case 'R': // "final"-aware revision - if (ctx->meta.final) - strncat(*dest, "final", maxlen); - else + if (ctx->meta.final) { + safe_strncat(*dest, "final", maxlen); + } else { snprintf(*dest + strlen(*dest), maxlen - strlen(*dest), "%d", ctx->meta.rc); + } break; case 'v': // version - strncat(*dest, ctx->meta.version, maxlen - 1); + safe_strncat(*dest, ctx->meta.version, maxlen); break; case 'P': // python version - strncat(*dest, ctx->meta.python, maxlen - 1); + safe_strncat(*dest, ctx->meta.python, maxlen); break; case 'p': // python version major/minor - strncat(*dest, ctx->meta.python_compact, maxlen - 1); + safe_strncat(*dest, ctx->meta.python_compact, maxlen); break; case 'a': // system architecture name - strncat(*dest, ctx->system.arch, maxlen - 1); + safe_strncat(*dest, ctx->system.arch, maxlen); break; case 'o': // system platform (OS) name - strncat(*dest, ctx->system.platform[DELIVERY_PLATFORM_RELEASE], maxlen - 1); + safe_strncat(*dest, ctx->system.platform[DELIVERY_PLATFORM_RELEASE], maxlen); break; case 't': // unix epoch snprintf(*dest + strlen(*dest), maxlen - strlen(*dest), "%ld", ctx->info.time_now); @@ -363,11 +366,11 @@ void delivery_defer_packages(struct Delivery *ctx, int type) { if (DEFER_CONDA == type) { dataptr = ctx->conda.conda_packages; deferred = ctx->conda.conda_packages_defer; - strncpy(mode, "conda", sizeof(mode) - 1); + safe_strncpy(mode, "conda", sizeof(mode)); } else if (DEFER_PIP == type) { dataptr = ctx->conda.pip_packages; deferred = ctx->conda.pip_packages_defer; - strncpy(mode, "pip", sizeof(mode) - 1); + safe_strncpy(mode, "pip", sizeof(mode)); } else { SYSERROR("BUG: type %d does not map to a supported package manager!", type); exit(1); @@ -397,11 +400,10 @@ void delivery_defer_packages(struct Delivery *ctx, int type) { while (*spec_end != '\0' && !isalnum(*spec_end)) { spec_end++; } - strncpy(package_name, name, spec_begin - name); - package_name[spec_begin - name] = '\0'; + size_t spec_len = spec_begin - name; + safe_strncpy(package_name, name, spec_len ? spec_len + 1 : sizeof(package_name)); } else { - strncpy(package_name, name, sizeof(package_name) - 1); - package_name[sizeof(package_name) - 1] = '\0'; + safe_strncpy(package_name, name, sizeof(package_name)); } remove_extras(package_name); @@ -412,8 +414,7 @@ void delivery_defer_packages(struct Delivery *ctx, int type) { struct Test *test = ctx->tests->test[x]; char nametmp[STASIS_NAME_MAX] = {0}; - strncpy(nametmp, package_name, sizeof(nametmp) - 1); - nametmp[sizeof(nametmp) - 1] = '\0'; + safe_strncpy(nametmp, package_name, sizeof(nametmp)); // Is the [test:NAME] in the package name? if (!strcmp(nametmp, test->name)) { @@ -479,6 +480,10 @@ void delivery_defer_packages(struct Delivery *ctx, int type) { } } + if (getenv("STASIS_ALWAYS_BUILD_FOR_HOST")) { + build_for_host = 1; + } + if (build_for_host) { printf("BUILD FOR HOST\n"); strlist_append(&deferred, name); diff --git a/src/lib/delivery/delivery_build.c b/src/lib/delivery/delivery_build.c index d8674e0..9ef5d92 100644 --- a/src/lib/delivery/delivery_build.c +++ b/src/lib/delivery/delivery_build.c @@ -16,11 +16,13 @@ int delivery_build_recipes(struct Delivery *ctx) { SYSERROR("BUG: recipe_clone() succeeded but recipe_dir is NULL: %s", strerror(errno)); return -1; } - int recipe_type = recipe_get_type(recipe_dir); + const int recipe_style = recipe_get_style(recipe_dir); + const int recipe_build_system = recipe_get_build_system(recipe_dir, recipe_style); + if(!pushd(recipe_dir)) { - if (RECIPE_TYPE_ASTROCONDA == recipe_type) { + if (RECIPE_STYLE_ASTROCONDA == recipe_style) { pushd(path_basename(ctx->tests->test[i]->repository)); - } else if (RECIPE_TYPE_CONDA_FORGE == recipe_type) { + } else if (RECIPE_STYLE_CONDA_FORGE == recipe_style) { pushd("recipe"); } @@ -56,23 +58,28 @@ int delivery_build_recipes(struct Delivery *ctx) { snprintf(recipe_buildno, sizeof(recipe_buildno), " number: 0"); unsigned flags = REPLACE_TRUNCATE_AFTER_MATCH; - //file_replace_text("meta.yaml", "{% set version = ", recipe_version); - if (ctx->meta.final) { // remove this. i.e. statis cannot deploy a release to conda-forge - snprintf(recipe_version, sizeof(recipe_version), "{%% set version = \"%s\" %%}", ctx->tests->test[i]->version); - // TODO: replace sha256 of tagged archive - // TODO: leave the recipe unchanged otherwise. in theory this should produce the same conda package hash as conda forge. - // For now, remove the sha256 requirement - file_replace_text("meta.yaml", "sha256:", "\n", flags); - } else { - file_replace_text("meta.yaml", "{% set version = ", recipe_version, flags); - file_replace_text("meta.yaml", " url:", recipe_git_url, flags); - //file_replace_text("meta.yaml", "sha256:", recipe_git_rev); - file_replace_text("meta.yaml", " sha256:", "\n", flags); - file_replace_text("meta.yaml", " number:", recipe_buildno, flags); + if (recipe_build_system == RECIPE_BUILD_CONDA_BUILD) { + if (ctx->meta.final) { // remove this. i.e. statis cannot deploy a release to conda-forge + snprintf(recipe_version, sizeof(recipe_version), "{%% set version = \"%s\" %%}", ctx->tests->test[i]->version); + // TODO: replace sha256 of tagged archive + // TODO: leave the recipe unchanged otherwise. in theory this should produce the same conda package hash as conda forge. + // For now, remove the sha256 requirement + file_replace_text("meta.yaml", "sha256:", "\n", flags); + } else { + file_replace_text("meta.yaml", "{% set version = ", recipe_version, flags); + file_replace_text("meta.yaml", " url:", recipe_git_url, flags); + file_replace_text("meta.yaml", " sha256:", "\n", flags); + file_replace_text("meta.yaml", " number:", recipe_buildno, flags); + } + } else if (recipe_build_system == RECIPE_BUILD_RATTLER) { + file_replace_text("recipe.yaml", " version:", ctx->tests->test[i]->version, flags); + file_replace_text("recipe.yaml", " url:", recipe_git_url, flags); + file_replace_text("recipe.yaml", " sha256:", "\n", flags); + file_replace_text("recipe.yaml", " number:", recipe_buildno, flags); } char command[PATH_MAX]; - if (RECIPE_TYPE_CONDA_FORGE == recipe_type) { + if (RECIPE_STYLE_CONDA_FORGE == recipe_style) { char arch[STASIS_NAME_MAX] = {0}; char platform[STASIS_NAME_MAX] = {0}; @@ -92,18 +99,34 @@ int delivery_build_recipes(struct Delivery *ctx) { } tolower_s(arch); - snprintf(command, sizeof(command), "mambabuild --python=%s -m ../.ci_support/%s_%s_.yaml .", - ctx->meta.python, platform, arch); + // default build tool is "conda build" aka "build" + char tool[STASIS_NAME_MAX] = "build"; + if (recipe_build_system == RECIPE_BUILD_CONDA_BUILD) { + if (strlist_contains(globals.conda_packages, "boa", NULL)) { + safe_strncpy(tool, "mambabuild", sizeof(tool)); + } + } else if (recipe_build_system == RECIPE_BUILD_RATTLER) { + snprintf(tool, sizeof(tool), "rattler-build"); + } + snprintf(command, sizeof(command), "%s --python=%s -m ../.ci_support/%s_%s_.yaml .", + tool, ctx->meta.python, platform, arch); + } else { + snprintf(command, sizeof(command), "build --python=%s .", ctx->meta.python); + } + + int status = 0; + if (recipe_build_system == RECIPE_BUILD_RATTLER) { + // rattler-build is a standalone program, not a conda sub-command + status = system(command); } else { - snprintf(command, sizeof(command), "mambabuild --python=%s .", ctx->meta.python); + status = conda_exec(command); } - int status = conda_exec(command); if (status) { guard_free(recipe_dir); return -1; } - if (RECIPE_TYPE_GENERIC != recipe_type) { + if (RECIPE_STYLE_GENERIC != recipe_style) { popd(); } popd(); diff --git a/src/lib/delivery/delivery_conda.c b/src/lib/delivery/delivery_conda.c index 117e6c9..4f2920e 100644 --- a/src/lib/delivery/delivery_conda.c +++ b/src/lib/delivery/delivery_conda.c @@ -103,24 +103,61 @@ void delivery_install_conda(char *install_script, char *conda_install_dir) { } } -void delivery_conda_enable(struct Delivery *ctx, char *conda_install_dir) { - if (conda_activate(conda_install_dir, "base")) { - SYSERROR("conda activation failed"); - exit(1); - } +void delivery_conda_enable(struct Delivery *ctx) { + setenv("MAMBA_ROOT_PREFIX", ctx->storage.conda_install_prefix, 1); // Setting the CONDARC environment variable appears to be the only consistent // way to make sure the file is used. Not setting this variable leads to strange // behavior, especially if a conda environment is already active when STASIS is loaded. char rcpath[PATH_MAX]; - snprintf(rcpath, sizeof(rcpath), "%s/%s", conda_install_dir, ".condarc"); + snprintf(rcpath, sizeof(rcpath), "%s/%s", ctx->storage.conda_install_prefix, ".condarc"); setenv("CONDARC", rcpath, 1); + setenv("MAMBARC", rcpath, 1); if (runtime_replace(&ctx->runtime.environ, __environ)) { SYSERROR("unable to replace runtime environment after activating conda"); exit(1); } - if (conda_setup_headless()) { + if (conda_activate(ctx->storage.conda_install_prefix, "base")) { + SYSERROR("conda activation failed"); + exit(1); + } + + char pinned[PATH_MAX]; + snprintf(pinned, sizeof(pinned), "%s/conda-meta/pinned", ctx->storage.conda_install_prefix); + touch(pinned); + if (errno == ENOENT) { + errno = 0; + } + + char *conda_version = strdup(ctx->conda.installer_version); + if (conda_version) { + char *rev = strpbrk(conda_version, "-"); + if (rev) { + *rev = '\0'; + } + + FILE *pinned_fp = fopen(pinned, "w+"); + if (!pinned_fp) { + SYSERROR("unable to open conda-meta/pinned file for writing: %s", strerror(errno)); + exit(1); + } + fprintf(pinned_fp, "conda=%s\n", conda_version); + fclose(pinned_fp); + guard_free(conda_version); + } + + if (conda_capable(&ctx->conda.capabilities, ctx->storage.conda_install_prefix)) { + SYSERROR("Conda capability check failed"); + exit(1); + } + + if (!ctx->conda.capabilities.usable) { + SYSERROR("Conda is broken"); + exit(1); + } + + if (conda_setup_headless(&ctx->conda.capabilities)) { // no COE check. this call must succeed. exit(1); } diff --git a/src/lib/delivery/delivery_populate.c b/src/lib/delivery/delivery_populate.c index 5ce11d7..314ef46 100644 --- a/src/lib/delivery/delivery_populate.c +++ b/src/lib/delivery/delivery_populate.c @@ -199,6 +199,44 @@ static void normalize_ini_list(struct INIFILE **inip, struct StrList **listp, ch (*listp) = list; } +static int check_package_spec_list(struct StrList *list, const char *ini_section, const char *ini_key) { + if (!list) { + // empty lists are OK + return 0; + } + for (size_t i = 0; i < strlist_count(list); i++) { + const char *item = strlist_item(list, i); + if (!item) { + continue; + } + const char *invalid_chars = "@:/<>!~"; + const char *invalid_spec = strpbrk(item, invalid_chars); + if (invalid_spec) { + SYSERROR("Invalid version specification detected at %s:%s[%zu], \"%s\"", ini_section, ini_key, i, item); + SYSERROR("Package specification may not contain the operator '%c' (or any of \"%s\")", *invalid_spec, invalid_chars); + SYSERROR(""); + SYSERROR("%s:%s supports:", ini_section, ini_key); + SYSERROR(" {package}[=={version|tag|branch|ref}]"); + SYSERROR(""); + char *reported_name = strdup(item); + if (!reported_name) { + SYSERROR("unable to allocate memory for reported_name"); + return -1; + } + const char *reported_name_end = reported_name; + while (!ispunct(*reported_name_end)) { + reported_name_end++; + } + reported_name[reported_name_end - reported_name] = '\0'; + SYSERROR("Set test:%s.repository to point to a valid Git repository URL to enable [tag|branch|ref]", reported_name); + + guard_free(reported_name); + return -1; + } + } + return 0; +} + int populate_delivery_ini(struct Delivery *ctx, int render_mode) { struct INIFILE *ini = ctx->_stasis_ini_fp.delivery; struct INIData *rtdata; @@ -235,12 +273,22 @@ int populate_delivery_ini(struct Delivery *ctx, int render_mode) { ctx->conda.installer_platform = ini_getval_str(ini, "conda", "installer_platform", render_mode, &err); ctx->conda.installer_arch = ini_getval_str(ini, "conda", "installer_arch", render_mode, &err); ctx->conda.installer_baseurl = ini_getval_str(ini, "conda", "installer_baseurl", render_mode, &err); + ctx->conda.conda_packages = ini_getval_strlist(ini, "conda", "conda_packages", LINE_SEP, render_mode, &err); normalize_ini_list(&ini, &ctx->conda.conda_packages, "conda", "conda_packages", render_mode); + if (check_package_spec_list(ctx->conda.conda_packages, "conda", "conda_packages")) { + return -1; + } + ctx->conda.conda_packages_purge = ini_getval_strlist(ini, "conda", "conda_packages_purge", LINE_SEP, render_mode, &err); normalize_ini_list(&ini, &ctx->conda.conda_packages_purge, "conda", "conda_package_purge", render_mode); + ctx->conda.pip_packages = ini_getval_strlist(ini, "conda", "pip_packages", LINE_SEP, render_mode, &err); normalize_ini_list(&ini, &ctx->conda.pip_packages, "conda", "pip_packages", render_mode); + if (check_package_spec_list(ctx->conda.pip_packages, "conda", "pip_packages")) { + return -1; + } + ctx->conda.pip_packages_purge = ini_getval_strlist(ini, "conda", "pip_packages_purge", LINE_SEP, render_mode, &err); normalize_ini_list(&ini, &ctx->conda.pip_packages_purge, "conda", "pip_packages_purge", render_mode); diff --git a/src/lib/delivery/include/delivery.h b/src/lib/delivery/include/delivery.h index 3103a86..7d846ab 100644 --- a/src/lib/delivery/include/delivery.h +++ b/src/lib/delivery/include/delivery.h @@ -10,6 +10,7 @@ #include "environment.h" #include "ini.h" #include "multiprocessing.h" +#include "conda.h" #define DELIVERY_PLATFORM_MAX 4 #define DELIVERY_PLATFORM_MAXLEN 65 @@ -157,6 +158,7 @@ struct Delivery { struct StrList *pip_packages_defer; ///< Python packages to be built for delivery struct StrList *pip_packages_purge; ///< Python packages to remove from a delivery (for: based_on) struct StrList *wheels_packages; ///< Wheel packages built for delivery + struct CondaCapabilities capabilities; ///< Capability information } conda; /*! \struct Runtime @@ -354,9 +356,8 @@ void delivery_defer_packages(struct Delivery *ctx, int type); /** * Configure and activate a Conda installation based on Delivery context * @param ctx pointer to Delivery context - * @param conda_install_dir path to Conda installation */ -void delivery_conda_enable(struct Delivery *ctx, char *conda_install_dir); +void delivery_conda_enable(struct Delivery *ctx); /** * Install Conda @@ -16,9 +16,9 @@ conda_fresh_start = true ; (list) Conda packages to be installed/overridden in the base environment conda_packages = - conda-build>=3.22.0 boa - conda-libmamba-solver + conda-build + rattler-build ; (list) Python packages to be installed/overridden in the base environment ;pip_packages = diff --git a/tests/data/compression/bz2 b/tests/data/compression/bz2 Binary files differnew file mode 100644 index 0000000..20d3517 --- /dev/null +++ b/tests/data/compression/bz2 diff --git a/tests/data/compression/gz b/tests/data/compression/gz Binary files differnew file mode 100644 index 0000000..2762cd9 --- /dev/null +++ b/tests/data/compression/gz diff --git a/tests/data/compression/none b/tests/data/compression/none new file mode 100644 index 0000000..ce01362 --- /dev/null +++ b/tests/data/compression/none @@ -0,0 +1 @@ +hello diff --git a/tests/data/compression/xz b/tests/data/compression/xz Binary files differnew file mode 100644 index 0000000..f54e3fc --- /dev/null +++ b/tests/data/compression/xz diff --git a/tests/data/compression/zip b/tests/data/compression/zip Binary files differnew file mode 100644 index 0000000..1f77925 --- /dev/null +++ b/tests/data/compression/zip diff --git a/tests/data/compression/zstd b/tests/data/compression/zstd Binary files differnew file mode 100644 index 0000000..a827868 --- /dev/null +++ b/tests/data/compression/zstd diff --git a/tests/data/gbo_ng.ini b/tests/data/gbo_ng.ini new file mode 100644 index 0000000..355e5ee --- /dev/null +++ b/tests/data/gbo_ng.ini @@ -0,0 +1,66 @@ +[meta] +mission = generic +name = GBO +version = 2.4.6 +rc = 1 +final = false +based_on = {{ env:TEST_DATA }}/gbo.yml +python = 3.11 + + +[conda] +installer_name = Miniforge3 +installer_version = 26.3.2-2 +installer_platform = {{env:STASIS_CONDA_PLATFORM}} +installer_arch = {{env:STASIS_CONDA_ARCH}} +installer_baseurl = https://github.com/conda-forge/miniforge/releases/download/{{conda.installer_version}} +;conda_packages = +pip_packages = + firewatch==0.0.4 + gwcs==0.22.1 + tweakwcs==0.9.0 + + +[runtime] +CPPFLAGS = ${CPPFLAGS} -fpermissive +PYTHONUNBUFFERED = 1 + + +[test:firewatch] +repository = https://github.com/astroconda/firewatch +script_setup = + pip install -e '.' +script = + firewatch -c conda-forge -p ${STASIS_CONDA_PLATFORM_SUBDIR} | grep -E ' python-[0-9]' + + +[test:tweakwcs] +repository = https://github.com/spacetelescope/tweakwcs +script_setup = + pip install -e '.[test]' +script = + pytest \ + -r fEsx \ + --basetemp="{{ func:basetemp_dir() }}" \ + --junitxml="{{ func:junitxml_file() }}" + + +[deploy:artifactory:delivery] +files = + {{ storage.output_dir }}/** +dest = {{ meta.mission }}/{{ info.build_name }}/ + + +[deploy:docker] +registry = bytesalad.stsci.edu +image_compression = zstd -v -9 -c +build_args = + SNAPSHOT_INPUT={{ info.release_name }}.yml + SNAPSHOT_PKGDIR=packages +tags = + {{ meta.name }}:{{ info.build_number }}-py{{ meta.python_compact }} + {{ deploy.docker.registry }}/{{ meta.name }}:{{ info.build_number }}-py{{ meta.python_compact }} +test_script = + source /etc/profile + python -m pip freeze + mamba info diff --git a/tests/include/testing.h b/tests/include/testing.h index d11398c..a669b5d 100644 --- a/tests/include/testing.h +++ b/tests/include/testing.h @@ -11,7 +11,7 @@ #ifdef STASIS_TEST_VERBOSE #define STASIS_TEST_MSG(MSG, ...) do { \ -fprintf(stderr, "%s:%d:%s(): ", path_basename(__FILE__), __LINE__, __FUNCTION__); \ +fprintf(stderr, "%s:%d:%s(): ", path_basename(__FILE__), __LINE__, __func__); \ fprintf(stderr, MSG LINE_SEP, __VA_ARGS__); \ } while (0) #else @@ -76,6 +76,7 @@ inline void stasis_testing_record_result_summary() { do_message = 1; #endif strcpy(status_msg, "PASS"); + do_reason = 0; passed++; } if (do_message) { diff --git a/tests/rt_generic_ng_based_on.sh b/tests/rt_generic_ng_based_on.sh new file mode 100644 index 0000000..de5af24 --- /dev/null +++ b/tests/rt_generic_ng_based_on.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +here="$(dirname ${BASH_SOURCE[0]})" +source $here/setup.sh + +TEST_NAME=gbo_ng +PYTHON_VERSIONS=( + 3.11 +) +setup_workspace "$TEST_NAME" +run_command install_stasis + +ln -s "$TEST_DATA"/"$TEST_NAME".yml +for py_version in "${PYTHON_VERSIONS[@]}"; do + run_command run_stasis --python "$py_version" \ + --no-docker \ + --no-artifactory \ + "$TEST_DATA"/"$TEST_NAME".ini +done + +check_output_add "(null)" +run_command check_output_stasis_dir stasis/*/output +check_output_reset + +# NOTE: indexer default output directory is "output" +check_output_add "(null)" +run_command run_stasis_indexer stasis +run_command check_output_indexed_dir output +check_output_reset + +teardown_workspace "$TEST_NAME" diff --git a/tests/test_conda.c b/tests/test_conda.c index 4d0b4d8..7de1275 100644 --- a/tests/test_conda.c +++ b/tests/test_conda.c @@ -2,15 +2,14 @@ #include "conda.h" #include "delivery.h" -char cwd_start[PATH_MAX]; -char cwd_workspace[PATH_MAX]; int conda_is_installed = 0; void test_micromamba() { + const char *cwd_workspace = TEST_WORKSPACE_DIR; char mm_prefix[PATH_MAX] = {0}; char c_prefix[PATH_MAX] = {0}; - snprintf(mm_prefix, strlen(cwd_workspace) + strlen("micromamba") + 2, "%s/%s", cwd_workspace, "micromamba"); - snprintf(c_prefix, strlen(mm_prefix) + strlen("conda") + 2, "%s/%s", mm_prefix, "conda"); + snprintf(mm_prefix, sizeof(mm_prefix), "%s/%s/%s", cwd_workspace, "tools", "micromamba"); + snprintf(c_prefix, sizeof(c_prefix), "%s/%s", mm_prefix, "conda"); struct testcase { struct MicromambaInfo mminfo; @@ -18,10 +17,10 @@ void test_micromamba() { int result; }; struct testcase tc[] = { - {.mminfo = {.download_dir = cwd_workspace, .micromamba_prefix = mm_prefix, .conda_prefix = c_prefix}, .cmd = "info", .result = 0}, - {.mminfo = {.download_dir = cwd_workspace, .micromamba_prefix = mm_prefix, .conda_prefix = c_prefix}, .cmd = "env list", .result = 0}, - {.mminfo = {.download_dir = cwd_workspace, .micromamba_prefix = mm_prefix, .conda_prefix = c_prefix}, .cmd = "run python3 -V", .result = 0}, - {.mminfo = {.download_dir = cwd_workspace, .micromamba_prefix = mm_prefix, .conda_prefix = c_prefix}, .cmd = "no_such_option", .result = 109}, + {.mminfo = {.download_dir = (char *) cwd_workspace, .micromamba_prefix = mm_prefix, .conda_prefix = c_prefix}, .cmd = "info", .result = 0}, + {.mminfo = {.download_dir = (char *) cwd_workspace, .micromamba_prefix = mm_prefix, .conda_prefix = c_prefix}, .cmd = "env list", .result = 0}, + {.mminfo = {.download_dir = (char *) cwd_workspace, .micromamba_prefix = mm_prefix, .conda_prefix = c_prefix}, .cmd = "run python3 -V", .result = 0}, + {.mminfo = {.download_dir = (char *) cwd_workspace, .micromamba_prefix = mm_prefix, .conda_prefix = c_prefix}, .cmd = "no_such_option", .result = 109}, }; for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) { @@ -67,7 +66,7 @@ void test_conda_activate() { STASIS_SKIP_IF(!conda_is_installed, "cannot run without conda"); STASIS_ASSERT_FATAL(conda_activate(ctx.storage.conda_install_prefix, "base") == 0, "unable to activate base environment"); - + runtime_replace(&ctx.runtime.environ, __environ); for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) { struct testcase *item = &tc[i]; char *value = getenv(item->key); @@ -102,11 +101,13 @@ void test_python_exec() { void test_conda_setup_headless() { globals.conda_packages = strlist_init(); globals.pip_packages = strlist_init(); - strlist_append(&globals.conda_packages, "boa"); strlist_append(&globals.conda_packages, "conda-build"); - strlist_append(&globals.conda_packages, "conda-verify"); strlist_append(&globals.pip_packages, "pytest"); - STASIS_ASSERT(conda_setup_headless() == 0, "headless configuration failed"); + struct CondaCapabilities cc; + int capable = conda_capable(&cc, ctx.storage.conda_install_prefix); + STASIS_ASSERT_FATAL(capable == EXIT_SUCCESS, "conda capability check function should not fail"); + STASIS_ASSERT(conda_setup_headless(&cc) == 0, "headless configuration failed"); + conda_capable_free(&cc); } void test_conda_env_create_from_uri() { @@ -207,18 +208,10 @@ int main(int argc, char *argv[]) { test_delivery_gather_tool_versions, }; - char ws[] = "workspace_XXXXXX"; - if (!mkdtemp(ws)) { - SYSERROR("unable to mkdtemp: %s", strerror(errno)); - exit(1); - } - getcwd(cwd_start, sizeof(cwd_start) - 1); - mkdir(ws, 0755); - chdir(ws); - getcwd(cwd_workspace, sizeof(cwd_workspace) - 1); - snprintf(conda_prefix, strlen(cwd_workspace) + strlen("conda") + 2, "%s/conda", cwd_workspace); + snprintf(conda_prefix, strlen(TEST_WORKSPACE_DIR) + strlen("conda") + 2, "%s/conda", TEST_WORKSPACE_DIR); + const char *installer_version = "26.3.2-2"; const char *mockinidata = "[meta]\n" "name = mock\n" "version = 1.0.0\n" @@ -227,18 +220,21 @@ int main(int argc, char *argv[]) { "python = 3.11\n" "[conda]\n" "installer_name = Miniforge3\n" - "installer_version = 24.3.0-0\n" + "installer_version = %s\n" "installer_platform = {{env:STASIS_CONDA_PLATFORM}}\n" "installer_arch = {{env:STASIS_CONDA_ARCH}}\n" - "installer_baseurl = https://github.com/conda-forge/miniforge/releases/download/24.3.0-0\n"; - stasis_testing_write_ascii("mock.ini", mockinidata); + "installer_baseurl = https://github.com/conda-forge/miniforge/releases/download/%s\n"; + char mockinidata_final[STASIS_BUFSIZ] = {0}; + snprintf(mockinidata_final, sizeof(mockinidata_final), mockinidata, installer_version, installer_version); + + stasis_testing_write_ascii("mock.ini", mockinidata_final); struct INIFILE *ini = ini_open("mock.ini"); ctx._stasis_ini_fp.delivery = ini; ctx._stasis_ini_fp.delivery_path = realpath("mock.ini", NULL); const char *sysconfdir = getenv("STASIS_SYSCONFDIR"); globals.sysconfdir = strdup(sysconfdir ? sysconfdir : STASIS_SYSCONFDIR); - ctx.storage.root = strdup(cwd_workspace); + ctx.storage.root = strdup(TEST_WORKSPACE_DIR); setenv("LANG", "C", 1); bootstrap_build_info(&ctx); @@ -246,10 +242,6 @@ int main(int argc, char *argv[]) { STASIS_TEST_RUN(tests); - chdir(cwd_start); - if (rmtree(cwd_workspace)) { - perror(cwd_workspace); - } delivery_free(&ctx); globals_free(); STASIS_TEST_END_MAIN(); diff --git a/tests/test_recipe.c b/tests/test_recipe.c index 3ea21ce..939ac7d 100644 --- a/tests/test_recipe.c +++ b/tests/test_recipe.c @@ -33,27 +33,27 @@ void test_recipe_clone() { {.recipe_dir = "recipe_condaforge", .url = "https://github.com/conda-forge/fitsverify-feedstock", .gitref = "HEAD", - .expect_type = RECIPE_TYPE_CONDA_FORGE, + .expect_type = RECIPE_STYLE_CONDA_FORGE, .expect_return = 0}, {.recipe_dir = "recipe_astroconda", .url = "https://github.com/astroconda/astroconda-contrib", .gitref = "HEAD", - .expect_type = RECIPE_TYPE_ASTROCONDA, + .expect_type = RECIPE_STYLE_ASTROCONDA, .expect_return = 0}, {.recipe_dir = "recipe_generic", .url = "local_repo", .gitref = "HEAD", - .expect_type = RECIPE_TYPE_GENERIC, + .expect_type = RECIPE_STYLE_GENERIC, .expect_return = 0}, {.recipe_dir = "recipe_unknown", .url = "https://github.com/astroconda/firewatch", .gitref = "HEAD", - .expect_type = RECIPE_TYPE_UNKNOWN, + .expect_type = RECIPE_STYLE_UNKNOWN, .expect_return = 0}, {.recipe_dir = "recipe_broken", .url = "123_BAD_BAD_BAD_456", .gitref = "HEAD", - .expect_type = RECIPE_TYPE_UNKNOWN, + .expect_type = RECIPE_STYLE_UNKNOWN, .expect_return = 128}, }; for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) { @@ -76,7 +76,7 @@ void test_recipe_clone() { // Ensure a path to the repository was returned in the result argument STASIS_ASSERT(result_path != NULL, "result path should not be NULL"); // Verify the repository was detected as the correct recipe type - STASIS_ASSERT(recipe_get_type(result_path) == test->expect_type, "repository detected as the wrong type"); + STASIS_ASSERT(recipe_get_style(result_path) == test->expect_type, "repository detected as the wrong type"); if (test->expect_return == 0) { // Verify the result path exists diff --git a/tests/test_str.c b/tests/test_str.c index 09d8809..26dcf4c 100644 --- a/tests/test_str.c +++ b/tests/test_str.c @@ -16,7 +16,7 @@ void test_to_short_version() { for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) { char *result = to_short_version(tc[i].data); STASIS_ASSERT_FATAL(result != NULL, "should not be NULL"); - //printf("%s[%zu], result: %s, expected: %s\n", __FUNCTION__, i, result, tc[i].expected); + printf("%s[%zu], result: %s, expected: %s\n", __func__, i, result, tc[i].expected); STASIS_ASSERT(strcmp(result, tc[i].expected) == 0, "unexpected result"); guard_free(result); } @@ -110,12 +110,17 @@ void test_strchrdel() { const struct testcase tc[] = { {.data ="aaaabbbbcccc", .input = "ac", .expected = "bbbb"}, {.data = "1I 2have 3a 4pencil 5box.", .input = "1245", .expected = "I have 3a pencil box."}, + {.data = "1I 2have 3a 4pencil 5box.", .input = "12345", .expected = "I have a pencil box."}, {.data = "\v\v\vI\t\f ha\tve a\t pen\tcil b\tox.", .input = " \f\t\v", "Ihaveapencilbox."}, }; for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) { char *data = strdup(tc[i].data); + printf("data before: %s\n", data); + debug_hexdump(data, strlen(data) + 1); strchrdel(data, tc[i].input); + printf("data after: %s\n", data); + debug_hexdump(data, strlen(data) + 1); STASIS_ASSERT(strcmp(data, tc[i].expected) == 0, "wrong status for condition"); guard_free(data); } @@ -184,7 +189,10 @@ void test_num_chars() { {.data = "abc\t\ndef\nabc\ndef\n", .input = '\t', .expected = 1}, }; for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) { - STASIS_ASSERT(num_chars(tc[i].data, tc[i].input) == tc[i].expected, "incorrect number of characters detected"); + const int count = num_chars(tc[i].data, tc[i].input); + SYSDEBUG("input[%zu] = '%s'", i, tc[i].data); + SYSDEBUG("result[%zu:'%c'] = %d", i, tc[i].input, count); + STASIS_ASSERT(count == tc[i].expected, "incorrect number of characters detected"); } } @@ -207,6 +215,13 @@ void test_split() { }; for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) { char **result = split((char *) tc[i].data, tc[i].delim, tc[i].max_split); + if (tc[i].data) { + SYSDEBUG("input[%zu] = %s", i, tc[i].data); + for (size_t j = 0; result[j] != NULL; j++) { + SYSDEBUG("result[%zu][%zu] = %s", i, j, result[j]); + debug_hexdump(result[j], strlen(result[j]) + 1); + } + } STASIS_ASSERT(strcmp_array((const char **) result, tc[i].expected) == 0, "Split failed"); guard_array_free(result); } @@ -225,8 +240,9 @@ void test_join() { {.data = NULL, .delim = NULL, .expected = ""}, }; for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) { - char *result; - result = join((char **) tc[i].data, tc[i].delim); + char *result = join((char **) tc[i].data, tc[i].delim); + SYSDEBUG("result[%zu] = '%s'", i, result ? result : "NULL"); + debug_hexdump(result, strlen(result ? result : "")); STASIS_ASSERT(strcmp(result ? result : "", tc[i].expected) == 0, "failed to join array"); guard_free(result); } @@ -245,6 +261,8 @@ void test_join_ex() { }; for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) { char *result = join_ex((char *) tc[i].delim, "a", "b", "c", "d", "e", NULL); + SYSDEBUG("result[%zu] = '%s'\n", i, result); + debug_hexdump(result, (int) strlen(result) + 1); STASIS_ASSERT(strcmp(result ? result : "", tc[i].expected) == 0, "failed to join array"); guard_free(result); } @@ -271,6 +289,9 @@ void test_substring_between() { }; for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) { char *result = substring_between((char *) tc[i].data, tc[i].delim); + SYSDEBUG("input[%zu] = '%s'", i, tc[i].data ? tc[i].data : "NULL"); + SYSDEBUG("result[%zu] = '%s'", i, result ? result : "NULL"); + debug_hexdump(result, result ? strlen(result) + 1 : 0); STASIS_ASSERT(strcmp(result ? result : "", tc[i].expected) == 0, "unable to extract substring"); guard_free(result); } @@ -290,6 +311,16 @@ void test_strdeldup() { }; for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) { char **result = strdeldup(tc[i].data); + if (tc[i].data && result) { + char *input_str = join((char **) tc[i].data, " "); + SYSDEBUG("input[%zu] = '%s'", i, input_str); + guard_free(input_str); + + char *result_str = join(result, " "); + SYSDEBUG("result[%zu] = '%s'", i, result_str); + debug_hexdump(result_str, strlen(result_str)); + guard_free(result_str); + } STASIS_ASSERT(strcmp_array((const char **) result, tc[i].expected) == 0, "incorrect number of duplicates removed"); guard_array_free(result); } @@ -316,11 +347,14 @@ void test_lstrip() { }; STASIS_ASSERT(lstrip(NULL) == NULL, "incorrect return type"); + const size_t maxlen = 64; for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) { - char *buf = calloc(255, sizeof(*buf)); - strncpy(buf, tc[i].data, 254); - buf[254] = '\0'; + char *buf = calloc(maxlen + 1, sizeof(*buf)); + strncpy(buf, tc[i].data, maxlen); char *result = lstrip(buf); + SYSDEBUG("input[%zu] = '%s'", i, buf); + SYSDEBUG("result[%zu] = '%s'", i, result ? result : "NULL"); + debug_hexdump(result, maxlen); STASIS_ASSERT(strcmp(result ? result : "", tc[i].expected) == 0, "incorrect strip-from-left"); guard_free(buf); } @@ -341,11 +375,14 @@ void test_strip() { }; STASIS_ASSERT(strip(NULL) == NULL, "incorrect return type"); + const ssize_t maxlen = 64; for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) { - char *buf = calloc(255, sizeof(*buf)); - strncpy(buf, tc[i].data, 254); - buf[254] = '\0'; + char *buf = calloc(maxlen + 1, sizeof(*buf)); + strncpy(buf, tc[i].data, maxlen); char *result = strip(buf); + SYSDEBUG("input[%zu] = '%s'", i, buf); + SYSDEBUG("result[%zu] = '%s'", i, result ? result : "NULL"); + debug_hexdump(result, maxlen); STASIS_ASSERT(strcmp(result ? result : "", tc[i].expected) == 0, "incorrect strip-from-right"); guard_free(buf); } diff --git a/tests/test_utils.c b/tests/test_utils.c index fc53f53..e1689dd 100644 --- a/tests/test_utils.c +++ b/tests/test_utils.c @@ -456,6 +456,52 @@ void test_pushd_popd_suggested_workflow() { } } +void test_is_file_compressed() { + const char *filenames[] = { + "zstd", "bz2", "gz", "xz", "zip", + }; + char datadir[PATH_MAX] = {0}; + snprintf(datadir, sizeof(datadir), "%s/compression", TEST_DATA_DIR); + + char inputfile[PATH_MAX] = {0}; + for (size_t i = 0; i < sizeof(filenames) / sizeof(*filenames); i++) { + snprintf(inputfile, sizeof(inputfile), "%s/%s", datadir, filenames[i]); + const int compressed = is_file_compressed(inputfile); + SYSDEBUG("[%zu] is %s compressed? => %s", i, inputfile, compressed ? "Yes" : "No"); + STASIS_ASSERT(compressed == true, "compression should have been detected"); + } + + snprintf(inputfile, sizeof(inputfile), "%s/none", datadir); + STASIS_ASSERT(is_file_compressed(inputfile) == false, "'none' file should not be detected as compressed data"); + + for (size_t i = 0; i < sizeof(filenames) / sizeof(*filenames); i++) { + char bytes[128]; + if (get_random_bytes(bytes, sizeof(bytes))) { + SYSERROR("get_random_bytes failed: %s, %s", bytes, strerror(errno)); + STASIS_ASSERT_FATAL(false, "get_random_bytes failed"); + return; + } + + FILE *fp = fopen(filenames[i], "wb"); + if (!fp) { + SYSERROR("fopen failed: %s, %s", filenames[i], strerror(errno)); + STASIS_ASSERT_FATAL(false, "fopen failed"); + return; + } + + bytes[0] = 'J'; + const size_t bytes_written = fwrite(bytes, 1, sizeof(bytes), fp); + if (bytes_written != sizeof(bytes)) { + SYSERROR("fwrite failed: %s, %s", bytes, strerror(errno)); + STASIS_ASSERT_FATAL(false, "fwrite failed"); + return; + } + fclose(fp); + + STASIS_ASSERT(is_file_compressed(filenames[i]) == false, "random data should not be detected as compressed"); + } +} + int main(int argc, char *argv[]) { STASIS_TEST_BEGIN_MAIN(); @@ -479,6 +525,7 @@ int main(int argc, char *argv[]) { test_dirstack, test_pushd_popd, test_pushd_popd_suggested_workflow, + test_is_file_compressed, }; const char *ws = "workspace"; getcwd(cwd_start, sizeof(cwd_start) - 1); diff --git a/tests/test_wheel.c b/tests/test_wheel.c index e486b05..c9aff6a 100644 --- a/tests/test_wheel.c +++ b/tests/test_wheel.c @@ -175,17 +175,19 @@ int main(int argc, char *argv[]) { delivery_get_conda_installer_url(&ctx, install_url, PATH_MAX); delivery_get_conda_installer(&ctx, install_url); delivery_install_conda(ctx.conda.installer_path, ctx.storage.conda_install_prefix); + delivery_conda_enable(&ctx); guard_free(install_url); if (conda_activate(ctx.storage.conda_install_prefix, "base")) { SYSERROR("conda_activate failed"); exit(1); } - if (conda_exec("install -y boa conda-build")) { + if (conda_exec("install -y conda-build")) { SYSERROR("conda_exec failed"); exit(1); } - if (conda_setup_headless()) { + + if (conda_setup_headless(&ctx.conda.capabilities)) { SYSERROR("conda_setup_headless failed"); exit(1); } |