From 5dfabb1f5eed4d9d789b3c0df467f728e4a7cdf1 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Mon, 7 Jul 2025 17:23:27 -0400 Subject: Remove dead code --- src/lib/delivery/delivery_populate.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/lib/delivery/delivery_populate.c b/src/lib/delivery/delivery_populate.c index 84676f1..53d44e3 100644 --- a/src/lib/delivery/delivery_populate.c +++ b/src/lib/delivery/delivery_populate.c @@ -162,8 +162,6 @@ int populate_delivery_ini(struct Delivery *ctx, int render_mode) { // keys in the configuration RuntimeEnv *rt = runtime_copy(__environ); while ((rtdata = ini_getall(ini, "runtime")) != NULL) { - char rec[STASIS_BUFSIZ]; - sprintf(rec, "%s=%s", lstrip(strip(rtdata->key)), lstrip(strip(rtdata->value))); runtime_set(rt, rtdata->key, rtdata->value); } runtime_apply(rt); -- cgit From 54aa6bfd1ee19c5d52602cdadd97a6c9aa965c0f Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Mon, 7 Jul 2025 17:29:46 -0400 Subject: Add delivery_export.c --- src/lib/delivery/CMakeLists.txt | 1 + src/lib/delivery/delivery_export.c | 58 +++++++++++++++++++++++++++++++++++++ src/lib/delivery/include/delivery.h | 24 +++++++++++++++ 3 files changed, 83 insertions(+) create mode 100644 src/lib/delivery/delivery_export.c diff --git a/src/lib/delivery/CMakeLists.txt b/src/lib/delivery/CMakeLists.txt index 78ed20f..559b2dc 100644 --- a/src/lib/delivery/CMakeLists.txt +++ b/src/lib/delivery/CMakeLists.txt @@ -1,4 +1,5 @@ add_library(stasis_delivery STATIC + delivery_export.c delivery_postprocess.c delivery_conda.c delivery_docker.c diff --git a/src/lib/delivery/delivery_export.c b/src/lib/delivery/delivery_export.c new file mode 100644 index 0000000..d982ad5 --- /dev/null +++ b/src/lib/delivery/delivery_export.c @@ -0,0 +1,58 @@ +#include "delivery.h" + +static void delivery_export_configuration(const struct Delivery *ctx) { + msg(STASIS_MSG_L2, "Exporting delivery configuration\n"); + if (!pushd(ctx->storage.cfgdump_dir)) { + char filename[PATH_MAX] = {0}; + sprintf(filename, "%s.ini", ctx->info.release_name); + FILE *spec = fopen(filename, "w+"); + if (!spec) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", filename); + exit(1); + } + ini_write(ctx->_stasis_ini_fp.delivery, &spec, INI_WRITE_RAW); + fclose(spec); + + memset(filename, 0, sizeof(filename)); + sprintf(filename, "%s-rendered.ini", ctx->info.release_name); + spec = fopen(filename, "w+"); + if (!spec) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", filename); + exit(1); + } + ini_write(ctx->_stasis_ini_fp.delivery, &spec, INI_WRITE_PRESERVE); + fclose(spec); + popd(); + } else { + SYSERROR("Failed to enter directory: %s", ctx->storage.delivery_dir); + exit(1); + } +} + +void delivery_export(const struct Delivery *ctx, char *envs[]) { + delivery_export_configuration(ctx); + + for (size_t i = 0; envs[i] != NULL; i++) { + char *name = envs[i]; + msg(STASIS_MSG_L2, "Exporting %s\n", name); + if (conda_env_export(name, ctx->storage.delivery_dir, name)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", name); + exit(1); + } + } +} + +void delivery_rewrite_stage1(struct Delivery *ctx, char *specfile) { + // Rewrite release environment output (i.e. set package origin(s) to point to the deployment server, etc.) + msg(STASIS_MSG_L3, "Rewriting release spec file (stage 1): %s\n", path_basename(specfile)); + delivery_rewrite_spec(ctx, specfile, DELIVERY_REWRITE_SPEC_STAGE_1); + + msg(STASIS_MSG_L1, "Rendering mission templates\n"); + delivery_mission_render_files(ctx); +} + +void delivery_rewrite_stage2(struct Delivery *ctx, char *specfile) { + msg(STASIS_MSG_L3, "Rewriting release spec file (stage 2): %s\n", path_basename(specfile)); + delivery_rewrite_spec(ctx, specfile, DELIVERY_REWRITE_SPEC_STAGE_2); +} + diff --git a/src/lib/delivery/include/delivery.h b/src/lib/delivery/include/delivery.h index 26a5499..69ec089 100644 --- a/src/lib/delivery/include/delivery.h +++ b/src/lib/delivery/include/delivery.h @@ -459,4 +459,28 @@ int delivery_series_sync(struct Delivery *ctx); */ int delivery_purge_packages(struct Delivery *ctx, const char *env_name, int use_pkg_manager); +/** + * Export delivery environments + * + * @param ctx Delivery context + * @param envs array of conda environment names + */ +void delivery_export(const struct Delivery *ctx, char *envs[]); + +/** + * STAGE 1: Rewrite delivery-related strings in specfile + * + * @param ctx Delivery context + * @param specfile path to YAML spec file + */ +void delivery_rewrite_stage1(struct Delivery *ctx, char *specfile); + +/** + * STAGE 2: Rewrite delivery-related strings in specfile + * + * @param ctx Delivery context + * @param specfile path to YAML spec file + */ +void delivery_rewrite_stage2(struct Delivery *ctx, char *specfile); + #endif //STASIS_DELIVERY_H -- cgit From 641c5d9cff3369b4cf0000a60180b47f3cefb0fa Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Mon, 7 Jul 2025 17:31:15 -0400 Subject: Move steps into functions --- src/cli/stasis/stasis_main.c | 717 +++++++++++++++++++++++-------------------- 1 file changed, 387 insertions(+), 330 deletions(-) diff --git a/src/cli/stasis/stasis_main.c b/src/cli/stasis/stasis_main.c index 459a6f6..d1c48e4 100644 --- a/src/cli/stasis/stasis_main.c +++ b/src/cli/stasis/stasis_main.c @@ -10,140 +10,7 @@ #include "system_requirements.h" #include "tpl.h" - -int main(int argc, char *argv[]) { - struct Delivery ctx; - struct Process proc = { - .f_stdout = "", - .f_stderr = "", - .redirect_stderr = 0, - }; - char env_name[STASIS_NAME_MAX] = {0}; - char env_name_testing[STASIS_NAME_MAX] = {0}; - char *delivery_input = NULL; - char *config_input = NULL; - char installer_url[PATH_MAX]; - char python_override_version[STASIS_NAME_MAX]; - int user_disabled_docker = false; - globals.cpu_limit = get_cpu_count(); - if (globals.cpu_limit > 1) { - globals.cpu_limit--; // max - 1 - } - - memset(env_name, 0, sizeof(env_name)); - memset(env_name_testing, 0, sizeof(env_name_testing)); - memset(installer_url, 0, sizeof(installer_url)); - memset(python_override_version, 0, sizeof(python_override_version)); - memset(&proc, 0, sizeof(proc)); - memset(&ctx, 0, sizeof(ctx)); - - int c; - int option_index = 0; - while ((c = getopt_long(argc, argv, "hVCc:p:vU", long_options, &option_index)) != -1) { - switch (c) { - case 'h': - usage(path_basename(argv[0])); - exit(0); - case 'V': - puts(VERSION); - exit(0); - case 'c': - config_input = strdup(optarg); - break; - case 'C': - globals.continue_on_error = true; - break; - case 'p': - strcpy(python_override_version, optarg); - break; - case 'l': - globals.cpu_limit = strtol(optarg, NULL, 10); - if (globals.cpu_limit <= 1) { - globals.cpu_limit = 1; - globals.enable_parallel = false; // No point - } - break; - case OPT_ALWAYS_UPDATE_BASE: - globals.always_update_base_environment = true; - break; - case OPT_FAIL_FAST: - globals.parallel_fail_fast = true; - break; - case OPT_POOL_STATUS_INTERVAL: - globals.pool_status_interval = (int) strtol(optarg, NULL, 10); - if (globals.pool_status_interval < 1) { - globals.pool_status_interval = 1; - } else if (globals.pool_status_interval > 60 * 10) { - // Possible poor choice alert - fprintf(stderr, "Caution: Excessive pausing between status updates may cause third-party CI/CD" - " jobs to fail if the stdout/stderr streams are idle for too long!\n"); - } - break; - case 'U': - setenv("PYTHONUNBUFFERED", "1", 1); - fflush(stdout); - fflush(stderr); - setvbuf(stdout, NULL, _IONBF, 0); - setvbuf(stderr, NULL, _IONBF, 0); - break; - case 'v': - globals.verbose = true; - break; - case OPT_OVERWRITE: - globals.enable_overwrite = true; - break; - case OPT_NO_DOCKER: - globals.enable_docker = false; - user_disabled_docker = true; - break; - case OPT_NO_ARTIFACTORY: - globals.enable_artifactory = false; - break; - case OPT_NO_ARTIFACTORY_BUILD_INFO: - globals.enable_artifactory_build_info = false; - break; - case OPT_NO_ARTIFACTORY_UPLOAD: - globals.enable_artifactory_build_info = false; - globals.enable_artifactory_upload = false; - break; - case OPT_NO_TESTING: - globals.enable_testing = false; - break; - case OPT_NO_REWRITE_SPEC_STAGE_2: - globals.enable_rewrite_spec_stage_2 = false; - break; - case OPT_NO_PARALLEL: - globals.enable_parallel = false; - break; - case '?': - default: - exit(1); - } - } - - if (optind < argc) { - while (optind < argc) { - // use first positional argument - delivery_input = argv[optind++]; - break; - } - } - - if (!delivery_input) { - fprintf(stderr, "error: a DELIVERY_FILE is required\n"); - usage(path_basename(argv[0])); - exit(1); - } - - printf(BANNER, VERSION, AUTHOR); - - check_system_path(); - - msg(STASIS_MSG_L1, "Setup\n"); - - tpl_setup_vars(&ctx); - tpl_setup_funcs(&ctx); - +static void setup_sysconfdir() { // Set up PREFIX/etc directory information // The user may manipulate the base directory path with STASIS_SYSCONFDIR // environment variable @@ -159,83 +26,103 @@ int main(int argc, char *argv[]) { msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Unable to resolve path to configuration directory: %s\n", stasis_sysconfdir_tmp); exit(1); } +} +static void setup_python_version_override(struct Delivery *ctx, const char *version) { // Override Python version from command-line, if any - if (strlen(python_override_version)) { - guard_free(ctx.meta.python); - ctx.meta.python = strdup(python_override_version); - guard_free(ctx.meta.python_compact); - ctx.meta.python_compact = to_short_version(ctx.meta.python); + if (strlen(version)) { + guard_free(ctx->meta.python); + ctx->meta.python = strdup(version); + if (!ctx->meta.python) { + SYSERROR("%s", "Unable to allocate bytes for python version override"); + } + guard_free(ctx->meta.python_compact); + ctx->meta.python_compact = to_short_version(ctx->meta.python); } +} - if (!config_input) { +static void configure_stasis_ini(struct Delivery *ctx, char **config_input) { + if (!*config_input) { // no configuration passed by argument. use basic config. char cfgfile[PATH_MAX * 2]; sprintf(cfgfile, "%s/%s", globals.sysconfdir, "stasis.ini"); if (!access(cfgfile, F_OK | R_OK)) { - config_input = strdup(cfgfile); + *config_input = strdup(cfgfile); } else { msg(STASIS_MSG_WARN, "STASIS global configuration is not readable, or does not exist: %s", cfgfile); } - } - - if (config_input) { - msg(STASIS_MSG_L2, "Reading STASIS global configuration: %s\n", config_input); - ctx._stasis_ini_fp.cfg = ini_open(config_input); - if (!ctx._stasis_ini_fp.cfg) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read config file: %s, %s\n", delivery_input, strerror(errno)); + } else { + msg(STASIS_MSG_L2, "Reading STASIS global configuration: %s\n", *config_input); + ctx->_stasis_ini_fp.cfg = ini_open(*config_input); + if (!ctx->_stasis_ini_fp.cfg) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read config file: %s, %s\n", *config_input, strerror(errno)); exit(1); } - ctx._stasis_ini_fp.cfg_path = strdup(config_input); - guard_free(config_input); + ctx->_stasis_ini_fp.cfg_path = strdup(*config_input); + guard_free(*config_input); } +} - msg(STASIS_MSG_L2, "Reading STASIS delivery configuration: %s\n", delivery_input); - ctx._stasis_ini_fp.delivery = ini_open(delivery_input); - if (!ctx._stasis_ini_fp.delivery) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read delivery file: %s, %s\n", delivery_input, strerror(errno)); +static void configure_delivery_ini(struct Delivery *ctx, char **delivery_input) { + msg(STASIS_MSG_L2, "Reading STASIS delivery configuration: %s\n", *delivery_input); + ctx->_stasis_ini_fp.delivery = ini_open(*delivery_input); + if (!ctx->_stasis_ini_fp.delivery) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read delivery file: %s, %s\n", *delivery_input, strerror(errno)); exit(1); } - ctx._stasis_ini_fp.delivery_path = strdup(delivery_input); + ctx->_stasis_ini_fp.delivery_path = strdup(*delivery_input); +} +static void configure_delivery_context(struct Delivery *ctx) { msg(STASIS_MSG_L2, "Bootstrapping delivery context\n"); - if (bootstrap_build_info(&ctx)) { + if (bootstrap_build_info(ctx)) { msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to bootstrap delivery context\n"); exit(1); } msg(STASIS_MSG_L2, "Initializing delivery context\n"); - if (delivery_init(&ctx, INI_READ_RENDER)) { + if (delivery_init(ctx, INI_READ_RENDER)) { msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to initialize delivery context\n"); exit(1); } - check_requirements(&ctx); +} +static void configure_jfrog_cli(struct Delivery *ctx) { msg(STASIS_MSG_L2, "Configuring JFrog CLI\n"); - if (delivery_init_artifactory(&ctx)) { + if (delivery_init_artifactory(ctx)) { msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "JFrog CLI configuration failed\n"); exit(1); } +} - runtime_apply(ctx.runtime.environ); - strcpy(env_name, ctx.info.release_name); - strcpy(env_name_testing, env_name); - strcat(env_name_testing, "-test"); - +static void check_release_history(struct Delivery *ctx) { // Safety gate: Avoid clobbering a delivered release unless the user wants that behavior msg(STASIS_MSG_L1, "Checking release history\n"); - if (!globals.enable_overwrite && delivery_exists(&ctx) == DELIVERY_FOUND) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Refusing to overwrite release: %s\nUse --overwrite to enable release clobbering.\n", ctx.info.release_name); + if (!globals.enable_overwrite && delivery_exists(ctx) == DELIVERY_FOUND) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Refusing to overwrite release: %s\nUse --overwrite to enable release clobbering.\n", ctx->info.release_name); exit(1); } +} + +static void check_conda_install_prefix(const struct Delivery *ctx) { + // Unlikely to occur: this should help prevent rmtree() from destroying your entire filesystem + // if path is "/" then, die + // or if empty string, die + if (!strcmp(ctx->storage.conda_install_prefix, DIR_SEP) || !strlen(ctx->storage.conda_install_prefix)) { + fprintf(stderr, "error: ctx.storage.conda_install_prefix is malformed!\n"); + exit(1); + } +} + +static void sync_release_history(struct Delivery *ctx) { if (globals.enable_artifactory) { // We need to download previous revisions to ensure processed packages are available at build-time // This is also a docker requirement. Python wheels must exist locally. - if (ctx.meta.rc > 1) { - msg(STASIS_MSG_L1, "Syncing delivery artifacts for %s\n", ctx.info.build_name); - if (delivery_series_sync(&ctx) != 0) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Unable to sync artifacts for %s\n", ctx.info.build_name); + if (ctx->meta.rc > 1) { + msg(STASIS_MSG_L1, "Syncing delivery artifacts for %s\n", ctx->info.build_name); + if (delivery_series_sync(ctx) != 0) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Unable to sync artifacts for %s\n", ctx->info.build_name); msg(STASIS_MSG_L3, "Case #1:\n" "\tIf this is a new 'version', and 'rc' is greater " "than 1, then no previous deliveries exist remotely. " @@ -248,45 +135,43 @@ int main(int argc, char *argv[]) { } } } +} - // Unlikely to occur: this should help prevent rmtree() from destroying your entire filesystem - // if path is "/" then, die - // or if empty string, die - if (!strcmp(ctx.storage.conda_install_prefix, DIR_SEP) || !strlen(ctx.storage.conda_install_prefix)) { - fprintf(stderr, "error: ctx.storage.conda_install_prefix is malformed!\n"); - exit(1); - } - +static void check_conda_prefix_length(const struct Delivery *ctx) { // 2 = #! // 5 = /bin\n - const size_t prefix_len = strlen(ctx.storage.conda_install_prefix) + 2 + 5; + const size_t prefix_len = strlen(ctx->storage.conda_install_prefix) + 2 + 5; const size_t prefix_len_max = 127; msg(STASIS_MSG_L1, "Checking length of conda installation prefix\n"); - if (!strcmp(ctx.system.platform[DELIVERY_PLATFORM], "Linux") && prefix_len > prefix_len_max) { + if (!strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Linux") && prefix_len > prefix_len_max) { msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "The shebang, '#!%s/bin/python\\n' is too long (%zu > %zu).\n", - ctx.storage.conda_install_prefix, prefix_len, prefix_len_max); + ctx->storage.conda_install_prefix, prefix_len, prefix_len_max); msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "Conda's workaround to handle long path names does not work consistently within STASIS.\n"); msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "Please try again from a different, \"shorter\", directory.\n"); exit(1); } +} +static void setup_conda(struct Delivery *ctx, char *installer_url) { msg(STASIS_MSG_L1, "Conda setup\n"); - delivery_get_conda_installer_url(&ctx, installer_url); + delivery_get_conda_installer_url(ctx, installer_url); msg(STASIS_MSG_L2, "Downloading: %s\n", installer_url); - if (delivery_get_conda_installer(&ctx, installer_url)) { + if (delivery_get_conda_installer(ctx, installer_url)) { msg(STASIS_MSG_ERROR, "download failed: %s\n", installer_url); exit(1); } - msg(STASIS_MSG_L2, "Installing: %s\n", ctx.conda.installer_name); - delivery_install_conda(ctx.conda.installer_path, ctx.storage.conda_install_prefix); + msg(STASIS_MSG_L2, "Installing: %s\n", ctx->conda.installer_name); + delivery_install_conda(ctx->conda.installer_path, ctx->storage.conda_install_prefix); - msg(STASIS_MSG_L2, "Configuring: %s\n", ctx.storage.conda_install_prefix); - delivery_conda_enable(&ctx, ctx.storage.conda_install_prefix); + msg(STASIS_MSG_L2, "Configuring: %s\n", ctx->storage.conda_install_prefix); + delivery_conda_enable(ctx, ctx->storage.conda_install_prefix); +} +static void configure_conda_base(struct Delivery *ctx, char *envs[]) { // // Implied environment creation modes/actions // @@ -303,269 +188,247 @@ int main(int argc, char *argv[]) { // 3b. Bugs, conflicts, and dependency resolution issues are inherited and // must be handled in the INI config msg(STASIS_MSG_L1, "Creating release environment(s)\n"); - char *mission_base = NULL; - if (isempty(ctx.meta.based_on)) { - guard_free(ctx.meta.based_on); + + if (isempty(ctx->meta.based_on)) { + // based_on was not set by the input file + + guard_free(ctx->meta.based_on); char *mission_base_orig = NULL; - if (asprintf(&mission_base_orig, "%s/%s/base.yml", ctx.storage.mission_dir, ctx.meta.mission) < 0) { - SYSERROR("Unable to allocate bytes for %s/%s/base.yml path\n", ctx.storage.mission_dir, ctx.meta.mission); + if (asprintf(&mission_base_orig, "%s/%s/base.yml", ctx->storage.mission_dir, ctx->meta.mission) < 0) { + SYSERROR("Unable to allocate bytes for %s/%s/base.yml path\n", ctx->storage.mission_dir, ctx->meta.mission); exit(1); } + // Does a base.yml exist in the mission directory? + // If not, do nothing. Otherwise, use the base.yml in the mission directory. if (access(mission_base_orig, F_OK) < 0) { - msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "Mission does not provide a base.yml configuration: %s (%s)\n", - ctx.meta.mission, ctx.storage.mission_dir); + msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "Mission does not provide a base.yml"); } else { msg(STASIS_MSG_L2, "Using base environment configuration: %s\n", mission_base_orig); - if (asprintf(&mission_base, "%s/%s-base.yml", ctx.storage.tmpdir, ctx.info.release_name) < 0) { + if (asprintf(&mission_base, "%s/%s-base.yml", ctx->storage.tmpdir, ctx->info.release_name) < 0) { SYSERROR("%s", "Unable to allocate bytes for temporary base.yml configuration"); remove(mission_base); exit(1); } copy2(mission_base_orig, mission_base, CT_OWNER | CT_PERM); - ctx.meta.based_on = mission_base; + ctx->meta.based_on = mission_base; } guard_free(mission_base_orig); } - if (!isempty(ctx.meta.based_on)) { - if (conda_env_exists(ctx.storage.conda_install_prefix, env_name) && conda_env_remove(env_name)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to remove release environment: %s\n", env_name); - exit(1); - } + msg(STASIS_MSG_L2, "Based on: %s\n", ctx->meta.based_on); - msg(STASIS_MSG_L2, "Based on: %s\n", ctx.meta.based_on); - if (conda_env_create_from_uri(env_name, ctx.meta.based_on, ctx.meta.python)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "unable to install release environment using configuration file\n"); - exit(1); - } + for (size_t i = 0; envs[i] != NULL; i += 2) { + char *title = envs[i]; + char *env = envs[i+1]; + // If based_on was populated above, or defined in the configuration: install its packages. + if (!isempty(ctx->meta.based_on)) { + if (conda_env_exists(ctx->storage.conda_install_prefix, env) && conda_env_remove(env)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to remove %s environment: %s\n", title); + exit(1); + } - if (conda_env_exists(ctx.storage.conda_install_prefix, env_name_testing) && conda_env_remove(env_name_testing)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to remove testing environment %s\n", env_name_testing); - exit(1); - } - if (conda_env_create_from_uri(env_name_testing, ctx.meta.based_on, ctx.meta.python)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "unable to install testing environment using configuration file\n"); - exit(1); - } - } else { - if (conda_env_create(env_name, ctx.meta.python, NULL)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to create release environment\n"); - exit(1); - } - if (conda_env_create(env_name_testing, ctx.meta.python, NULL)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to create testing environment\n"); - exit(1); + if (conda_env_create_from_uri(env, ctx->meta.based_on, ctx->meta.python)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "unable to install %s environment using configuration file\n", title); + exit(1); + } + } else { + // Otherwise, create the environments with the requested Python version and move on + if (conda_env_create(env, ctx->meta.python, NULL)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to create %s environment\n", title); + exit(1); + } } } // The base environment configuration not used past this point remove(mission_base); +} - const char *envs[] = {env_name_testing, env_name}; - for (size_t e = 0; e < sizeof(envs) / sizeof(*envs); e++) { - const char *name = envs[e]; - if (ctx.conda.conda_packages_purge && strlist_count(ctx.conda.conda_packages_purge)) { - msg(STASIS_MSG_L2, "Purging conda packages from %s\n", name); - if (delivery_purge_packages(&ctx, name, PKG_USE_CONDA)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "unable to purge requested conda packages from %s\n", name); - exit(1); - } - } - - if (ctx.conda.pip_packages_purge && strlist_count(ctx.conda.pip_packages_purge)) { - msg(STASIS_MSG_L2, "Purging pip packages from %s\n", name); - if (delivery_purge_packages(&ctx, env_name_testing, PKG_USE_PIP)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "unable to purge requested pip packages from %s\n", - env_name_testing); - exit(1); +static void configure_conda_purge(struct Delivery *ctx, char *envs[]) { + struct StrList *purge_list[] = { + ctx->conda.conda_packages_purge, + ctx->conda.pip_packages_purge + }; + for (size_t i = 0; i < sizeof(purge_list) / sizeof(purge_list[0]); i++) { + struct StrList *to_purge = purge_list[i]; + for (size_t e = 0; envs[e] != NULL; e += 2) { + //const char *title = envs[e]; // unused + const char *env = envs[e+1]; + if (to_purge && strlist_count(to_purge)) { + const char *pkg_manager_name[] = { + "conda", + "pip" + }; + const int pkg_manager_use[] = { + PKG_USE_CONDA, + PKG_USE_PIP + }; + const char *manager_str = pkg_manager_name[i]; + const int manager_flag = pkg_manager_use[i]; + msg(STASIS_MSG_L2, "Purging %s packages from %s\n", manager_str, env); + if (delivery_purge_packages(ctx, env, manager_flag)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "unable to purge requested %s packages from %s\n", manager_str, env); + exit(1); + } } } } +} +static void setup_activate_test_env(const struct Delivery *ctx, const char *env_name_testing) { // Activate test environment msg(STASIS_MSG_L1, "Activating test environment\n"); - if (conda_activate(ctx.storage.conda_install_prefix, env_name_testing)) { + if (conda_activate(ctx->storage.conda_install_prefix, env_name_testing)) { fprintf(stderr, "failed to activate test environment\n"); exit(1); } +} - if (delivery_gather_tool_versions(&ctx)) { - if (!ctx.conda.tool_version) { +static void configure_tool_versions(struct Delivery *ctx) { + if (delivery_gather_tool_versions(ctx)) { + if (!ctx->conda.tool_version) { msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Could not determine conda version\n"); exit(1); } - if (!ctx.conda.tool_build_version) { + if (!ctx->conda.tool_build_version) { msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Could not determine conda-build version\n"); exit(1); } } +} +static void install_build_package() { if (pip_exec("install build")) { msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "'build' tool installation failed\n"); exit(1); } +} - if (!isempty(ctx.meta.based_on)) { +static void configure_package_overlay(struct Delivery *ctx, const char *env_name) { + if (!isempty(ctx->meta.based_on)) { msg(STASIS_MSG_L1, "Generating package overlay from environment: %s\n", env_name); - if (delivery_overlay_packages_from_env(&ctx, env_name)) { + if (delivery_overlay_packages_from_env(ctx, env_name)) { msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "%s", "Failed to generate package overlay. Resulting environment integrity cannot be guaranteed.\n"); exit(1); } } +} +static void configure_deferred_packages(struct Delivery *ctx) { msg(STASIS_MSG_L1, "Filter deliverable packages\n"); - delivery_defer_packages(&ctx, DEFER_CONDA); - delivery_defer_packages(&ctx, DEFER_PIP); + delivery_defer_packages(ctx, DEFER_CONDA); + delivery_defer_packages(ctx, DEFER_PIP); +} + +static void show_overiew(struct Delivery *ctx) { msg(STASIS_MSG_L1, "Overview\n"); - delivery_meta_show(&ctx); - delivery_conda_show(&ctx); + delivery_meta_show(ctx); + delivery_conda_show(ctx); if (globals.verbose) { //delivery_runtime_show(&ctx); } +} +static void run_tests(struct Delivery *ctx) { // Execute configuration-defined tests if (globals.enable_testing) { - delivery_tests_show(&ctx); + delivery_tests_show(ctx); msg(STASIS_MSG_L1, "Begin test execution\n"); - delivery_tests_run(&ctx); + delivery_tests_run(ctx); msg(STASIS_MSG_L2, "Rewriting test results\n"); - delivery_fixup_test_results(&ctx); + delivery_fixup_test_results(ctx); } else { msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Test execution is disabled\n"); } +} - if (ctx.conda.conda_packages_defer && strlist_count(ctx.conda.conda_packages_defer)) { +static void build_conda_recipes(struct Delivery *ctx) { + if (ctx->conda.conda_packages_defer && strlist_count(ctx->conda.conda_packages_defer)) { msg(STASIS_MSG_L2, "Building Conda recipe(s)\n"); - if (delivery_build_recipes(&ctx)) { + if (delivery_build_recipes(ctx)) { exit(1); } msg(STASIS_MSG_L3, "Copying artifacts\n"); - if (delivery_copy_conda_artifacts(&ctx)) { + if (delivery_copy_conda_artifacts(ctx)) { exit(1); } msg(STASIS_MSG_L3, "Indexing artifacts\n"); - if (delivery_index_conda_artifacts(&ctx)) { + if (delivery_index_conda_artifacts(ctx)) { exit(1); } } +} - if (strlist_count(ctx.conda.pip_packages_defer)) { +static void build_wheel_packages(struct Delivery *ctx) { + if (strlist_count(ctx->conda.pip_packages_defer)) { msg(STASIS_MSG_L2, "Building Python wheels(s)\n"); - if (!((ctx.conda.wheels_packages = delivery_build_wheels(&ctx)))) { + if (!((ctx->conda.wheels_packages = delivery_build_wheels(ctx)))) { exit(1); } - if (delivery_index_wheel_artifacts(&ctx)) { + if (delivery_index_wheel_artifacts(ctx)) { exit(1); } } +} - // Populate the release environment - msg(STASIS_MSG_L1, "Populating release environment\n"); +static void release_install_conda_packages(struct Delivery *ctx, char *env_name) { msg(STASIS_MSG_L2, "Installing conda packages\n"); - if (strlist_count(ctx.conda.conda_packages)) { - if (delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_CONDA, (struct StrList *[]) {ctx.conda.conda_packages, NULL})) { + if (strlist_count(ctx->conda.conda_packages)) { + if (delivery_install_packages(ctx, ctx->storage.conda_install_prefix, env_name, INSTALL_PKG_CONDA, (struct StrList *[]) {ctx->conda.conda_packages, NULL})) { exit(1); } } - if (strlist_count(ctx.conda.conda_packages_defer)) { + if (strlist_count(ctx->conda.conda_packages_defer)) { msg(STASIS_MSG_L3, "Installing deferred conda packages\n"); - if (delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_CONDA | INSTALL_PKG_CONDA_DEFERRED, (struct StrList *[]) {ctx.conda.conda_packages_defer, NULL})) { + if (delivery_install_packages(ctx, ctx->storage.conda_install_prefix, env_name, INSTALL_PKG_CONDA | INSTALL_PKG_CONDA_DEFERRED, (struct StrList *[]) {ctx->conda.conda_packages_defer, NULL})) { exit(1); } } else { msg(STASIS_MSG_L3, "No deferred conda packages\n"); } +} +static void release_install_pip_packages(struct Delivery *ctx, char *env_name) { msg(STASIS_MSG_L2, "Installing pip packages\n"); - if (strlist_count(ctx.conda.pip_packages)) { - if (delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_PIP, (struct StrList *[]) {ctx.conda.pip_packages, NULL})) { + if (strlist_count(ctx->conda.pip_packages)) { + if (delivery_install_packages(ctx, ctx->storage.conda_install_prefix, env_name, INSTALL_PKG_PIP, (struct StrList *[]) {ctx->conda.pip_packages, NULL})) { exit(1); } } - if (strlist_count(ctx.conda.pip_packages_defer)) { + if (strlist_count(ctx->conda.pip_packages_defer)) { msg(STASIS_MSG_L3, "Installing deferred pip packages\n"); - if (delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_PIP | INSTALL_PKG_PIP_DEFERRED, (struct StrList *[]) {ctx.conda.pip_packages_defer, NULL})) { + if (delivery_install_packages(ctx, ctx->storage.conda_install_prefix, env_name, INSTALL_PKG_PIP | INSTALL_PKG_PIP_DEFERRED, (struct StrList *[]) {ctx->conda.pip_packages_defer, NULL})) { exit(1); } } else { msg(STASIS_MSG_L3, "No deferred pip packages\n"); } +} - conda_exec("list"); - - msg(STASIS_MSG_L1, "Creating release\n"); - msg(STASIS_MSG_L2, "Exporting delivery configuration\n"); - if (!pushd(ctx.storage.cfgdump_dir)) { - char filename[PATH_MAX] = {0}; - sprintf(filename, "%s.ini", ctx.info.release_name); - FILE *spec = fopen(filename, "w+"); - if (!spec) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", filename); - exit(1); - } - ini_write(ctx._stasis_ini_fp.delivery, &spec, INI_WRITE_RAW); - fclose(spec); - - memset(filename, 0, sizeof(filename)); - sprintf(filename, "%s-rendered.ini", ctx.info.release_name); - spec = fopen(filename, "w+"); - if (!spec) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", filename); - exit(1); - } - ini_write(ctx._stasis_ini_fp.delivery, &spec, INI_WRITE_PRESERVE); - fclose(spec); - popd(); - } else { - SYSERROR("Failed to enter directory: %s", ctx.storage.delivery_dir); - exit(1); - } - - msg(STASIS_MSG_L2, "Exporting %s\n", env_name_testing); - if (conda_env_export(env_name_testing, ctx.storage.delivery_dir, env_name_testing)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", env_name_testing); - exit(1); - } - - msg(STASIS_MSG_L2, "Exporting %s\n", env_name); - if (conda_env_export(env_name, ctx.storage.delivery_dir, env_name)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", env_name); - exit(1); - } - - // Rewrite release environment output (i.e. set package origin(s) to point to the deployment server, etc.) - char specfile[PATH_MAX]; - sprintf(specfile, "%s/%s.yml", ctx.storage.delivery_dir, env_name); - msg(STASIS_MSG_L3, "Rewriting release spec file (stage 1): %s\n", path_basename(specfile)); - delivery_rewrite_spec(&ctx, specfile, DELIVERY_REWRITE_SPEC_STAGE_1); - - msg(STASIS_MSG_L1, "Rendering mission templates\n"); - delivery_mission_render_files(&ctx); - - int want_docker = ini_section_search(&ctx._stasis_ini_fp.delivery, INI_SEARCH_BEGINS, "deploy:docker") ? true : false; - int want_artifactory = ini_section_search(&ctx._stasis_ini_fp.delivery, INI_SEARCH_BEGINS, "deploy:artifactory") ? true : false; +static void build_docker(struct Delivery *ctx, int disabled) { + int want_docker = ini_section_search(&ctx->_stasis_ini_fp.delivery, INI_SEARCH_BEGINS, "deploy:docker") ? true : false; if (want_docker) { - if (user_disabled_docker) { + if (disabled) { msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Docker image building is disabled by CLI argument\n"); } else { char dockerfile[PATH_MAX] = {0}; - sprintf(dockerfile, "%s/%s", ctx.storage.build_docker_dir, "Dockerfile"); + sprintf(dockerfile, "%s/%s", ctx->storage.build_docker_dir, "Dockerfile"); if (globals.enable_docker) { if (!access(dockerfile, F_OK)) { msg(STASIS_MSG_L1, "Building Docker image\n"); - if (delivery_docker(&ctx)) { + if (delivery_docker(ctx)) { msg(STASIS_MSG_L1 | STASIS_MSG_ERROR, "Failed to build docker image!\n"); COE_CHECK_ABORT(1, "Failed to build docker image"); } } else { - msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Docker image building is disabled. No Dockerfile found in %s\n", ctx.storage.build_docker_dir); + msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Docker image building is disabled. No Dockerfile found in %s\n", ctx->storage.build_docker_dir); } } else { msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Docker image building is disabled. System configuration error\n"); @@ -574,25 +437,219 @@ int main(int argc, char *argv[]) { } else { msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Docker image building is disabled. deploy:docker is not configured\n"); } +} + +static void generate_release(struct Delivery *ctx, char *env_name, char *env_name_testing, const int disable_docker) { + // Populate the release environment + msg(STASIS_MSG_L1, "Populating release environment\n"); + release_install_conda_packages(ctx, env_name); + release_install_pip_packages(ctx, env_name); + + conda_exec("list"); + + msg(STASIS_MSG_L1, "Creating release\n"); + delivery_export(ctx, (char *[]) {env_name, env_name_testing, NULL}); + + char specfile[PATH_MAX]; + sprintf(specfile, "%s/%s.yml", ctx->storage.delivery_dir, env_name); - msg(STASIS_MSG_L3, "Rewriting release spec file (stage 2): %s\n", path_basename(specfile)); - delivery_rewrite_spec(&ctx, specfile, DELIVERY_REWRITE_SPEC_STAGE_2); + delivery_rewrite_stage1(ctx, specfile); + build_docker(ctx, disable_docker); + delivery_rewrite_stage2(ctx, specfile); msg(STASIS_MSG_L1, "Dumping metadata\n"); - if (delivery_dump_metadata(&ctx)) { + if (delivery_dump_metadata(ctx)) { msg(STASIS_MSG_L1 | STASIS_MSG_ERROR, "Metadata dump failed\n"); } +} +static void transfer_artifacts(struct Delivery *ctx) { + int want_artifactory = ini_section_search(&ctx->_stasis_ini_fp.delivery, INI_SEARCH_BEGINS, "deploy:artifactory") ? true : false; if (want_artifactory) { if (globals.enable_artifactory && globals.enable_artifactory_upload) { msg(STASIS_MSG_L1, "Uploading artifacts\n"); - delivery_artifact_upload(&ctx); + delivery_artifact_upload(ctx); } else { msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Artifactory upload is disabled by CLI argument\n"); } } else { msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Artifactory upload is disabled. deploy:artifactory is not configured\n"); } +} + +int main(int argc, char *argv[]) { + struct Delivery ctx; + struct Process proc = { + .f_stdout = "", + .f_stderr = "", + .redirect_stderr = 0, + }; + char env_name[STASIS_NAME_MAX] = {0}; + char env_name_testing[STASIS_NAME_MAX] = {0}; + char *delivery_input = NULL; + char *config_input = NULL; + char installer_url[PATH_MAX]; + char python_override_version[STASIS_NAME_MAX]; + int user_disabled_docker = false; + globals.cpu_limit = get_cpu_count(); + if (globals.cpu_limit > 1) { + globals.cpu_limit--; // max - 1 + } + + memset(env_name, 0, sizeof(env_name)); + memset(env_name_testing, 0, sizeof(env_name_testing)); + memset(installer_url, 0, sizeof(installer_url)); + memset(python_override_version, 0, sizeof(python_override_version)); + memset(&proc, 0, sizeof(proc)); + memset(&ctx, 0, sizeof(ctx)); + + int c; + int option_index = 0; + while ((c = getopt_long(argc, argv, "hVCc:p:vU", long_options, &option_index)) != -1) { + switch (c) { + case 'h': + usage(path_basename(argv[0])); + exit(0); + case 'V': + puts(VERSION); + exit(0); + case 'c': + config_input = strdup(optarg); + break; + case 'C': + globals.continue_on_error = true; + break; + case 'p': + strcpy(python_override_version, optarg); + break; + case 'l': + globals.cpu_limit = strtol(optarg, NULL, 10); + if (globals.cpu_limit <= 1) { + globals.cpu_limit = 1; + globals.enable_parallel = false; // No point + } + break; + case OPT_ALWAYS_UPDATE_BASE: + globals.always_update_base_environment = true; + break; + case OPT_FAIL_FAST: + globals.parallel_fail_fast = true; + break; + case OPT_POOL_STATUS_INTERVAL: + globals.pool_status_interval = (int) strtol(optarg, NULL, 10); + if (globals.pool_status_interval < 1) { + globals.pool_status_interval = 1; + } else if (globals.pool_status_interval > 60 * 10) { + // Possible poor choice alert + fprintf(stderr, "Caution: Excessive pausing between status updates may cause third-party CI/CD" + " jobs to fail if the stdout/stderr streams are idle for too long!\n"); + } + break; + case 'U': + setenv("PYTHONUNBUFFERED", "1", 1); + fflush(stdout); + fflush(stderr); + setvbuf(stdout, NULL, _IONBF, 0); + setvbuf(stderr, NULL, _IONBF, 0); + break; + case 'v': + globals.verbose = true; + break; + case OPT_OVERWRITE: + globals.enable_overwrite = true; + break; + case OPT_NO_DOCKER: + globals.enable_docker = false; + user_disabled_docker = true; + break; + case OPT_NO_ARTIFACTORY: + globals.enable_artifactory = false; + break; + case OPT_NO_ARTIFACTORY_BUILD_INFO: + globals.enable_artifactory_build_info = false; + break; + case OPT_NO_ARTIFACTORY_UPLOAD: + globals.enable_artifactory_build_info = false; + globals.enable_artifactory_upload = false; + break; + case OPT_NO_TESTING: + globals.enable_testing = false; + break; + case OPT_NO_REWRITE_SPEC_STAGE_2: + globals.enable_rewrite_spec_stage_2 = false; + break; + case OPT_NO_PARALLEL: + globals.enable_parallel = false; + break; + case '?': + default: + exit(1); + } + } + + if (optind < argc) { + while (optind < argc) { + // use first positional argument + delivery_input = argv[optind++]; + break; + } + } + + if (!delivery_input) { + fprintf(stderr, "error: a DELIVERY_FILE is required\n"); + usage(path_basename(argv[0])); + exit(1); + } + + printf(BANNER, VERSION, AUTHOR); + + check_system_path(); + + msg(STASIS_MSG_L1, "Setup\n"); + + tpl_setup_vars(&ctx); + tpl_setup_funcs(&ctx); + + setup_sysconfdir(); + setup_python_version_override(&ctx, python_override_version); + + configure_stasis_ini(&ctx, &config_input); + configure_delivery_ini(&ctx, &delivery_input); + configure_delivery_context(&ctx); + + check_requirements(&ctx); + configure_jfrog_cli(&ctx); + + runtime_apply(ctx.runtime.environ); + strcpy(env_name, ctx.info.release_name); + strcpy(env_name_testing, env_name); + strcat(env_name_testing, "-test"); + char *envs[] = { + "release", env_name, + "testing", env_name_testing + }; + + check_release_history(&ctx); + sync_release_history(&ctx); + + check_conda_install_prefix(&ctx); + check_conda_prefix_length(&ctx); + setup_conda(&ctx, installer_url); + configure_conda_base(&ctx, envs); + configure_conda_purge(&ctx, envs); + setup_activate_test_env(&ctx, env_name_testing); + + configure_tool_versions(&ctx); + install_build_package(); + configure_package_overlay(&ctx, env_name); + configure_deferred_packages(&ctx); + + show_overiew(&ctx); + run_tests(&ctx); + build_conda_recipes(&ctx); + build_wheel_packages(&ctx); + generate_release(&ctx, env_name, env_name_testing, user_disabled_docker); + transfer_artifacts(&ctx); msg(STASIS_MSG_L1, "Cleaning up\n"); delivery_free(&ctx); -- cgit From dec6933405df0155a60c376e5345f71250b649b0 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Mon, 7 Jul 2025 17:31:55 -0400 Subject: Convert sprintf to snprintf --- src/lib/core/environment.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/lib/core/environment.c b/src/lib/core/environment.c index f5e8566..407c76c 100644 --- a/src/lib/core/environment.c +++ b/src/lib/core/environment.c @@ -106,14 +106,13 @@ void runtime_export(RuntimeEnv *env, char **keys) { if (keys != NULL) { for (size_t j = 0; keys[j] != NULL; j++) { if (strcmp(keys[j], key) == 0) { - //sprintf(output, "%s=\"%s\"\n%s %s", key, value ? value : "", export_command, key); - sprintf(output, "%s %s=\"%s\"", export_command, key, value ? value : ""); + snprintf(output, sizeof(output), "%s %s=\"%s\"", export_command, key, value ? value : ""); puts(output); } } } else { - sprintf(output, "%s %s=\"%s\"", export_command, key, value ? value : ""); + snprintf(output, sizeof(output), "%s %s=\"%s\"", export_command, key, value ? value : ""); puts(output); } guard_free(value); -- cgit From c3907905403c783aca3a02ca38cde6dc9f1a7cb7 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Mon, 7 Jul 2025 17:32:06 -0400 Subject: Grammar --- src/lib/core/environment.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/core/environment.c b/src/lib/core/environment.c index 407c76c..cb5a7a3 100644 --- a/src/lib/core/environment.c +++ b/src/lib/core/environment.c @@ -177,7 +177,7 @@ int runtime_replace(RuntimeEnv **dest, char **src) { } /** - * Determine whether or not a key exists in the runtime environment + * Determine whether a key exists in the runtime environment * * Example: * -- cgit From 6447853b3cf1cdf327758d847a09733e45e28dad Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Mon, 7 Jul 2025 17:38:16 -0400 Subject: Memory safety --- src/lib/core/environment.c | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/src/lib/core/environment.c b/src/lib/core/environment.c index cb5a7a3..bd3ac1b 100644 --- a/src/lib/core/environment.c +++ b/src/lib/core/environment.c @@ -244,7 +244,14 @@ char *runtime_get(RuntimeEnv *env, const char *key) { ssize_t key_offset = runtime_contains(env, key); if (key_offset != -1) { char **pair = split(strlist_item(env, key_offset), "=", 0); + if (!pair) { + return NULL; + } result = join(&pair[1], "="); + if (!result) { + guard_array_free(pair); + return NULL; + } guard_array_free(pair); } return result; @@ -347,9 +354,7 @@ char *runtime_expand_var(RuntimeEnv *env, char *input) { } // Append expanded environment variable to output strncat(expanded, tmp, STASIS_BUFSIZ - 1); - if (env) { - guard_free(tmp); - } + guard_free(tmp); } // Nothing to do so append input to output @@ -403,9 +408,22 @@ void runtime_set(RuntimeEnv *env, const char *_key, char *_value) { return; } char *key = strdup(_key); - ssize_t key_offset = runtime_contains(env, key); + if (!key) { + SYSERROR("%s", "unable to allocate memory for key"); + exit(1); + } char *value = runtime_expand_var(env, _value); - char *now = join((char *[]) {key, value, NULL}, "="); + if (!value) { + SYSERROR("%s", "unable to allocate memory for value"); + exit(1); + } + + lstrip(value); + char *now = join((char *[]) {key, value, NULL}, sep); + if (!now) { + SYSERROR("%s", "unable to allocate memory for join"); + exit(1); + } if (key_offset < 0) { strlist_append(&env, now); @@ -423,6 +441,10 @@ void runtime_set(RuntimeEnv *env, const char *_key, char *_value) { void runtime_apply(RuntimeEnv *env) { for (size_t i = 0; i < strlist_count(env); i++) { char **pair = split(strlist_item(env, i), "=", 1); + if (!pair) { + SYSERROR("%s", "unable to allocate memory for runtime_apply"); + return; + } setenv(pair[0], pair[1], 1); guard_array_free(pair); } -- cgit From de3e615c1971d844b8ab0c44332fa18466dca150 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Mon, 7 Jul 2025 17:38:57 -0400 Subject: Separator as variable --- src/lib/core/environment.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/core/environment.c b/src/lib/core/environment.c index bd3ac1b..057d89e 100644 --- a/src/lib/core/environment.c +++ b/src/lib/core/environment.c @@ -404,6 +404,7 @@ char *runtime_expand_var(RuntimeEnv *env, char *input) { * @param _value New environment variable value */ void runtime_set(RuntimeEnv *env, const char *_key, char *_value) { + const char *sep = "="; if (_key == NULL) { return; } -- cgit From fada0caa442c433a133735c79fd697f52696b705 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Mon, 7 Jul 2025 17:39:17 -0400 Subject: Return a copy of the input, not the input pointer --- src/lib/core/environment.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/lib/core/environment.c b/src/lib/core/environment.c index 057d89e..79dd559 100644 --- a/src/lib/core/environment.c +++ b/src/lib/core/environment.c @@ -291,8 +291,7 @@ char *runtime_expand_var(RuntimeEnv *env, char *input) { // If there's no environment variables to process return the input string if (strchr(input, delim) == NULL) { - //return strdup(input); - return input; + return strdup(input); } expanded = calloc(STASIS_BUFSIZ, sizeof(char)); -- cgit From dc6bbddab79bab20c5c2d2994cf6eae66bcd767e Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Mon, 7 Jul 2025 17:39:55 -0400 Subject: Use a copy of the environment variable if its present --- src/lib/core/environment.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/lib/core/environment.c b/src/lib/core/environment.c index 79dd559..02bbf82 100644 --- a/src/lib/core/environment.c +++ b/src/lib/core/environment.c @@ -341,7 +341,10 @@ char *runtime_expand_var(RuntimeEnv *env, char *input) { if (env) { tmp = runtime_get(env, var); } else { - tmp = getenv(var); + const char *v = getenv(var); + if (v) { + tmp = strdup(v); + } } if (tmp == NULL) { // This mimics shell behavior in general. -- cgit From b02d868a4f74194f910f8ce173cb5586bc62a31a Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Mon, 7 Jul 2025 17:40:28 -0400 Subject: Offset is constant --- src/lib/core/environment.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/core/environment.c b/src/lib/core/environment.c index 02bbf82..39d8116 100644 --- a/src/lib/core/environment.c +++ b/src/lib/core/environment.c @@ -410,6 +410,7 @@ void runtime_set(RuntimeEnv *env, const char *_key, char *_value) { if (_key == NULL) { return; } + const ssize_t key_offset = runtime_contains(env, _key); char *key = strdup(_key); if (!key) { SYSERROR("%s", "unable to allocate memory for key"); -- cgit From d0815e2b02b4f110fa140dd124cc0f5f236be05a Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Mon, 7 Jul 2025 17:40:51 -0400 Subject: Test for ini_getall() --- tests/test_ini.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_ini.c b/tests/test_ini.c index e4a7808..af47ddf 100644 --- a/tests/test_ini.c +++ b/tests/test_ini.c @@ -195,6 +195,23 @@ void test_ini_getval_wrappers() { ini_free(&ini); } +void test_ini_getall() { + const char *filename = "ini_open.ini"; + struct INIFILE *ini = NULL; + const char *data = "[default]\nhello=world!\nthis=is a test\nx=1\ny=0\n"; + + stasis_testing_write_ascii(filename, data); + ini = ini_open(filename); + STASIS_ASSERT_FATAL(ini != NULL, "failed to open ini file"); + + const struct INIData *d = NULL; + for (size_t i = 0; (d = ini_getall(ini, "default")) != NULL; i++) { + STASIS_ASSERT(d->key != NULL, "INIData key should not be NULL"); + STASIS_ASSERT(d->value != NULL, "INIData key should not be NULL"); + } + ini_free(&ini); +} + int main(int argc, char *argv[]) { STASIS_TEST_BEGIN_MAIN(); STASIS_TEST_FUNC *tests[] = { @@ -204,6 +221,7 @@ int main(int argc, char *argv[]) { test_ini_has_key, test_ini_setval_getval, test_ini_getval_wrappers, + test_ini_getall, }; STASIS_TEST_RUN(tests); STASIS_TEST_END_MAIN(); -- cgit From 7f8e8c82da33e64bd0a1c4376ea7bc184bded3ec Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Mon, 7 Jul 2025 19:06:27 -0400 Subject: Fix leak --- src/lib/core/environment.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/core/environment.c b/src/lib/core/environment.c index 39d8116..7ece5e6 100644 --- a/src/lib/core/environment.c +++ b/src/lib/core/environment.c @@ -436,6 +436,7 @@ void runtime_set(RuntimeEnv *env, const char *_key, char *_value) { } guard_free(now); guard_free(key); + guard_free(value); } /** -- cgit From 5aa0f7ffaf1352f39fbc42ed7bb38ecb17ed36a6 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Mon, 7 Jul 2025 19:06:55 -0400 Subject: Make want_(artifactory|docker) constant --- src/cli/stasis/stasis_main.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cli/stasis/stasis_main.c b/src/cli/stasis/stasis_main.c index d1c48e4..cfd6ba6 100644 --- a/src/cli/stasis/stasis_main.c +++ b/src/cli/stasis/stasis_main.c @@ -411,8 +411,8 @@ static void release_install_pip_packages(struct Delivery *ctx, char *env_name) { } } -static void build_docker(struct Delivery *ctx, int disabled) { - int want_docker = ini_section_search(&ctx->_stasis_ini_fp.delivery, INI_SEARCH_BEGINS, "deploy:docker") ? true : false; +static void build_docker(struct Delivery *ctx, const int disabled) { + const int want_docker = ini_section_search(&ctx->_stasis_ini_fp.delivery, INI_SEARCH_BEGINS, "deploy:docker") ? true : false; if (want_docker) { if (disabled) { @@ -464,7 +464,7 @@ static void generate_release(struct Delivery *ctx, char *env_name, char *env_nam } static void transfer_artifacts(struct Delivery *ctx) { - int want_artifactory = ini_section_search(&ctx->_stasis_ini_fp.delivery, INI_SEARCH_BEGINS, "deploy:artifactory") ? true : false; + const int want_artifactory = ini_section_search(&ctx->_stasis_ini_fp.delivery, INI_SEARCH_BEGINS, "deploy:artifactory") ? true : false; if (want_artifactory) { if (globals.enable_artifactory && globals.enable_artifactory_upload) { msg(STASIS_MSG_L1, "Uploading artifacts\n"); -- cgit From 1209bdcc6e6131d12fc0923d771e14d9d65635e2 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Mon, 7 Jul 2025 19:08:35 -0400 Subject: Fix leak in artifactory test --- tests/test_artifactory.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_artifactory.c b/tests/test_artifactory.c index 4af7eec..202a67c 100644 --- a/tests/test_artifactory.c +++ b/tests/test_artifactory.c @@ -116,6 +116,7 @@ int main(int argc, char *argv[]) { // Skip this suite if we're not configured to use it if (jfrt_auth_init(&gauth)) { SYSERROR("%s", "Not configured to test Artifactory. Skipping."); + guard_free(basedir); return STASIS_TEST_SUITE_SKIP; } guard_free(basedir); -- cgit From 76f50b85171ef007672a112da6faeeb4bb35da03 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 8 Jul 2025 13:54:26 -0400 Subject: Disable noisy debug statements --- src/lib/core/template.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/core/template.c b/src/lib/core/template.c index 68d20c9..dd3c7a2 100644 --- a/src/lib/core/template.c +++ b/src/lib/core/template.c @@ -280,8 +280,8 @@ char *tpl_render(char *str) { output[z] = pos[off]; z++; } - SYSDEBUG("template output length: %zu", strlen(output)); - SYSDEBUG("template output bytes: %zu", output_bytes); + //SYSDEBUG("template output length: %zu", strlen(output)); + //SYSDEBUG("template output bytes: %zu", output_bytes); return output; } -- cgit From f88373107794450f053220c2862415f00119b98c Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 8 Jul 2025 13:55:10 -0400 Subject: Set last records to NULL --- src/cli/stasis/stasis_main.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cli/stasis/stasis_main.c b/src/cli/stasis/stasis_main.c index cfd6ba6..d133f7e 100644 --- a/src/cli/stasis/stasis_main.c +++ b/src/cli/stasis/stasis_main.c @@ -626,7 +626,8 @@ int main(int argc, char *argv[]) { strcat(env_name_testing, "-test"); char *envs[] = { "release", env_name, - "testing", env_name_testing + "testing", env_name_testing, + NULL, NULL, }; check_release_history(&ctx); -- cgit From 5692c689a7b1da67724fc5c3c9bb3f3f1d7df0c9 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 8 Jul 2025 13:55:52 -0400 Subject: Enable debug messages in RT --- tests/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/setup.sh b/tests/setup.sh index 7e38cf9..41d8d49 100644 --- a/tests/setup.sh +++ b/tests/setup.sh @@ -78,7 +78,7 @@ teardown_workspace() { install_stasis() { pushd "$BUILD_DIR" - if ! cmake -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -DCMAKE_BUILD_TYPE=Debug "${TOPDIR}"/../..; then + if ! cmake -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -DCMAKE_BUILD_TYPE=Debug -DDEBUG_MESSAGES=ON "${TOPDIR}"/../..; then echo "cmake failed" >&2 return 1 fi -- cgit From 83f9592c4bc0f8e5a6a8e73ba9e3daa1da107262 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 8 Jul 2025 13:57:10 -0400 Subject: Write error message correctly to the buffer * If errmsg is NULL, allocate and write string * If errmsg is not NULL, write string --- src/lib/core/download.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/core/download.c b/src/lib/core/download.c index c3f8dca..b021860 100644 --- a/src/lib/core/download.c +++ b/src/lib/core/download.c @@ -41,10 +41,10 @@ long download(char *url, const char *filename, char **errmsg) { CURLcode curl_code = curl_easy_perform(c); SYSDEBUG("curl status code: %d", curl_code); if (curl_code != CURLE_OK) { - if (errmsg) { - strcpy(*errmsg, curl_easy_strerror(curl_code)); + if (!*errmsg) { + *errmsg = strdup(curl_easy_strerror(curl_code)); } else { - fprintf(stderr, "\nCURL ERROR: %s\n", curl_easy_strerror(curl_code)); + strncpy(*errmsg, curl_easy_strerror(curl_code), strlen(curl_easy_strerror(curl_code) + 1)); } goto failed; } -- cgit From fb7e221f89fdad477e6425d2673bdd959fb69d17 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 8 Jul 2025 13:57:57 -0400 Subject: Debug messages --- src/lib/delivery/delivery_init.c | 8 ++++++++ src/lib/delivery/delivery_populate.c | 1 + 2 files changed, 9 insertions(+) diff --git a/src/lib/delivery/delivery_init.c b/src/lib/delivery/delivery_init.c index 56c591a..a60d6af 100644 --- a/src/lib/delivery/delivery_init.c +++ b/src/lib/delivery/delivery_init.c @@ -287,18 +287,25 @@ int delivery_init(struct Delivery *ctx, int render_mode) { int bootstrap_build_info(struct Delivery *ctx) { struct Delivery local = {0}; + SYSDEBUG("ini_open(%s)", ctx->_stasis_ini_fp.cfg_path); local._stasis_ini_fp.cfg = ini_open(ctx->_stasis_ini_fp.cfg_path); + SYSDEBUG("ini_open(%s)", ctx->_stasis_ini_fp.delivery_path); local._stasis_ini_fp.delivery = ini_open(ctx->_stasis_ini_fp.delivery_path); + if (delivery_init_platform(&local)) { + SYSDEBUG("%s", "delivery_init_platform failed"); return -1; } if (populate_delivery_cfg(&local, INI_READ_RENDER)) { + SYSDEBUG("%s", "populate_delivery_cfg failed"); return -1; } if (populate_delivery_ini(&local, INI_READ_RENDER)) { + SYSDEBUG("%s", "populate_delivery_ini failed"); return -1; } if (populate_info(&local)) { + SYSDEBUG("%s", "populate_info failed"); return -1; } ctx->info.build_name = strdup(local.info.build_name); @@ -314,6 +321,7 @@ int bootstrap_build_info(struct Delivery *ctx) { memcpy(ctx->info.time_info, local.info.time_info, sizeof(*local.info.time_info)); ctx->info.time_now = local.info.time_now; ctx->info.time_str_epoch = strdup(local.info.time_str_epoch); + SYSDEBUG("%s", "delivery_free local resources"); delivery_free(&local); return 0; } diff --git a/src/lib/delivery/delivery_populate.c b/src/lib/delivery/delivery_populate.c index 53d44e3..28b2480 100644 --- a/src/lib/delivery/delivery_populate.c +++ b/src/lib/delivery/delivery_populate.c @@ -55,6 +55,7 @@ int populate_info(struct Delivery *ctx) { int populate_delivery_cfg(struct Delivery *ctx, int render_mode) { struct INIFILE *cfg = ctx->_stasis_ini_fp.cfg; if (!cfg) { + SYSDEBUG("%s", "cfg is NULL"); return -1; } int err = 0; -- cgit From 1fd91d5449004c8d02c58c05b9a303f520ee746e Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 8 Jul 2025 13:58:26 -0400 Subject: Bugfix: HTTP_ERROR did not check for negative values --- src/lib/core/include/core.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/core/include/core.h b/src/lib/core/include/core.h index 35a9506..92969d2 100644 --- a/src/lib/core/include/core.h +++ b/src/lib/core/include/core.h @@ -15,7 +15,7 @@ #define STASIS_NAME_MAX 255 #define STASIS_DIRSTACK_MAX 1024 #define STASIS_TIME_STR_MAX 128 -#define HTTP_ERROR(X) X >= 400 +#define HTTP_ERROR(X) (X >= 400 || X < 0) #include "config.h" #include "core_mem.h" -- cgit From 783a11a8537bc4fc6f211649c33d7e22e54d2dc0 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 8 Jul 2025 13:58:43 -0400 Subject: Free errmsg after use --- src/lib/core/conda.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/core/conda.c b/src/lib/core/conda.c index 268b433..de6130f 100644 --- a/src/lib/core/conda.c +++ b/src/lib/core/conda.c @@ -28,6 +28,7 @@ int micromamba(const struct MicromambaInfo *info, char *command, ...) { const long http_code = download(url, installer_path, &errmsg); if (HTTP_ERROR(http_code)) { fprintf(stderr, "download failed: %ld: %s\n", http_code, errmsg); + guard_free(errmsg); return -1; } } -- cgit From 49ea0d34021047fd8c48f6f0f5cb83533d33001c Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 8 Jul 2025 14:27:15 -0400 Subject: Bugfix: Remove errant else block * Add debug messages * Add error handling to cfg_path assignment --- src/cli/stasis/stasis_main.c | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/cli/stasis/stasis_main.c b/src/cli/stasis/stasis_main.c index d133f7e..65bf5a8 100644 --- a/src/cli/stasis/stasis_main.c +++ b/src/cli/stasis/stasis_main.c @@ -43,24 +43,29 @@ static void setup_python_version_override(struct Delivery *ctx, const char *vers static void configure_stasis_ini(struct Delivery *ctx, char **config_input) { if (!*config_input) { - // no configuration passed by argument. use basic config. + SYSDEBUG("%s", "No configuration passed by argument. Using basic config."); char cfgfile[PATH_MAX * 2]; sprintf(cfgfile, "%s/%s", globals.sysconfdir, "stasis.ini"); + SYSDEBUG("cfgfile: %s", cfgfile); if (!access(cfgfile, F_OK | R_OK)) { *config_input = strdup(cfgfile); } else { msg(STASIS_MSG_WARN, "STASIS global configuration is not readable, or does not exist: %s", cfgfile); } - } else { - msg(STASIS_MSG_L2, "Reading STASIS global configuration: %s\n", *config_input); - ctx->_stasis_ini_fp.cfg = ini_open(*config_input); - if (!ctx->_stasis_ini_fp.cfg) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read config file: %s, %s\n", *config_input, strerror(errno)); - exit(1); - } - ctx->_stasis_ini_fp.cfg_path = strdup(*config_input); - guard_free(*config_input); } + + msg(STASIS_MSG_L2, "Reading STASIS global configuration: %s\n", *config_input); + ctx->_stasis_ini_fp.cfg = ini_open(*config_input); + if (!ctx->_stasis_ini_fp.cfg) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read config file: %s, %s\n", *config_input, strerror(errno)); + exit(1); + } + ctx->_stasis_ini_fp.cfg_path = strdup(*config_input); + if (!ctx->_stasis_ini_fp.cfg_path) { + SYSERROR("Failed to allocate memory for config file name"); + exit(1); + } + guard_free(*config_input); } static void configure_delivery_ini(struct Delivery *ctx, char **delivery_input) { -- cgit From 1e572f3a653ddd6385a0ea49cacfbf2f3a32ce87 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 8 Jul 2025 14:27:45 -0400 Subject: Increase lines_on_error from 100 to 1000 * Debug messages are chatty --- src/cli/stasis/stasis_main.c | 2 +- tests/setup.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cli/stasis/stasis_main.c b/src/cli/stasis/stasis_main.c index 65bf5a8..2ce6831 100644 --- a/src/cli/stasis/stasis_main.c +++ b/src/cli/stasis/stasis_main.c @@ -62,7 +62,7 @@ static void configure_stasis_ini(struct Delivery *ctx, char **config_input) { } ctx->_stasis_ini_fp.cfg_path = strdup(*config_input); if (!ctx->_stasis_ini_fp.cfg_path) { - SYSERROR("Failed to allocate memory for config file name"); + SYSERROR("%s", "Failed to allocate memory for config file name"); exit(1); } guard_free(*config_input); diff --git a/tests/setup.sh b/tests/setup.sh index 41d8d49..bce2fbd 100644 --- a/tests/setup.sh +++ b/tests/setup.sh @@ -109,7 +109,7 @@ STASIS_TEST_RESULT_SKIP=0 run_command() { local logfile="$(mktemp).log" local cmd="${@}" - local lines_on_error=100 + local lines_on_error=1000 /bin/echo "Testing: $cmd " $cmd &>"$logfile" -- cgit