aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--src/cli/stasis/stasis_main.c725
-rw-r--r--src/lib/core/conda.c106
-rw-r--r--src/lib/core/download.c6
-rw-r--r--src/lib/core/environment.c50
-rw-r--r--src/lib/core/include/conda.h2
-rw-r--r--src/lib/core/include/core.h2
-rw-r--r--src/lib/core/include/str.h27
-rw-r--r--src/lib/core/include/utils.h45
-rw-r--r--src/lib/core/str.c31
-rw-r--r--src/lib/core/template.c21
-rw-r--r--src/lib/core/utils.c143
-rw-r--r--src/lib/delivery/CMakeLists.txt1
-rw-r--r--src/lib/delivery/delivery.c13
-rw-r--r--src/lib/delivery/delivery_build.c12
-rw-r--r--src/lib/delivery/delivery_export.c58
-rw-r--r--src/lib/delivery/delivery_init.c8
-rw-r--r--src/lib/delivery/delivery_install.c79
-rw-r--r--src/lib/delivery/delivery_populate.c3
-rw-r--r--src/lib/delivery/delivery_test.c12
-rw-r--r--src/lib/delivery/include/delivery.h24
-rw-r--r--tests/setup.sh4
-rw-r--r--tests/test_artifactory.c1
-rw-r--r--tests/test_ini.c18
-rw-r--r--tests/test_str.c7
24 files changed, 966 insertions, 432 deletions
diff --git a/src/cli/stasis/stasis_main.c b/src/cli/stasis/stasis_main.c
index 459a6f6..2ce6831 100644
--- a/src/cli/stasis/stasis_main.c
+++ b/src/cli/stasis/stasis_main.c
@@ -10,140 +10,7 @@
#include "system_requirements.h"
#include "tpl.h"
-
-int main(int argc, char *argv[]) {
- struct Delivery ctx;
- struct Process proc = {
- .f_stdout = "",
- .f_stderr = "",
- .redirect_stderr = 0,
- };
- char env_name[STASIS_NAME_MAX] = {0};
- char env_name_testing[STASIS_NAME_MAX] = {0};
- char *delivery_input = NULL;
- char *config_input = NULL;
- char installer_url[PATH_MAX];
- char python_override_version[STASIS_NAME_MAX];
- int user_disabled_docker = false;
- globals.cpu_limit = get_cpu_count();
- if (globals.cpu_limit > 1) {
- globals.cpu_limit--; // max - 1
- }
-
- memset(env_name, 0, sizeof(env_name));
- memset(env_name_testing, 0, sizeof(env_name_testing));
- memset(installer_url, 0, sizeof(installer_url));
- memset(python_override_version, 0, sizeof(python_override_version));
- memset(&proc, 0, sizeof(proc));
- memset(&ctx, 0, sizeof(ctx));
-
- int c;
- int option_index = 0;
- while ((c = getopt_long(argc, argv, "hVCc:p:vU", long_options, &option_index)) != -1) {
- switch (c) {
- case 'h':
- usage(path_basename(argv[0]));
- exit(0);
- case 'V':
- puts(VERSION);
- exit(0);
- case 'c':
- config_input = strdup(optarg);
- break;
- case 'C':
- globals.continue_on_error = true;
- break;
- case 'p':
- strcpy(python_override_version, optarg);
- break;
- case 'l':
- globals.cpu_limit = strtol(optarg, NULL, 10);
- if (globals.cpu_limit <= 1) {
- globals.cpu_limit = 1;
- globals.enable_parallel = false; // No point
- }
- break;
- case OPT_ALWAYS_UPDATE_BASE:
- globals.always_update_base_environment = true;
- break;
- case OPT_FAIL_FAST:
- globals.parallel_fail_fast = true;
- break;
- case OPT_POOL_STATUS_INTERVAL:
- globals.pool_status_interval = (int) strtol(optarg, NULL, 10);
- if (globals.pool_status_interval < 1) {
- globals.pool_status_interval = 1;
- } else if (globals.pool_status_interval > 60 * 10) {
- // Possible poor choice alert
- fprintf(stderr, "Caution: Excessive pausing between status updates may cause third-party CI/CD"
- " jobs to fail if the stdout/stderr streams are idle for too long!\n");
- }
- break;
- case 'U':
- setenv("PYTHONUNBUFFERED", "1", 1);
- fflush(stdout);
- fflush(stderr);
- setvbuf(stdout, NULL, _IONBF, 0);
- setvbuf(stderr, NULL, _IONBF, 0);
- break;
- case 'v':
- globals.verbose = true;
- break;
- case OPT_OVERWRITE:
- globals.enable_overwrite = true;
- break;
- case OPT_NO_DOCKER:
- globals.enable_docker = false;
- user_disabled_docker = true;
- break;
- case OPT_NO_ARTIFACTORY:
- globals.enable_artifactory = false;
- break;
- case OPT_NO_ARTIFACTORY_BUILD_INFO:
- globals.enable_artifactory_build_info = false;
- break;
- case OPT_NO_ARTIFACTORY_UPLOAD:
- globals.enable_artifactory_build_info = false;
- globals.enable_artifactory_upload = false;
- break;
- case OPT_NO_TESTING:
- globals.enable_testing = false;
- break;
- case OPT_NO_REWRITE_SPEC_STAGE_2:
- globals.enable_rewrite_spec_stage_2 = false;
- break;
- case OPT_NO_PARALLEL:
- globals.enable_parallel = false;
- break;
- case '?':
- default:
- exit(1);
- }
- }
-
- if (optind < argc) {
- while (optind < argc) {
- // use first positional argument
- delivery_input = argv[optind++];
- break;
- }
- }
-
- if (!delivery_input) {
- fprintf(stderr, "error: a DELIVERY_FILE is required\n");
- usage(path_basename(argv[0]));
- exit(1);
- }
-
- printf(BANNER, VERSION, AUTHOR);
-
- check_system_path();
-
- msg(STASIS_MSG_L1, "Setup\n");
-
- tpl_setup_vars(&ctx);
- tpl_setup_funcs(&ctx);
-
+static void setup_sysconfdir() {
// Set up PREFIX/etc directory information
// The user may manipulate the base directory path with STASIS_SYSCONFDIR
// environment variable
@@ -159,83 +26,108 @@ int main(int argc, char *argv[]) {
msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Unable to resolve path to configuration directory: %s\n", stasis_sysconfdir_tmp);
exit(1);
}
+}
+static void setup_python_version_override(struct Delivery *ctx, const char *version) {
// Override Python version from command-line, if any
- if (strlen(python_override_version)) {
- guard_free(ctx.meta.python);
- ctx.meta.python = strdup(python_override_version);
- guard_free(ctx.meta.python_compact);
- ctx.meta.python_compact = to_short_version(ctx.meta.python);
+ if (strlen(version)) {
+ guard_free(ctx->meta.python);
+ ctx->meta.python = strdup(version);
+ if (!ctx->meta.python) {
+ SYSERROR("%s", "Unable to allocate bytes for python version override");
+ }
+ guard_free(ctx->meta.python_compact);
+ ctx->meta.python_compact = to_short_version(ctx->meta.python);
}
+}
- if (!config_input) {
- // no configuration passed by argument. use basic config.
+static void configure_stasis_ini(struct Delivery *ctx, char **config_input) {
+ if (!*config_input) {
+ SYSDEBUG("%s", "No configuration passed by argument. Using basic config.");
char cfgfile[PATH_MAX * 2];
sprintf(cfgfile, "%s/%s", globals.sysconfdir, "stasis.ini");
+ SYSDEBUG("cfgfile: %s", cfgfile);
if (!access(cfgfile, F_OK | R_OK)) {
- config_input = strdup(cfgfile);
+ *config_input = strdup(cfgfile);
} else {
msg(STASIS_MSG_WARN, "STASIS global configuration is not readable, or does not exist: %s", cfgfile);
}
}
- if (config_input) {
- msg(STASIS_MSG_L2, "Reading STASIS global configuration: %s\n", config_input);
- ctx._stasis_ini_fp.cfg = ini_open(config_input);
- if (!ctx._stasis_ini_fp.cfg) {
- msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read config file: %s, %s\n", delivery_input, strerror(errno));
- exit(1);
- }
- ctx._stasis_ini_fp.cfg_path = strdup(config_input);
- guard_free(config_input);
+ msg(STASIS_MSG_L2, "Reading STASIS global configuration: %s\n", *config_input);
+ ctx->_stasis_ini_fp.cfg = ini_open(*config_input);
+ if (!ctx->_stasis_ini_fp.cfg) {
+ msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read config file: %s, %s\n", *config_input, strerror(errno));
+ exit(1);
+ }
+ ctx->_stasis_ini_fp.cfg_path = strdup(*config_input);
+ if (!ctx->_stasis_ini_fp.cfg_path) {
+ SYSERROR("%s", "Failed to allocate memory for config file name");
+ exit(1);
}
+ guard_free(*config_input);
+}
- msg(STASIS_MSG_L2, "Reading STASIS delivery configuration: %s\n", delivery_input);
- ctx._stasis_ini_fp.delivery = ini_open(delivery_input);
- if (!ctx._stasis_ini_fp.delivery) {
- msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read delivery file: %s, %s\n", delivery_input, strerror(errno));
+static void configure_delivery_ini(struct Delivery *ctx, char **delivery_input) {
+ msg(STASIS_MSG_L2, "Reading STASIS delivery configuration: %s\n", *delivery_input);
+ ctx->_stasis_ini_fp.delivery = ini_open(*delivery_input);
+ if (!ctx->_stasis_ini_fp.delivery) {
+ msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read delivery file: %s, %s\n", *delivery_input, strerror(errno));
exit(1);
}
- ctx._stasis_ini_fp.delivery_path = strdup(delivery_input);
+ ctx->_stasis_ini_fp.delivery_path = strdup(*delivery_input);
+}
+static void configure_delivery_context(struct Delivery *ctx) {
msg(STASIS_MSG_L2, "Bootstrapping delivery context\n");
- if (bootstrap_build_info(&ctx)) {
+ if (bootstrap_build_info(ctx)) {
msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to bootstrap delivery context\n");
exit(1);
}
msg(STASIS_MSG_L2, "Initializing delivery context\n");
- if (delivery_init(&ctx, INI_READ_RENDER)) {
+ if (delivery_init(ctx, INI_READ_RENDER)) {
msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to initialize delivery context\n");
exit(1);
}
- check_requirements(&ctx);
+}
+static void configure_jfrog_cli(struct Delivery *ctx) {
msg(STASIS_MSG_L2, "Configuring JFrog CLI\n");
- if (delivery_init_artifactory(&ctx)) {
+ if (delivery_init_artifactory(ctx)) {
msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "JFrog CLI configuration failed\n");
exit(1);
}
+}
- runtime_apply(ctx.runtime.environ);
- strcpy(env_name, ctx.info.release_name);
- strcpy(env_name_testing, env_name);
- strcat(env_name_testing, "-test");
-
+static void check_release_history(struct Delivery *ctx) {
// Safety gate: Avoid clobbering a delivered release unless the user wants that behavior
msg(STASIS_MSG_L1, "Checking release history\n");
- if (!globals.enable_overwrite && delivery_exists(&ctx) == DELIVERY_FOUND) {
- msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Refusing to overwrite release: %s\nUse --overwrite to enable release clobbering.\n", ctx.info.release_name);
+ if (!globals.enable_overwrite && delivery_exists(ctx) == DELIVERY_FOUND) {
+ msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Refusing to overwrite release: %s\nUse --overwrite to enable release clobbering.\n", ctx->info.release_name);
exit(1);
}
+}
+
+static void check_conda_install_prefix(const struct Delivery *ctx) {
+ // Unlikely to occur: this should help prevent rmtree() from destroying your entire filesystem
+ // if path is "/" then, die
+ // or if empty string, die
+ if (!strcmp(ctx->storage.conda_install_prefix, DIR_SEP) || !strlen(ctx->storage.conda_install_prefix)) {
+ fprintf(stderr, "error: ctx.storage.conda_install_prefix is malformed!\n");
+ exit(1);
+ }
+}
+
+static void sync_release_history(struct Delivery *ctx) {
if (globals.enable_artifactory) {
// We need to download previous revisions to ensure processed packages are available at build-time
// This is also a docker requirement. Python wheels must exist locally.
- if (ctx.meta.rc > 1) {
- msg(STASIS_MSG_L1, "Syncing delivery artifacts for %s\n", ctx.info.build_name);
- if (delivery_series_sync(&ctx) != 0) {
- msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Unable to sync artifacts for %s\n", ctx.info.build_name);
+ if (ctx->meta.rc > 1) {
+ msg(STASIS_MSG_L1, "Syncing delivery artifacts for %s\n", ctx->info.build_name);
+ if (delivery_series_sync(ctx) != 0) {
+ msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Unable to sync artifacts for %s\n", ctx->info.build_name);
msg(STASIS_MSG_L3, "Case #1:\n"
"\tIf this is a new 'version', and 'rc' is greater "
"than 1, then no previous deliveries exist remotely. "
@@ -248,45 +140,43 @@ int main(int argc, char *argv[]) {
}
}
}
+}
- // Unlikely to occur: this should help prevent rmtree() from destroying your entire filesystem
- // if path is "/" then, die
- // or if empty string, die
- if (!strcmp(ctx.storage.conda_install_prefix, DIR_SEP) || !strlen(ctx.storage.conda_install_prefix)) {
- fprintf(stderr, "error: ctx.storage.conda_install_prefix is malformed!\n");
- exit(1);
- }
-
+static void check_conda_prefix_length(const struct Delivery *ctx) {
// 2 = #!
// 5 = /bin\n
- const size_t prefix_len = strlen(ctx.storage.conda_install_prefix) + 2 + 5;
+ const size_t prefix_len = strlen(ctx->storage.conda_install_prefix) + 2 + 5;
const size_t prefix_len_max = 127;
msg(STASIS_MSG_L1, "Checking length of conda installation prefix\n");
- if (!strcmp(ctx.system.platform[DELIVERY_PLATFORM], "Linux") && prefix_len > prefix_len_max) {
+ if (!strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Linux") && prefix_len > prefix_len_max) {
msg(STASIS_MSG_L2 | STASIS_MSG_ERROR,
"The shebang, '#!%s/bin/python\\n' is too long (%zu > %zu).\n",
- ctx.storage.conda_install_prefix, prefix_len, prefix_len_max);
+ ctx->storage.conda_install_prefix, prefix_len, prefix_len_max);
msg(STASIS_MSG_L2 | STASIS_MSG_ERROR,
"Conda's workaround to handle long path names does not work consistently within STASIS.\n");
msg(STASIS_MSG_L2 | STASIS_MSG_ERROR,
"Please try again from a different, \"shorter\", directory.\n");
exit(1);
}
+}
+static void setup_conda(struct Delivery *ctx, char *installer_url) {
msg(STASIS_MSG_L1, "Conda setup\n");
- delivery_get_conda_installer_url(&ctx, installer_url);
+ delivery_get_conda_installer_url(ctx, installer_url);
msg(STASIS_MSG_L2, "Downloading: %s\n", installer_url);
- if (delivery_get_conda_installer(&ctx, installer_url)) {
+ if (delivery_get_conda_installer(ctx, installer_url)) {
msg(STASIS_MSG_ERROR, "download failed: %s\n", installer_url);
exit(1);
}
- msg(STASIS_MSG_L2, "Installing: %s\n", ctx.conda.installer_name);
- delivery_install_conda(ctx.conda.installer_path, ctx.storage.conda_install_prefix);
+ msg(STASIS_MSG_L2, "Installing: %s\n", ctx->conda.installer_name);
+ delivery_install_conda(ctx->conda.installer_path, ctx->storage.conda_install_prefix);
- msg(STASIS_MSG_L2, "Configuring: %s\n", ctx.storage.conda_install_prefix);
- delivery_conda_enable(&ctx, ctx.storage.conda_install_prefix);
+ msg(STASIS_MSG_L2, "Configuring: %s\n", ctx->storage.conda_install_prefix);
+ delivery_conda_enable(ctx, ctx->storage.conda_install_prefix);
+}
+static void configure_conda_base(struct Delivery *ctx, char *envs[]) {
//
// Implied environment creation modes/actions
//
@@ -303,269 +193,247 @@ int main(int argc, char *argv[]) {
// 3b. Bugs, conflicts, and dependency resolution issues are inherited and
// must be handled in the INI config
msg(STASIS_MSG_L1, "Creating release environment(s)\n");
-
char *mission_base = NULL;
- if (isempty(ctx.meta.based_on)) {
- guard_free(ctx.meta.based_on);
+
+ if (isempty(ctx->meta.based_on)) {
+ // based_on was not set by the input file
+
+ guard_free(ctx->meta.based_on);
char *mission_base_orig = NULL;
- if (asprintf(&mission_base_orig, "%s/%s/base.yml", ctx.storage.mission_dir, ctx.meta.mission) < 0) {
- SYSERROR("Unable to allocate bytes for %s/%s/base.yml path\n", ctx.storage.mission_dir, ctx.meta.mission);
+ if (asprintf(&mission_base_orig, "%s/%s/base.yml", ctx->storage.mission_dir, ctx->meta.mission) < 0) {
+ SYSERROR("Unable to allocate bytes for %s/%s/base.yml path\n", ctx->storage.mission_dir, ctx->meta.mission);
exit(1);
}
+ // Does a base.yml exist in the mission directory?
+ // If not, do nothing. Otherwise, use the base.yml in the mission directory.
if (access(mission_base_orig, F_OK) < 0) {
- msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "Mission does not provide a base.yml configuration: %s (%s)\n",
- ctx.meta.mission, ctx.storage.mission_dir);
+ msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "Mission does not provide a base.yml");
} else {
msg(STASIS_MSG_L2, "Using base environment configuration: %s\n", mission_base_orig);
- if (asprintf(&mission_base, "%s/%s-base.yml", ctx.storage.tmpdir, ctx.info.release_name) < 0) {
+ if (asprintf(&mission_base, "%s/%s-base.yml", ctx->storage.tmpdir, ctx->info.release_name) < 0) {
SYSERROR("%s", "Unable to allocate bytes for temporary base.yml configuration");
remove(mission_base);
exit(1);
}
copy2(mission_base_orig, mission_base, CT_OWNER | CT_PERM);
- ctx.meta.based_on = mission_base;
+ ctx->meta.based_on = mission_base;
}
guard_free(mission_base_orig);
}
- if (!isempty(ctx.meta.based_on)) {
- if (conda_env_exists(ctx.storage.conda_install_prefix, env_name) && conda_env_remove(env_name)) {
- msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to remove release environment: %s\n", env_name);
- exit(1);
- }
+ msg(STASIS_MSG_L2, "Based on: %s\n", ctx->meta.based_on);
- msg(STASIS_MSG_L2, "Based on: %s\n", ctx.meta.based_on);
- if (conda_env_create_from_uri(env_name, ctx.meta.based_on, ctx.meta.python)) {
- msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "unable to install release environment using configuration file\n");
- exit(1);
- }
+ for (size_t i = 0; envs[i] != NULL; i += 2) {
+ char *title = envs[i];
+ char *env = envs[i+1];
+ // If based_on was populated above, or defined in the configuration: install its packages.
+ if (!isempty(ctx->meta.based_on)) {
+ if (conda_env_exists(ctx->storage.conda_install_prefix, env) && conda_env_remove(env)) {
+ msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to remove %s environment: %s\n", title);
+ exit(1);
+ }
- if (conda_env_exists(ctx.storage.conda_install_prefix, env_name_testing) && conda_env_remove(env_name_testing)) {
- msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to remove testing environment %s\n", env_name_testing);
- exit(1);
- }
- if (conda_env_create_from_uri(env_name_testing, ctx.meta.based_on, ctx.meta.python)) {
- msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "unable to install testing environment using configuration file\n");
- exit(1);
- }
- } else {
- if (conda_env_create(env_name, ctx.meta.python, NULL)) {
- msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to create release environment\n");
- exit(1);
- }
- if (conda_env_create(env_name_testing, ctx.meta.python, NULL)) {
- msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to create testing environment\n");
- exit(1);
+ if (conda_env_create_from_uri(env, ctx->meta.based_on, ctx->meta.python)) {
+ msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "unable to install %s environment using configuration file\n", title);
+ exit(1);
+ }
+ } else {
+ // Otherwise, create the environments with the requested Python version and move on
+ if (conda_env_create(env, ctx->meta.python, NULL)) {
+ msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to create %s environment\n", title);
+ exit(1);
+ }
}
}
// The base environment configuration not used past this point
remove(mission_base);
+}
- const char *envs[] = {env_name_testing, env_name};
- for (size_t e = 0; e < sizeof(envs) / sizeof(*envs); e++) {
- const char *name = envs[e];
- if (ctx.conda.conda_packages_purge && strlist_count(ctx.conda.conda_packages_purge)) {
- msg(STASIS_MSG_L2, "Purging conda packages from %s\n", name);
- if (delivery_purge_packages(&ctx, name, PKG_USE_CONDA)) {
- msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "unable to purge requested conda packages from %s\n", name);
- exit(1);
- }
- }
-
- if (ctx.conda.pip_packages_purge && strlist_count(ctx.conda.pip_packages_purge)) {
- msg(STASIS_MSG_L2, "Purging pip packages from %s\n", name);
- if (delivery_purge_packages(&ctx, env_name_testing, PKG_USE_PIP)) {
- msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "unable to purge requested pip packages from %s\n",
- env_name_testing);
- exit(1);
+static void configure_conda_purge(struct Delivery *ctx, char *envs[]) {
+ struct StrList *purge_list[] = {
+ ctx->conda.conda_packages_purge,
+ ctx->conda.pip_packages_purge
+ };
+ for (size_t i = 0; i < sizeof(purge_list) / sizeof(purge_list[0]); i++) {
+ struct StrList *to_purge = purge_list[i];
+ for (size_t e = 0; envs[e] != NULL; e += 2) {
+ //const char *title = envs[e]; // unused
+ const char *env = envs[e+1];
+ if (to_purge && strlist_count(to_purge)) {
+ const char *pkg_manager_name[] = {
+ "conda",
+ "pip"
+ };
+ const int pkg_manager_use[] = {
+ PKG_USE_CONDA,
+ PKG_USE_PIP
+ };
+ const char *manager_str = pkg_manager_name[i];
+ const int manager_flag = pkg_manager_use[i];
+ msg(STASIS_MSG_L2, "Purging %s packages from %s\n", manager_str, env);
+ if (delivery_purge_packages(ctx, env, manager_flag)) {
+ msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "unable to purge requested %s packages from %s\n", manager_str, env);
+ exit(1);
+ }
}
}
}
+}
+static void setup_activate_test_env(const struct Delivery *ctx, const char *env_name_testing) {
// Activate test environment
msg(STASIS_MSG_L1, "Activating test environment\n");
- if (conda_activate(ctx.storage.conda_install_prefix, env_name_testing)) {
+ if (conda_activate(ctx->storage.conda_install_prefix, env_name_testing)) {
fprintf(stderr, "failed to activate test environment\n");
exit(1);
}
+}
- if (delivery_gather_tool_versions(&ctx)) {
- if (!ctx.conda.tool_version) {
+static void configure_tool_versions(struct Delivery *ctx) {
+ if (delivery_gather_tool_versions(ctx)) {
+ if (!ctx->conda.tool_version) {
msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Could not determine conda version\n");
exit(1);
}
- if (!ctx.conda.tool_build_version) {
+ if (!ctx->conda.tool_build_version) {
msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Could not determine conda-build version\n");
exit(1);
}
}
+}
+static void install_build_package() {
if (pip_exec("install build")) {
msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "'build' tool installation failed\n");
exit(1);
}
+}
- if (!isempty(ctx.meta.based_on)) {
+static void configure_package_overlay(struct Delivery *ctx, const char *env_name) {
+ if (!isempty(ctx->meta.based_on)) {
msg(STASIS_MSG_L1, "Generating package overlay from environment: %s\n", env_name);
- if (delivery_overlay_packages_from_env(&ctx, env_name)) {
+ if (delivery_overlay_packages_from_env(ctx, env_name)) {
msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "%s", "Failed to generate package overlay. Resulting environment integrity cannot be guaranteed.\n");
exit(1);
}
}
+}
+static void configure_deferred_packages(struct Delivery *ctx) {
msg(STASIS_MSG_L1, "Filter deliverable packages\n");
- delivery_defer_packages(&ctx, DEFER_CONDA);
- delivery_defer_packages(&ctx, DEFER_PIP);
+ delivery_defer_packages(ctx, DEFER_CONDA);
+ delivery_defer_packages(ctx, DEFER_PIP);
+}
+
+static void show_overiew(struct Delivery *ctx) {
msg(STASIS_MSG_L1, "Overview\n");
- delivery_meta_show(&ctx);
- delivery_conda_show(&ctx);
+ delivery_meta_show(ctx);
+ delivery_conda_show(ctx);
if (globals.verbose) {
//delivery_runtime_show(&ctx);
}
+}
+static void run_tests(struct Delivery *ctx) {
// Execute configuration-defined tests
if (globals.enable_testing) {
- delivery_tests_show(&ctx);
+ delivery_tests_show(ctx);
msg(STASIS_MSG_L1, "Begin test execution\n");
- delivery_tests_run(&ctx);
+ delivery_tests_run(ctx);
msg(STASIS_MSG_L2, "Rewriting test results\n");
- delivery_fixup_test_results(&ctx);
+ delivery_fixup_test_results(ctx);
} else {
msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Test execution is disabled\n");
}
+}
- if (ctx.conda.conda_packages_defer && strlist_count(ctx.conda.conda_packages_defer)) {
+static void build_conda_recipes(struct Delivery *ctx) {
+ if (ctx->conda.conda_packages_defer && strlist_count(ctx->conda.conda_packages_defer)) {
msg(STASIS_MSG_L2, "Building Conda recipe(s)\n");
- if (delivery_build_recipes(&ctx)) {
+ if (delivery_build_recipes(ctx)) {
exit(1);
}
msg(STASIS_MSG_L3, "Copying artifacts\n");
- if (delivery_copy_conda_artifacts(&ctx)) {
+ if (delivery_copy_conda_artifacts(ctx)) {
exit(1);
}
msg(STASIS_MSG_L3, "Indexing artifacts\n");
- if (delivery_index_conda_artifacts(&ctx)) {
+ if (delivery_index_conda_artifacts(ctx)) {
exit(1);
}
}
+}
- if (strlist_count(ctx.conda.pip_packages_defer)) {
+static void build_wheel_packages(struct Delivery *ctx) {
+ if (strlist_count(ctx->conda.pip_packages_defer)) {
msg(STASIS_MSG_L2, "Building Python wheels(s)\n");
- if (!((ctx.conda.wheels_packages = delivery_build_wheels(&ctx)))) {
+ if (!((ctx->conda.wheels_packages = delivery_build_wheels(ctx)))) {
exit(1);
}
- if (delivery_index_wheel_artifacts(&ctx)) {
+ if (delivery_index_wheel_artifacts(ctx)) {
exit(1);
}
}
+}
- // Populate the release environment
- msg(STASIS_MSG_L1, "Populating release environment\n");
+static void release_install_conda_packages(struct Delivery *ctx, char *env_name) {
msg(STASIS_MSG_L2, "Installing conda packages\n");
- if (strlist_count(ctx.conda.conda_packages)) {
- if (delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_CONDA, (struct StrList *[]) {ctx.conda.conda_packages, NULL})) {
+ if (strlist_count(ctx->conda.conda_packages)) {
+ if (delivery_install_packages(ctx, ctx->storage.conda_install_prefix, env_name, INSTALL_PKG_CONDA, (struct StrList *[]) {ctx->conda.conda_packages, NULL})) {
exit(1);
}
}
- if (strlist_count(ctx.conda.conda_packages_defer)) {
+ if (strlist_count(ctx->conda.conda_packages_defer)) {
msg(STASIS_MSG_L3, "Installing deferred conda packages\n");
- if (delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_CONDA | INSTALL_PKG_CONDA_DEFERRED, (struct StrList *[]) {ctx.conda.conda_packages_defer, NULL})) {
+ if (delivery_install_packages(ctx, ctx->storage.conda_install_prefix, env_name, INSTALL_PKG_CONDA | INSTALL_PKG_CONDA_DEFERRED, (struct StrList *[]) {ctx->conda.conda_packages_defer, NULL})) {
exit(1);
}
} else {
msg(STASIS_MSG_L3, "No deferred conda packages\n");
}
+}
+static void release_install_pip_packages(struct Delivery *ctx, char *env_name) {
msg(STASIS_MSG_L2, "Installing pip packages\n");
- if (strlist_count(ctx.conda.pip_packages)) {
- if (delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_PIP, (struct StrList *[]) {ctx.conda.pip_packages, NULL})) {
+ if (strlist_count(ctx->conda.pip_packages)) {
+ if (delivery_install_packages(ctx, ctx->storage.conda_install_prefix, env_name, INSTALL_PKG_PIP, (struct StrList *[]) {ctx->conda.pip_packages, NULL})) {
exit(1);
}
}
- if (strlist_count(ctx.conda.pip_packages_defer)) {
+ if (strlist_count(ctx->conda.pip_packages_defer)) {
msg(STASIS_MSG_L3, "Installing deferred pip packages\n");
- if (delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_PIP | INSTALL_PKG_PIP_DEFERRED, (struct StrList *[]) {ctx.conda.pip_packages_defer, NULL})) {
+ if (delivery_install_packages(ctx, ctx->storage.conda_install_prefix, env_name, INSTALL_PKG_PIP | INSTALL_PKG_PIP_DEFERRED, (struct StrList *[]) {ctx->conda.pip_packages_defer, NULL})) {
exit(1);
}
} else {
msg(STASIS_MSG_L3, "No deferred pip packages\n");
}
+}
- conda_exec("list");
-
- msg(STASIS_MSG_L1, "Creating release\n");
- msg(STASIS_MSG_L2, "Exporting delivery configuration\n");
- if (!pushd(ctx.storage.cfgdump_dir)) {
- char filename[PATH_MAX] = {0};
- sprintf(filename, "%s.ini", ctx.info.release_name);
- FILE *spec = fopen(filename, "w+");
- if (!spec) {
- msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", filename);
- exit(1);
- }
- ini_write(ctx._stasis_ini_fp.delivery, &spec, INI_WRITE_RAW);
- fclose(spec);
-
- memset(filename, 0, sizeof(filename));
- sprintf(filename, "%s-rendered.ini", ctx.info.release_name);
- spec = fopen(filename, "w+");
- if (!spec) {
- msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", filename);
- exit(1);
- }
- ini_write(ctx._stasis_ini_fp.delivery, &spec, INI_WRITE_PRESERVE);
- fclose(spec);
- popd();
- } else {
- SYSERROR("Failed to enter directory: %s", ctx.storage.delivery_dir);
- exit(1);
- }
-
- msg(STASIS_MSG_L2, "Exporting %s\n", env_name_testing);
- if (conda_env_export(env_name_testing, ctx.storage.delivery_dir, env_name_testing)) {
- msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", env_name_testing);
- exit(1);
- }
-
- msg(STASIS_MSG_L2, "Exporting %s\n", env_name);
- if (conda_env_export(env_name, ctx.storage.delivery_dir, env_name)) {
- msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", env_name);
- exit(1);
- }
-
- // Rewrite release environment output (i.e. set package origin(s) to point to the deployment server, etc.)
- char specfile[PATH_MAX];
- sprintf(specfile, "%s/%s.yml", ctx.storage.delivery_dir, env_name);
- msg(STASIS_MSG_L3, "Rewriting release spec file (stage 1): %s\n", path_basename(specfile));
- delivery_rewrite_spec(&ctx, specfile, DELIVERY_REWRITE_SPEC_STAGE_1);
-
- msg(STASIS_MSG_L1, "Rendering mission templates\n");
- delivery_mission_render_files(&ctx);
-
- int want_docker = ini_section_search(&ctx._stasis_ini_fp.delivery, INI_SEARCH_BEGINS, "deploy:docker") ? true : false;
- int want_artifactory = ini_section_search(&ctx._stasis_ini_fp.delivery, INI_SEARCH_BEGINS, "deploy:artifactory") ? true : false;
+static void build_docker(struct Delivery *ctx, const int disabled) {
+ const int want_docker = ini_section_search(&ctx->_stasis_ini_fp.delivery, INI_SEARCH_BEGINS, "deploy:docker") ? true : false;
if (want_docker) {
- if (user_disabled_docker) {
+ if (disabled) {
msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Docker image building is disabled by CLI argument\n");
} else {
char dockerfile[PATH_MAX] = {0};
- sprintf(dockerfile, "%s/%s", ctx.storage.build_docker_dir, "Dockerfile");
+ sprintf(dockerfile, "%s/%s", ctx->storage.build_docker_dir, "Dockerfile");
if (globals.enable_docker) {
if (!access(dockerfile, F_OK)) {
msg(STASIS_MSG_L1, "Building Docker image\n");
- if (delivery_docker(&ctx)) {
+ if (delivery_docker(ctx)) {
msg(STASIS_MSG_L1 | STASIS_MSG_ERROR, "Failed to build docker image!\n");
COE_CHECK_ABORT(1, "Failed to build docker image");
}
} else {
- msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Docker image building is disabled. No Dockerfile found in %s\n", ctx.storage.build_docker_dir);
+ msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Docker image building is disabled. No Dockerfile found in %s\n", ctx->storage.build_docker_dir);
}
} else {
msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Docker image building is disabled. System configuration error\n");
@@ -574,25 +442,220 @@ int main(int argc, char *argv[]) {
} else {
msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Docker image building is disabled. deploy:docker is not configured\n");
}
+}
+
+static void generate_release(struct Delivery *ctx, char *env_name, char *env_name_testing, const int disable_docker) {
+ // Populate the release environment
+ msg(STASIS_MSG_L1, "Populating release environment\n");
+ release_install_conda_packages(ctx, env_name);
+ release_install_pip_packages(ctx, env_name);
+
+ conda_exec("list");
+
+ msg(STASIS_MSG_L1, "Creating release\n");
+ delivery_export(ctx, (char *[]) {env_name, env_name_testing, NULL});
+
+ char specfile[PATH_MAX];
+ sprintf(specfile, "%s/%s.yml", ctx->storage.delivery_dir, env_name);
- msg(STASIS_MSG_L3, "Rewriting release spec file (stage 2): %s\n", path_basename(specfile));
- delivery_rewrite_spec(&ctx, specfile, DELIVERY_REWRITE_SPEC_STAGE_2);
+ delivery_rewrite_stage1(ctx, specfile);
+ build_docker(ctx, disable_docker);
+ delivery_rewrite_stage2(ctx, specfile);
msg(STASIS_MSG_L1, "Dumping metadata\n");
- if (delivery_dump_metadata(&ctx)) {
+ if (delivery_dump_metadata(ctx)) {
msg(STASIS_MSG_L1 | STASIS_MSG_ERROR, "Metadata dump failed\n");
}
+}
+static void transfer_artifacts(struct Delivery *ctx) {
+ const int want_artifactory = ini_section_search(&ctx->_stasis_ini_fp.delivery, INI_SEARCH_BEGINS, "deploy:artifactory") ? true : false;
if (want_artifactory) {
if (globals.enable_artifactory && globals.enable_artifactory_upload) {
msg(STASIS_MSG_L1, "Uploading artifacts\n");
- delivery_artifact_upload(&ctx);
+ delivery_artifact_upload(ctx);
} else {
msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Artifactory upload is disabled by CLI argument\n");
}
} else {
msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Artifactory upload is disabled. deploy:artifactory is not configured\n");
}
+}
+
+int main(int argc, char *argv[]) {
+ struct Delivery ctx;
+ struct Process proc = {
+ .f_stdout = "",
+ .f_stderr = "",
+ .redirect_stderr = 0,
+ };
+ char env_name[STASIS_NAME_MAX] = {0};
+ char env_name_testing[STASIS_NAME_MAX] = {0};
+ char *delivery_input = NULL;
+ char *config_input = NULL;
+ char installer_url[PATH_MAX];
+ char python_override_version[STASIS_NAME_MAX];
+ int user_disabled_docker = false;
+ globals.cpu_limit = get_cpu_count();
+ if (globals.cpu_limit > 1) {
+ globals.cpu_limit--; // max - 1
+ }
+
+ memset(env_name, 0, sizeof(env_name));
+ memset(env_name_testing, 0, sizeof(env_name_testing));
+ memset(installer_url, 0, sizeof(installer_url));
+ memset(python_override_version, 0, sizeof(python_override_version));
+ memset(&proc, 0, sizeof(proc));
+ memset(&ctx, 0, sizeof(ctx));
+
+ int c;
+ int option_index = 0;
+ while ((c = getopt_long(argc, argv, "hVCc:p:vU", long_options, &option_index)) != -1) {
+ switch (c) {
+ case 'h':
+ usage(path_basename(argv[0]));
+ exit(0);
+ case 'V':
+ puts(VERSION);
+ exit(0);
+ case 'c':
+ config_input = strdup(optarg);
+ break;
+ case 'C':
+ globals.continue_on_error = true;
+ break;
+ case 'p':
+ strcpy(python_override_version, optarg);
+ break;
+ case 'l':
+ globals.cpu_limit = strtol(optarg, NULL, 10);
+ if (globals.cpu_limit <= 1) {
+ globals.cpu_limit = 1;
+ globals.enable_parallel = false; // No point
+ }
+ break;
+ case OPT_ALWAYS_UPDATE_BASE:
+ globals.always_update_base_environment = true;
+ break;
+ case OPT_FAIL_FAST:
+ globals.parallel_fail_fast = true;
+ break;
+ case OPT_POOL_STATUS_INTERVAL:
+ globals.pool_status_interval = (int) strtol(optarg, NULL, 10);
+ if (globals.pool_status_interval < 1) {
+ globals.pool_status_interval = 1;
+ } else if (globals.pool_status_interval > 60 * 10) {
+ // Possible poor choice alert
+ fprintf(stderr, "Caution: Excessive pausing between status updates may cause third-party CI/CD"
+ " jobs to fail if the stdout/stderr streams are idle for too long!\n");
+ }
+ break;
+ case 'U':
+ setenv("PYTHONUNBUFFERED", "1", 1);
+ fflush(stdout);
+ fflush(stderr);
+ setvbuf(stdout, NULL, _IONBF, 0);
+ setvbuf(stderr, NULL, _IONBF, 0);
+ break;
+ case 'v':
+ globals.verbose = true;
+ break;
+ case OPT_OVERWRITE:
+ globals.enable_overwrite = true;
+ break;
+ case OPT_NO_DOCKER:
+ globals.enable_docker = false;
+ user_disabled_docker = true;
+ break;
+ case OPT_NO_ARTIFACTORY:
+ globals.enable_artifactory = false;
+ break;
+ case OPT_NO_ARTIFACTORY_BUILD_INFO:
+ globals.enable_artifactory_build_info = false;
+ break;
+ case OPT_NO_ARTIFACTORY_UPLOAD:
+ globals.enable_artifactory_build_info = false;
+ globals.enable_artifactory_upload = false;
+ break;
+ case OPT_NO_TESTING:
+ globals.enable_testing = false;
+ break;
+ case OPT_NO_REWRITE_SPEC_STAGE_2:
+ globals.enable_rewrite_spec_stage_2 = false;
+ break;
+ case OPT_NO_PARALLEL:
+ globals.enable_parallel = false;
+ break;
+ case '?':
+ default:
+ exit(1);
+ }
+ }
+
+ if (optind < argc) {
+ while (optind < argc) {
+ // use first positional argument
+ delivery_input = argv[optind++];
+ break;
+ }
+ }
+
+ if (!delivery_input) {
+ fprintf(stderr, "error: a DELIVERY_FILE is required\n");
+ usage(path_basename(argv[0]));
+ exit(1);
+ }
+
+ printf(BANNER, VERSION, AUTHOR);
+
+ check_system_path();
+
+ msg(STASIS_MSG_L1, "Setup\n");
+
+ tpl_setup_vars(&ctx);
+ tpl_setup_funcs(&ctx);
+
+ setup_sysconfdir();
+ setup_python_version_override(&ctx, python_override_version);
+
+ configure_stasis_ini(&ctx, &config_input);
+ configure_delivery_ini(&ctx, &delivery_input);
+ configure_delivery_context(&ctx);
+
+ check_requirements(&ctx);
+ configure_jfrog_cli(&ctx);
+
+ runtime_apply(ctx.runtime.environ);
+ strcpy(env_name, ctx.info.release_name);
+ strcpy(env_name_testing, env_name);
+ strcat(env_name_testing, "-test");
+ char *envs[] = {
+ "release", env_name,
+ "testing", env_name_testing,
+ NULL, NULL,
+ };
+
+ check_release_history(&ctx);
+ sync_release_history(&ctx);
+
+ check_conda_install_prefix(&ctx);
+ check_conda_prefix_length(&ctx);
+ setup_conda(&ctx, installer_url);
+ configure_conda_base(&ctx, envs);
+ configure_conda_purge(&ctx, envs);
+ setup_activate_test_env(&ctx, env_name_testing);
+
+ configure_tool_versions(&ctx);
+ install_build_package();
+ configure_package_overlay(&ctx, env_name);
+ configure_deferred_packages(&ctx);
+
+ show_overiew(&ctx);
+ run_tests(&ctx);
+ build_conda_recipes(&ctx);
+ build_wheel_packages(&ctx);
+ generate_release(&ctx, env_name, env_name_testing, user_disabled_docker);
+ transfer_artifacts(&ctx);
msg(STASIS_MSG_L1, "Cleaning up\n");
delivery_free(&ctx);
diff --git a/src/lib/core/conda.c b/src/lib/core/conda.c
index c81e6cc..de6130f 100644
--- a/src/lib/core/conda.c
+++ b/src/lib/core/conda.c
@@ -4,7 +4,7 @@
#include "conda.h"
-int micromamba(struct MicromambaInfo *info, char *command, ...) {
+int micromamba(const struct MicromambaInfo *info, char *command, ...) {
struct utsname sys;
uname(&sys);
@@ -24,7 +24,13 @@ int micromamba(struct MicromambaInfo *info, char *command, ...) {
sprintf(installer_path, "%s/latest", getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp");
if (access(installer_path, F_OK)) {
- download(url, installer_path, NULL);
+ char *errmsg = NULL;
+ const long http_code = download(url, installer_path, &errmsg);
+ if (HTTP_ERROR(http_code)) {
+ fprintf(stderr, "download failed: %ld: %s\n", http_code, errmsg);
+ guard_free(errmsg);
+ return -1;
+ }
}
char mmbin[PATH_MAX];
@@ -62,17 +68,40 @@ int micromamba(struct MicromambaInfo *info, char *command, ...) {
}
int python_exec(const char *args) {
- char command[PATH_MAX] = {0};
- snprintf(command, sizeof(command) - 1, "python %s", args);
+ const char *command_base = "python ";
+ const char *command_fmt = "%s%s";
+
+ const int len = snprintf(NULL, 0, command_fmt, command_base, args);
+ char *command = calloc(len + 1, sizeof(*command));
+ if (!command) {
+ SYSERROR("Unable to allocate %d bytes for command string", len);
+ return -1;
+ }
+
+ snprintf(command, len + 1, command_fmt, command_base, args);
msg(STASIS_MSG_L3, "Executing: %s\n", command);
- return system(command);
+
+ const int result = system(command);
+ guard_free(command);
+ return result;
}
int pip_exec(const char *args) {
- char command[PATH_MAX] = {0};
- snprintf(command, sizeof(command) - 1, "python -m pip %s", args);
+ const char *command_base = "python -m pip ";
+ const char *command_fmt = "%s%s";
+
+ const int len = snprintf(NULL, 0, command_fmt, command_base, args);
+ char *command = calloc(len + 1, sizeof(*command));
+ if (!command) {
+ SYSERROR("Unable to allocate %d bytes for command string", len);
+ return -1;
+ }
+ snprintf(command, len + 1, command_fmt, command_base, args);
msg(STASIS_MSG_L3, "Executing: %s\n", command);
- return system(command);
+
+ const int result = system(command);
+ guard_free(command);
+ return result;
}
static const char *PKG_ERROR_STR[] = {
@@ -177,7 +206,6 @@ int pkg_index_provides(int mode, const char *index, const char *spec) {
}
int conda_exec(const char *args) {
- char command[PATH_MAX];
const char *mamba_commands[] = {
"build",
"install",
@@ -202,15 +230,24 @@ int conda_exec(const char *args) {
}
}
- snprintf(command, sizeof(command) - 1, "%s %s", conda_as, args);
+ const char *command_fmt = "%s %s";
+ const int len = snprintf(NULL, 0, command_fmt, conda_as, args);
+ char *command = calloc(len + 1, sizeof(*command));
+ if (!command) {
+ return -1;
+ }
+
+ snprintf(command, len + 1, command_fmt, conda_as, args);
msg(STASIS_MSG_L3, "Executing: %s\n", command);
- return system(command);
+ const int result = system(command);
+ guard_free(command);
+ return result;
}
static int conda_prepend_bin(const char *root) {
char conda_bin[PATH_MAX] = {0};
- snprintf(conda_bin, sizeof(conda_bin) - 1, "%s/bin", root);
+ snprintf(conda_bin, sizeof(conda_bin), "%s/bin", root);
if (env_manipulate_pathstr("PATH", conda_bin, PM_PREPEND | PM_ONCE)) {
return -1;
}
@@ -220,7 +257,7 @@ static int conda_prepend_bin(const char *root) {
static int conda_prepend_condabin(const char *root) {
char conda_condabin[PATH_MAX] = {0};
- snprintf(conda_condabin, sizeof(conda_condabin) - 1, "%s/condabin", root);
+ snprintf(conda_condabin, sizeof(conda_condabin), "%s/condabin", root);
if (env_manipulate_pathstr("PATH", conda_condabin, PM_PREPEND | PM_ONCE)) {
return -1;
}
@@ -329,7 +366,7 @@ int conda_activate(const char *root, const char *env_name) {
return -1;
}
- snprintf(command, sizeof(command) - 1,
+ snprintf(command, sizeof(command),
"set -a\n"
"source %s\n"
"__conda_exe() (\n"
@@ -499,10 +536,8 @@ int conda_setup_headless() {
}
int conda_env_create_from_uri(char *name, char *uri, char *python_version) {
- char env_command[PATH_MAX];
char *uri_fs = NULL;
-
// Convert a bare system path to a file:// path
if (!strstr(uri, "://")) {
uri_fs = calloc(strlen(uri) + strlen("file://") + 1, sizeof(*uri_fs));
@@ -523,25 +558,50 @@ int conda_env_create_from_uri(char *name, char *uri, char *python_version) {
// We'll create a new file with the same random bits, ending with .yml
strcat(tempfile, ".yml");
char *errmsg = NULL;
- download(uri_fs ? uri_fs : uri, tempfile, &errmsg);
+ const long http_code = download(uri_fs ? uri_fs : uri, tempfile, &errmsg);
+ if (HTTP_ERROR(http_code)) {
+ if (errmsg) {
+ fprintf(stderr, "download failed: %ld: %s\n", http_code, errmsg);
+ guard_free(errmsg);
+ }
+ guard_free(uri_fs);
+ return -1;
+ }
guard_free(uri_fs);
// Rewrite python version
char spec[255] = {0};
- snprintf(spec, sizeof(spec) - 1, "- python=%s\n", python_version);
+ snprintf(spec, sizeof(spec), "- python=%s\n", python_version);
file_replace_text(tempfile, "- python\n", spec, 0);
- sprintf(env_command, "env create -n '%s' --file='%s'", name, tempfile);
- int status = conda_exec(env_command);
+ const char *fmt = "env create -n '%s' --file='%s'";
+ int len = snprintf(NULL, 0, fmt, name, tempfile);
+ char *env_command = calloc(len + 1, sizeof(*env_command));
+ if (!env_command) {
+ return -1;
+ }
+
+ snprintf(env_command, len + 1, fmt, name, tempfile);
+ const int status = conda_exec(env_command);
unlink(tempfile);
+ guard_free(env_command);
return status;
}
int conda_env_create(char *name, char *python_version, char *packages) {
- char env_command[PATH_MAX];
- sprintf(env_command, "create -n %s python=%s %s", name, python_version, packages ? packages : "");
- return conda_exec(env_command);
+ const char *fmt = "create -n %s python=%s %s";
+ const int len = snprintf(NULL, 0, fmt, name, python_version, packages ? packages : "");
+ char *env_command = calloc(len + 1, sizeof(*env_command));
+ if (!env_command) {
+ return -1;
+ }
+
+ snprintf(env_command, len + 1, fmt, name, python_version, packages ? packages : "");
+ const int result = conda_exec(env_command);
+ guard_free(env_command);
+
+ return result;
}
int conda_env_remove(char *name) {
diff --git a/src/lib/core/download.c b/src/lib/core/download.c
index c3f8dca..b021860 100644
--- a/src/lib/core/download.c
+++ b/src/lib/core/download.c
@@ -41,10 +41,10 @@ long download(char *url, const char *filename, char **errmsg) {
CURLcode curl_code = curl_easy_perform(c);
SYSDEBUG("curl status code: %d", curl_code);
if (curl_code != CURLE_OK) {
- if (errmsg) {
- strcpy(*errmsg, curl_easy_strerror(curl_code));
+ if (!*errmsg) {
+ *errmsg = strdup(curl_easy_strerror(curl_code));
} else {
- fprintf(stderr, "\nCURL ERROR: %s\n", curl_easy_strerror(curl_code));
+ strncpy(*errmsg, curl_easy_strerror(curl_code), strlen(curl_easy_strerror(curl_code) + 1));
}
goto failed;
}
diff --git a/src/lib/core/environment.c b/src/lib/core/environment.c
index f5e8566..7ece5e6 100644
--- a/src/lib/core/environment.c
+++ b/src/lib/core/environment.c
@@ -106,14 +106,13 @@ void runtime_export(RuntimeEnv *env, char **keys) {
if (keys != NULL) {
for (size_t j = 0; keys[j] != NULL; j++) {
if (strcmp(keys[j], key) == 0) {
- //sprintf(output, "%s=\"%s\"\n%s %s", key, value ? value : "", export_command, key);
- sprintf(output, "%s %s=\"%s\"", export_command, key, value ? value : "");
+ snprintf(output, sizeof(output), "%s %s=\"%s\"", export_command, key, value ? value : "");
puts(output);
}
}
}
else {
- sprintf(output, "%s %s=\"%s\"", export_command, key, value ? value : "");
+ snprintf(output, sizeof(output), "%s %s=\"%s\"", export_command, key, value ? value : "");
puts(output);
}
guard_free(value);
@@ -178,7 +177,7 @@ int runtime_replace(RuntimeEnv **dest, char **src) {
}
/**
- * Determine whether or not a key exists in the runtime environment
+ * Determine whether a key exists in the runtime environment
*
* Example:
*
@@ -245,7 +244,14 @@ char *runtime_get(RuntimeEnv *env, const char *key) {
ssize_t key_offset = runtime_contains(env, key);
if (key_offset != -1) {
char **pair = split(strlist_item(env, key_offset), "=", 0);
+ if (!pair) {
+ return NULL;
+ }
result = join(&pair[1], "=");
+ if (!result) {
+ guard_array_free(pair);
+ return NULL;
+ }
guard_array_free(pair);
}
return result;
@@ -285,8 +291,7 @@ char *runtime_expand_var(RuntimeEnv *env, char *input) {
// If there's no environment variables to process return the input string
if (strchr(input, delim) == NULL) {
- //return strdup(input);
- return input;
+ return strdup(input);
}
expanded = calloc(STASIS_BUFSIZ, sizeof(char));
@@ -336,7 +341,10 @@ char *runtime_expand_var(RuntimeEnv *env, char *input) {
if (env) {
tmp = runtime_get(env, var);
} else {
- tmp = getenv(var);
+ const char *v = getenv(var);
+ if (v) {
+ tmp = strdup(v);
+ }
}
if (tmp == NULL) {
// This mimics shell behavior in general.
@@ -348,9 +356,7 @@ char *runtime_expand_var(RuntimeEnv *env, char *input) {
}
// Append expanded environment variable to output
strncat(expanded, tmp, STASIS_BUFSIZ - 1);
- if (env) {
- guard_free(tmp);
- }
+ guard_free(tmp);
}
// Nothing to do so append input to output
@@ -400,13 +406,28 @@ char *runtime_expand_var(RuntimeEnv *env, char *input) {
* @param _value New environment variable value
*/
void runtime_set(RuntimeEnv *env, const char *_key, char *_value) {
+ const char *sep = "=";
if (_key == NULL) {
return;
}
+ const ssize_t key_offset = runtime_contains(env, _key);
char *key = strdup(_key);
- ssize_t key_offset = runtime_contains(env, key);
+ if (!key) {
+ SYSERROR("%s", "unable to allocate memory for key");
+ exit(1);
+ }
char *value = runtime_expand_var(env, _value);
- char *now = join((char *[]) {key, value, NULL}, "=");
+ if (!value) {
+ SYSERROR("%s", "unable to allocate memory for value");
+ exit(1);
+ }
+
+ lstrip(value);
+ char *now = join((char *[]) {key, value, NULL}, sep);
+ if (!now) {
+ SYSERROR("%s", "unable to allocate memory for join");
+ exit(1);
+ }
if (key_offset < 0) {
strlist_append(&env, now);
@@ -415,6 +436,7 @@ void runtime_set(RuntimeEnv *env, const char *_key, char *_value) {
}
guard_free(now);
guard_free(key);
+ guard_free(value);
}
/**
@@ -424,6 +446,10 @@ void runtime_set(RuntimeEnv *env, const char *_key, char *_value) {
void runtime_apply(RuntimeEnv *env) {
for (size_t i = 0; i < strlist_count(env); i++) {
char **pair = split(strlist_item(env, i), "=", 1);
+ if (!pair) {
+ SYSERROR("%s", "unable to allocate memory for runtime_apply");
+ return;
+ }
setenv(pair[0], pair[1], 1);
guard_array_free(pair);
}
diff --git a/src/lib/core/include/conda.h b/src/lib/core/include/conda.h
index ea8613f..f3d481c 100644
--- a/src/lib/core/include/conda.h
+++ b/src/lib/core/include/conda.h
@@ -38,7 +38,7 @@ struct MicromambaInfo {
* @param ... variadic arguments
* @return exit code
*/
-int micromamba(struct MicromambaInfo *info, char *command, ...);
+int micromamba(const struct MicromambaInfo *info, char *command, ...);
/**
* Execute Python
diff --git a/src/lib/core/include/core.h b/src/lib/core/include/core.h
index 35a9506..92969d2 100644
--- a/src/lib/core/include/core.h
+++ b/src/lib/core/include/core.h
@@ -15,7 +15,7 @@
#define STASIS_NAME_MAX 255
#define STASIS_DIRSTACK_MAX 1024
#define STASIS_TIME_STR_MAX 128
-#define HTTP_ERROR(X) X >= 400
+#define HTTP_ERROR(X) (X >= 400 || X < 0)
#include "config.h"
#include "core_mem.h"
diff --git a/src/lib/core/include/str.h b/src/lib/core/include/str.h
index bb96db0..be497ed 100644
--- a/src/lib/core/include/str.h
+++ b/src/lib/core/include/str.h
@@ -293,18 +293,27 @@ int isdigit_s(const char *s);
char *tolower_s(char *s);
/**
- * Return a copy of the input string with "." characters removed
+ * Reduce a version string to the major[minor] format used by Python
*
- * ~~~{.c}
- * char *version = strdup("1.2.3");
- * char *version_short = to_short_version(str);
- * // version_short is "123"
- * free(version_short);
+ * @code{.c}
+ * #include <stdio.h>
+ * #include "str.h"
*
- * ~~~
+ * int main(int argc, char *argv[]) {
+ * char python_version[] = "3.13.3"
+ * char *python_short_version = to_short_version(python_version); // "313"
+ * if (!python_short_version) {
+ * perror("unable to allocate memory for shortened python version");
+ * return 1;
+ * }
+ * free(python_short_version);
+ * return 0;
+ * }
+ * @endcode
*
- * @param s input string
- * @return pointer to new string
+ * @param s python version string
+ * @return the shortened version string
+ * @return NULL on error
*/
char *to_short_version(const char *s);
diff --git a/src/lib/core/include/utils.h b/src/lib/core/include/utils.h
index b405b2a..a9bcd2f 100644
--- a/src/lib/core/include/utils.h
+++ b/src/lib/core/include/utils.h
@@ -419,4 +419,49 @@ int gen_file_extension_str(char *filename, const char *extension);
char *remove_extras(char *s);
void debug_hexdump(char *data, int len);
+
+/**
+ * Realloc helper
+ *
+ * @code{.c}
+ * #include <stdio.h>
+ * #include <stdlib.h>
+ * #include <string.h>
+ * #include "utils.h"
+ *
+ * int main(int argc, char *argv[]) {
+ * size_t sz = 10;
+ * char *data = calloc(sz, sizeof(*data));
+ *
+ * // populate data
+ * strncat(data, "/path/to/", sz - 1);
+ *
+ * // Double the allocation size for data
+ * if (grow(sz * 2, &sz, &data)) {
+ * // memory error
+ * }
+ *
+ * // sz is now 20
+ * strncat(data, "filename", sz - 1 - strlen(data));
+ *
+ * puts(data);
+ * // output: "/path/to/filename"
+ * }
+ * @endcode
+ *
+ * @param size_new increase by `size_new` bytes
+ * @param size_orig address of variable containing the original allocation size (modified)
+ * @param data address to write data
+ * @return 0 on success
+ * @return -1 on error
+ */
+int grow(size_t size_new, size_t *size_orig, char **data);
+
+int in_ascii_range(char c, char lower, char upper);
+
+#define GIT_HASH_LEN 40
+int is_git_sha(char const *hash);
+
+int check_python_package_dependencies(const char *srcdir);
+
#endif //STASIS_UTILS_H
diff --git a/src/lib/core/str.c b/src/lib/core/str.c
index 1d0b268..9524886 100644
--- a/src/lib/core/str.c
+++ b/src/lib/core/str.c
@@ -640,12 +640,35 @@ char *tolower_s(char *s) {
}
char *to_short_version(const char *s) {
- char *result = strdup(s);
- if (!result) {
- return NULL;
+ char *result = NULL;
+ if (num_chars(s, '.') > 1) {
+ char **version_data = split((char *) s, ".", 1);
+ if (!version_data) {
+ goto to_short_version_failed;
+ }
+ if (version_data[1]) {
+ char *dot = strchr(version_data[1], '.');
+ if (dot) {
+ *dot = '\0';
+ }
+ }
+ result = join(version_data, "");
+ if (!result) {
+ guard_array_free(version_data);
+ goto to_short_version_failed;
+ }
+ guard_array_free(version_data);
+ } else {
+ result = strdup(s);
+ if (!result) {
+ goto to_short_version_failed;
+ }
+ strchrdel(result, ".");
}
- strchrdel(result, ".");
+
return result;
+ to_short_version_failed:
+ return NULL;
}
void unindent(char *s) {
diff --git a/src/lib/core/template.c b/src/lib/core/template.c
index ba45a5a..dd3c7a2 100644
--- a/src/lib/core/template.c
+++ b/src/lib/core/template.c
@@ -137,23 +137,6 @@ struct tplfunc_frame *tpl_getfunc(char *key) {
return result;
}
-static int grow(size_t z, size_t *output_bytes, char **output) {
- if (z >= *output_bytes) {
- size_t new_size = *output_bytes + z + 1;
- SYSDEBUG("template output buffer new size: %zu\n", new_size);
-
- char *tmp = realloc(*output, new_size);
- if (!tmp) {
- perror("realloc failed");
- return -1;
- } else if (tmp != *output) {
- *output = tmp;
- }
- *output_bytes = new_size;
- }
- return 0;
-}
-
char *tpl_render(char *str) {
if (!str) {
return NULL;
@@ -297,8 +280,8 @@ char *tpl_render(char *str) {
output[z] = pos[off];
z++;
}
- SYSDEBUG("template output length: %zu", strlen(output));
- SYSDEBUG("template output bytes: %zu", output_bytes);
+ //SYSDEBUG("template output length: %zu", strlen(output));
+ //SYSDEBUG("template output bytes: %zu", output_bytes);
return output;
}
diff --git a/src/lib/core/utils.c b/src/lib/core/utils.c
index a9c9ea5..62f3bec 100644
--- a/src/lib/core/utils.c
+++ b/src/lib/core/utils.c
@@ -892,10 +892,11 @@ int gen_file_extension_str(char *filename, const char *extension) {
return replace_text(ext_orig, ext_orig, extension, 0);
}
+#define DEBUG_HEXDUMP_FMT_BYTES 6
#define DEBUG_HEXDUMP_ADDR_MAXLEN 20
#define DEBUG_HEXDUMP_BYTES_MAXLEN (16 * 3 + 2)
#define DEBUG_HEXDUMP_ASCII_MAXLEN (16 + 1)
-#define DEBUG_HEXDUMP_OUTPUT_MAXLEN (DEBUG_HEXDUMP_ADDR_MAXLEN + DEBUG_HEXDUMP_BYTES_MAXLEN + DEBUG_HEXDUMP_ASCII_MAXLEN + 1)
+#define DEBUG_HEXDUMP_OUTPUT_MAXLEN (DEBUG_HEXDUMP_FMT_BYTES + DEBUG_HEXDUMP_ADDR_MAXLEN + DEBUG_HEXDUMP_BYTES_MAXLEN + DEBUG_HEXDUMP_ASCII_MAXLEN + 1)
void debug_hexdump(char *data, int len) {
int count = 0;
@@ -936,10 +937,144 @@ void debug_hexdump(char *data, int len) {
// Add group padding
strcat(bytes, " ");
}
- int padding = 16 - count;
+ const int padding = 16 - count;
for (int i = 0; i < padding; i++) {
strcat(bytes, " ");
}
- sprintf(output, "%s | %s | %s", addr, bytes, ascii);
+ snprintf(output, DEBUG_HEXDUMP_FMT_BYTES + sizeof(addr) + sizeof(bytes) + sizeof(ascii), "%s | %s | %s", addr, bytes, ascii);
puts(output);
-} \ No newline at end of file
+}
+
+int grow(const size_t size_new, size_t *size_orig, char **data) {
+ if (!*data) {
+ return 0;
+ }
+ if (size_new >= *size_orig) {
+ const size_t new_size = *size_orig + size_new + 1;
+ SYSDEBUG("template data buffer new size: %zu\n", new_size);
+
+ char *tmp = realloc(*data, new_size);
+ if (!tmp) {
+ perror("realloc failed");
+ return -1;
+ }
+ if (tmp != *data) {
+ *data = tmp;
+ }
+ *size_orig = new_size;
+ }
+ return 0;
+}
+
+int in_ascii_range(const char c, char lower, char upper) {
+ if (!(c >= lower && c <= upper)) {
+ return 0;
+ }
+ return 1;
+}
+
+int is_git_sha(char const *hash) {
+ size_t result = 0;
+ size_t len = strlen(hash);
+
+ if (len > GIT_HASH_LEN) {
+ // too long to be a git commit hash
+ return 0;
+ }
+ for (size_t i = 0; i < len; i++) {
+ if (in_ascii_range(hash[i], 'a', 'f')
+ || in_ascii_range(hash[i], 'A', 'F')
+ || in_ascii_range(hash[i], '0', '9')) {
+ result++;
+ }
+ }
+ if (result < len) {
+ return 0;
+ }
+ return 1;
+}
+
+static int read_vcs_records(const size_t line, char **data) {
+ (void) line; // unused
+ const char *vcs_name[] = {
+ "git",
+ "svn",
+ "hg",
+ "bzr",
+ };
+ for (size_t i = 0; i < sizeof(vcs_name) / sizeof(vcs_name[0]); i++) {
+ const char *vcs = vcs_name[i];
+ char *data_local = strdup(*data);
+ if (!data_local) {
+ fprintf(stderr, "Out of memory\n");
+ return -1;
+ }
+
+ // Remove leading/trailing blanks
+ lstrip(data_local);
+ strip(data_local);
+
+ // Ignore file comment(s)
+ if (startswith(data_local, "#") || startswith(data_local, ";")) {
+ // continue
+ return 1;
+ }
+
+ // Begin matching VCS package syntax
+ const char *match_vcs = strstr(data_local,vcs);
+ if (match_vcs) {
+ const char *match_protocol_sep = strstr(match_vcs, "+");
+ if (match_protocol_sep) {
+ const char *match_protocol = strstr(match_protocol_sep, "://");
+ if (match_protocol) {
+ guard_free(data_local);
+ // match found
+ return 0;
+ }
+ }
+ }
+ guard_free(data_local);
+ }
+
+ // no match, continue
+ return 1;
+}
+int check_python_package_dependencies(const char *srcdir) {
+ const char *configs[] = {
+ "pyproject.toml",
+ "setup.cfg",
+ "setup.py"
+ };
+
+ for (size_t i = 0; i < sizeof(configs) / sizeof(configs[0]); i++) {
+ char path[PATH_MAX] = {0};
+ const char *configfile = configs[i];
+
+ snprintf(path, sizeof(path), "%s/%s", srcdir, configfile);
+ if (access(path, F_OK) < 0) {
+ continue;
+ }
+
+ //char **data = file_readlines(path, 0, 0, NULL);
+ struct StrList *data = strlist_init();
+ int err = 0;
+ if ((err = strlist_append_file(data, path, read_vcs_records))) {
+ guard_strlist_free(&data);
+ return -1;
+ }
+ const size_t count = strlist_count(data);
+ if (count) {
+ printf("\nERROR: VCS requirement(s) detected in %s:\n", configfile);
+ for (size_t j = 0; j < count; j++) {
+ char *record = strlist_item(data, j);
+ lstrip(record);
+ strip(record);
+ printf("[%zu] %s\n", j, record);
+ }
+ guard_strlist_free(&data);
+ return 1;
+ }
+ guard_strlist_free(&data);
+ }
+ return 0;
+}
diff --git a/src/lib/delivery/CMakeLists.txt b/src/lib/delivery/CMakeLists.txt
index 78ed20f..559b2dc 100644
--- a/src/lib/delivery/CMakeLists.txt
+++ b/src/lib/delivery/CMakeLists.txt
@@ -1,4 +1,5 @@
add_library(stasis_delivery STATIC
+ delivery_export.c
delivery_postprocess.c
delivery_conda.c
delivery_docker.c
diff --git a/src/lib/delivery/delivery.c b/src/lib/delivery/delivery.c
index d480ab4..7ec2e04 100644
--- a/src/lib/delivery/delivery.c
+++ b/src/lib/delivery/delivery.c
@@ -225,7 +225,15 @@ void delivery_defer_packages(struct Delivery *ctx, int type) {
// Override test->version when a version is provided by the (pip|conda)_package list item
guard_free(test->version);
if (spec_begin && spec_end) {
- test->version = strdup(spec_end);
+ char *version_at = strrchr(spec_end, '@');
+ if (version_at) {
+ if (strlen(version_at)) {
+ version_at++;
+ }
+ test->version = strdup(version_at);
+ } else {
+ test->version = strdup(spec_end);
+ }
} else {
// There are too many possible default branches nowadays: master, main, develop, xyz, etc.
// HEAD is a safe bet.
@@ -233,6 +241,9 @@ void delivery_defer_packages(struct Delivery *ctx, int type) {
}
// Is the list item a git+schema:// URL?
+ // TODO: nametmp is just the name so this will never work. but do we want it to? this looks like
+ // TODO: an unsafe feature. We shouldn't be able to change what's in the config. we should
+ // TODO: be getting what we asked for, or exit the program with an error.
if (strstr(nametmp, "git+") && strstr(nametmp, "://")) {
char *xrepo = strstr(nametmp, "+");
if (xrepo) {
diff --git a/src/lib/delivery/delivery_build.c b/src/lib/delivery/delivery_build.c
index 2d891d2..c5093d4 100644
--- a/src/lib/delivery/delivery_build.c
+++ b/src/lib/delivery/delivery_build.c
@@ -173,6 +173,18 @@ struct StrList *delivery_build_wheels(struct Delivery *ctx) {
memset(outdir, 0, sizeof(outdir));
memset(cmd, 0, sizeof(outdir));
+ const int dep_status = check_python_package_dependencies(".");
+ if (dep_status) {
+ fprintf(stderr, "\nPlease replace all occurrences above with standard package specs:\n"
+ "\n"
+ " package==x.y.z\n"
+ " package>=x.y.z\n"
+ " package<=x.y.z\n"
+ " ...\n"
+ "\n");
+ COE_CHECK_ABORT(dep_status, "Unreproducible delivery");
+ }
+
strcpy(dname, ctx->tests[i].name);
tolower_s(dname);
sprintf(outdir, "%s/%s", ctx->storage.wheel_artifact_dir, dname);
diff --git a/src/lib/delivery/delivery_export.c b/src/lib/delivery/delivery_export.c
new file mode 100644
index 0000000..d982ad5
--- /dev/null
+++ b/src/lib/delivery/delivery_export.c
@@ -0,0 +1,58 @@
+#include "delivery.h"
+
+static void delivery_export_configuration(const struct Delivery *ctx) {
+ msg(STASIS_MSG_L2, "Exporting delivery configuration\n");
+ if (!pushd(ctx->storage.cfgdump_dir)) {
+ char filename[PATH_MAX] = {0};
+ sprintf(filename, "%s.ini", ctx->info.release_name);
+ FILE *spec = fopen(filename, "w+");
+ if (!spec) {
+ msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", filename);
+ exit(1);
+ }
+ ini_write(ctx->_stasis_ini_fp.delivery, &spec, INI_WRITE_RAW);
+ fclose(spec);
+
+ memset(filename, 0, sizeof(filename));
+ sprintf(filename, "%s-rendered.ini", ctx->info.release_name);
+ spec = fopen(filename, "w+");
+ if (!spec) {
+ msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", filename);
+ exit(1);
+ }
+ ini_write(ctx->_stasis_ini_fp.delivery, &spec, INI_WRITE_PRESERVE);
+ fclose(spec);
+ popd();
+ } else {
+ SYSERROR("Failed to enter directory: %s", ctx->storage.delivery_dir);
+ exit(1);
+ }
+}
+
+void delivery_export(const struct Delivery *ctx, char *envs[]) {
+ delivery_export_configuration(ctx);
+
+ for (size_t i = 0; envs[i] != NULL; i++) {
+ char *name = envs[i];
+ msg(STASIS_MSG_L2, "Exporting %s\n", name);
+ if (conda_env_export(name, ctx->storage.delivery_dir, name)) {
+ msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", name);
+ exit(1);
+ }
+ }
+}
+
+void delivery_rewrite_stage1(struct Delivery *ctx, char *specfile) {
+ // Rewrite release environment output (i.e. set package origin(s) to point to the deployment server, etc.)
+ msg(STASIS_MSG_L3, "Rewriting release spec file (stage 1): %s\n", path_basename(specfile));
+ delivery_rewrite_spec(ctx, specfile, DELIVERY_REWRITE_SPEC_STAGE_1);
+
+ msg(STASIS_MSG_L1, "Rendering mission templates\n");
+ delivery_mission_render_files(ctx);
+}
+
+void delivery_rewrite_stage2(struct Delivery *ctx, char *specfile) {
+ msg(STASIS_MSG_L3, "Rewriting release spec file (stage 2): %s\n", path_basename(specfile));
+ delivery_rewrite_spec(ctx, specfile, DELIVERY_REWRITE_SPEC_STAGE_2);
+}
+
diff --git a/src/lib/delivery/delivery_init.c b/src/lib/delivery/delivery_init.c
index 56c591a..a60d6af 100644
--- a/src/lib/delivery/delivery_init.c
+++ b/src/lib/delivery/delivery_init.c
@@ -287,18 +287,25 @@ int delivery_init(struct Delivery *ctx, int render_mode) {
int bootstrap_build_info(struct Delivery *ctx) {
struct Delivery local = {0};
+ SYSDEBUG("ini_open(%s)", ctx->_stasis_ini_fp.cfg_path);
local._stasis_ini_fp.cfg = ini_open(ctx->_stasis_ini_fp.cfg_path);
+ SYSDEBUG("ini_open(%s)", ctx->_stasis_ini_fp.delivery_path);
local._stasis_ini_fp.delivery = ini_open(ctx->_stasis_ini_fp.delivery_path);
+
if (delivery_init_platform(&local)) {
+ SYSDEBUG("%s", "delivery_init_platform failed");
return -1;
}
if (populate_delivery_cfg(&local, INI_READ_RENDER)) {
+ SYSDEBUG("%s", "populate_delivery_cfg failed");
return -1;
}
if (populate_delivery_ini(&local, INI_READ_RENDER)) {
+ SYSDEBUG("%s", "populate_delivery_ini failed");
return -1;
}
if (populate_info(&local)) {
+ SYSDEBUG("%s", "populate_info failed");
return -1;
}
ctx->info.build_name = strdup(local.info.build_name);
@@ -314,6 +321,7 @@ int bootstrap_build_info(struct Delivery *ctx) {
memcpy(ctx->info.time_info, local.info.time_info, sizeof(*local.info.time_info));
ctx->info.time_now = local.info.time_now;
ctx->info.time_str_epoch = strdup(local.info.time_str_epoch);
+ SYSDEBUG("%s", "delivery_free local resources");
delivery_free(&local);
return 0;
}
diff --git a/src/lib/delivery/delivery_install.c b/src/lib/delivery/delivery_install.c
index 246c604..f1637a3 100644
--- a/src/lib/delivery/delivery_install.c
+++ b/src/lib/delivery/delivery_install.c
@@ -36,7 +36,7 @@ static char *have_spec_in_config(const struct Delivery *ctx, const char *name) {
strncpy(package, config_spec, sizeof(package) - 1);
}
remove_extras(package);
- if (strncmp(package, name, strlen(package)) == 0) {
+ if (strncmp(package, name, strlen(name)) == 0) {
return config_spec;
}
}
@@ -189,8 +189,7 @@ int delivery_purge_packages(struct Delivery *ctx, const char *env_name, int use_
}
int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, char *env_name, int type, struct StrList **manifest) {
- char cmd[PATH_MAX];
- char pkgs[STASIS_BUFSIZ];
+ char command_base[PATH_MAX];
const char *env_current = getenv("CONDA_DEFAULT_ENV");
if (env_current) {
@@ -203,9 +202,8 @@ int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, cha
}
}
- memset(cmd, 0, sizeof(cmd));
- memset(pkgs, 0, sizeof(pkgs));
- strcat(cmd, "install");
+ memset(command_base, 0, sizeof(command_base));
+ strcat(command_base, "install");
typedef int (*Runner)(const char *);
Runner runner = NULL;
@@ -216,17 +214,23 @@ int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, cha
}
if (INSTALL_PKG_CONDA_DEFERRED & type) {
- strcat(cmd, " --use-local");
+ strcat(command_base, " --use-local");
} else if (INSTALL_PKG_PIP_DEFERRED & type) {
// Don't change the baseline package set unless we're working with a
// new build. Release candidates will need to keep packages as stable
// as possible between releases.
if (!ctx->meta.based_on) {
- strcat(cmd, " --upgrade");
+ strcat(command_base, " --upgrade");
}
+ sprintf(command_base + strlen(command_base), " --extra-index-url 'file://%s'", ctx->storage.wheel_artifact_dir);
}
- sprintf(cmd + strlen(cmd), " --extra-index-url 'file://%s'", ctx->storage.wheel_artifact_dir);
+ size_t args_alloc_len = STASIS_BUFSIZ;
+ char *args = calloc(args_alloc_len + 1, sizeof(*args));
+ if (!args) {
+ SYSERROR("%s", "Unable to allocate bytes for command arguments");
+ return -1;
+ }
for (size_t x = 0; manifest[x] != NULL; x++) {
char *name = NULL;
@@ -239,10 +243,11 @@ int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, cha
if (INSTALL_PKG_PIP_DEFERRED & type) {
struct Test *info = requirement_from_test(ctx, name);
if (info) {
- if (!strcmp(info->version, "HEAD")) {
+ if (!strcmp(info->version, "HEAD") || is_git_sha(info->version)) {
struct StrList *tag_data = strlist_init();
if (!tag_data) {
SYSERROR("%s", "Unable to allocate memory for tag data\n");
+ guard_free(args);
return -1;
}
strlist_append_tokenize(tag_data, info->repository_info_tag, "-");
@@ -273,9 +278,9 @@ int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, cha
fprintf(stderr, "No wheel packages found that match the description of '%s'", info->name);
} else {
// found
- guard_strlist_free(&tag_data);
info->version = strdup(whl->version);
}
+ guard_strlist_free(&tag_data);
wheel_free(&whl);
}
@@ -290,26 +295,66 @@ int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, cha
}
}
- snprintf(cmd + strlen(cmd),
- sizeof(cmd) - strlen(cmd) - strlen(info->name) - strlen(info->version) + 5,
- " '%s==%s'", req, info->version);
+ const char *fmt_append = "%s '%s==%s'";
+ const char *fmt = " '%s==%s'";
+ const int required_len = snprintf(NULL, 0, fmt_append, args, req, info->version);
+ if (required_len > (int) args_alloc_len) {
+ if (grow(required_len, &args_alloc_len, &args)) {
+ SYSERROR("Unable to allocate %d bytes for command arguments", required_len);
+ guard_free(args);
+ return -1;
+ }
+ }
+ snprintf(args + strlen(args), required_len + 1, fmt, req, info->version);
} else {
fprintf(stderr, "Deferred package '%s' is not present in the tested package list!\n", name);
+ guard_free(args);
return -1;
}
} else {
if (startswith(name, "--") || startswith(name, "-")) {
- sprintf(cmd + strlen(cmd), " %s", name);
+ const char *fmt_append = "%s %s";
+ const char *fmt = " %s";
+ const int required_len = snprintf(NULL, 0, fmt_append, args, name);
+ if (required_len > (int) args_alloc_len) {
+ if (grow(required_len, &args_alloc_len, &args)) {
+ SYSERROR("Unable to allocate %d bytes for command arguments", required_len);
+ guard_free(args);
+ return -1;
+ }
+ }
+ snprintf(args + strlen(args), required_len + 1, fmt, name);
} else {
- sprintf(cmd + strlen(cmd), " '%s'", name);
+ const char *fmt_append = "%s '%s'";
+ const char *fmt = " '%s'";
+ const int required_len = snprintf(NULL, 0, fmt_append, args, name);
+ if (required_len > (int) args_alloc_len) {
+ if (grow(required_len, &args_alloc_len, &args)) {
+ SYSERROR("Unable to allocate %d bytes for command arguments", required_len);
+ guard_free(args);
+ return -1;
+ }
+ }
+ snprintf(args + strlen(args), required_len + 1, fmt, name);
}
}
}
- int status = runner(cmd);
+ char *command = NULL;
+ if (asprintf(&command, "%s %s", command_base, args) < 0) {
+ SYSERROR("%s", "Unable to allocate bytes for command\n");
+ guard_free(args);
+ return -1;
+ }
+
+ int status = runner(command);
+ guard_free(args);
+ guard_free(command);
if (status) {
+ // fail quickly
return status;
}
}
+ guard_free(args);
return 0;
}
diff --git a/src/lib/delivery/delivery_populate.c b/src/lib/delivery/delivery_populate.c
index 84676f1..28b2480 100644
--- a/src/lib/delivery/delivery_populate.c
+++ b/src/lib/delivery/delivery_populate.c
@@ -55,6 +55,7 @@ int populate_info(struct Delivery *ctx) {
int populate_delivery_cfg(struct Delivery *ctx, int render_mode) {
struct INIFILE *cfg = ctx->_stasis_ini_fp.cfg;
if (!cfg) {
+ SYSDEBUG("%s", "cfg is NULL");
return -1;
}
int err = 0;
@@ -162,8 +163,6 @@ int populate_delivery_ini(struct Delivery *ctx, int render_mode) {
// keys in the configuration
RuntimeEnv *rt = runtime_copy(__environ);
while ((rtdata = ini_getall(ini, "runtime")) != NULL) {
- char rec[STASIS_BUFSIZ];
- sprintf(rec, "%s=%s", lstrip(strip(rtdata->key)), lstrip(strip(rtdata->value)));
runtime_set(rt, rtdata->key, rtdata->value);
}
runtime_apply(rt);
diff --git a/src/lib/delivery/delivery_test.c b/src/lib/delivery/delivery_test.c
index e80e0ec..6e0a226 100644
--- a/src/lib/delivery/delivery_test.c
+++ b/src/lib/delivery/delivery_test.c
@@ -97,6 +97,18 @@ void delivery_tests_run(struct Delivery *ctx) {
if (pushd(destdir)) {
COE_CHECK_ABORT(1, "Unable to enter repository directory\n");
} else {
+ int dep_status = check_python_package_dependencies(".");
+ if (dep_status) {
+ fprintf(stderr, "\nPlease replace all occurrences above with standard package specs:\n"
+ "\n"
+ " package==x.y.z\n"
+ " package>=x.y.z\n"
+ " package<=x.y.z\n"
+ " ...\n"
+ "\n");
+ COE_CHECK_ABORT(dep_status, "Unreproducible delivery");
+ }
+
char *cmd = calloc(strlen(test->script) + STASIS_BUFSIZ, sizeof(*cmd));
if (!cmd) {
SYSERROR("Unable to allocate test script buffer: %s", strerror(errno));
diff --git a/src/lib/delivery/include/delivery.h b/src/lib/delivery/include/delivery.h
index 26a5499..69ec089 100644
--- a/src/lib/delivery/include/delivery.h
+++ b/src/lib/delivery/include/delivery.h
@@ -459,4 +459,28 @@ int delivery_series_sync(struct Delivery *ctx);
*/
int delivery_purge_packages(struct Delivery *ctx, const char *env_name, int use_pkg_manager);
+/**
+ * Export delivery environments
+ *
+ * @param ctx Delivery context
+ * @param envs array of conda environment names
+ */
+void delivery_export(const struct Delivery *ctx, char *envs[]);
+
+/**
+ * STAGE 1: Rewrite delivery-related strings in specfile
+ *
+ * @param ctx Delivery context
+ * @param specfile path to YAML spec file
+ */
+void delivery_rewrite_stage1(struct Delivery *ctx, char *specfile);
+
+/**
+ * STAGE 2: Rewrite delivery-related strings in specfile
+ *
+ * @param ctx Delivery context
+ * @param specfile path to YAML spec file
+ */
+void delivery_rewrite_stage2(struct Delivery *ctx, char *specfile);
+
#endif //STASIS_DELIVERY_H
diff --git a/tests/setup.sh b/tests/setup.sh
index 7e38cf9..bce2fbd 100644
--- a/tests/setup.sh
+++ b/tests/setup.sh
@@ -78,7 +78,7 @@ teardown_workspace() {
install_stasis() {
pushd "$BUILD_DIR"
- if ! cmake -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -DCMAKE_BUILD_TYPE=Debug "${TOPDIR}"/../..; then
+ if ! cmake -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -DCMAKE_BUILD_TYPE=Debug -DDEBUG_MESSAGES=ON "${TOPDIR}"/../..; then
echo "cmake failed" >&2
return 1
fi
@@ -109,7 +109,7 @@ STASIS_TEST_RESULT_SKIP=0
run_command() {
local logfile="$(mktemp).log"
local cmd="${@}"
- local lines_on_error=100
+ local lines_on_error=1000
/bin/echo "Testing: $cmd "
$cmd &>"$logfile"
diff --git a/tests/test_artifactory.c b/tests/test_artifactory.c
index 4af7eec..202a67c 100644
--- a/tests/test_artifactory.c
+++ b/tests/test_artifactory.c
@@ -116,6 +116,7 @@ int main(int argc, char *argv[]) {
// Skip this suite if we're not configured to use it
if (jfrt_auth_init(&gauth)) {
SYSERROR("%s", "Not configured to test Artifactory. Skipping.");
+ guard_free(basedir);
return STASIS_TEST_SUITE_SKIP;
}
guard_free(basedir);
diff --git a/tests/test_ini.c b/tests/test_ini.c
index e4a7808..af47ddf 100644
--- a/tests/test_ini.c
+++ b/tests/test_ini.c
@@ -195,6 +195,23 @@ void test_ini_getval_wrappers() {
ini_free(&ini);
}
+void test_ini_getall() {
+ const char *filename = "ini_open.ini";
+ struct INIFILE *ini = NULL;
+ const char *data = "[default]\nhello=world!\nthis=is a test\nx=1\ny=0\n";
+
+ stasis_testing_write_ascii(filename, data);
+ ini = ini_open(filename);
+ STASIS_ASSERT_FATAL(ini != NULL, "failed to open ini file");
+
+ const struct INIData *d = NULL;
+ for (size_t i = 0; (d = ini_getall(ini, "default")) != NULL; i++) {
+ STASIS_ASSERT(d->key != NULL, "INIData key should not be NULL");
+ STASIS_ASSERT(d->value != NULL, "INIData key should not be NULL");
+ }
+ ini_free(&ini);
+}
+
int main(int argc, char *argv[]) {
STASIS_TEST_BEGIN_MAIN();
STASIS_TEST_FUNC *tests[] = {
@@ -204,6 +221,7 @@ int main(int argc, char *argv[]) {
test_ini_has_key,
test_ini_setval_getval,
test_ini_getval_wrappers,
+ test_ini_getall,
};
STASIS_TEST_RUN(tests);
STASIS_TEST_END_MAIN();
diff --git a/tests/test_str.c b/tests/test_str.c
index ad0c07a..a98a34d 100644
--- a/tests/test_str.c
+++ b/tests/test_str.c
@@ -6,16 +6,17 @@ void test_to_short_version() {
const char *expected;
};
- struct testcase tc[] = {
- {.data = "1.2.3", .expected = "123"},
+ const struct testcase tc[] = {
+ {.data = "1.2.3", .expected = "12"},
{.data = "py3.12", .expected = "py312"},
- {.data = "generic-1.2.3", .expected = "generic-123"},
+ {.data = "generic-1.2.3", .expected = "generic-12"},
{.data = "nothing to do", .expected = "nothing to do"},
};
for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) {
char *result = to_short_version(tc[i].data);
STASIS_ASSERT_FATAL(result != NULL, "should not be NULL");
+ //printf("%s[%zu], result: %s, expected: %s\n", __FUNCTION__, i, result, tc[i].expected);
STASIS_ASSERT(strcmp(result, tc[i].expected) == 0, "unexpected result");
guard_free(result);
}