diff options
| -rw-r--r-- | src/cli/stasis/stasis_main.c | 725 | ||||
| -rw-r--r-- | src/lib/core/conda.c | 106 | ||||
| -rw-r--r-- | src/lib/core/download.c | 6 | ||||
| -rw-r--r-- | src/lib/core/environment.c | 50 | ||||
| -rw-r--r-- | src/lib/core/include/conda.h | 2 | ||||
| -rw-r--r-- | src/lib/core/include/core.h | 2 | ||||
| -rw-r--r-- | src/lib/core/include/str.h | 27 | ||||
| -rw-r--r-- | src/lib/core/include/utils.h | 45 | ||||
| -rw-r--r-- | src/lib/core/str.c | 31 | ||||
| -rw-r--r-- | src/lib/core/template.c | 21 | ||||
| -rw-r--r-- | src/lib/core/utils.c | 143 | ||||
| -rw-r--r-- | src/lib/delivery/CMakeLists.txt | 1 | ||||
| -rw-r--r-- | src/lib/delivery/delivery.c | 13 | ||||
| -rw-r--r-- | src/lib/delivery/delivery_build.c | 12 | ||||
| -rw-r--r-- | src/lib/delivery/delivery_export.c | 58 | ||||
| -rw-r--r-- | src/lib/delivery/delivery_init.c | 8 | ||||
| -rw-r--r-- | src/lib/delivery/delivery_install.c | 79 | ||||
| -rw-r--r-- | src/lib/delivery/delivery_populate.c | 3 | ||||
| -rw-r--r-- | src/lib/delivery/delivery_test.c | 12 | ||||
| -rw-r--r-- | src/lib/delivery/include/delivery.h | 24 | ||||
| -rw-r--r-- | tests/setup.sh | 4 | ||||
| -rw-r--r-- | tests/test_artifactory.c | 1 | ||||
| -rw-r--r-- | tests/test_ini.c | 18 | ||||
| -rw-r--r-- | tests/test_str.c | 7 |
24 files changed, 966 insertions, 432 deletions
diff --git a/src/cli/stasis/stasis_main.c b/src/cli/stasis/stasis_main.c index 459a6f6..2ce6831 100644 --- a/src/cli/stasis/stasis_main.c +++ b/src/cli/stasis/stasis_main.c @@ -10,140 +10,7 @@ #include "system_requirements.h" #include "tpl.h" - -int main(int argc, char *argv[]) { - struct Delivery ctx; - struct Process proc = { - .f_stdout = "", - .f_stderr = "", - .redirect_stderr = 0, - }; - char env_name[STASIS_NAME_MAX] = {0}; - char env_name_testing[STASIS_NAME_MAX] = {0}; - char *delivery_input = NULL; - char *config_input = NULL; - char installer_url[PATH_MAX]; - char python_override_version[STASIS_NAME_MAX]; - int user_disabled_docker = false; - globals.cpu_limit = get_cpu_count(); - if (globals.cpu_limit > 1) { - globals.cpu_limit--; // max - 1 - } - - memset(env_name, 0, sizeof(env_name)); - memset(env_name_testing, 0, sizeof(env_name_testing)); - memset(installer_url, 0, sizeof(installer_url)); - memset(python_override_version, 0, sizeof(python_override_version)); - memset(&proc, 0, sizeof(proc)); - memset(&ctx, 0, sizeof(ctx)); - - int c; - int option_index = 0; - while ((c = getopt_long(argc, argv, "hVCc:p:vU", long_options, &option_index)) != -1) { - switch (c) { - case 'h': - usage(path_basename(argv[0])); - exit(0); - case 'V': - puts(VERSION); - exit(0); - case 'c': - config_input = strdup(optarg); - break; - case 'C': - globals.continue_on_error = true; - break; - case 'p': - strcpy(python_override_version, optarg); - break; - case 'l': - globals.cpu_limit = strtol(optarg, NULL, 10); - if (globals.cpu_limit <= 1) { - globals.cpu_limit = 1; - globals.enable_parallel = false; // No point - } - break; - case OPT_ALWAYS_UPDATE_BASE: - globals.always_update_base_environment = true; - break; - case OPT_FAIL_FAST: - globals.parallel_fail_fast = true; - break; - case OPT_POOL_STATUS_INTERVAL: - globals.pool_status_interval = (int) strtol(optarg, NULL, 10); - if (globals.pool_status_interval < 1) { - globals.pool_status_interval = 1; - } else if (globals.pool_status_interval > 60 * 10) { - // Possible poor choice alert - fprintf(stderr, "Caution: Excessive pausing between status updates may cause third-party CI/CD" - " jobs to fail if the stdout/stderr streams are idle for too long!\n"); - } - break; - case 'U': - setenv("PYTHONUNBUFFERED", "1", 1); - fflush(stdout); - fflush(stderr); - setvbuf(stdout, NULL, _IONBF, 0); - setvbuf(stderr, NULL, _IONBF, 0); - break; - case 'v': - globals.verbose = true; - break; - case OPT_OVERWRITE: - globals.enable_overwrite = true; - break; - case OPT_NO_DOCKER: - globals.enable_docker = false; - user_disabled_docker = true; - break; - case OPT_NO_ARTIFACTORY: - globals.enable_artifactory = false; - break; - case OPT_NO_ARTIFACTORY_BUILD_INFO: - globals.enable_artifactory_build_info = false; - break; - case OPT_NO_ARTIFACTORY_UPLOAD: - globals.enable_artifactory_build_info = false; - globals.enable_artifactory_upload = false; - break; - case OPT_NO_TESTING: - globals.enable_testing = false; - break; - case OPT_NO_REWRITE_SPEC_STAGE_2: - globals.enable_rewrite_spec_stage_2 = false; - break; - case OPT_NO_PARALLEL: - globals.enable_parallel = false; - break; - case '?': - default: - exit(1); - } - } - - if (optind < argc) { - while (optind < argc) { - // use first positional argument - delivery_input = argv[optind++]; - break; - } - } - - if (!delivery_input) { - fprintf(stderr, "error: a DELIVERY_FILE is required\n"); - usage(path_basename(argv[0])); - exit(1); - } - - printf(BANNER, VERSION, AUTHOR); - - check_system_path(); - - msg(STASIS_MSG_L1, "Setup\n"); - - tpl_setup_vars(&ctx); - tpl_setup_funcs(&ctx); - +static void setup_sysconfdir() { // Set up PREFIX/etc directory information // The user may manipulate the base directory path with STASIS_SYSCONFDIR // environment variable @@ -159,83 +26,108 @@ int main(int argc, char *argv[]) { msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Unable to resolve path to configuration directory: %s\n", stasis_sysconfdir_tmp); exit(1); } +} +static void setup_python_version_override(struct Delivery *ctx, const char *version) { // Override Python version from command-line, if any - if (strlen(python_override_version)) { - guard_free(ctx.meta.python); - ctx.meta.python = strdup(python_override_version); - guard_free(ctx.meta.python_compact); - ctx.meta.python_compact = to_short_version(ctx.meta.python); + if (strlen(version)) { + guard_free(ctx->meta.python); + ctx->meta.python = strdup(version); + if (!ctx->meta.python) { + SYSERROR("%s", "Unable to allocate bytes for python version override"); + } + guard_free(ctx->meta.python_compact); + ctx->meta.python_compact = to_short_version(ctx->meta.python); } +} - if (!config_input) { - // no configuration passed by argument. use basic config. +static void configure_stasis_ini(struct Delivery *ctx, char **config_input) { + if (!*config_input) { + SYSDEBUG("%s", "No configuration passed by argument. Using basic config."); char cfgfile[PATH_MAX * 2]; sprintf(cfgfile, "%s/%s", globals.sysconfdir, "stasis.ini"); + SYSDEBUG("cfgfile: %s", cfgfile); if (!access(cfgfile, F_OK | R_OK)) { - config_input = strdup(cfgfile); + *config_input = strdup(cfgfile); } else { msg(STASIS_MSG_WARN, "STASIS global configuration is not readable, or does not exist: %s", cfgfile); } } - if (config_input) { - msg(STASIS_MSG_L2, "Reading STASIS global configuration: %s\n", config_input); - ctx._stasis_ini_fp.cfg = ini_open(config_input); - if (!ctx._stasis_ini_fp.cfg) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read config file: %s, %s\n", delivery_input, strerror(errno)); - exit(1); - } - ctx._stasis_ini_fp.cfg_path = strdup(config_input); - guard_free(config_input); + msg(STASIS_MSG_L2, "Reading STASIS global configuration: %s\n", *config_input); + ctx->_stasis_ini_fp.cfg = ini_open(*config_input); + if (!ctx->_stasis_ini_fp.cfg) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read config file: %s, %s\n", *config_input, strerror(errno)); + exit(1); + } + ctx->_stasis_ini_fp.cfg_path = strdup(*config_input); + if (!ctx->_stasis_ini_fp.cfg_path) { + SYSERROR("%s", "Failed to allocate memory for config file name"); + exit(1); } + guard_free(*config_input); +} - msg(STASIS_MSG_L2, "Reading STASIS delivery configuration: %s\n", delivery_input); - ctx._stasis_ini_fp.delivery = ini_open(delivery_input); - if (!ctx._stasis_ini_fp.delivery) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read delivery file: %s, %s\n", delivery_input, strerror(errno)); +static void configure_delivery_ini(struct Delivery *ctx, char **delivery_input) { + msg(STASIS_MSG_L2, "Reading STASIS delivery configuration: %s\n", *delivery_input); + ctx->_stasis_ini_fp.delivery = ini_open(*delivery_input); + if (!ctx->_stasis_ini_fp.delivery) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read delivery file: %s, %s\n", *delivery_input, strerror(errno)); exit(1); } - ctx._stasis_ini_fp.delivery_path = strdup(delivery_input); + ctx->_stasis_ini_fp.delivery_path = strdup(*delivery_input); +} +static void configure_delivery_context(struct Delivery *ctx) { msg(STASIS_MSG_L2, "Bootstrapping delivery context\n"); - if (bootstrap_build_info(&ctx)) { + if (bootstrap_build_info(ctx)) { msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to bootstrap delivery context\n"); exit(1); } msg(STASIS_MSG_L2, "Initializing delivery context\n"); - if (delivery_init(&ctx, INI_READ_RENDER)) { + if (delivery_init(ctx, INI_READ_RENDER)) { msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to initialize delivery context\n"); exit(1); } - check_requirements(&ctx); +} +static void configure_jfrog_cli(struct Delivery *ctx) { msg(STASIS_MSG_L2, "Configuring JFrog CLI\n"); - if (delivery_init_artifactory(&ctx)) { + if (delivery_init_artifactory(ctx)) { msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "JFrog CLI configuration failed\n"); exit(1); } +} - runtime_apply(ctx.runtime.environ); - strcpy(env_name, ctx.info.release_name); - strcpy(env_name_testing, env_name); - strcat(env_name_testing, "-test"); - +static void check_release_history(struct Delivery *ctx) { // Safety gate: Avoid clobbering a delivered release unless the user wants that behavior msg(STASIS_MSG_L1, "Checking release history\n"); - if (!globals.enable_overwrite && delivery_exists(&ctx) == DELIVERY_FOUND) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Refusing to overwrite release: %s\nUse --overwrite to enable release clobbering.\n", ctx.info.release_name); + if (!globals.enable_overwrite && delivery_exists(ctx) == DELIVERY_FOUND) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Refusing to overwrite release: %s\nUse --overwrite to enable release clobbering.\n", ctx->info.release_name); exit(1); } +} + +static void check_conda_install_prefix(const struct Delivery *ctx) { + // Unlikely to occur: this should help prevent rmtree() from destroying your entire filesystem + // if path is "/" then, die + // or if empty string, die + if (!strcmp(ctx->storage.conda_install_prefix, DIR_SEP) || !strlen(ctx->storage.conda_install_prefix)) { + fprintf(stderr, "error: ctx.storage.conda_install_prefix is malformed!\n"); + exit(1); + } +} + +static void sync_release_history(struct Delivery *ctx) { if (globals.enable_artifactory) { // We need to download previous revisions to ensure processed packages are available at build-time // This is also a docker requirement. Python wheels must exist locally. - if (ctx.meta.rc > 1) { - msg(STASIS_MSG_L1, "Syncing delivery artifacts for %s\n", ctx.info.build_name); - if (delivery_series_sync(&ctx) != 0) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Unable to sync artifacts for %s\n", ctx.info.build_name); + if (ctx->meta.rc > 1) { + msg(STASIS_MSG_L1, "Syncing delivery artifacts for %s\n", ctx->info.build_name); + if (delivery_series_sync(ctx) != 0) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Unable to sync artifacts for %s\n", ctx->info.build_name); msg(STASIS_MSG_L3, "Case #1:\n" "\tIf this is a new 'version', and 'rc' is greater " "than 1, then no previous deliveries exist remotely. " @@ -248,45 +140,43 @@ int main(int argc, char *argv[]) { } } } +} - // Unlikely to occur: this should help prevent rmtree() from destroying your entire filesystem - // if path is "/" then, die - // or if empty string, die - if (!strcmp(ctx.storage.conda_install_prefix, DIR_SEP) || !strlen(ctx.storage.conda_install_prefix)) { - fprintf(stderr, "error: ctx.storage.conda_install_prefix is malformed!\n"); - exit(1); - } - +static void check_conda_prefix_length(const struct Delivery *ctx) { // 2 = #! // 5 = /bin\n - const size_t prefix_len = strlen(ctx.storage.conda_install_prefix) + 2 + 5; + const size_t prefix_len = strlen(ctx->storage.conda_install_prefix) + 2 + 5; const size_t prefix_len_max = 127; msg(STASIS_MSG_L1, "Checking length of conda installation prefix\n"); - if (!strcmp(ctx.system.platform[DELIVERY_PLATFORM], "Linux") && prefix_len > prefix_len_max) { + if (!strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Linux") && prefix_len > prefix_len_max) { msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "The shebang, '#!%s/bin/python\\n' is too long (%zu > %zu).\n", - ctx.storage.conda_install_prefix, prefix_len, prefix_len_max); + ctx->storage.conda_install_prefix, prefix_len, prefix_len_max); msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "Conda's workaround to handle long path names does not work consistently within STASIS.\n"); msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "Please try again from a different, \"shorter\", directory.\n"); exit(1); } +} +static void setup_conda(struct Delivery *ctx, char *installer_url) { msg(STASIS_MSG_L1, "Conda setup\n"); - delivery_get_conda_installer_url(&ctx, installer_url); + delivery_get_conda_installer_url(ctx, installer_url); msg(STASIS_MSG_L2, "Downloading: %s\n", installer_url); - if (delivery_get_conda_installer(&ctx, installer_url)) { + if (delivery_get_conda_installer(ctx, installer_url)) { msg(STASIS_MSG_ERROR, "download failed: %s\n", installer_url); exit(1); } - msg(STASIS_MSG_L2, "Installing: %s\n", ctx.conda.installer_name); - delivery_install_conda(ctx.conda.installer_path, ctx.storage.conda_install_prefix); + msg(STASIS_MSG_L2, "Installing: %s\n", ctx->conda.installer_name); + delivery_install_conda(ctx->conda.installer_path, ctx->storage.conda_install_prefix); - msg(STASIS_MSG_L2, "Configuring: %s\n", ctx.storage.conda_install_prefix); - delivery_conda_enable(&ctx, ctx.storage.conda_install_prefix); + msg(STASIS_MSG_L2, "Configuring: %s\n", ctx->storage.conda_install_prefix); + delivery_conda_enable(ctx, ctx->storage.conda_install_prefix); +} +static void configure_conda_base(struct Delivery *ctx, char *envs[]) { // // Implied environment creation modes/actions // @@ -303,269 +193,247 @@ int main(int argc, char *argv[]) { // 3b. Bugs, conflicts, and dependency resolution issues are inherited and // must be handled in the INI config msg(STASIS_MSG_L1, "Creating release environment(s)\n"); - char *mission_base = NULL; - if (isempty(ctx.meta.based_on)) { - guard_free(ctx.meta.based_on); + + if (isempty(ctx->meta.based_on)) { + // based_on was not set by the input file + + guard_free(ctx->meta.based_on); char *mission_base_orig = NULL; - if (asprintf(&mission_base_orig, "%s/%s/base.yml", ctx.storage.mission_dir, ctx.meta.mission) < 0) { - SYSERROR("Unable to allocate bytes for %s/%s/base.yml path\n", ctx.storage.mission_dir, ctx.meta.mission); + if (asprintf(&mission_base_orig, "%s/%s/base.yml", ctx->storage.mission_dir, ctx->meta.mission) < 0) { + SYSERROR("Unable to allocate bytes for %s/%s/base.yml path\n", ctx->storage.mission_dir, ctx->meta.mission); exit(1); } + // Does a base.yml exist in the mission directory? + // If not, do nothing. Otherwise, use the base.yml in the mission directory. if (access(mission_base_orig, F_OK) < 0) { - msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "Mission does not provide a base.yml configuration: %s (%s)\n", - ctx.meta.mission, ctx.storage.mission_dir); + msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "Mission does not provide a base.yml"); } else { msg(STASIS_MSG_L2, "Using base environment configuration: %s\n", mission_base_orig); - if (asprintf(&mission_base, "%s/%s-base.yml", ctx.storage.tmpdir, ctx.info.release_name) < 0) { + if (asprintf(&mission_base, "%s/%s-base.yml", ctx->storage.tmpdir, ctx->info.release_name) < 0) { SYSERROR("%s", "Unable to allocate bytes for temporary base.yml configuration"); remove(mission_base); exit(1); } copy2(mission_base_orig, mission_base, CT_OWNER | CT_PERM); - ctx.meta.based_on = mission_base; + ctx->meta.based_on = mission_base; } guard_free(mission_base_orig); } - if (!isempty(ctx.meta.based_on)) { - if (conda_env_exists(ctx.storage.conda_install_prefix, env_name) && conda_env_remove(env_name)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to remove release environment: %s\n", env_name); - exit(1); - } + msg(STASIS_MSG_L2, "Based on: %s\n", ctx->meta.based_on); - msg(STASIS_MSG_L2, "Based on: %s\n", ctx.meta.based_on); - if (conda_env_create_from_uri(env_name, ctx.meta.based_on, ctx.meta.python)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "unable to install release environment using configuration file\n"); - exit(1); - } + for (size_t i = 0; envs[i] != NULL; i += 2) { + char *title = envs[i]; + char *env = envs[i+1]; + // If based_on was populated above, or defined in the configuration: install its packages. + if (!isempty(ctx->meta.based_on)) { + if (conda_env_exists(ctx->storage.conda_install_prefix, env) && conda_env_remove(env)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to remove %s environment: %s\n", title); + exit(1); + } - if (conda_env_exists(ctx.storage.conda_install_prefix, env_name_testing) && conda_env_remove(env_name_testing)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to remove testing environment %s\n", env_name_testing); - exit(1); - } - if (conda_env_create_from_uri(env_name_testing, ctx.meta.based_on, ctx.meta.python)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "unable to install testing environment using configuration file\n"); - exit(1); - } - } else { - if (conda_env_create(env_name, ctx.meta.python, NULL)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to create release environment\n"); - exit(1); - } - if (conda_env_create(env_name_testing, ctx.meta.python, NULL)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to create testing environment\n"); - exit(1); + if (conda_env_create_from_uri(env, ctx->meta.based_on, ctx->meta.python)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "unable to install %s environment using configuration file\n", title); + exit(1); + } + } else { + // Otherwise, create the environments with the requested Python version and move on + if (conda_env_create(env, ctx->meta.python, NULL)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to create %s environment\n", title); + exit(1); + } } } // The base environment configuration not used past this point remove(mission_base); +} - const char *envs[] = {env_name_testing, env_name}; - for (size_t e = 0; e < sizeof(envs) / sizeof(*envs); e++) { - const char *name = envs[e]; - if (ctx.conda.conda_packages_purge && strlist_count(ctx.conda.conda_packages_purge)) { - msg(STASIS_MSG_L2, "Purging conda packages from %s\n", name); - if (delivery_purge_packages(&ctx, name, PKG_USE_CONDA)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "unable to purge requested conda packages from %s\n", name); - exit(1); - } - } - - if (ctx.conda.pip_packages_purge && strlist_count(ctx.conda.pip_packages_purge)) { - msg(STASIS_MSG_L2, "Purging pip packages from %s\n", name); - if (delivery_purge_packages(&ctx, env_name_testing, PKG_USE_PIP)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "unable to purge requested pip packages from %s\n", - env_name_testing); - exit(1); +static void configure_conda_purge(struct Delivery *ctx, char *envs[]) { + struct StrList *purge_list[] = { + ctx->conda.conda_packages_purge, + ctx->conda.pip_packages_purge + }; + for (size_t i = 0; i < sizeof(purge_list) / sizeof(purge_list[0]); i++) { + struct StrList *to_purge = purge_list[i]; + for (size_t e = 0; envs[e] != NULL; e += 2) { + //const char *title = envs[e]; // unused + const char *env = envs[e+1]; + if (to_purge && strlist_count(to_purge)) { + const char *pkg_manager_name[] = { + "conda", + "pip" + }; + const int pkg_manager_use[] = { + PKG_USE_CONDA, + PKG_USE_PIP + }; + const char *manager_str = pkg_manager_name[i]; + const int manager_flag = pkg_manager_use[i]; + msg(STASIS_MSG_L2, "Purging %s packages from %s\n", manager_str, env); + if (delivery_purge_packages(ctx, env, manager_flag)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "unable to purge requested %s packages from %s\n", manager_str, env); + exit(1); + } } } } +} +static void setup_activate_test_env(const struct Delivery *ctx, const char *env_name_testing) { // Activate test environment msg(STASIS_MSG_L1, "Activating test environment\n"); - if (conda_activate(ctx.storage.conda_install_prefix, env_name_testing)) { + if (conda_activate(ctx->storage.conda_install_prefix, env_name_testing)) { fprintf(stderr, "failed to activate test environment\n"); exit(1); } +} - if (delivery_gather_tool_versions(&ctx)) { - if (!ctx.conda.tool_version) { +static void configure_tool_versions(struct Delivery *ctx) { + if (delivery_gather_tool_versions(ctx)) { + if (!ctx->conda.tool_version) { msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Could not determine conda version\n"); exit(1); } - if (!ctx.conda.tool_build_version) { + if (!ctx->conda.tool_build_version) { msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Could not determine conda-build version\n"); exit(1); } } +} +static void install_build_package() { if (pip_exec("install build")) { msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "'build' tool installation failed\n"); exit(1); } +} - if (!isempty(ctx.meta.based_on)) { +static void configure_package_overlay(struct Delivery *ctx, const char *env_name) { + if (!isempty(ctx->meta.based_on)) { msg(STASIS_MSG_L1, "Generating package overlay from environment: %s\n", env_name); - if (delivery_overlay_packages_from_env(&ctx, env_name)) { + if (delivery_overlay_packages_from_env(ctx, env_name)) { msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "%s", "Failed to generate package overlay. Resulting environment integrity cannot be guaranteed.\n"); exit(1); } } +} +static void configure_deferred_packages(struct Delivery *ctx) { msg(STASIS_MSG_L1, "Filter deliverable packages\n"); - delivery_defer_packages(&ctx, DEFER_CONDA); - delivery_defer_packages(&ctx, DEFER_PIP); + delivery_defer_packages(ctx, DEFER_CONDA); + delivery_defer_packages(ctx, DEFER_PIP); +} + +static void show_overiew(struct Delivery *ctx) { msg(STASIS_MSG_L1, "Overview\n"); - delivery_meta_show(&ctx); - delivery_conda_show(&ctx); + delivery_meta_show(ctx); + delivery_conda_show(ctx); if (globals.verbose) { //delivery_runtime_show(&ctx); } +} +static void run_tests(struct Delivery *ctx) { // Execute configuration-defined tests if (globals.enable_testing) { - delivery_tests_show(&ctx); + delivery_tests_show(ctx); msg(STASIS_MSG_L1, "Begin test execution\n"); - delivery_tests_run(&ctx); + delivery_tests_run(ctx); msg(STASIS_MSG_L2, "Rewriting test results\n"); - delivery_fixup_test_results(&ctx); + delivery_fixup_test_results(ctx); } else { msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Test execution is disabled\n"); } +} - if (ctx.conda.conda_packages_defer && strlist_count(ctx.conda.conda_packages_defer)) { +static void build_conda_recipes(struct Delivery *ctx) { + if (ctx->conda.conda_packages_defer && strlist_count(ctx->conda.conda_packages_defer)) { msg(STASIS_MSG_L2, "Building Conda recipe(s)\n"); - if (delivery_build_recipes(&ctx)) { + if (delivery_build_recipes(ctx)) { exit(1); } msg(STASIS_MSG_L3, "Copying artifacts\n"); - if (delivery_copy_conda_artifacts(&ctx)) { + if (delivery_copy_conda_artifacts(ctx)) { exit(1); } msg(STASIS_MSG_L3, "Indexing artifacts\n"); - if (delivery_index_conda_artifacts(&ctx)) { + if (delivery_index_conda_artifacts(ctx)) { exit(1); } } +} - if (strlist_count(ctx.conda.pip_packages_defer)) { +static void build_wheel_packages(struct Delivery *ctx) { + if (strlist_count(ctx->conda.pip_packages_defer)) { msg(STASIS_MSG_L2, "Building Python wheels(s)\n"); - if (!((ctx.conda.wheels_packages = delivery_build_wheels(&ctx)))) { + if (!((ctx->conda.wheels_packages = delivery_build_wheels(ctx)))) { exit(1); } - if (delivery_index_wheel_artifacts(&ctx)) { + if (delivery_index_wheel_artifacts(ctx)) { exit(1); } } +} - // Populate the release environment - msg(STASIS_MSG_L1, "Populating release environment\n"); +static void release_install_conda_packages(struct Delivery *ctx, char *env_name) { msg(STASIS_MSG_L2, "Installing conda packages\n"); - if (strlist_count(ctx.conda.conda_packages)) { - if (delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_CONDA, (struct StrList *[]) {ctx.conda.conda_packages, NULL})) { + if (strlist_count(ctx->conda.conda_packages)) { + if (delivery_install_packages(ctx, ctx->storage.conda_install_prefix, env_name, INSTALL_PKG_CONDA, (struct StrList *[]) {ctx->conda.conda_packages, NULL})) { exit(1); } } - if (strlist_count(ctx.conda.conda_packages_defer)) { + if (strlist_count(ctx->conda.conda_packages_defer)) { msg(STASIS_MSG_L3, "Installing deferred conda packages\n"); - if (delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_CONDA | INSTALL_PKG_CONDA_DEFERRED, (struct StrList *[]) {ctx.conda.conda_packages_defer, NULL})) { + if (delivery_install_packages(ctx, ctx->storage.conda_install_prefix, env_name, INSTALL_PKG_CONDA | INSTALL_PKG_CONDA_DEFERRED, (struct StrList *[]) {ctx->conda.conda_packages_defer, NULL})) { exit(1); } } else { msg(STASIS_MSG_L3, "No deferred conda packages\n"); } +} +static void release_install_pip_packages(struct Delivery *ctx, char *env_name) { msg(STASIS_MSG_L2, "Installing pip packages\n"); - if (strlist_count(ctx.conda.pip_packages)) { - if (delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_PIP, (struct StrList *[]) {ctx.conda.pip_packages, NULL})) { + if (strlist_count(ctx->conda.pip_packages)) { + if (delivery_install_packages(ctx, ctx->storage.conda_install_prefix, env_name, INSTALL_PKG_PIP, (struct StrList *[]) {ctx->conda.pip_packages, NULL})) { exit(1); } } - if (strlist_count(ctx.conda.pip_packages_defer)) { + if (strlist_count(ctx->conda.pip_packages_defer)) { msg(STASIS_MSG_L3, "Installing deferred pip packages\n"); - if (delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_PIP | INSTALL_PKG_PIP_DEFERRED, (struct StrList *[]) {ctx.conda.pip_packages_defer, NULL})) { + if (delivery_install_packages(ctx, ctx->storage.conda_install_prefix, env_name, INSTALL_PKG_PIP | INSTALL_PKG_PIP_DEFERRED, (struct StrList *[]) {ctx->conda.pip_packages_defer, NULL})) { exit(1); } } else { msg(STASIS_MSG_L3, "No deferred pip packages\n"); } +} - conda_exec("list"); - - msg(STASIS_MSG_L1, "Creating release\n"); - msg(STASIS_MSG_L2, "Exporting delivery configuration\n"); - if (!pushd(ctx.storage.cfgdump_dir)) { - char filename[PATH_MAX] = {0}; - sprintf(filename, "%s.ini", ctx.info.release_name); - FILE *spec = fopen(filename, "w+"); - if (!spec) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", filename); - exit(1); - } - ini_write(ctx._stasis_ini_fp.delivery, &spec, INI_WRITE_RAW); - fclose(spec); - - memset(filename, 0, sizeof(filename)); - sprintf(filename, "%s-rendered.ini", ctx.info.release_name); - spec = fopen(filename, "w+"); - if (!spec) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", filename); - exit(1); - } - ini_write(ctx._stasis_ini_fp.delivery, &spec, INI_WRITE_PRESERVE); - fclose(spec); - popd(); - } else { - SYSERROR("Failed to enter directory: %s", ctx.storage.delivery_dir); - exit(1); - } - - msg(STASIS_MSG_L2, "Exporting %s\n", env_name_testing); - if (conda_env_export(env_name_testing, ctx.storage.delivery_dir, env_name_testing)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", env_name_testing); - exit(1); - } - - msg(STASIS_MSG_L2, "Exporting %s\n", env_name); - if (conda_env_export(env_name, ctx.storage.delivery_dir, env_name)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", env_name); - exit(1); - } - - // Rewrite release environment output (i.e. set package origin(s) to point to the deployment server, etc.) - char specfile[PATH_MAX]; - sprintf(specfile, "%s/%s.yml", ctx.storage.delivery_dir, env_name); - msg(STASIS_MSG_L3, "Rewriting release spec file (stage 1): %s\n", path_basename(specfile)); - delivery_rewrite_spec(&ctx, specfile, DELIVERY_REWRITE_SPEC_STAGE_1); - - msg(STASIS_MSG_L1, "Rendering mission templates\n"); - delivery_mission_render_files(&ctx); - - int want_docker = ini_section_search(&ctx._stasis_ini_fp.delivery, INI_SEARCH_BEGINS, "deploy:docker") ? true : false; - int want_artifactory = ini_section_search(&ctx._stasis_ini_fp.delivery, INI_SEARCH_BEGINS, "deploy:artifactory") ? true : false; +static void build_docker(struct Delivery *ctx, const int disabled) { + const int want_docker = ini_section_search(&ctx->_stasis_ini_fp.delivery, INI_SEARCH_BEGINS, "deploy:docker") ? true : false; if (want_docker) { - if (user_disabled_docker) { + if (disabled) { msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Docker image building is disabled by CLI argument\n"); } else { char dockerfile[PATH_MAX] = {0}; - sprintf(dockerfile, "%s/%s", ctx.storage.build_docker_dir, "Dockerfile"); + sprintf(dockerfile, "%s/%s", ctx->storage.build_docker_dir, "Dockerfile"); if (globals.enable_docker) { if (!access(dockerfile, F_OK)) { msg(STASIS_MSG_L1, "Building Docker image\n"); - if (delivery_docker(&ctx)) { + if (delivery_docker(ctx)) { msg(STASIS_MSG_L1 | STASIS_MSG_ERROR, "Failed to build docker image!\n"); COE_CHECK_ABORT(1, "Failed to build docker image"); } } else { - msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Docker image building is disabled. No Dockerfile found in %s\n", ctx.storage.build_docker_dir); + msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Docker image building is disabled. No Dockerfile found in %s\n", ctx->storage.build_docker_dir); } } else { msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Docker image building is disabled. System configuration error\n"); @@ -574,25 +442,220 @@ int main(int argc, char *argv[]) { } else { msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Docker image building is disabled. deploy:docker is not configured\n"); } +} + +static void generate_release(struct Delivery *ctx, char *env_name, char *env_name_testing, const int disable_docker) { + // Populate the release environment + msg(STASIS_MSG_L1, "Populating release environment\n"); + release_install_conda_packages(ctx, env_name); + release_install_pip_packages(ctx, env_name); + + conda_exec("list"); + + msg(STASIS_MSG_L1, "Creating release\n"); + delivery_export(ctx, (char *[]) {env_name, env_name_testing, NULL}); + + char specfile[PATH_MAX]; + sprintf(specfile, "%s/%s.yml", ctx->storage.delivery_dir, env_name); - msg(STASIS_MSG_L3, "Rewriting release spec file (stage 2): %s\n", path_basename(specfile)); - delivery_rewrite_spec(&ctx, specfile, DELIVERY_REWRITE_SPEC_STAGE_2); + delivery_rewrite_stage1(ctx, specfile); + build_docker(ctx, disable_docker); + delivery_rewrite_stage2(ctx, specfile); msg(STASIS_MSG_L1, "Dumping metadata\n"); - if (delivery_dump_metadata(&ctx)) { + if (delivery_dump_metadata(ctx)) { msg(STASIS_MSG_L1 | STASIS_MSG_ERROR, "Metadata dump failed\n"); } +} +static void transfer_artifacts(struct Delivery *ctx) { + const int want_artifactory = ini_section_search(&ctx->_stasis_ini_fp.delivery, INI_SEARCH_BEGINS, "deploy:artifactory") ? true : false; if (want_artifactory) { if (globals.enable_artifactory && globals.enable_artifactory_upload) { msg(STASIS_MSG_L1, "Uploading artifacts\n"); - delivery_artifact_upload(&ctx); + delivery_artifact_upload(ctx); } else { msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Artifactory upload is disabled by CLI argument\n"); } } else { msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Artifactory upload is disabled. deploy:artifactory is not configured\n"); } +} + +int main(int argc, char *argv[]) { + struct Delivery ctx; + struct Process proc = { + .f_stdout = "", + .f_stderr = "", + .redirect_stderr = 0, + }; + char env_name[STASIS_NAME_MAX] = {0}; + char env_name_testing[STASIS_NAME_MAX] = {0}; + char *delivery_input = NULL; + char *config_input = NULL; + char installer_url[PATH_MAX]; + char python_override_version[STASIS_NAME_MAX]; + int user_disabled_docker = false; + globals.cpu_limit = get_cpu_count(); + if (globals.cpu_limit > 1) { + globals.cpu_limit--; // max - 1 + } + + memset(env_name, 0, sizeof(env_name)); + memset(env_name_testing, 0, sizeof(env_name_testing)); + memset(installer_url, 0, sizeof(installer_url)); + memset(python_override_version, 0, sizeof(python_override_version)); + memset(&proc, 0, sizeof(proc)); + memset(&ctx, 0, sizeof(ctx)); + + int c; + int option_index = 0; + while ((c = getopt_long(argc, argv, "hVCc:p:vU", long_options, &option_index)) != -1) { + switch (c) { + case 'h': + usage(path_basename(argv[0])); + exit(0); + case 'V': + puts(VERSION); + exit(0); + case 'c': + config_input = strdup(optarg); + break; + case 'C': + globals.continue_on_error = true; + break; + case 'p': + strcpy(python_override_version, optarg); + break; + case 'l': + globals.cpu_limit = strtol(optarg, NULL, 10); + if (globals.cpu_limit <= 1) { + globals.cpu_limit = 1; + globals.enable_parallel = false; // No point + } + break; + case OPT_ALWAYS_UPDATE_BASE: + globals.always_update_base_environment = true; + break; + case OPT_FAIL_FAST: + globals.parallel_fail_fast = true; + break; + case OPT_POOL_STATUS_INTERVAL: + globals.pool_status_interval = (int) strtol(optarg, NULL, 10); + if (globals.pool_status_interval < 1) { + globals.pool_status_interval = 1; + } else if (globals.pool_status_interval > 60 * 10) { + // Possible poor choice alert + fprintf(stderr, "Caution: Excessive pausing between status updates may cause third-party CI/CD" + " jobs to fail if the stdout/stderr streams are idle for too long!\n"); + } + break; + case 'U': + setenv("PYTHONUNBUFFERED", "1", 1); + fflush(stdout); + fflush(stderr); + setvbuf(stdout, NULL, _IONBF, 0); + setvbuf(stderr, NULL, _IONBF, 0); + break; + case 'v': + globals.verbose = true; + break; + case OPT_OVERWRITE: + globals.enable_overwrite = true; + break; + case OPT_NO_DOCKER: + globals.enable_docker = false; + user_disabled_docker = true; + break; + case OPT_NO_ARTIFACTORY: + globals.enable_artifactory = false; + break; + case OPT_NO_ARTIFACTORY_BUILD_INFO: + globals.enable_artifactory_build_info = false; + break; + case OPT_NO_ARTIFACTORY_UPLOAD: + globals.enable_artifactory_build_info = false; + globals.enable_artifactory_upload = false; + break; + case OPT_NO_TESTING: + globals.enable_testing = false; + break; + case OPT_NO_REWRITE_SPEC_STAGE_2: + globals.enable_rewrite_spec_stage_2 = false; + break; + case OPT_NO_PARALLEL: + globals.enable_parallel = false; + break; + case '?': + default: + exit(1); + } + } + + if (optind < argc) { + while (optind < argc) { + // use first positional argument + delivery_input = argv[optind++]; + break; + } + } + + if (!delivery_input) { + fprintf(stderr, "error: a DELIVERY_FILE is required\n"); + usage(path_basename(argv[0])); + exit(1); + } + + printf(BANNER, VERSION, AUTHOR); + + check_system_path(); + + msg(STASIS_MSG_L1, "Setup\n"); + + tpl_setup_vars(&ctx); + tpl_setup_funcs(&ctx); + + setup_sysconfdir(); + setup_python_version_override(&ctx, python_override_version); + + configure_stasis_ini(&ctx, &config_input); + configure_delivery_ini(&ctx, &delivery_input); + configure_delivery_context(&ctx); + + check_requirements(&ctx); + configure_jfrog_cli(&ctx); + + runtime_apply(ctx.runtime.environ); + strcpy(env_name, ctx.info.release_name); + strcpy(env_name_testing, env_name); + strcat(env_name_testing, "-test"); + char *envs[] = { + "release", env_name, + "testing", env_name_testing, + NULL, NULL, + }; + + check_release_history(&ctx); + sync_release_history(&ctx); + + check_conda_install_prefix(&ctx); + check_conda_prefix_length(&ctx); + setup_conda(&ctx, installer_url); + configure_conda_base(&ctx, envs); + configure_conda_purge(&ctx, envs); + setup_activate_test_env(&ctx, env_name_testing); + + configure_tool_versions(&ctx); + install_build_package(); + configure_package_overlay(&ctx, env_name); + configure_deferred_packages(&ctx); + + show_overiew(&ctx); + run_tests(&ctx); + build_conda_recipes(&ctx); + build_wheel_packages(&ctx); + generate_release(&ctx, env_name, env_name_testing, user_disabled_docker); + transfer_artifacts(&ctx); msg(STASIS_MSG_L1, "Cleaning up\n"); delivery_free(&ctx); diff --git a/src/lib/core/conda.c b/src/lib/core/conda.c index c81e6cc..de6130f 100644 --- a/src/lib/core/conda.c +++ b/src/lib/core/conda.c @@ -4,7 +4,7 @@ #include "conda.h" -int micromamba(struct MicromambaInfo *info, char *command, ...) { +int micromamba(const struct MicromambaInfo *info, char *command, ...) { struct utsname sys; uname(&sys); @@ -24,7 +24,13 @@ int micromamba(struct MicromambaInfo *info, char *command, ...) { sprintf(installer_path, "%s/latest", getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"); if (access(installer_path, F_OK)) { - download(url, installer_path, NULL); + char *errmsg = NULL; + const long http_code = download(url, installer_path, &errmsg); + if (HTTP_ERROR(http_code)) { + fprintf(stderr, "download failed: %ld: %s\n", http_code, errmsg); + guard_free(errmsg); + return -1; + } } char mmbin[PATH_MAX]; @@ -62,17 +68,40 @@ int micromamba(struct MicromambaInfo *info, char *command, ...) { } int python_exec(const char *args) { - char command[PATH_MAX] = {0}; - snprintf(command, sizeof(command) - 1, "python %s", args); + const char *command_base = "python "; + const char *command_fmt = "%s%s"; + + const int len = snprintf(NULL, 0, command_fmt, command_base, args); + char *command = calloc(len + 1, sizeof(*command)); + if (!command) { + SYSERROR("Unable to allocate %d bytes for command string", len); + return -1; + } + + snprintf(command, len + 1, command_fmt, command_base, args); msg(STASIS_MSG_L3, "Executing: %s\n", command); - return system(command); + + const int result = system(command); + guard_free(command); + return result; } int pip_exec(const char *args) { - char command[PATH_MAX] = {0}; - snprintf(command, sizeof(command) - 1, "python -m pip %s", args); + const char *command_base = "python -m pip "; + const char *command_fmt = "%s%s"; + + const int len = snprintf(NULL, 0, command_fmt, command_base, args); + char *command = calloc(len + 1, sizeof(*command)); + if (!command) { + SYSERROR("Unable to allocate %d bytes for command string", len); + return -1; + } + snprintf(command, len + 1, command_fmt, command_base, args); msg(STASIS_MSG_L3, "Executing: %s\n", command); - return system(command); + + const int result = system(command); + guard_free(command); + return result; } static const char *PKG_ERROR_STR[] = { @@ -177,7 +206,6 @@ int pkg_index_provides(int mode, const char *index, const char *spec) { } int conda_exec(const char *args) { - char command[PATH_MAX]; const char *mamba_commands[] = { "build", "install", @@ -202,15 +230,24 @@ int conda_exec(const char *args) { } } - snprintf(command, sizeof(command) - 1, "%s %s", conda_as, args); + const char *command_fmt = "%s %s"; + const int len = snprintf(NULL, 0, command_fmt, conda_as, args); + char *command = calloc(len + 1, sizeof(*command)); + if (!command) { + return -1; + } + + snprintf(command, len + 1, command_fmt, conda_as, args); msg(STASIS_MSG_L3, "Executing: %s\n", command); - return system(command); + const int result = system(command); + guard_free(command); + return result; } static int conda_prepend_bin(const char *root) { char conda_bin[PATH_MAX] = {0}; - snprintf(conda_bin, sizeof(conda_bin) - 1, "%s/bin", root); + snprintf(conda_bin, sizeof(conda_bin), "%s/bin", root); if (env_manipulate_pathstr("PATH", conda_bin, PM_PREPEND | PM_ONCE)) { return -1; } @@ -220,7 +257,7 @@ static int conda_prepend_bin(const char *root) { static int conda_prepend_condabin(const char *root) { char conda_condabin[PATH_MAX] = {0}; - snprintf(conda_condabin, sizeof(conda_condabin) - 1, "%s/condabin", root); + snprintf(conda_condabin, sizeof(conda_condabin), "%s/condabin", root); if (env_manipulate_pathstr("PATH", conda_condabin, PM_PREPEND | PM_ONCE)) { return -1; } @@ -329,7 +366,7 @@ int conda_activate(const char *root, const char *env_name) { return -1; } - snprintf(command, sizeof(command) - 1, + snprintf(command, sizeof(command), "set -a\n" "source %s\n" "__conda_exe() (\n" @@ -499,10 +536,8 @@ int conda_setup_headless() { } int conda_env_create_from_uri(char *name, char *uri, char *python_version) { - char env_command[PATH_MAX]; char *uri_fs = NULL; - // Convert a bare system path to a file:// path if (!strstr(uri, "://")) { uri_fs = calloc(strlen(uri) + strlen("file://") + 1, sizeof(*uri_fs)); @@ -523,25 +558,50 @@ int conda_env_create_from_uri(char *name, char *uri, char *python_version) { // We'll create a new file with the same random bits, ending with .yml strcat(tempfile, ".yml"); char *errmsg = NULL; - download(uri_fs ? uri_fs : uri, tempfile, &errmsg); + const long http_code = download(uri_fs ? uri_fs : uri, tempfile, &errmsg); + if (HTTP_ERROR(http_code)) { + if (errmsg) { + fprintf(stderr, "download failed: %ld: %s\n", http_code, errmsg); + guard_free(errmsg); + } + guard_free(uri_fs); + return -1; + } guard_free(uri_fs); // Rewrite python version char spec[255] = {0}; - snprintf(spec, sizeof(spec) - 1, "- python=%s\n", python_version); + snprintf(spec, sizeof(spec), "- python=%s\n", python_version); file_replace_text(tempfile, "- python\n", spec, 0); - sprintf(env_command, "env create -n '%s' --file='%s'", name, tempfile); - int status = conda_exec(env_command); + const char *fmt = "env create -n '%s' --file='%s'"; + int len = snprintf(NULL, 0, fmt, name, tempfile); + char *env_command = calloc(len + 1, sizeof(*env_command)); + if (!env_command) { + return -1; + } + + snprintf(env_command, len + 1, fmt, name, tempfile); + const int status = conda_exec(env_command); unlink(tempfile); + guard_free(env_command); return status; } int conda_env_create(char *name, char *python_version, char *packages) { - char env_command[PATH_MAX]; - sprintf(env_command, "create -n %s python=%s %s", name, python_version, packages ? packages : ""); - return conda_exec(env_command); + const char *fmt = "create -n %s python=%s %s"; + const int len = snprintf(NULL, 0, fmt, name, python_version, packages ? packages : ""); + char *env_command = calloc(len + 1, sizeof(*env_command)); + if (!env_command) { + return -1; + } + + snprintf(env_command, len + 1, fmt, name, python_version, packages ? packages : ""); + const int result = conda_exec(env_command); + guard_free(env_command); + + return result; } int conda_env_remove(char *name) { diff --git a/src/lib/core/download.c b/src/lib/core/download.c index c3f8dca..b021860 100644 --- a/src/lib/core/download.c +++ b/src/lib/core/download.c @@ -41,10 +41,10 @@ long download(char *url, const char *filename, char **errmsg) { CURLcode curl_code = curl_easy_perform(c); SYSDEBUG("curl status code: %d", curl_code); if (curl_code != CURLE_OK) { - if (errmsg) { - strcpy(*errmsg, curl_easy_strerror(curl_code)); + if (!*errmsg) { + *errmsg = strdup(curl_easy_strerror(curl_code)); } else { - fprintf(stderr, "\nCURL ERROR: %s\n", curl_easy_strerror(curl_code)); + strncpy(*errmsg, curl_easy_strerror(curl_code), strlen(curl_easy_strerror(curl_code) + 1)); } goto failed; } diff --git a/src/lib/core/environment.c b/src/lib/core/environment.c index f5e8566..7ece5e6 100644 --- a/src/lib/core/environment.c +++ b/src/lib/core/environment.c @@ -106,14 +106,13 @@ void runtime_export(RuntimeEnv *env, char **keys) { if (keys != NULL) { for (size_t j = 0; keys[j] != NULL; j++) { if (strcmp(keys[j], key) == 0) { - //sprintf(output, "%s=\"%s\"\n%s %s", key, value ? value : "", export_command, key); - sprintf(output, "%s %s=\"%s\"", export_command, key, value ? value : ""); + snprintf(output, sizeof(output), "%s %s=\"%s\"", export_command, key, value ? value : ""); puts(output); } } } else { - sprintf(output, "%s %s=\"%s\"", export_command, key, value ? value : ""); + snprintf(output, sizeof(output), "%s %s=\"%s\"", export_command, key, value ? value : ""); puts(output); } guard_free(value); @@ -178,7 +177,7 @@ int runtime_replace(RuntimeEnv **dest, char **src) { } /** - * Determine whether or not a key exists in the runtime environment + * Determine whether a key exists in the runtime environment * * Example: * @@ -245,7 +244,14 @@ char *runtime_get(RuntimeEnv *env, const char *key) { ssize_t key_offset = runtime_contains(env, key); if (key_offset != -1) { char **pair = split(strlist_item(env, key_offset), "=", 0); + if (!pair) { + return NULL; + } result = join(&pair[1], "="); + if (!result) { + guard_array_free(pair); + return NULL; + } guard_array_free(pair); } return result; @@ -285,8 +291,7 @@ char *runtime_expand_var(RuntimeEnv *env, char *input) { // If there's no environment variables to process return the input string if (strchr(input, delim) == NULL) { - //return strdup(input); - return input; + return strdup(input); } expanded = calloc(STASIS_BUFSIZ, sizeof(char)); @@ -336,7 +341,10 @@ char *runtime_expand_var(RuntimeEnv *env, char *input) { if (env) { tmp = runtime_get(env, var); } else { - tmp = getenv(var); + const char *v = getenv(var); + if (v) { + tmp = strdup(v); + } } if (tmp == NULL) { // This mimics shell behavior in general. @@ -348,9 +356,7 @@ char *runtime_expand_var(RuntimeEnv *env, char *input) { } // Append expanded environment variable to output strncat(expanded, tmp, STASIS_BUFSIZ - 1); - if (env) { - guard_free(tmp); - } + guard_free(tmp); } // Nothing to do so append input to output @@ -400,13 +406,28 @@ char *runtime_expand_var(RuntimeEnv *env, char *input) { * @param _value New environment variable value */ void runtime_set(RuntimeEnv *env, const char *_key, char *_value) { + const char *sep = "="; if (_key == NULL) { return; } + const ssize_t key_offset = runtime_contains(env, _key); char *key = strdup(_key); - ssize_t key_offset = runtime_contains(env, key); + if (!key) { + SYSERROR("%s", "unable to allocate memory for key"); + exit(1); + } char *value = runtime_expand_var(env, _value); - char *now = join((char *[]) {key, value, NULL}, "="); + if (!value) { + SYSERROR("%s", "unable to allocate memory for value"); + exit(1); + } + + lstrip(value); + char *now = join((char *[]) {key, value, NULL}, sep); + if (!now) { + SYSERROR("%s", "unable to allocate memory for join"); + exit(1); + } if (key_offset < 0) { strlist_append(&env, now); @@ -415,6 +436,7 @@ void runtime_set(RuntimeEnv *env, const char *_key, char *_value) { } guard_free(now); guard_free(key); + guard_free(value); } /** @@ -424,6 +446,10 @@ void runtime_set(RuntimeEnv *env, const char *_key, char *_value) { void runtime_apply(RuntimeEnv *env) { for (size_t i = 0; i < strlist_count(env); i++) { char **pair = split(strlist_item(env, i), "=", 1); + if (!pair) { + SYSERROR("%s", "unable to allocate memory for runtime_apply"); + return; + } setenv(pair[0], pair[1], 1); guard_array_free(pair); } diff --git a/src/lib/core/include/conda.h b/src/lib/core/include/conda.h index ea8613f..f3d481c 100644 --- a/src/lib/core/include/conda.h +++ b/src/lib/core/include/conda.h @@ -38,7 +38,7 @@ struct MicromambaInfo { * @param ... variadic arguments * @return exit code */ -int micromamba(struct MicromambaInfo *info, char *command, ...); +int micromamba(const struct MicromambaInfo *info, char *command, ...); /** * Execute Python diff --git a/src/lib/core/include/core.h b/src/lib/core/include/core.h index 35a9506..92969d2 100644 --- a/src/lib/core/include/core.h +++ b/src/lib/core/include/core.h @@ -15,7 +15,7 @@ #define STASIS_NAME_MAX 255 #define STASIS_DIRSTACK_MAX 1024 #define STASIS_TIME_STR_MAX 128 -#define HTTP_ERROR(X) X >= 400 +#define HTTP_ERROR(X) (X >= 400 || X < 0) #include "config.h" #include "core_mem.h" diff --git a/src/lib/core/include/str.h b/src/lib/core/include/str.h index bb96db0..be497ed 100644 --- a/src/lib/core/include/str.h +++ b/src/lib/core/include/str.h @@ -293,18 +293,27 @@ int isdigit_s(const char *s); char *tolower_s(char *s); /** - * Return a copy of the input string with "." characters removed + * Reduce a version string to the major[minor] format used by Python * - * ~~~{.c} - * char *version = strdup("1.2.3"); - * char *version_short = to_short_version(str); - * // version_short is "123" - * free(version_short); + * @code{.c} + * #include <stdio.h> + * #include "str.h" * - * ~~~ + * int main(int argc, char *argv[]) { + * char python_version[] = "3.13.3" + * char *python_short_version = to_short_version(python_version); // "313" + * if (!python_short_version) { + * perror("unable to allocate memory for shortened python version"); + * return 1; + * } + * free(python_short_version); + * return 0; + * } + * @endcode * - * @param s input string - * @return pointer to new string + * @param s python version string + * @return the shortened version string + * @return NULL on error */ char *to_short_version(const char *s); diff --git a/src/lib/core/include/utils.h b/src/lib/core/include/utils.h index b405b2a..a9bcd2f 100644 --- a/src/lib/core/include/utils.h +++ b/src/lib/core/include/utils.h @@ -419,4 +419,49 @@ int gen_file_extension_str(char *filename, const char *extension); char *remove_extras(char *s); void debug_hexdump(char *data, int len); + +/** + * Realloc helper + * + * @code{.c} + * #include <stdio.h> + * #include <stdlib.h> + * #include <string.h> + * #include "utils.h" + * + * int main(int argc, char *argv[]) { + * size_t sz = 10; + * char *data = calloc(sz, sizeof(*data)); + * + * // populate data + * strncat(data, "/path/to/", sz - 1); + * + * // Double the allocation size for data + * if (grow(sz * 2, &sz, &data)) { + * // memory error + * } + * + * // sz is now 20 + * strncat(data, "filename", sz - 1 - strlen(data)); + * + * puts(data); + * // output: "/path/to/filename" + * } + * @endcode + * + * @param size_new increase by `size_new` bytes + * @param size_orig address of variable containing the original allocation size (modified) + * @param data address to write data + * @return 0 on success + * @return -1 on error + */ +int grow(size_t size_new, size_t *size_orig, char **data); + +int in_ascii_range(char c, char lower, char upper); + +#define GIT_HASH_LEN 40 +int is_git_sha(char const *hash); + +int check_python_package_dependencies(const char *srcdir); + #endif //STASIS_UTILS_H diff --git a/src/lib/core/str.c b/src/lib/core/str.c index 1d0b268..9524886 100644 --- a/src/lib/core/str.c +++ b/src/lib/core/str.c @@ -640,12 +640,35 @@ char *tolower_s(char *s) { } char *to_short_version(const char *s) { - char *result = strdup(s); - if (!result) { - return NULL; + char *result = NULL; + if (num_chars(s, '.') > 1) { + char **version_data = split((char *) s, ".", 1); + if (!version_data) { + goto to_short_version_failed; + } + if (version_data[1]) { + char *dot = strchr(version_data[1], '.'); + if (dot) { + *dot = '\0'; + } + } + result = join(version_data, ""); + if (!result) { + guard_array_free(version_data); + goto to_short_version_failed; + } + guard_array_free(version_data); + } else { + result = strdup(s); + if (!result) { + goto to_short_version_failed; + } + strchrdel(result, "."); } - strchrdel(result, "."); + return result; + to_short_version_failed: + return NULL; } void unindent(char *s) { diff --git a/src/lib/core/template.c b/src/lib/core/template.c index ba45a5a..dd3c7a2 100644 --- a/src/lib/core/template.c +++ b/src/lib/core/template.c @@ -137,23 +137,6 @@ struct tplfunc_frame *tpl_getfunc(char *key) { return result; } -static int grow(size_t z, size_t *output_bytes, char **output) { - if (z >= *output_bytes) { - size_t new_size = *output_bytes + z + 1; - SYSDEBUG("template output buffer new size: %zu\n", new_size); - - char *tmp = realloc(*output, new_size); - if (!tmp) { - perror("realloc failed"); - return -1; - } else if (tmp != *output) { - *output = tmp; - } - *output_bytes = new_size; - } - return 0; -} - char *tpl_render(char *str) { if (!str) { return NULL; @@ -297,8 +280,8 @@ char *tpl_render(char *str) { output[z] = pos[off]; z++; } - SYSDEBUG("template output length: %zu", strlen(output)); - SYSDEBUG("template output bytes: %zu", output_bytes); + //SYSDEBUG("template output length: %zu", strlen(output)); + //SYSDEBUG("template output bytes: %zu", output_bytes); return output; } diff --git a/src/lib/core/utils.c b/src/lib/core/utils.c index a9c9ea5..62f3bec 100644 --- a/src/lib/core/utils.c +++ b/src/lib/core/utils.c @@ -892,10 +892,11 @@ int gen_file_extension_str(char *filename, const char *extension) { return replace_text(ext_orig, ext_orig, extension, 0); } +#define DEBUG_HEXDUMP_FMT_BYTES 6 #define DEBUG_HEXDUMP_ADDR_MAXLEN 20 #define DEBUG_HEXDUMP_BYTES_MAXLEN (16 * 3 + 2) #define DEBUG_HEXDUMP_ASCII_MAXLEN (16 + 1) -#define DEBUG_HEXDUMP_OUTPUT_MAXLEN (DEBUG_HEXDUMP_ADDR_MAXLEN + DEBUG_HEXDUMP_BYTES_MAXLEN + DEBUG_HEXDUMP_ASCII_MAXLEN + 1) +#define DEBUG_HEXDUMP_OUTPUT_MAXLEN (DEBUG_HEXDUMP_FMT_BYTES + DEBUG_HEXDUMP_ADDR_MAXLEN + DEBUG_HEXDUMP_BYTES_MAXLEN + DEBUG_HEXDUMP_ASCII_MAXLEN + 1) void debug_hexdump(char *data, int len) { int count = 0; @@ -936,10 +937,144 @@ void debug_hexdump(char *data, int len) { // Add group padding strcat(bytes, " "); } - int padding = 16 - count; + const int padding = 16 - count; for (int i = 0; i < padding; i++) { strcat(bytes, " "); } - sprintf(output, "%s | %s | %s", addr, bytes, ascii); + snprintf(output, DEBUG_HEXDUMP_FMT_BYTES + sizeof(addr) + sizeof(bytes) + sizeof(ascii), "%s | %s | %s", addr, bytes, ascii); puts(output); -}
\ No newline at end of file +} + +int grow(const size_t size_new, size_t *size_orig, char **data) { + if (!*data) { + return 0; + } + if (size_new >= *size_orig) { + const size_t new_size = *size_orig + size_new + 1; + SYSDEBUG("template data buffer new size: %zu\n", new_size); + + char *tmp = realloc(*data, new_size); + if (!tmp) { + perror("realloc failed"); + return -1; + } + if (tmp != *data) { + *data = tmp; + } + *size_orig = new_size; + } + return 0; +} + +int in_ascii_range(const char c, char lower, char upper) { + if (!(c >= lower && c <= upper)) { + return 0; + } + return 1; +} + +int is_git_sha(char const *hash) { + size_t result = 0; + size_t len = strlen(hash); + + if (len > GIT_HASH_LEN) { + // too long to be a git commit hash + return 0; + } + for (size_t i = 0; i < len; i++) { + if (in_ascii_range(hash[i], 'a', 'f') + || in_ascii_range(hash[i], 'A', 'F') + || in_ascii_range(hash[i], '0', '9')) { + result++; + } + } + if (result < len) { + return 0; + } + return 1; +} + +static int read_vcs_records(const size_t line, char **data) { + (void) line; // unused + const char *vcs_name[] = { + "git", + "svn", + "hg", + "bzr", + }; + for (size_t i = 0; i < sizeof(vcs_name) / sizeof(vcs_name[0]); i++) { + const char *vcs = vcs_name[i]; + char *data_local = strdup(*data); + if (!data_local) { + fprintf(stderr, "Out of memory\n"); + return -1; + } + + // Remove leading/trailing blanks + lstrip(data_local); + strip(data_local); + + // Ignore file comment(s) + if (startswith(data_local, "#") || startswith(data_local, ";")) { + // continue + return 1; + } + + // Begin matching VCS package syntax + const char *match_vcs = strstr(data_local,vcs); + if (match_vcs) { + const char *match_protocol_sep = strstr(match_vcs, "+"); + if (match_protocol_sep) { + const char *match_protocol = strstr(match_protocol_sep, "://"); + if (match_protocol) { + guard_free(data_local); + // match found + return 0; + } + } + } + guard_free(data_local); + } + + // no match, continue + return 1; +} +int check_python_package_dependencies(const char *srcdir) { + const char *configs[] = { + "pyproject.toml", + "setup.cfg", + "setup.py" + }; + + for (size_t i = 0; i < sizeof(configs) / sizeof(configs[0]); i++) { + char path[PATH_MAX] = {0}; + const char *configfile = configs[i]; + + snprintf(path, sizeof(path), "%s/%s", srcdir, configfile); + if (access(path, F_OK) < 0) { + continue; + } + + //char **data = file_readlines(path, 0, 0, NULL); + struct StrList *data = strlist_init(); + int err = 0; + if ((err = strlist_append_file(data, path, read_vcs_records))) { + guard_strlist_free(&data); + return -1; + } + const size_t count = strlist_count(data); + if (count) { + printf("\nERROR: VCS requirement(s) detected in %s:\n", configfile); + for (size_t j = 0; j < count; j++) { + char *record = strlist_item(data, j); + lstrip(record); + strip(record); + printf("[%zu] %s\n", j, record); + } + guard_strlist_free(&data); + return 1; + } + guard_strlist_free(&data); + } + return 0; +} diff --git a/src/lib/delivery/CMakeLists.txt b/src/lib/delivery/CMakeLists.txt index 78ed20f..559b2dc 100644 --- a/src/lib/delivery/CMakeLists.txt +++ b/src/lib/delivery/CMakeLists.txt @@ -1,4 +1,5 @@ add_library(stasis_delivery STATIC + delivery_export.c delivery_postprocess.c delivery_conda.c delivery_docker.c diff --git a/src/lib/delivery/delivery.c b/src/lib/delivery/delivery.c index d480ab4..7ec2e04 100644 --- a/src/lib/delivery/delivery.c +++ b/src/lib/delivery/delivery.c @@ -225,7 +225,15 @@ void delivery_defer_packages(struct Delivery *ctx, int type) { // Override test->version when a version is provided by the (pip|conda)_package list item guard_free(test->version); if (spec_begin && spec_end) { - test->version = strdup(spec_end); + char *version_at = strrchr(spec_end, '@'); + if (version_at) { + if (strlen(version_at)) { + version_at++; + } + test->version = strdup(version_at); + } else { + test->version = strdup(spec_end); + } } else { // There are too many possible default branches nowadays: master, main, develop, xyz, etc. // HEAD is a safe bet. @@ -233,6 +241,9 @@ void delivery_defer_packages(struct Delivery *ctx, int type) { } // Is the list item a git+schema:// URL? + // TODO: nametmp is just the name so this will never work. but do we want it to? this looks like + // TODO: an unsafe feature. We shouldn't be able to change what's in the config. we should + // TODO: be getting what we asked for, or exit the program with an error. if (strstr(nametmp, "git+") && strstr(nametmp, "://")) { char *xrepo = strstr(nametmp, "+"); if (xrepo) { diff --git a/src/lib/delivery/delivery_build.c b/src/lib/delivery/delivery_build.c index 2d891d2..c5093d4 100644 --- a/src/lib/delivery/delivery_build.c +++ b/src/lib/delivery/delivery_build.c @@ -173,6 +173,18 @@ struct StrList *delivery_build_wheels(struct Delivery *ctx) { memset(outdir, 0, sizeof(outdir)); memset(cmd, 0, sizeof(outdir)); + const int dep_status = check_python_package_dependencies("."); + if (dep_status) { + fprintf(stderr, "\nPlease replace all occurrences above with standard package specs:\n" + "\n" + " package==x.y.z\n" + " package>=x.y.z\n" + " package<=x.y.z\n" + " ...\n" + "\n"); + COE_CHECK_ABORT(dep_status, "Unreproducible delivery"); + } + strcpy(dname, ctx->tests[i].name); tolower_s(dname); sprintf(outdir, "%s/%s", ctx->storage.wheel_artifact_dir, dname); diff --git a/src/lib/delivery/delivery_export.c b/src/lib/delivery/delivery_export.c new file mode 100644 index 0000000..d982ad5 --- /dev/null +++ b/src/lib/delivery/delivery_export.c @@ -0,0 +1,58 @@ +#include "delivery.h" + +static void delivery_export_configuration(const struct Delivery *ctx) { + msg(STASIS_MSG_L2, "Exporting delivery configuration\n"); + if (!pushd(ctx->storage.cfgdump_dir)) { + char filename[PATH_MAX] = {0}; + sprintf(filename, "%s.ini", ctx->info.release_name); + FILE *spec = fopen(filename, "w+"); + if (!spec) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", filename); + exit(1); + } + ini_write(ctx->_stasis_ini_fp.delivery, &spec, INI_WRITE_RAW); + fclose(spec); + + memset(filename, 0, sizeof(filename)); + sprintf(filename, "%s-rendered.ini", ctx->info.release_name); + spec = fopen(filename, "w+"); + if (!spec) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", filename); + exit(1); + } + ini_write(ctx->_stasis_ini_fp.delivery, &spec, INI_WRITE_PRESERVE); + fclose(spec); + popd(); + } else { + SYSERROR("Failed to enter directory: %s", ctx->storage.delivery_dir); + exit(1); + } +} + +void delivery_export(const struct Delivery *ctx, char *envs[]) { + delivery_export_configuration(ctx); + + for (size_t i = 0; envs[i] != NULL; i++) { + char *name = envs[i]; + msg(STASIS_MSG_L2, "Exporting %s\n", name); + if (conda_env_export(name, ctx->storage.delivery_dir, name)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", name); + exit(1); + } + } +} + +void delivery_rewrite_stage1(struct Delivery *ctx, char *specfile) { + // Rewrite release environment output (i.e. set package origin(s) to point to the deployment server, etc.) + msg(STASIS_MSG_L3, "Rewriting release spec file (stage 1): %s\n", path_basename(specfile)); + delivery_rewrite_spec(ctx, specfile, DELIVERY_REWRITE_SPEC_STAGE_1); + + msg(STASIS_MSG_L1, "Rendering mission templates\n"); + delivery_mission_render_files(ctx); +} + +void delivery_rewrite_stage2(struct Delivery *ctx, char *specfile) { + msg(STASIS_MSG_L3, "Rewriting release spec file (stage 2): %s\n", path_basename(specfile)); + delivery_rewrite_spec(ctx, specfile, DELIVERY_REWRITE_SPEC_STAGE_2); +} + diff --git a/src/lib/delivery/delivery_init.c b/src/lib/delivery/delivery_init.c index 56c591a..a60d6af 100644 --- a/src/lib/delivery/delivery_init.c +++ b/src/lib/delivery/delivery_init.c @@ -287,18 +287,25 @@ int delivery_init(struct Delivery *ctx, int render_mode) { int bootstrap_build_info(struct Delivery *ctx) { struct Delivery local = {0}; + SYSDEBUG("ini_open(%s)", ctx->_stasis_ini_fp.cfg_path); local._stasis_ini_fp.cfg = ini_open(ctx->_stasis_ini_fp.cfg_path); + SYSDEBUG("ini_open(%s)", ctx->_stasis_ini_fp.delivery_path); local._stasis_ini_fp.delivery = ini_open(ctx->_stasis_ini_fp.delivery_path); + if (delivery_init_platform(&local)) { + SYSDEBUG("%s", "delivery_init_platform failed"); return -1; } if (populate_delivery_cfg(&local, INI_READ_RENDER)) { + SYSDEBUG("%s", "populate_delivery_cfg failed"); return -1; } if (populate_delivery_ini(&local, INI_READ_RENDER)) { + SYSDEBUG("%s", "populate_delivery_ini failed"); return -1; } if (populate_info(&local)) { + SYSDEBUG("%s", "populate_info failed"); return -1; } ctx->info.build_name = strdup(local.info.build_name); @@ -314,6 +321,7 @@ int bootstrap_build_info(struct Delivery *ctx) { memcpy(ctx->info.time_info, local.info.time_info, sizeof(*local.info.time_info)); ctx->info.time_now = local.info.time_now; ctx->info.time_str_epoch = strdup(local.info.time_str_epoch); + SYSDEBUG("%s", "delivery_free local resources"); delivery_free(&local); return 0; } diff --git a/src/lib/delivery/delivery_install.c b/src/lib/delivery/delivery_install.c index 246c604..f1637a3 100644 --- a/src/lib/delivery/delivery_install.c +++ b/src/lib/delivery/delivery_install.c @@ -36,7 +36,7 @@ static char *have_spec_in_config(const struct Delivery *ctx, const char *name) { strncpy(package, config_spec, sizeof(package) - 1); } remove_extras(package); - if (strncmp(package, name, strlen(package)) == 0) { + if (strncmp(package, name, strlen(name)) == 0) { return config_spec; } } @@ -189,8 +189,7 @@ int delivery_purge_packages(struct Delivery *ctx, const char *env_name, int use_ } int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, char *env_name, int type, struct StrList **manifest) { - char cmd[PATH_MAX]; - char pkgs[STASIS_BUFSIZ]; + char command_base[PATH_MAX]; const char *env_current = getenv("CONDA_DEFAULT_ENV"); if (env_current) { @@ -203,9 +202,8 @@ int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, cha } } - memset(cmd, 0, sizeof(cmd)); - memset(pkgs, 0, sizeof(pkgs)); - strcat(cmd, "install"); + memset(command_base, 0, sizeof(command_base)); + strcat(command_base, "install"); typedef int (*Runner)(const char *); Runner runner = NULL; @@ -216,17 +214,23 @@ int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, cha } if (INSTALL_PKG_CONDA_DEFERRED & type) { - strcat(cmd, " --use-local"); + strcat(command_base, " --use-local"); } else if (INSTALL_PKG_PIP_DEFERRED & type) { // Don't change the baseline package set unless we're working with a // new build. Release candidates will need to keep packages as stable // as possible between releases. if (!ctx->meta.based_on) { - strcat(cmd, " --upgrade"); + strcat(command_base, " --upgrade"); } + sprintf(command_base + strlen(command_base), " --extra-index-url 'file://%s'", ctx->storage.wheel_artifact_dir); } - sprintf(cmd + strlen(cmd), " --extra-index-url 'file://%s'", ctx->storage.wheel_artifact_dir); + size_t args_alloc_len = STASIS_BUFSIZ; + char *args = calloc(args_alloc_len + 1, sizeof(*args)); + if (!args) { + SYSERROR("%s", "Unable to allocate bytes for command arguments"); + return -1; + } for (size_t x = 0; manifest[x] != NULL; x++) { char *name = NULL; @@ -239,10 +243,11 @@ int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, cha if (INSTALL_PKG_PIP_DEFERRED & type) { struct Test *info = requirement_from_test(ctx, name); if (info) { - if (!strcmp(info->version, "HEAD")) { + if (!strcmp(info->version, "HEAD") || is_git_sha(info->version)) { struct StrList *tag_data = strlist_init(); if (!tag_data) { SYSERROR("%s", "Unable to allocate memory for tag data\n"); + guard_free(args); return -1; } strlist_append_tokenize(tag_data, info->repository_info_tag, "-"); @@ -273,9 +278,9 @@ int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, cha fprintf(stderr, "No wheel packages found that match the description of '%s'", info->name); } else { // found - guard_strlist_free(&tag_data); info->version = strdup(whl->version); } + guard_strlist_free(&tag_data); wheel_free(&whl); } @@ -290,26 +295,66 @@ int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, cha } } - snprintf(cmd + strlen(cmd), - sizeof(cmd) - strlen(cmd) - strlen(info->name) - strlen(info->version) + 5, - " '%s==%s'", req, info->version); + const char *fmt_append = "%s '%s==%s'"; + const char *fmt = " '%s==%s'"; + const int required_len = snprintf(NULL, 0, fmt_append, args, req, info->version); + if (required_len > (int) args_alloc_len) { + if (grow(required_len, &args_alloc_len, &args)) { + SYSERROR("Unable to allocate %d bytes for command arguments", required_len); + guard_free(args); + return -1; + } + } + snprintf(args + strlen(args), required_len + 1, fmt, req, info->version); } else { fprintf(stderr, "Deferred package '%s' is not present in the tested package list!\n", name); + guard_free(args); return -1; } } else { if (startswith(name, "--") || startswith(name, "-")) { - sprintf(cmd + strlen(cmd), " %s", name); + const char *fmt_append = "%s %s"; + const char *fmt = " %s"; + const int required_len = snprintf(NULL, 0, fmt_append, args, name); + if (required_len > (int) args_alloc_len) { + if (grow(required_len, &args_alloc_len, &args)) { + SYSERROR("Unable to allocate %d bytes for command arguments", required_len); + guard_free(args); + return -1; + } + } + snprintf(args + strlen(args), required_len + 1, fmt, name); } else { - sprintf(cmd + strlen(cmd), " '%s'", name); + const char *fmt_append = "%s '%s'"; + const char *fmt = " '%s'"; + const int required_len = snprintf(NULL, 0, fmt_append, args, name); + if (required_len > (int) args_alloc_len) { + if (grow(required_len, &args_alloc_len, &args)) { + SYSERROR("Unable to allocate %d bytes for command arguments", required_len); + guard_free(args); + return -1; + } + } + snprintf(args + strlen(args), required_len + 1, fmt, name); } } } - int status = runner(cmd); + char *command = NULL; + if (asprintf(&command, "%s %s", command_base, args) < 0) { + SYSERROR("%s", "Unable to allocate bytes for command\n"); + guard_free(args); + return -1; + } + + int status = runner(command); + guard_free(args); + guard_free(command); if (status) { + // fail quickly return status; } } + guard_free(args); return 0; } diff --git a/src/lib/delivery/delivery_populate.c b/src/lib/delivery/delivery_populate.c index 84676f1..28b2480 100644 --- a/src/lib/delivery/delivery_populate.c +++ b/src/lib/delivery/delivery_populate.c @@ -55,6 +55,7 @@ int populate_info(struct Delivery *ctx) { int populate_delivery_cfg(struct Delivery *ctx, int render_mode) { struct INIFILE *cfg = ctx->_stasis_ini_fp.cfg; if (!cfg) { + SYSDEBUG("%s", "cfg is NULL"); return -1; } int err = 0; @@ -162,8 +163,6 @@ int populate_delivery_ini(struct Delivery *ctx, int render_mode) { // keys in the configuration RuntimeEnv *rt = runtime_copy(__environ); while ((rtdata = ini_getall(ini, "runtime")) != NULL) { - char rec[STASIS_BUFSIZ]; - sprintf(rec, "%s=%s", lstrip(strip(rtdata->key)), lstrip(strip(rtdata->value))); runtime_set(rt, rtdata->key, rtdata->value); } runtime_apply(rt); diff --git a/src/lib/delivery/delivery_test.c b/src/lib/delivery/delivery_test.c index e80e0ec..6e0a226 100644 --- a/src/lib/delivery/delivery_test.c +++ b/src/lib/delivery/delivery_test.c @@ -97,6 +97,18 @@ void delivery_tests_run(struct Delivery *ctx) { if (pushd(destdir)) { COE_CHECK_ABORT(1, "Unable to enter repository directory\n"); } else { + int dep_status = check_python_package_dependencies("."); + if (dep_status) { + fprintf(stderr, "\nPlease replace all occurrences above with standard package specs:\n" + "\n" + " package==x.y.z\n" + " package>=x.y.z\n" + " package<=x.y.z\n" + " ...\n" + "\n"); + COE_CHECK_ABORT(dep_status, "Unreproducible delivery"); + } + char *cmd = calloc(strlen(test->script) + STASIS_BUFSIZ, sizeof(*cmd)); if (!cmd) { SYSERROR("Unable to allocate test script buffer: %s", strerror(errno)); diff --git a/src/lib/delivery/include/delivery.h b/src/lib/delivery/include/delivery.h index 26a5499..69ec089 100644 --- a/src/lib/delivery/include/delivery.h +++ b/src/lib/delivery/include/delivery.h @@ -459,4 +459,28 @@ int delivery_series_sync(struct Delivery *ctx); */ int delivery_purge_packages(struct Delivery *ctx, const char *env_name, int use_pkg_manager); +/** + * Export delivery environments + * + * @param ctx Delivery context + * @param envs array of conda environment names + */ +void delivery_export(const struct Delivery *ctx, char *envs[]); + +/** + * STAGE 1: Rewrite delivery-related strings in specfile + * + * @param ctx Delivery context + * @param specfile path to YAML spec file + */ +void delivery_rewrite_stage1(struct Delivery *ctx, char *specfile); + +/** + * STAGE 2: Rewrite delivery-related strings in specfile + * + * @param ctx Delivery context + * @param specfile path to YAML spec file + */ +void delivery_rewrite_stage2(struct Delivery *ctx, char *specfile); + #endif //STASIS_DELIVERY_H diff --git a/tests/setup.sh b/tests/setup.sh index 7e38cf9..bce2fbd 100644 --- a/tests/setup.sh +++ b/tests/setup.sh @@ -78,7 +78,7 @@ teardown_workspace() { install_stasis() { pushd "$BUILD_DIR" - if ! cmake -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -DCMAKE_BUILD_TYPE=Debug "${TOPDIR}"/../..; then + if ! cmake -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -DCMAKE_BUILD_TYPE=Debug -DDEBUG_MESSAGES=ON "${TOPDIR}"/../..; then echo "cmake failed" >&2 return 1 fi @@ -109,7 +109,7 @@ STASIS_TEST_RESULT_SKIP=0 run_command() { local logfile="$(mktemp).log" local cmd="${@}" - local lines_on_error=100 + local lines_on_error=1000 /bin/echo "Testing: $cmd " $cmd &>"$logfile" diff --git a/tests/test_artifactory.c b/tests/test_artifactory.c index 4af7eec..202a67c 100644 --- a/tests/test_artifactory.c +++ b/tests/test_artifactory.c @@ -116,6 +116,7 @@ int main(int argc, char *argv[]) { // Skip this suite if we're not configured to use it if (jfrt_auth_init(&gauth)) { SYSERROR("%s", "Not configured to test Artifactory. Skipping."); + guard_free(basedir); return STASIS_TEST_SUITE_SKIP; } guard_free(basedir); diff --git a/tests/test_ini.c b/tests/test_ini.c index e4a7808..af47ddf 100644 --- a/tests/test_ini.c +++ b/tests/test_ini.c @@ -195,6 +195,23 @@ void test_ini_getval_wrappers() { ini_free(&ini); } +void test_ini_getall() { + const char *filename = "ini_open.ini"; + struct INIFILE *ini = NULL; + const char *data = "[default]\nhello=world!\nthis=is a test\nx=1\ny=0\n"; + + stasis_testing_write_ascii(filename, data); + ini = ini_open(filename); + STASIS_ASSERT_FATAL(ini != NULL, "failed to open ini file"); + + const struct INIData *d = NULL; + for (size_t i = 0; (d = ini_getall(ini, "default")) != NULL; i++) { + STASIS_ASSERT(d->key != NULL, "INIData key should not be NULL"); + STASIS_ASSERT(d->value != NULL, "INIData key should not be NULL"); + } + ini_free(&ini); +} + int main(int argc, char *argv[]) { STASIS_TEST_BEGIN_MAIN(); STASIS_TEST_FUNC *tests[] = { @@ -204,6 +221,7 @@ int main(int argc, char *argv[]) { test_ini_has_key, test_ini_setval_getval, test_ini_getval_wrappers, + test_ini_getall, }; STASIS_TEST_RUN(tests); STASIS_TEST_END_MAIN(); diff --git a/tests/test_str.c b/tests/test_str.c index ad0c07a..a98a34d 100644 --- a/tests/test_str.c +++ b/tests/test_str.c @@ -6,16 +6,17 @@ void test_to_short_version() { const char *expected; }; - struct testcase tc[] = { - {.data = "1.2.3", .expected = "123"}, + const struct testcase tc[] = { + {.data = "1.2.3", .expected = "12"}, {.data = "py3.12", .expected = "py312"}, - {.data = "generic-1.2.3", .expected = "generic-123"}, + {.data = "generic-1.2.3", .expected = "generic-12"}, {.data = "nothing to do", .expected = "nothing to do"}, }; for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) { char *result = to_short_version(tc[i].data); STASIS_ASSERT_FATAL(result != NULL, "should not be NULL"); + //printf("%s[%zu], result: %s, expected: %s\n", __FUNCTION__, i, result, tc[i].expected); STASIS_ASSERT(strcmp(result, tc[i].expected) == 0, "unexpected result"); guard_free(result); } |
