diff options
Diffstat (limited to 'src/cli')
| -rw-r--r-- | src/cli/stasis/args.c | 4 | ||||
| -rw-r--r-- | src/cli/stasis/include/args.h | 2 | ||||
| -rw-r--r-- | src/cli/stasis/stasis_main.c | 741 | ||||
| -rw-r--r-- | src/cli/stasis_indexer/callbacks.c | 4 | ||||
| -rw-r--r-- | src/cli/stasis_indexer/helpers.c | 36 | ||||
| -rw-r--r-- | src/cli/stasis_indexer/include/helpers.h | 8 | ||||
| -rw-r--r-- | src/cli/stasis_indexer/include/junitxml_report.h | 2 | ||||
| -rw-r--r-- | src/cli/stasis_indexer/include/readmes.h | 2 | ||||
| -rw-r--r-- | src/cli/stasis_indexer/include/website.h | 2 | ||||
| -rw-r--r-- | src/cli/stasis_indexer/junitxml_report.c | 22 | ||||
| -rw-r--r-- | src/cli/stasis_indexer/readmes.c | 51 | ||||
| -rw-r--r-- | src/cli/stasis_indexer/stasis_indexer_main.c | 55 | ||||
| -rw-r--r-- | src/cli/stasis_indexer/website.c | 6 |
13 files changed, 515 insertions, 420 deletions
diff --git a/src/cli/stasis/args.c b/src/cli/stasis/args.c index f3ce823..172981a 100644 --- a/src/cli/stasis/args.c +++ b/src/cli/stasis/args.c @@ -13,6 +13,7 @@ struct option long_options[] = { {"unbuffered", no_argument, 0, 'U'}, {"update-base", no_argument, 0, OPT_ALWAYS_UPDATE_BASE}, {"fail-fast", no_argument, 0, OPT_FAIL_FAST}, + {"task-timeout", required_argument, 0, OPT_TASK_TIMEOUT}, {"overwrite", no_argument, 0, OPT_OVERWRITE}, {"no-docker", no_argument, 0, OPT_NO_DOCKER}, {"no-artifactory", no_argument, 0, OPT_NO_ARTIFACTORY}, @@ -20,6 +21,7 @@ struct option long_options[] = { {"no-artifactory-upload", no_argument, 0, OPT_NO_ARTIFACTORY_UPLOAD}, {"no-testing", no_argument, 0, OPT_NO_TESTING}, {"no-parallel", no_argument, 0, OPT_NO_PARALLEL}, + {"no-task-logging", no_argument, 0, OPT_NO_TASK_LOGGING}, {"no-rewrite", no_argument, 0, OPT_NO_REWRITE_SPEC_STAGE_2}, {0, 0, 0, 0}, }; @@ -36,6 +38,7 @@ const char *long_options_help[] = { "Disable line buffering", "Update conda installation prior to STASIS environment creation", "On error, immediately terminate all tasks", + "Terminate task after timeout is reached (#s, #m, #h)", "Overwrite an existing release", "Do not build docker images", "Do not upload artifacts to Artifactory", @@ -43,6 +46,7 @@ const char *long_options_help[] = { "Do not upload artifacts to Artifactory (dry-run)", "Do not execute test scripts", "Do not execute tests in parallel", + "Do not log task output (write to stdout)", "Do not rewrite paths and URLs in output files", NULL, }; diff --git a/src/cli/stasis/include/args.h b/src/cli/stasis/include/args.h index 5bad752..5536735 100644 --- a/src/cli/stasis/include/args.h +++ b/src/cli/stasis/include/args.h @@ -17,6 +17,8 @@ #define OPT_FAIL_FAST 1009 #define OPT_NO_PARALLEL 1010 #define OPT_POOL_STATUS_INTERVAL 1011 +#define OPT_NO_TASK_LOGGING 1012 +#define OPT_TASK_TIMEOUT 1013 extern struct option long_options[]; void usage(char *progname); diff --git a/src/cli/stasis/stasis_main.c b/src/cli/stasis/stasis_main.c index 7f0b88a..44ee6d7 100644 --- a/src/cli/stasis/stasis_main.c +++ b/src/cli/stasis/stasis_main.c @@ -10,140 +10,7 @@ #include "system_requirements.h" #include "tpl.h" - -int main(int argc, char *argv[]) { - struct Delivery ctx; - struct Process proc = { - .f_stdout = "", - .f_stderr = "", - .redirect_stderr = 0, - }; - char env_name[STASIS_NAME_MAX] = {0}; - char env_name_testing[STASIS_NAME_MAX] = {0}; - char *delivery_input = NULL; - char *config_input = NULL; - char installer_url[PATH_MAX]; - char python_override_version[STASIS_NAME_MAX]; - int user_disabled_docker = false; - globals.cpu_limit = get_cpu_count(); - if (globals.cpu_limit > 1) { - globals.cpu_limit--; // max - 1 - } - - memset(env_name, 0, sizeof(env_name)); - memset(env_name_testing, 0, sizeof(env_name_testing)); - memset(installer_url, 0, sizeof(installer_url)); - memset(python_override_version, 0, sizeof(python_override_version)); - memset(&proc, 0, sizeof(proc)); - memset(&ctx, 0, sizeof(ctx)); - - int c; - int option_index = 0; - while ((c = getopt_long(argc, argv, "hVCc:p:vU", long_options, &option_index)) != -1) { - switch (c) { - case 'h': - usage(path_basename(argv[0])); - exit(0); - case 'V': - puts(VERSION); - exit(0); - case 'c': - config_input = strdup(optarg); - break; - case 'C': - globals.continue_on_error = true; - break; - case 'p': - strcpy(python_override_version, optarg); - break; - case 'l': - globals.cpu_limit = strtol(optarg, NULL, 10); - if (globals.cpu_limit <= 1) { - globals.cpu_limit = 1; - globals.enable_parallel = false; // No point - } - break; - case OPT_ALWAYS_UPDATE_BASE: - globals.always_update_base_environment = true; - break; - case OPT_FAIL_FAST: - globals.parallel_fail_fast = true; - break; - case OPT_POOL_STATUS_INTERVAL: - globals.pool_status_interval = (int) strtol(optarg, NULL, 10); - if (globals.pool_status_interval < 1) { - globals.pool_status_interval = 1; - } else if (globals.pool_status_interval > 60 * 10) { - // Possible poor choice alert - fprintf(stderr, "Caution: Excessive pausing between status updates may cause third-party CI/CD" - " jobs to fail if the stdout/stderr streams are idle for too long!\n"); - } - break; - case 'U': - setenv("PYTHONUNBUFFERED", "1", 1); - fflush(stdout); - fflush(stderr); - setvbuf(stdout, NULL, _IONBF, 0); - setvbuf(stderr, NULL, _IONBF, 0); - break; - case 'v': - globals.verbose = true; - break; - case OPT_OVERWRITE: - globals.enable_overwrite = true; - break; - case OPT_NO_DOCKER: - globals.enable_docker = false; - user_disabled_docker = true; - break; - case OPT_NO_ARTIFACTORY: - globals.enable_artifactory = false; - break; - case OPT_NO_ARTIFACTORY_BUILD_INFO: - globals.enable_artifactory_build_info = false; - break; - case OPT_NO_ARTIFACTORY_UPLOAD: - globals.enable_artifactory_build_info = false; - globals.enable_artifactory_upload = false; - break; - case OPT_NO_TESTING: - globals.enable_testing = false; - break; - case OPT_NO_REWRITE_SPEC_STAGE_2: - globals.enable_rewrite_spec_stage_2 = false; - break; - case OPT_NO_PARALLEL: - globals.enable_parallel = false; - break; - case '?': - default: - exit(1); - } - } - - if (optind < argc) { - while (optind < argc) { - // use first positional argument - delivery_input = argv[optind++]; - break; - } - } - - if (!delivery_input) { - fprintf(stderr, "error: a DELIVERY_FILE is required\n"); - usage(path_basename(argv[0])); - exit(1); - } - - printf(BANNER, VERSION, AUTHOR); - - check_system_path(); - - msg(STASIS_MSG_L1, "Setup\n"); - - tpl_setup_vars(&ctx); - tpl_setup_funcs(&ctx); - +static void setup_sysconfdir() { // Set up PREFIX/etc directory information // The user may manipulate the base directory path with STASIS_SYSCONFDIR // environment variable @@ -159,83 +26,108 @@ int main(int argc, char *argv[]) { msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Unable to resolve path to configuration directory: %s\n", stasis_sysconfdir_tmp); exit(1); } +} +static void setup_python_version_override(struct Delivery *ctx, const char *version) { // Override Python version from command-line, if any - if (strlen(python_override_version)) { - guard_free(ctx.meta.python); - ctx.meta.python = strdup(python_override_version); - guard_free(ctx.meta.python_compact); - ctx.meta.python_compact = to_short_version(ctx.meta.python); + if (strlen(version)) { + guard_free(ctx->meta.python); + ctx->meta.python = strdup(version); + if (!ctx->meta.python) { + SYSERROR("%s", "Unable to allocate bytes for python version override"); + } + guard_free(ctx->meta.python_compact); + ctx->meta.python_compact = to_short_version(ctx->meta.python); } +} - if (!config_input) { - // no configuration passed by argument. use basic config. +static void configure_stasis_ini(struct Delivery *ctx, char **config_input) { + if (!*config_input) { + SYSDEBUG("%s", "No configuration passed by argument. Using basic config."); char cfgfile[PATH_MAX * 2]; sprintf(cfgfile, "%s/%s", globals.sysconfdir, "stasis.ini"); + SYSDEBUG("cfgfile: %s", cfgfile); if (!access(cfgfile, F_OK | R_OK)) { - config_input = strdup(cfgfile); + *config_input = strdup(cfgfile); } else { msg(STASIS_MSG_WARN, "STASIS global configuration is not readable, or does not exist: %s", cfgfile); } } - if (config_input) { - msg(STASIS_MSG_L2, "Reading STASIS global configuration: %s\n", config_input); - ctx._stasis_ini_fp.cfg = ini_open(config_input); - if (!ctx._stasis_ini_fp.cfg) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read config file: %s, %s\n", delivery_input, strerror(errno)); - exit(1); - } - ctx._stasis_ini_fp.cfg_path = strdup(config_input); - guard_free(config_input); + msg(STASIS_MSG_L2, "Reading STASIS global configuration: %s\n", *config_input); + ctx->_stasis_ini_fp.cfg = ini_open(*config_input); + if (!ctx->_stasis_ini_fp.cfg) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read config file: %s, %s\n", *config_input, strerror(errno)); + exit(1); + } + ctx->_stasis_ini_fp.cfg_path = strdup(*config_input); + if (!ctx->_stasis_ini_fp.cfg_path) { + SYSERROR("%s", "Failed to allocate memory for config file name"); + exit(1); } + guard_free(*config_input); +} - msg(STASIS_MSG_L2, "Reading STASIS delivery configuration: %s\n", delivery_input); - ctx._stasis_ini_fp.delivery = ini_open(delivery_input); - if (!ctx._stasis_ini_fp.delivery) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read delivery file: %s, %s\n", delivery_input, strerror(errno)); +static void configure_delivery_ini(struct Delivery *ctx, char **delivery_input) { + msg(STASIS_MSG_L2, "Reading STASIS delivery configuration: %s\n", *delivery_input); + ctx->_stasis_ini_fp.delivery = ini_open(*delivery_input); + if (!ctx->_stasis_ini_fp.delivery) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read delivery file: %s, %s\n", *delivery_input, strerror(errno)); exit(1); } - ctx._stasis_ini_fp.delivery_path = strdup(delivery_input); + ctx->_stasis_ini_fp.delivery_path = strdup(*delivery_input); +} +static void configure_delivery_context(struct Delivery *ctx) { msg(STASIS_MSG_L2, "Bootstrapping delivery context\n"); - if (bootstrap_build_info(&ctx)) { + if (bootstrap_build_info(ctx)) { msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to bootstrap delivery context\n"); exit(1); } msg(STASIS_MSG_L2, "Initializing delivery context\n"); - if (delivery_init(&ctx, INI_READ_RENDER)) { + if (delivery_init(ctx, INI_READ_RENDER)) { msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to initialize delivery context\n"); exit(1); } - check_requirements(&ctx); +} +static void configure_jfrog_cli(struct Delivery *ctx) { msg(STASIS_MSG_L2, "Configuring JFrog CLI\n"); - if (delivery_init_artifactory(&ctx)) { + if (delivery_init_artifactory(ctx)) { msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "JFrog CLI configuration failed\n"); exit(1); } +} - runtime_apply(ctx.runtime.environ); - strcpy(env_name, ctx.info.release_name); - strcpy(env_name_testing, env_name); - strcat(env_name_testing, "-test"); - +static void check_release_history(struct Delivery *ctx) { // Safety gate: Avoid clobbering a delivered release unless the user wants that behavior msg(STASIS_MSG_L1, "Checking release history\n"); - if (!globals.enable_overwrite && delivery_exists(&ctx) == DELIVERY_FOUND) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Refusing to overwrite release: %s\nUse --overwrite to enable release clobbering.\n", ctx.info.release_name); + if (!globals.enable_overwrite && delivery_exists(ctx) == DELIVERY_FOUND) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Refusing to overwrite release: %s\nUse --overwrite to enable release clobbering.\n", ctx->info.release_name); exit(1); } +} + +static void check_conda_install_prefix(const struct Delivery *ctx) { + // Unlikely to occur: this should help prevent rmtree() from destroying your entire filesystem + // if path is "/" then, die + // or if empty string, die + if (!strcmp(ctx->storage.conda_install_prefix, DIR_SEP) || !strlen(ctx->storage.conda_install_prefix)) { + fprintf(stderr, "error: ctx.storage.conda_install_prefix is malformed!\n"); + exit(1); + } +} + +static void sync_release_history(struct Delivery *ctx) { if (globals.enable_artifactory) { // We need to download previous revisions to ensure processed packages are available at build-time // This is also a docker requirement. Python wheels must exist locally. - if (ctx.meta.rc > 1) { - msg(STASIS_MSG_L1, "Syncing delivery artifacts for %s\n", ctx.info.build_name); - if (delivery_series_sync(&ctx) != 0) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Unable to sync artifacts for %s\n", ctx.info.build_name); + if (ctx->meta.rc > 1) { + msg(STASIS_MSG_L1, "Syncing delivery artifacts for %s\n", ctx->info.build_name); + if (delivery_series_sync(ctx) != 0) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Unable to sync artifacts for %s\n", ctx->info.build_name); msg(STASIS_MSG_L3, "Case #1:\n" "\tIf this is a new 'version', and 'rc' is greater " "than 1, then no previous deliveries exist remotely. " @@ -248,45 +140,43 @@ int main(int argc, char *argv[]) { } } } +} - // Unlikely to occur: this should help prevent rmtree() from destroying your entire filesystem - // if path is "/" then, die - // or if empty string, die - if (!strcmp(ctx.storage.conda_install_prefix, DIR_SEP) || !strlen(ctx.storage.conda_install_prefix)) { - fprintf(stderr, "error: ctx.storage.conda_install_prefix is malformed!\n"); - exit(1); - } - +static void check_conda_prefix_length(const struct Delivery *ctx) { // 2 = #! // 5 = /bin\n - const size_t prefix_len = strlen(ctx.storage.conda_install_prefix) + 2 + 5; + const size_t prefix_len = strlen(ctx->storage.conda_install_prefix) + 2 + 5; const size_t prefix_len_max = 127; msg(STASIS_MSG_L1, "Checking length of conda installation prefix\n"); - if (!strcmp(ctx.system.platform[DELIVERY_PLATFORM], "Linux") && prefix_len > prefix_len_max) { + if (!strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Linux") && prefix_len > prefix_len_max) { msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "The shebang, '#!%s/bin/python\\n' is too long (%zu > %zu).\n", - ctx.storage.conda_install_prefix, prefix_len, prefix_len_max); + ctx->storage.conda_install_prefix, prefix_len, prefix_len_max); msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "Conda's workaround to handle long path names does not work consistently within STASIS.\n"); msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "Please try again from a different, \"shorter\", directory.\n"); exit(1); } +} +static void setup_conda(struct Delivery *ctx, char *installer_url) { msg(STASIS_MSG_L1, "Conda setup\n"); - delivery_get_conda_installer_url(&ctx, installer_url); + delivery_get_conda_installer_url(ctx, installer_url); msg(STASIS_MSG_L2, "Downloading: %s\n", installer_url); - if (delivery_get_conda_installer(&ctx, installer_url)) { + if (delivery_get_conda_installer(ctx, installer_url)) { msg(STASIS_MSG_ERROR, "download failed: %s\n", installer_url); exit(1); } - msg(STASIS_MSG_L2, "Installing: %s\n", ctx.conda.installer_name); - delivery_install_conda(ctx.conda.installer_path, ctx.storage.conda_install_prefix); + msg(STASIS_MSG_L2, "Installing: %s\n", ctx->conda.installer_name); + delivery_install_conda(ctx->conda.installer_path, ctx->storage.conda_install_prefix); - msg(STASIS_MSG_L2, "Configuring: %s\n", ctx.storage.conda_install_prefix); - delivery_conda_enable(&ctx, ctx.storage.conda_install_prefix); + msg(STASIS_MSG_L2, "Configuring: %s\n", ctx->storage.conda_install_prefix); + delivery_conda_enable(ctx, ctx->storage.conda_install_prefix); +} +static void configure_conda_base(struct Delivery *ctx, char *envs[]) { // // Implied environment creation modes/actions // @@ -303,268 +193,247 @@ int main(int argc, char *argv[]) { // 3b. Bugs, conflicts, and dependency resolution issues are inherited and // must be handled in the INI config msg(STASIS_MSG_L1, "Creating release environment(s)\n"); - char *mission_base = NULL; - if (isempty(ctx.meta.based_on)) { - guard_free(ctx.meta.based_on); + + if (isempty(ctx->meta.based_on)) { + // based_on was not set by the input file + + guard_free(ctx->meta.based_on); char *mission_base_orig = NULL; - if (asprintf(&mission_base_orig, "%s/%s/base.yml", ctx.storage.mission_dir, ctx.meta.mission) < 0) { - SYSERROR("Unable to allocate bytes for %s/%s/base.yml path\n", ctx.storage.mission_dir, ctx.meta.mission); + if (asprintf(&mission_base_orig, "%s/%s/base.yml", ctx->storage.mission_dir, ctx->meta.mission) < 0) { + SYSERROR("Unable to allocate bytes for %s/%s/base.yml path\n", ctx->storage.mission_dir, ctx->meta.mission); exit(1); } + // Does a base.yml exist in the mission directory? + // If not, do nothing. Otherwise, use the base.yml in the mission directory. if (access(mission_base_orig, F_OK) < 0) { - msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "Mission does not provide a base.yml configuration: %s (%s)\n", - ctx.meta.mission, ctx.storage.mission_dir); + msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "Mission does not provide a base.yml"); } else { msg(STASIS_MSG_L2, "Using base environment configuration: %s\n", mission_base_orig); - if (asprintf(&mission_base, "%s/%s-base.yml", ctx.storage.tmpdir, ctx.info.release_name) < 0) { + if (asprintf(&mission_base, "%s/%s-base.yml", ctx->storage.tmpdir, ctx->info.release_name) < 0) { SYSERROR("%s", "Unable to allocate bytes for temporary base.yml configuration"); remove(mission_base); exit(1); } copy2(mission_base_orig, mission_base, CT_OWNER | CT_PERM); - ctx.meta.based_on = mission_base; + ctx->meta.based_on = mission_base; } guard_free(mission_base_orig); } - if (!isempty(ctx.meta.based_on)) { - if (conda_env_exists(ctx.storage.conda_install_prefix, env_name) && conda_env_remove(env_name)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to remove release environment: %s\n", env_name); - exit(1); - } + msg(STASIS_MSG_L2, "Based on: %s\n", ctx->meta.based_on); - msg(STASIS_MSG_L2, "Based on: %s\n", ctx.meta.based_on); - if (conda_env_create_from_uri(env_name, ctx.meta.based_on, ctx.meta.python)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "unable to install release environment using configuration file\n"); - exit(1); - } + for (size_t i = 0; envs[i] != NULL; i += 2) { + char *title = envs[i]; + char *env = envs[i+1]; + // If based_on was populated above, or defined in the configuration: install its packages. + if (!isempty(ctx->meta.based_on)) { + if (conda_env_exists(ctx->storage.conda_install_prefix, env) && conda_env_remove(env)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to remove %s environment: %s\n", title); + exit(1); + } - if (conda_env_exists(ctx.storage.conda_install_prefix, env_name_testing) && conda_env_remove(env_name_testing)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to remove testing environment %s\n", env_name_testing); - exit(1); - } - if (conda_env_create_from_uri(env_name_testing, ctx.meta.based_on, ctx.meta.python)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "unable to install testing environment using configuration file\n"); - exit(1); - } - } else { - if (conda_env_create(env_name, ctx.meta.python, NULL)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to create release environment\n"); - exit(1); - } - if (conda_env_create(env_name_testing, ctx.meta.python, NULL)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to create testing environment\n"); - exit(1); + if (conda_env_create_from_uri(env, ctx->meta.based_on, ctx->meta.python)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "unable to install %s environment using configuration file\n", title); + exit(1); + } + } else { + // Otherwise, create the environments with the requested Python version and move on + if (conda_env_create(env, ctx->meta.python, NULL)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to create %s environment\n", title); + exit(1); + } } } // The base environment configuration not used past this point remove(mission_base); +} - const char *envs[] = {env_name_testing, env_name}; - for (size_t e = 0; e < sizeof(envs) / sizeof(*envs); e++) { - const char *name = envs[e]; - if (ctx.conda.conda_packages_purge && strlist_count(ctx.conda.conda_packages_purge)) { - msg(STASIS_MSG_L2, "Purging conda packages from %s\n", name); - if (delivery_purge_packages(&ctx, name, PKG_USE_CONDA)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "unable to purge requested conda packages from %s\n", name); - exit(1); - } - } - - if (ctx.conda.pip_packages_purge && strlist_count(ctx.conda.pip_packages_purge)) { - msg(STASIS_MSG_L2, "Purging pip packages from %s\n", name); - if (delivery_purge_packages(&ctx, env_name_testing, PKG_USE_PIP)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "unable to purge requested pip packages from %s\n", - env_name_testing); - exit(1); +static void configure_conda_purge(struct Delivery *ctx, char *envs[]) { + struct StrList *purge_list[] = { + ctx->conda.conda_packages_purge, + ctx->conda.pip_packages_purge + }; + for (size_t i = 0; i < sizeof(purge_list) / sizeof(purge_list[0]); i++) { + struct StrList *to_purge = purge_list[i]; + for (size_t e = 0; envs[e] != NULL; e += 2) { + //const char *title = envs[e]; // unused + const char *env = envs[e+1]; + if (to_purge && strlist_count(to_purge)) { + const char *pkg_manager_name[] = { + "conda", + "pip" + }; + const int pkg_manager_use[] = { + PKG_USE_CONDA, + PKG_USE_PIP + }; + const char *manager_str = pkg_manager_name[i]; + const int manager_flag = pkg_manager_use[i]; + msg(STASIS_MSG_L2, "Purging %s packages from %s\n", manager_str, env); + if (delivery_purge_packages(ctx, env, manager_flag)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "unable to purge requested %s packages from %s\n", manager_str, env); + exit(1); + } } } } +} +static void setup_activate_test_env(const struct Delivery *ctx, const char *env_name_testing) { // Activate test environment msg(STASIS_MSG_L1, "Activating test environment\n"); - if (conda_activate(ctx.storage.conda_install_prefix, env_name_testing)) { + if (conda_activate(ctx->storage.conda_install_prefix, env_name_testing)) { fprintf(stderr, "failed to activate test environment\n"); exit(1); } +} - if (delivery_gather_tool_versions(&ctx)) { - if (!ctx.conda.tool_version) { +static void configure_tool_versions(struct Delivery *ctx) { + if (delivery_gather_tool_versions(ctx)) { + if (!ctx->conda.tool_version) { msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Could not determine conda version\n"); exit(1); } - if (!ctx.conda.tool_build_version) { + if (!ctx->conda.tool_build_version) { msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Could not determine conda-build version\n"); exit(1); } } +} +static void install_build_package() { if (pip_exec("install build")) { msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "'build' tool installation failed\n"); exit(1); } +} - if (!isempty(ctx.meta.based_on)) { +static void configure_package_overlay(struct Delivery *ctx, const char *env_name) { + if (!isempty(ctx->meta.based_on)) { msg(STASIS_MSG_L1, "Generating package overlay from environment: %s\n", env_name); - if (delivery_overlay_packages_from_env(&ctx, env_name)) { + if (delivery_overlay_packages_from_env(ctx, env_name)) { msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "%s", "Failed to generate package overlay. Resulting environment integrity cannot be guaranteed.\n"); exit(1); } } +} +static void configure_deferred_packages(struct Delivery *ctx) { msg(STASIS_MSG_L1, "Filter deliverable packages\n"); - delivery_defer_packages(&ctx, DEFER_CONDA); - delivery_defer_packages(&ctx, DEFER_PIP); + delivery_defer_packages(ctx, DEFER_CONDA); + delivery_defer_packages(ctx, DEFER_PIP); +} + +static void show_overiew(struct Delivery *ctx) { msg(STASIS_MSG_L1, "Overview\n"); - delivery_meta_show(&ctx); - delivery_conda_show(&ctx); + delivery_meta_show(ctx); + delivery_conda_show(ctx); if (globals.verbose) { //delivery_runtime_show(&ctx); } +} +static void run_tests(struct Delivery *ctx) { // Execute configuration-defined tests if (globals.enable_testing) { - delivery_tests_show(&ctx); + delivery_tests_show(ctx); msg(STASIS_MSG_L1, "Begin test execution\n"); - delivery_tests_run(&ctx); + delivery_tests_run(ctx); msg(STASIS_MSG_L2, "Rewriting test results\n"); - delivery_fixup_test_results(&ctx); + delivery_fixup_test_results(ctx); } else { msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Test execution is disabled\n"); } +} - if (ctx.conda.conda_packages_defer && strlist_count(ctx.conda.conda_packages_defer)) { +static void build_conda_recipes(struct Delivery *ctx) { + if (ctx->conda.conda_packages_defer && strlist_count(ctx->conda.conda_packages_defer)) { msg(STASIS_MSG_L2, "Building Conda recipe(s)\n"); - if (delivery_build_recipes(&ctx)) { + if (delivery_build_recipes(ctx)) { exit(1); } msg(STASIS_MSG_L3, "Copying artifacts\n"); - if (delivery_copy_conda_artifacts(&ctx)) { + if (delivery_copy_conda_artifacts(ctx)) { exit(1); } msg(STASIS_MSG_L3, "Indexing artifacts\n"); - if (delivery_index_conda_artifacts(&ctx)) { + if (delivery_index_conda_artifacts(ctx)) { exit(1); } } +} - if (strlist_count(ctx.conda.pip_packages_defer)) { - if (!((ctx.conda.wheels_packages = delivery_build_wheels(&ctx)))) { +static void build_wheel_packages(struct Delivery *ctx) { + if (strlist_count(ctx->conda.pip_packages_defer)) { + msg(STASIS_MSG_L2, "Building Python wheels(s)\n"); + if (!((ctx->conda.wheels_packages = delivery_build_wheels(ctx)))) { exit(1); } - if (delivery_index_wheel_artifacts(&ctx)) { + if (delivery_index_wheel_artifacts(ctx)) { exit(1); } } +} - // Populate the release environment - msg(STASIS_MSG_L1, "Populating release environment\n"); +static void release_install_conda_packages(struct Delivery *ctx, char *env_name) { msg(STASIS_MSG_L2, "Installing conda packages\n"); - if (strlist_count(ctx.conda.conda_packages)) { - if (delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_CONDA, (struct StrList *[]) {ctx.conda.conda_packages, NULL})) { + if (strlist_count(ctx->conda.conda_packages)) { + if (delivery_install_packages(ctx, ctx->storage.conda_install_prefix, env_name, INSTALL_PKG_CONDA, (struct StrList *[]) {ctx->conda.conda_packages, NULL})) { exit(1); } } - if (strlist_count(ctx.conda.conda_packages_defer)) { + if (strlist_count(ctx->conda.conda_packages_defer)) { msg(STASIS_MSG_L3, "Installing deferred conda packages\n"); - if (delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_CONDA | INSTALL_PKG_CONDA_DEFERRED, (struct StrList *[]) {ctx.conda.conda_packages_defer, NULL})) { + if (delivery_install_packages(ctx, ctx->storage.conda_install_prefix, env_name, INSTALL_PKG_CONDA | INSTALL_PKG_CONDA_DEFERRED, (struct StrList *[]) {ctx->conda.conda_packages_defer, NULL})) { exit(1); } } else { msg(STASIS_MSG_L3, "No deferred conda packages\n"); } +} +static void release_install_pip_packages(struct Delivery *ctx, char *env_name) { msg(STASIS_MSG_L2, "Installing pip packages\n"); - if (strlist_count(ctx.conda.pip_packages)) { - if (delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_PIP, (struct StrList *[]) {ctx.conda.pip_packages, NULL})) { + if (strlist_count(ctx->conda.pip_packages)) { + if (delivery_install_packages(ctx, ctx->storage.conda_install_prefix, env_name, INSTALL_PKG_PIP, (struct StrList *[]) {ctx->conda.pip_packages, NULL})) { exit(1); } } - if (strlist_count(ctx.conda.pip_packages_defer)) { + if (strlist_count(ctx->conda.pip_packages_defer)) { msg(STASIS_MSG_L3, "Installing deferred pip packages\n"); - if (delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_PIP | INSTALL_PKG_PIP_DEFERRED, (struct StrList *[]) {ctx.conda.pip_packages_defer, NULL})) { + if (delivery_install_packages(ctx, ctx->storage.conda_install_prefix, env_name, INSTALL_PKG_PIP | INSTALL_PKG_PIP_DEFERRED, (struct StrList *[]) {ctx->conda.pip_packages_defer, NULL})) { exit(1); } } else { msg(STASIS_MSG_L3, "No deferred pip packages\n"); } +} - conda_exec("list"); - - msg(STASIS_MSG_L1, "Creating release\n"); - msg(STASIS_MSG_L2, "Exporting delivery configuration\n"); - if (!pushd(ctx.storage.cfgdump_dir)) { - char filename[PATH_MAX] = {0}; - sprintf(filename, "%s.ini", ctx.info.release_name); - FILE *spec = fopen(filename, "w+"); - if (!spec) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", filename); - exit(1); - } - ini_write(ctx._stasis_ini_fp.delivery, &spec, INI_WRITE_RAW); - fclose(spec); - - memset(filename, 0, sizeof(filename)); - sprintf(filename, "%s-rendered.ini", ctx.info.release_name); - spec = fopen(filename, "w+"); - if (!spec) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", filename); - exit(1); - } - ini_write(ctx._stasis_ini_fp.delivery, &spec, INI_WRITE_PRESERVE); - fclose(spec); - popd(); - } else { - SYSERROR("Failed to enter directory: %s", ctx.storage.delivery_dir); - exit(1); - } - - msg(STASIS_MSG_L2, "Exporting %s\n", env_name_testing); - if (conda_env_export(env_name_testing, ctx.storage.delivery_dir, env_name_testing)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", env_name_testing); - exit(1); - } - - msg(STASIS_MSG_L2, "Exporting %s\n", env_name); - if (conda_env_export(env_name, ctx.storage.delivery_dir, env_name)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", env_name); - exit(1); - } - - // Rewrite release environment output (i.e. set package origin(s) to point to the deployment server, etc.) - char specfile[PATH_MAX]; - sprintf(specfile, "%s/%s.yml", ctx.storage.delivery_dir, env_name); - msg(STASIS_MSG_L3, "Rewriting release spec file (stage 1): %s\n", path_basename(specfile)); - delivery_rewrite_spec(&ctx, specfile, DELIVERY_REWRITE_SPEC_STAGE_1); - - msg(STASIS_MSG_L1, "Rendering mission templates\n"); - delivery_mission_render_files(&ctx); - - int want_docker = ini_section_search(&ctx._stasis_ini_fp.delivery, INI_SEARCH_BEGINS, "deploy:docker") ? true : false; - int want_artifactory = ini_section_search(&ctx._stasis_ini_fp.delivery, INI_SEARCH_BEGINS, "deploy:artifactory") ? true : false; +static void build_docker(struct Delivery *ctx, const int disabled) { + const int want_docker = ini_section_search(&ctx->_stasis_ini_fp.delivery, INI_SEARCH_BEGINS, "deploy:docker") ? true : false; if (want_docker) { - if (user_disabled_docker) { + if (disabled) { msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Docker image building is disabled by CLI argument\n"); } else { char dockerfile[PATH_MAX] = {0}; - sprintf(dockerfile, "%s/%s", ctx.storage.build_docker_dir, "Dockerfile"); + sprintf(dockerfile, "%s/%s", ctx->storage.build_docker_dir, "Dockerfile"); if (globals.enable_docker) { if (!access(dockerfile, F_OK)) { msg(STASIS_MSG_L1, "Building Docker image\n"); - if (delivery_docker(&ctx)) { + if (delivery_docker(ctx)) { msg(STASIS_MSG_L1 | STASIS_MSG_ERROR, "Failed to build docker image!\n"); COE_CHECK_ABORT(1, "Failed to build docker image"); } } else { - msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Docker image building is disabled. No Dockerfile found in %s\n", ctx.storage.build_docker_dir); + msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Docker image building is disabled. No Dockerfile found in %s\n", ctx->storage.build_docker_dir); } } else { msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Docker image building is disabled. System configuration error\n"); @@ -573,25 +442,235 @@ int main(int argc, char *argv[]) { } else { msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Docker image building is disabled. deploy:docker is not configured\n"); } +} + +static void generate_release(struct Delivery *ctx, char *env_name, char *env_name_testing, const int disable_docker) { + // Populate the release environment + msg(STASIS_MSG_L1, "Populating release environment\n"); + release_install_conda_packages(ctx, env_name); + release_install_pip_packages(ctx, env_name); + + conda_exec("list"); - msg(STASIS_MSG_L3, "Rewriting release spec file (stage 2): %s\n", path_basename(specfile)); - delivery_rewrite_spec(&ctx, specfile, DELIVERY_REWRITE_SPEC_STAGE_2); + msg(STASIS_MSG_L1, "Creating release\n"); + delivery_export(ctx, (char *[]) {env_name, env_name_testing, NULL}); + + char specfile[PATH_MAX]; + sprintf(specfile, "%s/%s.yml", ctx->storage.delivery_dir, env_name); + + delivery_rewrite_stage1(ctx, specfile); + build_docker(ctx, disable_docker); + delivery_rewrite_stage2(ctx, specfile); msg(STASIS_MSG_L1, "Dumping metadata\n"); - if (delivery_dump_metadata(&ctx)) { + if (delivery_dump_metadata(ctx)) { msg(STASIS_MSG_L1 | STASIS_MSG_ERROR, "Metadata dump failed\n"); } +} +static void transfer_artifacts(struct Delivery *ctx) { + const int want_artifactory = ini_section_search(&ctx->_stasis_ini_fp.delivery, INI_SEARCH_BEGINS, "deploy:artifactory") ? true : false; if (want_artifactory) { if (globals.enable_artifactory && globals.enable_artifactory_upload) { msg(STASIS_MSG_L1, "Uploading artifacts\n"); - delivery_artifact_upload(&ctx); + delivery_artifact_upload(ctx); } else { msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Artifactory upload is disabled by CLI argument\n"); } } else { msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Artifactory upload is disabled. deploy:artifactory is not configured\n"); } +} + +int main(int argc, char *argv[]) { + struct Delivery ctx; + struct Process proc = { + .f_stdout = "", + .f_stderr = "", + .redirect_stderr = 0, + }; + char env_name[STASIS_NAME_MAX] = {0}; + char env_name_testing[STASIS_NAME_MAX] = {0}; + char *delivery_input = NULL; + char *config_input = NULL; + char installer_url[PATH_MAX]; + char python_override_version[STASIS_NAME_MAX]; + int user_disabled_docker = false; + globals.cpu_limit = get_cpu_count(); + if (globals.cpu_limit > 1) { + globals.cpu_limit--; // max - 1 + } + + memset(env_name, 0, sizeof(env_name)); + memset(env_name_testing, 0, sizeof(env_name_testing)); + memset(installer_url, 0, sizeof(installer_url)); + memset(python_override_version, 0, sizeof(python_override_version)); + memset(&proc, 0, sizeof(proc)); + memset(&ctx, 0, sizeof(ctx)); + + int c; + int option_index = 0; + while ((c = getopt_long(argc, argv, "hVCc:p:vU", long_options, &option_index)) != -1) { + switch (c) { + case 'h': + usage(path_basename(argv[0])); + exit(0); + case 'V': + puts(VERSION); + exit(0); + case 'c': + config_input = strdup(optarg); + break; + case 'C': + globals.continue_on_error = true; + break; + case 'p': + strcpy(python_override_version, optarg); + break; + case 'l': + globals.cpu_limit = strtol(optarg, NULL, 10); + if (globals.cpu_limit <= 1) { + globals.cpu_limit = 1; + globals.enable_parallel = false; // No point + } + break; + case OPT_ALWAYS_UPDATE_BASE: + globals.always_update_base_environment = true; + break; + case OPT_FAIL_FAST: + globals.parallel_fail_fast = true; + break; + case OPT_TASK_TIMEOUT: + globals.task_timeout = str_to_timeout(optarg); + if (globals.task_timeout < 0) { + fprintf(stderr, "Invalid timeout: %s\n", optarg); + if (globals.task_timeout == STR_TO_TIMEOUT_INVALID_TIME_SCALE) { + fprintf(stderr, "Use format '#s' (seconds), '#m' (minutes), '#h' (hours)\n"); + } else if (globals.task_timeout == STR_TO_TIMEOUT_NEGATIVE) { + fprintf(stderr, "Timeout cannot be negative\n"); + } + exit(1); + } + break; + case OPT_POOL_STATUS_INTERVAL: + globals.pool_status_interval = (int) strtol(optarg, NULL, 10); + if (globals.pool_status_interval < 1) { + globals.pool_status_interval = 1; + } else if (globals.pool_status_interval > 60 * 10) { + // Possible poor choice alert + fprintf(stderr, "Caution: Excessive pausing between status updates may cause third-party CI/CD" + " jobs to fail if the stdout/stderr streams are idle for too long!\n"); + } + break; + case 'U': + setenv("PYTHONUNBUFFERED", "1", 1); + fflush(stdout); + fflush(stderr); + setvbuf(stdout, NULL, _IONBF, 0); + setvbuf(stderr, NULL, _IONBF, 0); + break; + case 'v': + globals.verbose = true; + break; + case OPT_OVERWRITE: + globals.enable_overwrite = true; + break; + case OPT_NO_DOCKER: + globals.enable_docker = false; + user_disabled_docker = true; + break; + case OPT_NO_ARTIFACTORY: + globals.enable_artifactory = false; + break; + case OPT_NO_ARTIFACTORY_BUILD_INFO: + globals.enable_artifactory_build_info = false; + break; + case OPT_NO_ARTIFACTORY_UPLOAD: + globals.enable_artifactory_build_info = false; + globals.enable_artifactory_upload = false; + break; + case OPT_NO_TESTING: + globals.enable_testing = false; + break; + case OPT_NO_REWRITE_SPEC_STAGE_2: + globals.enable_rewrite_spec_stage_2 = false; + break; + case OPT_NO_PARALLEL: + globals.enable_parallel = false; + break; + case OPT_NO_TASK_LOGGING: + globals.enable_task_logging = false; + break; + case '?': + default: + exit(1); + } + } + + if (optind < argc) { + while (optind < argc) { + // use first positional argument + delivery_input = argv[optind++]; + break; + } + } + + if (!delivery_input) { + fprintf(stderr, "error: a DELIVERY_FILE is required\n"); + usage(path_basename(argv[0])); + exit(1); + } + + printf(BANNER, VERSION, AUTHOR); + + check_system_path(); + + msg(STASIS_MSG_L1, "Setup\n"); + + tpl_setup_vars(&ctx); + tpl_setup_funcs(&ctx); + + setup_sysconfdir(); + setup_python_version_override(&ctx, python_override_version); + + configure_stasis_ini(&ctx, &config_input); + configure_delivery_ini(&ctx, &delivery_input); + configure_delivery_context(&ctx); + + check_requirements(&ctx); + configure_jfrog_cli(&ctx); + + runtime_apply(ctx.runtime.environ); + strcpy(env_name, ctx.info.release_name); + strcpy(env_name_testing, env_name); + strcat(env_name_testing, "-test"); + char *envs[] = { + "release", env_name, + "testing", env_name_testing, + NULL, NULL, + }; + + check_release_history(&ctx); + sync_release_history(&ctx); + + check_conda_install_prefix(&ctx); + check_conda_prefix_length(&ctx); + setup_conda(&ctx, installer_url); + configure_conda_base(&ctx, envs); + configure_conda_purge(&ctx, envs); + setup_activate_test_env(&ctx, env_name_testing); + + configure_tool_versions(&ctx); + install_build_package(); + configure_package_overlay(&ctx, env_name); + configure_deferred_packages(&ctx); + + show_overiew(&ctx); + run_tests(&ctx); + build_conda_recipes(&ctx); + build_wheel_packages(&ctx); + generate_release(&ctx, env_name, env_name_testing, user_disabled_docker); + transfer_artifacts(&ctx); msg(STASIS_MSG_L1, "Cleaning up\n"); delivery_free(&ctx); diff --git a/src/cli/stasis_indexer/callbacks.c b/src/cli/stasis_indexer/callbacks.c index 603aef9..20674f0 100644 --- a/src/cli/stasis_indexer/callbacks.c +++ b/src/cli/stasis_indexer/callbacks.c @@ -7,9 +7,9 @@ // qsort callback to sort delivery contexts by compact python version int callback_sort_deliveries_cmpfn(const void *a, const void *b) { - const struct Delivery *delivery1 = (struct Delivery *) a; + const struct Delivery *delivery1 = *(struct Delivery **) a; const size_t delivery1_python = strtoul(delivery1->meta.python_compact, NULL, 10); - const struct Delivery *delivery2 = (struct Delivery *) b; + const struct Delivery *delivery2 = *(struct Delivery **) b; const size_t delivery2_python = strtoul(delivery2->meta.python_compact, NULL, 10); if (delivery2_python > delivery1_python) { diff --git a/src/cli/stasis_indexer/helpers.c b/src/cli/stasis_indexer/helpers.c index 018a8f6..6dc653d 100644 --- a/src/cli/stasis_indexer/helpers.c +++ b/src/cli/stasis_indexer/helpers.c @@ -5,24 +5,24 @@ #include "core.h" #include "helpers.h" -struct StrList *get_architectures(struct Delivery ctx[], const size_t nelem) { +struct StrList *get_architectures(struct Delivery **ctx, const size_t nelem) { struct StrList *architectures = strlist_init(); for (size_t i = 0; i < nelem; i++) { - if (ctx[i].system.arch) { - if (!strstr_array(architectures->data, ctx[i].system.arch)) { - strlist_append(&architectures, ctx[i].system.arch); + if (ctx[i]->system.arch) { + if (!strstr_array(architectures->data, ctx[i]->system.arch)) { + strlist_append(&architectures, ctx[i]->system.arch); } } } return architectures; } -struct StrList *get_platforms(struct Delivery ctx[], const size_t nelem) { +struct StrList *get_platforms(struct Delivery **ctx, const size_t nelem) { struct StrList *platforms = strlist_init(); for (size_t i = 0; i < nelem; i++) { - if (ctx[i].system.platform) { - if (!strstr_array(platforms->data, ctx[i].system.platform[DELIVERY_PLATFORM_RELEASE])) { - strlist_append(&platforms, ctx[i].system.platform[DELIVERY_PLATFORM_RELEASE]); + if (ctx[i]->system.platform) { + if (!strstr_array(platforms->data, ctx[i]->system.platform[DELIVERY_PLATFORM_RELEASE])) { + strlist_append(&platforms, ctx[i]->system.platform[DELIVERY_PLATFORM_RELEASE]); } } } @@ -177,19 +177,19 @@ int micromamba_configure(const struct Delivery *ctx, struct MicromambaInfo *m) { return status; } -int get_latest_rc(struct Delivery ctx[], const size_t nelem) { +int get_latest_rc(struct Delivery **ctx, const size_t nelem) { int result = 0; for (size_t i = 0; i < nelem; i++) { - if (ctx[i].meta.rc > result) { - result = ctx[i].meta.rc; + if (ctx[i]->meta.rc > result) { + result = ctx[i]->meta.rc; } } return result; } int sort_by_latest_rc(const void *a, const void *b) { - const struct Delivery *aa = a; - const struct Delivery *bb = b; + const struct Delivery *aa = *(struct Delivery **) a; + const struct Delivery *bb = *(struct Delivery **) b; if (aa->meta.rc > bb->meta.rc) { return -1; } else if (aa->meta.rc < bb->meta.rc) { @@ -214,11 +214,11 @@ int sort_by_latest_rc(const void *a, const void *b) { } } -struct Delivery *get_latest_deliveries(struct Delivery ctx[], size_t nelem) { +struct Delivery **get_latest_deliveries(struct Delivery **ctx, size_t nelem, size_t *result_nelem) { int latest = 0; size_t n = 0; - struct Delivery *result = calloc(nelem + 1, sizeof(*result)); + struct Delivery **result = calloc(nelem + 1, sizeof(*result)); if (!result) { fprintf(stderr, "Unable to allocate %zu bytes for result delivery array: %s\n", nelem * sizeof(*result), strerror(errno)); return NULL; @@ -227,11 +227,15 @@ struct Delivery *get_latest_deliveries(struct Delivery ctx[], size_t nelem) { latest = get_latest_rc(ctx, nelem); qsort(ctx, nelem, sizeof(*ctx), sort_by_latest_rc); for (size_t i = 0; i < nelem; i++) { - if (ctx[i].meta.rc == latest) { + if (ctx[i]->meta.rc == latest) { result[n] = ctx[i]; n++; } } + + if (result_nelem) { + *result_nelem = n; + } return result; } diff --git a/src/cli/stasis_indexer/include/helpers.h b/src/cli/stasis_indexer/include/helpers.h index 9cefc05..6e2f93c 100644 --- a/src/cli/stasis_indexer/include/helpers.h +++ b/src/cli/stasis_indexer/include/helpers.h @@ -13,12 +13,12 @@ for ((COUNTER) = 0; (X)[COUNTER].MEMBER != NULL; (COUNTER)++) {} \ } while(0) -struct StrList *get_architectures(struct Delivery ctx[], size_t nelem); -struct StrList *get_platforms(struct Delivery ctx[], size_t nelem); +struct StrList *get_architectures(struct Delivery **ctx, size_t nelem); +struct StrList *get_platforms(struct Delivery **ctx, size_t nelem); int get_pandoc_version(size_t *result); int pandoc_exec(const char *in_file, const char *out_file, const char *css_file, const char *title); -int get_latest_rc(struct Delivery ctx[], size_t nelem); -struct Delivery *get_latest_deliveries(struct Delivery ctx[], size_t nelem); +int get_latest_rc(struct Delivery **ctx, size_t nelem); +struct Delivery **get_latest_deliveries(struct Delivery **ctx, size_t nelem, size_t *result_nelem); int get_files(struct StrList **out, const char *path, const char *pattern, ...); struct StrList *get_docker_images(struct Delivery *ctx, char *pattern); int load_metadata(struct Delivery *ctx, const char *filename); diff --git a/src/cli/stasis_indexer/include/junitxml_report.h b/src/cli/stasis_indexer/include/junitxml_report.h index 6d2a248..5747359 100644 --- a/src/cli/stasis_indexer/include/junitxml_report.h +++ b/src/cli/stasis_indexer/include/junitxml_report.h @@ -3,6 +3,6 @@ #include "helpers.h" -int indexer_junitxml_report(struct Delivery ctx[], size_t nelem); +int indexer_junitxml_report(struct Delivery **ctx, size_t nelem); #endif //JUNITXML_REPORT_H diff --git a/src/cli/stasis_indexer/include/readmes.h b/src/cli/stasis_indexer/include/readmes.h index d4fa7ac..e14e681 100644 --- a/src/cli/stasis_indexer/include/readmes.h +++ b/src/cli/stasis_indexer/include/readmes.h @@ -3,6 +3,6 @@ #include "helpers.h" -int indexer_readmes(struct Delivery ctx[], size_t nelem); +int indexer_readmes(struct Delivery **ctx, size_t nelem); #endif //READMES_H diff --git a/src/cli/stasis_indexer/include/website.h b/src/cli/stasis_indexer/include/website.h index e67d58b..83657a1 100644 --- a/src/cli/stasis_indexer/include/website.h +++ b/src/cli/stasis_indexer/include/website.h @@ -3,6 +3,6 @@ #include "helpers.h" -int indexer_make_website(const struct Delivery *ctx); +int indexer_make_website(struct Delivery **ctx); #endif //WEBSITE_H diff --git a/src/cli/stasis_indexer/junitxml_report.c b/src/cli/stasis_indexer/junitxml_report.c index 904a3e5..21cf729 100644 --- a/src/cli/stasis_indexer/junitxml_report.c +++ b/src/cli/stasis_indexer/junitxml_report.c @@ -96,17 +96,17 @@ static int write_report_output(struct Delivery *ctx, FILE *destfp, const char *x return 0; } -int indexer_junitxml_report(struct Delivery ctx[], const size_t nelem) { +int indexer_junitxml_report(struct Delivery **ctx, const size_t nelem) { char indexfile[PATH_MAX] = {0}; - sprintf(indexfile, "%s/README.md", ctx->storage.results_dir); + sprintf(indexfile, "%s/README.md", (*ctx)->storage.results_dir); - struct StrList *file_listing = listdir(ctx->storage.results_dir); + struct StrList *file_listing = listdir((*ctx)->storage.results_dir); if (!file_listing) { // no test results to process return 0; } - if (!pushd(ctx->storage.results_dir)) { + if (!pushd((*ctx)->storage.results_dir)) { FILE *indexfp = fopen(indexfile, "w+"); if (!indexfp) { fprintf(stderr, "Unable to open %s for writing\n", indexfile); @@ -114,21 +114,21 @@ int indexer_junitxml_report(struct Delivery ctx[], const size_t nelem) { } printf("Index %s opened for writing\n", indexfile); - int current_rc = ctx->meta.rc; + int current_rc = (*ctx)->meta.rc; for (size_t d = 0; d < nelem; d++) { char pattern[PATH_MAX] = {0}; - snprintf(pattern, sizeof(pattern) - 1, "*%s*", ctx[d].info.release_name); + snprintf(pattern, sizeof(pattern) - 1, "*%s*", ctx[d]->info.release_name); // if the result directory contains tests for this release name, print them if (!is_file_in_listing(file_listing, pattern)) { // no test results continue; } - if (current_rc > ctx[d].meta.rc) { - current_rc = ctx[d].meta.rc; + if (current_rc > ctx[d]->meta.rc) { + current_rc = ctx[d]->meta.rc; fprintf(indexfp, "\n\n---\n\n"); } - fprintf(indexfp, "### %s\n", ctx[d].info.release_name); + fprintf(indexfp, "### %s\n", ctx[d]->info.release_name); fprintf(indexfp, "\n|Suite|Duration|Total|Pass|Fail|Skip|Error|\n"); fprintf(indexfp, "|:----|:------:|:---:|:--:|:--:|:--:|:---:|\n"); @@ -139,7 +139,7 @@ int indexer_junitxml_report(struct Delivery ctx[], const size_t nelem) { continue; } if (!fnmatch(pattern, filename, 0)) { - if (write_report_output(&ctx[d], indexfp, filename)) { + if (write_report_output(ctx[d], indexfp, filename)) { // warn only SYSERROR("Unable to write xml report file using %s", filename); } @@ -150,7 +150,7 @@ int indexer_junitxml_report(struct Delivery ctx[], const size_t nelem) { fclose(indexfp); popd(); } else { - fprintf(stderr, "Unable to enter delivery directory: %s\n", ctx->storage.delivery_dir); + fprintf(stderr, "Unable to enter delivery directory: %s\n", (*ctx)->storage.delivery_dir); guard_strlist_free(&file_listing); return -1; } diff --git a/src/cli/stasis_indexer/readmes.c b/src/cli/stasis_indexer/readmes.c index 413a6a3..edc6312 100644 --- a/src/cli/stasis_indexer/readmes.c +++ b/src/cli/stasis_indexer/readmes.c @@ -1,8 +1,9 @@ #include "core.h" #include "readmes.h" -int indexer_readmes(struct Delivery ctx[], const size_t nelem) { - struct Delivery *latest_deliveries = get_latest_deliveries(ctx, nelem); +int indexer_readmes(struct Delivery **ctx, const size_t nelem) { + size_t nelem_real = 0; + struct Delivery **latest_deliveries = get_latest_deliveries(ctx, nelem, &nelem_real); if (!latest_deliveries) { if (errno) { return -1; @@ -11,17 +12,17 @@ int indexer_readmes(struct Delivery ctx[], const size_t nelem) { } char indexfile[PATH_MAX] = {0}; - sprintf(indexfile, "%s/README.md", ctx->storage.delivery_dir); + sprintf(indexfile, "%s/README.md", (*ctx)->storage.delivery_dir); FILE *indexfp = fopen(indexfile, "w+"); if (!indexfp) { fprintf(stderr, "Unable to open %s for writing\n", indexfile); return -1; } - struct StrList *archs = get_architectures(latest_deliveries, nelem); - struct StrList *platforms = get_platforms(latest_deliveries, nelem); + struct StrList *archs = get_architectures(latest_deliveries, nelem_real); + struct StrList *platforms = get_platforms(latest_deliveries, nelem_real); - fprintf(indexfp, "# %s-%s\n\n", ctx->meta.name, ctx->meta.version); + fprintf(indexfp, "# %s-%s\n\n", (*ctx)->meta.name, (*ctx)->meta.version); fprintf(indexfp, "## Current Release\n\n"); strlist_sort(platforms, STASIS_SORT_ALPHA); strlist_sort(archs, STASIS_SORT_ALPHA); @@ -31,10 +32,10 @@ int indexer_readmes(struct Delivery ctx[], const size_t nelem) { for (size_t a = 0; a < strlist_count(archs); a++) { char *arch = strlist_item(archs, a); int have_combo = 0; - for (size_t i = 0; i < nelem; i++) { - if (latest_deliveries[i].system.platform) { - if (strstr(latest_deliveries[i].system.platform[DELIVERY_PLATFORM_RELEASE], platform) && - strstr(latest_deliveries[i].system.arch, arch)) { + for (size_t i = 0; i < nelem_real; i++) { + if (latest_deliveries[i]->system.platform) { + if (strstr(latest_deliveries[i]->system.platform[DELIVERY_PLATFORM_RELEASE], platform) && + strstr(latest_deliveries[i]->system.arch, arch)) { have_combo = 1; } } @@ -43,36 +44,36 @@ int indexer_readmes(struct Delivery ctx[], const size_t nelem) { continue; } fprintf(indexfp, "### %s-%s\n\n", platform, arch); - for (size_t i = 0; i < nelem; i++) { + for (size_t i = 0; i < nelem_real; i++) { char link_name[PATH_MAX] = {0}; char readme_name[PATH_MAX] = {0}; char conf_name[PATH_MAX] = {0}; char conf_name_relative[PATH_MAX] = {0}; - if (!latest_deliveries[i].meta.name) { + if (!latest_deliveries[i]->meta.name) { continue; } - sprintf(link_name, "latest-py%s-%s-%s.yml", latest_deliveries[i].meta.python_compact, latest_deliveries[i].system.platform[DELIVERY_PLATFORM_RELEASE], latest_deliveries[i].system.arch); - sprintf(readme_name, "README-py%s-%s-%s.md", latest_deliveries[i].meta.python_compact, latest_deliveries[i].system.platform[DELIVERY_PLATFORM_RELEASE], latest_deliveries[i].system.arch); - sprintf(conf_name, "%s.ini", latest_deliveries[i].info.release_name); - sprintf(conf_name_relative, "../config/%s.ini", latest_deliveries[i].info.release_name); + sprintf(link_name, "latest-py%s-%s-%s.yml", latest_deliveries[i]->meta.python_compact, latest_deliveries[i]->system.platform[DELIVERY_PLATFORM_RELEASE], latest_deliveries[i]->system.arch); + sprintf(readme_name, "README-py%s-%s-%s.md", latest_deliveries[i]->meta.python_compact, latest_deliveries[i]->system.platform[DELIVERY_PLATFORM_RELEASE], latest_deliveries[i]->system.arch); + sprintf(conf_name, "%s.ini", latest_deliveries[i]->info.release_name); + sprintf(conf_name_relative, "../config/%s.ini", latest_deliveries[i]->info.release_name); if (strstr(link_name, platform) && strstr(link_name, arch)) { - fprintf(indexfp, "- Python %s\n", latest_deliveries[i].meta.python); + fprintf(indexfp, "- Python %s\n", latest_deliveries[i]->meta.python); fprintf(indexfp, " - Info: [README](%s)\n", readme_name); fprintf(indexfp, " - Release: [Conda Environment YAML](%s)\n", link_name); fprintf(indexfp, " - Receipt: [STASIS input file](%s)\n", conf_name_relative); char *pattern = NULL; asprintf(&pattern, "*%s*%s*", - latest_deliveries[i].info.build_number, - strstr(ctx->rules.release_fmt, "%p") ? latest_deliveries[i].meta.python_compact : "" ); + latest_deliveries[i]->info.build_number, + strstr((*ctx)->rules.release_fmt, "%p") ? latest_deliveries[i]->meta.python_compact : "" ); if (!pattern) { SYSERROR("%s", "Unable to allocate bytes for pattern"); return -1; } - struct StrList *docker_images = get_docker_images(&latest_deliveries[i], pattern); + struct StrList *docker_images = get_docker_images(latest_deliveries[i], pattern); if (docker_images && strlist_count(docker_images) - && !strcmp(latest_deliveries[i].system.platform[DELIVERY_PLATFORM_RELEASE], "linux")) { + && !strcmp(latest_deliveries[i]->system.platform[DELIVERY_PLATFORM_RELEASE], "linux")) { fprintf(indexfp, " - Docker: "); fprintf(indexfp, "[Archive](../packages/docker/%s)\n", path_basename(strlist_item(docker_images, 0))); } @@ -86,9 +87,9 @@ int indexer_readmes(struct Delivery ctx[], const size_t nelem) { } fprintf(indexfp, "## Releases\n"); - int current_rc = ctx->meta.rc; - for (size_t i = 0; ctx[i].meta.name != NULL; i++) { - struct Delivery *current = &ctx[i]; + int current_rc = (*ctx)->meta.rc; + for (size_t i = 0; i < nelem; i++) { + struct Delivery *current = ctx[i]; if (current_rc > current->meta.rc) { current_rc = current->meta.rc; fprintf(indexfp, "\n\n---\n\n"); @@ -101,7 +102,7 @@ int indexer_readmes(struct Delivery ctx[], const size_t nelem) { char *pattern = NULL; asprintf(&pattern, "*%s*%s*", current->info.build_number, - strstr(ctx->rules.release_fmt, "%p") ? current->meta.python_compact : "" ); + strstr((*ctx)->rules.release_fmt, "%p") ? current->meta.python_compact : "" ); if (!pattern) { SYSERROR("%s", "Unable to allocate bytes for pattern"); return -1; diff --git a/src/cli/stasis_indexer/stasis_indexer_main.c b/src/cli/stasis_indexer/stasis_indexer_main.c index 279af5a..840e897 100644 --- a/src/cli/stasis_indexer/stasis_indexer_main.c +++ b/src/cli/stasis_indexer/stasis_indexer_main.c @@ -8,15 +8,11 @@ #include "delivery.h" int indexer_combine_rootdirs(const char *dest, char **rootdirs, const size_t rootdirs_total) { - char cmd[PATH_MAX]; - char destdir_bare[PATH_MAX]; - char destdir_with_output[PATH_MAX]; + char cmd[PATH_MAX] = {0}; + char destdir_bare[PATH_MAX] = {0}; + char destdir_with_output[PATH_MAX] = {0}; char *destdir = destdir_bare; - memset(cmd, 0, sizeof(cmd)); - memset(destdir_bare, 0, sizeof(destdir_bare)); - memset(destdir_with_output, 0, sizeof(destdir_bare)); - strcpy(destdir_bare, dest); strcpy(destdir_with_output, dest); strcat(destdir_with_output, "/output"); @@ -25,7 +21,7 @@ int indexer_combine_rootdirs(const char *dest, char **rootdirs, const size_t roo destdir = destdir_with_output; } - sprintf(cmd, "rsync -ah%s --delete --exclude 'tools/' --exclude 'tmp/' --exclude 'build/' ", globals.verbose ? "v" : "q"); + snprintf(cmd, sizeof(cmd), "rsync -ah%s --delete --exclude 'tools/' --exclude 'tmp/' --exclude 'build/' ", globals.verbose ? "v" : "q"); for (size_t i = 0; i < rootdirs_total; i++) { char srcdir_bare[PATH_MAX] = {0}; char srcdir_with_output[PATH_MAX] = {0}; @@ -42,9 +38,9 @@ int indexer_combine_rootdirs(const char *dest, char **rootdirs, const size_t roo if (!access(srcdir_with_output, F_OK)) { srcdir = srcdir_with_output; } - snprintf(cmd + strlen(cmd), sizeof(srcdir) - strlen(srcdir) + 4, "'%s'/ ", srcdir); + snprintf(cmd + strlen(cmd), sizeof(cmd) - strlen(cmd), "'%s'/ ", srcdir); } - snprintf(cmd + strlen(cmd), sizeof(cmd) - strlen(destdir) + 1, " %s/", destdir); + snprintf(cmd + strlen(cmd), sizeof(cmd) - strlen(cmd), " %s/", destdir); if (globals.verbose) { puts(cmd); @@ -67,27 +63,28 @@ int indexer_conda(const struct Delivery *ctx, struct MicromambaInfo m) { return status; } -int indexer_symlinks(struct Delivery *ctx, const size_t nelem) { - struct Delivery *data = NULL; - data = get_latest_deliveries(ctx, nelem); +int indexer_symlinks(struct Delivery **ctx, const size_t nelem) { + struct Delivery **data = NULL; + size_t nelem_real = 0; + data = get_latest_deliveries(ctx, nelem, &nelem_real); //int latest = get_latest_rc(ctx, nelem); - if (!pushd(ctx->storage.delivery_dir)) { - for (size_t i = 0; i < nelem; i++) { + if (!pushd((*ctx)->storage.delivery_dir)) { + for (size_t i = 0; i < nelem_real; i++) { char link_name_spec[PATH_MAX]; char link_name_readme[PATH_MAX]; char file_name_spec[PATH_MAX]; char file_name_readme[PATH_MAX]; - if (!data[i].meta.name) { + if (!data[i]->meta.name) { continue; } - sprintf(link_name_spec, "latest-py%s-%s-%s.yml", data[i].meta.python_compact, data[i].system.platform[DELIVERY_PLATFORM_RELEASE], data[i].system.arch); - sprintf(file_name_spec, "%s.yml", data[i].info.release_name); + sprintf(link_name_spec, "latest-py%s-%s-%s.yml", data[i]->meta.python_compact, data[i]->system.platform[DELIVERY_PLATFORM_RELEASE], data[i]->system.arch); + sprintf(file_name_spec, "%s.yml", data[i]->info.release_name); - sprintf(link_name_readme, "README-py%s-%s-%s.md", data[i].meta.python_compact, data[i].system.platform[DELIVERY_PLATFORM_RELEASE], data[i].system.arch); - sprintf(file_name_readme, "README-%s.md", data[i].info.release_name); + sprintf(link_name_readme, "README-py%s-%s-%s.md", data[i]->meta.python_compact, data[i]->system.platform[DELIVERY_PLATFORM_RELEASE], data[i]->system.arch); + sprintf(file_name_readme, "README-%s.md", data[i]->info.release_name); if (!access(link_name_spec, F_OK)) { if (unlink(link_name_spec)) { @@ -116,7 +113,7 @@ int indexer_symlinks(struct Delivery *ctx, const size_t nelem) { } popd(); } else { - fprintf(stderr, "Unable to enter delivery directory: %s\n", ctx->storage.delivery_dir); + fprintf(stderr, "Unable to enter delivery directory: %s\n", (*ctx)->storage.delivery_dir); guard_free(data); return -1; } @@ -325,7 +322,7 @@ int main(const int argc, char *argv[]) { get_files(&metafiles, ctx.storage.meta_dir, "*.stasis"); strlist_sort(metafiles, STASIS_SORT_LEN_ASCENDING); - struct Delivery *local = calloc(strlist_count(metafiles) + 1, sizeof(*local)); + struct Delivery **local = calloc(strlist_count(metafiles) + 1, sizeof(*local)); if (!local) { SYSERROR("%s", "Unable to allocate bytes for local delivery context array"); exit(1); @@ -334,11 +331,15 @@ int main(const int argc, char *argv[]) { for (size_t i = 0; i < strlist_count(metafiles); i++) { char *item = strlist_item(metafiles, i); // Copy the pre-filled contents of the main delivery context - memcpy(&local[i], &ctx, sizeof(ctx)); + local[i] = delivery_duplicate(&ctx); + if (!local[i]) { + SYSERROR("Unable to duplicate delivery context %zu", i); + exit(1); + } if (globals.verbose) { puts(item); } - load_metadata(&local[i], item); + load_metadata(local[i], item); } qsort(local, strlist_count(metafiles), sizeof(*local), callback_sort_deliveries_cmpfn); @@ -430,10 +431,14 @@ int main(const int argc, char *argv[]) { guard_free(destdir); guard_array_free(rootdirs); - guard_strlist_free(&metafiles); guard_free(m.micromamba_prefix); delivery_free(&ctx); + for (size_t i = 0; i < strlist_count(metafiles); i++) { + delivery_free(local[i]); + guard_free(local[i]); + } guard_free(local); + guard_strlist_free(&metafiles); globals_free(); msg(STASIS_MSG_L1, "Done!\n"); diff --git a/src/cli/stasis_indexer/website.c b/src/cli/stasis_indexer/website.c index 55f0c45..966391e 100644 --- a/src/cli/stasis_indexer/website.c +++ b/src/cli/stasis_indexer/website.c @@ -1,7 +1,7 @@ #include "core.h" #include "website.h" -int indexer_make_website(const struct Delivery *ctx) { +int indexer_make_website(struct Delivery **ctx) { char *css_filename = calloc(PATH_MAX, sizeof(*css_filename)); if (!css_filename) { SYSERROR("unable to allocate string for CSS file path: %s", strerror(errno)); @@ -12,8 +12,8 @@ int indexer_make_website(const struct Delivery *ctx) { const int have_css = access(css_filename, F_OK | R_OK) == 0; struct StrList *dirs = strlist_init(); - strlist_append(&dirs, ctx->storage.delivery_dir); - strlist_append(&dirs, ctx->storage.results_dir); + strlist_append(&dirs, (*ctx)->storage.delivery_dir); + strlist_append(&dirs, (*ctx)->storage.results_dir); struct StrList *inputs = NULL; for (size_t i = 0; i < strlist_count(dirs); i++) { |
