diff options
| -rw-r--r-- | include/conda.h | 8 | ||||
| -rw-r--r-- | include/deliverable.h | 7 | ||||
| -rw-r--r-- | src/CMakeLists.txt | 1 | ||||
| -rw-r--r-- | src/conda.c | 59 | ||||
| -rw-r--r-- | src/deliverable.c | 75 | ||||
| -rw-r--r-- | src/ini.c | 25 | ||||
| -rw-r--r-- | src/main.c | 133 |
7 files changed, 216 insertions, 92 deletions
diff --git a/include/conda.h b/include/conda.h index bc74b94..3565bda 100644 --- a/include/conda.h +++ b/include/conda.h @@ -16,9 +16,9 @@ int pip_exec(const char *args); int conda_exec(const char *args); int conda_activate(const char *root, const char *env_name); void conda_setup_headless(); -void conda_env_create_from_uri(char *name, char *uri); -void conda_env_create(char *name, char *python_version, char *packages); -void conda_env_remove(char *name); -void conda_env_export(char *name, char *output_dir, char *output_filename); +int conda_env_create_from_uri(char *name, char *uri); +int conda_env_create(char *name, char *python_version, char *packages); +int conda_env_remove(char *name); +int conda_env_export(char *name, char *output_dir, char *output_filename); int conda_index(const char *path); #endif //OMC_CONDA_H diff --git a/include/deliverable.h b/include/deliverable.h index 02c13e1..b585054 100644 --- a/include/deliverable.h +++ b/include/deliverable.h @@ -72,7 +72,6 @@ struct Delivery { char *mission; ///< hst, jwst, roman char *codename; ///< HST uses codenames bool final; ///< is this a final release? - bool continue_on_error; ///< keep going, or don't } meta; /*! \struct Info @@ -187,7 +186,7 @@ int delivery_index_wheel_artifacts(struct Delivery *ctx); * @return header on success * @return NULL on error */ -char *delivery_get_spec_header(struct Delivery *ctx); +char *delivery_get_release_header(struct Delivery *ctx); /** * Finalizes a delivery artifact for distribution @@ -216,7 +215,7 @@ int delivery_copy_conda_artifacts(struct Delivery *ctx); * Retrieve Conda installer * @param installer_url URL to installation script */ -void delivery_get_installer(char *installer_url); +int delivery_get_installer(char *installer_url); /** * Generate URL based on Delivery context @@ -237,7 +236,7 @@ void delivery_get_installer_url(struct Delivery *delivery, char *result); * @param type INSTALL_PKG_PIP_DEFERRED * @param manifest pointer to array of StrList (package list(s)) */ -void delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, char *env_name, int type, struct StrList *manifest[]); +int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, char *env_name, int type, struct StrList *manifest[]); /** * Update "conda index" on Conda artifact storage diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8f996f2..e22e749 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,5 +1,6 @@ include_directories(${CMAKE_SOURCE_DIR}/include) include_directories(${PROJECT_BINARY_DIR}) +add_compile_options(-Wall -Wextra -Wno-format-truncation) add_executable(omc main.c str.c diff --git a/src/conda.c b/src/conda.c index 79e4949..a361267 100644 --- a/src/conda.c +++ b/src/conda.c @@ -5,6 +5,8 @@ #include <unistd.h> #include "conda.h" +extern struct OMC_GLOBAL globals; + int python_exec(const char *args) { char command[PATH_MAX]; memset(command, 0, sizeof(command)); @@ -104,7 +106,7 @@ int conda_activate(const char *root, const char *env_name) { // Parse the log file: // 1. Extract the environment keys and values from the sub-shell - // 2. Apply it to ohmycal's runtime environment + // 2. Apply it to OMC's runtime environment // 3. Now we're ready to execute conda commands anywhere fp = fopen(proc.stdout, "r"); if (!fp) { @@ -115,7 +117,7 @@ int conda_activate(const char *root, const char *env_name) { while (!feof(fp)) { char buf[OMC_BUFSIZ] = {0}; int ch = 0; - int z = 0; + size_t z = 0; while (z < sizeof(buf) && (ch = (int) fgetc(fp)) != 0) { if (ch == EOF) { ch = 0; @@ -160,47 +162,50 @@ void conda_setup_headless() { conda_exec("config --system --set rollback_enabled false"); conda_exec("config --system --set report_errors false"); - // make this configurable - //if (conda_exec("update --all")) { - // fprintf(stderr, "conda update was unsuccessful\n"); - // exit(1); - //} + if (globals.verbose) { + char *rcfile = getenv("CONDARC"); + if (rcfile) { + msg(OMC_MSG_L1, "Dump %s\n", rcfile); + char **condarc_contents = file_readlines(rcfile, 0, 0, NULL); + for (size_t i = 0; condarc_contents[i] != NULL; i++) { + msg(OMC_MSG_L2, "%s", condarc_contents[i]); + free(condarc_contents[i]); + } + free(condarc_contents); + } + + } + + if (globals.always_update_base_environment) { + if (conda_exec("update --all")) { + fprintf(stderr, "conda update was unsuccessful\n"); + exit(1); + } + } } -void conda_env_create_from_uri(char *name, char *uri) { +int conda_env_create_from_uri(char *name, char *uri) { char env_command[PATH_MAX]; sprintf(env_command, "env create -n %s -f %s", name, uri); - if (conda_exec(env_command)) { - fprintf(stderr, "derived environment creation failed\n"); - exit(1); - } + return conda_exec(env_command); } -void conda_env_create(char *name, char *python_version, char *packages) { +int conda_env_create(char *name, char *python_version, char *packages) { char env_command[PATH_MAX]; sprintf(env_command, "create -n %s python=%s %s", name, python_version, packages ? packages : ""); - if (conda_exec(env_command)) { - fprintf(stderr, "conda environment creation failed\n"); - exit(1); - } + return conda_exec(env_command); } -void conda_env_remove(char *name) { +int conda_env_remove(char *name) { char env_command[PATH_MAX]; sprintf(env_command, "env remove -n %s", name); - if (conda_exec(env_command)) { - fprintf(stderr, "conda environment removal failed\n"); - exit(1); - } + return conda_exec(env_command); } -void conda_env_export(char *name, char *output_dir, char *output_filename) { +int conda_env_export(char *name, char *output_dir, char *output_filename) { char env_command[PATH_MAX]; sprintf(env_command, "env export -n %s -f %s/%s.yml", name, output_dir, output_filename); - if (conda_exec(env_command)) { - fprintf(stderr, "conda environment export failed\n"); - exit(1); - } + return conda_exec(env_command); } int conda_index(const char *path) { diff --git a/src/deliverable.c b/src/deliverable.c index f8e5145..cf25e25 100644 --- a/src/deliverable.c +++ b/src/deliverable.c @@ -420,7 +420,7 @@ static char *requirement_from_test(struct Delivery *ctx, const char *name) { return result; } -void delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, char *env_name, int type, struct StrList **manifest) { +int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, char *env_name, int type, struct StrList **manifest) { char cmd[PATH_MAX]; char pkgs[OMC_BUFSIZ]; char *env_current = getenv("CONDA_DEFAULT_ENV"); @@ -458,6 +458,9 @@ void delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, ch for (size_t p = 0; p < strlist_count(manifest[x]); p++) { name = strlist_item(manifest[x], p); strip(name); + if (!strlen(name)) { + continue; + } if (INSTALL_PKG_PIP_DEFERRED & type) { //DIR *dp; //struct dirent *rec; @@ -499,11 +502,12 @@ void delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, ch } } } - if (runner(cmd)) { - fprintf(stderr, "failed to install package: %s\n", name); - exit(1); + int status = runner(cmd); + if (status) { + return status; } } + return 0; } void delivery_get_installer_url(struct Delivery *delivery, char *result) { @@ -524,13 +528,18 @@ void delivery_get_installer_url(struct Delivery *delivery, char *result) { } -void delivery_get_installer(char *installer_url) { - if (access(path_basename(installer_url), F_OK)) { - if (download(installer_url, path_basename(installer_url))) { - fprintf(stderr, "download failed: %s\n", installer_url); - exit(1); +int delivery_get_installer(char *installer_url) { + char *script = path_basename(installer_url); + if (access(script, F_OK)) { + // Script doesn't exist + if (download(installer_url, script)) { + // download failed + return -1; } + } else { + msg(OMC_MSG_RESTRICT | OMC_MSG_L3, "Skipped, installer already exists\n", script); } + return 0; } int delivery_copy_conda_artifacts(struct Delivery *ctx) { @@ -542,8 +551,11 @@ int delivery_copy_conda_artifacts(struct Delivery *ctx) { memset(subdir, 0, sizeof(subdir)); sprintf(conda_build_dir, "%s/%s", ctx->storage.conda_install_prefix, "conda-bld"); + // One must run conda build at least once to create the "conda-bld" directory. + // When this directory is missing there can be no build artifacts. if (access(conda_build_dir, F_OK) < 0) { - // Conda build was never executed + msg(OMC_MSG_RESTRICT | OMC_MSG_WARN | OMC_MSG_L3, + "Skipped: 'conda build' has never been executed.\n"); return 0; } @@ -710,6 +722,21 @@ char *delivery_get_spec_header(struct Delivery *ctx) { void delivery_rewrite_spec(struct Delivery *ctx, char *filename) { char *package_name = NULL; char output[PATH_MAX]; + char *header = NULL; + char *tempfile = NULL; + FILE *tp = NULL; + + header = delivery_get_release_header(ctx); + if (!header) { + msg(OMC_MSG_ERROR, "failed to generate release header string\n", filename); + exit(1); + } + tempfile = xmkstemp(&tp); + if (!tempfile || !tp) { + msg(OMC_MSG_ERROR, "%s: unable to create temporary file\n", strerror(errno)); + exit(1); + } + fprintf(tp, "%s", header); sprintf(output, " - %s", ctx->storage.conda_staging_url); file_replace_text(filename, " - local", output); @@ -782,16 +809,21 @@ void delivery_tests_run(struct Delivery *ctx) { msg(OMC_MSG_L3, "Cloning %s\n", ctx->tests[i].repository); git_clone(&proc, ctx->tests[i].repository, destdir, ctx->tests[i].version); - if (pushd(destdir) && !ctx->meta.continue_on_error) { - fprintf(stderr, "unable to enter repository directory\n"); - exit(1); + if (pushd(destdir)) { + COE_CHECK_ABORT(!globals.continue_on_error, "Unable to enter repository directory\n"); } else { #if 1 - msg(OMC_MSG_L3, "Running\n"); + int status; + char cmd[PATH_MAX]; + msg(OMC_MSG_L3, "Testing %s\n", ctx->tests[i].name); memset(&proc, 0, sizeof(proc)); - if (shell2(&proc, ctx->tests[i].script) && !ctx->meta.continue_on_error) { - fprintf(stderr, "continue on error is not enabled. aborting.\n"); - exit(1); + + // enable trace mode before executing each test script + memset(cmd, 0, sizeof(cmd)); + sprintf(cmd, "set -x ; %s", ctx->tests[i].script); + status = shell2(&proc, cmd); + if (status) { + COE_CHECK_ABORT(!globals.continue_on_error, "Test failure"); } popd(); #else @@ -800,6 +832,15 @@ void delivery_tests_run(struct Delivery *ctx) { } } } +} +void delivery_gather_tool_versions(struct Delivery *ctx) { + // Extract version from tool output + ctx->conda.tool_version = shell_output("conda --version"); + if (ctx->conda.tool_version) + strip(ctx->conda.tool_version); + ctx->conda.tool_build_version = shell_output("conda build --version"); + if (ctx->conda.tool_build_version) + strip(ctx->conda.tool_version); } @@ -56,7 +56,7 @@ struct INIData *ini_getall(struct INIFILE *ini, char *section_name) { static size_t i = 0; section = ini_section_search(&ini, section_name); - if (section->data[i]) { + if (section->data_count) { result = section->data[i]; i++; } else { @@ -138,8 +138,7 @@ int ini_data_record(struct INIFILE **ini, char *section_name, char *key, char *v struct INIData **tmp = realloc(section->data, (section->data_count + 1) * sizeof(**section->data)); if (!tmp) { - perror(__FUNCTION__); - exit(1); + return 1; } section->data = tmp; if (!ini_data_get((*ini), section_name, key)) { @@ -157,7 +156,7 @@ int ini_data_record(struct INIFILE **ini, char *section_name, char *key, char *v value_tmp = realloc(data->value, value_len_new + 2); if (!value_tmp) { perror(__FUNCTION__ ); - exit(1); + return -1; } data->value = value_tmp; */ @@ -170,13 +169,19 @@ int ini_data_record(struct INIFILE **ini, char *section_name, char *key, char *v void ini_section_record(struct INIFILE **ini, char *key) { struct INISection **tmp = realloc((*ini)->section, ((*ini)->section_count + 1) * sizeof((*ini)->section)); if (!tmp) { - perror(__FUNCTION__); - exit(1); + return 1; } (*ini)->section = tmp; (*ini)->section[(*ini)->section_count] = calloc(1, sizeof(*(*ini)->section[0])); + if (!(*ini)->section[(*ini)->section_count]) { + return -1; + } (*ini)->section[(*ini)->section_count]->key = strdup(key); + if (!(*ini)->section[(*ini)->section_count]->key) { + return -1; + } (*ini)->section_count++; + return 0; } void ini_show(struct INIFILE *ini) { @@ -238,6 +243,9 @@ struct INIFILE *ini_open(const char *filename) { char current_section[OMC_BUFSIZ] = {0}; char *key_last = NULL; struct INIFILE *ini = ini_init(); + if (ini == NULL) { + return NULL; + } ini_section_init(&ini); @@ -249,8 +257,9 @@ struct INIFILE *ini_open(const char *filename) { // Open the configuration file for reading fp = fopen(filename, "r"); if (!fp) { - perror(filename); - exit(1); + ini_free(&ini); + ini = NULL; + return NULL; } // Read file @@ -1,10 +1,8 @@ -#define GNU_SOURCE 1 +#define GNU_SOURCE #include <stdio.h> #include <stdlib.h> #include <string.h> -#include <unistd.h> #include <limits.h> -#include <time.h> #include <sys/utsname.h> #include <getopt.h> #include "omc.h" @@ -115,10 +113,10 @@ int main(int argc, char *argv[], char *arge[]) { .stderr = "", .redirect_stderr = 0, }; - char env_name[PATH_MAX]; - char env_name_testing[PATH_MAX]; - char *delivery_input = argv[1]; - char *config_input = argv[2]; + char env_name[255] = {0}; + char env_name_testing[255] = {0}; + char *delivery_input = NULL; // = argv[1]; + char *config_input = NULL; // = argv[2]; char installer_url[PATH_MAX]; char python_override_version[NAME_MAX]; unsigned char arg_continue_on_error = 0; @@ -218,18 +216,38 @@ int main(int argc, char *argv[], char *arge[]) { if (config_input) { msg(OMC_MSG_L2, "Reading OMC global configuration: %s\n", config_input); cfg = ini_open(config_input); - //ini_show(cfg); + if (!cfg) { + msg(OMC_MSG_ERROR | OMC_MSG_L2, "Failed to read config file: %s, %s", delivery_input, strerror(errno)); + exit(1); + } } msg(OMC_MSG_L2, "Reading OMC delivery configuration: %s\n", delivery_input); ini = ini_open(delivery_input); + if (!ini) { + msg(OMC_MSG_ERROR | OMC_MSG_L2, "Failed to read delivery file: %s, %s", delivery_input, strerror(errno)); + exit(1); + } printf(BANNER, VERSION, AUTHOR); - delivery_init(&ctx, ini, cfg); + if (delivery_init(&ctx, ini, cfg)) { + msg(OMC_MSG_ERROR | OMC_MSG_L1, "Failed to initialize delivery context\n"); + exit(1); + } + + globals.always_update_base_environment = arg_always_update_base_environment; + globals.continue_on_error = arg_continue_on_error; + + // Override Python version from command-line, if any + if (strlen(python_override_version) && ctx.meta.python) { + free(ctx.meta.python); + ctx.meta.python = strdup(python_override_version); + } + runtime_apply(ctx.runtime.environ); - sprintf(env_name, "%s", ctx.info.release_name); - sprintf(env_name_testing, "%s_test", env_name); + snprintf(env_name, sizeof(env_name_testing) - 1, "%s", ctx.info.release_name); + snprintf(env_name_testing, sizeof(env_name) - 1, "%s_test", env_name); msg(OMC_MSG_L1, "Overview\n"); delivery_meta_show(&ctx); @@ -239,7 +257,10 @@ int main(int argc, char *argv[], char *arge[]) { msg(OMC_MSG_L1, "Conda setup\n"); delivery_get_installer_url(&ctx, installer_url); msg(OMC_MSG_L2, "Downloading: %s\n", installer_url); - delivery_get_installer(installer_url); + if (delivery_get_installer(installer_url)) { + msg(OMC_MSG_ERROR, "download failed: %s\n", installer_url); + exit(1); + } // Unlikely to occur: this should help prevent rmtree() from destroying your entire filesystem // if path is "/" then, die @@ -255,16 +276,51 @@ int main(int argc, char *argv[], char *arge[]) { msg(OMC_MSG_L2, "Configuring: %s\n", ctx.storage.conda_install_prefix); delivery_conda_enable(&ctx, ctx.storage.conda_install_prefix); + char *pathvar = NULL; + pathvar = getenv("PATH"); + if (!pathvar) { + msg(OMC_MSG_ERROR | OMC_MSG_L2, "PATH variable is not set. Cannot continue.\n"); + exit(1); + } else { + char pathvar_tmp[OMC_BUFSIZ]; + sprintf(pathvar_tmp, "%s/bin:%s", ctx.storage.conda_install_prefix, pathvar); + setenv("PATH", pathvar_tmp, 1); + } + + msg(OMC_MSG_L2, "Installing build tools\n"); + if (conda_exec("install boa conda-build conda-verify")) { + msg(OMC_MSG_ERROR | OMC_MSG_L2, "conda-build installation failed\n"); + exit(1); + } msg(OMC_MSG_L1, "Creating release environment(s)\n"); if (ctx.meta.based_on && strlen(ctx.meta.based_on)) { - conda_env_remove(env_name); - conda_env_create_from_uri(env_name, ctx.meta.based_on); + if (conda_env_remove(env_name)) { + msg(OMC_MSG_ERROR | OMC_MSG_L2, "failed to remove release environment: %s\n", env_name_testing); + exit(1); + } + msg(OMC_MSG_L2, "Based on release: %s\n", ctx.meta.based_on); + if (conda_env_create_from_uri(env_name, ctx.meta.based_on)) { + msg(OMC_MSG_ERROR | OMC_MSG_L2, "unable to install release environment using configuration file\n"); + exit(1); + } - conda_env_remove(env_name_testing); - conda_env_create_from_uri(env_name_testing, ctx.meta.based_on); + if (conda_env_remove(env_name_testing)) { + msg(OMC_MSG_ERROR | OMC_MSG_L2, "failed to remove testing environment\n"); + exit(1); + } + if (conda_env_create_from_uri(env_name_testing, ctx.meta.based_on)) { + msg(OMC_MSG_ERROR | OMC_MSG_L2, "unable to install testing environment using configuration file\n"); + exit(1); + } } else { - conda_env_create(env_name, ctx.meta.python, NULL); - conda_env_create(env_name_testing, ctx.meta.python, NULL); + if (conda_env_create(env_name, ctx.meta.python, NULL)) { + msg(OMC_MSG_ERROR | OMC_MSG_L2, "failed to create release environment\n"); + exit(1); + } + if (conda_env_create(env_name_testing, ctx.meta.python, NULL)) { + msg(OMC_MSG_ERROR | OMC_MSG_L2, "failed to create release environment\n"); + exit(1); + } } // Activate test environment @@ -274,15 +330,15 @@ int main(int argc, char *argv[], char *arge[]) { exit(1); } - msg(OMC_MSG_L2, "Installing build tools\n"); - if (conda_exec("install boa conda-build conda-verify")) { - msg(OMC_MSG_ERROR | OMC_MSG_L2, "conda-build installation failed"); + delivery_gather_tool_versions(&ctx); + if (!ctx.conda.tool_version) { + msg(OMC_MSG_ERROR | OMC_MSG_L2, "Could not determine conda version\n"); + exit(1); + } + if (!ctx.conda.tool_build_version) { + msg(OMC_MSG_ERROR | OMC_MSG_L2, "Could not determine conda-build version\n"); exit(1); } - - // TODO: extract version from tool output - ctx.conda.tool_version = NULL; - ctx.conda.tool_build_version = NULL; if (pip_exec("install build")) { msg(OMC_MSG_ERROR | OMC_MSG_L2, "'build' tool installation failed"); @@ -311,7 +367,7 @@ int main(int argc, char *argv[], char *arge[]) { // } //} - if (ctx.conda.conda_packages_defer) { + if (ctx.conda.conda_packages_defer && strlist_count(ctx.conda.conda_packages_defer)) { msg(OMC_MSG_L2, "Building Conda recipe(s)\n"); if (delivery_build_recipes(&ctx)) { exit(1); @@ -328,11 +384,18 @@ int main(int argc, char *argv[], char *arge[]) { // Populate the release environment msg(OMC_MSG_L1, "Populating release environment\n"); - msg(OMC_MSG_L2, "Installing conda packages\n"); - delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_CONDA, (struct StrList *[]) {ctx.conda.conda_packages, NULL}); - msg(OMC_MSG_L3, "Installing deferred conda packages\n"); - delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_CONDA | INSTALL_PKG_CONDA_DEFERRED, (struct StrList *[]) {ctx.conda.conda_packages_defer, NULL}); + if (delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_CONDA, (struct StrList *[]) {ctx.conda.conda_packages, NULL})) { + exit(1); + } + if (strlist_count(ctx.conda.conda_packages_defer)) { + msg(OMC_MSG_L3, "Installing deferred conda packages\n"); + if (delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_CONDA | INSTALL_PKG_CONDA_DEFERRED, (struct StrList *[]) {ctx.conda.conda_packages_defer, NULL})) { + exit(1); + } + } else { + msg(OMC_MSG_L3, "No deferred conda packages\n"); + } msg(OMC_MSG_L2, "Installing pip packages\n"); delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_PIP, (struct StrList *[]) {ctx.conda.pip_packages, NULL}); msg(OMC_MSG_L3, "Installing deferred pip packages\n"); @@ -342,10 +405,16 @@ int main(int argc, char *argv[], char *arge[]) { msg(OMC_MSG_L1, "Creating release\n"); msg(OMC_MSG_L2, "Exporting %s\n", env_name_testing); - conda_env_export(env_name_testing, ctx.storage.delivery_dir, env_name_testing); + if (conda_env_export(env_name_testing, ctx.storage.delivery_dir, env_name_testing)) { + msg(OMC_MSG_ERROR | OMC_MSG_L2, "failed %s\n", env_name_testing); + exit(1); + } msg(OMC_MSG_L2, "Exporting %s\n", env_name); - conda_env_export(env_name, ctx.storage.delivery_dir, env_name); + if (conda_env_export(env_name, ctx.storage.delivery_dir, env_name)) { + msg(OMC_MSG_ERROR | OMC_MSG_L2, "failed %s\n", env_name); + exit(1); + } // Rewrite release environment output (i.e. set package origin(s) to point to the deployment server, etc.) char specfile[PATH_MAX]; |