// // Created by jhunk on 10/5/23. // #include "deliverable.h" #include "str.h" #include "strlist.h" #include "wheel.h" #define getter(XINI, SECTION_NAME, KEY, TYPE) \ { \ if (ini_getval(XINI, SECTION_NAME, KEY, TYPE, &val)) { \ fprintf(stderr, "%s:%s not defined\n", SECTION_NAME, KEY); \ } \ } #define conv_int(X, DEST) X->DEST = val.as_int; #define conv_str(X, DEST) X->DEST = runtime_expand_var(NULL, val.as_char_p); #define conv_str_noexpand(X, DEST) if (val.as_char_p) X->DEST = strdup(val.as_char_p); #define conv_strlist(X, DEST, TOK) { \ char *rtevnop = runtime_expand_var(NULL, val.as_char_p); \ if (!X->DEST) \ X->DEST = strlist_init(); \ strlist_append_tokenize(X->DEST, rtevnop, TOK); \ free(rtevnop);\ } #define conv_bool(X, DEST) X->DEST = val.as_bool; #define guard_runtime_free(X) if (X) { runtime_free(X); X = NULL; } #define guard_strlist_free(X) if (X) { strlist_free(X); X = NULL; } #define guard_free(X) if (X) { free(X); X = NULL; } void delivery_free(struct Delivery *ctx) { guard_free(ctx->system.arch); guard_free(ctx->meta.name) guard_free(ctx->meta.version) guard_free(ctx->meta.codename) guard_free(ctx->meta.mission) guard_free(ctx->meta.python) guard_free(ctx->meta.mission) guard_free(ctx->meta.python_compact); guard_runtime_free(ctx->runtime.environ) guard_free(ctx->storage.delivery_dir) guard_free(ctx->storage.conda_install_prefix) guard_free(ctx->storage.conda_artifact_dir) guard_free(ctx->storage.conda_staging_dir) guard_free(ctx->storage.conda_staging_url) guard_free(ctx->storage.wheel_artifact_dir) guard_free(ctx->storage.wheel_staging_dir) guard_free(ctx->storage.wheel_staging_url) guard_free(ctx->storage.build_dir) guard_free(ctx->storage.build_recipes_dir) guard_free(ctx->storage.build_sources_dir) guard_free(ctx->storage.build_testing_dir) guard_free(ctx->conda.installer_baseurl) guard_free(ctx->conda.installer_name) guard_free(ctx->conda.installer_version) guard_free(ctx->conda.installer_platform) guard_free(ctx->conda.installer_arch) guard_free(ctx->conda.tool_version); guard_free(ctx->conda.tool_build_version); guard_strlist_free(ctx->conda.conda_packages) guard_strlist_free(ctx->conda.conda_packages_defer) guard_strlist_free(ctx->conda.pip_packages) guard_strlist_free(ctx->conda.pip_packages_defer) for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { guard_free(ctx->tests[i].name) guard_free(ctx->tests[i].version) guard_free(ctx->tests[i].repository) guard_free(ctx->tests[i].script) guard_free(ctx->tests[i].build_recipe) // test-specific runtime variables guard_runtime_free(ctx->tests[i].runtime.environ) } } void delivery_init_dirs(struct Delivery *ctx) { mkdir("build", 0755); mkdir("build/recipes", 0755); mkdir("build/sources", 0755); mkdir("build/testing", 0755); ctx->storage.build_dir = realpath("build", NULL); ctx->storage.build_recipes_dir = realpath("build/recipes", NULL); ctx->storage.build_sources_dir = realpath("build/sources", NULL); ctx->storage.build_testing_dir = realpath("build/testing", NULL); mkdir("output", 0755); mkdir("output/omc", 0755); mkdir("output/packages", 0755); mkdir("output/packages/conda", 0755); mkdir("output/packages/wheels", 0755); ctx->storage.delivery_dir = realpath("output/omc", NULL); ctx->storage.conda_artifact_dir = realpath("output/packages/conda", NULL); ctx->storage.wheel_artifact_dir = realpath("output/packages/wheels", NULL); mkdir(CONDA_INSTALL_PREFIX, 0755); ctx->storage.conda_install_prefix = realpath(CONDA_INSTALL_PREFIX, NULL); } int delivery_init(struct Delivery *ctx, struct INIFILE *ini, struct INIFILE *cfg) { RuntimeEnv *rt; struct INIData *rtdata; union INIVal val; // Record timestamp used for release time(&ctx->info.time_now); ctx->info.time_info = localtime(&ctx->info.time_now); if (cfg) { getter(cfg, "default", "conda_staging_dir", INIVAL_TYPE_STR); conv_str(ctx, storage.conda_staging_dir); getter(cfg, "default", "conda_staging_url", INIVAL_TYPE_STR); conv_str(ctx, storage.conda_staging_url); getter(cfg, "default", "wheel_staging_dir", INIVAL_TYPE_STR); conv_str(ctx, storage.wheel_staging_dir); getter(cfg, "default", "wheel_staging_url", INIVAL_TYPE_STR); conv_str(ctx, storage.wheel_staging_url); } delivery_init_dirs(ctx); // Populate runtime variables first they may be interpreted by other // keys in the configuration rt = runtime_copy(__environ); while ((rtdata = ini_getall(ini, "runtime")) != NULL) { char rec[OMC_BUFSIZ]; sprintf(rec, "%s=%s", lstrip(strip(rtdata->key)), lstrip(strip(rtdata->value))); runtime_set(rt, rtdata->key, rtdata->value); } runtime_apply(rt); ctx->runtime.environ = rt; getter(ini, "meta", "mission", INIVAL_TYPE_STR) conv_str(ctx, meta.mission) if (!strcasecmp(ctx->meta.mission, "hst")) { getter(ini, "meta", "codename", INIVAL_TYPE_STR) conv_str(ctx, meta.codename) } else { ctx->meta.codename = NULL; } if (!strcasecmp(ctx->meta.mission, "jwst")) { getter(ini, "meta", "version", INIVAL_TYPE_STR) conv_str(ctx, meta.version) } else { ctx->meta.version = NULL; } getter(ini, "meta", "name", INIVAL_TYPE_STR) conv_str(ctx, meta.name) getter(ini, "meta", "rc", INIVAL_TYPE_INT) conv_int(ctx, meta.rc) getter(ini, "meta", "final", INIVAL_TYPE_BOOL) conv_bool(ctx, meta.final) getter(ini, "meta", "continue_on_error", INIVAL_TYPE_BOOL) conv_bool(ctx, meta.continue_on_error) getter(ini, "meta", "based_on", INIVAL_TYPE_STR) conv_str(ctx, meta.based_on) getter(ini, "meta", "python", INIVAL_TYPE_STR) conv_str(ctx, meta.python) getter(ini, "conda", "installer_baseurl", INIVAL_TYPE_STR) conv_str(ctx, conda.installer_baseurl) getter(ini, "conda", "installer_name", INIVAL_TYPE_STR) conv_str(ctx, conda.installer_name) getter(ini, "conda", "installer_version", INIVAL_TYPE_STR) conv_str(ctx, conda.installer_version) getter(ini, "conda", "installer_platform", INIVAL_TYPE_STR) conv_str(ctx, conda.installer_platform) getter(ini, "conda", "installer_arch", INIVAL_TYPE_STR) conv_str(ctx, conda.installer_arch) getter(ini, "conda", "conda_packages", INIVAL_TYPE_STR_ARRAY) conv_strlist(ctx, conda.conda_packages, "\n") getter(ini, "conda", "pip_packages", INIVAL_TYPE_STR_ARRAY) conv_strlist(ctx, conda.pip_packages, "\n") ctx->conda.conda_packages_defer = strlist_init(); ctx->conda.pip_packages_defer = strlist_init(); for (size_t z = 0, i = 0; i < ini->section_count; i++ ) { if (startswith(ini->section[i]->key, "test:")) { val.as_char_p = strchr(ini->section[i]->key, ':') + 1; conv_str(ctx, tests[z].name) getter(ini, ini->section[i]->key, "version", INIVAL_TYPE_STR) conv_str(ctx, tests[z].version) getter(ini, ini->section[i]->key, "repository", INIVAL_TYPE_STR) conv_str(ctx, tests[z].repository) getter(ini, ini->section[i]->key, "script", INIVAL_TYPE_STR) conv_str_noexpand(ctx, tests[z].script) getter(ini, ini->section[i]->key, "build_recipe", INIVAL_TYPE_STR); conv_str(ctx, tests[z].build_recipe) z++; } } char env_name[NAME_MAX]; char env_date[NAME_MAX]; ctx->meta.python_compact = to_short_version(ctx->meta.python); if (!strcasecmp(ctx->meta.mission, "hst") && ctx->meta.final) { memset(env_date, 0, sizeof(env_date)); strftime(env_date, sizeof(env_date) - 1, "%Y%m%d", ctx->info.time_info); sprintf(env_name, "%s_%s_rc%d", ctx->meta.name, env_date, ctx->meta.rc); } else if (!strcasecmp(ctx->meta.mission, "hst")) { sprintf(env_name, "%s_%s_%s_%s_rc%d", ctx->meta.name, ctx->meta.codename, ctx->system.platform[DELIVERY_PLATFORM_RELEASE], ctx->meta.python_compact, ctx->meta.rc); } else if (!strcasecmp(ctx->meta.mission, "jwst") && ctx->meta.final) { sprintf(env_name, "%s_%s_final", ctx->meta.name, ctx->meta.version); } else if (!strcasecmp(ctx->meta.mission, "jwst")) { sprintf(env_name, "%s_%s_rc%d", ctx->meta.name, ctx->meta.version, ctx->meta.rc); } if (strlen(env_name)) { ctx->info.release_name = strdup(env_name); } return 0; } void delivery_meta_show(struct Delivery *ctx) { printf("====DELIVERY====\n"); printf("%-20s %-10s\n", "Target Python:", ctx->meta.python); printf("%-20s %-10s\n", "Name:", ctx->meta.name); printf("%-20s %-10s\n", "Mission:", ctx->meta.mission); if (ctx->meta.codename) { printf("%-20s %-10s\n", "Codename:", ctx->meta.codename); } if (ctx->meta.version) { printf("%-20s %-10s\n", "Version", ctx->meta.version); } if (!ctx->meta.final) { printf("%-20s %-10d\n", "RC Level:", ctx->meta.rc); } printf("%-20s %-10s\n", "Final Release:", ctx->meta.final ? "Yes" : "No"); printf("%-20s %-10s\n", "Based On:", ctx->meta.based_on ? ctx->meta.based_on : "New"); } void delivery_conda_show(struct Delivery *ctx) { printf("====CONDA====\n"); printf("%-20s %-10s\n", "Installer:", ctx->conda.installer_baseurl); puts("Native Packages:"); for (size_t i = 0; i < strlist_count(ctx->conda.conda_packages); i++) { char *token = strlist_item(ctx->conda.conda_packages, i); if (isempty(token) || isblank(*token) || startswith(token, "-")) { continue; } printf("%21s%s\n", "", token); } puts("PyPi Packages:"); for (size_t i = 0; i < strlist_count(ctx->conda.pip_packages); i++) { char *token = strlist_item(ctx->conda.pip_packages, i); if (isempty(token) || isblank(*token) || startswith(token, "-")) { continue; } printf("%21s%s\n", "", token); } } void delivery_tests_show(struct Delivery *ctx) { printf("====TESTS====\n"); for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { if (!ctx->tests[i].name) { continue; } printf("%-20s %-10s %s\n", ctx->tests[i].name, ctx->tests[i].version, ctx->tests[i].repository); } } int delivery_build_recipes(struct Delivery *ctx) { for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { char *recipe_dir = NULL; if (ctx->tests[i].build_recipe) { // build a conda recipe int recipe_type; int status; if (recipe_clone(ctx->storage.build_recipes_dir, ctx->tests[i].build_recipe, NULL, &recipe_dir)) { fprintf(stderr, "Encountered an issue while cloning recipe for: %s\n", ctx->tests[i].name); return -1; } recipe_type = recipe_get_type(recipe_dir); pushd(recipe_dir); { if (RECIPE_TYPE_ASTROCONDA == recipe_type) { pushd(path_basename(ctx->tests[i].repository)); } else if (RECIPE_TYPE_CONDA_FORGE == recipe_type) { pushd("recipe"); } char recipe_version[100]; char recipe_buildno[100]; char recipe_git_url[PATH_MAX]; char recipe_git_rev[PATH_MAX]; sprintf(recipe_version, "{%% set version = GIT_DESCRIBE_TAG ~ \".dev\" ~ GIT_DESCRIBE_NUMBER ~ \"+\" ~ GIT_DESCRIBE_HASH %%}"); sprintf(recipe_git_url, " git_url: %s", ctx->tests[i].repository); sprintf(recipe_git_rev, " git_rev: %s", ctx->tests[i].version); sprintf(recipe_buildno, " number: 0"); //file_replace_text("meta.yaml", "{% set version = ", recipe_version); if (ctx->meta.final) { sprintf(recipe_version, "{%% set version = \"%s\" %%}", ctx->tests[i].version); // TODO: replace sha256 of tagged archive // TODO: leave the recipe unchanged otherwise. in theory this should produce the same conda package hash as conda forge. // For now, remove the sha256 requirement file_replace_text("meta.yaml", " sha256:", "\n"); } else { file_replace_text("meta.yaml", "{% set version = ", recipe_version); file_replace_text("meta.yaml", " url:", recipe_git_url); file_replace_text("meta.yaml", " sha256:", recipe_git_rev); file_replace_text("meta.yaml", " number:", recipe_buildno); } char command[PATH_MAX]; sprintf(command, "build --python=%s .", ctx->meta.python); status = conda_exec(command); if (status && !ctx->meta.continue_on_error) { return -1; } if (RECIPE_TYPE_GENERIC != recipe_type) { popd(); } popd(); } } if (recipe_dir) { free(recipe_dir); } } return 0; } struct StrList *delivery_build_wheels(struct Delivery *ctx) { struct StrList *result = NULL; struct Process proc; memset(&proc, 0, sizeof(proc)); result = strlist_init(); if (!result) { perror("unable to allocate memory for string list"); result = NULL; return NULL; } for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { if (!ctx->tests[i].build_recipe && ctx->tests[i].repository) { // build from source char srcdir[PATH_MAX]; char wheeldir[PATH_MAX]; memset(srcdir, 0, sizeof(srcdir)); memset(wheeldir, 0, sizeof(wheeldir)); sprintf(srcdir, "%s/%s", ctx->storage.build_sources_dir, ctx->tests[i].name); git_clone(&proc, ctx->tests[i].repository, srcdir, ctx->tests[i].version); pushd(srcdir); { if (python_exec("-m build -w ")) { fprintf(stderr, "failed to generate wheel package for %s-%s\n", ctx->tests[i].name, ctx->tests[i].version); if (!ctx->meta.continue_on_error) { strlist_free(result); result = NULL; return NULL; } } else { DIR *dp; struct dirent *rec; dp = opendir("dist"); if (!dp) { fprintf(stderr, "wheel artifact directory does not exist: %s\n", ctx->storage.wheel_artifact_dir); strlist_free(result); return NULL; } while ((rec = readdir(dp)) != NULL) { if (strstr(rec->d_name, ctx->tests[i].name)) { strlist_append(result, rec->d_name); } } } popd(); } } } return result; } static char *requirement_from_test(struct Delivery *ctx, const char *name) { static char result[PATH_MAX]; memset(result, 0, sizeof(result)); for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { if (!strcmp(ctx->tests[i].name, name)) { sprintf(result, "git+%s@%s", ctx->tests[i].repository, ctx->tests[i].version); break; } } if (!strlen(result)) { return NULL; } return result; } int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, char *env_name, int type, struct StrList **manifest) { char cmd[PATH_MAX]; char pkgs[OMC_BUFSIZ]; char *env_current = getenv("CONDA_DEFAULT_ENV"); if (env_current) { // The requested environment is not the current environment if (strcmp(env_current, env_name) != 0) { // Activate the requested environment printf("Activating: %s\n", env_name); conda_activate(conda_install_dir, env_name); runtime_replace(&ctx->runtime.environ, __environ); } } memset(cmd, 0, sizeof(cmd)); memset(pkgs, 0, sizeof(pkgs)); strcat(cmd, "install"); typedef int (*Runner)(const char *); Runner runner = NULL; if (INSTALL_PKG_CONDA & type) { runner = conda_exec; } else if (INSTALL_PKG_PIP & type) { runner = pip_exec; } if (INSTALL_PKG_CONDA_DEFERRED & type) { strcat(cmd, " --use-local"); } else if (INSTALL_PKG_PIP_DEFERRED & type) { strcat(cmd, " --upgrade"); } for (size_t x = 0; manifest[x] != NULL; x++) { char *name = NULL; for (size_t p = 0; p < strlist_count(manifest[x]); p++) { name = strlist_item(manifest[x], p); strip(name); if (!strlen(name)) { continue; } if (INSTALL_PKG_PIP_DEFERRED & type) { //DIR *dp; //struct dirent *rec; //dp = opendir(ctx->storage.wheel_artifact_dir); //if (!dp) { // perror(ctx->storage.wheel_artifact_dir); // exit(1); //} //char pyver_compact[100]; //sprintf(pyver_compact, "-cp%s", ctx->meta.python); //strchrdel(pyver_compact, "."); //while ((rec = readdir(dp)) != NULL) { // struct Wheel *wheelfile = NULL; // if (!strcmp(rec->d_name, ".") || !strcmp(rec->d_name, "..")) { // continue; // } // if (DT_DIR == rec->d_type && startswith(rec->d_name, name)) { // wheelfile = get_wheel_file(ctx->storage.wheel_artifact_dir, name, (char *[]) {pyver_compact, NULL}); // if (wheelfile) { // sprintf(cmd + strlen(cmd), " %s/%s", wheelfile->path_name, wheelfile->file_name); // free(wheelfile); // break; // } // } //} //closedir(dp); char *requirement = requirement_from_test(ctx, name); if (requirement) { sprintf(cmd + strlen(cmd), " '%s'", requirement); } } else { if (startswith(name, "--") || startswith(name, "-")) { sprintf(cmd + strlen(cmd), " %s", name); } else { sprintf(cmd + strlen(cmd), " '%s'", name); } } } int status = runner(cmd); if (status) { return status; } } return 0; } void delivery_get_installer_url(struct Delivery *delivery, char *result) { if (delivery->conda.installer_version) { // Use version specified by configuration file sprintf(result, "%s/%s-%s-%s-%s.sh", delivery->conda.installer_baseurl, delivery->conda.installer_name, delivery->conda.installer_version, delivery->conda.installer_platform, delivery->conda.installer_arch); } else { // Use latest installer sprintf(result, "%s/%s-%s-%s.sh", delivery->conda.installer_baseurl, delivery->conda.installer_name, delivery->conda.installer_platform, delivery->conda.installer_arch); } } int delivery_get_installer(char *installer_url) { char *script = path_basename(installer_url); if (access(script, F_OK)) { // Script doesn't exist if (download(installer_url, script)) { // download failed return -1; } } else { msg(OMC_MSG_RESTRICT | OMC_MSG_L3, "Skipped, installer already exists\n", script); } return 0; } int delivery_copy_conda_artifacts(struct Delivery *ctx) { char cmd[PATH_MAX]; char conda_build_dir[PATH_MAX]; char subdir[PATH_MAX]; memset(cmd, 0, sizeof(cmd)); memset(conda_build_dir, 0, sizeof(conda_build_dir)); memset(subdir, 0, sizeof(subdir)); sprintf(conda_build_dir, "%s/%s", ctx->storage.conda_install_prefix, "conda-bld"); // One must run conda build at least once to create the "conda-bld" directory. // When this directory is missing there can be no build artifacts. if (access(conda_build_dir, F_OK) < 0) { msg(OMC_MSG_RESTRICT | OMC_MSG_WARN | OMC_MSG_L3, "Skipped: 'conda build' has never been executed.\n"); return 0; } snprintf(cmd, sizeof(cmd) - 1, "rsync -avi --progress %s/%s %s", conda_build_dir, ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], ctx->storage.conda_artifact_dir); return system(cmd); } int delivery_copy_wheel_artifacts(struct Delivery *ctx) { char cmd[PATH_MAX]; memset(cmd, 0, sizeof(cmd)); snprintf(cmd, sizeof(cmd) - 1, "rsync -avi --progress %s/*/dist/*.whl %s", ctx->storage.build_sources_dir, ctx->storage.wheel_artifact_dir); return system(cmd); } int delivery_index_wheel_artifacts(struct Delivery *ctx) { struct dirent *rec; DIR *dp; dp = opendir(ctx->storage.wheel_artifact_dir); if (!dp) { return -1; } while ((rec = readdir(dp)) != NULL) { // skip directories if (DT_DIR == rec->d_type || !endswith(rec->d_name, ".whl")) { continue; } char name[NAME_MAX]; strcpy(name, rec->d_name); char **parts = split(name, "-", 1); strcpy(name, parts[0]); split_free(parts); tolower_s(name); char path_dest[PATH_MAX]; sprintf(path_dest, "%s/%s/", ctx->storage.wheel_artifact_dir, name); mkdir(path_dest, 0755); sprintf(path_dest + strlen(path_dest), "%s", rec->d_name); char path_src[PATH_MAX]; sprintf(path_src, "%s/%s", ctx->storage.wheel_artifact_dir, rec->d_name); rename(path_src, path_dest); } return 0; } void delivery_install_conda(char *install_script, char *conda_install_dir) { struct Process proc; memset(&proc, 0, sizeof(proc)); if (!access(conda_install_dir, F_OK)) { if (rmtree(conda_install_dir)) { perror("unable to remove previous installation"); exit(1); } } // -b = batch mode if (shell_safe(&proc, (char *[]) {find_program("bash"), install_script, "-b", "-p", conda_install_dir, NULL})) { fprintf(stderr, "conda installation failed\n"); exit(1); } } void delivery_conda_enable(struct Delivery *ctx, char *conda_install_dir) { if (conda_activate(conda_install_dir, "base")) { fprintf(stderr, "conda activation failed\n"); exit(1); } if (runtime_replace(&ctx->runtime.environ, __environ)) { perror("unable to replace runtime environment after activating conda"); exit(1); } conda_setup_headless(); } void delivery_defer_packages(struct Delivery *ctx, int type) { struct StrList *dataptr = NULL; struct StrList *deferred = NULL; char *name = NULL; char cmd[PATH_MAX]; memset(cmd, 0, sizeof(cmd)); char mode[10]; if (DEFER_CONDA == type) { dataptr = ctx->conda.conda_packages; deferred = ctx->conda.conda_packages_defer; strcpy(mode, "conda"); } else if (DEFER_PIP == type) { dataptr = ctx->conda.pip_packages; deferred = ctx->conda.pip_packages_defer; strcpy(mode, "pip"); } msg(OMC_MSG_L2, "Filtering %s packages by test definition...\n", mode); struct StrList *filtered = NULL; filtered = strlist_init(); for (size_t i = 0, z = 0; i < strlist_count(dataptr); i++) { name = strlist_item(dataptr, i); if (!strlen(name) || isblank(*name) || isspace(*name)) { continue; } msg(OMC_MSG_L3, "package '%s': ", name); int ignore_pkg = 0; for (size_t x = 0; x < sizeof(ctx->tests) / sizeof(ctx->tests[0]); x++) { if (ctx->tests[x].name) { if (startswith(ctx->tests[x].name, name)) { ignore_pkg = 1; z++; break; } } } if (ignore_pkg) { printf("BUILD FOR HOST\n"); strlist_append(deferred, name); } else { printf("USE EXISTING\n"); strlist_append(filtered, name); } } if (!strlist_count(deferred)) { msg(OMC_MSG_WARN, "No packages were filtered by test definitions"); } else { if (DEFER_CONDA == type) { strlist_free(ctx->conda.conda_packages); ctx->conda.conda_packages = strlist_copy(filtered); } else if (DEFER_PIP == type) { strlist_free(ctx->conda.pip_packages); ctx->conda.pip_packages = strlist_copy(filtered); } } if (filtered) { strlist_free(filtered); } } char *delivery_get_spec_header(struct Delivery *ctx) { char output[BUFSIZ]; char stamp[100]; const char *header = "# delivery_name: %s\n" "# creation_time: %s\n" "# conda_version: %s\n" "# condabuild_version: %s\n"; strftime(stamp, sizeof(stamp) - 1, "%c", ctx->info.time_info); sprintf(output, header, ctx->info.release_name, stamp, ctx->conda.tool_version, ctx->conda.tool_build_version); return strdup(output); } void delivery_rewrite_spec(struct Delivery *ctx, char *filename) { char *package_name = NULL; char output[PATH_MAX]; char *header = NULL; char *tempfile = NULL; FILE *tp = NULL; header = delivery_get_release_header(ctx); if (!header) { msg(OMC_MSG_ERROR, "failed to generate release header string\n", filename); exit(1); } tempfile = xmkstemp(&tp); if (!tempfile || !tp) { msg(OMC_MSG_ERROR, "%s: unable to create temporary file\n", strerror(errno)); exit(1); } fprintf(tp, "%s", header); sprintf(output, " - %s", ctx->storage.conda_staging_url); file_replace_text(filename, " - local", output); for (size_t i = 0; i < strlist_count(ctx->conda.pip_packages_defer); i++) { package_name = strlist_item(ctx->conda.pip_packages_defer, i); char target[PATH_MAX]; char replacement[PATH_MAX]; struct Wheel *wheelfile; memset(target, 0, sizeof(target)); memset(replacement, 0, sizeof(replacement)); sprintf(target, " - %s", package_name); // TODO: I still want to use wheels for this but tagging semantics are getting in the way. // When someone pushes a lightweight tag setuptools_scm will not resolve the expected // refs unless the following is present in pyproject.toml, setup.cfg, or setup.py: // // git_describe_command = "git describe --tags" # at the bare minimum // //char abi[NAME_MAX]; //strcpy(abi, ctx->meta.python); //strchrdel(abi, "."); //char source_dir[PATH_MAX]; //sprintf(source_dir, "%s/%s", ctx->storage.build_sources_dir, package_name); //wheelfile = get_wheel_file(ctx->storage.wheel_artifact_dir, package_name, (char *[]) {git_describe(source_dir), abi, ctx->system.arch, NULL}); //if (wheelfile) { // sprintf(replacement, " - %s/%s", ctx->storage.wheel_staging_url, wheelfile->file_name); // file_replace_text(filename, target, replacement); //} // end of TODO char *requirement = requirement_from_test(ctx, package_name); if (requirement) { sprintf(replacement, " - %s", requirement); file_replace_text(filename, target, replacement); } else { fprintf(stderr, "an error occurred while rewriting a release artifact: %s\n", filename); fprintf(stderr, "mapping a replacement value for package defined by '[test:%s]' failed: %s\n", package_name, package_name); fprintf(stderr, "target string in artifact was:\n%s\n", target); exit(1); } } } int delivery_index_conda_artifacts(struct Delivery *ctx) { return conda_index(ctx->storage.conda_artifact_dir); } void delivery_tests_run(struct Delivery *ctx) { struct Process proc; if (!ctx->tests[0].name) { msg(OMC_MSG_WARN | OMC_MSG_L2, "no tests are defined!\n"); } else { for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { if (!ctx->tests[i].name && !ctx->tests[i].repository && !ctx->tests[i].script) { // unused entry continue; } msg(OMC_MSG_L2, "%s %s\n", ctx->tests[i].name, ctx->tests[i].version); if (!ctx->tests[i].script || !strlen(ctx->tests[i].script)) { msg(OMC_MSG_WARN | OMC_MSG_L3, "Nothing to do. To fix, declare a 'script' in section: [test:%s]\n", ctx->tests[i].name); continue; } char destdir[PATH_MAX]; sprintf(destdir, "%s/%s", ctx->storage.build_sources_dir, path_basename(ctx->tests[i].repository)); msg(OMC_MSG_L3, "Cloning %s\n", ctx->tests[i].repository); git_clone(&proc, ctx->tests[i].repository, destdir, ctx->tests[i].version); if (pushd(destdir)) { COE_CHECK_ABORT(!globals.continue_on_error, "Unable to enter repository directory\n"); } else { #if 1 int status; char cmd[PATH_MAX]; msg(OMC_MSG_L3, "Testing %s\n", ctx->tests[i].name); memset(&proc, 0, sizeof(proc)); // enable trace mode before executing each test script memset(cmd, 0, sizeof(cmd)); sprintf(cmd, "set -x ; %s", ctx->tests[i].script); status = shell2(&proc, cmd); if (status) { COE_CHECK_ABORT(!globals.continue_on_error, "Test failure"); } popd(); #else msg(OMC_MSG_WARNING | OMC_MSG_L3, "TESTING DISABLED BY CODE!\n"); #endif } } } } void delivery_gather_tool_versions(struct Delivery *ctx) { // Extract version from tool output ctx->conda.tool_version = shell_output("conda --version"); if (ctx->conda.tool_version) strip(ctx->conda.tool_version); ctx->conda.tool_build_version = shell_output("conda build --version"); if (ctx->conda.tool_build_version) strip(ctx->conda.tool_version); }