#include "delivery.h" int delivery_build_recipes(struct Delivery *ctx) { for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { char *recipe_dir = NULL; if (ctx->tests[i].build_recipe) { // build a conda recipe if (recipe_clone(ctx->storage.build_recipes_dir, ctx->tests[i].build_recipe, NULL, &recipe_dir)) { fprintf(stderr, "Encountered an issue while cloning recipe for: %s\n", ctx->tests[i].name); return -1; } if (!recipe_dir) { fprintf(stderr, "BUG: recipe_clone() succeeded but recipe_dir is NULL: %s\n", strerror(errno)); return -1; } int recipe_type = recipe_get_type(recipe_dir); if(!pushd(recipe_dir)) { if (RECIPE_TYPE_ASTROCONDA == recipe_type) { pushd(path_basename(ctx->tests[i].repository)); } else if (RECIPE_TYPE_CONDA_FORGE == recipe_type) { pushd("recipe"); } char recipe_version[100]; char recipe_buildno[100]; char recipe_git_url[PATH_MAX]; char recipe_git_rev[PATH_MAX]; char tag[100]; if (ctx->tests[i].repository_info_tag) { const int is_long_tag = num_chars(ctx->tests[i].repository_info_tag, '-') > 1; if (is_long_tag) { const size_t len = strcspn(ctx->tests[i].repository_info_tag, "-"); strncpy(tag, ctx->tests[i].repository_info_tag, len); tag[len] = '\0'; } else { strcpy(tag, ctx->tests[i].repository_info_tag); tag[strlen(ctx->tests[i].repository_info_tag)] = '\0'; } } else { strcpy(tag, ctx->tests[i].version); } //sprintf(recipe_version, "{%% set version = GIT_DESCRIBE_TAG ~ \".dev\" ~ GIT_DESCRIBE_NUMBER ~ \"+\" ~ GIT_DESCRIBE_HASH %%}"); //sprintf(recipe_git_url, " git_url: %s", ctx->tests[i].repository); //sprintf(recipe_git_rev, " git_rev: %s", ctx->tests[i].version); // TODO: Conditionally download archives if github.com is the origin. Else, use raw git_* keys ^^^ sprintf(recipe_version, "{%% set version = \"%s\" %%}", tag); sprintf(recipe_git_url, " url: %s/archive/refs/tags/{{ version }}.tar.gz", ctx->tests[i].repository); strcpy(recipe_git_rev, ""); sprintf(recipe_buildno, " number: 0"); unsigned flags = REPLACE_TRUNCATE_AFTER_MATCH; //file_replace_text("meta.yaml", "{% set version = ", recipe_version); if (ctx->meta.final) { // remove this. i.e. statis cannot deploy a release to conda-forge sprintf(recipe_version, "{%% set version = \"%s\" %%}", ctx->tests[i].version); // TODO: replace sha256 of tagged archive // TODO: leave the recipe unchanged otherwise. in theory this should produce the same conda package hash as conda forge. // For now, remove the sha256 requirement file_replace_text("meta.yaml", "sha256:", "\n", flags); } else { file_replace_text("meta.yaml", "{% set version = ", recipe_version, flags); file_replace_text("meta.yaml", " url:", recipe_git_url, flags); //file_replace_text("meta.yaml", "sha256:", recipe_git_rev); file_replace_text("meta.yaml", " sha256:", "\n", flags); file_replace_text("meta.yaml", " number:", recipe_buildno, flags); } char command[PATH_MAX]; if (RECIPE_TYPE_CONDA_FORGE == recipe_type) { char arch[STASIS_NAME_MAX] = {0}; char platform[STASIS_NAME_MAX] = {0}; strcpy(platform, ctx->system.platform[DELIVERY_PLATFORM]); if (strstr(platform, "Darwin")) { memset(platform, 0, sizeof(platform)); strcpy(platform, "osx"); } tolower_s(platform); if (strstr(ctx->system.arch, "arm64")) { strcpy(arch, "arm64"); } else if (strstr(ctx->system.arch, "64")) { strcpy(arch, "64"); } else { strcat(arch, "32"); // blind guess } tolower_s(arch); sprintf(command, "mambabuild --python=%s -m ../.ci_support/%s_%s_.yaml .", ctx->meta.python, platform, arch); } else { sprintf(command, "mambabuild --python=%s .", ctx->meta.python); } int status = conda_exec(command); if (status) { guard_free(recipe_dir); return -1; } if (RECIPE_TYPE_GENERIC != recipe_type) { popd(); } popd(); } else { fprintf(stderr, "Unable to enter recipe directory %s: %s\n", recipe_dir, strerror(errno)); guard_free(recipe_dir); return -1; } } guard_free(recipe_dir); } return 0; } int filter_repo_tags(char *repo, struct StrList *patterns) { int result = 0; if (!pushd(repo)) { int list_status = 0; char *tags_raw = shell_output("git tag -l", &list_status); struct StrList *tags = strlist_init(); strlist_append_tokenize(tags, tags_raw, LINE_SEP); for (size_t i = 0; tags && i < strlist_count(tags); i++) { char *tag = strlist_item(tags, i); for (size_t p = 0; p < strlist_count(patterns); p++) { char *pattern = strlist_item(patterns, p); int match = fnmatch(pattern, tag, 0); if (!match) { char cmd[PATH_MAX] = {0}; sprintf(cmd, "git tag -d %s", tag); result += system(cmd); break; } } } guard_strlist_free(&tags); guard_free(tags_raw); popd(); } else { result = -1; } return result; } static int read_without_line_endings(const size_t line, char ** arg) { (void) line; if (*arg) { strip(*arg); if (isempty(*arg)) { return 1; // skip } } return 0; } int manylinux_exec(const char *image, const char *script, const char *copy_to_container_dir, const char *copy_from_container_dir, const char *copy_to_host_dir) { int result = -1; // fail by default char *container_name = NULL; char *source_copy_command = NULL; char *copy_command = NULL; char *rm_command = NULL; char *nop_create_command = NULL; char *nop_rm_command = NULL; char *volume_rm_command = NULL; char *find_command = NULL; char *wheel_paths_filename = NULL; char *args = NULL; const uid_t uid = geteuid(); char suffix[7] = {0}; // setup if (get_random_bytes(suffix, sizeof(suffix))) { SYSERROR("%s", "unable to acquire value from random generator"); goto manylinux_fail; } if (asprintf(&container_name, "manylinux_build_%d_%zd_%s", uid, time(NULL), suffix) < 0) { SYSERROR("%s", "unable to allocate memory for container name"); goto manylinux_fail; } if (asprintf(&args, "--name %s -w /build -v %s:/build", container_name, container_name) < 0) { SYSERROR("%s", "unable to allocate memory for docker arguments"); goto manylinux_fail; } if (!strstr(image, "manylinux")) { SYSERROR("expected a manylinux image, but got %s", image); goto manylinux_fail; } if (asprintf(&nop_create_command, "run --name nop_%s -v %s:/build busybox", container_name, container_name) < 0) { SYSERROR("%s", "unable to allocate memory for nop container command"); goto manylinux_fail; } if (asprintf(&source_copy_command, "cp %s nop_%s:/build", copy_to_container_dir, container_name) < 0) { SYSERROR("%s", "unable to allocate memory for source copy command"); goto manylinux_fail; } if (asprintf(&nop_rm_command, "rm nop_%s", container_name) < 0) { SYSERROR("%s", "unable to allocate memory for nop container command"); goto manylinux_fail; } if (asprintf(&wheel_paths_filename, "wheel_paths_%s.txt", container_name) < 0) { SYSERROR("%s", "unable to allocate memory for wheel paths file name"); goto manylinux_fail; } if (asprintf(&find_command, "run --rm -t -v %s:/build busybox sh -c 'find %s -name \"*.whl\"' > %s", container_name, copy_from_container_dir, wheel_paths_filename) < 0) { SYSERROR("%s", "unable to allocate memory for find command"); goto manylinux_fail; } // execute if (docker_exec(nop_create_command, 0)) { SYSERROR("%s", "docker nop container creation failed"); goto manylinux_fail; } if (docker_exec(source_copy_command, 0)) { SYSERROR("%s", "docker source copy operation failed"); goto manylinux_fail; } if (docker_exec(nop_rm_command, STASIS_DOCKER_QUIET)) { SYSERROR("%s", "docker nop container removal failed"); goto manylinux_fail; } if (docker_script(image, args, (char *) script, 0)) { SYSERROR("%s", "manylinux execution failed"); goto manylinux_fail; } if (docker_exec(find_command, 0)) { SYSERROR("%s", "docker find command failed"); goto manylinux_fail; } struct StrList *wheel_paths = strlist_init(); if (!wheel_paths) { SYSERROR("%s", "wheel_paths not initialized"); goto manylinux_fail; } if (strlist_append_file(wheel_paths, wheel_paths_filename, read_without_line_endings)) { SYSERROR("%s", "wheel_paths append failed"); goto manylinux_fail; } for (size_t i = 0; i < strlist_count(wheel_paths); i++) { const char *item = strlist_item(wheel_paths, i); if (asprintf(©_command, "cp %s:%s %s", container_name, item, copy_to_host_dir) < 0) { SYSERROR("%s", "unable to allocate memory for docker copy command"); goto manylinux_fail; } if (docker_exec(copy_command, 0)) { SYSERROR("%s", "docker copy operation failed"); goto manylinux_fail; } guard_free(copy_command); } // Success result = 0; manylinux_fail: if (wheel_paths_filename) { remove(wheel_paths_filename); } if (container_name) { // Keep going on failure unless memory related. // We don't want build debris everywhere. if (asprintf(&rm_command, "rm %s", container_name) < 0) { SYSERROR("%s", "unable to allocate memory for rm command"); goto late_fail; } if (docker_exec(rm_command, STASIS_DOCKER_QUIET)) { SYSERROR("%s", "docker container removal operation failed"); } if (asprintf(&volume_rm_command, "volume rm -f %s", container_name) < 0) { SYSERROR("%s", "unable to allocate memory for docker volume removal command"); goto late_fail; } if (docker_exec(volume_rm_command, STASIS_DOCKER_QUIET)) { SYSERROR("%s", "docker volume removal operation failed"); } } late_fail: guard_free(container_name); guard_free(args); guard_free(copy_command); guard_free(rm_command); guard_free(volume_rm_command); guard_free(source_copy_command); guard_free(nop_create_command); guard_free(nop_rm_command); guard_free(find_command); guard_free(wheel_paths_filename); guard_strlist_free(&wheel_paths); return result; } int delivery_build_wheels_manylinux(struct Delivery *ctx, const char *outdir) { msg(STASIS_MSG_L1, "Building wheels\n"); const char *manylinux_image = globals.wheel_builder_manylinux_image; if (!manylinux_image) { SYSERROR("%s", "manylinux_image not initialized"); return -1; } int manylinux_build_status = 0; msg(STASIS_MSG_L2, "Using: %s\n", manylinux_image); const struct Meta *meta = &ctx->meta; const char *script_fmt = "set -e -x\n" "git config --global --add safe.directory /build\n" "python%s -m pip install auditwheel build\n" "python%s -m build -w .\n" "auditwheel show dist/*.whl\n" "auditwheel repair --allow-pure-python-wheel dist/*.whl\n"; char *script = NULL; if (asprintf(&script, script_fmt, meta->python, meta->python) < 0) { SYSERROR("%s", "unable to allocate memory for build script"); return -1; } manylinux_build_status = manylinux_exec( manylinux_image, script, "./", "/build/wheelhouse", outdir); if (manylinux_build_status) { msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "manylinux build failed (%d)", manylinux_build_status); guard_free(script); return -1; } guard_free(script); return 0; } struct StrList *delivery_build_wheels(struct Delivery *ctx) { const int on_linux = strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Linux") == 0; const int docker_usable = ctx->deploy.docker.capabilities.usable; int use_builder_build = strcmp(globals.wheel_builder, "native") == 0; const int use_builder_cibuildwheel = strcmp(globals.wheel_builder, "cibuildwheel") == 0 && on_linux && docker_usable; const int use_builder_manylinux = strcmp(globals.wheel_builder, "manylinux") == 0 && on_linux && docker_usable; if (!use_builder_build && !use_builder_cibuildwheel && !use_builder_manylinux) { msg(STASIS_MSG_WARN, "Cannot build wheel for platform using: %\n", globals.wheel_builder); msg(STASIS_MSG_WARN, "Falling back to native toolchain.\n", globals.wheel_builder); use_builder_build = 1; } struct StrList *result = NULL; struct Process proc = {0}; result = strlist_init(); if (!result) { perror("unable to allocate memory for string list"); return NULL; } for (size_t p = 0; p < strlist_count(ctx->conda.pip_packages_defer); p++) { char name[100] = {0}; char *fullspec = strlist_item(ctx->conda.pip_packages_defer, p); strncpy(name, fullspec, sizeof(name) - 1); remove_extras(name); char *spec = find_version_spec(name); if (spec) { *spec = '\0'; } for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { if ((ctx->tests[i].name && !strcmp(name, ctx->tests[i].name)) && (!ctx->tests[i].build_recipe && ctx->tests[i].repository)) { // build from source char srcdir[PATH_MAX]; char wheeldir[PATH_MAX]; memset(srcdir, 0, sizeof(srcdir)); memset(wheeldir, 0, sizeof(wheeldir)); sprintf(srcdir, "%s/%s", ctx->storage.build_sources_dir, ctx->tests[i].name); if (git_clone(&proc, ctx->tests[i].repository, srcdir, ctx->tests[i].version)) { SYSERROR("Unable to checkout tag '%s' for package '%s' from repository '%s'\n", ctx->tests[i].version, ctx->tests[i].name, ctx->tests[i].repository); return NULL; } if (!ctx->tests[i].repository_info_tag) { ctx->tests[i].repository_info_tag = strdup(git_describe(srcdir)); } if (!ctx->tests[i].repository_info_ref) { ctx->tests[i].repository_info_ref = strdup(git_rev_parse(srcdir, ctx->tests[i].version)); } if (ctx->tests[i].repository_remove_tags && strlist_count(ctx->tests[i].repository_remove_tags)) { filter_repo_tags(srcdir, ctx->tests[i].repository_remove_tags); } if (!pushd(srcdir)) { char dname[NAME_MAX]; char outdir[PATH_MAX]; char *cmd = NULL; memset(dname, 0, sizeof(dname)); memset(outdir, 0, sizeof(outdir)); const int dep_status = check_python_package_dependencies("."); if (dep_status) { fprintf(stderr, "\nPlease replace all occurrences above with standard package specs:\n" "\n" " package==x.y.z\n" " package>=x.y.z\n" " package<=x.y.z\n" " ...\n" "\n"); COE_CHECK_ABORT(dep_status, "Unreproducible delivery"); } strcpy(dname, ctx->tests[i].name); tolower_s(dname); sprintf(outdir, "%s/%s", ctx->storage.wheel_artifact_dir, dname); if (mkdirs(outdir, 0755)) { fprintf(stderr, "failed to create output directory: %s\n", outdir); guard_strlist_free(&result); return NULL; } if (use_builder_manylinux) { if (delivery_build_wheels_manylinux(ctx, outdir)) { fprintf(stderr, "failed to generate wheel package for %s-%s\n", ctx->tests[i].name, ctx->tests[i].version); guard_strlist_free(&result); guard_free(cmd); return NULL; } } else if (use_builder_build || use_builder_cibuildwheel) { if (use_builder_build) { if (asprintf(&cmd, "-m build -w -o %s", outdir) < 0) { SYSERROR("%s", "Unable to allocate memory for build command"); return NULL; } } else if (use_builder_cibuildwheel) { if (asprintf(&cmd, "-m cibuildwheel --output-dir %s --only cp%s-manylinux_%s", outdir, ctx->meta.python_compact, ctx->system.arch) < 0) { SYSERROR("%s", "Unable to allocate memory for cibuildwheel command"); return NULL; } } if (python_exec(cmd)) { fprintf(stderr, "failed to generate wheel package for %s-%s\n", ctx->tests[i].name, ctx->tests[i].version); guard_strlist_free(&result); guard_free(cmd); return NULL; } } else { SYSERROR("unknown wheel builder backend: %s", globals.wheel_builder); return NULL; } guard_free(cmd); popd(); } else { fprintf(stderr, "Unable to enter source directory %s: %s\n", srcdir, strerror(errno)); guard_strlist_free(&result); return NULL; } } } } return result; }