diff options
Diffstat (limited to 'src/lib/delivery/delivery_build.c')
| -rw-r--r-- | src/lib/delivery/delivery_build.c | 296 |
1 files changed, 277 insertions, 19 deletions
diff --git a/src/lib/delivery/delivery_build.c b/src/lib/delivery/delivery_build.c index 8370e6d..86555bd 100644 --- a/src/lib/delivery/delivery_build.c +++ b/src/lib/delivery/delivery_build.c @@ -25,11 +25,26 @@ int delivery_build_recipes(struct Delivery *ctx) { char recipe_git_url[PATH_MAX]; char recipe_git_rev[PATH_MAX]; + char tag[100]; + if (ctx->tests[i].repository_info_tag) { + const int is_long_tag = num_chars(ctx->tests[i].repository_info_tag, '-') > 1; + if (is_long_tag) { + const size_t len = strcspn(ctx->tests[i].repository_info_tag, "-"); + strncpy(tag, ctx->tests[i].repository_info_tag, len); + tag[len] = '\0'; + } else { + strcpy(tag, ctx->tests[i].repository_info_tag); + tag[strlen(ctx->tests[i].repository_info_tag)] = '\0'; + } + } else { + strcpy(tag, ctx->tests[i].version); + } + //sprintf(recipe_version, "{%% set version = GIT_DESCRIBE_TAG ~ \".dev\" ~ GIT_DESCRIBE_NUMBER ~ \"+\" ~ GIT_DESCRIBE_HASH %%}"); //sprintf(recipe_git_url, " git_url: %s", ctx->tests[i].repository); //sprintf(recipe_git_rev, " git_rev: %s", ctx->tests[i].version); // TODO: Conditionally download archives if github.com is the origin. Else, use raw git_* keys ^^^ - sprintf(recipe_version, "{%% set version = \"%s\" %%}", ctx->tests[i].repository_info_tag ? ctx->tests[i].repository_info_tag : ctx->tests[i].version); + sprintf(recipe_version, "{%% set version = \"%s\" %%}", tag); sprintf(recipe_git_url, " url: %s/archive/refs/tags/{{ version }}.tar.gz", ctx->tests[i].repository); strcpy(recipe_git_rev, ""); sprintf(recipe_buildno, " number: 0"); @@ -127,7 +142,232 @@ int filter_repo_tags(char *repo, struct StrList *patterns) { return result; } +static int read_without_line_endings(const size_t line, char ** arg) { + (void) line; + if (*arg) { + strip(*arg); + if (isempty(*arg)) { + return 1; // skip + } + } + return 0; +} + +int manylinux_exec(const char *image, const char *script, const char *copy_to_container_dir, const char *copy_from_container_dir, const char *copy_to_host_dir) { + int result = -1; // fail by default + char *container_name = NULL; + char *source_copy_command = NULL; + char *copy_command = NULL; + char *rm_command = NULL; + char *nop_create_command = NULL; + char *nop_rm_command = NULL; + char *volume_rm_command = NULL; + char *find_command = NULL; + char *wheel_paths_filename = NULL; + char *args = NULL; + + const uid_t uid = geteuid(); + char suffix[7] = {0}; + + // setup + + if (get_random_bytes(suffix, sizeof(suffix))) { + SYSERROR("%s", "unable to acquire value from random generator"); + goto manylinux_fail; + } + + if (asprintf(&container_name, "manylinux_build_%d_%zd_%s", uid, time(NULL), suffix) < 0) { + SYSERROR("%s", "unable to allocate memory for container name"); + goto manylinux_fail; + } + + if (asprintf(&args, "--name %s -w /build -v %s:/build", container_name, container_name) < 0) { + SYSERROR("%s", "unable to allocate memory for docker arguments"); + goto manylinux_fail; + } + + if (!strstr(image, "manylinux")) { + SYSERROR("expected a manylinux image, but got %s", image); + goto manylinux_fail; + } + + if (asprintf(&nop_create_command, "run --name nop_%s -v %s:/build busybox", container_name, container_name) < 0) { + SYSERROR("%s", "unable to allocate memory for nop container command"); + goto manylinux_fail; + } + + if (asprintf(&source_copy_command, "cp %s nop_%s:/build", copy_to_container_dir, container_name) < 0) { + SYSERROR("%s", "unable to allocate memory for source copy command"); + goto manylinux_fail; + } + + if (asprintf(&nop_rm_command, "rm nop_%s", container_name) < 0) { + SYSERROR("%s", "unable to allocate memory for nop container command"); + goto manylinux_fail; + } + + if (asprintf(&wheel_paths_filename, "wheel_paths_%s.txt", container_name) < 0) { + SYSERROR("%s", "unable to allocate memory for wheel paths file name"); + goto manylinux_fail; + } + + if (asprintf(&find_command, "run --rm -t -v %s:/build busybox sh -c 'find %s -name \"*.whl\"' > %s", container_name, copy_from_container_dir, wheel_paths_filename) < 0) { + SYSERROR("%s", "unable to allocate memory for find command"); + goto manylinux_fail; + } + + // execute + + if (docker_exec(nop_create_command, 0)) { + SYSERROR("%s", "docker nop container creation failed"); + goto manylinux_fail; + } + + if (docker_exec(source_copy_command, 0)) { + SYSERROR("%s", "docker source copy operation failed"); + goto manylinux_fail; + } + + if (docker_exec(nop_rm_command, STASIS_DOCKER_QUIET)) { + SYSERROR("%s", "docker nop container removal failed"); + goto manylinux_fail; + } + + if (docker_script(image, args, (char *) script, 0)) { + SYSERROR("%s", "manylinux execution failed"); + goto manylinux_fail; + } + + if (docker_exec(find_command, 0)) { + SYSERROR("%s", "docker find command failed"); + goto manylinux_fail; + } + + struct StrList *wheel_paths = strlist_init(); + if (!wheel_paths) { + SYSERROR("%s", "wheel_paths not initialized"); + goto manylinux_fail; + } + + if (strlist_append_file(wheel_paths, wheel_paths_filename, read_without_line_endings)) { + SYSERROR("%s", "wheel_paths append failed"); + goto manylinux_fail; + } + + for (size_t i = 0; i < strlist_count(wheel_paths); i++) { + const char *item = strlist_item(wheel_paths, i); + if (asprintf(©_command, "cp %s:%s %s", container_name, item, copy_to_host_dir) < 0) { + SYSERROR("%s", "unable to allocate memory for docker copy command"); + goto manylinux_fail; + } + + if (docker_exec(copy_command, 0)) { + SYSERROR("%s", "docker copy operation failed"); + goto manylinux_fail; + } + guard_free(copy_command); + } + + // Success + result = 0; + + manylinux_fail: + if (wheel_paths_filename) { + remove(wheel_paths_filename); + } + + if (container_name) { + // Keep going on failure unless memory related. + // We don't want build debris everywhere. + if (asprintf(&rm_command, "rm %s", container_name) < 0) { + SYSERROR("%s", "unable to allocate memory for rm command"); + goto late_fail; + } + + if (docker_exec(rm_command, STASIS_DOCKER_QUIET)) { + SYSERROR("%s", "docker container removal operation failed"); + } + + if (asprintf(&volume_rm_command, "volume rm -f %s", container_name) < 0) { + SYSERROR("%s", "unable to allocate memory for docker volume removal command"); + goto late_fail; + } + + if (docker_exec(volume_rm_command, STASIS_DOCKER_QUIET)) { + SYSERROR("%s", "docker volume removal operation failed"); + } + } + + late_fail: + guard_free(container_name); + guard_free(args); + guard_free(copy_command); + guard_free(rm_command); + guard_free(volume_rm_command); + guard_free(source_copy_command); + guard_free(nop_create_command); + guard_free(nop_rm_command); + guard_free(find_command); + guard_free(wheel_paths_filename); + guard_strlist_free(&wheel_paths); + return result; +} + +int delivery_build_wheels_manylinux(struct Delivery *ctx, const char *outdir) { + msg(STASIS_MSG_L1, "Building wheels\n"); + + const char *manylinux_image = globals.wheel_builder_manylinux_image; + if (!manylinux_image) { + SYSERROR("%s", "manylinux_image not initialized"); + return -1; + } + + int manylinux_build_status = 0; + + msg(STASIS_MSG_L2, "Using: %s\n", manylinux_image); + const struct Meta *meta = &ctx->meta; + const char *script_fmt = + "set -e -x\n" + "git config --global --add safe.directory /build\n" + "python%s -m pip install auditwheel build\n" + "python%s -m build -w .\n" + "auditwheel show dist/*.whl\n" + "auditwheel repair --allow-pure-python-wheel dist/*.whl\n"; + char *script = NULL; + if (asprintf(&script, script_fmt, + meta->python, meta->python) < 0) { + SYSERROR("%s", "unable to allocate memory for build script"); + return -1; + } + manylinux_build_status = manylinux_exec( + manylinux_image, + script, + "./", + "/build/wheelhouse", + outdir); + + if (manylinux_build_status) { + msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "manylinux build failed (%d)", manylinux_build_status); + guard_free(script); + return -1; + } + guard_free(script); + return 0; +} + struct StrList *delivery_build_wheels(struct Delivery *ctx) { + const int on_linux = strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Linux") == 0; + const int docker_usable = ctx->deploy.docker.capabilities.usable; + int use_builder_build = strcmp(globals.wheel_builder, "native") == 0; + const int use_builder_cibuildwheel = strcmp(globals.wheel_builder, "cibuildwheel") == 0 && on_linux && docker_usable; + const int use_builder_manylinux = strcmp(globals.wheel_builder, "manylinux") == 0 && on_linux && docker_usable; + + if (!use_builder_build && !use_builder_cibuildwheel && !use_builder_manylinux) { + msg(STASIS_MSG_WARN, "Cannot build wheel for platform using: %\n", globals.wheel_builder); + msg(STASIS_MSG_WARN, "Falling back to native toolchain.\n", globals.wheel_builder); + use_builder_build = 1; + } + struct StrList *result = NULL; struct Process proc = {0}; @@ -161,6 +401,12 @@ struct StrList *delivery_build_wheels(struct Delivery *ctx) { return NULL; } + if (!ctx->tests[i].repository_info_tag) { + ctx->tests[i].repository_info_tag = strdup(git_describe(srcdir)); + } + if (!ctx->tests[i].repository_info_ref) { + ctx->tests[i].repository_info_ref = strdup(git_rev_parse(srcdir, ctx->tests[i].version)); + } if (ctx->tests[i].repository_remove_tags && strlist_count(ctx->tests[i].repository_remove_tags)) { filter_repo_tags(srcdir, ctx->tests[i].repository_remove_tags); } @@ -192,28 +438,41 @@ struct StrList *delivery_build_wheels(struct Delivery *ctx) { guard_strlist_free(&result); return NULL; } - - if (asprintf(&cmd, "-m build -w -o %s", outdir) < 0) { - SYSERROR("%s", "Unable to allocate memory for build command"); - return NULL; - } - if (!strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Linux") - && ctx->deploy.docker.capabilities.usable) { - guard_free(cmd); - if (asprintf(&cmd, "-m cibuildwheel --output-dir %s --only cp%s-manylinux_%s", - outdir, ctx->meta.python_compact, ctx->system.arch) < 0) { - SYSERROR("%s", "Unable to allocate memory for cibuildwheel command"); + if (use_builder_manylinux) { + if (delivery_build_wheels_manylinux(ctx, outdir)) { + fprintf(stderr, "failed to generate wheel package for %s-%s\n", ctx->tests[i].name, + ctx->tests[i].version); + guard_strlist_free(&result); + guard_free(cmd); return NULL; } - } + } else if (use_builder_build || use_builder_cibuildwheel) { + if (use_builder_build) { + if (asprintf(&cmd, "-m build -w -o %s", outdir) < 0) { + SYSERROR("%s", "Unable to allocate memory for build command"); + return NULL; + } + } else if (use_builder_cibuildwheel) { + if (asprintf(&cmd, "-m cibuildwheel --output-dir %s --only cp%s-manylinux_%s", + outdir, ctx->meta.python_compact, ctx->system.arch) < 0) { + SYSERROR("%s", "Unable to allocate memory for cibuildwheel command"); + return NULL; + } + } - if (python_exec(cmd)) { - fprintf(stderr, "failed to generate wheel package for %s-%s\n", ctx->tests[i].name, - ctx->tests[i].version); - guard_strlist_free(&result); - guard_free(cmd); + if (python_exec(cmd)) { + fprintf(stderr, "failed to generate wheel package for %s-%s\n", ctx->tests[i].name, + ctx->tests[i].version); + guard_strlist_free(&result); + guard_free(cmd); + return NULL; + } + } else { + SYSERROR("unknown wheel builder backend: %s", globals.wheel_builder); return NULL; } + + guard_free(cmd); popd(); } else { fprintf(stderr, "Unable to enter source directory %s: %s\n", srcdir, strerror(errno)); @@ -225,4 +484,3 @@ struct StrList *delivery_build_wheels(struct Delivery *ctx) { } return result; } - |