aboutsummaryrefslogtreecommitdiff
path: root/src/lib/delivery
diff options
context:
space:
mode:
Diffstat (limited to 'src/lib/delivery')
-rw-r--r--src/lib/delivery/delivery_build.c296
-rw-r--r--src/lib/delivery/delivery_docker.c2
-rw-r--r--src/lib/delivery/delivery_init.c6
-rw-r--r--src/lib/delivery/delivery_install.c3
-rw-r--r--src/lib/delivery/delivery_populate.c48
5 files changed, 329 insertions, 26 deletions
diff --git a/src/lib/delivery/delivery_build.c b/src/lib/delivery/delivery_build.c
index 8370e6d..86555bd 100644
--- a/src/lib/delivery/delivery_build.c
+++ b/src/lib/delivery/delivery_build.c
@@ -25,11 +25,26 @@ int delivery_build_recipes(struct Delivery *ctx) {
char recipe_git_url[PATH_MAX];
char recipe_git_rev[PATH_MAX];
+ char tag[100];
+ if (ctx->tests[i].repository_info_tag) {
+ const int is_long_tag = num_chars(ctx->tests[i].repository_info_tag, '-') > 1;
+ if (is_long_tag) {
+ const size_t len = strcspn(ctx->tests[i].repository_info_tag, "-");
+ strncpy(tag, ctx->tests[i].repository_info_tag, len);
+ tag[len] = '\0';
+ } else {
+ strcpy(tag, ctx->tests[i].repository_info_tag);
+ tag[strlen(ctx->tests[i].repository_info_tag)] = '\0';
+ }
+ } else {
+ strcpy(tag, ctx->tests[i].version);
+ }
+
//sprintf(recipe_version, "{%% set version = GIT_DESCRIBE_TAG ~ \".dev\" ~ GIT_DESCRIBE_NUMBER ~ \"+\" ~ GIT_DESCRIBE_HASH %%}");
//sprintf(recipe_git_url, " git_url: %s", ctx->tests[i].repository);
//sprintf(recipe_git_rev, " git_rev: %s", ctx->tests[i].version);
// TODO: Conditionally download archives if github.com is the origin. Else, use raw git_* keys ^^^
- sprintf(recipe_version, "{%% set version = \"%s\" %%}", ctx->tests[i].repository_info_tag ? ctx->tests[i].repository_info_tag : ctx->tests[i].version);
+ sprintf(recipe_version, "{%% set version = \"%s\" %%}", tag);
sprintf(recipe_git_url, " url: %s/archive/refs/tags/{{ version }}.tar.gz", ctx->tests[i].repository);
strcpy(recipe_git_rev, "");
sprintf(recipe_buildno, " number: 0");
@@ -127,7 +142,232 @@ int filter_repo_tags(char *repo, struct StrList *patterns) {
return result;
}
+static int read_without_line_endings(const size_t line, char ** arg) {
+ (void) line;
+ if (*arg) {
+ strip(*arg);
+ if (isempty(*arg)) {
+ return 1; // skip
+ }
+ }
+ return 0;
+}
+
+int manylinux_exec(const char *image, const char *script, const char *copy_to_container_dir, const char *copy_from_container_dir, const char *copy_to_host_dir) {
+ int result = -1; // fail by default
+ char *container_name = NULL;
+ char *source_copy_command = NULL;
+ char *copy_command = NULL;
+ char *rm_command = NULL;
+ char *nop_create_command = NULL;
+ char *nop_rm_command = NULL;
+ char *volume_rm_command = NULL;
+ char *find_command = NULL;
+ char *wheel_paths_filename = NULL;
+ char *args = NULL;
+
+ const uid_t uid = geteuid();
+ char suffix[7] = {0};
+
+ // setup
+
+ if (get_random_bytes(suffix, sizeof(suffix))) {
+ SYSERROR("%s", "unable to acquire value from random generator");
+ goto manylinux_fail;
+ }
+
+ if (asprintf(&container_name, "manylinux_build_%d_%zd_%s", uid, time(NULL), suffix) < 0) {
+ SYSERROR("%s", "unable to allocate memory for container name");
+ goto manylinux_fail;
+ }
+
+ if (asprintf(&args, "--name %s -w /build -v %s:/build", container_name, container_name) < 0) {
+ SYSERROR("%s", "unable to allocate memory for docker arguments");
+ goto manylinux_fail;
+ }
+
+ if (!strstr(image, "manylinux")) {
+ SYSERROR("expected a manylinux image, but got %s", image);
+ goto manylinux_fail;
+ }
+
+ if (asprintf(&nop_create_command, "run --name nop_%s -v %s:/build busybox", container_name, container_name) < 0) {
+ SYSERROR("%s", "unable to allocate memory for nop container command");
+ goto manylinux_fail;
+ }
+
+ if (asprintf(&source_copy_command, "cp %s nop_%s:/build", copy_to_container_dir, container_name) < 0) {
+ SYSERROR("%s", "unable to allocate memory for source copy command");
+ goto manylinux_fail;
+ }
+
+ if (asprintf(&nop_rm_command, "rm nop_%s", container_name) < 0) {
+ SYSERROR("%s", "unable to allocate memory for nop container command");
+ goto manylinux_fail;
+ }
+
+ if (asprintf(&wheel_paths_filename, "wheel_paths_%s.txt", container_name) < 0) {
+ SYSERROR("%s", "unable to allocate memory for wheel paths file name");
+ goto manylinux_fail;
+ }
+
+ if (asprintf(&find_command, "run --rm -t -v %s:/build busybox sh -c 'find %s -name \"*.whl\"' > %s", container_name, copy_from_container_dir, wheel_paths_filename) < 0) {
+ SYSERROR("%s", "unable to allocate memory for find command");
+ goto manylinux_fail;
+ }
+
+ // execute
+
+ if (docker_exec(nop_create_command, 0)) {
+ SYSERROR("%s", "docker nop container creation failed");
+ goto manylinux_fail;
+ }
+
+ if (docker_exec(source_copy_command, 0)) {
+ SYSERROR("%s", "docker source copy operation failed");
+ goto manylinux_fail;
+ }
+
+ if (docker_exec(nop_rm_command, STASIS_DOCKER_QUIET)) {
+ SYSERROR("%s", "docker nop container removal failed");
+ goto manylinux_fail;
+ }
+
+ if (docker_script(image, args, (char *) script, 0)) {
+ SYSERROR("%s", "manylinux execution failed");
+ goto manylinux_fail;
+ }
+
+ if (docker_exec(find_command, 0)) {
+ SYSERROR("%s", "docker find command failed");
+ goto manylinux_fail;
+ }
+
+ struct StrList *wheel_paths = strlist_init();
+ if (!wheel_paths) {
+ SYSERROR("%s", "wheel_paths not initialized");
+ goto manylinux_fail;
+ }
+
+ if (strlist_append_file(wheel_paths, wheel_paths_filename, read_without_line_endings)) {
+ SYSERROR("%s", "wheel_paths append failed");
+ goto manylinux_fail;
+ }
+
+ for (size_t i = 0; i < strlist_count(wheel_paths); i++) {
+ const char *item = strlist_item(wheel_paths, i);
+ if (asprintf(&copy_command, "cp %s:%s %s", container_name, item, copy_to_host_dir) < 0) {
+ SYSERROR("%s", "unable to allocate memory for docker copy command");
+ goto manylinux_fail;
+ }
+
+ if (docker_exec(copy_command, 0)) {
+ SYSERROR("%s", "docker copy operation failed");
+ goto manylinux_fail;
+ }
+ guard_free(copy_command);
+ }
+
+ // Success
+ result = 0;
+
+ manylinux_fail:
+ if (wheel_paths_filename) {
+ remove(wheel_paths_filename);
+ }
+
+ if (container_name) {
+ // Keep going on failure unless memory related.
+ // We don't want build debris everywhere.
+ if (asprintf(&rm_command, "rm %s", container_name) < 0) {
+ SYSERROR("%s", "unable to allocate memory for rm command");
+ goto late_fail;
+ }
+
+ if (docker_exec(rm_command, STASIS_DOCKER_QUIET)) {
+ SYSERROR("%s", "docker container removal operation failed");
+ }
+
+ if (asprintf(&volume_rm_command, "volume rm -f %s", container_name) < 0) {
+ SYSERROR("%s", "unable to allocate memory for docker volume removal command");
+ goto late_fail;
+ }
+
+ if (docker_exec(volume_rm_command, STASIS_DOCKER_QUIET)) {
+ SYSERROR("%s", "docker volume removal operation failed");
+ }
+ }
+
+ late_fail:
+ guard_free(container_name);
+ guard_free(args);
+ guard_free(copy_command);
+ guard_free(rm_command);
+ guard_free(volume_rm_command);
+ guard_free(source_copy_command);
+ guard_free(nop_create_command);
+ guard_free(nop_rm_command);
+ guard_free(find_command);
+ guard_free(wheel_paths_filename);
+ guard_strlist_free(&wheel_paths);
+ return result;
+}
+
+int delivery_build_wheels_manylinux(struct Delivery *ctx, const char *outdir) {
+ msg(STASIS_MSG_L1, "Building wheels\n");
+
+ const char *manylinux_image = globals.wheel_builder_manylinux_image;
+ if (!manylinux_image) {
+ SYSERROR("%s", "manylinux_image not initialized");
+ return -1;
+ }
+
+ int manylinux_build_status = 0;
+
+ msg(STASIS_MSG_L2, "Using: %s\n", manylinux_image);
+ const struct Meta *meta = &ctx->meta;
+ const char *script_fmt =
+ "set -e -x\n"
+ "git config --global --add safe.directory /build\n"
+ "python%s -m pip install auditwheel build\n"
+ "python%s -m build -w .\n"
+ "auditwheel show dist/*.whl\n"
+ "auditwheel repair --allow-pure-python-wheel dist/*.whl\n";
+ char *script = NULL;
+ if (asprintf(&script, script_fmt,
+ meta->python, meta->python) < 0) {
+ SYSERROR("%s", "unable to allocate memory for build script");
+ return -1;
+ }
+ manylinux_build_status = manylinux_exec(
+ manylinux_image,
+ script,
+ "./",
+ "/build/wheelhouse",
+ outdir);
+
+ if (manylinux_build_status) {
+ msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "manylinux build failed (%d)", manylinux_build_status);
+ guard_free(script);
+ return -1;
+ }
+ guard_free(script);
+ return 0;
+}
+
struct StrList *delivery_build_wheels(struct Delivery *ctx) {
+ const int on_linux = strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Linux") == 0;
+ const int docker_usable = ctx->deploy.docker.capabilities.usable;
+ int use_builder_build = strcmp(globals.wheel_builder, "native") == 0;
+ const int use_builder_cibuildwheel = strcmp(globals.wheel_builder, "cibuildwheel") == 0 && on_linux && docker_usable;
+ const int use_builder_manylinux = strcmp(globals.wheel_builder, "manylinux") == 0 && on_linux && docker_usable;
+
+ if (!use_builder_build && !use_builder_cibuildwheel && !use_builder_manylinux) {
+ msg(STASIS_MSG_WARN, "Cannot build wheel for platform using: %\n", globals.wheel_builder);
+ msg(STASIS_MSG_WARN, "Falling back to native toolchain.\n", globals.wheel_builder);
+ use_builder_build = 1;
+ }
+
struct StrList *result = NULL;
struct Process proc = {0};
@@ -161,6 +401,12 @@ struct StrList *delivery_build_wheels(struct Delivery *ctx) {
return NULL;
}
+ if (!ctx->tests[i].repository_info_tag) {
+ ctx->tests[i].repository_info_tag = strdup(git_describe(srcdir));
+ }
+ if (!ctx->tests[i].repository_info_ref) {
+ ctx->tests[i].repository_info_ref = strdup(git_rev_parse(srcdir, ctx->tests[i].version));
+ }
if (ctx->tests[i].repository_remove_tags && strlist_count(ctx->tests[i].repository_remove_tags)) {
filter_repo_tags(srcdir, ctx->tests[i].repository_remove_tags);
}
@@ -192,28 +438,41 @@ struct StrList *delivery_build_wheels(struct Delivery *ctx) {
guard_strlist_free(&result);
return NULL;
}
-
- if (asprintf(&cmd, "-m build -w -o %s", outdir) < 0) {
- SYSERROR("%s", "Unable to allocate memory for build command");
- return NULL;
- }
- if (!strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Linux")
- && ctx->deploy.docker.capabilities.usable) {
- guard_free(cmd);
- if (asprintf(&cmd, "-m cibuildwheel --output-dir %s --only cp%s-manylinux_%s",
- outdir, ctx->meta.python_compact, ctx->system.arch) < 0) {
- SYSERROR("%s", "Unable to allocate memory for cibuildwheel command");
+ if (use_builder_manylinux) {
+ if (delivery_build_wheels_manylinux(ctx, outdir)) {
+ fprintf(stderr, "failed to generate wheel package for %s-%s\n", ctx->tests[i].name,
+ ctx->tests[i].version);
+ guard_strlist_free(&result);
+ guard_free(cmd);
return NULL;
}
- }
+ } else if (use_builder_build || use_builder_cibuildwheel) {
+ if (use_builder_build) {
+ if (asprintf(&cmd, "-m build -w -o %s", outdir) < 0) {
+ SYSERROR("%s", "Unable to allocate memory for build command");
+ return NULL;
+ }
+ } else if (use_builder_cibuildwheel) {
+ if (asprintf(&cmd, "-m cibuildwheel --output-dir %s --only cp%s-manylinux_%s",
+ outdir, ctx->meta.python_compact, ctx->system.arch) < 0) {
+ SYSERROR("%s", "Unable to allocate memory for cibuildwheel command");
+ return NULL;
+ }
+ }
- if (python_exec(cmd)) {
- fprintf(stderr, "failed to generate wheel package for %s-%s\n", ctx->tests[i].name,
- ctx->tests[i].version);
- guard_strlist_free(&result);
- guard_free(cmd);
+ if (python_exec(cmd)) {
+ fprintf(stderr, "failed to generate wheel package for %s-%s\n", ctx->tests[i].name,
+ ctx->tests[i].version);
+ guard_strlist_free(&result);
+ guard_free(cmd);
+ return NULL;
+ }
+ } else {
+ SYSERROR("unknown wheel builder backend: %s", globals.wheel_builder);
return NULL;
}
+
+ guard_free(cmd);
popd();
} else {
fprintf(stderr, "Unable to enter source directory %s: %s\n", srcdir, strerror(errno));
@@ -225,4 +484,3 @@ struct StrList *delivery_build_wheels(struct Delivery *ctx) {
}
return result;
}
-
diff --git a/src/lib/delivery/delivery_docker.c b/src/lib/delivery/delivery_docker.c
index 57015ad..2c43caf 100644
--- a/src/lib/delivery/delivery_docker.c
+++ b/src/lib/delivery/delivery_docker.c
@@ -111,7 +111,7 @@ int delivery_docker(struct Delivery *ctx) {
msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "Image test script has no content\n");
} else {
int state;
- if ((state = docker_script(tag, ctx->deploy.docker.test_script, 0))) {
+ if ((state = docker_script(tag, "--rm", ctx->deploy.docker.test_script, 0))) {
msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "Non-zero exit (%d) from test script. %s image archive will not be generated.\n", state >> 8, tag);
// test failed -- don't save the image
return -1;
diff --git a/src/lib/delivery/delivery_init.c b/src/lib/delivery/delivery_init.c
index a60d6af..eb0b527 100644
--- a/src/lib/delivery/delivery_init.c
+++ b/src/lib/delivery/delivery_init.c
@@ -150,7 +150,7 @@ void delivery_init_dirs_stage1(struct Delivery *ctx) {
}
int delivery_init_platform(struct Delivery *ctx) {
- msg(STASIS_MSG_L2, "Setting architecture\n");
+ SYSDEBUG("%s", "Setting architecture\n");
char archsuffix[20];
struct utsname uts;
if (uname(&uts)) {
@@ -179,7 +179,7 @@ int delivery_init_platform(struct Delivery *ctx) {
strcpy(archsuffix, ctx->system.arch);
}
- msg(STASIS_MSG_L2, "Setting platform\n");
+ SYSDEBUG("%s", "Setting platform\n");
strcpy(ctx->system.platform[DELIVERY_PLATFORM], uts.sysname);
if (!strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Darwin")) {
sprintf(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], "osx-%s", archsuffix);
@@ -287,6 +287,8 @@ int delivery_init(struct Delivery *ctx, int render_mode) {
int bootstrap_build_info(struct Delivery *ctx) {
struct Delivery local = {0};
+ memcpy(&local.deploy.docker.capabilities, &ctx->deploy.docker.capabilities, sizeof(local.deploy.docker.capabilities));
+
SYSDEBUG("ini_open(%s)", ctx->_stasis_ini_fp.cfg_path);
local._stasis_ini_fp.cfg = ini_open(ctx->_stasis_ini_fp.cfg_path);
SYSDEBUG("ini_open(%s)", ctx->_stasis_ini_fp.delivery_path);
diff --git a/src/lib/delivery/delivery_install.c b/src/lib/delivery/delivery_install.c
index f1637a3..f40a509 100644
--- a/src/lib/delivery/delivery_install.c
+++ b/src/lib/delivery/delivery_install.c
@@ -277,7 +277,8 @@ int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, cha
// not found
fprintf(stderr, "No wheel packages found that match the description of '%s'", info->name);
} else {
- // found
+ // found, replace the original version with newly detected version
+ guard_free(info->version);
info->version = strdup(whl->version);
}
guard_strlist_free(&tag_data);
diff --git a/src/lib/delivery/delivery_populate.c b/src/lib/delivery/delivery_populate.c
index 15ab6bd..4ea93c1 100644
--- a/src/lib/delivery/delivery_populate.c
+++ b/src/lib/delivery/delivery_populate.c
@@ -85,6 +85,45 @@ int populate_delivery_cfg(struct Delivery *ctx, int render_mode) {
}
globals.pip_packages = ini_getval_strlist(cfg, "default", "pip_packages", LINE_SEP, render_mode, &err);
+ err = 0;
+ if (!globals.wheel_builder) {
+ globals.wheel_builder = ini_getval_str(cfg, "default", "wheel_builder", render_mode, &err);
+ if (err) {
+ msg(STASIS_MSG_WARN, "wheel_builder is undefined. Falling back to system toolchain: 'build'.\n");
+ globals.wheel_builder = strdup("build");
+ if (!globals.wheel_builder) {
+ SYSERROR("%s", "unable to allocate memory for default wheel_builder value");
+ return -1;
+ }
+ }
+ }
+
+ err = 0;
+ if (!globals.wheel_builder_manylinux_image) {
+ globals.wheel_builder_manylinux_image = ini_getval_str(cfg, "default", "wheel_builder_manylinux_image", render_mode, &err);
+ }
+
+ if (err && globals.wheel_builder && strcmp(globals.wheel_builder, "manylinux") == 0) {
+ SYSERROR("%s", "default:wheel_builder is set to 'manylinux', however default:wheel_builder_manylinux_image is not configured");
+ return -1;
+ }
+
+ if (strcmp(globals.wheel_builder, "manylinux") == 0) {
+ char *manifest_inspect_cmd = NULL;
+ if (asprintf(&manifest_inspect_cmd, "manifest inspect '%s'", globals.wheel_builder_manylinux_image) < 0) {
+ SYSERROR("%s", "unable to allocate memory for docker command");
+ guard_free(manifest_inspect_cmd);
+ return -1;
+ }
+ if (ctx->deploy.docker.capabilities.usable && docker_exec(manifest_inspect_cmd, STASIS_DOCKER_QUIET_STDOUT)) {
+ SYSERROR("Image provided by default:wheel_builder_manylinux_image does not exist: %s", globals.wheel_builder_manylinux_image);
+ guard_free(manifest_inspect_cmd);
+ return -1;
+ }
+ guard_free(manifest_inspect_cmd);
+ }
+
+
if (globals.jfrog.jfrog_artifactory_base_url) {
guard_free(globals.jfrog.jfrog_artifactory_base_url);
}
@@ -200,7 +239,9 @@ int populate_delivery_ini(struct Delivery *ctx, int render_mode) {
normalize_ini_list(&ini, &ctx->conda.pip_packages_purge, "conda", "pip_packages_purge", render_mode);
// Delivery metadata consumed
- populate_mission_ini(&ctx, render_mode);
+ if (populate_mission_ini(&ctx, render_mode)) {
+ return -1;
+ }
if (ctx->info.release_name) {
guard_free(ctx->info.release_name);
@@ -320,6 +361,7 @@ int populate_mission_ini(struct Delivery **ctx, int render_mode) {
int err = 0;
if ((*ctx)->_stasis_ini_fp.mission) {
+ // mission configurations are optional
return 0;
}
@@ -333,12 +375,12 @@ int populate_mission_ini(struct Delivery **ctx, int render_mode) {
globals.sysconfdir, "mission", (*ctx)->meta.mission, (*ctx)->meta.mission);
}
- msg(STASIS_MSG_L2, "Reading mission configuration: %s\n", missionfile);
+ SYSDEBUG("Reading mission configuration: %s\n", missionfile);
(*ctx)->_stasis_ini_fp.mission = ini_open(missionfile);
struct INIFILE *ini = (*ctx)->_stasis_ini_fp.mission;
if (!ini) {
msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read mission configuration: %s, %s\n", missionfile, strerror(errno));
- exit(1);
+ return -1;
}
(*ctx)->_stasis_ini_fp.mission_path = strdup(missionfile);
generated by cgit (git 2.34.1) at 2026-04-08 07:35:14 -0400