Merge pull request #128 from jhunkeler/manylinuxHEADnextmaster

Manylinux

5 files changed, 329 insertions, 26 deletions

diff --git a/src/lib/delivery/delivery_build.c b/src/lib/delivery/delivery_build.c
index 8370e6d..86555bd 100644
--- a/src/lib/delivery/delivery_build.c
+++ b/src/lib/delivery/delivery_build.c
@@ -25,11 +25,26 @@ int delivery_build_recipes(struct Delivery *ctx) {
                 char recipe_git_url[PATH_MAX];
                 char recipe_git_rev[PATH_MAX];
 
+                char tag[100];
+                if (ctx->tests[i].repository_info_tag) {
+                    const int is_long_tag = num_chars(ctx->tests[i].repository_info_tag, '-') > 1;
+                    if (is_long_tag) {
+                        const size_t len = strcspn(ctx->tests[i].repository_info_tag, "-");
+                        strncpy(tag, ctx->tests[i].repository_info_tag, len);
+                        tag[len] = '\0';
+                    } else {
+                        strcpy(tag, ctx->tests[i].repository_info_tag);
+                        tag[strlen(ctx->tests[i].repository_info_tag)] = '\0';
+                    }
+                } else {
+                    strcpy(tag, ctx->tests[i].version);
+		}
+
                 //sprintf(recipe_version, "{%% set version = GIT_DESCRIBE_TAG ~ \".dev\" ~ GIT_DESCRIBE_NUMBER ~ \"+\" ~ GIT_DESCRIBE_HASH %%}");
                 //sprintf(recipe_git_url, "  git_url: %s", ctx->tests[i].repository);
                 //sprintf(recipe_git_rev, "  git_rev: %s", ctx->tests[i].version);
                 // TODO: Conditionally download archives if github.com is the origin. Else, use raw git_* keys ^^^
-                sprintf(recipe_version, "{%% set version = \"%s\" %%}", ctx->tests[i].repository_info_tag ? ctx->tests[i].repository_info_tag : ctx->tests[i].version);
+                sprintf(recipe_version, "{%% set version = \"%s\" %%}", tag);
                 sprintf(recipe_git_url, "  url: %s/archive/refs/tags/{{ version }}.tar.gz", ctx->tests[i].repository);
                 strcpy(recipe_git_rev, "");
                 sprintf(recipe_buildno, "  number: 0");
@@ -127,7 +142,232 @@ int filter_repo_tags(char *repo, struct StrList *patterns) {
     return result;
 }
 
+static int read_without_line_endings(const size_t line, char ** arg) {
+    (void) line;
+    if (*arg) {
+        strip(*arg);
+        if (isempty(*arg)) {
+            return 1; // skip
+        }
+    }
+    return 0;
+}
+
+int manylinux_exec(const char *image, const char *script, const char *copy_to_container_dir, const char *copy_from_container_dir, const char *copy_to_host_dir) {
+    int result = -1; // fail by default
+    char *container_name = NULL;
+    char *source_copy_command = NULL;
+    char *copy_command = NULL;
+    char *rm_command = NULL;
+    char *nop_create_command = NULL;
+    char *nop_rm_command = NULL;
+    char *volume_rm_command = NULL;
+    char *find_command = NULL;
+    char *wheel_paths_filename = NULL;
+    char *args = NULL;
+
+    const uid_t uid = geteuid();
+    char suffix[7] = {0};
+
+    // setup
+
+    if (get_random_bytes(suffix, sizeof(suffix))) {
+        SYSERROR("%s", "unable to acquire value from random generator");
+        goto manylinux_fail;
+    }
+
+    if (asprintf(&container_name, "manylinux_build_%d_%zd_%s", uid, time(NULL), suffix) < 0) {
+        SYSERROR("%s", "unable to allocate memory for container name");
+        goto manylinux_fail;
+    }
+
+    if (asprintf(&args, "--name %s -w /build -v %s:/build", container_name, container_name) < 0) {
+        SYSERROR("%s", "unable to allocate memory for docker arguments");
+        goto manylinux_fail;
+    }
+
+    if (!strstr(image, "manylinux")) {
+        SYSERROR("expected a manylinux image, but got %s", image);
+        goto manylinux_fail;
+    }
+
+    if (asprintf(&nop_create_command, "run --name nop_%s -v %s:/build busybox", container_name, container_name) < 0) {
+        SYSERROR("%s", "unable to allocate memory for nop container command");
+        goto manylinux_fail;
+    }
+
+    if (asprintf(&source_copy_command, "cp %s nop_%s:/build", copy_to_container_dir, container_name) < 0) {
+        SYSERROR("%s", "unable to allocate memory for source copy command");
+        goto manylinux_fail;
+    }
+
+    if (asprintf(&nop_rm_command, "rm nop_%s", container_name) < 0) {
+        SYSERROR("%s", "unable to allocate memory for nop container command");
+        goto manylinux_fail;
+    }
+
+    if (asprintf(&wheel_paths_filename, "wheel_paths_%s.txt", container_name) < 0) {
+        SYSERROR("%s", "unable to allocate memory for wheel paths file name");
+        goto manylinux_fail;
+    }
+
+    if (asprintf(&find_command, "run --rm -t -v %s:/build busybox sh -c 'find %s -name \"*.whl\"' > %s", container_name, copy_from_container_dir, wheel_paths_filename) < 0) {
+        SYSERROR("%s", "unable to allocate memory for find command");
+        goto manylinux_fail;
+    }
+
+    // execute
+
+    if (docker_exec(nop_create_command, 0)) {
+        SYSERROR("%s", "docker nop container creation failed");
+        goto manylinux_fail;
+    }
+
+    if (docker_exec(source_copy_command, 0)) {
+        SYSERROR("%s", "docker source copy operation failed");
+        goto manylinux_fail;
+    }
+
+    if (docker_exec(nop_rm_command, STASIS_DOCKER_QUIET)) {
+        SYSERROR("%s", "docker nop container removal failed");
+        goto manylinux_fail;
+    }
+
+    if (docker_script(image, args, (char *) script, 0)) {
+        SYSERROR("%s", "manylinux execution failed");
+        goto manylinux_fail;
+    }
+
+    if (docker_exec(find_command, 0)) {
+        SYSERROR("%s", "docker find command failed");
+        goto manylinux_fail;
+    }
+
+    struct StrList *wheel_paths = strlist_init();
+    if (!wheel_paths) {
+        SYSERROR("%s", "wheel_paths not initialized");
+        goto manylinux_fail;
+    }
+
+    if (strlist_append_file(wheel_paths, wheel_paths_filename, read_without_line_endings)) {
+        SYSERROR("%s", "wheel_paths append failed");
+        goto manylinux_fail;
+    }
+
+    for (size_t i = 0; i < strlist_count(wheel_paths); i++) {
+        const char *item = strlist_item(wheel_paths, i);
+        if (asprintf(&copy_command, "cp %s:%s %s", container_name, item, copy_to_host_dir) < 0) {
+            SYSERROR("%s", "unable to allocate memory for docker copy command");
+            goto manylinux_fail;
+        }
+
+        if (docker_exec(copy_command, 0)) {
+            SYSERROR("%s", "docker copy operation failed");
+            goto manylinux_fail;
+        }
+        guard_free(copy_command);
+    }
+
+    // Success
+    result = 0;
+
+    manylinux_fail:
+    if (wheel_paths_filename) {
+        remove(wheel_paths_filename);
+    }
+
+    if (container_name) {
+        // Keep going on failure unless memory related.
+        // We don't want build debris everywhere.
+        if (asprintf(&rm_command, "rm %s", container_name) < 0) {
+            SYSERROR("%s", "unable to allocate memory for rm command");
+            goto late_fail;
+        }
+
+        if (docker_exec(rm_command, STASIS_DOCKER_QUIET)) {
+            SYSERROR("%s", "docker container removal operation failed");
+        }
+
+        if (asprintf(&volume_rm_command, "volume rm -f %s", container_name) < 0) {
+            SYSERROR("%s", "unable to allocate memory for docker volume removal command");
+            goto late_fail;
+        }
+
+        if (docker_exec(volume_rm_command, STASIS_DOCKER_QUIET)) {
+            SYSERROR("%s", "docker volume removal operation failed");
+        }
+    }
+
+    late_fail:
+    guard_free(container_name);
+    guard_free(args);
+    guard_free(copy_command);
+    guard_free(rm_command);
+    guard_free(volume_rm_command);
+    guard_free(source_copy_command);
+    guard_free(nop_create_command);
+    guard_free(nop_rm_command);
+    guard_free(find_command);
+    guard_free(wheel_paths_filename);
+    guard_strlist_free(&wheel_paths);
+    return result;
+}
+
+int delivery_build_wheels_manylinux(struct Delivery *ctx, const char *outdir) {
+    msg(STASIS_MSG_L1, "Building wheels\n");
+
+    const char *manylinux_image = globals.wheel_builder_manylinux_image;
+    if (!manylinux_image) {
+        SYSERROR("%s", "manylinux_image not initialized");
+        return -1;
+    }
+
+    int manylinux_build_status = 0;
+
+    msg(STASIS_MSG_L2, "Using: %s\n", manylinux_image);
+    const struct Meta *meta = &ctx->meta;
+    const char *script_fmt =
+        "set -e -x\n"
+        "git config --global --add safe.directory /build\n"
+        "python%s -m pip install auditwheel build\n"
+        "python%s -m build -w .\n"
+        "auditwheel show dist/*.whl\n"
+        "auditwheel repair --allow-pure-python-wheel dist/*.whl\n";
+    char *script = NULL;
+    if (asprintf(&script, script_fmt,
+        meta->python, meta->python) < 0) {
+        SYSERROR("%s", "unable to allocate memory for build script");
+        return -1;
+    }
+    manylinux_build_status = manylinux_exec(
+        manylinux_image,
+        script,
+        "./",
+        "/build/wheelhouse",
+        outdir);
+
+    if (manylinux_build_status) {
+        msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "manylinux build failed (%d)", manylinux_build_status);
+        guard_free(script);
+        return -1;
+    }
+    guard_free(script);
+    return 0;
+}
+
 struct StrList *delivery_build_wheels(struct Delivery *ctx) {
+    const int on_linux = strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Linux") == 0;
+    const int docker_usable = ctx->deploy.docker.capabilities.usable;
+    int use_builder_build = strcmp(globals.wheel_builder, "native") == 0;
+    const int use_builder_cibuildwheel = strcmp(globals.wheel_builder, "cibuildwheel") == 0 && on_linux && docker_usable;
+    const int use_builder_manylinux = strcmp(globals.wheel_builder, "manylinux") == 0 && on_linux && docker_usable;
+
+    if (!use_builder_build && !use_builder_cibuildwheel && !use_builder_manylinux) {
+        msg(STASIS_MSG_WARN, "Cannot build wheel for platform using: %\n", globals.wheel_builder);
+        msg(STASIS_MSG_WARN, "Falling back to native toolchain.\n", globals.wheel_builder);
+        use_builder_build = 1;
+    }
+
     struct StrList *result = NULL;
     struct Process proc = {0};
 
@@ -161,6 +401,12 @@ struct StrList *delivery_build_wheels(struct Delivery *ctx) {
                     return NULL;
                 }
 
+                if (!ctx->tests[i].repository_info_tag) {
+                    ctx->tests[i].repository_info_tag = strdup(git_describe(srcdir));
+                }
+                if (!ctx->tests[i].repository_info_ref) {
+                    ctx->tests[i].repository_info_ref = strdup(git_rev_parse(srcdir, ctx->tests[i].version));
+                }
                 if (ctx->tests[i].repository_remove_tags && strlist_count(ctx->tests[i].repository_remove_tags)) {
                     filter_repo_tags(srcdir, ctx->tests[i].repository_remove_tags);
                 }
@@ -192,28 +438,41 @@ struct StrList *delivery_build_wheels(struct Delivery *ctx) {
                         guard_strlist_free(&result);
                         return NULL;
                     }
-
-                    if (asprintf(&cmd, "-m build -w -o %s", outdir) < 0) {
-                        SYSERROR("%s", "Unable to allocate memory for build command");
-                        return NULL;
-                    }
-                    if (!strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Linux")
-                        && ctx->deploy.docker.capabilities.usable) {
-                        guard_free(cmd);
-                        if (asprintf(&cmd, "-m cibuildwheel --output-dir %s --only cp%s-manylinux_%s",
-                            outdir, ctx->meta.python_compact, ctx->system.arch) < 0) {
-                            SYSERROR("%s", "Unable to allocate memory for cibuildwheel command");
+                    if (use_builder_manylinux) {
+                        if (delivery_build_wheels_manylinux(ctx, outdir)) {
+                            fprintf(stderr, "failed to generate wheel package for %s-%s\n", ctx->tests[i].name,
+                                    ctx->tests[i].version);
+                            guard_strlist_free(&result);
+                            guard_free(cmd);
                             return NULL;
                         }
-                    }
+                    } else if (use_builder_build || use_builder_cibuildwheel) {
+                        if (use_builder_build) {
+                            if (asprintf(&cmd, "-m build -w -o %s", outdir) < 0) {
+                                SYSERROR("%s", "Unable to allocate memory for build command");
+                                return NULL;
+                            }
+                        } else if (use_builder_cibuildwheel) {
+                            if (asprintf(&cmd, "-m cibuildwheel --output-dir %s --only cp%s-manylinux_%s",
+                                outdir, ctx->meta.python_compact, ctx->system.arch) < 0) {
+                                SYSERROR("%s", "Unable to allocate memory for cibuildwheel command");
+                                return NULL;
+                            }
+                        }
 
-                    if (python_exec(cmd)) {
-                        fprintf(stderr, "failed to generate wheel package for %s-%s\n", ctx->tests[i].name,
-                                ctx->tests[i].version);
-                        guard_strlist_free(&result);
-                        guard_free(cmd);
+                        if (python_exec(cmd)) {
+                            fprintf(stderr, "failed to generate wheel package for %s-%s\n", ctx->tests[i].name,
+                                    ctx->tests[i].version);
+                            guard_strlist_free(&result);
+                            guard_free(cmd);
+                            return NULL;
+                        }
+                    } else {
+                        SYSERROR("unknown wheel builder backend: %s", globals.wheel_builder);
                         return NULL;
                     }
+
+                    guard_free(cmd);
                     popd();
                 } else {
                     fprintf(stderr, "Unable to enter source directory %s: %s\n", srcdir, strerror(errno));
@@ -225,4 +484,3 @@ struct StrList *delivery_build_wheels(struct Delivery *ctx) {
     }
     return result;
 }
-
diff --git a/src/lib/delivery/delivery_docker.c b/src/lib/delivery/delivery_docker.c
index 57015ad..2c43caf 100644
--- a/src/lib/delivery/delivery_docker.c
+++ b/src/lib/delivery/delivery_docker.c
@@ -111,7 +111,7 @@ int delivery_docker(struct Delivery *ctx) {
             msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "Image test script has no content\n");
         } else {
             int state;
-            if ((state = docker_script(tag, ctx->deploy.docker.test_script, 0))) {
+            if ((state = docker_script(tag, "--rm", ctx->deploy.docker.test_script, 0))) {
                 msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "Non-zero exit (%d) from test script. %s image archive will not be generated.\n", state >> 8, tag);
                 // test failed -- don't save the image
                 return -1;
diff --git a/src/lib/delivery/delivery_init.c b/src/lib/delivery/delivery_init.c
index a60d6af..eb0b527 100644
--- a/src/lib/delivery/delivery_init.c
+++ b/src/lib/delivery/delivery_init.c
@@ -150,7 +150,7 @@ void delivery_init_dirs_stage1(struct Delivery *ctx) {
 }
 
 int delivery_init_platform(struct Delivery *ctx) {
-    msg(STASIS_MSG_L2, "Setting architecture\n");
+    SYSDEBUG("%s", "Setting architecture\n");
     char archsuffix[20];
     struct utsname uts;
     if (uname(&uts)) {
@@ -179,7 +179,7 @@ int delivery_init_platform(struct Delivery *ctx) {
         strcpy(archsuffix, ctx->system.arch);
     }
 
-    msg(STASIS_MSG_L2, "Setting platform\n");
+    SYSDEBUG("%s", "Setting platform\n");
     strcpy(ctx->system.platform[DELIVERY_PLATFORM], uts.sysname);
     if (!strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Darwin")) {
         sprintf(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], "osx-%s", archsuffix);
@@ -287,6 +287,8 @@ int delivery_init(struct Delivery *ctx, int render_mode) {
 
 int bootstrap_build_info(struct Delivery *ctx) {
     struct Delivery local = {0};
+    memcpy(&local.deploy.docker.capabilities, &ctx->deploy.docker.capabilities, sizeof(local.deploy.docker.capabilities));
+
     SYSDEBUG("ini_open(%s)", ctx->_stasis_ini_fp.cfg_path);
     local._stasis_ini_fp.cfg = ini_open(ctx->_stasis_ini_fp.cfg_path);
     SYSDEBUG("ini_open(%s)", ctx->_stasis_ini_fp.delivery_path);
diff --git a/src/lib/delivery/delivery_install.c b/src/lib/delivery/delivery_install.c
index f1637a3..f40a509 100644
--- a/src/lib/delivery/delivery_install.c
+++ b/src/lib/delivery/delivery_install.c
@@ -277,7 +277,8 @@ int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, cha
                             // not found
                             fprintf(stderr, "No wheel packages found that match the description of '%s'", info->name);
                         } else {
-                            // found
+                            // found, replace the original version with newly detected version
+                            guard_free(info->version);
                             info->version = strdup(whl->version);
                         }
                         guard_strlist_free(&tag_data);
diff --git a/src/lib/delivery/delivery_populate.c b/src/lib/delivery/delivery_populate.c
index 15ab6bd..4ea93c1 100644
--- a/src/lib/delivery/delivery_populate.c
+++ b/src/lib/delivery/delivery_populate.c
@@ -85,6 +85,45 @@ int populate_delivery_cfg(struct Delivery *ctx, int render_mode) {
     }
     globals.pip_packages = ini_getval_strlist(cfg, "default", "pip_packages", LINE_SEP, render_mode, &err);
 
+    err = 0;
+    if (!globals.wheel_builder) {
+        globals.wheel_builder = ini_getval_str(cfg, "default", "wheel_builder", render_mode, &err);
+        if (err) {
+            msg(STASIS_MSG_WARN, "wheel_builder is undefined. Falling back to system toolchain: 'build'.\n");
+            globals.wheel_builder = strdup("build");
+            if (!globals.wheel_builder) {
+                SYSERROR("%s", "unable to allocate memory for default wheel_builder value");
+                return -1;
+            }
+        }
+    }
+
+    err = 0;
+    if (!globals.wheel_builder_manylinux_image) {
+        globals.wheel_builder_manylinux_image = ini_getval_str(cfg, "default", "wheel_builder_manylinux_image", render_mode, &err);
+    }
+
+    if (err && globals.wheel_builder && strcmp(globals.wheel_builder, "manylinux") == 0) {
+        SYSERROR("%s", "default:wheel_builder is set to 'manylinux', however default:wheel_builder_manylinux_image is not configured");
+        return -1;
+    }
+
+    if (strcmp(globals.wheel_builder, "manylinux") == 0) {
+        char *manifest_inspect_cmd = NULL;
+        if (asprintf(&manifest_inspect_cmd, "manifest inspect '%s'", globals.wheel_builder_manylinux_image) < 0) {
+            SYSERROR("%s", "unable to allocate memory for docker command");
+            guard_free(manifest_inspect_cmd);
+            return -1;
+        }
+        if (ctx->deploy.docker.capabilities.usable && docker_exec(manifest_inspect_cmd, STASIS_DOCKER_QUIET_STDOUT)) {
+            SYSERROR("Image provided by default:wheel_builder_manylinux_image does not exist: %s", globals.wheel_builder_manylinux_image);
+            guard_free(manifest_inspect_cmd);
+            return -1;
+        }
+        guard_free(manifest_inspect_cmd);
+    }
+
+
     if (globals.jfrog.jfrog_artifactory_base_url) {
         guard_free(globals.jfrog.jfrog_artifactory_base_url);
     }
@@ -200,7 +239,9 @@ int populate_delivery_ini(struct Delivery *ctx, int render_mode) {
     normalize_ini_list(&ini, &ctx->conda.pip_packages_purge, "conda", "pip_packages_purge", render_mode);
 
     // Delivery metadata consumed
-    populate_mission_ini(&ctx, render_mode);
+    if (populate_mission_ini(&ctx, render_mode)) {
+        return -1;
+    }
 
     if (ctx->info.release_name) {
         guard_free(ctx->info.release_name);
@@ -320,6 +361,7 @@ int populate_mission_ini(struct Delivery **ctx, int render_mode) {
     int err = 0;
 
     if ((*ctx)->_stasis_ini_fp.mission) {
+        // mission configurations are optional
         return 0;
     }
 
@@ -333,12 +375,12 @@ int populate_mission_ini(struct Delivery **ctx, int render_mode) {
                 globals.sysconfdir, "mission", (*ctx)->meta.mission, (*ctx)->meta.mission);
     }
 
-    msg(STASIS_MSG_L2, "Reading mission configuration: %s\n", missionfile);
+    SYSDEBUG("Reading mission configuration: %s\n", missionfile);
     (*ctx)->_stasis_ini_fp.mission = ini_open(missionfile);
     struct INIFILE *ini = (*ctx)->_stasis_ini_fp.mission;
     if (!ini) {
         msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read mission configuration: %s, %s\n", missionfile, strerror(errno));
-        exit(1);
+        return -1;
     }
     (*ctx)->_stasis_ini_fp.mission_path = strdup(missionfile);