diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/conda.c | 62 | ||||
| -rw-r--r-- | src/delivery.c | 77 | ||||
| -rw-r--r-- | src/globals.c | 1 | ||||
| -rw-r--r-- | src/stasis_indexer.c | 80 | ||||
| -rw-r--r-- | src/stasis_main.c | 2 | ||||
| -rw-r--r-- | src/utils.c | 7 |
6 files changed, 160 insertions, 69 deletions
diff --git a/src/conda.c b/src/conda.c index 342b6af..1e8b03f 100644 --- a/src/conda.c +++ b/src/conda.c @@ -5,6 +5,61 @@ #include <unistd.h> #include "conda.h" +int micromamba(struct MicromambaInfo *info, char *command, ...) { + struct utsname sys; + uname(&sys); + + tolower_s(sys.sysname); + if (!strcmp(sys.sysname, "darwin")) { + strcpy(sys.sysname, "osx"); + } + + if (!strcmp(sys.machine, "x86_64")) { + strcpy(sys.machine, "64"); + } + + char url[PATH_MAX]; + sprintf(url, "https://micro.mamba.pm/api/micromamba/%s-%s/latest", sys.sysname, sys.machine); + + char installer_path[PATH_MAX]; + sprintf(installer_path, "%s/latest", getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"); + + if (access(installer_path, F_OK)) { + download(url, installer_path, NULL); + } + + char mmbin[PATH_MAX]; + sprintf(mmbin, "%s/micromamba", info->micromamba_prefix); + + if (access(mmbin, F_OK)) { + char untarcmd[PATH_MAX]; + mkdirs(info->micromamba_prefix, 0755); + sprintf(untarcmd, "tar -xvf %s -C %s --strip-components=1 bin/micromamba 1>/dev/null", installer_path, info->micromamba_prefix); + system(untarcmd); + } + + char cmd[STASIS_BUFSIZ]; + memset(cmd, 0, sizeof(cmd)); + sprintf(cmd, "%s -r %s -p %s ", mmbin, info->conda_prefix, info->conda_prefix); + va_list args; + va_start(args, command); + vsprintf(cmd + strlen(cmd), command, args); + va_end(args); + + mkdirs(info->conda_prefix, 0755); + + char rcpath[PATH_MAX]; + sprintf(rcpath, "%s/.condarc", info->conda_prefix); + touch(rcpath); + + setenv("CONDARC", rcpath, 1); + setenv("MAMBA_ROOT_PREFIX", info->conda_prefix, 1); + int status = system(cmd); + unsetenv("MAMBA_ROOT_PREFIX"); + + return status; +} + int python_exec(const char *args) { char command[PATH_MAX]; memset(command, 0, sizeof(command)); @@ -114,12 +169,12 @@ int conda_activate(const char *root, const char *env_name) { perror(logfile); return -1; } - int i = 0; + while (!feof(fp)) { char buf[STASIS_BUFSIZ] = {0}; int ch = 0; size_t z = 0; - // We are ingesting output from "env -0", can't use fgets() + // We are ingesting output from "env -0" and can't use fgets() // Copy each character into the buffer until we encounter '\0' or EOF while (z < sizeof(buf) && (ch = (int) fgetc(fp)) != 0) { if (ch == EOF) { @@ -141,15 +196,12 @@ int conda_activate(const char *root, const char *env_name) { } if (!part[0]) { msg(STASIS_MSG_WARN | STASIS_MSG_L1, "Invalid environment variable key ignored: '%s'\n", buf); - i++; } else if (!part[1]) { msg(STASIS_MSG_WARN | STASIS_MSG_L1, "Invalid environment variable value ignored: '%s'\n", buf); - i++; } else { setenv(part[0], part[1], 1); } GENERIC_ARRAY_FREE(part); - i++; } fclose(fp); remove(logfile); diff --git a/src/delivery.c b/src/delivery.c index e77d41c..278647c 100644 --- a/src/delivery.c +++ b/src/delivery.c @@ -162,6 +162,9 @@ void delivery_free(struct Delivery *ctx) { guard_free(ctx->storage.build_docker_dir); guard_free(ctx->storage.mission_dir); guard_free(ctx->storage.docker_artifact_dir); + guard_free(ctx->storage.meta_dir); + guard_free(ctx->storage.package_dir); + guard_free(ctx->storage.cfgdump_dir); guard_free(ctx->info.time_str_epoch); guard_free(ctx->info.build_name); guard_free(ctx->info.build_number); @@ -186,6 +189,7 @@ void delivery_free(struct Delivery *ctx) { guard_free(ctx->tests[i].repository); guard_free(ctx->tests[i].repository_info_ref); guard_free(ctx->tests[i].repository_info_tag); + guard_strlist_free(&ctx->tests[i].repository_remove_tags); guard_free(ctx->tests[i].script); guard_free(ctx->tests[i].build_recipe); // test-specific runtime variables @@ -349,7 +353,17 @@ int delivery_init_platform(struct Delivery *ctx) { tolower_s(ctx->system.platform[DELIVERY_PLATFORM_RELEASE]); } + long cpu_count = get_cpu_count(); + if (!cpu_count) { + fprintf(stderr, "Unable to determine CPU count. Falling back to 1.\n"); + cpu_count = 1; + } + char ncpus[100] = {0}; + sprintf(ncpus, "%ld", cpu_count); + // Declare some important bits as environment variables + setenv("CPU_COUNT", ncpus, 1); + setenv("STASIS_CPU_COUNT", ncpus, 1); setenv("STASIS_ARCH", ctx->system.arch, 1); setenv("STASIS_PLATFORM", ctx->system.platform[DELIVERY_PLATFORM], 1); setenv("STASIS_CONDA_ARCH", ctx->system.arch, 1); @@ -1031,7 +1045,30 @@ int delivery_build_recipes(struct Delivery *ctx) { } char command[PATH_MAX]; - sprintf(command, "mambabuild --python=%s .", ctx->meta.python); + if (RECIPE_TYPE_CONDA_FORGE == recipe_type) { + char arch[STASIS_NAME_MAX] = {0}; + char platform[STASIS_NAME_MAX] = {0}; + + strcpy(platform, ctx->system.platform[DELIVERY_PLATFORM]); + if (strstr(platform, "Darwin")) { + memset(platform, 0, sizeof(platform)); + strcpy(platform, "osx"); + } + tolower_s(platform); + if (strstr(ctx->system.arch, "arm64")) { + strcpy(arch, "arm64"); + } else if (strstr(ctx->system.arch, "64")) { + strcpy(arch, "64"); + } else { + strcat(arch, "32"); // blind guess + } + tolower_s(arch); + + sprintf(command, "mambabuild --python=%s -m ../.ci_support/%s_%s_.yaml .", + ctx->meta.python, platform, arch); + } else { + sprintf(command, "mambabuild --python=%s .", ctx->meta.python); + } status = conda_exec(command); if (status) { return -1; @@ -1650,6 +1687,13 @@ void delivery_tests_run(struct Delivery *ctx) { struct Process proc; memset(&proc, 0, sizeof(proc)); + if (!globals.workaround.conda_reactivate) { + globals.workaround.conda_reactivate = calloc(PATH_MAX, sizeof(*globals.workaround.conda_reactivate)); + } else { + memset(globals.workaround.conda_reactivate, 0, PATH_MAX); + } + snprintf(globals.workaround.conda_reactivate, PATH_MAX - 1, "\nset +x\neval `conda shell.posix reactivate`\nset -x\n"); + if (!ctx->tests[0].name) { msg(STASIS_MSG_WARN | STASIS_MSG_L2, "no tests are defined!\n"); } else { @@ -1691,7 +1735,7 @@ void delivery_tests_run(struct Delivery *ctx) { } else { #if 1 int status; - char cmd[PATH_MAX]; + char *cmd = calloc(strlen(ctx->tests[i].script) + STASIS_BUFSIZ, sizeof(*cmd)); msg(STASIS_MSG_L3, "Testing %s\n", ctx->tests[i].name); memset(&proc, 0, sizeof(proc)); @@ -1711,22 +1755,45 @@ void delivery_tests_run(struct Delivery *ctx) { } // enable trace mode before executing each test script - memset(cmd, 0, sizeof(cmd)); - sprintf(cmd, "set -x ; %s", ctx->tests[i].script); + strcpy(cmd, ctx->tests[i].script); char *cmd_rendered = tpl_render(cmd); if (cmd_rendered) { if (strcmp(cmd_rendered, cmd) != 0) { strcpy(cmd, cmd_rendered); + cmd[strlen(cmd_rendered) ? strlen(cmd_rendered) - 1 : 0] = 0; } guard_free(cmd_rendered); } - status = shell(&proc, cmd); + FILE *runner_fp; + char *runner_filename = xmkstemp(&runner_fp, "w"); + + fprintf(runner_fp, "#!/bin/bash\n" + "eval `conda shell.posix reactivate`\n" + "set -x\n" + "%s\n", + cmd); + fclose(runner_fp); + chmod(runner_filename, 0755); + + puts(cmd); + char runner_cmd[PATH_MAX] = {0}; + sprintf(runner_cmd, "%s", runner_filename); + status = shell(&proc, runner_cmd); if (status) { msg(STASIS_MSG_ERROR, "Script failure: %s\n%s\n\nExit code: %d\n", ctx->tests[i].name, ctx->tests[i].script, status); + remove(runner_filename); + popd(); + guard_free(cmd); + tpl_free(); + delivery_free(ctx); + globals_free(); COE_CHECK_ABORT(1, "Test failure"); } + guard_free(cmd); + remove(runner_filename); + guard_free(runner_filename); if (toxconf) { remove(toxconf); diff --git a/src/globals.c b/src/globals.c index 7ed7c3c..297598f 100644 --- a/src/globals.c +++ b/src/globals.c @@ -54,4 +54,5 @@ void globals_free() { guard_free(globals.jfrog.jfrog_artifactory_product); guard_free(globals.jfrog.remote_filename); guard_free(globals.workaround.tox_posargs); + guard_free(globals.workaround.conda_reactivate); } diff --git a/src/stasis_indexer.c b/src/stasis_indexer.c index fb231e0..a7b0fce 100644 --- a/src/stasis_indexer.c +++ b/src/stasis_indexer.c @@ -200,57 +200,6 @@ struct Delivery **get_latest_deliveries(struct Delivery ctx[], size_t nelem) { return result; } -int micromamba(const char *write_to, const char *prefix, char *command, ...) { - struct utsname sys; - uname(&sys); - - tolower_s(sys.sysname); - if (!strcmp(sys.sysname, "darwin")) { - strcpy(sys.sysname, "osx"); - } - - if (!strcmp(sys.machine, "x86_64")) { - strcpy(sys.machine, "64"); - } - - char url[PATH_MAX]; - sprintf(url, "https://micro.mamba.pm/api/micromamba/%s-%s/latest", sys.sysname, sys.machine); - if (access("latest", F_OK)) { - download(url, "latest", NULL); - } - - char mmbin[PATH_MAX]; - sprintf(mmbin, "%s/micromamba", write_to); - - if (access(mmbin, F_OK)) { - char untarcmd[PATH_MAX]; - mkdirs(write_to, 0755); - sprintf(untarcmd, "tar -xvf latest -C %s --strip-components=1 bin/micromamba 1>/dev/null", write_to); - system(untarcmd); - } - - char cmd[STASIS_BUFSIZ]; - memset(cmd, 0, sizeof(cmd)); - sprintf(cmd, "%s -r %s -p %s ", mmbin, prefix, prefix); - va_list args; - va_start(args, command); - vsprintf(cmd + strlen(cmd), command, args); - va_end(args); - - mkdirs(prefix, 0755); - - char rcpath[PATH_MAX]; - sprintf(rcpath, "%s/.condarc", prefix); - touch(rcpath); - - setenv("CONDARC", rcpath, 1); - setenv("MAMBA_ROOT_PREFIX", prefix, 1); - int status = system(cmd); - unsetenv("MAMBA_ROOT_PREFIX"); - - return status; -} - int indexer_make_website(struct Delivery *ctx) { char cmd[PATH_MAX]; char *inputs[] = { @@ -259,6 +208,7 @@ int indexer_make_website(struct Delivery *ctx) { }; if (!find_program("pandoc")) { + fprintf(stderr, "pandoc is not installed: unable to generate HTML indexes\n"); return 0; } @@ -291,16 +241,17 @@ int indexer_make_website(struct Delivery *ctx) { int indexer_conda(struct Delivery *ctx) { int status = 0; - char prefix[PATH_MAX]; - sprintf(prefix, "%s/%s", ctx->storage.tmpdir, "indexer"); + char micromamba_prefix[PATH_MAX] = {0}; + sprintf(micromamba_prefix, "%s/bin", ctx->storage.tools_dir); + struct MicromambaInfo m = {.conda_prefix = globals.conda_install_prefix, .micromamba_prefix = micromamba_prefix}; - status += micromamba(ctx->storage.tmpdir, prefix, "config prepend --env channels conda-forge"); + status += micromamba(&m, "config prepend --env channels conda-forge"); if (!globals.verbose) { - status += micromamba(ctx->storage.tmpdir, prefix, "config set --env quiet true"); + status += micromamba(&m, "config set --env quiet true"); } - status += micromamba(ctx->storage.tmpdir, prefix, "config set --env always_yes true"); - status += micromamba(ctx->storage.tmpdir, prefix, "install conda-build"); - status += micromamba(ctx->storage.tmpdir, prefix, "run conda index %s", ctx->storage.conda_artifact_dir); + status += micromamba(&m, "config set --env always_yes true"); + status += micromamba(&m, "install conda-build"); + status += micromamba(&m, "run conda index %s", ctx->storage.conda_artifact_dir); return status; } @@ -558,6 +509,8 @@ void indexer_init_dirs(struct Delivery *ctx, const char *workdir) { exit(1); } path_store(&ctx->storage.output_dir, PATH_MAX, ctx->storage.root, "output"); + path_store(&ctx->storage.tools_dir, PATH_MAX, ctx->storage.output_dir, "tools"); + path_store(&globals.conda_install_prefix, PATH_MAX, ctx->storage.tools_dir, "conda"); path_store(&ctx->storage.cfgdump_dir, PATH_MAX, ctx->storage.output_dir, "config"); path_store(&ctx->storage.meta_dir, PATH_MAX, ctx->storage.output_dir, "meta"); path_store(&ctx->storage.delivery_dir, PATH_MAX, ctx->storage.output_dir, "delivery"); @@ -565,6 +518,15 @@ void indexer_init_dirs(struct Delivery *ctx, const char *workdir) { path_store(&ctx->storage.results_dir, PATH_MAX, ctx->storage.output_dir, "results"); path_store(&ctx->storage.wheel_artifact_dir, PATH_MAX, ctx->storage.package_dir, "wheels"); path_store(&ctx->storage.conda_artifact_dir, PATH_MAX, ctx->storage.package_dir, "conda"); + + char newpath[PATH_MAX] = {0}; + if (getenv("PATH")) { + sprintf(newpath, "%s/bin:%s", ctx->storage.tools_dir, getenv("PATH")); + setenv("PATH", newpath, 1); + } else { + SYSERROR("%s", "environment variable PATH is undefined. Unable to continue."); + exit(1); + } } int main(int argc, char *argv[]) { @@ -627,7 +589,7 @@ int main(int argc, char *argv[]) { } char *workdir; - char workdir_template[PATH_MAX]; + char workdir_template[PATH_MAX] = {0}; char *system_tmp = getenv("TMPDIR"); if (system_tmp) { strcat(workdir_template, system_tmp); diff --git a/src/stasis_main.c b/src/stasis_main.c index 8b13b98..c550982 100644 --- a/src/stasis_main.c +++ b/src/stasis_main.c @@ -248,6 +248,7 @@ int main(int argc, char *argv[]) { tpl_register("storage.build_sources_dir", &ctx.storage.build_sources_dir); tpl_register("storage.build_docker_dir", &ctx.storage.build_docker_dir); tpl_register("storage.results_dir", &ctx.storage.results_dir); + tpl_register("storage.tools_dir", &ctx.storage.tools_dir); tpl_register("conda.installer_baseurl", &ctx.conda.installer_baseurl); tpl_register("conda.installer_name", &ctx.conda.installer_name); tpl_register("conda.installer_version", &ctx.conda.installer_version); @@ -257,6 +258,7 @@ int main(int argc, char *argv[]) { tpl_register("deploy.jfrog.url", &globals.jfrog.url); tpl_register("deploy.docker.registry", &ctx.deploy.docker.registry); tpl_register("workaround.tox_posargs", &globals.workaround.tox_posargs); + tpl_register("workaround.conda_reactivate", &globals.workaround.conda_reactivate); // Set up PREFIX/etc directory information // The user may manipulate the base directory path with STASIS_SYSCONFDIR diff --git a/src/utils.c b/src/utils.c index fb22f59..2143c52 100644 --- a/src/utils.c +++ b/src/utils.c @@ -758,3 +758,10 @@ struct StrList *listdir(const char *path) { return node; } +long get_cpu_count() { +#if defined(STASIS_OS_LINUX) || defined(STASIS_OS_DARWIN) + return sysconf(_SC_NPROCESSORS_ONLN); +#else + return 0; +#endif +}
\ No newline at end of file |