diff options
41 files changed, 2871 insertions, 400 deletions
diff --git a/.github/workflows/cmake-multi-platform.yml b/.github/workflows/cmake-multi-platform.yml index b94f6a8..e3f0ce7 100644 --- a/.github/workflows/cmake-multi-platform.yml +++ b/.github/workflows/cmake-multi-platform.yml @@ -49,9 +49,16 @@ jobs: libcurl4-openssl-dev libxml2-dev libxml2-utils + libzip-dev pandoc rsync + - name: Install macOS dependencies + if: matrix.os == 'macos-latest' + run: > + brew install + libzip + - name: Configure CMake run: > cmake -B ${{ steps.strings.outputs.build-output-dir }} @@ -64,10 +71,12 @@ jobs: -S ${{ github.workspace }} - name: Build + env: + PKG_CONFIG_PATH: /opt/homebrew/lib/pkgconfig:/usr/lib/x86_64-linux-gnu/pkgconfig run: cmake --build ${{ steps.strings.outputs.build-output-dir }} --config ${{ matrix.build_type }} - name: Test working-directory: ${{ steps.strings.outputs.build-output-dir }} - run: ctest -V --build-config ${{ matrix.build_type }} --output-junit results.xml --test-output-size-passed 65536 --test-output-size-failed 65536 + run: ctest --build-config ${{ matrix.build_type }} --output-on-failure --output-junit results.xml --test-output-size-passed 65536 --test-output-size-failed 65536 env: STASIS_SYSCONFDIR: ../../.. diff --git a/CMakeLists.txt b/CMakeLists.txt index 074c2ee..bd214ca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,15 +8,24 @@ set(CMAKE_C_STANDARD 99) find_package(LibXml2) find_package(CURL) -option(ASAN OFF) +option(ASAN "Address Analyzer" OFF) +set(ASAN_OPTIONS "-fsanitize=address,leak,null,undefined") if (ASAN) - add_compile_options(-fsanitize=address) - add_link_options(-fsanitize=address) + add_compile_options(${ASAN_OPTIONS} -fno-omit-frame-pointer -g -O0) + add_link_options(${ASAN_OPTIONS}) endif() +pkg_check_modules(ZIP libzip) +if (NOT ZIP_FOUND) + message(FATAL_ERROR "libzip is required (https://libzip.org)") +endif () + +include_directories(${ZIP_INCLUDEDIR}) +link_directories(${ZIP_LIBRARY_DIRS}) +include_directories(${CURL_INCLUDE_DIR}) link_libraries(CURL::libcurl) -link_libraries(LibXml2::LibXml2) include_directories(${LIBXML2_INCLUDE_DIR}) +link_libraries(LibXml2::LibXml2) option(FORTIFY_SOURCE OFF) if (FORTIFY_SOURCE) @@ -8,6 +8,7 @@ STASIS consolidates the steps required to build, test, and deploy calibration pi - cmake - libcurl - libxml2 +- libzip - rsync # Installation @@ -147,30 +148,32 @@ stasis mydelivery.ini ## Command Line Options -| Long Option | Short Option | Purpose | -|:----------------------------|:------------:|:---------------------------------------------------------------| -| --help | -h | Display usage statement | -| --version | -V | Display program version | -| --continue-on-error | -C | Allow tests to fail | -| --config ARG | -c ARG | Read STASIS configuration file | -| --cpu-limit ARG | -l ARG | Number of processes to spawn concurrently (default: cpus - 1) | -| --pool-status-interval ARG | n/a | Report task status every n seconds (default: 30) | -| --python ARG | -p ARG | Override version of Python in configuration | -| --verbose | -v | Increase output verbosity | -| --unbuffered | -U | Disable line buffering | -| --update-base | n/a | Update conda installation prior to STATIS environment creation | -| --fail-fast | n/a | On test error, terminate all tasks | -| --task-timeout ARG | n/a | Terminate task after timeout is reached (#s, #m, #h) | -| --overwrite | n/a | Overwrite an existing release | -| --no-docker | n/a | Do not build docker images | -| --no-artifactory | n/a | Do not upload artifacts to Artifactory | -| --no-artifactory-build-info | n/a | Do not upload build info objects to Artifactory | -| --no-artifactory-upload | n/a | Do not upload artifacts to Artifactory (dry-run) | -| --no-testing | n/a | Do not execute test scripts | -| --no-parallel | n/a | Do not execute tests in parallel | -| --no-task-logging | n/a | Do not log task output (write to stdout) | -| --no-rewrite | n/a | Do not rewrite paths and URLs in output files | -| DELIVERY_FILE | n/a | STASIS delivery file | +| Long Option | Short Option | Purpose | +|:------------------------------------|:------------:|:---------------------------------------------------------------| +| --help | -h | Display usage statement | +| --version | -V | Display program version | +| --continue-on-error | -C | Allow tests to fail | +| --config ARG | -c ARG | Read STASIS configuration file | +| --cpu-limit ARG | -l ARG | Number of processes to spawn concurrently (default: cpus - 1) | +| --pool-status-interval ARG | n/a | Report task status every n seconds (default: 30) | +| --python ARG | -p ARG | Override version of Python in configuration | +| --verbose | -v | Increase output verbosity | +| --unbuffered | -U | Disable line buffering | +| --update-base | n/a | Update conda installation prior to STATIS environment creation | +| --fail-fast | n/a | On test error, terminate all tasks | +| --task-timeout ARG | n/a | Terminate task after timeout is reached (#s, #m, #h) | +| --overwrite | n/a | Overwrite an existing release | +| --wheel-builder ARG | n/a | Wheel building backend (build, cibuildwheel, manylinux) | +| --wheel-builder-manylinux-image ARG | n/a | Manylinux image name | +| --no-docker | n/a | Do not build docker images | +| --no-artifactory | n/a | Do not upload artifacts to Artifactory | +| --no-artifactory-build-info | n/a | Do not upload build info objects to Artifactory | +| --no-artifactory-upload | n/a | Do not upload artifacts to Artifactory (dry-run) | +| --no-testing | n/a | Do not execute test scripts | +| --no-parallel | n/a | Do not execute tests in parallel | +| --no-task-logging | n/a | Do not log task output (write to stdout) | +| --no-rewrite | n/a | Do not rewrite paths and URLs in output files | +| DELIVERY_FILE | n/a | STASIS delivery file | ## Indexer Command Line Options diff --git a/src/cli/stasis/args.c b/src/cli/stasis/args.c index 172981a..dbc9c2f 100644 --- a/src/cli/stasis/args.c +++ b/src/cli/stasis/args.c @@ -15,6 +15,8 @@ struct option long_options[] = { {"fail-fast", no_argument, 0, OPT_FAIL_FAST}, {"task-timeout", required_argument, 0, OPT_TASK_TIMEOUT}, {"overwrite", no_argument, 0, OPT_OVERWRITE}, + {"wheel-builder", required_argument, 0, OPT_WHEEL_BUILDER}, + {"wheel-builder-manylinux-image", required_argument, 0, OPT_WHEEL_BUILDER_MANYLINUX_IMAGE}, {"no-docker", no_argument, 0, OPT_NO_DOCKER}, {"no-artifactory", no_argument, 0, OPT_NO_ARTIFACTORY}, {"no-artifactory-build-info", no_argument, 0, OPT_NO_ARTIFACTORY_BUILD_INFO}, @@ -40,6 +42,8 @@ const char *long_options_help[] = { "On error, immediately terminate all tasks", "Terminate task after timeout is reached (#s, #m, #h)", "Overwrite an existing release", + "Wheel building backend (build, cibuildwheel, manylinux)", + "Manylinux image name", "Do not build docker images", "Do not upload artifacts to Artifactory", "Do not upload build info objects to Artifactory", diff --git a/src/cli/stasis/include/args.h b/src/cli/stasis/include/args.h index 5536735..e789261 100644 --- a/src/cli/stasis/include/args.h +++ b/src/cli/stasis/include/args.h @@ -19,6 +19,8 @@ #define OPT_POOL_STATUS_INTERVAL 1011 #define OPT_NO_TASK_LOGGING 1012 #define OPT_TASK_TIMEOUT 1013 +#define OPT_WHEEL_BUILDER 1014 +#define OPT_WHEEL_BUILDER_MANYLINUX_IMAGE 1015 extern struct option long_options[]; void usage(char *progname); diff --git a/src/cli/stasis/stasis_main.c b/src/cli/stasis/stasis_main.c index 633d014..44efc4a 100644 --- a/src/cli/stasis/stasis_main.c +++ b/src/cli/stasis/stasis_main.c @@ -54,15 +54,16 @@ static void configure_stasis_ini(struct Delivery *ctx, char **config_input) { } } - msg(STASIS_MSG_L2, "Reading STASIS global configuration: %s\n", *config_input); + SYSDEBUG("Reading STASIS global configuration: %s\n", *config_input); ctx->_stasis_ini_fp.cfg = ini_open(*config_input); if (!ctx->_stasis_ini_fp.cfg) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read config file: %s, %s\n", *config_input, strerror(errno)); + msg(STASIS_MSG_ERROR, "Failed to read global config file: %s, %s\n", *config_input, strerror(errno)); + SYSERROR("Failed to read global config file: %s\n", *config_input); exit(1); } ctx->_stasis_ini_fp.cfg_path = strdup(*config_input); if (!ctx->_stasis_ini_fp.cfg_path) { - SYSERROR("%s", "Failed to allocate memory for config file name"); + SYSERROR("%s", "Failed to allocate memory delivery context global config file name"); exit(1); } guard_free(*config_input); @@ -102,9 +103,9 @@ static void configure_jfrog_cli(struct Delivery *ctx) { static void check_release_history(struct Delivery *ctx) { // Safety gate: Avoid clobbering a delivered release unless the user wants that behavior - msg(STASIS_MSG_L1, "Checking release history\n"); + msg(STASIS_MSG_L2, "Checking release history\n"); if (!globals.enable_overwrite && delivery_exists(ctx) == DELIVERY_FOUND) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Refusing to overwrite release: %s\nUse --overwrite to enable release clobbering.\n", ctx->info.release_name); + msg(STASIS_MSG_ERROR, "Refusing to overwrite release: %s\nUse --overwrite to enable release clobbering.\n", ctx->info.release_name); exit(1); } @@ -147,14 +148,14 @@ static void check_conda_prefix_length(const struct Delivery *ctx) { // 5 = /bin\n const size_t prefix_len = strlen(ctx->storage.conda_install_prefix) + 2 + 5; const size_t prefix_len_max = 127; - msg(STASIS_MSG_L1, "Checking length of conda installation prefix\n"); + msg(STASIS_MSG_L2, "Checking length of conda installation prefix\n"); if (!strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Linux") && prefix_len > prefix_len_max) { - msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, + msg(STASIS_MSG_L3 | STASIS_MSG_ERROR, "The shebang, '#!%s/bin/python\\n' is too long (%zu > %zu).\n", ctx->storage.conda_install_prefix, prefix_len, prefix_len_max); - msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, + msg(STASIS_MSG_L3 | STASIS_MSG_ERROR, "Conda's workaround to handle long path names does not work consistently within STASIS.\n"); - msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, + msg(STASIS_MSG_L3 | STASIS_MSG_ERROR, "Please try again from a different, \"shorter\", directory.\n"); exit(1); } @@ -304,7 +305,8 @@ static void configure_tool_versions(struct Delivery *ctx) { } } -static void install_build_package() { +static void install_packaging_tools() { + msg(STASIS_MSG_L1, "Installing packaging tool(s)\n"); if (pip_exec("install build")) { msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "'build' tool installation failed\n"); exit(1); @@ -331,8 +333,7 @@ static void configure_deferred_packages(struct Delivery *ctx) { delivery_defer_packages(ctx, DEFER_PIP); } -static void show_overiew(struct Delivery *ctx) { - +static void show_overview(struct Delivery *ctx) { msg(STASIS_MSG_L1, "Overview\n"); delivery_meta_show(ctx); delivery_conda_show(ctx); @@ -512,9 +513,11 @@ int main(int argc, char *argv[]) { memset(&proc, 0, sizeof(proc)); memset(&ctx, 0, sizeof(ctx)); + setup_sysconfdir(); + int c; int option_index = 0; - while ((c = getopt_long(argc, argv, "hVCc:p:vU", long_options, &option_index)) != -1) { + while ((c = getopt_long(argc, argv, "hVCc:p:vUl:", long_options, &option_index)) != -1) { switch (c) { case 'h': usage(path_basename(argv[0])); @@ -605,6 +608,12 @@ int main(int argc, char *argv[]) { case OPT_NO_TASK_LOGGING: globals.enable_task_logging = false; break; + case OPT_WHEEL_BUILDER: + globals.wheel_builder = strdup(optarg); + break; + case OPT_WHEEL_BUILDER_MANYLINUX_IMAGE: + globals.wheel_builder_manylinux_image = strdup(optarg); + break; case '?': default: exit(1); @@ -627,21 +636,19 @@ int main(int argc, char *argv[]) { printf(BANNER, VERSION, AUTHOR); + setup_python_version_override(&ctx, python_override_version); + configure_stasis_ini(&ctx, &config_input); check_system_path(); + check_requirements(&ctx); msg(STASIS_MSG_L1, "Setup\n"); tpl_setup_vars(&ctx); tpl_setup_funcs(&ctx); - setup_sysconfdir(); - setup_python_version_override(&ctx, python_override_version); - - configure_stasis_ini(&ctx, &config_input); configure_delivery_ini(&ctx, &delivery_input); configure_delivery_context(&ctx); - check_requirements(&ctx); configure_jfrog_cli(&ctx); runtime_apply(ctx.runtime.environ); @@ -665,11 +672,11 @@ int main(int argc, char *argv[]) { setup_activate_test_env(&ctx, env_name_testing); configure_tool_versions(&ctx); - install_build_package(); + install_packaging_tools(); configure_package_overlay(&ctx, env_name); configure_deferred_packages(&ctx); - show_overiew(&ctx); + show_overview(&ctx); run_tests(&ctx); build_conda_recipes(&ctx); build_wheel_packages(&ctx); diff --git a/src/cli/stasis/system_requirements.c b/src/cli/stasis/system_requirements.c index cb0ebd5..0f0aae8 100644 --- a/src/cli/stasis/system_requirements.c +++ b/src/cli/stasis/system_requirements.c @@ -27,36 +27,46 @@ void check_system_requirements(struct Delivery *ctx) { }; msg(STASIS_MSG_L1, "Checking system requirements\n"); + + msg(STASIS_MSG_L2, "Tools\n"); for (size_t i = 0; tools_required[i] != NULL; i++) { + msg(STASIS_MSG_L3, "%s: ", tools_required[i]); if (!find_program(tools_required[i])) { - msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "'%s' must be installed.\n", tools_required[i]); + msg(STASIS_MSG_ERROR, "'%s' must be installed.\n", tools_required[i]); exit(1); } + msg(STASIS_MSG_RESTRICT, "found\n"); } if (!globals.tmpdir && !ctx->storage.tmpdir) { delivery_init_tmpdir(ctx); } - if (!docker_capable(&ctx->deploy.docker.capabilities)) { + msg(STASIS_MSG_L2, "Docker\n"); + if (docker_capable(&ctx->deploy.docker.capabilities)) { struct DockerCapabilities *dcap = &ctx->deploy.docker.capabilities; - msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "Docker is broken\n"); - msg(STASIS_MSG_L3, "Available: %s\n", dcap->available ? "Yes" : "No"); - msg(STASIS_MSG_L3, "Usable: %s\n", dcap->usable ? "Yes" : "No"); + msg(STASIS_MSG_L3, "Available: %s%s%s\n", dcap->available ? STASIS_COLOR_GREEN : STASIS_COLOR_RED, dcap->available ? "Yes" : "No", STASIS_COLOR_RESET); + msg(STASIS_MSG_L3, "Usable: %s%s%s\n", dcap->usable ? STASIS_COLOR_GREEN : STASIS_COLOR_RED, dcap->usable ? "Yes" : "No", STASIS_COLOR_RESET); msg(STASIS_MSG_L3, "Podman [Docker Emulation]: %s\n", dcap->podman ? "Yes" : "No"); msg(STASIS_MSG_L3, "Build plugin(s): "); - if (dcap->usable) { + if (dcap->build) { if (dcap->build & STASIS_DOCKER_BUILD) { - printf("build "); + msg(STASIS_MSG_RESTRICT, "build "); } if (dcap->build & STASIS_DOCKER_BUILD_X) { - printf("buildx "); + msg(STASIS_MSG_RESTRICT, "buildx "); } - puts(""); + msg(STASIS_MSG_RESTRICT,"\n"); } else { - printf("N/A\n"); + msg(STASIS_MSG_RESTRICT, "%sN/A%s\n", STASIS_COLOR_YELLOW, STASIS_COLOR_RESET); } + if (!dcap->usable) { + // disable docker builds + globals.enable_docker = false; + } + } else { + msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "Docker is broken\n"); // disable docker builds globals.enable_docker = false; } diff --git a/src/lib/core/CMakeLists.txt b/src/lib/core/CMakeLists.txt index eb7a908..0fb273c 100644 --- a/src/lib/core/CMakeLists.txt +++ b/src/lib/core/CMakeLists.txt @@ -11,6 +11,7 @@ add_library(stasis_core STATIC download.c recipe.c relocation.c + wheelinfo.c wheel.c copy.c artifactory.c @@ -27,5 +28,8 @@ add_library(stasis_core STATIC target_include_directories(stasis_core PRIVATE ${core_INCLUDE} ${delivery_INCLUDE} + ${ZIP_INCLUDEDIR} ${CMAKE_CURRENT_SOURCE_DIR}/include ) +target_link_libraries(stasis_core PRIVATE + ${ZIP_LIBRARIES}) diff --git a/src/lib/core/docker.c b/src/lib/core/docker.c index 4723446..39357ad 100644 --- a/src/lib/core/docker.c +++ b/src/lib/core/docker.c @@ -1,17 +1,30 @@ #include "docker.h" -int docker_exec(const char *args, unsigned flags) { +int docker_exec(const char *args, const unsigned flags) { struct Process proc; char cmd[PATH_MAX]; memset(&proc, 0, sizeof(proc)); memset(cmd, 0, sizeof(cmd)); snprintf(cmd, sizeof(cmd) - 1, "docker %s", args); + + unsigned final_flags = 0; if (flags & STASIS_DOCKER_QUIET) { + final_flags |= STASIS_DOCKER_QUIET_STDOUT; + final_flags |= STASIS_DOCKER_QUIET_STDERR; + } else { + final_flags = flags; + } + + if (final_flags & STASIS_DOCKER_QUIET_STDOUT) { strcpy(proc.f_stdout, "/dev/null"); + } + if (final_flags & STASIS_DOCKER_QUIET_STDERR) { strcpy(proc.f_stderr, "/dev/null"); - } else { + } + + if (!final_flags) { msg(STASIS_MSG_L2, "Executing: %s\n", cmd); } @@ -19,11 +32,11 @@ int docker_exec(const char *args, unsigned flags) { return proc.returncode; } -int docker_script(const char *image, char *data, unsigned flags) { +int docker_script(const char *image, char *args, char *data, const unsigned flags) { (void)flags; // TODO: placeholder char cmd[PATH_MAX] = {0}; - snprintf(cmd, sizeof(cmd) - 1, "docker run --rm -i %s /bin/sh -", image); + snprintf(cmd, sizeof(cmd) - 1, "docker run -i %s \"%s\" /bin/sh -", args ? args : "", image); FILE *outfile = popen(cmd, "w"); if (!outfile) { diff --git a/src/lib/core/envctl.c b/src/lib/core/envctl.c index b036611..d8d1b3d 100644 --- a/src/lib/core/envctl.c +++ b/src/lib/core/envctl.c @@ -92,23 +92,28 @@ unsigned envctl_get_flags(const struct EnvCtl *envctl, const char *name) { } void envctl_do_required(const struct EnvCtl *envctl, int verbose) { + int failed = 0; for (size_t i = 0; i < envctl->num_used; i++) { - struct EnvCtl_Item *item = envctl->item[i]; + const struct EnvCtl_Item *item = envctl->item[i]; const char *name = item->name; envctl_except_fn *callback = item->callback; if (verbose) { - msg(STASIS_MSG_L2, "Verifying %s\n", name); + msg(STASIS_MSG_L2, "Verifying %s [%s]\n", name, item->flags & STASIS_ENVCTL_REQUIRED ? "required" : "optional"); } - int code = callback((const void *) item, (const void *) name); + const int code = callback((const void *) item, (const void *) name); if (code == STASIS_ENVCTL_RET_IGNORE || code == STASIS_ENVCTL_RET_SUCCESS) { continue; } if (code == STASIS_ENVCTL_RET_FAIL) { - fprintf(stderr, "\n%s must be set. Exiting.\n", name); - exit(1); + msg(STASIS_MSG_ERROR, "\n%s%s must be defined.\n", name, STASIS_COLOR_RESET); + failed++; } - fprintf(stderr, "\nan unknown envctl callback code occurred: %d\n", code); + msg(STASIS_MSG_ERROR, "\nan unknown envctl callback code occurred: %d\n", code); + } + + if (failed) { + msg(STASIS_MSG_ERROR, "Environment check failed with %d error(s)\n", failed); exit(1); } } diff --git a/src/lib/core/globals.c b/src/lib/core/globals.c index 834213b..63555a2 100644 --- a/src/lib/core/globals.c +++ b/src/lib/core/globals.c @@ -53,6 +53,8 @@ void globals_free() { guard_free(globals.conda_install_prefix); guard_strlist_free(&globals.conda_packages); guard_strlist_free(&globals.pip_packages); + guard_free(globals.wheel_builder); + guard_free(globals.wheel_builder_manylinux_image); guard_free(globals.jfrog.arch); guard_free(globals.jfrog.os); guard_free(globals.jfrog.url); diff --git a/src/lib/core/include/core.h b/src/lib/core/include/core.h index 5a3fa85..c895267 100644 --- a/src/lib/core/include/core.h +++ b/src/lib/core/include/core.h @@ -51,7 +51,9 @@ struct STASIS_GLOBAL { char *tmpdir; //!< Path to temporary storage directory char *conda_install_prefix; //!< Path to install conda char *sysconfdir; //!< Path where STASIS reads its configuration files (mission directory, etc) - int task_timeout; ///< Time in seconds before task is terminated + int task_timeout; ///!< Time in seconds before task is terminated + char *wheel_builder; ///!< Backend to build wheels (build, cibuildwheel, manylinux) + char *wheel_builder_manylinux_image; ///!< Image to use for a Manylinux build struct { char *tox_posargs; char *conda_reactivate; diff --git a/src/lib/core/include/core_mem.h b/src/lib/core/include/core_mem.h index dd79e72..b67130c 100644 --- a/src/lib/core/include/core_mem.h +++ b/src/lib/core/include/core_mem.h @@ -22,4 +22,11 @@ guard_free(ARR); \ } while (0) +#define guard_array_n_free(ARR, LEN) do { \ + for (size_t ARR_I = 0; ARR && ARR_I < LEN ; ARR_I++) { \ + guard_free(ARR[ARR_I]); \ + } \ + guard_free(ARR); \ +} while (0) + #endif //STASIS_CORE_MEM_H diff --git a/src/lib/core/include/docker.h b/src/lib/core/include/docker.h index 7585d86..dd67f21 100644 --- a/src/lib/core/include/docker.h +++ b/src/lib/core/include/docker.h @@ -6,6 +6,8 @@ //! Flag to squelch output from docker_exec() #define STASIS_DOCKER_QUIET 1 << 1 +#define STASIS_DOCKER_QUIET_STDOUT 1 << 2 +#define STASIS_DOCKER_QUIET_STDERR 1 << 3 //! Flag for older style docker build #define STASIS_DOCKER_BUILD 1 << 1 @@ -83,7 +85,7 @@ int docker_exec(const char *args, unsigned flags); * @return */ int docker_build(const char *dirpath, const char *args, int engine); -int docker_script(const char *image, char *data, unsigned flags); +int docker_script(const char *image, char *args, char *data, unsigned flags); int docker_save(const char *image, const char *destdir, const char *compression_program); void docker_sanitize_tag(char *str); int docker_validate_compression_program(char *prog); diff --git a/src/lib/core/include/strlist.h b/src/lib/core/include/strlist.h index 18c60eb..b2d7da7 100644 --- a/src/lib/core/include/strlist.h +++ b/src/lib/core/include/strlist.h @@ -46,6 +46,8 @@ void strlist_append_strlist(struct StrList *pStrList1, struct StrList *pStrList2 void strlist_append(struct StrList **pStrList, char *str); void strlist_append_array(struct StrList *pStrList, char **arr); void strlist_append_tokenize(struct StrList *pStrList, char *str, char *delim); +void strlist_append_tokenize_raw(struct StrList *pStrList, char *str, char *delim); +int strlist_appendf(struct StrList **pStrList, const char *fmt, ...); struct StrList *strlist_copy(struct StrList *pStrList); int strlist_cmp(struct StrList *a, struct StrList *b); void strlist_free(struct StrList **pStrList); diff --git a/src/lib/core/include/utils.h b/src/lib/core/include/utils.h index ea98faf..335a7e4 100644 --- a/src/lib/core/include/utils.h +++ b/src/lib/core/include/utils.h @@ -27,6 +27,15 @@ #define LINE_SEP "\n" #endif +#if defined(STASIS_OS_LINUX) +#define STASIS_RANDOM_GENERATOR_FILE "/dev/urandom" +#elif defined(STASIS_OS_DARWIN) +#define STASIS_RANDOM_GENERATOR_FILE "/dev/random" +#else +#define STASIS_RANDOM_GENERATOR_FILE NULL +#define NEED_SRAND 1 +#endif + #define STASIS_XML_PRETTY_PRINT_PROG "xmllint" #define STASIS_XML_PRETTY_PRINT_ARGS "--format" @@ -470,4 +479,7 @@ void seconds_to_human_readable(int v, char *result, size_t maxlen); #define STR_TO_TIMEOUT_INVALID_TIME_SCALE (-2) int str_to_timeout(char *s); +const char *get_random_generator_file(); +int get_random_bytes(char *result, size_t maxlen); + #endif //STASIS_UTILS_H diff --git a/src/lib/core/include/wheel.h b/src/lib/core/include/wheel.h index 1a689e9..765f2c3 100644 --- a/src/lib/core/include/wheel.h +++ b/src/lib/core/include/wheel.h @@ -1,36 +1,257 @@ -//! @file wheel.h -#ifndef STASIS_WHEEL_H -#define STASIS_WHEEL_H +#ifndef WHEEL_H +#define WHEEL_H -#include <dirent.h> -#include <string.h> #include <stdio.h> -#include "str.h" -#define WHEEL_MATCH_EXACT 0 ///< Match when all patterns are present -#define WHEEL_MATCH_ANY 1 ///< Match when any patterns are present +#include <stdlib.h> +#include <string.h> +#include <fnmatch.h> +#include <zip.h> + +#define WHEEL_MAXELEM 255 + +#define WHEEL_FROM_DIST 0 +#define WHEEL_FROM_METADATA 1 + +enum { + WHEEL_META_METADATA_VERSION=0, + WHEEL_META_NAME, + WHEEL_META_VERSION, + WHEEL_META_AUTHOR, + WHEEL_META_AUTHOR_EMAIL, + WHEEL_META_MAINTAINER, + WHEEL_META_MAINTAINER_EMAIL, + WHEEL_META_SUMMARY, + WHEEL_META_LICENSE, + WHEEL_META_LICENSE_EXPRESSION, + WHEEL_META_LICENSE_FILE, + WHEEL_META_HOME_PAGE, + WHEEL_META_DOWNLOAD_URL, + WHEEL_META_PROJECT_URL, + WHEEL_META_CLASSIFIER, + WHEEL_META_REQUIRES_PYTHON, + WHEEL_META_REQUIRES_EXTERNAL, + WHEEL_META_IMPORT_NAME, + WHEEL_META_IMPORT_NAMESPACE, + WHEEL_META_REQUIRES_DIST, + WHEEL_META_PROVIDES, + WHEEL_META_PROVIDES_DIST, + WHEEL_META_PROVIDES_EXTRA, + WHEEL_META_OBSOLETES, + WHEEL_META_OBSOLETES_DIST, + WHEEL_META_PLATFORM, + WHEEL_META_SUPPORTED_PLATFORM, + WHEEL_META_KEYWORDS, + WHEEL_META_DYNAMIC, + WHEEL_META_DESCRIPTION_CONTENT_TYPE, + WHEEL_META_DESCRIPTION, + WHEEL_META_END_ENUM, // NOP +}; + + +enum { + WHEEL_DIST_VERSION=0, + WHEEL_DIST_GENERATOR, + WHEEL_DIST_ROOT_IS_PURELIB, + WHEEL_DIST_TAG, + WHEEL_DIST_ZIP_SAFE, + WHEEL_DIST_TOP_LEVEL, + WHEEL_DIST_ENTRY_POINT, + WHEEL_DIST_RECORD, + WHEEL_DIST_END_ENUM, // NOP +}; + +struct WheelMetadata_ProvidesExtra { + char *target; + struct StrList *requires_dist; + int count; +}; + +struct WheelMetadata { + char *metadata_version; + char *name; + char *version; + char *summary; + struct StrList *author; + struct StrList *author_email; + struct StrList *maintainer; + struct StrList *maintainer_email; + char *license; + char *license_expression; + char *home_page; + char * download_url; + struct StrList *project_url; + struct StrList *classifier; + struct StrList *requires_python; + struct StrList *requires_external; + char *description_content_type; + struct StrList *license_file; + struct StrList *import_name; + struct StrList *import_namespace; + struct StrList *requires_dist; + struct StrList *provides; + struct StrList *provides_dist; + struct StrList *obsoletes; + struct StrList *obsoletes_dist; + char *description; + struct StrList *platform; + struct StrList *supported_platform; + struct StrList *keywords; + struct StrList *dynamic; + + struct WheelMetadata_ProvidesExtra **provides_extra; +}; + +struct WheelRecord { + char *filename; + char *checksum; + size_t size; +}; + +struct WheelEntryPoint { + char *name; + char *function; + char *type; +}; + +/* +Wheel-Version: 1.0 +Generator: setuptools (75.8.0) +Root-Is-Purelib: false +Tag: cp313-cp313-manylinux_2_17_x86_64 +Tag: cp313-cp313-manylinux2014_x86_64 +*/ struct Wheel { - char *distribution; ///< Package name - char *version; ///< Package version - char *build_tag; ///< Package build tag (optional) - char *python_tag; ///< Package Python tag (pyXY) - char *abi_tag; ///< Package ABI tag (cpXY, abiX, none) - char *platform_tag; ///< Package platform tag (linux_x86_64, any) - char *path_name; ///< Path to package on-disk - char *file_name; ///< Name of package on-disk + char *wheel_version; + char *generator; + char *root_is_pure_lib; + struct StrList *tag; + struct StrList *top_level; + int zip_safe; + struct WheelMetadata *metadata; + struct WheelRecord **record; + size_t num_record; + struct WheelEntryPoint **entry_point; + size_t num_entry_point; +}; + +#define METADATA_MULTILINE_PREFIX " " + + +static inline int consume_append(char **dest, const char *src, const char *accept) { + const char *start = src; + if (!strncmp(src, METADATA_MULTILINE_PREFIX, strlen(METADATA_MULTILINE_PREFIX))) { + start += strlen(METADATA_MULTILINE_PREFIX); + } + + const char *end = strpbrk(start, accept); + size_t cut_len = end ? (size_t)(end - start) : strlen(start); + size_t dest_len = strlen(*dest); + + char *tmp = realloc(*dest, strlen(*dest) + cut_len + 2); + if (!tmp) { + return -1; + } + *dest = tmp; + memcpy(*dest + dest_len, start, cut_len); + + dest_len += cut_len; + (*dest)[dest_len ? dest_len : 0] = '\n'; + (*dest)[dest_len + 1] = '\0'; + return 0; +} + +#define WHEEL_KEY_UNKNOWN (-1) +enum { + WHEELVAL_STR = 0, + WHEELVAL_STRLIST, + WHEELVAL_OBJ_EXTRA, + WHEELVAL_OBJ_RECORD, + WHEELVAL_OBJ_ENTRY_POINT, +}; + +struct WheelValue { + int type; + size_t count; + void *data; }; +enum { + WHEEL_PACKAGE_E_SUCCESS=0, + WHEEL_PACKAGE_E_FILENAME=-1, + WHEEL_PACKAGE_E_ALLOC=-2, + WHEEL_PACKAGE_E_GET=-3, + WHEEL_PACKAGE_E_GET_METADATA=-4, + WHEEL_PACKAGE_E_GET_TOP_LEVEL=-5, + WHEEL_PACKAGE_E_GET_RECORDS=-6, + WHEEL_PACKAGE_E_GET_ENTRY_POINT=-7, +}; + +/** + * Populate a `Wheel` structure using a Python wheel file as input. + * + * @param pkg pointer to a `Wheel` (may be initialized to `NULL`) + * @param filename path to a Python wheel file + * @return a WHEEL_PACKAGE_E_ error code + */ +int wheel_package(struct Wheel **pkg, const char *filename); + +/** + * Frees a `Wheel` structure + * @param pkg pointer to an initialized `Wheel` + */ +void wheel_package_free(struct Wheel **pkg); + + +/** + * Get wheel data by name + * @param pkg pointer to an initialized `Wheel` + * @param from `WHEEL_FROM_DIST`, `WHEEL_FROM_META` + * @param key name of key in DIST or META data + * @return a populated `WheelValue` (stack) + */ +struct WheelValue wheel_get_value_by_name(const struct Wheel *pkg, int from, const char *key); + + /** - * Extract metadata from a Python Wheel file name + * Get wheel data by internal identifier + * @param pkg pointer to an initialized `Wheel` + * @param from `WHEEL_FROM_DIST`, `WHEEL_FROM_META` + * @param id `WHEEL_META_VERSION`, `WHEEL_DIST_VERSION` (see wheel.h) + * @return a populated `WheelValue` (stack) + */ +struct WheelValue wheel_get_value_by_id(const struct Wheel *pkg, int from, ssize_t id); + +/** + * Returns the error code assocated with the `WheelValue`, if possible + * @param val a populated `WheelValue` + * @return error code (see wheel.h) + */ +int wheel_value_error(struct WheelValue const *val); + +/** + * Retreive the key name string for a given id + * @param from `WHEEL_FROM_DIST`, `WHEEL_FROM_META` + * @param id `WHEEL_META_VERSION`, `WHEEL_DIST_VERSION` (see wheel.h) + * @return the key name, or NULL + */ +const char *wheel_get_key_by_id(int from, ssize_t id); + +/** + * Get the contents of a file within a Python wheel + * @param wheelfile path to Python wheel file + * @param filename path to file inside of wheel file archive + * @param contents pointer to store file contents + * @return 0 on success, -1 on error + */ +int wheel_get_file_contents(const char *wheelfile, const char *filename, char **contents); + +/** + * Display the values of a `Wheel` structure in human readable format * - * @param basepath directory containing a wheel file - * @param name of wheel file - * @param to_match a NULL terminated array of patterns (i.e. platform, arch, version, etc) - * @param match_mode WHEEL_MATCH_EXACT - * @param match_mode WHEEL_MATCH ANY - * @return pointer to populated Wheel on success - * @return NULL on error - */ -struct Wheel *get_wheel_info(const char *basepath, const char *name, char *to_match[], unsigned match_mode); -void wheel_free(struct Wheel **wheel); -#endif //STASIS_WHEEL_H + * @param wheel + * @return 0 on success, -1 on error + */ +int wheel_show_info(const struct Wheel *wheel); + +#endif //WHEEL_H diff --git a/src/lib/core/include/wheelinfo.h b/src/lib/core/include/wheelinfo.h new file mode 100644 index 0000000..8009e91 --- /dev/null +++ b/src/lib/core/include/wheelinfo.h @@ -0,0 +1,36 @@ +//! @file wheel.h +#ifndef STASIS_WHEEL_H +#define STASIS_WHEEL_H + +#include <dirent.h> +#include <string.h> +#include <stdio.h> +#include "str.h" +#define WHEEL_MATCH_EXACT 0 ///< Match when all patterns are present +#define WHEEL_MATCH_ANY 1 ///< Match when any patterns are present + +struct WheelInfo { + char *distribution; ///< Package name + char *version; ///< Package version + char *build_tag; ///< Package build tag (optional) + char *python_tag; ///< Package Python tag (pyXY) + char *abi_tag; ///< Package ABI tag (cpXY, abiX, none) + char *platform_tag; ///< Package platform tag (linux_x86_64, any) + char *path_name; ///< Path to package on-disk + char *file_name; ///< Name of package on-disk +}; + +/** + * Extract metadata from a Python Wheel file name + * + * @param basepath directory containing a wheel file + * @param name of wheel file + * @param to_match a NULL terminated array of patterns (i.e. platform, arch, version, etc) + * @param match_mode WHEEL_MATCH_EXACT + * @param match_mode WHEEL_MATCH ANY + * @return pointer to populated Wheel on success + * @return NULL on error + */ +struct WheelInfo *wheelinfo_get(const char *basepath, const char *name, char *to_match[], unsigned match_mode); +void wheelinfo_free(struct WheelInfo **wheel); +#endif //STASIS_WHEEL_H diff --git a/src/lib/core/multiprocessing.c b/src/lib/core/multiprocessing.c index 298484a..7ae23c9 100644 --- a/src/lib/core/multiprocessing.c +++ b/src/lib/core/multiprocessing.c @@ -345,12 +345,12 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) { if (slot->pid == MP_POOL_PID_UNUSED) { // Child is already used up, skip it hang_check++; - SYSDEBUG("slot %zu: hang_check=%zu", i, hang_check); if (hang_check >= pool->num_used) { // If you join a pool that's already finished it will spin // forever. This protects the program from entering an // infinite loop. - fprintf(stderr, "%s is deadlocked\n", pool->ident); + SYSDEBUG("slot %zu: hang_check=%zu >= pool->num_used=%zu", i, hang_check, pool->num_used); + SYSERROR("%s is deadlocked\n", pool->ident); failures++; goto pool_deadlocked; } diff --git a/src/lib/core/strlist.c b/src/lib/core/strlist.c index a0db5f3..f3754c3 100644 --- a/src/lib/core/strlist.c +++ b/src/lib/core/strlist.c @@ -47,7 +47,6 @@ void strlist_append(struct StrList **pStrList, char *str) { (*pStrList)->data = tmp; (*pStrList)->data[(*pStrList)->num_inuse] = strdup(str); (*pStrList)->data[(*pStrList)->num_alloc] = NULL; - strcpy((*pStrList)->data[(*pStrList)->num_inuse], str); (*pStrList)->num_inuse++; (*pStrList)->num_alloc++; } @@ -231,6 +230,51 @@ void strlist_append_strlist(struct StrList *pStrList1, struct StrList *pStrList2 } /** + * Append the contents of a newline delimited string without + * modifying the input `str` + * @param pStrList `StrList` + * @param str + * @param delim + */ +void strlist_append_tokenize_raw(struct StrList *pStrList, char *str, char *delim) { + if (!str || !delim) { + return; + } + + char *tmp = strdup(str); + char **token = split(tmp, delim, 0); + if (token) { + for (size_t i = 0; token[i] != NULL; i++) { + strlist_append(&pStrList, token[i]); + } + guard_array_free(token); + } + guard_free(tmp); +} + +/** + * Append a formatted string + * Behavior is identical to asprintf-family of functions + * @param pStrList `StrList` + * @param fmt printf format string + * @param ... format arguments + * @return same as vasnprintf + */ +int strlist_appendf(struct StrList **pStrList, const char *fmt, ...) { + char *s = NULL; + va_list ap; + va_start(ap, fmt); + const int len = vasprintf(&s, fmt, ap); + va_end(ap); + + if (pStrList && *pStrList && len >= 0) { + strlist_append(pStrList, s); + } + guard_free(s); + return len; +} + +/** * Produce a new copy of a `StrList` * @param pStrList `StrList` * @return `StrList` copy diff --git a/src/lib/core/template_func_proto.c b/src/lib/core/template_func_proto.c index 8324389..52a11b5 100644 --- a/src/lib/core/template_func_proto.c +++ b/src/lib/core/template_func_proto.c @@ -28,9 +28,9 @@ int get_github_release_notes_auto_tplfunc_entrypoint(void *frame, void *data_out const struct Delivery *ctx = (struct Delivery *) f->data_in; struct StrList *notes_list = strlist_init(); - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(*ctx->tests); i++) { + for (size_t i = 0; i < ctx->tests->num_used; i++) { // Get test context - const struct Test *test = &ctx->tests[i]; + const struct Test *test = ctx->tests->test[i]; if (test->name && test->version && test->repository) { char *repository = strdup(test->repository); char *match = strstr(repository, "spacetelescope/"); @@ -55,8 +55,8 @@ int get_github_release_notes_auto_tplfunc_entrypoint(void *frame, void *data_out strlist_append(¬es_list, note); guard_free(note); } - guard_free(repository); } + guard_free(repository); } } // Return all notes as a single string diff --git a/src/lib/core/utils.c b/src/lib/core/utils.c index 00d747f..e106193 100644 --- a/src/lib/core/utils.c +++ b/src/lib/core/utils.c @@ -376,7 +376,8 @@ char *git_describe(const char *path) { return NULL; } - FILE *pp = popen("git describe --first-parent --always --tags", "r"); + // TODO: Use `-C [path]` if the version of git installed supports it + FILE *pp = popen("git describe --first-parent --long --always --tags", "r"); if (!pp) { return NULL; } @@ -401,6 +402,7 @@ char *git_rev_parse(const char *path, char *args) { return NULL; } + // TODO: Use `-C [path]` if the version of git installed supports it sprintf(cmd, "git rev-parse %s", args); FILE *pp = popen(cmd, "r"); if (!pp) { @@ -1120,3 +1122,53 @@ void seconds_to_human_readable(const int v, char *result, const size_t maxlen) { snprintf(result + strlen(result), maxlen, "%ds", seconds); } +const char *get_random_generator_file() { + return STASIS_RANDOM_GENERATOR_FILE; +} + +#ifdef NEED_SRAND +static char stasis_srand_initialized = 0; +#endif + +int get_random_bytes(char *result, size_t maxlen) { +#ifdef NEED_SRAND + if (!srand_initialized) { + srand(time(NULL)); + srand_initialized = 1; + } +#endif + size_t bytes = 0; + const char *filename = get_random_generator_file(); + FILE *fp = NULL; + if (filename != NULL) { + fp = fopen(filename, "rb"); + if (!fp) { + SYSERROR("%s", "unable to open random generator"); + return -1; + } + } + + do { + int ch = 0; + if (fp) { + ch = fgetc(fp); + } else { + ch = rand() % 255; + } + if (fp && ferror(fp)) { + SYSERROR("%s", "unable to read from random generator"); + return -1; + } + if (isalnum(ch)) { + result[bytes] = (char) ch; + bytes++; + } + } while (bytes < maxlen); + + if (fp) { + fclose(fp); + } + result[bytes ? bytes - 1 : 0] = '\0'; + return 0; +} + diff --git a/src/lib/core/wheel.c b/src/lib/core/wheel.c index c7e485a..78209f1 100644 --- a/src/lib/core/wheel.c +++ b/src/lib/core/wheel.c @@ -1,126 +1,1356 @@ #include "wheel.h" -struct Wheel *get_wheel_info(const char *basepath, const char *name, char *to_match[], unsigned match_mode) { - struct dirent *rec; - struct Wheel *result = NULL; - char package_path[PATH_MAX]; - char package_name[NAME_MAX]; +#include <ctype.h> - strcpy(package_name, name); - tolower_s(package_name); - sprintf(package_path, "%s/%s", basepath, package_name); +#include "str.h" +#include "strlist.h" - DIR *dp = opendir(package_path); - if (!dp) { - return NULL; +const char *WHEEL_META_KEY[] = { + "Metadata-Version", + "Name", + "Version", + "Author", + "Author-email", + "Maintainer", + "Maintainer-email", + "Summary", + "License", + "License-Expression", + "License-File", + "Home-page", + "Download-URL", + "Project-URL", + "Classifier", + "Requires-Python", + "Requires-External", + "Import-Name", + "Import-Namespace", + "Requires-Dist", + "Provides", + "Provides-Dist", + "Provides-Extra", + "Obsoletes", + "Obsoletes-Dist", + "Platform", + "Supported-Platform", + "Keywords", + "Dynamic", + "Description-Content-Type", + "Description", + NULL, +}; + +const char *WHEEL_DIST_KEY[] = { + "Wheel-Version", + "Generator", + "Root-Is-Purelib", + "Tag", + "Zip-Safe", + "Top-Level", + "Entry-points", + "Record", + NULL, +}; + +static ssize_t wheel_parse_wheel(struct Wheel * pkg, const char * data) { + int read_as = 0; + struct StrList *lines = strlist_init(); + if (!lines) { + return -1; } + strlist_append_tokenize(lines, (char *) data, "\r\n"); - while ((rec = readdir(dp)) != NULL) { - if (!strcmp(rec->d_name, ".") || !strcmp(rec->d_name, "..")) { + for (size_t i = 0; i < strlist_count(lines); i++) { + char *line = strlist_item(lines, i); + if (isempty(line)) { continue; } - char filename[NAME_MAX]; - strcpy(filename, rec->d_name); - char *ext = strstr(filename, ".whl"); - if (ext) { - *ext = '\0'; + + char **pair = split(line, ":", 1); + if (pair) { + char *key = strdup(strip(pair[0])); + char *value = strdup(lstrip(pair[1])); + + if (!key || !value) { + return -1; + } + + if (!strcasecmp(key, WHEEL_DIST_KEY[WHEEL_DIST_VERSION])) { + read_as = WHEEL_DIST_VERSION; + } else if (!strcasecmp(key, WHEEL_DIST_KEY[WHEEL_DIST_GENERATOR])) { + read_as = WHEEL_DIST_GENERATOR; + } else if (!strcasecmp(key, WHEEL_DIST_KEY[WHEEL_DIST_ROOT_IS_PURELIB])) { + read_as = WHEEL_DIST_ROOT_IS_PURELIB; + } else if (!strcasecmp(key, WHEEL_DIST_KEY[WHEEL_DIST_TAG])) { + read_as = WHEEL_DIST_TAG; + } + + switch (read_as) { + case WHEEL_DIST_VERSION: { + pkg->wheel_version = strdup(value); + if (!pkg->wheel_version) { + // memory error + return -1; + } + break; + } + case WHEEL_DIST_GENERATOR: { + pkg->generator = strdup(value); + if (!pkg->generator) { + // memory error + return -1; + } + break; + } + case WHEEL_DIST_ROOT_IS_PURELIB: { + pkg->root_is_pure_lib = strdup(value); + if (!pkg->root_is_pure_lib) { + // memory error + return -1; + } + break; + } + case WHEEL_DIST_TAG: { + if (!pkg->tag) { + pkg->tag = strlist_init(); + if (!pkg->tag) { + return -1; + } + } + strlist_append(&pkg->tag, value); + break; + } + default: + fprintf(stderr, "warning: unhandled wheel key on line %zu:\nbuffer contents: '%s'\n", i, value); + break; + } + guard_free(key); + guard_free(value); + } + guard_array_free(pair); + } + guard_strlist_free(&lines); + return data ? (ssize_t) strlen(data) : -1; +} + + +static inline int is_continuation(const char *s) { + return s && (s[0] == ' ' || s[0] == '\t'); +} + +static ssize_t wheel_parse_metadata(struct WheelMetadata * const pkg, const char * const data) { + int read_as = WHEEL_KEY_UNKNOWN; + // triggers + int reading_multiline = 0; + int reading_extra = 0; + size_t provides_extra_i = 0; + int reading_description = 0; + size_t base_description_len = 1024; + size_t len_description = 0; + struct WheelMetadata_ProvidesExtra *current_extra = NULL; + + if (!data) { + // data can't be NULL + return -1; + } + + pkg->provides_extra = calloc(WHEEL_MAXELEM + 1, sizeof(pkg->provides_extra[0])); + if (!pkg->provides_extra) { + // memory error + return -1; + } + + struct StrList *lines = strlist_init(); + if (!lines) { + // memory error + return -1; + } + + strlist_append_tokenize_raw(lines, (char *) data, "\r\n"); + for (size_t i = 0; i < strlist_count(lines); i++) { + const char *line = strlist_item(lines, i); + char *key = NULL; + char *value = NULL; + + reading_multiline = is_continuation(line); + if (!reading_multiline && line[0] == '\0') { + reading_description = 1; + read_as = WHEEL_META_DESCRIPTION; + } + + char **pair = split((char *) line, ":", 1); + if (!pair) { + // memory error + return -1; + } + + if (!reading_description && !reading_multiline && pair[1]) { + key = strip(strdup(pair[0])); + value = lstrip(strdup(pair[1])); + + if (!key || !value) { + // memory error + return -1; + } + + if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_METADATA_VERSION])) { + read_as = WHEEL_META_METADATA_VERSION; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_NAME])) { + read_as = WHEEL_META_NAME; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_VERSION])) { + read_as = WHEEL_META_VERSION; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_SUMMARY])) { + read_as = WHEEL_META_SUMMARY; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_AUTHOR])) { + read_as = WHEEL_META_AUTHOR; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_AUTHOR_EMAIL])) { + read_as = WHEEL_META_AUTHOR_EMAIL; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_MAINTAINER])) { + read_as = WHEEL_META_MAINTAINER; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_MAINTAINER_EMAIL])) { + read_as = WHEEL_META_MAINTAINER_EMAIL; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_LICENSE])) { + read_as = WHEEL_META_LICENSE; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_LICENSE_EXPRESSION])) { + read_as = WHEEL_META_LICENSE_EXPRESSION; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_HOME_PAGE])) { + read_as = WHEEL_META_HOME_PAGE; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_DOWNLOAD_URL])) { + read_as = WHEEL_META_DOWNLOAD_URL; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_PROJECT_URL])) { + read_as = WHEEL_META_PROJECT_URL; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_CLASSIFIER])) { + read_as = WHEEL_META_CLASSIFIER; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_REQUIRES_PYTHON])) { + read_as = WHEEL_META_REQUIRES_PYTHON; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_REQUIRES_EXTERNAL])) { + read_as = WHEEL_META_REQUIRES_EXTERNAL; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_DESCRIPTION_CONTENT_TYPE])) { + read_as = WHEEL_META_DESCRIPTION_CONTENT_TYPE; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_LICENSE_FILE])) { + read_as = WHEEL_META_LICENSE_FILE; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_REQUIRES_DIST])) { + read_as = WHEEL_META_REQUIRES_DIST; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_PROVIDES])) { + read_as = WHEEL_META_PROVIDES; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_IMPORT_NAME])) { + read_as = WHEEL_META_IMPORT_NAME; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_IMPORT_NAMESPACE])) { + read_as = WHEEL_META_IMPORT_NAMESPACE; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_PROVIDES_DIST])) { + read_as = WHEEL_META_PROVIDES_DIST; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_PROVIDES_EXTRA])) { + read_as = WHEEL_META_PROVIDES_EXTRA; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_PLATFORM])) { + read_as = WHEEL_META_PLATFORM; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_SUPPORTED_PLATFORM])) { + read_as = WHEEL_META_SUPPORTED_PLATFORM; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_KEYWORDS])) { + read_as = WHEEL_META_KEYWORDS; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_DYNAMIC])) { + read_as = WHEEL_META_DYNAMIC; + } else { + read_as = WHEEL_KEY_UNKNOWN; + } } else { - // not a wheel file. nothing to do - continue; + value = strdup(line); + if (!value) { + // memory error + return -1; + } } - size_t match = 0; - size_t pattern_count = 0; - for (; to_match[pattern_count] != NULL; pattern_count++) { - if (strstr(filename, to_match[pattern_count])) { - match++; + switch (read_as) { + case WHEEL_META_METADATA_VERSION: { + pkg->metadata_version = strdup(value); + if (!pkg->metadata_version) { + // memory error + return -1; + } + break; + } + case WHEEL_META_NAME: { + pkg->name = strdup(value); + if (!pkg->name) { + // memory error + return -1; + } + break; + } + case WHEEL_META_VERSION: { + pkg->version = strdup(value); + if (!pkg->version) { + // memory error + return -1; + } + break; + } + case WHEEL_META_SUMMARY: { + pkg->summary = strdup(value); + if (!pkg->summary) { + // memory error + return -1; + } + break; + } + case WHEEL_META_AUTHOR: { + if (!pkg->author) { + pkg->author = strlist_init(); + if (!pkg->author) { + // memory error + return -1; + } + } + strlist_append_tokenize(pkg->author, value, ","); + break; + } + case WHEEL_META_AUTHOR_EMAIL: { + if (!pkg->author_email) { + pkg->author_email = strlist_init(); + if (!pkg->author_email) { + // memory error + return -1; + } + } + strlist_append_tokenize(pkg->author_email, value, ","); + break; + } + case WHEEL_META_MAINTAINER: { + if (!pkg->maintainer) { + pkg->maintainer = strlist_init(); + if (!pkg->maintainer) { + // memory error + return -1; + } + } + strlist_append_tokenize(pkg->maintainer, value, ","); + break; + } + case WHEEL_META_MAINTAINER_EMAIL: { + if (!pkg->maintainer_email) { + pkg->maintainer_email = strlist_init(); + if (!pkg->maintainer_email) { + // memory error + return -1; + } + } + strlist_append_tokenize(pkg->maintainer_email, value, ","); + break; + } + case WHEEL_META_LICENSE: { + if (!reading_multiline) { + pkg->license = strdup(value); + if (!pkg->license) { + // memory error + return -1; + } + } else { + if (pkg->license) { + consume_append(&pkg->license, line, "\r\n"); + } else { + // previously unhandled memory error + return -1; + } + } + break; + } + case WHEEL_META_LICENSE_EXPRESSION: { + pkg->license_expression = strdup(value); + if (!pkg->license_expression) { + // memory error + return -1; + } + break; + } + case WHEEL_META_HOME_PAGE: { + pkg->home_page = strdup(value); + if (!pkg->home_page) { + // memory error + return -1; + } + break; + } + case WHEEL_META_DOWNLOAD_URL: + pkg->download_url = strdup(value); + if (!pkg->download_url) { + // memory error + return -1; + } + break; + case WHEEL_META_PROJECT_URL: { + if (!pkg->project_url) { + pkg->project_url = strlist_init(); + if (!pkg->project_url) { + // memory_error + return -1; + } + } + strlist_append(&pkg->project_url, value); + break; + } + case WHEEL_META_CLASSIFIER: { + if (!pkg->classifier) { + pkg->classifier = strlist_init(); + if (!pkg->classifier) { + // memory error + return -1; + } + } + strlist_append(&pkg->classifier, value); + break; + } + case WHEEL_META_REQUIRES_PYTHON: { + if (!pkg->requires_python) { + pkg->requires_python = strlist_init(); + if (!pkg->requires_python) { + // memory error + return -1; + } + } + strlist_append(&pkg->requires_python, value); + break; + } + case WHEEL_META_REQUIRES_EXTERNAL: { + if (!pkg->requires_external) { + pkg->requires_external = strlist_init(); + if (!pkg->requires_external) { + // memory error + return -1; + } + } + strlist_append(&pkg->requires_external, value); + break; + } + case WHEEL_META_DESCRIPTION_CONTENT_TYPE: { + pkg->description_content_type = strdup(value); + if (!pkg->description_content_type) { + // memory error + return -1; + } + break; + } + case WHEEL_META_LICENSE_FILE: { + if (!pkg->license_file) { + pkg->license_file = strlist_init(); + if (!pkg->license_file) { + // memory error + return -1; + } + } + strlist_append(&pkg->license_file, value); + break; + } + case WHEEL_META_IMPORT_NAME: { + if (!pkg->import_name) { + pkg->import_name = strlist_init(); + if (!pkg->import_name) { + // memory error + return -1; + } + } + strlist_append(&pkg->import_name, value); + break; + } + case WHEEL_META_IMPORT_NAMESPACE: { + if (!pkg->import_namespace) { + pkg->import_namespace = strlist_init(); + if (!pkg->import_namespace) { + // memory error + return -1; + } + } + strlist_append(&pkg->import_namespace, value); + break; + } + case WHEEL_META_REQUIRES_DIST: { + if (!pkg->requires_dist) { + pkg->requires_dist = strlist_init(); + if (!pkg->requires_dist) { + // memory error + return -1; + } + } + if (reading_extra) { + if (strrchr(value, ';')) { + *strrchr(value, ';') = 0; + } + if (!current_extra) { + reading_extra = 0; + break; + } + if (!current_extra->requires_dist) { + current_extra->requires_dist = strlist_init(); + if (!current_extra->requires_dist) { + // memory error + return -1; + } + } + strlist_append(¤t_extra->requires_dist, value); + } else { + strlist_append(&pkg->requires_dist, value); + reading_extra = 0; + } + break; + } + case WHEEL_META_PROVIDES: { + if (!pkg->provides) { + pkg->provides = strlist_init(); + if (!pkg->provides) { + // memory error + return -1; + } + } + strlist_append(&pkg->provides, value); + break; + } + case WHEEL_META_PROVIDES_DIST: { + if (!pkg->provides_dist) { + pkg->provides_dist = strlist_init(); + if (!pkg->provides_dist) { + // memory error + return -1; + } + } + strlist_append(&pkg->provides_dist, value); + break; + } + case WHEEL_META_PROVIDES_EXTRA: + pkg->provides_extra[provides_extra_i] = calloc(1, sizeof(*pkg->provides_extra[0])); + if (!pkg->provides_extra[provides_extra_i]) { + // memory error + return -1; + } + current_extra = pkg->provides_extra[provides_extra_i]; + current_extra->target = strdup(value); + if (!current_extra->target) { + // memory error + return -1; + } + reading_extra = 1; + provides_extra_i++; + break; + case WHEEL_META_PLATFORM:{ + if (!pkg->platform) { + pkg->platform = strlist_init(); + if (!pkg->platform) { + // memory error + return -1; + } + } + strlist_append(&pkg->platform, value); + break; + } + case WHEEL_META_SUPPORTED_PLATFORM: { + if (!pkg->supported_platform) { + pkg->supported_platform = strlist_init(); + if (!pkg->supported_platform) { + // memory error + return -1; + } + } + strlist_append(&pkg->supported_platform, value); + break; + } + case WHEEL_META_KEYWORDS: { + if (!pkg->keywords) { + pkg->keywords = strlist_init(); + if (!pkg->keywords) { + // memory error + return -1; + } + } + strlist_append_tokenize(pkg->keywords, value, ","); + break; + } + case WHEEL_META_DYNAMIC: { + if (!pkg->dynamic) { + pkg->dynamic = strlist_init(); + if (!pkg->dynamic) { + // memory error + return -1; + } + } + strlist_append(&pkg->dynamic, value); + break; + } + case WHEEL_META_DESCRIPTION: { + // reading_description will never be reset to zero + reading_description = 1; + if (!pkg->description) { + pkg->description = malloc(base_description_len + 1); + if (!pkg->description) { + return -1; + } + len_description = snprintf(pkg->description, base_description_len, "%s\n", line); + } else { + const size_t next_len = snprintf(NULL, 0, "%s\n%s\n", pkg->description, line); + if (next_len + 1 > base_description_len) { + base_description_len *= 2; + char *tmp = realloc(pkg->description, base_description_len + 1); + if (!tmp) { + // memory error + guard_free(pkg->description); + return -1; + } + pkg->description = tmp; + } + len_description += snprintf(pkg->description + len_description, next_len + 1, "%s\n", line); + + //consume_append(&pkg->description, line, "\r\n"); + } + break; } + case WHEEL_KEY_UNKNOWN: + default: + fprintf(stderr, "warning: unhandled metadata key on line %zu:\nbuffer contents: '%s'\n", i, value); + break; } + guard_free(key); + guard_free(value); + guard_array_free(pair); + if (reading_multiline) { + guard_free(value); + } + } + guard_strlist_free(&lines); + + return 0; +} - if (!startswith(rec->d_name, name)) { +int wheel_get_file_contents(const char *wheelfile, const char *filename, char **contents) { + int status = 0; + int err = 0; + struct zip_stat archive_info; + zip_t *archive = zip_open(wheelfile, 0, &err); + if (!archive) { + return status; + } + + zip_stat_init(&archive_info); + for (size_t i = 0; zip_stat_index(archive, i, 0, &archive_info) >= 0; i++) { + char internal_path[1024] = {0}; + snprintf(internal_path, sizeof(internal_path), "%s", filename); + const int match = fnmatch(internal_path, archive_info.name, 0); + if (match == FNM_NOMATCH) { continue; } + if (match < 0) { + goto GWM_FAIL; + } - if (match_mode == WHEEL_MATCH_EXACT && match != pattern_count) { - continue; + zip_file_t *handle = zip_fopen_index(archive, i, 0); + if (!handle) { + goto GWM_FAIL; } - result = calloc(1, sizeof(*result)); - if (!result) { - SYSERROR("Unable to allocate %zu bytes for wheel struct", sizeof(*result)); - closedir(dp); - return NULL; - } - - result->path_name = realpath(package_path, NULL); - if (!result->path_name) { - SYSERROR("Unable to resolve absolute path to %s: %s", filename, strerror(errno)); - wheel_free(&result); - closedir(dp); - return NULL; - } - result->file_name = strdup(rec->d_name); - if (!result->file_name) { - SYSERROR("Unable to allocate bytes for %s: %s", rec->d_name, strerror(errno)); - wheel_free(&result); - closedir(dp); - return NULL; - } - - size_t parts_total; - char **parts = split(filename, "-", 0); - if (!parts) { - // This shouldn't happen unless a wheel file is present in the - // directory with a malformed file name, or we've managed to - // exhaust the system's memory - SYSERROR("%s has no '-' separators! (Delete this file and try again)", filename); - wheel_free(&result); - closedir(dp); - return NULL; - } - - for (parts_total = 0; parts[parts_total] != NULL; parts_total++) {} - if (parts_total == 5) { - // no build tag - result->distribution = strdup(parts[0]); - result->version = strdup(parts[1]); - result->build_tag = NULL; - result->python_tag = strdup(parts[2]); - result->abi_tag = strdup(parts[3]); - result->platform_tag = strdup(parts[4]); - } else if (parts_total == 6) { - // has build tag - result->distribution = strdup(parts[0]); - result->version = strdup(parts[1]); - result->build_tag = strdup(parts[2]); - result->python_tag = strdup(parts[3]); - result->abi_tag = strdup(parts[4]); - result->platform_tag = strdup(parts[5]); - } else { - SYSERROR("Unknown wheel name format: %s. Expected 5 or 6 strings " - "separated by '-', but got %zu instead", filename, parts_total); - guard_array_free(parts); - wheel_free(&result); - closedir(dp); - return NULL; - } - guard_array_free(parts); + *contents = calloc(archive_info.size + 1, sizeof(**contents)); + if (!*contents) { + zip_fclose(handle); + goto GWM_FAIL; + } + + if (zip_fread(handle, *contents, archive_info.size) < 0) { + zip_fclose(handle); + guard_free(*contents); + goto GWM_FAIL; + } + zip_fclose(handle); break; } - closedir(dp); + + goto GWM_END; + GWM_FAIL: + status = -1; + + GWM_END: + zip_close(archive); + return status; +} + +static int wheel_metadata_get(const struct Wheel *pkg, const char *wheel_filename) { + char *data = NULL; + if (wheel_get_file_contents(wheel_filename, "*.dist-info/METADATA", &data)) { + return -1; + } + const ssize_t result = wheel_parse_metadata(pkg->metadata, data); + char *data_orig = data; + guard_free(data_orig); + return (int) result; +} + +static struct WheelValue wheel_data_dump(const struct Wheel *pkg, const ssize_t key) { + struct WheelValue result; + result.type = WHEELVAL_STR; + result.count = 0; + switch (key) { + case WHEEL_DIST_VERSION: + result.data = pkg->wheel_version; + break; + case WHEEL_DIST_GENERATOR: + result.data = pkg->generator; + break; + case WHEEL_DIST_ZIP_SAFE: + result.data = pkg->zip_safe ? "True" : "False"; + break; + case WHEEL_DIST_ROOT_IS_PURELIB: + result.data = pkg->root_is_pure_lib ? "True" : "False"; + break; + case WHEEL_DIST_TOP_LEVEL: + result.type = WHEELVAL_STRLIST; + result.data = pkg->top_level; + break; + case WHEEL_DIST_TAG: + result.type = WHEELVAL_STRLIST; + result.data = pkg->tag; + break; + case WHEEL_DIST_RECORD: + result.type = WHEELVAL_OBJ_RECORD; + result.data = pkg->record; + result.count = pkg->num_record; + break; + case WHEEL_DIST_ENTRY_POINT: + result.type = WHEELVAL_OBJ_ENTRY_POINT; + result.data = pkg->entry_point; + result.count = pkg->num_entry_point; + break; + default: + result.data = NULL; + result.type = WHEEL_KEY_UNKNOWN; + break; + } + + switch (result.type) { + case WHEELVAL_STR: + result.count = result.data != NULL ? strlen(result.data) : 0; + break; + case WHEELVAL_STRLIST: + result.count = result.data != NULL ? strlist_count(result.data) : 0; + break; + default: + break; + } + return result; } -void wheel_free(struct Wheel **wheel) { - struct Wheel *w = (*wheel); - guard_free(w->path_name); - guard_free(w->file_name); - guard_free(w->distribution); - guard_free(w->version); - guard_free(w->build_tag); - guard_free(w->python_tag); - guard_free(w->abi_tag); - guard_free(w->python_tag); - guard_free(w->platform_tag); - guard_free(w); +static struct WheelValue wheel_metadata_dump(const struct Wheel *pkg, const ssize_t key) { + const struct WheelMetadata *meta = pkg->metadata; + struct WheelValue result; + result.type = WHEELVAL_STR; + result.count = 0; + switch (key) { + case WHEEL_META_METADATA_VERSION: + result.data = meta->metadata_version; + break; + case WHEEL_META_NAME: + result.data = meta->name; + break; + case WHEEL_META_VERSION: + result.data = meta->version; + break; + case WHEEL_META_SUMMARY: + result.data = meta->summary; + break; + case WHEEL_META_AUTHOR: + result.type = WHEELVAL_STRLIST; + result.data = meta->author; + break; + case WHEEL_META_AUTHOR_EMAIL: + result.type = WHEELVAL_STRLIST; + result.data = meta->author_email; + break; + case WHEEL_META_MAINTAINER: + result.type = WHEELVAL_STRLIST; + result.data = meta->maintainer; + break; + case WHEEL_META_MAINTAINER_EMAIL: + result.type = WHEELVAL_STRLIST; + result.data = meta->maintainer_email; + break; + case WHEEL_META_LICENSE: + result.data = meta->license; + break; + case WHEEL_META_HOME_PAGE: + result.data = meta->home_page; + break; + case WHEEL_META_DOWNLOAD_URL: + result.data = meta->download_url; + break; + case WHEEL_META_PROJECT_URL: + result.type = WHEELVAL_STRLIST; + result.data = meta->project_url; + break; + case WHEEL_META_CLASSIFIER: + result.type = WHEELVAL_STRLIST; + result.data = meta->classifier; + break; + case WHEEL_META_REQUIRES_PYTHON: + result.type = WHEELVAL_STRLIST; + result.data = meta->requires_python; + break; + case WHEEL_META_DESCRIPTION_CONTENT_TYPE: + result.data = meta->description_content_type; + break; + case WHEEL_META_LICENSE_FILE: + result.type = WHEELVAL_STRLIST; + result.data = meta->license_file; + break; + case WHEEL_META_LICENSE_EXPRESSION: + result.data = meta->license_expression; + break; + case WHEEL_META_IMPORT_NAME: + result.type = WHEELVAL_STRLIST; + result.data = meta->import_name; + break; + case WHEEL_META_IMPORT_NAMESPACE: + result.type = WHEELVAL_STRLIST; + result.data = meta->import_namespace; + break; + case WHEEL_META_REQUIRES_DIST: + result.type = WHEELVAL_STRLIST; + result.data = meta->requires_dist; + break; + case WHEEL_META_PROVIDES_DIST: + result.type = WHEELVAL_STRLIST; + result.data = meta->provides_dist; + break; + case WHEEL_META_PROVIDES_EXTRA: + result.type = WHEELVAL_OBJ_EXTRA; + result.data = (struct WheelMetadata_ProvidesExtra *) meta->provides_extra; + result.count = result.data != NULL ? (size_t) ((struct WheelMetadata_ProvidesExtra *) result.data)->count : 0; + break; + case WHEEL_META_OBSOLETES: + result.type = WHEELVAL_STRLIST; + result.data = meta->obsoletes; + break; + case WHEEL_META_OBSOLETES_DIST: + result.type = WHEELVAL_STRLIST; + result.data = meta->obsoletes_dist; + break; + case WHEEL_META_DESCRIPTION: + result.type = WHEELVAL_STR; + result.data = meta->description; + break; + case WHEEL_META_PLATFORM: + result.type = WHEELVAL_STRLIST; + result.data = meta->platform; + break; + case WHEEL_META_SUPPORTED_PLATFORM: + result.type = WHEELVAL_STRLIST; + result.data = meta->supported_platform; + break; + case WHEEL_META_KEYWORDS: + result.type = WHEELVAL_STRLIST; + result.data = meta->keywords; + break; + case WHEEL_META_DYNAMIC: + result.type = WHEELVAL_STRLIST; + result.data = meta->dynamic; + break; + case WHEEL_KEY_UNKNOWN: + default: + result.data = NULL; + break; + } + + switch (result.type) { + case WHEELVAL_STR: + result.count = result.data != NULL ? strlen(result.data) : 0; + break; + case WHEELVAL_STRLIST: + result.count = result.data != NULL ? strlist_count(result.data) : 0; + break; + default: + break; + } + + return result; +} + +static ssize_t get_key_index(const char **arr, const char *key) { + for (ssize_t i = 0; arr[i] != NULL; i++) { + if (strcmp(arr[i], key) == 0) { + return i; + } + } + return -1; +} + +const char *wheel_get_key_by_id(const int from, const ssize_t id) { + if (from == WHEEL_FROM_DIST) { + if (id >= 0 && id < WHEEL_DIST_END_ENUM) { + return WHEEL_DIST_KEY[id]; + } + } + if (from == WHEEL_FROM_METADATA) { + if (id >= 0 && id < WHEEL_META_END_ENUM) { + return WHEEL_META_KEY[id]; + } + } + return NULL; +} + +struct WheelValue wheel_get_value_by_name(const struct Wheel *pkg, const int from, const char *key) { + struct WheelValue result = {0}; + ssize_t id; + + if (from == WHEEL_FROM_DIST) { + id = get_key_index(WHEEL_DIST_KEY, key); + result = wheel_data_dump(pkg, id); + } else if (from == WHEEL_FROM_METADATA) { + id = get_key_index(WHEEL_META_KEY, key); + result = wheel_metadata_dump(pkg, id); + } else { + result.data = NULL; + result.type = WHEEL_KEY_UNKNOWN; + } + + return result; +} + +struct WheelValue wheel_get_value_by_id(const struct Wheel *pkg, const int from, const ssize_t id) { + struct WheelValue result = {0}; + + if (from == WHEEL_FROM_DIST) { + result = wheel_data_dump(pkg, id); + } else if (from == WHEEL_FROM_METADATA) { + result = wheel_metadata_dump(pkg, id); + } else { + result.data = NULL; + result.type = -1; + } + + return result; } + +void wheel_record_free(struct WheelRecord **record) { + guard_free((*record)->filename); + guard_free((*record)->checksum); + guard_free(*record); +} + +void wheel_entry_point_free(struct WheelEntryPoint **entry) { + guard_free((*entry)->name); + guard_free((*entry)->function); + guard_free((*entry)->type); + guard_free(*entry); +} + +void wheel_metadata_free(struct WheelMetadata *meta) { + guard_free(meta->license); + guard_free(meta->license_expression); + guard_free(meta->version); + guard_free(meta->name); + guard_free(meta->description); + guard_free(meta->metadata_version); + guard_free(meta->summary); + guard_free(meta->description_content_type); + guard_free(meta->home_page); + guard_free(meta->download_url); + + guard_strlist_free(&meta->author_email); + guard_strlist_free(&meta->author); + guard_strlist_free(&meta->maintainer); + guard_strlist_free(&meta->maintainer_email); + guard_strlist_free(&meta->requires_python); + guard_strlist_free(&meta->requires_external); + guard_strlist_free(&meta->project_url); + guard_strlist_free(&meta->classifier); + guard_strlist_free(&meta->requires_dist); + guard_strlist_free(&meta->keywords); + guard_strlist_free(&meta->license_file); + + for (size_t i = 0; meta->provides_extra[i] != NULL; i++) { + guard_free(meta->provides_extra[i]->target); + guard_strlist_free(&meta->provides_extra[i]->requires_dist); + guard_free(meta->provides_extra[i]); + } + guard_free(meta->provides_extra); + + guard_free(meta); +} + +void wheel_package_free(struct Wheel **pkg) { + guard_free((*pkg)->wheel_version); + guard_free((*pkg)->generator); + guard_free((*pkg)->root_is_pure_lib); + wheel_metadata_free((*pkg)->metadata); + + guard_strlist_free(&(*pkg)->tag); + guard_strlist_free(&(*pkg)->top_level); + for (size_t i = 0; (*pkg)->record && (*pkg)->record[i] != NULL; i++) { + wheel_record_free(&(*pkg)->record[i]); + } + guard_free((*pkg)->record); + + for (size_t i = 0; (*pkg)->entry_point && (*pkg)->entry_point[i] != NULL; i++) { + wheel_entry_point_free(&(*pkg)->entry_point[i]); + } + guard_free((*pkg)->entry_point); + guard_free((*pkg)); +} + +int wheel_get_top_level(struct Wheel *pkg, const char *filename) { + char *data = NULL; + if (wheel_get_file_contents(filename, "*.dist-info/top_level.txt", &data)) { + guard_free(data); + return -1; + } + if (!pkg->top_level) { + pkg->top_level = strlist_init(); + } + strlist_append_tokenize(pkg->top_level, data, "\r\n"); + guard_free(data); + return 0; +} + +int wheel_get_zip_safe(struct Wheel *pkg, const char *filename) { + char *data = NULL; + const int exists = wheel_get_file_contents(filename, "*.dist-info/zip-safe", &data) == 0; + guard_free(data); + + pkg->zip_safe = 0; + if (exists) { + pkg->zip_safe = 1; + } + + return 0; +} + +int wheel_get_records(struct Wheel *pkg, const char *filename) { + char *data = NULL; + const int exists = wheel_get_file_contents(filename, "*.dist-info/RECORD", &data) == 0; + + if (!exists) { + guard_free(data); + return 1; + } + + const size_t records_initial_count = 2; + pkg->record = calloc(records_initial_count, sizeof(*pkg->record)); + if (!pkg->record) { + guard_free(data); + return 1; + } + size_t records_count = 0; + + const char *token = NULL; + char *data_orig = data; + while ((token = strsep(&data, "\r\n")) != NULL) { + if (!strlen(token)) { + continue; + } + + pkg->record[records_count] = calloc(1, sizeof(*pkg->record[0])); + if (!pkg->record[records_count]) { + return -1; + } + + struct WheelRecord *record = pkg->record[records_count]; + for (size_t x = 0; x < 3; x++) { + const char *next_comma = strpbrk(token, ","); + if (next_comma) { + if (x == 0) { + record->filename = strndup(token, next_comma - token); + } else if (x == 1) { + record->checksum = strndup(token, next_comma - token); + } + token = next_comma + 1; + } else { + record->size = strtol(token, NULL, 10); + } + } + records_count++; + + struct WheelRecord **tmp = realloc(pkg->record, (records_count + 2) * sizeof(*pkg->record)); + if (tmp == NULL) { + guard_free(data); + return -1; + } + pkg->record = tmp; + pkg->record[records_count + 1] = NULL; + } + + pkg->num_record = records_count; + guard_free(data_orig); + return 0; +} + +int wheel_get(struct Wheel **pkg, const char *filename) { + char *data = NULL; + if (wheel_get_file_contents(filename, "*.dist-info/WHEEL", &data) < 0) { + return -1; + } + const ssize_t result = wheel_parse_wheel(*pkg, data); + guard_free(data); + return (int) result; +} + +int wheel_get_entry_point(struct Wheel *pkg, const char *filename) { + char *data = NULL; + if (wheel_get_file_contents(filename, "*.dist-info/entry_points.txt", &data) < 0) { + return -1; + } + + struct StrList *lines = strlist_init(); + if (!lines) { + goto GEP_FAIL; + } + strlist_append_tokenize(lines, data, "\r\n"); + + const size_t line_count = strlist_count(lines); + size_t usable_lines = line_count; + for (size_t i = 0; i < line_count; i++) { + const char *item = strlist_item(lines, i); + if (isempty((char *) item) || item[0] == '[') { + usable_lines--; + } + } + + pkg->entry_point = calloc(usable_lines + 1, sizeof(*pkg->entry_point)); + if (!pkg->entry_point) { + goto GEP_FAIL; + } + + for (size_t i = 0; i < usable_lines; i++) { + pkg->entry_point[i] = calloc(1, sizeof(*pkg->entry_point[0])); + if (!pkg->entry_point[i]) { + goto GEP_FAIL; + } + } + + size_t x = 0; + char section[255] = {0}; + for (size_t i = 0; i < line_count; i++) { + const char *item = strlist_item(lines, i); + if (isempty((char *) item)) { + continue; + } + + if (strpbrk(item, "[")) { + const size_t start = strcspn((char *) item, "[") + 1; + if (start) { + const size_t len = strcspn((char *) item, "]"); + strncpy(section, item + start, len - start); + section[len - start] = '\0'; + continue; + } + } + + pkg->entry_point[x]->type = strdup(section); + if (!pkg->entry_point[x]->type) { + goto GEP_FAIL; + } + + char **pair = split((char *) item, "=", 1); + if (!pair) { + wheel_entry_point_free(&pkg->entry_point[x]); + goto GEP_FAIL; + } + + pkg->entry_point[x]->name = strdup(strip(pair[0])); + if (!pkg->entry_point[x]->name) { + wheel_entry_point_free(&pkg->entry_point[x]); + guard_array_free(pair); + goto GEP_FAIL; + } + + pkg->entry_point[x]->function = strdup(lstrip(pair[1])); + if (!pkg->entry_point[x]->function) { + wheel_entry_point_free(&pkg->entry_point[x]); + guard_array_free(pair); + goto GEP_FAIL; + } + guard_array_free(pair); + x++; + } + pkg->num_entry_point = x; + guard_strlist_free(&lines); + guard_free(data); + return 0; + + GEP_FAIL: + guard_strlist_free(&lines); + guard_free(data); + return -1; +} + +int wheel_value_error(struct WheelValue const *val) { + if (val) { + if (val->type < 0 && val->data == NULL) { + return 1; + } + } + return 0; +} + +int wheel_show_info(const struct Wheel *wheel) { + printf("WHEEL INFO\n\n"); + for (ssize_t i = 0; i < WHEEL_DIST_END_ENUM; i++) { + const char *key = wheel_get_key_by_id(WHEEL_FROM_DIST, i); + if (!key) { + fprintf(stderr, "wheel_get_key_by_id(%zi) failed\n", i); + return -1; + } + + printf("%s: ", key); + fflush(stdout); + const struct WheelValue dist = wheel_get_value_by_id(wheel, WHEEL_FROM_DIST, i); + if (wheel_value_error(&dist)) { + fprintf(stderr, "wheel_get_value_by_id(%zi) failed\n", i); + return -1; + } + switch (dist.type) { + case WHEELVAL_STR: { + char *s = dist.data; + if (s != NULL && !isempty(s)) { + printf("%s\n", s); + } else { + printf("[N/A]\n"); + } + break; + } + case WHEELVAL_STRLIST: { + struct StrList *list = dist.data; + if (list) { + printf("\n"); + for (size_t x = 0; x < strlist_count(list); x++) { + const char *item = strlist_item(list, x); + printf(" %s\n", item); + } + } else { + printf("[N/A]\n"); + } + break; + } + case WHEELVAL_OBJ_RECORD: { + struct WheelRecord **record = dist.data; + if (record && *record) { + printf("\n"); + for (size_t x = 0; x < dist.count; x++) { + printf(" [%zu] %s (size: %zu bytes, checksum: %s)\n", x, wheel->record[x]->filename, wheel->record[x]->size, strlen(wheel->record[x]->checksum) ? wheel->record[x]->checksum : "N/A"); + } + } else { + printf("[N/A]\n"); + } + break; + } + case WHEELVAL_OBJ_ENTRY_POINT: { + struct WheelEntryPoint **entry = dist.data; + if (entry && *entry) { + printf("\n"); + for (size_t x = 0; x < dist.count; x++) { + printf(" [%zu] type: %s, name: %s, function: %s\n", x, entry[x]->type, entry[x]->name, entry[x]->function); + } + } else { + printf("[N/A]\n"); + } + break; + } + default: + printf("[no handler]\n"); + break; + } + + } + + printf("\nPACKAGE INFO\n\n"); + for (ssize_t i = 0; i < WHEEL_META_END_ENUM; i++) { + const char *key = wheel_get_key_by_id(WHEEL_FROM_METADATA, i); + if (!key) { + fprintf(stderr, "wheel_get_key_by_id(%zi) failed\n", i); + return -1; + } + printf("%s: ", key); + fflush(stdout); + + const struct WheelValue pkg = wheel_get_value_by_id(wheel, WHEEL_FROM_METADATA, i); + if (wheel_value_error(&pkg)) { + fprintf(stderr, "wheel_get_value_by_id(%zi) failed\n", i); + return -1; + } + switch (pkg.type) { + case WHEELVAL_STR: { + char *s = pkg.data; + if (s != NULL && !isempty(s)) { + printf("%s\n", s); + } else { + printf("[N/A]\n"); + } + break; + } + case WHEELVAL_STRLIST: { + struct StrList *list = pkg.data; + if (list) { + printf("\n"); + for (size_t x = 0; x < strlist_count(list); x++) { + const char *item = strlist_item(list, x); + printf(" %s\n", item); + } + } else { + printf("[N/A]\n"); + } + break; + } + case WHEELVAL_OBJ_EXTRA: { + const struct WheelMetadata_ProvidesExtra **extra = pkg.data; + printf("\n"); + if (*extra) { + for (size_t x = 0; extra[x] != NULL; x++) { + printf(" + %s\n", extra[x]->target); + for (size_t z = 0; z < strlist_count(extra[x]->requires_dist); z++) { + const char *item = strlist_item(extra[x]->requires_dist, z); + printf(" `- %s\n", item); + } + } + } else { + printf("[N/A]\n"); + } + break; + } + default: + break; + } + } + return 0; +} + +int wheel_package(struct Wheel **pkg, const char *filename) { + if (!filename) { + return WHEEL_PACKAGE_E_FILENAME; + } + if (!*pkg) { + *pkg = calloc(1, sizeof(**pkg)); + if (!*pkg) { + return WHEEL_PACKAGE_E_ALLOC; + } + + (*pkg)->metadata = calloc(1, sizeof(*(*pkg)->metadata)); + if (!(*pkg)->metadata) { + guard_free(*pkg); + return WHEEL_PACKAGE_E_ALLOC; + } + } + if (wheel_get(pkg, filename) < 0) { + return WHEEL_PACKAGE_E_GET; + } + if (wheel_metadata_get(*pkg, filename) < 0) { + return WHEEL_PACKAGE_E_GET_METADATA; + } + if (wheel_get_top_level(*pkg, filename) < 0) { + return WHEEL_PACKAGE_E_GET_TOP_LEVEL; + } + if (wheel_get_records(*pkg, filename) < 0) { + return WHEEL_PACKAGE_E_GET_RECORDS; + } + if (wheel_get_entry_point(*pkg, filename) < 0) { + return WHEEL_PACKAGE_E_GET_ENTRY_POINT; + } + + // Optional marker + wheel_get_zip_safe(*pkg, filename); + + return WHEEL_PACKAGE_E_SUCCESS; +} + + diff --git a/src/lib/core/wheelinfo.c b/src/lib/core/wheelinfo.c new file mode 100644 index 0000000..1a93a82 --- /dev/null +++ b/src/lib/core/wheelinfo.c @@ -0,0 +1,129 @@ +#include "wheelinfo.h" + +struct WheelInfo *wheelinfo_get(const char *basepath, const char *name, char *to_match[], unsigned match_mode) { + struct dirent *rec; + struct WheelInfo *result = NULL; + char package_path[PATH_MAX]; + char package_name[NAME_MAX]; + + strcpy(package_name, name); + tolower_s(package_name); + sprintf(package_path, "%s/%s", basepath, package_name); + + DIR *dp = opendir(package_path); + if (!dp) { + return NULL; + } + + while ((rec = readdir(dp)) != NULL) { + if (!strcmp(rec->d_name, ".") || !strcmp(rec->d_name, "..")) { + continue; + } + char filename[NAME_MAX]; + strcpy(filename, rec->d_name); + char *ext = strstr(filename, ".whl"); + if (ext) { + *ext = '\0'; + } else { + // not a wheel file. nothing to do + continue; + } + + size_t match = 0; + size_t pattern_count = 0; + for (; to_match[pattern_count] != NULL; pattern_count++) { + if (strstr(filename, to_match[pattern_count])) { + match++; + } + } + + if (!startswith(rec->d_name, name)) { + continue; + } + + if (match_mode == WHEEL_MATCH_EXACT && match != pattern_count) { + continue; + } + + result = calloc(1, sizeof(*result)); + if (!result) { + SYSERROR("Unable to allocate %zu bytes for wheel struct", sizeof(*result)); + closedir(dp); + return NULL; + } + + result->path_name = realpath(package_path, NULL); + if (!result->path_name) { + SYSERROR("Unable to resolve absolute path to %s: %s", filename, strerror(errno)); + wheelinfo_free(&result); + closedir(dp); + return NULL; + } + result->file_name = strdup(rec->d_name); + if (!result->file_name) { + SYSERROR("Unable to allocate bytes for %s: %s", rec->d_name, strerror(errno)); + wheelinfo_free(&result); + closedir(dp); + return NULL; + } + + size_t parts_total; + char **parts = split(filename, "-", 0); + if (!parts) { + // This shouldn't happen unless a wheel file is present in the + // directory with a malformed file name, or we've managed to + // exhaust the system's memory + SYSERROR("%s has no '-' separators! (Delete this file and try again)", filename); + wheelinfo_free(&result); + closedir(dp); + return NULL; + } + + for (parts_total = 0; parts[parts_total] != NULL; parts_total++) {} + if (parts_total == 5) { + // no build tag + result->distribution = strdup(parts[0]); + result->version = strdup(parts[1]); + result->build_tag = NULL; + result->python_tag = strdup(parts[2]); + result->abi_tag = strdup(parts[3]); + result->platform_tag = strdup(parts[4]); + } else if (parts_total == 6) { + // has build tag + result->distribution = strdup(parts[0]); + result->version = strdup(parts[1]); + result->build_tag = strdup(parts[2]); + result->python_tag = strdup(parts[3]); + result->abi_tag = strdup(parts[4]); + result->platform_tag = strdup(parts[5]); + } else { + SYSERROR("Unknown wheel name format: %s. Expected 5 or 6 strings " + "separated by '-', but got %zu instead", filename, parts_total); + guard_array_free(parts); + wheelinfo_free(&result); + closedir(dp); + return NULL; + } + guard_array_free(parts); + break; + } + closedir(dp); + return result; +} + +void wheelinfo_free(struct WheelInfo **wheel) { + struct WheelInfo *w = (*wheel); + if (!w) { + return; + } + guard_free(w->path_name); + guard_free(w->file_name); + guard_free(w->distribution); + guard_free(w->version); + guard_free(w->build_tag); + guard_free(w->python_tag); + guard_free(w->abi_tag); + guard_free(w->python_tag); + guard_free(w->platform_tag); + guard_free(w); +} diff --git a/src/lib/delivery/delivery.c b/src/lib/delivery/delivery.c index 600ddf9..11dd7b0 100644 --- a/src/lib/delivery/delivery.c +++ b/src/lib/delivery/delivery.c @@ -153,21 +153,21 @@ struct Delivery *delivery_duplicate(const struct Delivery *ctx) { result->deploy.jfrog_auth.url = strdup_maybe(ctx->deploy.jfrog_auth.url); result->deploy.jfrog_auth.user = strdup_maybe(ctx->deploy.jfrog_auth.user); - for (size_t i = 0; i < sizeof(result->tests) / sizeof(result->tests[0]); i++) { - result->tests[i].disable = ctx->tests[i].disable; - result->tests[i].parallel = ctx->tests[i].parallel; - result->tests[i].build_recipe = strdup_maybe(ctx->tests[i].build_recipe); - result->tests[i].name = strdup_maybe(ctx->tests[i].name); - result->tests[i].version = strdup_maybe(ctx->tests[i].version); - result->tests[i].repository = strdup_maybe(ctx->tests[i].repository); - result->tests[i].repository_info_ref = strdup_maybe(ctx->tests[i].repository_info_ref); - result->tests[i].repository_info_tag = strdup_maybe(ctx->tests[i].repository_info_tag); - result->tests[i].repository_remove_tags = strlist_copy(ctx->tests[i].repository_remove_tags); - if (ctx->tests[i].runtime.environ) { - result->tests[i].runtime.environ = runtime_copy(ctx->tests[i].runtime.environ->data); + for (size_t i = 0; result->tests && i < result->tests->num_used; i++) { + result->tests->test[i]->disable = ctx->tests->test[i]->disable; + result->tests->test[i]->parallel = ctx->tests->test[i]->parallel; + result->tests->test[i]->build_recipe = strdup_maybe(ctx->tests->test[i]->build_recipe); + result->tests->test[i]->name = strdup_maybe(ctx->tests->test[i]->name); + result->tests->test[i]->version = strdup_maybe(ctx->tests->test[i]->version); + result->tests->test[i]->repository = strdup_maybe(ctx->tests->test[i]->repository); + result->tests->test[i]->repository_info_ref = strdup_maybe(ctx->tests->test[i]->repository_info_ref); + result->tests->test[i]->repository_info_tag = strdup_maybe(ctx->tests->test[i]->repository_info_tag); + result->tests->test[i]->repository_remove_tags = strlist_copy(ctx->tests->test[i]->repository_remove_tags); + if (ctx->tests->test[i]->runtime->environ) { + result->tests->test[i]->runtime->environ = runtime_copy(ctx->tests->test[i]->runtime->environ->data); } - result->tests[i].script = strdup_maybe(ctx->tests[i].script); - result->tests[i].script_setup = strdup_maybe(ctx->tests[i].script_setup); + result->tests->test[i]->script = strdup_maybe(ctx->tests->test[i]->script); + result->tests->test[i]->script_setup = strdup_maybe(ctx->tests->test[i]->script_setup); } return result; @@ -175,7 +175,7 @@ struct Delivery *delivery_duplicate(const struct Delivery *ctx) { void delivery_free(struct Delivery *ctx) { guard_free(ctx->system.arch); - guard_array_free(ctx->system.platform); + guard_array_n_free(ctx->system.platform, DELIVERY_PLATFORM_MAX); guard_free(ctx->meta.name); guard_free(ctx->meta.version); guard_free(ctx->meta.codename); @@ -230,18 +230,24 @@ void delivery_free(struct Delivery *ctx) { guard_strlist_free(&ctx->conda.pip_packages_purge); guard_strlist_free(&ctx->conda.wheels_packages); - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { - guard_free(ctx->tests[i].name); - guard_free(ctx->tests[i].version); - guard_free(ctx->tests[i].repository); - guard_free(ctx->tests[i].repository_info_ref); - guard_free(ctx->tests[i].repository_info_tag); - guard_strlist_free(&ctx->tests[i].repository_remove_tags); - guard_free(ctx->tests[i].script); - guard_free(ctx->tests[i].script_setup); - guard_free(ctx->tests[i].build_recipe); + for (size_t i = 0; ctx->tests && i < ctx->tests->num_used; i++) { + guard_free(ctx->tests->test[i]->name); + guard_free(ctx->tests->test[i]->version); + guard_free(ctx->tests->test[i]->repository); + guard_free(ctx->tests->test[i]->repository_info_ref); + guard_free(ctx->tests->test[i]->repository_info_tag); + guard_strlist_free(&ctx->tests->test[i]->repository_remove_tags); + guard_free(ctx->tests->test[i]->script); + guard_free(ctx->tests->test[i]->script_setup); + guard_free(ctx->tests->test[i]->build_recipe); // test-specific runtime variables - guard_runtime_free(ctx->tests[i].runtime.environ); + guard_runtime_free(ctx->tests->test[i]->runtime->environ); + guard_free(ctx->tests->test[i]->runtime); + guard_free(ctx->tests->test[i]); + } + if (ctx->tests) { + guard_free(ctx->tests->test); + guard_free(ctx->tests); } guard_free(ctx->rules.release_fmt); @@ -388,8 +394,8 @@ void delivery_defer_packages(struct Delivery *ctx, int type) { msg(STASIS_MSG_L3, "package '%s': ", package_name); // When spec is present in name, set tests->version to the version detected in the name - for (size_t x = 0; x < sizeof(ctx->tests) / sizeof(ctx->tests[0]) && ctx->tests[x].name != NULL; x++) { - struct Test *test = &ctx->tests[x]; + for (size_t x = 0; x < ctx->tests->num_used; x++) { + struct Test *test = ctx->tests->test[x]; char nametmp[1024] = {0}; strncpy(nametmp, package_name, sizeof(nametmp) - 1); diff --git a/src/lib/delivery/delivery_build.c b/src/lib/delivery/delivery_build.c index 8370e6d..0013e96 100644 --- a/src/lib/delivery/delivery_build.c +++ b/src/lib/delivery/delivery_build.c @@ -1,11 +1,11 @@ #include "delivery.h" int delivery_build_recipes(struct Delivery *ctx) { - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { + for (size_t i = 0; i < ctx->tests->num_used; i++) { char *recipe_dir = NULL; - if (ctx->tests[i].build_recipe) { // build a conda recipe - if (recipe_clone(ctx->storage.build_recipes_dir, ctx->tests[i].build_recipe, NULL, &recipe_dir)) { - fprintf(stderr, "Encountered an issue while cloning recipe for: %s\n", ctx->tests[i].name); + if (ctx->tests->test[i]->build_recipe) { // build a conda recipe + if (recipe_clone(ctx->storage.build_recipes_dir, ctx->tests->test[i]->build_recipe, NULL, &recipe_dir)) { + fprintf(stderr, "Encountered an issue while cloning recipe for: %s\n", ctx->tests->test[i]->name); return -1; } if (!recipe_dir) { @@ -15,29 +15,48 @@ int delivery_build_recipes(struct Delivery *ctx) { int recipe_type = recipe_get_type(recipe_dir); if(!pushd(recipe_dir)) { if (RECIPE_TYPE_ASTROCONDA == recipe_type) { - pushd(path_basename(ctx->tests[i].repository)); + pushd(path_basename(ctx->tests->test[i]->repository)); } else if (RECIPE_TYPE_CONDA_FORGE == recipe_type) { pushd("recipe"); } - char recipe_version[100]; - char recipe_buildno[100]; + char recipe_version[200]; + char recipe_buildno[200]; char recipe_git_url[PATH_MAX]; char recipe_git_rev[PATH_MAX]; + char tag[100] = {0}; + if (ctx->tests->test[i]->repository_info_tag) { + const int is_long_tag = num_chars(ctx->tests->test[i]->repository_info_tag, '-') > 1; + if (is_long_tag) { + const size_t len = strcspn(ctx->tests->test[i]->repository_info_tag, "-"); + strncpy(tag, ctx->tests->test[i]->repository_info_tag, len); + tag[len] = '\0'; + } else { + strncpy(tag, ctx->tests->test[i]->repository_info_tag, sizeof(tag) - 1); + tag[strlen(ctx->tests->test[i]->repository_info_tag)] = '\0'; + } + } else { + strcpy(tag, ctx->tests->test[i]->version); + } + //sprintf(recipe_version, "{%% set version = GIT_DESCRIBE_TAG ~ \".dev\" ~ GIT_DESCRIBE_NUMBER ~ \"+\" ~ GIT_DESCRIBE_HASH %%}"); - //sprintf(recipe_git_url, " git_url: %s", ctx->tests[i].repository); - //sprintf(recipe_git_rev, " git_rev: %s", ctx->tests[i].version); + //sprintf(recipe_git_url, " git_url: %s", ctx->tests->test[i]->repository); + //sprintf(recipe_git_rev, " git_rev: %s", ctx->tests->test[i]->version); // TODO: Conditionally download archives if github.com is the origin. Else, use raw git_* keys ^^^ - sprintf(recipe_version, "{%% set version = \"%s\" %%}", ctx->tests[i].repository_info_tag ? ctx->tests[i].repository_info_tag : ctx->tests[i].version); - sprintf(recipe_git_url, " url: %s/archive/refs/tags/{{ version }}.tar.gz", ctx->tests[i].repository); + // 03/2026 - How can we know if the repository URL supports archive downloads? + // Perhaps we can key it to the recipe type, because the archive is a requirement imposed + // by conda-forge. Hmm. + + sprintf(recipe_version, "{%% set version = \"%s\" %%}", tag); + sprintf(recipe_git_url, " url: %s/archive/refs/tags/{{ version }}.tar.gz", ctx->tests->test[i]->repository); strcpy(recipe_git_rev, ""); sprintf(recipe_buildno, " number: 0"); unsigned flags = REPLACE_TRUNCATE_AFTER_MATCH; //file_replace_text("meta.yaml", "{% set version = ", recipe_version); if (ctx->meta.final) { // remove this. i.e. statis cannot deploy a release to conda-forge - sprintf(recipe_version, "{%% set version = \"%s\" %%}", ctx->tests[i].version); + sprintf(recipe_version, "{%% set version = \"%s\" %%}", ctx->tests->test[i]->version); // TODO: replace sha256 of tagged archive // TODO: leave the recipe unchanged otherwise. in theory this should produce the same conda package hash as conda forge. // For now, remove the sha256 requirement @@ -127,7 +146,232 @@ int filter_repo_tags(char *repo, struct StrList *patterns) { return result; } +static int read_without_line_endings(const size_t line, char ** arg) { + (void) line; + if (*arg) { + strip(*arg); + if (isempty(*arg)) { + return 1; // skip + } + } + return 0; +} + +int manylinux_exec(const char *image, const char *script, const char *copy_to_container_dir, const char *copy_from_container_dir, const char *copy_to_host_dir) { + int result = -1; // fail by default + char *container_name = NULL; + char *source_copy_command = NULL; + char *copy_command = NULL; + char *rm_command = NULL; + char *nop_create_command = NULL; + char *nop_rm_command = NULL; + char *volume_rm_command = NULL; + char *find_command = NULL; + char *wheel_paths_filename = NULL; + char *args = NULL; + + const uid_t uid = geteuid(); + char suffix[7] = {0}; + + // setup + + if (get_random_bytes(suffix, sizeof(suffix))) { + SYSERROR("%s", "unable to acquire value from random generator"); + goto manylinux_fail; + } + + if (asprintf(&container_name, "manylinux_build_%d_%zd_%s", uid, time(NULL), suffix) < 0) { + SYSERROR("%s", "unable to allocate memory for container name"); + goto manylinux_fail; + } + + if (asprintf(&args, "--name %s -w /build -v %s:/build", container_name, container_name) < 0) { + SYSERROR("%s", "unable to allocate memory for docker arguments"); + goto manylinux_fail; + } + + if (!strstr(image, "manylinux")) { + SYSERROR("expected a manylinux image, but got %s", image); + goto manylinux_fail; + } + + if (asprintf(&nop_create_command, "run --name nop_%s -v %s:/build busybox", container_name, container_name) < 0) { + SYSERROR("%s", "unable to allocate memory for nop container command"); + goto manylinux_fail; + } + + if (asprintf(&source_copy_command, "cp %s nop_%s:/build", copy_to_container_dir, container_name) < 0) { + SYSERROR("%s", "unable to allocate memory for source copy command"); + goto manylinux_fail; + } + + if (asprintf(&nop_rm_command, "rm nop_%s", container_name) < 0) { + SYSERROR("%s", "unable to allocate memory for nop container command"); + goto manylinux_fail; + } + + if (asprintf(&wheel_paths_filename, "%s/wheel_paths_%s.txt", globals.tmpdir, container_name) < 0) { + SYSERROR("%s", "unable to allocate memory for wheel paths file name"); + goto manylinux_fail; + } + + if (asprintf(&find_command, "run --rm -t -v %s:/build busybox sh -c 'find %s -name \"*.whl\"' > %s", container_name, copy_from_container_dir, wheel_paths_filename) < 0) { + SYSERROR("%s", "unable to allocate memory for find command"); + goto manylinux_fail; + } + + // execute + + if (docker_exec(nop_create_command, 0)) { + SYSERROR("%s", "docker nop container creation failed"); + goto manylinux_fail; + } + + if (docker_exec(source_copy_command, 0)) { + SYSERROR("%s", "docker source copy operation failed"); + goto manylinux_fail; + } + + if (docker_exec(nop_rm_command, STASIS_DOCKER_QUIET)) { + SYSERROR("%s", "docker nop container removal failed"); + goto manylinux_fail; + } + + if (docker_script(image, args, (char *) script, 0)) { + SYSERROR("%s", "manylinux execution failed"); + goto manylinux_fail; + } + + if (docker_exec(find_command, 0)) { + SYSERROR("%s", "docker find command failed"); + goto manylinux_fail; + } + + struct StrList *wheel_paths = strlist_init(); + if (!wheel_paths) { + SYSERROR("%s", "wheel_paths not initialized"); + goto manylinux_fail; + } + + if (strlist_append_file(wheel_paths, wheel_paths_filename, read_without_line_endings)) { + SYSERROR("%s", "wheel_paths append failed"); + goto manylinux_fail; + } + + for (size_t i = 0; i < strlist_count(wheel_paths); i++) { + const char *item = strlist_item(wheel_paths, i); + if (asprintf(©_command, "cp %s:%s %s", container_name, item, copy_to_host_dir) < 0) { + SYSERROR("%s", "unable to allocate memory for docker copy command"); + goto manylinux_fail; + } + + if (docker_exec(copy_command, 0)) { + SYSERROR("%s", "docker copy operation failed"); + goto manylinux_fail; + } + guard_free(copy_command); + } + + // Success + result = 0; + + manylinux_fail: + if (wheel_paths_filename) { + remove(wheel_paths_filename); + } + + if (container_name) { + // Keep going on failure unless memory related. + // We don't want build debris everywhere. + if (asprintf(&rm_command, "rm %s", container_name) < 0) { + SYSERROR("%s", "unable to allocate memory for rm command"); + goto late_fail; + } + + if (docker_exec(rm_command, STASIS_DOCKER_QUIET)) { + SYSERROR("%s", "docker container removal operation failed"); + } + + if (asprintf(&volume_rm_command, "volume rm -f %s", container_name) < 0) { + SYSERROR("%s", "unable to allocate memory for docker volume removal command"); + goto late_fail; + } + + if (docker_exec(volume_rm_command, STASIS_DOCKER_QUIET)) { + SYSERROR("%s", "docker volume removal operation failed"); + } + } + + late_fail: + guard_free(container_name); + guard_free(args); + guard_free(copy_command); + guard_free(rm_command); + guard_free(volume_rm_command); + guard_free(source_copy_command); + guard_free(nop_create_command); + guard_free(nop_rm_command); + guard_free(find_command); + guard_free(wheel_paths_filename); + guard_strlist_free(&wheel_paths); + return result; +} + +int delivery_build_wheels_manylinux(struct Delivery *ctx, const char *outdir) { + msg(STASIS_MSG_L1, "Building wheels\n"); + + const char *manylinux_image = globals.wheel_builder_manylinux_image; + if (!manylinux_image) { + SYSERROR("%s", "manylinux_image not initialized"); + return -1; + } + + int manylinux_build_status = 0; + + msg(STASIS_MSG_L2, "Using: %s\n", manylinux_image); + const struct Meta *meta = &ctx->meta; + const char *script_fmt = + "set -e -x\n" + "git config --global --add safe.directory /build\n" + "python%s -m pip install auditwheel build\n" + "python%s -m build -w .\n" + "auditwheel show dist/*.whl\n" + "auditwheel repair --allow-pure-python-wheel dist/*.whl\n"; + char *script = NULL; + if (asprintf(&script, script_fmt, + meta->python, meta->python) < 0) { + SYSERROR("%s", "unable to allocate memory for build script"); + return -1; + } + manylinux_build_status = manylinux_exec( + manylinux_image, + script, + "./", + "/build/wheelhouse", + outdir); + + if (manylinux_build_status) { + msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "manylinux build failed (%d)", manylinux_build_status); + guard_free(script); + return -1; + } + guard_free(script); + return 0; +} + struct StrList *delivery_build_wheels(struct Delivery *ctx) { + const int on_linux = strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Linux") == 0; + const int docker_usable = ctx->deploy.docker.capabilities.usable; + int use_builder_build = strcmp(globals.wheel_builder, "native") == 0; + const int use_builder_cibuildwheel = strcmp(globals.wheel_builder, "cibuildwheel") == 0 && on_linux && docker_usable; + const int use_builder_manylinux = strcmp(globals.wheel_builder, "manylinux") == 0 && on_linux && docker_usable; + + if (!use_builder_build && !use_builder_cibuildwheel && !use_builder_manylinux) { + msg(STASIS_MSG_WARN, "Cannot build wheel for platform using: %\n", globals.wheel_builder); + msg(STASIS_MSG_WARN, "Falling back to native toolchain.\n", globals.wheel_builder); + use_builder_build = 1; + } + struct StrList *result = NULL; struct Process proc = {0}; @@ -147,22 +391,28 @@ struct StrList *delivery_build_wheels(struct Delivery *ctx) { *spec = '\0'; } - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { - if ((ctx->tests[i].name && !strcmp(name, ctx->tests[i].name)) && (!ctx->tests[i].build_recipe && ctx->tests[i].repository)) { // build from source + for (size_t i = 0; i < ctx->tests->num_used; i++) { + if ((ctx->tests->test[i]->name && !strcmp(name, ctx->tests->test[i]->name)) && (!ctx->tests->test[i]->build_recipe && ctx->tests->test[i]->repository)) { // build from source char srcdir[PATH_MAX]; char wheeldir[PATH_MAX]; memset(srcdir, 0, sizeof(srcdir)); memset(wheeldir, 0, sizeof(wheeldir)); - sprintf(srcdir, "%s/%s", ctx->storage.build_sources_dir, ctx->tests[i].name); - if (git_clone(&proc, ctx->tests[i].repository, srcdir, ctx->tests[i].version)) { + sprintf(srcdir, "%s/%s", ctx->storage.build_sources_dir, ctx->tests->test[i]->name); + if (git_clone(&proc, ctx->tests->test[i]->repository, srcdir, ctx->tests->test[i]->version)) { SYSERROR("Unable to checkout tag '%s' for package '%s' from repository '%s'\n", - ctx->tests[i].version, ctx->tests[i].name, ctx->tests[i].repository); + ctx->tests->test[i]->version, ctx->tests->test[i]->name, ctx->tests->test[i]->repository); return NULL; } - if (ctx->tests[i].repository_remove_tags && strlist_count(ctx->tests[i].repository_remove_tags)) { - filter_repo_tags(srcdir, ctx->tests[i].repository_remove_tags); + if (!ctx->tests->test[i]->repository_info_tag) { + ctx->tests->test[i]->repository_info_tag = strdup(git_describe(srcdir)); + } + if (!ctx->tests->test[i]->repository_info_ref) { + ctx->tests->test[i]->repository_info_ref = strdup(git_rev_parse(srcdir, ctx->tests->test[i]->version)); + } + if (ctx->tests->test[i]->repository_remove_tags && strlist_count(ctx->tests->test[i]->repository_remove_tags)) { + filter_repo_tags(srcdir, ctx->tests->test[i]->repository_remove_tags); } if (!pushd(srcdir)) { @@ -184,7 +434,7 @@ struct StrList *delivery_build_wheels(struct Delivery *ctx) { COE_CHECK_ABORT(dep_status, "Unreproducible delivery"); } - strcpy(dname, ctx->tests[i].name); + strcpy(dname, ctx->tests->test[i]->name); tolower_s(dname); sprintf(outdir, "%s/%s", ctx->storage.wheel_artifact_dir, dname); if (mkdirs(outdir, 0755)) { @@ -192,28 +442,41 @@ struct StrList *delivery_build_wheels(struct Delivery *ctx) { guard_strlist_free(&result); return NULL; } - - if (asprintf(&cmd, "-m build -w -o %s", outdir) < 0) { - SYSERROR("%s", "Unable to allocate memory for build command"); - return NULL; - } - if (!strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Linux") - && ctx->deploy.docker.capabilities.usable) { - guard_free(cmd); - if (asprintf(&cmd, "-m cibuildwheel --output-dir %s --only cp%s-manylinux_%s", - outdir, ctx->meta.python_compact, ctx->system.arch) < 0) { - SYSERROR("%s", "Unable to allocate memory for cibuildwheel command"); + if (use_builder_manylinux) { + if (delivery_build_wheels_manylinux(ctx, outdir)) { + fprintf(stderr, "failed to generate wheel package for %s-%s\n", ctx->tests->test[i]->name, + ctx->tests->test[i]->version); + guard_strlist_free(&result); + guard_free(cmd); return NULL; } - } + } else if (use_builder_build || use_builder_cibuildwheel) { + if (use_builder_build) { + if (asprintf(&cmd, "-m build -w -o %s", outdir) < 0) { + SYSERROR("%s", "Unable to allocate memory for build command"); + return NULL; + } + } else if (use_builder_cibuildwheel) { + if (asprintf(&cmd, "-m cibuildwheel --output-dir %s --only cp%s-manylinux_%s", + outdir, ctx->meta.python_compact, ctx->system.arch) < 0) { + SYSERROR("%s", "Unable to allocate memory for cibuildwheel command"); + return NULL; + } + } - if (python_exec(cmd)) { - fprintf(stderr, "failed to generate wheel package for %s-%s\n", ctx->tests[i].name, - ctx->tests[i].version); - guard_strlist_free(&result); - guard_free(cmd); + if (python_exec(cmd)) { + fprintf(stderr, "failed to generate wheel package for %s-%s\n", ctx->tests->test[i]->name, + ctx->tests->test[i]->version); + guard_strlist_free(&result); + guard_free(cmd); + return NULL; + } + } else { + SYSERROR("unknown wheel builder backend: %s", globals.wheel_builder); return NULL; } + + guard_free(cmd); popd(); } else { fprintf(stderr, "Unable to enter source directory %s: %s\n", srcdir, strerror(errno)); @@ -225,4 +488,3 @@ struct StrList *delivery_build_wheels(struct Delivery *ctx) { } return result; } - diff --git a/src/lib/delivery/delivery_docker.c b/src/lib/delivery/delivery_docker.c index 57015ad..2c43caf 100644 --- a/src/lib/delivery/delivery_docker.c +++ b/src/lib/delivery/delivery_docker.c @@ -111,7 +111,7 @@ int delivery_docker(struct Delivery *ctx) { msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "Image test script has no content\n"); } else { int state; - if ((state = docker_script(tag, ctx->deploy.docker.test_script, 0))) { + if ((state = docker_script(tag, "--rm", ctx->deploy.docker.test_script, 0))) { msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "Non-zero exit (%d) from test script. %s image archive will not be generated.\n", state >> 8, tag); // test failed -- don't save the image return -1; diff --git a/src/lib/delivery/delivery_init.c b/src/lib/delivery/delivery_init.c index a60d6af..1666f0a 100644 --- a/src/lib/delivery/delivery_init.c +++ b/src/lib/delivery/delivery_init.c @@ -119,7 +119,7 @@ void delivery_init_dirs_stage1(struct Delivery *ctx) { } if (access(ctx->storage.mission_dir, F_OK)) { - msg(STASIS_MSG_L1, "%s: %s\n", ctx->storage.mission_dir, strerror(errno)); + msg(STASIS_MSG_L1, "%s: %s: mission directory does not exist\n", ctx->storage.mission_dir, strerror(errno)); exit(1); } @@ -150,7 +150,7 @@ void delivery_init_dirs_stage1(struct Delivery *ctx) { } int delivery_init_platform(struct Delivery *ctx) { - msg(STASIS_MSG_L2, "Setting architecture\n"); + SYSDEBUG("%s", "Setting architecture"); char archsuffix[20]; struct utsname uts; if (uname(&uts)) { @@ -179,7 +179,7 @@ int delivery_init_platform(struct Delivery *ctx) { strcpy(archsuffix, ctx->system.arch); } - msg(STASIS_MSG_L2, "Setting platform\n"); + SYSDEBUG("%s", "Setting platform"); strcpy(ctx->system.platform[DELIVERY_PLATFORM], uts.sysname); if (!strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Darwin")) { sprintf(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], "osx-%s", archsuffix); @@ -287,6 +287,8 @@ int delivery_init(struct Delivery *ctx, int render_mode) { int bootstrap_build_info(struct Delivery *ctx) { struct Delivery local = {0}; + memcpy(&local.deploy.docker.capabilities, &ctx->deploy.docker.capabilities, sizeof(local.deploy.docker.capabilities)); + SYSDEBUG("ini_open(%s)", ctx->_stasis_ini_fp.cfg_path); local._stasis_ini_fp.cfg = ini_open(ctx->_stasis_ini_fp.cfg_path); SYSDEBUG("ini_open(%s)", ctx->_stasis_ini_fp.delivery_path); @@ -294,18 +296,22 @@ int bootstrap_build_info(struct Delivery *ctx) { if (delivery_init_platform(&local)) { SYSDEBUG("%s", "delivery_init_platform failed"); + delivery_free(&local); return -1; } if (populate_delivery_cfg(&local, INI_READ_RENDER)) { SYSDEBUG("%s", "populate_delivery_cfg failed"); + delivery_free(&local); return -1; } if (populate_delivery_ini(&local, INI_READ_RENDER)) { SYSDEBUG("%s", "populate_delivery_ini failed"); + delivery_free(&local); return -1; } if (populate_info(&local)) { SYSDEBUG("%s", "populate_info failed"); + delivery_free(&local); return -1; } ctx->info.build_name = strdup(local.info.build_name); @@ -315,6 +321,7 @@ int bootstrap_build_info(struct Delivery *ctx) { ctx->info.time_info = malloc(sizeof(*ctx->info.time_info)); if (!ctx->info.time_info) { SYSERROR("Unable to allocate %zu bytes for tm struct: %s", sizeof(*local.info.time_info), strerror(errno)); + delivery_free(&local); return -1; } } diff --git a/src/lib/delivery/delivery_install.c b/src/lib/delivery/delivery_install.c index f1637a3..2de80cf 100644 --- a/src/lib/delivery/delivery_install.c +++ b/src/lib/delivery/delivery_install.c @@ -2,7 +2,7 @@ static struct Test *requirement_from_test(struct Delivery *ctx, const char *name) { struct Test *result = NULL; - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { + for (size_t i = 0; i < ctx->tests->num_used; i++) { char *package_name = strdup(name); if (package_name) { char *spec = find_version_spec(package_name); @@ -11,8 +11,8 @@ static struct Test *requirement_from_test(struct Delivery *ctx, const char *name } remove_extras(package_name); - if (ctx->tests[i].name && !strcmp(package_name, ctx->tests[i].name)) { - result = &ctx->tests[i]; + if (ctx->tests->test[i]->name && !strcmp(package_name, ctx->tests->test[i]->name)) { + result = ctx->tests->test[i]; guard_free(package_name); break; } @@ -252,7 +252,7 @@ int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, cha } strlist_append_tokenize(tag_data, info->repository_info_tag, "-"); - struct Wheel *whl = NULL; + struct WheelInfo *whl = NULL; char *post_commit = NULL; char *hash = NULL; if (strlist_count(tag_data) > 1) { @@ -264,7 +264,7 @@ int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, cha // equal to the tag; setuptools_scm auto-increments the value, the user can change it manually, // etc. errno = 0; - whl = get_wheel_info(ctx->storage.wheel_artifact_dir, info->name, + whl = wheelinfo_get(ctx->storage.wheel_artifact_dir, info->name, (char *[]) {ctx->meta.python_compact, ctx->system.arch, "none", "any", post_commit, hash, @@ -277,11 +277,12 @@ int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, cha // not found fprintf(stderr, "No wheel packages found that match the description of '%s'", info->name); } else { - // found + // found, replace the original version with newly detected version + guard_free(info->version); info->version = strdup(whl->version); } guard_strlist_free(&tag_data); - wheel_free(&whl); + wheelinfo_free(&whl); } char req[255] = {0}; diff --git a/src/lib/delivery/delivery_populate.c b/src/lib/delivery/delivery_populate.c index 15ab6bd..d41e3a4 100644 --- a/src/lib/delivery/delivery_populate.c +++ b/src/lib/delivery/delivery_populate.c @@ -85,6 +85,45 @@ int populate_delivery_cfg(struct Delivery *ctx, int render_mode) { } globals.pip_packages = ini_getval_strlist(cfg, "default", "pip_packages", LINE_SEP, render_mode, &err); + err = 0; + if (!globals.wheel_builder) { + globals.wheel_builder = ini_getval_str(cfg, "default", "wheel_builder", render_mode, &err); + if (err) { + msg(STASIS_MSG_WARN, "wheel_builder is undefined. Falling back to system toolchain: 'build'.\n"); + globals.wheel_builder = strdup("build"); + if (!globals.wheel_builder) { + SYSERROR("%s", "unable to allocate memory for default wheel_builder value"); + return -1; + } + } + } + + err = 0; + if (!globals.wheel_builder_manylinux_image) { + globals.wheel_builder_manylinux_image = ini_getval_str(cfg, "default", "wheel_builder_manylinux_image", render_mode, &err); + } + + if (err && globals.wheel_builder && strcmp(globals.wheel_builder, "manylinux") == 0) { + SYSERROR("%s", "default:wheel_builder is set to 'manylinux', however default:wheel_builder_manylinux_image is not configured"); + return -1; + } + + if (strcmp(globals.wheel_builder, "manylinux") == 0) { + char *manifest_inspect_cmd = NULL; + if (asprintf(&manifest_inspect_cmd, "manifest inspect '%s'", globals.wheel_builder_manylinux_image) < 0) { + SYSERROR("%s", "unable to allocate memory for docker command"); + guard_free(manifest_inspect_cmd); + return -1; + } + if (ctx->deploy.docker.capabilities.usable && docker_exec(manifest_inspect_cmd, STASIS_DOCKER_QUIET_STDOUT)) { + SYSERROR("Image provided by default:wheel_builder_manylinux_image does not exist: %s", globals.wheel_builder_manylinux_image); + guard_free(manifest_inspect_cmd); + return -1; + } + guard_free(manifest_inspect_cmd); + } + + if (globals.jfrog.jfrog_artifactory_base_url) { guard_free(globals.jfrog.jfrog_artifactory_base_url); } @@ -154,6 +193,7 @@ static void normalize_ini_list(struct INIFILE **inip, struct StrList **listp, ch (*inip) = ini; (*listp) = list; } + int populate_delivery_ini(struct Delivery *ctx, int render_mode) { struct INIFILE *ini = ctx->_stasis_ini_fp.delivery; struct INIData *rtdata; @@ -200,7 +240,9 @@ int populate_delivery_ini(struct Delivery *ctx, int render_mode) { normalize_ini_list(&ini, &ctx->conda.pip_packages_purge, "conda", "pip_packages_purge", render_mode); // Delivery metadata consumed - populate_mission_ini(&ctx, render_mode); + if (populate_mission_ini(&ctx, render_mode)) { + return -1; + } if (ctx->info.release_name) { guard_free(ctx->info.release_name); @@ -236,11 +278,17 @@ int populate_delivery_ini(struct Delivery *ctx, int render_mode) { ctx->conda.pip_packages_defer = strlist_init(); } - for (size_t z = 0, i = 0; i < ini->section_count; i++) { + ctx->tests = tests_init(TEST_NUM_ALLOC_INITIAL); + for (size_t i = 0; i < ini->section_count; i++) { char *section_name = ini->section[i]->key; if (startswith(section_name, "test:")) { union INIVal val; - struct Test *test = &ctx->tests[z]; + struct Test *test = test_init(); + if (!test) { + SYSERROR("%s", "unable to allocate memory for test structure"); + return -1; + } + val.as_char_p = strchr(ini->section[i]->key, ':') + 1; if (val.as_char_p && isempty(val.as_char_p)) { return 1; @@ -258,7 +306,8 @@ int populate_delivery_ini(struct Delivery *ctx, int render_mode) { } test->repository_remove_tags = ini_getval_strlist(ini, section_name, "repository_remove_tags", LINE_SEP, render_mode, &err); test->build_recipe = ini_getval_str(ini, section_name, "build_recipe", render_mode, &err); - test->runtime.environ = ini_getval_strlist(ini, section_name, "runtime", LINE_SEP, render_mode, &err); + + test->runtime->environ = ini_getval_strlist(ini, section_name, "runtime", LINE_SEP, render_mode, &err); const char *timeout_str = ini_getval_str(ini, section_name, "timeout", render_mode, &err); if (timeout_str) { test->timeout = str_to_timeout((char *) timeout_str); @@ -271,7 +320,7 @@ int populate_delivery_ini(struct Delivery *ctx, int render_mode) { return 1; } } - z++; + tests_add(ctx->tests, test); } } @@ -320,6 +369,7 @@ int populate_mission_ini(struct Delivery **ctx, int render_mode) { int err = 0; if ((*ctx)->_stasis_ini_fp.mission) { + // mission configurations are optional return 0; } @@ -333,12 +383,12 @@ int populate_mission_ini(struct Delivery **ctx, int render_mode) { globals.sysconfdir, "mission", (*ctx)->meta.mission, (*ctx)->meta.mission); } - msg(STASIS_MSG_L2, "Reading mission configuration: %s\n", missionfile); + SYSDEBUG("Reading mission configuration: %s\n", missionfile); (*ctx)->_stasis_ini_fp.mission = ini_open(missionfile); struct INIFILE *ini = (*ctx)->_stasis_ini_fp.mission; if (!ini) { msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read mission configuration: %s, %s\n", missionfile, strerror(errno)); - exit(1); + return -1; } (*ctx)->_stasis_ini_fp.mission_path = strdup(missionfile); diff --git a/src/lib/delivery/delivery_postprocess.c b/src/lib/delivery/delivery_postprocess.c index 5029e02..a7bb2b4 100644 --- a/src/lib/delivery/delivery_postprocess.c +++ b/src/lib/delivery/delivery_postprocess.c @@ -28,7 +28,7 @@ int delivery_dump_metadata(struct Delivery *ctx) { return -1; } if (globals.verbose) { - printf("%s\n", filename); + msg(STASIS_MSG_L2, "%s\n", filename); } fprintf(fp, "name %s\n", ctx->meta.name); fprintf(fp, "version %s\n", ctx->meta.version); diff --git a/src/lib/delivery/delivery_show.c b/src/lib/delivery/delivery_show.c index adfa1be..f4ac825 100644 --- a/src/lib/delivery/delivery_show.c +++ b/src/lib/delivery/delivery_show.c @@ -84,13 +84,13 @@ void delivery_conda_show(struct Delivery *ctx) { void delivery_tests_show(struct Delivery *ctx) { printf("\n====TESTS====\n"); - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { - if (!ctx->tests[i].name) { + for (size_t i = 0; i < ctx->tests->num_used; i++) { + if (!ctx->tests->test[i]->name) { continue; } - printf("%-20s %-20s %s\n", ctx->tests[i].name, - ctx->tests[i].version, - ctx->tests[i].repository); + printf("%-20s %-20s %s\n", ctx->tests->test[i]->name, + ctx->tests->test[i]->version, + ctx->tests->test[i]->repository); } } diff --git a/src/lib/delivery/delivery_test.c b/src/lib/delivery/delivery_test.c index 500ade9..3ba9d56 100644 --- a/src/lib/delivery/delivery_test.c +++ b/src/lib/delivery/delivery_test.c @@ -1,5 +1,59 @@ #include "delivery.h" +struct Tests *tests_init(const size_t num_tests) { + struct Tests *tests = calloc(1, sizeof(*tests)); + if (!tests) { + return NULL; + } + + tests->test = calloc(num_tests, sizeof(*tests->test)); + if (!tests->test) { + return NULL; + } + tests->num_used = 0; + tests->num_alloc = num_tests; + + return tests; +} + +int tests_add(struct Tests *tests, struct Test *x) { + if (tests->num_used >= tests->num_alloc) { +#ifdef DEBUG + const size_t old_alloc = tests->num_alloc; +#endif + struct Test **tmp = realloc(tests->test, tests->num_alloc++ * sizeof(*tests->test)); + SYSDEBUG("Increasing size of test array: %zu -> %zu", old_alloc, tests->num_alloc); + if (!tmp) { + SYSDEBUG("Failed to allocate %zu bytes for test array", tests->num_alloc * sizeof(*tests->test)); + return -1; + } + tests->test = tmp; + } + + SYSDEBUG("Adding test: '%s'", x->name); + tests->test[tests->num_used++] = x; + return 0; +} + +struct Test *test_init() { + struct Test *result = calloc(1, sizeof(*result)); + result->runtime = calloc(1, sizeof(*result->runtime)); + + return result; +} + +void test_free(struct Test **x) { + struct Test *test = *x; + guard_free(test); +} + +void tests_free(struct Tests **x) { + for (size_t i = 0; i < (*x)->num_alloc; i++) { + test_free(&(*x)->test[i]); + } + guard_free((*x)->test); +} + void delivery_tests_run(struct Delivery *ctx) { static const int SETUP = 0; static const int PARALLEL = 1; @@ -16,7 +70,7 @@ void delivery_tests_run(struct Delivery *ctx) { // amount of debug information. snprintf(globals.workaround.conda_reactivate, PATH_MAX - 1, "\nset +x; mamba activate ${CONDA_DEFAULT_ENV}; set -x\n"); - if (!ctx->tests[0].name) { + if (!ctx->tests || !ctx->tests->num_used) { msg(STASIS_MSG_WARN | STASIS_MSG_L2, "no tests are defined!\n"); } else { pool[PARALLEL] = mp_pool_init("parallel", ctx->storage.tmpdir); @@ -60,8 +114,8 @@ void delivery_tests_run(struct Delivery *ctx) { // Iterate over our test records, retrieving the source code for each package, and assigning its scripted tasks // to the appropriate processing pool - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { - struct Test *test = &ctx->tests[i]; + for (size_t i = 0; i < ctx->tests->num_used; i++) { + struct Test *test = ctx->tests->test[i]; if (!test->name && !test->repository && !test->script) { // skip unused test records continue; @@ -181,8 +235,8 @@ void delivery_tests_run(struct Delivery *ctx) { // Configure "script_setup" tasks // Directories should exist now, so no need to go through initializing everything all over again. - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { - struct Test *test = &ctx->tests[i]; + for (size_t i = 0; i < ctx->tests->num_used; i++) { + const struct Test *test = ctx->tests->test[i]; if (test->script_setup) { char destdir[PATH_MAX]; sprintf(destdir, "%s/%s", ctx->storage.build_sources_dir, path_basename(test->repository)); diff --git a/src/lib/delivery/include/delivery.h b/src/lib/delivery/include/delivery.h index cae4b02..b5799ac 100644 --- a/src/lib/delivery/include/delivery.h +++ b/src/lib/delivery/include/delivery.h @@ -19,6 +19,8 @@ #include "multiprocessing.h" #include "recipe.h" #include "wheel.h" +#include "wheelinfo.h" +#include "environment.h" #define DELIVERY_PLATFORM_MAX 4 #define DELIVERY_PLATFORM_MAXLEN 65 @@ -44,6 +46,28 @@ struct Content { char *data; }; +//! Number of test records to allocate (grows dynamically) +#define TEST_NUM_ALLOC_INITIAL 10 + +/*! \struct Test + * \brief Test information + */ +struct Test { + char *name; ///< Name of package + char *version; ///< Version of package + char *repository; ///< Git repository of package + char *script_setup; ///< Commands to execute before the main script + char *script; ///< Commands to execute + bool disable; ///< Toggle a test block + bool parallel; ///< Toggle parallel or serial execution + char *build_recipe; ///< Conda recipe to build (optional) + char *repository_info_ref; ///< Git commit hash + char *repository_info_tag; ///< Git tag (first parent) + struct StrList *repository_remove_tags; ///< Git tags to remove (to fix duplicate commit tags) + struct Runtime *runtime; ///< Environment variables specific to the test context + int timeout; ///< Timeout in seconds +}; ///< An array of tests + /*! \struct Delivery * \brief A structure describing a full delivery object */ @@ -153,24 +177,11 @@ struct Delivery { RuntimeEnv *environ; ///< Environment variables } runtime; - /*! \struct Test - * \brief Test information - */ - struct Test { - char *name; ///< Name of package - char *version; ///< Version of package - char *repository; ///< Git repository of package - char *script_setup; ///< Commands to execute before the main script - char *script; ///< Commands to execute - bool disable; ///< Toggle a test block - bool parallel; ///< Toggle parallel or serial execution - char *build_recipe; ///< Conda recipe to build (optional) - char *repository_info_ref; ///< Git commit hash - char *repository_info_tag; ///< Git tag (first parent) - struct StrList *repository_remove_tags; ///< Git tags to remove (to fix duplicate commit tags) - struct Runtime runtime; ///< Environment variables specific to the test context - int timeout; ///< Timeout in seconds - } tests[1000]; ///< An array of tests + struct Tests { + struct Test **test; + size_t num_used; + size_t num_alloc; + } *tests; struct Deploy { struct JFRT_Auth jfrog_auth; @@ -489,4 +500,32 @@ void delivery_rewrite_stage2(struct Delivery *ctx, char *specfile); */ struct Delivery *delivery_duplicate(const struct Delivery *ctx); +/** + * Initialize a `Tests` structure + * @param num_tests number of test records + * @return a an initialized `Tests` structure + */ +struct Tests *tests_init(size_t num_tests); + +/** + * Add a `Test` structure to `Tests` + * @param tests list to add to + * @param x test to add to list + * @return 0=success, -1=error + */ +int tests_add(struct Tests *tests, struct Test *x); + +/** + * Free a `Test` structure + * @param x pointer to `Test` + */ +void test_free(struct Test **x); + +/** + * Initialize a `Test` structure + * @return an initialized `Test` structure + */ +struct Test *test_init(); + + #endif //STASIS_DELIVERY_H @@ -23,6 +23,18 @@ conda_packages = ; (list) Python packages to be installed/overridden in the base environment ;pip_packages = +; (string) Python wheel builder [Linux only] +; DEFAULT: system +; OPTIONS: +; system = Build using local system toolchain +; cibuildwheel = Build using cibuildwheel and docker +; manylinux = Build using manylinux and docker +wheel_builder = manylinux + +; (string) Manylinux image [Linux only] +; When wheel_builder is set to "manylinux", use the following image +wheel_builder_manylinux_image = quay.io/pypa/manylinux2014 + [jfrog_cli_download] url = https://releases.jfrog.io/artifactory product = jfrog-cli diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 08ef833..dd68231 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -44,7 +44,7 @@ foreach(source_file ${source_files}) add_test(${test_executable} ${test_executable}) set_tests_properties(${test_executable} PROPERTIES - TIMEOUT 240) + TIMEOUT 600) set_tests_properties(${test_executable} PROPERTIES SKIP_RETURN_CODE 127) diff --git a/tests/test_docker.c b/tests/test_docker.c index d60522f..b0cf381 100644 --- a/tests/test_docker.c +++ b/tests/test_docker.c @@ -41,7 +41,7 @@ void test_docker_build_and_script_and_save() { if (!pushd("test_docker_build")) { stasis_testing_write_ascii("Dockerfile", dockerfile_contents); STASIS_ASSERT(docker_build(".", "-t test_docker_build", cap_suite.build) == 0, "docker build test failed"); - STASIS_ASSERT(docker_script("test_docker_build", "uname -a", 0) == 0, "simple docker container script execution failed"); + STASIS_ASSERT(docker_script("test_docker_build", "--rm", "uname -a", 0) == 0, "simple docker container script execution failed"); STASIS_ASSERT(docker_save("test_docker_build", ".", STASIS_DOCKER_IMAGE_COMPRESSION) == 0, "saving a simple image failed"); STASIS_ASSERT(docker_exec("load < test_docker_build.tar.*", 0) == 0, "loading a simple image failed"); docker_exec("image rm -f test_docker_build", 0); diff --git a/tests/test_strlist.c b/tests/test_strlist.c index 47722c0..38343f4 100644 --- a/tests/test_strlist.c +++ b/tests/test_strlist.c @@ -200,6 +200,20 @@ void test_strlist_append_tokenize() { guard_strlist_free(&list); } +void test_strlist_appendf() { + const char *fmt = "%c %s %d"; + struct StrList *list; + list = strlist_init(); + const int len = strlist_appendf(NULL, fmt, 'a', "abc", strlen(fmt)); + STASIS_ASSERT(strlist_appendf(&list, fmt, 'a', "abc", strlen(fmt)) == len, "length of formatted string should be 7"); + const char *item = strlist_item(list, 0); + STASIS_ASSERT(item != NULL, "valid pointer expected, item should not be NULL"); + STASIS_ASSERT(strncmp(item, "a", 1) == 0, "first character should be 'a'"); + STASIS_ASSERT(strncmp(item + 2, "abc", 3) == 0, "string should be 'abc'"); + STASIS_ASSERT(strncmp(item + 6, "8", 1) == 0, "length of the raw format should be 8"); + guard_strlist_free(&list); +} + void test_strlist_copy() { struct StrList *list = strlist_init(); struct StrList *list_copy; @@ -628,6 +642,7 @@ void test_strlist_item_as_long_double() { int main(int argc, char *argv[]) { STASIS_TEST_BEGIN_MAIN(); STASIS_TEST_FUNC *tests[] = { + test_strlist_appendf, test_strlist_init, test_strlist_free, test_strlist_append, diff --git a/tests/test_utils.c b/tests/test_utils.c index 0e2eb7b..cfe79e0 100644 --- a/tests/test_utils.c +++ b/tests/test_utils.c @@ -213,17 +213,21 @@ void test_git_clone_and_describe() { // test git_describe is functional char *taginfo_none = git_describe("."); STASIS_ASSERT(taginfo_none != NULL, "should be a git hash, not NULL"); + puts(taginfo_none); + STASIS_ASSERT(is_git_sha(taginfo_none) == true, "not a git hash"); system("git tag -a 1.0.0 -m Mock"); system("git push --tags origin"); - char *taginfo = git_describe("."); + const char *taginfo = git_describe("."); + puts(taginfo); STASIS_ASSERT(taginfo != NULL, "should be 1.0.0, not NULL"); - STASIS_ASSERT(strcmp(taginfo, "1.0.0") == 0, "just-created tag was not described correctly"); + STASIS_ASSERT(startswith(taginfo, "1.0.0") == true, "just-created tag was not described correctly"); chdir(".."); char *taginfo_outer = git_describe(repo); + puts(taginfo_outer); STASIS_ASSERT(taginfo_outer != NULL, "should be 1.0.0, not NULL"); - STASIS_ASSERT(strcmp(taginfo_outer, "1.0.0") == 0, "just-created tag was not described correctly (out-of-dir invocation)"); + STASIS_ASSERT(startswith(taginfo_outer, "1.0.0") == true, "just-created tag was not described correctly (out-of-dir invocation)"); char *taginfo_bad = git_describe("abc1234_not_here_or_there"); STASIS_ASSERT(taginfo_bad == NULL, "a repository that shouldn't exist... exists and has a tag."); diff --git a/tests/test_wheel.c b/tests/test_wheel.c index 6818b22..1eabb1b 100644 --- a/tests/test_wheel.c +++ b/tests/test_wheel.c @@ -1,91 +1,216 @@ +#include "delivery.h" #include "testing.h" +#include "str.h" #include "wheel.h" -void test_get_wheel_file() { - struct testcase { - const char *filename; - struct Wheel expected; - }; - struct testcase tc[] = { - { - // Test for "build tags" - .filename = "btpackage-1.2.3-mytag-py2.py3-none-any.whl", - .expected = { - .file_name = "btpackage-1.2.3-mytag-py2.py3-none-any.whl", - .version = "1.2.3", - .distribution = "btpackage", - .build_tag = "mytag", - .platform_tag = "any", - .python_tag = "py2.py3", - .abi_tag = "none", - .path_name = ".", - } - }, - { - // Test for universal package format - .filename = "anypackage-1.2.3-py2.py3-none-any.whl", - .expected = { - .file_name = "anypackage-1.2.3-py2.py3-none-any.whl", - .version = "1.2.3", - .distribution = "anypackage", - .build_tag = NULL, - .platform_tag = "any", - .python_tag = "py2.py3", - .abi_tag = "none", - .path_name = ".", - } - }, - { - // Test for binary package format - .filename = "binpackage-1.2.3-cp311-cp311-linux_x86_64.whl", - .expected = { - .file_name = "binpackage-1.2.3-cp311-cp311-linux_x86_64.whl", - .version = "1.2.3", - .distribution = "binpackage", - .build_tag = NULL, - .platform_tag = "linux_x86_64", - .python_tag = "cp311", - .abi_tag = "cp311", - .path_name = ".", - } - }, - }; +char cwd_start[PATH_MAX]; +char cwd_workspace[PATH_MAX]; +int conda_is_installed = 0; +static char conda_prefix[PATH_MAX] = {0}; +struct Delivery ctx; +static const char *testpkg_filename = "testpkg/dist/testpkg-1.0.0-py3-none-any.whl"; + + +static void test_wheel_package() { + const char *filename = testpkg_filename; + struct Wheel *wheel = NULL; + int state = wheel_package(&wheel, filename); + STASIS_ASSERT(state != WHEEL_PACKAGE_E_ALLOC, "Cannot fail to allocate memory for package structure"); + STASIS_ASSERT(state != WHEEL_PACKAGE_E_GET, "Cannot fail to parse wheel"); + STASIS_ASSERT(state != WHEEL_PACKAGE_E_GET_METADATA, "Cannot fail to read wheel metadata"); + STASIS_ASSERT(state != WHEEL_PACKAGE_E_GET_RECORDS, "Cannot fail reading wheel path records"); + STASIS_ASSERT(state != WHEEL_PACKAGE_E_GET_ENTRY_POINT, "Cannot fail reading wheel entry points"); + STASIS_ASSERT(state == WHEEL_PACKAGE_E_SUCCESS, "Wheel file should be usable"); + STASIS_ASSERT(wheel != NULL, "wheel cannot be NULL"); + STASIS_ASSERT(wheel != NULL, "wheel_package failed to initialize wheel struct"); + STASIS_ASSERT(wheel->record != NULL, "Record cannot be NULL"); + STASIS_ASSERT(wheel->num_record > 0, "Record count cannot be zero"); + STASIS_ASSERT(wheel->tag != NULL, "Package tag list cannot be NULL"); + STASIS_ASSERT(wheel->generator != NULL, "Generator field cannot be NULL"); + STASIS_ASSERT(wheel->top_level != NULL, "Top level directory name cannot be NULL"); + STASIS_ASSERT(wheel->wheel_version != NULL, "Wheel version cannot be NULL"); + STASIS_ASSERT(wheel->metadata != NULL, "Metadata cannot be NULL"); + STASIS_ASSERT(wheel->metadata->name != NULL, "Metadata::name cannot be NULL"); + STASIS_ASSERT(wheel->metadata->version != NULL, "Metadata::version cannot be NULL"); + STASIS_ASSERT(wheel->metadata->metadata_version != NULL, "Metadata::version (of metadata) cannot be NULL"); + + // Implied test against key/id getters. If wheel_show_info segfaults, that functionality is broken. + STASIS_ASSERT(wheel_show_info(wheel) == 0, "wheel_show_info should never fail. Enum(s) might be out of sync with META_*_KEYS array(s)"); + + // Get data from DIST + const struct WheelValue dist_version = wheel_get_value_by_name(wheel, WHEEL_FROM_DIST, "Wheel-Version"); + STASIS_ASSERT(dist_version.type == WHEELVAL_STR, "Wheel dist version value must be a string"); + STASIS_ASSERT_FATAL(dist_version.data != NULL, "Wheel dist version value must not be NULL"); + STASIS_ASSERT(dist_version.count, "Wheel value must be populated"); + + // Get data from METADATA + const struct WheelValue meta_name = wheel_get_value_by_name(wheel, WHEEL_FROM_METADATA, "Metadata-Version"); + STASIS_ASSERT(meta_name.type == WHEELVAL_STR, "Wheel metadata version value must be a string"); + STASIS_ASSERT_FATAL(meta_name.data != NULL, "Wheel metadata version value must not be NULL"); + STASIS_ASSERT(meta_name.count, "Wheel metadata version value must be populated"); + + wheel_package_free(&wheel); + STASIS_ASSERT(wheel == NULL, "wheel struct should be NULL after free"); +} + +static void mock_python_package() { + const char *pyproject_toml_data = "[build-system]\n" + "requires = [\"setuptools >= 77.0.3\"]\n" + "build-backend = \"setuptools.build_meta\"\n" + "\n" + "[project]\n" + "name = \"testpkg\"\n" + "version = \"1.0.0\"\n" + "authors = [{name = \"STASIS Team\", email = \"stasis@not-a-real-domain.tld\"}]\n" + "description = \"A STASIS test package\"\n" + "readme = \"README.md\"\n" + "license = \"BSD-3-Clause\"\n" + "classifiers = [\"Programming Language :: Python :: 3\"]\n" + "\n" + "[project.urls]\n" + "Homepage = \"https://not-a-real-address.tld\"\n" + "Documentation = \"https://not-a-real-address.tld/docs\"\n" + "Repository = \"https://not-a-real-address.tld/repo.git\"\n" + "Issues = \"https://not-a-real-address.tld/tracker\"\n" + "Changelog = \"https://not-a-real-address.tld/changes\"\n"; + const char *readme = "# testpkg\n\nThis is a test package, for testing.\n"; - struct Wheel *doesnotexist = get_wheel_info("doesnotexist", "doesnotexist-0.0.1-py2.py3-none-any.whl", (char *[]) {"not", NULL}, WHEEL_MATCH_ANY); - STASIS_ASSERT(doesnotexist == NULL, "returned non-NULL on error"); - - for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) { - struct testcase *test = &tc[i]; - struct Wheel *wheel = get_wheel_info(".", test->expected.distribution, (char *[]) {(char *) test->expected.version, NULL}, WHEEL_MATCH_ANY); - STASIS_ASSERT(wheel != NULL, "result should not be NULL!"); - STASIS_ASSERT(wheel->file_name && strcmp(wheel->file_name, test->expected.file_name) == 0, "mismatched file name"); - STASIS_ASSERT(wheel->version && strcmp(wheel->version, test->expected.version) == 0, "mismatched version"); - STASIS_ASSERT(wheel->distribution && strcmp(wheel->distribution, test->expected.distribution) == 0, "mismatched distribution (package name)"); - STASIS_ASSERT(wheel->platform_tag && strcmp(wheel->platform_tag, test->expected.platform_tag) == 0, "mismatched platform tag ([platform]_[architecture])"); - STASIS_ASSERT(wheel->python_tag && strcmp(wheel->python_tag, test->expected.python_tag) == 0, "mismatched python tag (python version)"); - STASIS_ASSERT(wheel->abi_tag && strcmp(wheel->abi_tag, test->expected.abi_tag) == 0, "mismatched abi tag (python compatibility version)"); - if (wheel->build_tag) { - STASIS_ASSERT(strcmp(wheel->build_tag, test->expected.build_tag) == 0, - "mismatched build tag (optional arbitrary string)"); - } - wheel_free(&wheel); + mkdir("testpkg", 0755); + mkdir("testpkg/src", 0755); + mkdir("testpkg/src/testpkg", 0755); + if (touch("testpkg/src/testpkg/__init__.py")) { + fprintf(stderr, "unable to write __init__.py"); + exit(1); + } + if (touch("testpkg/README.md")) { + fprintf(stderr, "unable to write README.md"); + exit(1); + } + if (stasis_testing_write_ascii("testpkg/pyproject.toml", pyproject_toml_data)) { + perror("unable to write pyproject.toml"); + exit(1); + } + if (stasis_testing_write_ascii("testpkg/README.md", readme)) { + perror("unable to write readme"); + exit(1); + } + if (pip_exec("install build")) { + fprintf(stderr, "unable to install build tool using pip\n"); + exit(1); + } + if (python_exec("-m build -w ./testpkg")) { + fprintf(stderr, "unable build test package"); + exit(1); } } int main(int argc, char *argv[]) { STASIS_TEST_BEGIN_MAIN(); STASIS_TEST_FUNC *tests[] = { - test_get_wheel_file, + test_wheel_package, }; - // Create mock package directories, and files - mkdir("binpackage", 0755); - touch("binpackage/binpackage-1.2.3-cp311-cp311-linux_x86_64.whl"); - mkdir("anypackage", 0755); - touch("anypackage/anypackage-1.2.3-py2.py3-none-any.whl"); - mkdir("btpackage", 0755); - touch("btpackage/btpackage-1.2.3-mytag-py2.py3-none-any.whl"); + char ws[] = "workspace_XXXXXX"; + if (!mkdtemp(ws)) { + perror("mkdtemp"); + exit(1); + } + getcwd(cwd_start, sizeof(cwd_start) - 1); + mkdir(ws, 0755); + chdir(ws); + getcwd(cwd_workspace, sizeof(cwd_workspace) - 1); + + snprintf(conda_prefix, strlen(cwd_workspace) + strlen("conda") + 2, "%s/conda", cwd_workspace); + + const char *mockinidata = "[meta]\n" + "name = mock\n" + "version = 1.0.0\n" + "rc = 1\n" + "mission = generic\n" + "python = 3.11\n" + "[conda]\n" + "installer_name = Miniforge3\n" + "installer_version = 24.3.0-0\n" + "installer_platform = {{env:STASIS_CONDA_PLATFORM}}\n" + "installer_arch = {{env:STASIS_CONDA_ARCH}}\n" + "installer_baseurl = https://github.com/conda-forge/miniforge/releases/download/24.3.0-0\n"; + stasis_testing_write_ascii("mock.ini", mockinidata); + struct INIFILE *ini = ini_open("mock.ini"); + ctx._stasis_ini_fp.delivery = ini; + ctx._stasis_ini_fp.delivery_path = realpath("mock.ini", NULL); + + const char *sysconfdir = getenv("STASIS_SYSCONFDIR"); + globals.sysconfdir = strdup(sysconfdir ? sysconfdir : STASIS_SYSCONFDIR); + ctx.storage.root = strdup(cwd_workspace); + char *cfgfile = join((char *[]) {globals.sysconfdir, "stasis.ini", NULL}, "/"); + if (!cfgfile) { + perror("unable to create path to global config"); + exit(1); + } + + ctx._stasis_ini_fp.cfg = ini_open(cfgfile); + if (!ctx._stasis_ini_fp.cfg) { + fprintf(stderr, "unable to open config file, %s\n", cfgfile); + exit(1); + } + ctx._stasis_ini_fp.cfg_path = realpath(cfgfile, NULL); + if (!ctx._stasis_ini_fp.cfg_path) { + fprintf(stderr, "unable to determine absolute path of config, %s\n", cfgfile); + exit(1); + } + guard_free(cfgfile); + + setenv("LANG", "C", 1); + if (bootstrap_build_info(&ctx)) { + fprintf(stderr, "bootstrap_build_info failed\n"); + exit(1); + } + if (delivery_init(&ctx, INI_READ_RENDER)) { + fprintf(stderr, "delivery_init failed\n"); + exit(1); + } + + char *install_url = calloc(255, sizeof(install_url)); + delivery_get_conda_installer_url(&ctx, install_url); + delivery_get_conda_installer(&ctx, install_url); + delivery_install_conda(ctx.conda.installer_path, ctx.storage.conda_install_prefix); + guard_free(install_url); + + if (conda_activate(ctx.storage.conda_install_prefix, "base")) { + fprintf(stderr, "conda_activate failed\n"); + exit(1); + } + if (conda_exec("install -y boa conda-build")) { + fprintf(stderr, "conda_exec failed\n"); + exit(1); + } + if (conda_setup_headless()) { + fprintf(stderr, "conda_setup_headless failed\n"); + exit(1); + } + if (conda_env_create("testpkg", ctx.meta.python, NULL)) { + fprintf(stderr, "conda_env_create failed\n"); + exit(1); + } + if (conda_activate(ctx.storage.conda_install_prefix, "testpkg")) { + fprintf(stderr, "conda_activate failed\n"); + exit(1); + } + + mock_python_package(); STASIS_TEST_RUN(tests); + + if (chdir(cwd_start) < 0) { + fprintf(stderr, "chdir failed\n"); + exit(1); + } + if (rmtree(cwd_workspace)) { + perror(cwd_workspace); + } + delivery_free(&ctx); + globals_free(); + STASIS_TEST_END_MAIN(); + }
\ No newline at end of file diff --git a/tests/test_wheelinfo.c b/tests/test_wheelinfo.c new file mode 100644 index 0000000..1abbeac --- /dev/null +++ b/tests/test_wheelinfo.c @@ -0,0 +1,91 @@ +#include "testing.h" +#include "wheelinfo.h" + +void test_wheelinfo_get() { + struct testcase { + const char *filename; + struct WheelInfo expected; + }; + struct testcase tc[] = { + { + // Test for "build tags" + .filename = "btpackage-1.2.3-mytag-py2.py3-none-any.whl", + .expected = { + .file_name = "btpackage-1.2.3-mytag-py2.py3-none-any.whl", + .version = "1.2.3", + .distribution = "btpackage", + .build_tag = "mytag", + .platform_tag = "any", + .python_tag = "py2.py3", + .abi_tag = "none", + .path_name = ".", + } + }, + { + // Test for universal package format + .filename = "anypackage-1.2.3-py2.py3-none-any.whl", + .expected = { + .file_name = "anypackage-1.2.3-py2.py3-none-any.whl", + .version = "1.2.3", + .distribution = "anypackage", + .build_tag = NULL, + .platform_tag = "any", + .python_tag = "py2.py3", + .abi_tag = "none", + .path_name = ".", + } + }, + { + // Test for binary package format + .filename = "binpackage-1.2.3-cp311-cp311-linux_x86_64.whl", + .expected = { + .file_name = "binpackage-1.2.3-cp311-cp311-linux_x86_64.whl", + .version = "1.2.3", + .distribution = "binpackage", + .build_tag = NULL, + .platform_tag = "linux_x86_64", + .python_tag = "cp311", + .abi_tag = "cp311", + .path_name = ".", + } + }, + }; + + struct WheelInfo *doesnotexist = wheelinfo_get("doesnotexist", "doesnotexist-0.0.1-py2.py3-none-any.whl", (char *[]) {"not", NULL}, WHEEL_MATCH_ANY); + STASIS_ASSERT(doesnotexist == NULL, "returned non-NULL on error"); + + for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) { + struct testcase *test = &tc[i]; + struct WheelInfo *wheel = wheelinfo_get(".", test->expected.distribution, (char *[]) {(char *) test->expected.version, NULL}, WHEEL_MATCH_ANY); + STASIS_ASSERT(wheel != NULL, "result should not be NULL!"); + STASIS_ASSERT(wheel->file_name && strcmp(wheel->file_name, test->expected.file_name) == 0, "mismatched file name"); + STASIS_ASSERT(wheel->version && strcmp(wheel->version, test->expected.version) == 0, "mismatched version"); + STASIS_ASSERT(wheel->distribution && strcmp(wheel->distribution, test->expected.distribution) == 0, "mismatched distribution (package name)"); + STASIS_ASSERT(wheel->platform_tag && strcmp(wheel->platform_tag, test->expected.platform_tag) == 0, "mismatched platform tag ([platform]_[architecture])"); + STASIS_ASSERT(wheel->python_tag && strcmp(wheel->python_tag, test->expected.python_tag) == 0, "mismatched python tag (python version)"); + STASIS_ASSERT(wheel->abi_tag && strcmp(wheel->abi_tag, test->expected.abi_tag) == 0, "mismatched abi tag (python compatibility version)"); + if (wheel->build_tag) { + STASIS_ASSERT(strcmp(wheel->build_tag, test->expected.build_tag) == 0, + "mismatched build tag (optional arbitrary string)"); + } + wheelinfo_free(&wheel); + } +} + +int main(int argc, char *argv[]) { + STASIS_TEST_BEGIN_MAIN(); + STASIS_TEST_FUNC *tests[] = { + test_wheelinfo_get, + }; + + // Create mock package directories, and files + mkdir("binpackage", 0755); + touch("binpackage/binpackage-1.2.3-cp311-cp311-linux_x86_64.whl"); + mkdir("anypackage", 0755); + touch("anypackage/anypackage-1.2.3-py2.py3-none-any.whl"); + mkdir("btpackage", 0755); + touch("btpackage/btpackage-1.2.3-mytag-py2.py3-none-any.whl"); + + STASIS_TEST_RUN(tests); + STASIS_TEST_END_MAIN(); +}
\ No newline at end of file |