diff options
103 files changed, 8788 insertions, 2014 deletions
diff --git a/.github/workflows/cmake-multi-platform.yml b/.github/workflows/cmake-multi-platform.yml index ee601b1..2191559 100644 --- a/.github/workflows/cmake-multi-platform.yml +++ b/.github/workflows/cmake-multi-platform.yml @@ -29,7 +29,9 @@ jobs: c_compiler: gcc steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 - name: Set reusable strings id: strings @@ -49,24 +51,42 @@ jobs: libcurl4-openssl-dev libxml2-dev libxml2-utils + libzip-dev pandoc rsync + - name: Install macOS dependencies + if: matrix.os == 'macos-latest' + run: > + brew install + libzip + - name: Configure CMake run: > cmake -B ${{ steps.strings.outputs.build-output-dir }} -DCMAKE_C_COMPILER=${{ matrix.c_compiler }} -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} + -DASAN=ON -DTESTS=ON + -DTESTS_VERBOSE=ON -DTESTS_RT=ON -DDEBUG_MESSAGES=ON -S ${{ github.workspace }} - name: Build + env: + PKG_CONFIG_PATH: /opt/homebrew/lib/pkgconfig:/usr/lib/x86_64-linux-gnu/pkgconfig run: cmake --build ${{ steps.strings.outputs.build-output-dir }} --config ${{ matrix.build_type }} - name: Test working-directory: ${{ steps.strings.outputs.build-output-dir }} - run: ctest -V --build-config ${{ matrix.build_type }} --output-junit results.xml --test-output-size-passed 65536 --test-output-size-failed 65536 + run: | + export ASAN_OPTIONS="verify_asan_link_order=0 strict_string_checks=1 detect_stack_use_after_return=1" + if [[ "$RUNNER_OS" == "macOS" ]]; then + ASAN_OPTIONS="$ASAN_OPTIONS detect_leaks=0" + else + ASAN_OPTIONS="$ASAN_OPTIONS detect_leaks=1" + fi + ctest --build-config ${{ matrix.build_type }} --output-on-failure --output-junit results.xml --test-output-size-passed 65536 --test-output-size-failed 65536 env: STASIS_SYSCONFDIR: ../../.. diff --git a/CMakeLists.txt b/CMakeLists.txt index 8f5dfa9..817f88d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,15 @@ cmake_minimum_required(VERSION 3.15) project(STASIS C) +list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") + include(GNUInstallDirs) +include(GitVersion) +get_version_from_git() +configure_file( + ${CMAKE_SOURCE_DIR}/include/version.h.in + ${CMAKE_BINARY_DIR}/include/version.h +) +message("-- STASIS version: ${PROJECT_VERSION} (${PROJECT_VERSION_BRANCH})") set(nix_cflags -Wall -Wextra -fPIC -D_GNU_SOURCE) set(win_cflags /Wall) @@ -8,15 +17,28 @@ set(CMAKE_C_STANDARD 99) find_package(LibXml2) find_package(CURL) -option(ASAN OFF) +option(ASAN "Address Analyzer" OFF) +set(ASAN_OPTIONS "-fsanitize=address,null,undefined") +message(CHECK_START "Enable Address Analyzer (ASAN)") if (ASAN) - add_compile_options(-fsanitize=address) - add_link_options(-fsanitize=address) + message(CHECK_PASS "Yes") + add_compile_options(${ASAN_OPTIONS} -fno-omit-frame-pointer -g -O0) + add_link_options(${ASAN_OPTIONS}) +else() + message(CHECK_FAIL "No") endif() +pkg_check_modules(ZIP libzip) +if (NOT ZIP_FOUND) + message(FATAL_ERROR "libzip is required (https://libzip.org)") +endif () + +include_directories(${ZIP_INCLUDEDIR}) +link_directories(${ZIP_LIBRARY_DIRS}) +include_directories(${CURL_INCLUDE_DIR}) link_libraries(CURL::libcurl) -link_libraries(LibXml2::LibXml2) include_directories(${LIBXML2_INCLUDE_DIR}) +link_libraries(LibXml2::LibXml2) option(FORTIFY_SOURCE OFF) if (FORTIFY_SOURCE) @@ -54,12 +76,22 @@ message(CHECK_START "Run unit tests") if (TESTS) message(CHECK_PASS "yes") enable_testing() + + message(CHECK_START "Verbose test output") + option(TESTS_VERBOSE OFF) + if (TESTS_VERBOSE) + message(CHECK_PASS "yes") + else() + message(CHECK_PASS "no") + endif() + message(CHECK_START "Run regression tests") if (TESTS_RT) message(CHECK_PASS "yes") else() message(CHECK_PASS "no") endif() + add_subdirectory(tests) else() message(CHECK_PASS "no") @@ -8,6 +8,7 @@ STASIS consolidates the steps required to build, test, and deploy calibration pi - cmake - libcurl - libxml2 +- libzip - rsync # Installation @@ -147,28 +148,32 @@ stasis mydelivery.ini ## Command Line Options -| Long Option | Short Option | Purpose | -|:---------------------------|:------------:|:---------------------------------------------------------------| -| --help | -h | Display usage statement | -| --version | -V | Display program version | -| --continue-on-error | -C | Allow tests to fail | -| --config ARG | -c ARG | Read STASIS configuration file | -| --cpu-limit ARG | -l ARG | Number of processes to spawn concurrently (default: cpus - 1) | -| --pool-status-interval ARG | n/a | Report task status every n seconds (default: 30) | -| --python ARG | -p ARG | Override version of Python in configuration | -| --verbose | -v | Increase output verbosity | -| --unbuffered | -U | Disable line buffering | -| --update-base | n/a | Update conda installation prior to STATIS environment creation | -| --fail-fast | n/a | On test error, terminate all tasks | -| --overwrite | n/a | Overwrite an existing release | -| --no-docker | n/a | Do not build docker images | -| --no-artifactory | n/a | Do not upload artifacts to Artifactory | -| --no-artifactory-build-info| n/a | Do not upload build info objects to Artifactory | -| --no-artifactory-upload | n/a | Do not upload artifacts to Artifactory (dry-run) | -| --no-testing | n/a | Do not execute test scripts | -| --no-parallel | n/a | Do not execute tests in parallel | -| --no-rewrite | n/a | Do not rewrite paths and URLs in output files | -| DELIVERY_FILE | n/a | STASIS delivery file | +| Long Option | Short Option | Purpose | +|:------------------------------------|:------------:|:---------------------------------------------------------------| +| --help | -h | Display usage statement | +| --version | -V | Display program version | +| --continue-on-error | -C | Allow tests to fail | +| --config ARG | -c ARG | Read STASIS configuration file | +| --cpu-limit ARG | -l ARG | Number of processes to spawn concurrently (default: cpus - 1) | +| --pool-status-interval ARG | n/a | Report task status every n seconds (default: 30) | +| --python ARG | -p ARG | Override version of Python in configuration | +| --verbose | -v | Increase output verbosity | +| --unbuffered | -U | Disable line buffering | +| --update-base | n/a | Update conda installation prior to STATIS environment creation | +| --fail-fast | n/a | On test error, terminate all tasks | +| --task-timeout ARG | n/a | Terminate task after timeout is reached (#s, #m, #h) | +| --overwrite | n/a | Overwrite an existing release | +| --wheel-builder ARG | n/a | Wheel building backend (build, cibuildwheel, manylinux) | +| --wheel-builder-manylinux-image ARG | n/a | Manylinux image name | +| --no-docker | n/a | Do not build docker images | +| --no-artifactory | n/a | Do not upload artifacts to Artifactory | +| --no-artifactory-build-info | n/a | Do not upload build info objects to Artifactory | +| --no-artifactory-upload | n/a | Do not upload artifacts to Artifactory (dry-run) | +| --no-testing | n/a | Do not execute test scripts | +| --no-parallel | n/a | Do not execute tests in parallel | +| --no-task-logging | n/a | Do not log task output (write to stdout) | +| --no-rewrite | n/a | Do not rewrite paths and URLs in output files | +| DELIVERY_FILE | n/a | STASIS delivery file | ## Indexer Command Line Options @@ -198,6 +203,9 @@ stasis mydelivery.ini | STASIS_JF_CLIENT_CERT_CERT_PATH | Path to OpenSSL cert files | | STASIS_JF_CLIENT_CERT_KEY_PATH | OpenSSL key file (in cert path) | | STASIS_JF_REPO | Artifactory "generic" repository to write to | +| STASIS_DOWNLOAD_TIMEOUT | Number of seconds before timing out a remote file download | +| STASIS_DOWNLOAD_RETRY_MAX | Number of retries before giving up on a remote file download | +| STASIS_DOWNLOAD_RETRY_SECONDS | Number of seconds to wait before retrying a remote file download | ## Main configuration (stasis.ini) @@ -285,6 +293,7 @@ Sections starting with `test:` will be used during the testing phase of the stas |--------------|---------|-------------------------------------------------------------|----------| | disable | Boolean | Disable `script` execution (`script_setup` always executes) | N | | parallel | Boolean | Execute test block in parallel (default) or sequentially | N | +| timeout | String | Kill test script after `n[hms]` | N | | build_recipe | String | Git repository path to package's conda recipe | N | | repository | String | Git repository path or URL to clone | Y | | version | String | Git commit or tag to check out | Y | diff --git a/cmake/GitVersion.cmake b/cmake/GitVersion.cmake new file mode 100644 index 0000000..005667d --- /dev/null +++ b/cmake/GitVersion.cmake @@ -0,0 +1,48 @@ +function(set_version_fallback) + message(WARNING "Version information not available. Using fallback...") + set(PROJECT_VERSION "0.0.0") + set(PROJECT_VERSION ${PROJECT_VERSION} PARENT_SCOPE) + set(PROJECT_VERSION_BRANCH "unknown") + set(PROJECT_VERSION_BRANCH ${PROJECT_VERSION_BRANCH} PARENT_SCOPE) +endfunction() + +function(get_version_from_git) + find_package(Git QUIET) + if(NOT Git_FOUND) + message(WARNING "Git not found.") + set_version_fallback() + return() + endif() + + execute_process( + COMMAND ${GIT_EXECUTABLE} describe --long --dirty --tags --always + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + OUTPUT_VARIABLE GIT_TAG + OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE GIT_RESULT + ) + if(NOT GIT_RESULT EQUAL 0) + message(WARNING "Failed to get git describe info") + set_version_fallback() + return() + endif() + + execute_process( + COMMAND ${GIT_EXECUTABLE} rev-parse --abbrev-ref HEAD + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + OUTPUT_VARIABLE GIT_BRANCH + OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE GIT_RESULT + ) + if(NOT GIT_RESULT EQUAL 0) + message(WARNING "Failed to get git branch") + set_version_fallback() + return() + endif() + + string(REGEX REPLACE "^v" "" CLEAN_TAG "${GIT_TAG}") + set(PROJECT_VERSION ${CLEAN_TAG}) + set(PROJECT_VERSION ${CLEAN_TAG} PARENT_SCOPE) + set(PROJECT_VERSION_BRANCH ${GIT_BRANCH}) + set(PROJECT_VERSION_BRANCH ${GIT_BRANCH} PARENT_SCOPE) +endfunction()
\ No newline at end of file diff --git a/include/version.h.in b/include/version.h.in new file mode 100644 index 0000000..b33d7c7 --- /dev/null +++ b/include/version.h.in @@ -0,0 +1,6 @@ +#ifndef STASIS_VERSION_H + +#define STASIS_VERSION "@PROJECT_VERSION@" +#define STASIS_VERSION_BRANCH "@PROJECT_VERSION_BRANCH@" + +#endif // STASIS_VERSION_H
\ No newline at end of file diff --git a/mission/hst/base.yml b/mission/hst/base.yml index af115cf..b528b70 100644 --- a/mission/hst/base.yml +++ b/mission/hst/base.yml @@ -15,7 +15,6 @@ dependencies: - fitsblender - gwcs - hasp - - nictools - spherical_geometry - stistools - stregion @@ -29,4 +28,4 @@ dependencies: - ullyses - ullyses-utils - wfc3tools - - wfpc2tools
\ No newline at end of file + - wfpc2tools diff --git a/src/cli/stasis/args.c b/src/cli/stasis/args.c index f3ce823..eb096bc 100644 --- a/src/cli/stasis/args.c +++ b/src/cli/stasis/args.c @@ -13,13 +13,17 @@ struct option long_options[] = { {"unbuffered", no_argument, 0, 'U'}, {"update-base", no_argument, 0, OPT_ALWAYS_UPDATE_BASE}, {"fail-fast", no_argument, 0, OPT_FAIL_FAST}, + {"task-timeout", required_argument, 0, OPT_TASK_TIMEOUT}, {"overwrite", no_argument, 0, OPT_OVERWRITE}, + {"wheel-builder", required_argument, 0, OPT_WHEEL_BUILDER}, + {"wheel-builder-manylinux-image", required_argument, 0, OPT_WHEEL_BUILDER_MANYLINUX_IMAGE}, {"no-docker", no_argument, 0, OPT_NO_DOCKER}, {"no-artifactory", no_argument, 0, OPT_NO_ARTIFACTORY}, {"no-artifactory-build-info", no_argument, 0, OPT_NO_ARTIFACTORY_BUILD_INFO}, {"no-artifactory-upload", no_argument, 0, OPT_NO_ARTIFACTORY_UPLOAD}, {"no-testing", no_argument, 0, OPT_NO_TESTING}, {"no-parallel", no_argument, 0, OPT_NO_PARALLEL}, + {"no-task-logging", no_argument, 0, OPT_NO_TASK_LOGGING}, {"no-rewrite", no_argument, 0, OPT_NO_REWRITE_SPEC_STAGE_2}, {0, 0, 0, 0}, }; @@ -36,13 +40,17 @@ const char *long_options_help[] = { "Disable line buffering", "Update conda installation prior to STASIS environment creation", "On error, immediately terminate all tasks", + "Terminate task after timeout is reached (#s, #m, #h)", "Overwrite an existing release", + "Wheel building backend (build, cibuildwheel, manylinux)", + "Manylinux image name", "Do not build docker images", "Do not upload artifacts to Artifactory", "Do not upload build info objects to Artifactory", "Do not upload artifacts to Artifactory (dry-run)", "Do not execute test scripts", "Do not execute tests in parallel", + "Do not log task output (write to stdout)", "Do not rewrite paths and URLs in output files", NULL, }; @@ -51,7 +59,7 @@ static int get_option_max_width(struct option option[]) { int i = 0; int max = 0; const int indent = 4; - while (option[i].name != 0) { + while (option[i].name != NULL) { int len = (int) strlen(option[i].name); if (option[i].has_arg) { len += indent; @@ -77,28 +85,28 @@ void usage(char *progname) { int width = get_option_max_width(long_options); for (int x = 0; long_options[x].name != 0; x++) { char tmp[STASIS_NAME_MAX] = {0}; - char output[sizeof(tmp)] = {0}; + char output[STASIS_NAME_MAX] = {0}; char opt_long[50] = {0}; // --? [ARG]? char opt_short[50] = {0}; // -? [ARG]? - strcat(opt_long, "--"); - strcat(opt_long, long_options[x].name); + strncat(opt_long, "--", sizeof(opt_long) - strlen(opt_long) - 1); + strncat(opt_long, long_options[x].name, sizeof(opt_long) - strlen(opt_long) - 1); if (long_options[x].has_arg) { - strcat(opt_long, " ARG"); + strncat(opt_long, " ARG", sizeof(opt_long) - strlen(opt_long) - 1); } if (long_options[x].val <= 'z') { - strcat(opt_short, "-"); + strncat(opt_short, "-", sizeof(opt_short) - strlen(opt_short) - 1); opt_short[1] = (char) long_options[x].val; if (long_options[x].has_arg) { - strcat(opt_short, " ARG"); + strncat(opt_short, " ARG", sizeof(opt_short) - strlen(opt_short) - 1); } } else { - strcat(opt_short, " "); + strncat(opt_short, " ", sizeof(opt_short) - strlen(opt_short) - 1); } - sprintf(tmp, " %%-%ds\t%%s\t\t%%s", width + 4); - sprintf(output, tmp, opt_long, opt_short, long_options_help[x]); + snprintf(tmp, sizeof(tmp) - strlen(tmp), " %%-%ds\t%%s\t\t%%s", width + 4); + snprintf(output, sizeof(output), tmp, opt_long, opt_short, long_options_help[x]); puts(output); } } diff --git a/src/cli/stasis/include/args.h b/src/cli/stasis/include/args.h index 5bad752..e789261 100644 --- a/src/cli/stasis/include/args.h +++ b/src/cli/stasis/include/args.h @@ -17,6 +17,10 @@ #define OPT_FAIL_FAST 1009 #define OPT_NO_PARALLEL 1010 #define OPT_POOL_STATUS_INTERVAL 1011 +#define OPT_NO_TASK_LOGGING 1012 +#define OPT_TASK_TIMEOUT 1013 +#define OPT_WHEEL_BUILDER 1014 +#define OPT_WHEEL_BUILDER_MANYLINUX_IMAGE 1015 extern struct option long_options[]; void usage(char *progname); diff --git a/src/cli/stasis/stasis_main.c b/src/cli/stasis/stasis_main.c index 7f0b88a..fb4ed80 100644 --- a/src/cli/stasis/stasis_main.c +++ b/src/cli/stasis/stasis_main.c @@ -7,143 +7,11 @@ // local includes #include "args.h" +#include "conda.h" #include "system_requirements.h" #include "tpl.h" - -int main(int argc, char *argv[]) { - struct Delivery ctx; - struct Process proc = { - .f_stdout = "", - .f_stderr = "", - .redirect_stderr = 0, - }; - char env_name[STASIS_NAME_MAX] = {0}; - char env_name_testing[STASIS_NAME_MAX] = {0}; - char *delivery_input = NULL; - char *config_input = NULL; - char installer_url[PATH_MAX]; - char python_override_version[STASIS_NAME_MAX]; - int user_disabled_docker = false; - globals.cpu_limit = get_cpu_count(); - if (globals.cpu_limit > 1) { - globals.cpu_limit--; // max - 1 - } - - memset(env_name, 0, sizeof(env_name)); - memset(env_name_testing, 0, sizeof(env_name_testing)); - memset(installer_url, 0, sizeof(installer_url)); - memset(python_override_version, 0, sizeof(python_override_version)); - memset(&proc, 0, sizeof(proc)); - memset(&ctx, 0, sizeof(ctx)); - - int c; - int option_index = 0; - while ((c = getopt_long(argc, argv, "hVCc:p:vU", long_options, &option_index)) != -1) { - switch (c) { - case 'h': - usage(path_basename(argv[0])); - exit(0); - case 'V': - puts(VERSION); - exit(0); - case 'c': - config_input = strdup(optarg); - break; - case 'C': - globals.continue_on_error = true; - break; - case 'p': - strcpy(python_override_version, optarg); - break; - case 'l': - globals.cpu_limit = strtol(optarg, NULL, 10); - if (globals.cpu_limit <= 1) { - globals.cpu_limit = 1; - globals.enable_parallel = false; // No point - } - break; - case OPT_ALWAYS_UPDATE_BASE: - globals.always_update_base_environment = true; - break; - case OPT_FAIL_FAST: - globals.parallel_fail_fast = true; - break; - case OPT_POOL_STATUS_INTERVAL: - globals.pool_status_interval = (int) strtol(optarg, NULL, 10); - if (globals.pool_status_interval < 1) { - globals.pool_status_interval = 1; - } else if (globals.pool_status_interval > 60 * 10) { - // Possible poor choice alert - fprintf(stderr, "Caution: Excessive pausing between status updates may cause third-party CI/CD" - " jobs to fail if the stdout/stderr streams are idle for too long!\n"); - } - break; - case 'U': - setenv("PYTHONUNBUFFERED", "1", 1); - fflush(stdout); - fflush(stderr); - setvbuf(stdout, NULL, _IONBF, 0); - setvbuf(stderr, NULL, _IONBF, 0); - break; - case 'v': - globals.verbose = true; - break; - case OPT_OVERWRITE: - globals.enable_overwrite = true; - break; - case OPT_NO_DOCKER: - globals.enable_docker = false; - user_disabled_docker = true; - break; - case OPT_NO_ARTIFACTORY: - globals.enable_artifactory = false; - break; - case OPT_NO_ARTIFACTORY_BUILD_INFO: - globals.enable_artifactory_build_info = false; - break; - case OPT_NO_ARTIFACTORY_UPLOAD: - globals.enable_artifactory_build_info = false; - globals.enable_artifactory_upload = false; - break; - case OPT_NO_TESTING: - globals.enable_testing = false; - break; - case OPT_NO_REWRITE_SPEC_STAGE_2: - globals.enable_rewrite_spec_stage_2 = false; - break; - case OPT_NO_PARALLEL: - globals.enable_parallel = false; - break; - case '?': - default: - exit(1); - } - } - - if (optind < argc) { - while (optind < argc) { - // use first positional argument - delivery_input = argv[optind++]; - break; - } - } - - if (!delivery_input) { - fprintf(stderr, "error: a DELIVERY_FILE is required\n"); - usage(path_basename(argv[0])); - exit(1); - } - - printf(BANNER, VERSION, AUTHOR); - - check_system_path(); - - msg(STASIS_MSG_L1, "Setup\n"); - - tpl_setup_vars(&ctx); - tpl_setup_funcs(&ctx); - +static void setup_sysconfdir() { // Set up PREFIX/etc directory information // The user may manipulate the base directory path with STASIS_SYSCONFDIR // environment variable @@ -153,140 +21,166 @@ int main(int argc, char *argv[]) { } else { strncpy(stasis_sysconfdir_tmp, STASIS_SYSCONFDIR, sizeof(stasis_sysconfdir_tmp) - 1); } + stasis_sysconfdir_tmp[sizeof(stasis_sysconfdir_tmp) - 1] = '\0'; globals.sysconfdir = realpath(stasis_sysconfdir_tmp, NULL); if (!globals.sysconfdir) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Unable to resolve path to configuration directory: %s\n", stasis_sysconfdir_tmp); + SYSERROR("Unable to resolve path to configuration directory: %s", stasis_sysconfdir_tmp); exit(1); } +} +static void setup_python_version_override(struct Delivery *ctx, const char *version) { // Override Python version from command-line, if any - if (strlen(python_override_version)) { - guard_free(ctx.meta.python); - ctx.meta.python = strdup(python_override_version); - guard_free(ctx.meta.python_compact); - ctx.meta.python_compact = to_short_version(ctx.meta.python); + if (version && strlen(version)) { + guard_free(ctx->meta.python); + ctx->meta.python = strdup(version); + if (!ctx->meta.python) { + SYSERROR("Unable to allocate bytes for python version override"); + exit(1); + } + guard_free(ctx->meta.python_compact); + ctx->meta.python_compact = to_short_version(ctx->meta.python); + if (!ctx->meta.python_compact) { + SYSERROR("Unable to allocate bytes for python compact version override"); + exit(1); + } } +} - if (!config_input) { - // no configuration passed by argument. use basic config. +static void configure_stasis_ini(struct Delivery *ctx, char **config_input) { + if (!*config_input) { + SYSDEBUG("No configuration passed by argument. Using basic config."); char cfgfile[PATH_MAX * 2]; - sprintf(cfgfile, "%s/%s", globals.sysconfdir, "stasis.ini"); + snprintf(cfgfile, sizeof(cfgfile), "%s/%s", globals.sysconfdir, "stasis.ini"); + SYSDEBUG("cfgfile: %s", cfgfile); if (!access(cfgfile, F_OK | R_OK)) { - config_input = strdup(cfgfile); + *config_input = strdup(cfgfile); } else { - msg(STASIS_MSG_WARN, "STASIS global configuration is not readable, or does not exist: %s", cfgfile); + SYSWARN("STASIS global configuration is not readable, or does not exist: %s", cfgfile); } } - if (config_input) { - msg(STASIS_MSG_L2, "Reading STASIS global configuration: %s\n", config_input); - ctx._stasis_ini_fp.cfg = ini_open(config_input); - if (!ctx._stasis_ini_fp.cfg) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read config file: %s, %s\n", delivery_input, strerror(errno)); - exit(1); - } - ctx._stasis_ini_fp.cfg_path = strdup(config_input); - guard_free(config_input); + SYSDEBUG("Reading STASIS global configuration: %s", *config_input); + ctx->_stasis_ini_fp.cfg = ini_open(*config_input); + if (!ctx->_stasis_ini_fp.cfg) { + SYSERROR("Failed to read global config file: %s, %s", *config_input, strerror(errno)); + exit(1); + } + ctx->_stasis_ini_fp.cfg_path = strdup(*config_input); + if (!ctx->_stasis_ini_fp.cfg_path) { + SYSERROR("Failed to allocate memory delivery context global config file name"); + exit(1); } + guard_free(*config_input); +} - msg(STASIS_MSG_L2, "Reading STASIS delivery configuration: %s\n", delivery_input); - ctx._stasis_ini_fp.delivery = ini_open(delivery_input); - if (!ctx._stasis_ini_fp.delivery) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read delivery file: %s, %s\n", delivery_input, strerror(errno)); +static void configure_delivery_ini(struct Delivery *ctx, char **delivery_input) { + msg(STASIS_MSG_L2, "Reading STASIS delivery configuration: %s\n", *delivery_input); + ctx->_stasis_ini_fp.delivery = ini_open(*delivery_input); + if (!ctx->_stasis_ini_fp.delivery) { + SYSERROR("Failed to read delivery file: %s, %s", *delivery_input, strerror(errno)); exit(1); } - ctx._stasis_ini_fp.delivery_path = strdup(delivery_input); + ctx->_stasis_ini_fp.delivery_path = strdup(*delivery_input); +} +static void configure_delivery_context(struct Delivery *ctx) { msg(STASIS_MSG_L2, "Bootstrapping delivery context\n"); - if (bootstrap_build_info(&ctx)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to bootstrap delivery context\n"); + if (bootstrap_build_info(ctx)) { + SYSERROR("Failed to bootstrap delivery context"); exit(1); } msg(STASIS_MSG_L2, "Initializing delivery context\n"); - if (delivery_init(&ctx, INI_READ_RENDER)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to initialize delivery context\n"); + if (delivery_init(ctx, INI_READ_RENDER)) { + SYSERROR("Failed to initialize delivery context"); exit(1); } - check_requirements(&ctx); +} +static void configure_jfrog_cli(struct Delivery *ctx) { msg(STASIS_MSG_L2, "Configuring JFrog CLI\n"); - if (delivery_init_artifactory(&ctx)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "JFrog CLI configuration failed\n"); + if (delivery_init_artifactory(ctx)) { + SYSERROR("JFrog CLI configuration failed"); exit(1); } +} - runtime_apply(ctx.runtime.environ); - strcpy(env_name, ctx.info.release_name); - strcpy(env_name_testing, env_name); - strcat(env_name_testing, "-test"); - +static void check_release_history(struct Delivery *ctx) { // Safety gate: Avoid clobbering a delivered release unless the user wants that behavior - msg(STASIS_MSG_L1, "Checking release history\n"); - if (!globals.enable_overwrite && delivery_exists(&ctx) == DELIVERY_FOUND) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Refusing to overwrite release: %s\nUse --overwrite to enable release clobbering.\n", ctx.info.release_name); + msg(STASIS_MSG_L2, "Checking release history\n"); + if (!globals.enable_overwrite && delivery_exists(ctx) == DELIVERY_FOUND) { + SYSERROR("Refusing to overwrite release: %s\nUse --overwrite to enable release clobbering.", ctx->info.release_name); exit(1); } +} + +static void check_conda_install_prefix(const struct Delivery *ctx) { + // Unlikely to occur: this should help prevent rmtree() from destroying your entire filesystem + // if path is "/" then, die + // or if empty string, die + if (!strcmp(ctx->storage.conda_install_prefix, DIR_SEP) || !strlen(ctx->storage.conda_install_prefix)) { + SYSERROR("error: ctx.storage.conda_install_prefix is malformed!"); + exit(1); + } +} + +static void sync_release_history(struct Delivery *ctx) { if (globals.enable_artifactory) { // We need to download previous revisions to ensure processed packages are available at build-time // This is also a docker requirement. Python wheels must exist locally. - if (ctx.meta.rc > 1) { - msg(STASIS_MSG_L1, "Syncing delivery artifacts for %s\n", ctx.info.build_name); - if (delivery_series_sync(&ctx) != 0) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Unable to sync artifacts for %s\n", ctx.info.build_name); - msg(STASIS_MSG_L3, "Case #1:\n" - "\tIf this is a new 'version', and 'rc' is greater " - "than 1, then no previous deliveries exist remotely. " - "Reset 'rc' to 1.\n"); - msg(STASIS_MSG_L3, "Case #2:\n" - "\tThe Artifactory server %s is unreachable, or the credentials used " - "are invalid.\n", globals.jfrog.url); + if (ctx->meta.rc > 1) { + msg(STASIS_MSG_L1, "Syncing delivery artifacts for %s\n", ctx->info.build_name); + if (delivery_series_sync(ctx) != 0) { + SYSERROR("Unable to sync artifacts for %s", ctx->info.build_name); + SYSERROR("Case #1:\n" + "\tIf this is a new 'version', and 'rc' is greater " + "than 1, then no previous deliveries exist remotely. " + "Reset 'rc' to 1."); + SYSERROR("Case #2:\n" + "\tThe Artifactory server %s is unreachable, or the credentials used " + "are invalid.", globals.jfrog.url); // No continue-on-error check. Without the previous delivery nothing can be done. exit(1); } } } +} - // Unlikely to occur: this should help prevent rmtree() from destroying your entire filesystem - // if path is "/" then, die - // or if empty string, die - if (!strcmp(ctx.storage.conda_install_prefix, DIR_SEP) || !strlen(ctx.storage.conda_install_prefix)) { - fprintf(stderr, "error: ctx.storage.conda_install_prefix is malformed!\n"); - exit(1); - } - +static void check_conda_prefix_length(const struct Delivery *ctx) { // 2 = #! // 5 = /bin\n - const size_t prefix_len = strlen(ctx.storage.conda_install_prefix) + 2 + 5; + const size_t prefix_len = strlen(ctx->storage.conda_install_prefix) + 2 + 5; const size_t prefix_len_max = 127; - msg(STASIS_MSG_L1, "Checking length of conda installation prefix\n"); - if (!strcmp(ctx.system.platform[DELIVERY_PLATFORM], "Linux") && prefix_len > prefix_len_max) { - msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, - "The shebang, '#!%s/bin/python\\n' is too long (%zu > %zu).\n", - ctx.storage.conda_install_prefix, prefix_len, prefix_len_max); - msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, - "Conda's workaround to handle long path names does not work consistently within STASIS.\n"); - msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, - "Please try again from a different, \"shorter\", directory.\n"); + msg(STASIS_MSG_L2, "Checking length of conda installation prefix\n"); + if (!strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Linux") && prefix_len > prefix_len_max) { + SYSERROR("The shebang, '#!%s/bin/python\\n' is too long (%zu > %zu).", + ctx->storage.conda_install_prefix, prefix_len, prefix_len_max); + SYSERROR("Conda's workaround to handle long path names does not work consistently within STASIS."); + SYSERROR("Please try again from a different, \"shorter\", directory."); exit(1); } +} +static void setup_conda(struct Delivery *ctx, char *installer_url, const size_t maxlen) { msg(STASIS_MSG_L1, "Conda setup\n"); - delivery_get_conda_installer_url(&ctx, installer_url); + delivery_get_conda_installer_url(ctx, installer_url, maxlen); msg(STASIS_MSG_L2, "Downloading: %s\n", installer_url); - if (delivery_get_conda_installer(&ctx, installer_url)) { - msg(STASIS_MSG_ERROR, "download failed: %s\n", installer_url); + if (delivery_get_conda_installer(ctx, installer_url)) { + SYSERROR("download failed: %s", installer_url); exit(1); } - msg(STASIS_MSG_L2, "Installing: %s\n", ctx.conda.installer_name); - delivery_install_conda(ctx.conda.installer_path, ctx.storage.conda_install_prefix); + msg(STASIS_MSG_L2, "Installing: %s\n", ctx->conda.installer_name); + delivery_install_conda(ctx->conda.installer_path, ctx->storage.conda_install_prefix); - msg(STASIS_MSG_L2, "Configuring: %s\n", ctx.storage.conda_install_prefix); - delivery_conda_enable(&ctx, ctx.storage.conda_install_prefix); + msg(STASIS_MSG_L2, "Configuring: %s\n", ctx->storage.conda_install_prefix); + delivery_conda_enable(ctx, ctx->storage.conda_install_prefix); +} +static void configure_conda_base(struct Delivery *ctx, char *envs[]) { // // Implied environment creation modes/actions // @@ -303,296 +197,553 @@ int main(int argc, char *argv[]) { // 3b. Bugs, conflicts, and dependency resolution issues are inherited and // must be handled in the INI config msg(STASIS_MSG_L1, "Creating release environment(s)\n"); - char *mission_base = NULL; - if (isempty(ctx.meta.based_on)) { - guard_free(ctx.meta.based_on); + + if (isempty(ctx->meta.based_on)) { + // based_on was not set by the input file + + guard_free(ctx->meta.based_on); char *mission_base_orig = NULL; - if (asprintf(&mission_base_orig, "%s/%s/base.yml", ctx.storage.mission_dir, ctx.meta.mission) < 0) { - SYSERROR("Unable to allocate bytes for %s/%s/base.yml path\n", ctx.storage.mission_dir, ctx.meta.mission); + if (asprintf(&mission_base_orig, "%s/%s/base.yml", ctx->storage.mission_dir, ctx->meta.mission) < 0) { + SYSERROR("Unable to allocate bytes for %s/%s/base.yml path", ctx->storage.mission_dir, ctx->meta.mission); exit(1); } + // Does a base.yml exist in the mission directory? + // If not, do nothing. Otherwise, use the base.yml in the mission directory. if (access(mission_base_orig, F_OK) < 0) { - msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "Mission does not provide a base.yml configuration: %s (%s)\n", - ctx.meta.mission, ctx.storage.mission_dir); + SYSWARN("Mission does not provide a base.yml"); } else { msg(STASIS_MSG_L2, "Using base environment configuration: %s\n", mission_base_orig); - if (asprintf(&mission_base, "%s/%s-base.yml", ctx.storage.tmpdir, ctx.info.release_name) < 0) { - SYSERROR("%s", "Unable to allocate bytes for temporary base.yml configuration"); + if (asprintf(&mission_base, "%s/%s-base.yml", ctx->storage.tmpdir, ctx->info.release_name) < 0) { + SYSERROR("Unable to allocate bytes for temporary base.yml configuration"); remove(mission_base); exit(1); } copy2(mission_base_orig, mission_base, CT_OWNER | CT_PERM); - ctx.meta.based_on = mission_base; + ctx->meta.based_on = mission_base; } guard_free(mission_base_orig); } - if (!isempty(ctx.meta.based_on)) { - if (conda_env_exists(ctx.storage.conda_install_prefix, env_name) && conda_env_remove(env_name)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to remove release environment: %s\n", env_name); - exit(1); - } + msg(STASIS_MSG_L2, "Based on: %s\n", ctx->meta.based_on); - msg(STASIS_MSG_L2, "Based on: %s\n", ctx.meta.based_on); - if (conda_env_create_from_uri(env_name, ctx.meta.based_on, ctx.meta.python)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "unable to install release environment using configuration file\n"); - exit(1); - } + for (size_t i = 0; envs[i] != NULL; i += 2) { + char *title = envs[i]; + char *env = envs[i+1]; + // If based_on was populated above, or defined in the configuration: install its packages. + if (!isempty(ctx->meta.based_on)) { + if (conda_env_exists(ctx->storage.conda_install_prefix, env) && conda_env_remove(env)) { + SYSERROR("failed to remove %s environment: %s", title); + exit(1); + } - if (conda_env_exists(ctx.storage.conda_install_prefix, env_name_testing) && conda_env_remove(env_name_testing)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to remove testing environment %s\n", env_name_testing); - exit(1); - } - if (conda_env_create_from_uri(env_name_testing, ctx.meta.based_on, ctx.meta.python)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "unable to install testing environment using configuration file\n"); - exit(1); - } - } else { - if (conda_env_create(env_name, ctx.meta.python, NULL)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to create release environment\n"); - exit(1); - } - if (conda_env_create(env_name_testing, ctx.meta.python, NULL)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to create testing environment\n"); - exit(1); + if (conda_env_create_from_uri(env, ctx->meta.based_on, ctx->meta.python)) { + SYSERROR("unable to install %s environment using configuration file", title); + exit(1); + } + } else { + // Otherwise, create the environments with the requested Python version and move on + if (conda_env_create(env, ctx->meta.python, NULL)) { + SYSERROR("failed to create %s environment", title); + exit(1); + } } } // The base environment configuration not used past this point remove(mission_base); +} - const char *envs[] = {env_name_testing, env_name}; - for (size_t e = 0; e < sizeof(envs) / sizeof(*envs); e++) { - const char *name = envs[e]; - if (ctx.conda.conda_packages_purge && strlist_count(ctx.conda.conda_packages_purge)) { - msg(STASIS_MSG_L2, "Purging conda packages from %s\n", name); - if (delivery_purge_packages(&ctx, name, PKG_USE_CONDA)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "unable to purge requested conda packages from %s\n", name); - exit(1); - } - } - - if (ctx.conda.pip_packages_purge && strlist_count(ctx.conda.pip_packages_purge)) { - msg(STASIS_MSG_L2, "Purging pip packages from %s\n", name); - if (delivery_purge_packages(&ctx, env_name_testing, PKG_USE_PIP)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "unable to purge requested pip packages from %s\n", - env_name_testing); - exit(1); +static void configure_conda_purge(struct Delivery *ctx, char *envs[]) { + struct StrList *purge_list[] = { + ctx->conda.conda_packages_purge, + ctx->conda.pip_packages_purge + }; + for (size_t i = 0; i < sizeof(purge_list) / sizeof(purge_list[0]); i++) { + struct StrList *to_purge = purge_list[i]; + for (size_t e = 0; envs[e] != NULL; e += 2) { + //const char *title = envs[e]; // unused + const char *env = envs[e+1]; + if (to_purge && strlist_count(to_purge)) { + const char *pkg_manager_name[] = { + "conda", + "pip" + }; + const int pkg_manager_use[] = { + PKG_USE_CONDA, + PKG_USE_PIP + }; + const char *manager_str = pkg_manager_name[i]; + const int manager_flag = pkg_manager_use[i]; + msg(STASIS_MSG_L2, "Purging %s packages from %s\n", manager_str, env); + if (delivery_purge_packages(ctx, env, manager_flag)) { + SYSERROR("unable to purge requested %s packages from %s", manager_str, env); + exit(1); + } } } } +} +static void setup_activate_test_env(const struct Delivery *ctx, const char *env_name_testing) { // Activate test environment msg(STASIS_MSG_L1, "Activating test environment\n"); - if (conda_activate(ctx.storage.conda_install_prefix, env_name_testing)) { - fprintf(stderr, "failed to activate test environment\n"); + if (conda_activate(ctx->storage.conda_install_prefix, env_name_testing)) { + SYSERROR("failed to activate test environment"); exit(1); } +} - if (delivery_gather_tool_versions(&ctx)) { - if (!ctx.conda.tool_version) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Could not determine conda version\n"); +static void configure_tool_versions(struct Delivery *ctx) { + if (delivery_gather_tool_versions(ctx)) { + if (!ctx->conda.tool_version) { + SYSERROR("Could not determine conda version"); exit(1); } - if (!ctx.conda.tool_build_version) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Could not determine conda-build version\n"); + if (!ctx->conda.tool_build_version) { + SYSERROR("Could not determine conda-build version"); exit(1); } } +} +static void install_packaging_tools() { + msg(STASIS_MSG_L1, "Installing packaging tool(s)\n"); if (pip_exec("install build")) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "'build' tool installation failed\n"); + SYSERROR("'build' tool installation failed"); exit(1); } + if (pip_exec("install cibuildwheel")) { + SYSERROR("'cibuildwheel' tool installation failed"); + exit(1); + } +} + +static void force_conda_package_reinstallation_on_mismatch(struct Delivery *ctx, const char *env_name) { + const size_t conda_package_count = strlist_count(ctx->conda.conda_packages); + if (conda_package_count) { + msg(STASIS_MSG_L1, "Enforcing Conda package versions: %s\n", env_name); + for (size_t i = 0; i < conda_package_count; i++) { + const char *item = strlist_item(ctx->conda.conda_packages, i); + if (!item) { + SYSERROR("NULL record in conda package list"); + exit(1); + } + char *pkg_name = strdup(item); + if (!pkg_name) { + SYSERROR("unable to allocate memory for package name"); + exit(1); + } + const char *spec = find_version_spec(pkg_name); + if (spec) { + pkg_name[spec - pkg_name] = '\0'; + } + + msg(STASIS_MSG_L2, "%s\n", pkg_name); + if (delivery_conda_enforce_package_version(ctx, env_name, pkg_name)) { + SYSERROR("Failed to determine conda package version: %s", pkg_name); + guard_free(pkg_name); + exit(1); + } + guard_free(pkg_name); + } + } +} - if (!isempty(ctx.meta.based_on)) { +static void configure_package_overlay(struct Delivery *ctx, const char *env_name) { + if (!isempty(ctx->meta.based_on)) { msg(STASIS_MSG_L1, "Generating package overlay from environment: %s\n", env_name); - if (delivery_overlay_packages_from_env(&ctx, env_name)) { - msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "%s", "Failed to generate package overlay. Resulting environment integrity cannot be guaranteed.\n"); + if (delivery_overlay_packages_from_env(ctx, env_name)) { + SYSERROR("Failed to generate package overlay. Resulting environment integrity cannot be guaranteed."); exit(1); } } +} +static void configure_deferred_packages(struct Delivery *ctx) { msg(STASIS_MSG_L1, "Filter deliverable packages\n"); - delivery_defer_packages(&ctx, DEFER_CONDA); - delivery_defer_packages(&ctx, DEFER_PIP); + delivery_defer_packages(ctx, DEFER_CONDA); + delivery_defer_packages(ctx, DEFER_PIP); +} +static void show_overview(struct Delivery *ctx) { msg(STASIS_MSG_L1, "Overview\n"); - delivery_meta_show(&ctx); - delivery_conda_show(&ctx); + delivery_meta_show(ctx); + delivery_conda_show(ctx); if (globals.verbose) { //delivery_runtime_show(&ctx); } +} +static void run_tests(struct Delivery *ctx) { // Execute configuration-defined tests if (globals.enable_testing) { - delivery_tests_show(&ctx); + delivery_tests_show(ctx); msg(STASIS_MSG_L1, "Begin test execution\n"); - delivery_tests_run(&ctx); + delivery_tests_run(ctx); msg(STASIS_MSG_L2, "Rewriting test results\n"); - delivery_fixup_test_results(&ctx); + delivery_fixup_test_results(ctx); } else { - msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Test execution is disabled\n"); + SYSWARN("Test execution is disabled"); } +} - if (ctx.conda.conda_packages_defer && strlist_count(ctx.conda.conda_packages_defer)) { +static void build_conda_recipes(struct Delivery *ctx) { + if (ctx->conda.conda_packages_defer && strlist_count(ctx->conda.conda_packages_defer)) { msg(STASIS_MSG_L2, "Building Conda recipe(s)\n"); - if (delivery_build_recipes(&ctx)) { + if (delivery_build_recipes(ctx)) { exit(1); } msg(STASIS_MSG_L3, "Copying artifacts\n"); - if (delivery_copy_conda_artifacts(&ctx)) { + if (delivery_copy_conda_artifacts(ctx)) { exit(1); } msg(STASIS_MSG_L3, "Indexing artifacts\n"); - if (delivery_index_conda_artifacts(&ctx)) { + if (delivery_index_conda_artifacts(ctx)) { exit(1); } } +} - if (strlist_count(ctx.conda.pip_packages_defer)) { - if (!((ctx.conda.wheels_packages = delivery_build_wheels(&ctx)))) { +static void build_wheel_packages(struct Delivery *ctx) { + if (strlist_count(ctx->conda.pip_packages_defer)) { + msg(STASIS_MSG_L2, "Building Python wheels(s)\n"); + if (!((ctx->conda.wheels_packages = delivery_build_wheels(ctx)))) { exit(1); } - if (delivery_index_wheel_artifacts(&ctx)) { + if (delivery_index_wheel_artifacts(ctx)) { exit(1); } } +} - // Populate the release environment - msg(STASIS_MSG_L1, "Populating release environment\n"); +static void release_install_conda_packages(struct Delivery *ctx, char *env_name) { msg(STASIS_MSG_L2, "Installing conda packages\n"); - if (strlist_count(ctx.conda.conda_packages)) { - if (delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_CONDA, (struct StrList *[]) {ctx.conda.conda_packages, NULL})) { + if (strlist_count(ctx->conda.conda_packages)) { + if (delivery_install_packages(ctx, ctx->storage.conda_install_prefix, env_name, INSTALL_PKG_CONDA, (struct StrList *[]) {ctx->conda.conda_packages, NULL})) { exit(1); } + force_conda_package_reinstallation_on_mismatch(ctx, env_name); } - if (strlist_count(ctx.conda.conda_packages_defer)) { + if (strlist_count(ctx->conda.conda_packages_defer)) { msg(STASIS_MSG_L3, "Installing deferred conda packages\n"); - if (delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_CONDA | INSTALL_PKG_CONDA_DEFERRED, (struct StrList *[]) {ctx.conda.conda_packages_defer, NULL})) { + if (delivery_install_packages(ctx, ctx->storage.conda_install_prefix, env_name, INSTALL_PKG_CONDA | INSTALL_PKG_CONDA_DEFERRED, (struct StrList *[]) {ctx->conda.conda_packages_defer, NULL})) { exit(1); } } else { msg(STASIS_MSG_L3, "No deferred conda packages\n"); } +} +static void release_install_pip_packages(struct Delivery *ctx, char *env_name) { msg(STASIS_MSG_L2, "Installing pip packages\n"); - if (strlist_count(ctx.conda.pip_packages)) { - if (delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_PIP, (struct StrList *[]) {ctx.conda.pip_packages, NULL})) { + if (strlist_count(ctx->conda.pip_packages)) { + if (delivery_install_packages(ctx, ctx->storage.conda_install_prefix, env_name, INSTALL_PKG_PIP, (struct StrList *[]) {ctx->conda.pip_packages, NULL})) { exit(1); } } - if (strlist_count(ctx.conda.pip_packages_defer)) { + if (strlist_count(ctx->conda.pip_packages_defer)) { msg(STASIS_MSG_L3, "Installing deferred pip packages\n"); - if (delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_PIP | INSTALL_PKG_PIP_DEFERRED, (struct StrList *[]) {ctx.conda.pip_packages_defer, NULL})) { + if (delivery_install_packages(ctx, ctx->storage.conda_install_prefix, env_name, INSTALL_PKG_PIP | INSTALL_PKG_PIP_DEFERRED, (struct StrList *[]) {ctx->conda.pip_packages_defer, NULL})) { exit(1); } } else { msg(STASIS_MSG_L3, "No deferred pip packages\n"); } +} - conda_exec("list"); - - msg(STASIS_MSG_L1, "Creating release\n"); - msg(STASIS_MSG_L2, "Exporting delivery configuration\n"); - if (!pushd(ctx.storage.cfgdump_dir)) { - char filename[PATH_MAX] = {0}; - sprintf(filename, "%s.ini", ctx.info.release_name); - FILE *spec = fopen(filename, "w+"); - if (!spec) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", filename); - exit(1); - } - ini_write(ctx._stasis_ini_fp.delivery, &spec, INI_WRITE_RAW); - fclose(spec); - - memset(filename, 0, sizeof(filename)); - sprintf(filename, "%s-rendered.ini", ctx.info.release_name); - spec = fopen(filename, "w+"); - if (!spec) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", filename); - exit(1); - } - ini_write(ctx._stasis_ini_fp.delivery, &spec, INI_WRITE_PRESERVE); - fclose(spec); - popd(); - } else { - SYSERROR("Failed to enter directory: %s", ctx.storage.delivery_dir); - exit(1); - } - - msg(STASIS_MSG_L2, "Exporting %s\n", env_name_testing); - if (conda_env_export(env_name_testing, ctx.storage.delivery_dir, env_name_testing)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", env_name_testing); - exit(1); - } - - msg(STASIS_MSG_L2, "Exporting %s\n", env_name); - if (conda_env_export(env_name, ctx.storage.delivery_dir, env_name)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", env_name); - exit(1); - } - - // Rewrite release environment output (i.e. set package origin(s) to point to the deployment server, etc.) - char specfile[PATH_MAX]; - sprintf(specfile, "%s/%s.yml", ctx.storage.delivery_dir, env_name); - msg(STASIS_MSG_L3, "Rewriting release spec file (stage 1): %s\n", path_basename(specfile)); - delivery_rewrite_spec(&ctx, specfile, DELIVERY_REWRITE_SPEC_STAGE_1); - - msg(STASIS_MSG_L1, "Rendering mission templates\n"); - delivery_mission_render_files(&ctx); - - int want_docker = ini_section_search(&ctx._stasis_ini_fp.delivery, INI_SEARCH_BEGINS, "deploy:docker") ? true : false; - int want_artifactory = ini_section_search(&ctx._stasis_ini_fp.delivery, INI_SEARCH_BEGINS, "deploy:artifactory") ? true : false; +static void build_docker(struct Delivery *ctx, const int disabled) { + const int want_docker = ini_section_search(&ctx->_stasis_ini_fp.delivery, INI_SEARCH_BEGINS, "deploy:docker") ? true : false; if (want_docker) { - if (user_disabled_docker) { - msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Docker image building is disabled by CLI argument\n"); + if (disabled) { + SYSWARN("Docker image building is disabled by CLI argument"); } else { char dockerfile[PATH_MAX] = {0}; - sprintf(dockerfile, "%s/%s", ctx.storage.build_docker_dir, "Dockerfile"); + snprintf(dockerfile, sizeof(dockerfile), "%s/%s", ctx->storage.build_docker_dir, "Dockerfile"); if (globals.enable_docker) { if (!access(dockerfile, F_OK)) { msg(STASIS_MSG_L1, "Building Docker image\n"); - if (delivery_docker(&ctx)) { - msg(STASIS_MSG_L1 | STASIS_MSG_ERROR, "Failed to build docker image!\n"); + if (delivery_docker(ctx)) { + SYSERROR("Failed to build docker image!"); COE_CHECK_ABORT(1, "Failed to build docker image"); } } else { - msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Docker image building is disabled. No Dockerfile found in %s\n", ctx.storage.build_docker_dir); + SYSWARN("Docker image building is disabled. No Dockerfile found in %s", ctx->storage.build_docker_dir); } } else { - msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Docker image building is disabled. System configuration error\n"); + SYSWARN("Docker image building is disabled. System configuration error"); } } } else { - msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Docker image building is disabled. deploy:docker is not configured\n"); + SYSWARN("Docker image building is disabled. deploy:docker is not configured"); } +} + +static void generate_release(struct Delivery *ctx, char *env_name, char *env_name_testing, const int disable_docker) { + // Populate the release environment + msg(STASIS_MSG_L1, "Populating release environment\n"); + release_install_conda_packages(ctx, env_name); + release_install_pip_packages(ctx, env_name); + + conda_exec("list"); - msg(STASIS_MSG_L3, "Rewriting release spec file (stage 2): %s\n", path_basename(specfile)); - delivery_rewrite_spec(&ctx, specfile, DELIVERY_REWRITE_SPEC_STAGE_2); + msg(STASIS_MSG_L1, "Creating release\n"); + delivery_export(ctx, (char *[]) {env_name, env_name_testing, NULL}); + + char specfile[PATH_MAX]; + snprintf(specfile, sizeof(specfile), "%s/%s.yml", ctx->storage.delivery_dir, env_name); + + delivery_rewrite_stage1(ctx, specfile); + build_docker(ctx, disable_docker); + delivery_rewrite_stage2(ctx, specfile); msg(STASIS_MSG_L1, "Dumping metadata\n"); - if (delivery_dump_metadata(&ctx)) { - msg(STASIS_MSG_L1 | STASIS_MSG_ERROR, "Metadata dump failed\n"); + if (delivery_dump_metadata(ctx)) { + SYSERROR("Metadata dump failed"); } +} +static void transfer_artifacts(struct Delivery *ctx) { + const int want_artifactory = ini_section_search(&ctx->_stasis_ini_fp.delivery, INI_SEARCH_BEGINS, "deploy:artifactory") ? true : false; if (want_artifactory) { if (globals.enable_artifactory && globals.enable_artifactory_upload) { msg(STASIS_MSG_L1, "Uploading artifacts\n"); - delivery_artifact_upload(&ctx); + delivery_artifact_upload(ctx); } else { - msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Artifactory upload is disabled by CLI argument\n"); + SYSWARN("Artifactory upload is disabled by CLI argument"); } } else { - msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Artifactory upload is disabled. deploy:artifactory is not configured\n"); + SYSWARN("Artifactory upload is disabled. deploy:artifactory is not configured"); + } +} + +int main(int argc, char *argv[]) { + struct Delivery ctx; + struct Process proc = { + .f_stdout = "", + .f_stderr = "", + .redirect_stderr = 0, + }; + char env_name[STASIS_NAME_MAX] = {0}; + char env_name_testing[STASIS_NAME_MAX] = {0}; + char *delivery_input = NULL; + char *config_input = NULL; + char installer_url[PATH_MAX]; + char python_override_version[STASIS_NAME_MAX]; + int user_disabled_docker = false; + globals.cpu_limit = get_cpu_count(); + if (globals.cpu_limit > 1) { + globals.cpu_limit--; // max - 1 } + memset(env_name, 0, sizeof(env_name)); + memset(env_name_testing, 0, sizeof(env_name_testing)); + memset(installer_url, 0, sizeof(installer_url)); + memset(python_override_version, 0, sizeof(python_override_version)); + memset(&proc, 0, sizeof(proc)); + memset(&ctx, 0, sizeof(ctx)); + + setup_sysconfdir(); + + int c; + int option_index = 0; + while ((c = getopt_long(argc, argv, "hVCc:p:vUl:", long_options, &option_index)) != -1) { + switch (c) { + case 'h': + usage(path_basename(argv[0])); + exit(0); + case 'V': + puts(VERSION); + exit(0); + case 'c': + config_input = strdup(optarg); + break; + case 'C': + globals.continue_on_error = true; + break; + case 'p': + strncpy(python_override_version, optarg, sizeof(python_override_version) - 1); + python_override_version[sizeof(python_override_version) - 1] = '\0'; + break; + case 'l': + globals.cpu_limit = strtol(optarg, NULL, 10); + if (globals.cpu_limit <= 1) { + globals.cpu_limit = 1; + globals.enable_parallel = false; // No point + } + break; + case OPT_ALWAYS_UPDATE_BASE: + globals.always_update_base_environment = true; + break; + case OPT_FAIL_FAST: + globals.parallel_fail_fast = true; + break; + case OPT_TASK_TIMEOUT: + globals.task_timeout = str_to_timeout(optarg); + if (globals.task_timeout < 0) { + SYSERROR("Invalid timeout: %s", optarg); + if (globals.task_timeout == STR_TO_TIMEOUT_INVALID_TIME_SCALE) { + SYSERROR("Use format '#s' (seconds), '#m' (minutes), '#h' (hours)"); + } else if (globals.task_timeout == STR_TO_TIMEOUT_NEGATIVE) { + SYSERROR("Timeout cannot be negative"); + } + exit(1); + } + break; + case OPT_POOL_STATUS_INTERVAL: + globals.pool_status_interval = (int) strtol(optarg, NULL, 10); + if (globals.pool_status_interval < 1) { + globals.pool_status_interval = 1; + } else if (globals.pool_status_interval > 60 * 10) { + // Possible poor choice alert + SYSWARN("Excessive pausing between status updates may cause third-party CI/CD" + " jobs to fail if the stdout/stderr streams are idle for too long!"); + } + break; + case 'U': + setenv("PYTHONUNBUFFERED", "1", 1); + fflush(stdout); + fflush(stderr); + setvbuf(stdout, NULL, _IONBF, 0); + setvbuf(stderr, NULL, _IONBF, 0); + break; + case 'v': + globals.verbose = true; + LOG_LEVEL++; + break; + case OPT_OVERWRITE: + globals.enable_overwrite = true; + break; + case OPT_NO_DOCKER: + globals.enable_docker = false; + user_disabled_docker = true; + break; + case OPT_NO_ARTIFACTORY: + globals.enable_artifactory = false; + break; + case OPT_NO_ARTIFACTORY_BUILD_INFO: + globals.enable_artifactory_build_info = false; + break; + case OPT_NO_ARTIFACTORY_UPLOAD: + globals.enable_artifactory_build_info = false; + globals.enable_artifactory_upload = false; + break; + case OPT_NO_TESTING: + globals.enable_testing = false; + break; + case OPT_NO_REWRITE_SPEC_STAGE_2: + globals.enable_rewrite_spec_stage_2 = false; + break; + case OPT_NO_PARALLEL: + globals.enable_parallel = false; + break; + case OPT_NO_TASK_LOGGING: + globals.enable_task_logging = false; + break; + case OPT_WHEEL_BUILDER: + globals.wheel_builder = strdup(optarg); + break; + case OPT_WHEEL_BUILDER_MANYLINUX_IMAGE: + globals.wheel_builder_manylinux_image = strdup(optarg); + break; + case '?': + default: + exit(1); + } + } + + if (optind < argc) { + while (optind < argc) { + // use first positional argument + delivery_input = argv[optind++]; + break; + } + } + + if (!delivery_input) { + SYSERROR("a DELIVERY_FILE is required"); + usage(path_basename(argv[0])); + exit(1); + } + + char *version = center_text(VERSION, strlen(STASIS_BANNER_HEADER)); + if (!version) { + SYSERROR("version too long?"); + version = strdup(VERSION); + if (!version) { + SYSERROR("unable to allocate uncentered fallback version string"); + exit(1); + } + } + printf(BANNER, version, AUTHOR); + guard_free(version); + + SYSDEBUG("LOG_LEVEL is %s", log_get_level_str()); + + setup_python_version_override(&ctx, python_override_version); + configure_stasis_ini(&ctx, &config_input); + check_system_path(); + + msg(STASIS_MSG_L1, "Setup\n"); + + tpl_setup_vars(&ctx); + tpl_setup_funcs(&ctx); + + configure_delivery_ini(&ctx, &delivery_input); + configure_delivery_context(&ctx); + check_requirements(&ctx); + + configure_jfrog_cli(&ctx); + /* + delivery_free(&ctx); + tpl_free(); + globals_free(); + return 0; + */ + + runtime_apply(ctx.runtime.environ); + strncpy(env_name, ctx.info.release_name, sizeof(env_name) - 1); + env_name[sizeof(env_name) - 1] = '\0'; + + strncpy(env_name_testing, env_name, sizeof(env_name_testing) - 1); + env_name_testing[sizeof(env_name_testing) - 1] = '\0'; + + strncat(env_name_testing, "-test", sizeof(env_name_testing) - strlen(env_name_testing) - 1); + env_name_testing[sizeof(env_name_testing) - 1] = '\0'; + + char *envs[] = { + "release", env_name, + "testing", env_name_testing, + NULL, NULL, + }; + + check_release_history(&ctx); + sync_release_history(&ctx); + + check_conda_install_prefix(&ctx); + check_conda_prefix_length(&ctx); + setup_conda(&ctx, installer_url, sizeof(installer_url)); + configure_conda_base(&ctx, envs); + configure_conda_purge(&ctx, envs); + setup_activate_test_env(&ctx, env_name_testing); + + configure_tool_versions(&ctx); + install_packaging_tools(); + configure_package_overlay(&ctx, env_name); + configure_deferred_packages(&ctx); + + show_overview(&ctx); + run_tests(&ctx); + build_conda_recipes(&ctx); + build_wheel_packages(&ctx); + generate_release(&ctx, env_name, env_name_testing, user_disabled_docker); + transfer_artifacts(&ctx); + msg(STASIS_MSG_L1, "Cleaning up\n"); delivery_free(&ctx); globals_free(); diff --git a/src/cli/stasis/system_requirements.c b/src/cli/stasis/system_requirements.c index d8d7df3..a48c113 100644 --- a/src/cli/stasis/system_requirements.c +++ b/src/cli/stasis/system_requirements.c @@ -3,20 +3,83 @@ void check_system_env_requirements() { msg(STASIS_MSG_L1, "Checking environment\n"); globals.envctl = envctl_init(); - envctl_register(&globals.envctl, STASIS_ENVCTL_PASSTHRU, NULL, "TMPDIR"); - envctl_register(&globals.envctl, STASIS_ENVCTL_PASSTHRU, NULL, "STASIS_ROOT"); - envctl_register(&globals.envctl, STASIS_ENVCTL_PASSTHRU, NULL, "STASIS_SYSCONFDIR"); - envctl_register(&globals.envctl, STASIS_ENVCTL_PASSTHRU, NULL, "STASIS_CPU_COUNT"); - envctl_register(&globals.envctl, STASIS_ENVCTL_REQUIRED | STASIS_ENVCTL_REDACT, callback_except_gh, "STASIS_GH_TOKEN"); - envctl_register(&globals.envctl, STASIS_ENVCTL_REQUIRED, callback_except_jf, "STASIS_JF_ARTIFACTORY_URL"); - envctl_register(&globals.envctl, STASIS_ENVCTL_REDACT, NULL, "STASIS_JF_ACCESS_TOKEN"); - envctl_register(&globals.envctl, STASIS_ENVCTL_PASSTHRU, NULL, "STASIS_JF_USER"); - envctl_register(&globals.envctl, STASIS_ENVCTL_REDACT, NULL, "STASIS_JF_PASSWORD"); - envctl_register(&globals.envctl, STASIS_ENVCTL_REDACT, NULL, "STASIS_JF_SSH_KEY_PATH"); - envctl_register(&globals.envctl, STASIS_ENVCTL_REDACT, NULL, "STASIS_JF_SSH_PASSPHRASE"); - envctl_register(&globals.envctl, STASIS_ENVCTL_REDACT, NULL, "STASIS_JF_CLIENT_CERT_CERT_PATH"); - envctl_register(&globals.envctl, STASIS_ENVCTL_REDACT, NULL, "STASIS_JF_CLIENT_CERT_KEY_PATH"); - envctl_register(&globals.envctl, STASIS_ENVCTL_REQUIRED, callback_except_jf, "STASIS_JF_REPO"); + if (!globals.envctl) { + SYSERROR("envctl_init failed"); + exit(1); + } + + int status = 0; + status = envctl_register(&globals.envctl, STASIS_ENVCTL_PASSTHRU, NULL, "TMPDIR"); + if (status) { + SYSERROR("envctl_register failed"); + exit(1); + } + status = envctl_register(&globals.envctl, STASIS_ENVCTL_PASSTHRU, NULL, "STASIS_ROOT"); + if (status) { + SYSERROR("envctl_register failed"); + exit(1); + } + status = envctl_register(&globals.envctl, STASIS_ENVCTL_PASSTHRU, NULL, "STASIS_SYSCONFDIR"); + if (status) { + SYSERROR("envctl_register failed"); + exit(1); + } + status = envctl_register(&globals.envctl, STASIS_ENVCTL_PASSTHRU, NULL, "STASIS_CPU_COUNT"); + if (status) { + SYSERROR("envctl_register failed"); + exit(1); + } + status = envctl_register(&globals.envctl, STASIS_ENVCTL_REQUIRED | STASIS_ENVCTL_REDACT, callback_except_gh, "STASIS_GH_TOKEN"); + if (status) { + SYSERROR("envctl_register failed"); + exit(1); + } + status = envctl_register(&globals.envctl, STASIS_ENVCTL_REQUIRED, callback_except_jf, "STASIS_JF_ARTIFACTORY_URL"); + if (status) { + SYSERROR("envctl_register failed"); + exit(1); + } + status = envctl_register(&globals.envctl, STASIS_ENVCTL_REDACT, NULL, "STASIS_JF_ACCESS_TOKEN"); + if (status) { + SYSERROR("envctl_register failed"); + exit(1); + } + status = envctl_register(&globals.envctl, STASIS_ENVCTL_PASSTHRU, NULL, "STASIS_JF_USER"); + if (status) { + SYSERROR("envctl_register failed"); + exit(1); + } + status = envctl_register(&globals.envctl, STASIS_ENVCTL_REDACT, NULL, "STASIS_JF_PASSWORD"); + if (status) { + SYSERROR("envctl_register failed"); + exit(1); + } + status = envctl_register(&globals.envctl, STASIS_ENVCTL_REDACT, NULL, "STASIS_JF_SSH_KEY_PATH"); + if (status) { + SYSERROR("envctl_register failed"); + exit(1); + } + status = envctl_register(&globals.envctl, STASIS_ENVCTL_REDACT, NULL, "STASIS_JF_SSH_PASSPHRASE"); + if (status) { + SYSERROR("envctl_register failed"); + exit(1); + } + status = envctl_register(&globals.envctl, STASIS_ENVCTL_REDACT, NULL, "STASIS_JF_CLIENT_CERT_CERT_PATH"); + if (status) { + SYSERROR("envctl_register failed"); + exit(1); + } + status = envctl_register(&globals.envctl, STASIS_ENVCTL_REDACT, NULL, "STASIS_JF_CLIENT_CERT_KEY_PATH"); + if (status) { + SYSERROR("envctl_register failed"); + exit(1); + } + status = envctl_register(&globals.envctl, STASIS_ENVCTL_REQUIRED, callback_except_jf, "STASIS_JF_REPO"); + if (status) { + SYSERROR("envctl_register failed"); + exit(1); + } + envctl_do_required(globals.envctl, globals.verbose); } @@ -27,36 +90,42 @@ void check_system_requirements(struct Delivery *ctx) { }; msg(STASIS_MSG_L1, "Checking system requirements\n"); + + msg(STASIS_MSG_L2, "Tools\n"); for (size_t i = 0; tools_required[i] != NULL; i++) { + msg(STASIS_MSG_L3, "%s: ", tools_required[i]); if (!find_program(tools_required[i])) { - msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "'%s' must be installed.\n", tools_required[i]); + SYSERROR("'%s' must be installed.", tools_required[i]); exit(1); } + msg(STASIS_MSG_RESTRICT, "found\n"); } - if (!globals.tmpdir && !ctx->storage.tmpdir) { - delivery_init_tmpdir(ctx); - } - - struct DockerCapabilities dcap; - if (!docker_capable(&dcap)) { - msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "Docker is broken\n"); - msg(STASIS_MSG_L3, "Available: %s\n", dcap.available ? "Yes" : "No"); - msg(STASIS_MSG_L3, "Usable: %s\n", dcap.usable ? "Yes" : "No"); - msg(STASIS_MSG_L3, "Podman [Docker Emulation]: %s\n", dcap.podman ? "Yes" : "No"); + msg(STASIS_MSG_L2, "Docker\n"); + if (docker_capable(&ctx->deploy.docker.capabilities)) { + struct DockerCapabilities *dcap = &ctx->deploy.docker.capabilities; + msg(STASIS_MSG_L3, "Available: %s%s%s\n", dcap->available ? STASIS_COLOR_GREEN : STASIS_COLOR_RED, dcap->available ? "Yes" : "No", STASIS_COLOR_RESET); + msg(STASIS_MSG_L3, "Usable: %s%s%s\n", dcap->usable ? STASIS_COLOR_GREEN : STASIS_COLOR_RED, dcap->usable ? "Yes" : "No", STASIS_COLOR_RESET); + msg(STASIS_MSG_L3, "Podman [Docker Emulation]: %s\n", dcap->podman ? "Yes" : "No"); msg(STASIS_MSG_L3, "Build plugin(s): "); - if (dcap.usable) { - if (dcap.build & STASIS_DOCKER_BUILD) { - printf("build "); + if (dcap->build) { + if (dcap->build & STASIS_DOCKER_BUILD) { + msg(STASIS_MSG_RESTRICT, "build "); } - if (dcap.build & STASIS_DOCKER_BUILD_X) { - printf("buildx "); + if (dcap->build & STASIS_DOCKER_BUILD_X) { + msg(STASIS_MSG_RESTRICT, "buildx "); } - puts(""); + msg(STASIS_MSG_RESTRICT,"\n"); } else { - printf("N/A\n"); + msg(STASIS_MSG_RESTRICT, "%sN/A%s\n", STASIS_COLOR_YELLOW, STASIS_COLOR_RESET); } + if (!dcap->usable) { + // disable docker builds + globals.enable_docker = false; + } + } else { + SYSWARN("Docker is broken"); // disable docker builds globals.enable_docker = false; } @@ -71,7 +140,7 @@ void check_system_path() { char *pathvar = NULL; pathvar = getenv("PATH"); if (!pathvar) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "PATH variable is not set. Cannot continue.\n"); + SYSERROR("PATH variable is not set. Cannot continue."); exit(1); } }
\ No newline at end of file diff --git a/src/cli/stasis_indexer/args.c b/src/cli/stasis_indexer/args.c index 2d92ab0..0d0e9b9 100644 --- a/src/cli/stasis_indexer/args.c +++ b/src/cli/stasis_indexer/args.c @@ -22,6 +22,10 @@ const char *long_options_help[] = { void usage(char *name) { const int maxopts = sizeof(long_options) / sizeof(long_options[0]); char *opts = calloc(maxopts + 1, sizeof(char)); + if (!opts) { + SYSERROR("Unable to allocate memory for options array"); + exit(1); + } for (int i = 0; i < maxopts; i++) { opts[i] = (char) long_options[i].val; } @@ -30,7 +34,7 @@ void usage(char *name) { for (int i = 0; i < maxopts - 1; i++) { char line[255] = {0}; - sprintf(line, " --%s -%c %-20s", long_options[i].name, long_options[i].val, long_options_help[i]); + snprintf(line, sizeof(line), " --%s -%c %-20s", long_options[i].name, long_options[i].val, long_options_help[i]); puts(line); } diff --git a/src/cli/stasis_indexer/callbacks.c b/src/cli/stasis_indexer/callbacks.c index 603aef9..20674f0 100644 --- a/src/cli/stasis_indexer/callbacks.c +++ b/src/cli/stasis_indexer/callbacks.c @@ -7,9 +7,9 @@ // qsort callback to sort delivery contexts by compact python version int callback_sort_deliveries_cmpfn(const void *a, const void *b) { - const struct Delivery *delivery1 = (struct Delivery *) a; + const struct Delivery *delivery1 = *(struct Delivery **) a; const size_t delivery1_python = strtoul(delivery1->meta.python_compact, NULL, 10); - const struct Delivery *delivery2 = (struct Delivery *) b; + const struct Delivery *delivery2 = *(struct Delivery **) b; const size_t delivery2_python = strtoul(delivery2->meta.python_compact, NULL, 10); if (delivery2_python > delivery1_python) { diff --git a/src/cli/stasis_indexer/helpers.c b/src/cli/stasis_indexer/helpers.c index 018a8f6..92e2dd4 100644 --- a/src/cli/stasis_indexer/helpers.c +++ b/src/cli/stasis_indexer/helpers.c @@ -2,27 +2,29 @@ // Created by jhunk on 11/15/24. // +#include <fnmatch.h> + #include "core.h" #include "helpers.h" -struct StrList *get_architectures(struct Delivery ctx[], const size_t nelem) { +struct StrList *get_architectures(struct Delivery **ctx, const size_t nelem) { struct StrList *architectures = strlist_init(); for (size_t i = 0; i < nelem; i++) { - if (ctx[i].system.arch) { - if (!strstr_array(architectures->data, ctx[i].system.arch)) { - strlist_append(&architectures, ctx[i].system.arch); + if (ctx[i]->system.arch) { + if (!strstr_array(architectures->data, ctx[i]->system.arch)) { + strlist_append(&architectures, ctx[i]->system.arch); } } } return architectures; } -struct StrList *get_platforms(struct Delivery ctx[], const size_t nelem) { +struct StrList *get_platforms(struct Delivery **ctx, const size_t nelem) { struct StrList *platforms = strlist_init(); for (size_t i = 0; i < nelem; i++) { - if (ctx[i].system.platform) { - if (!strstr_array(platforms->data, ctx[i].system.platform[DELIVERY_PLATFORM_RELEASE])) { - strlist_append(&platforms, ctx[i].system.platform[DELIVERY_PLATFORM_RELEASE]); + if (ctx[i]->system.platform) { + if (!strstr_array(platforms->data, ctx[i]->system.platform[DELIVERY_PLATFORM_RELEASE])) { + strlist_append(&platforms, ctx[i]->system.platform[DELIVERY_PLATFORM_RELEASE]); } } } @@ -87,7 +89,7 @@ int get_pandoc_version(size_t *result) { int pandoc_exec(const char *in_file, const char *out_file, const char *css_file, const char *title) { if (!find_program("pandoc")) { - fprintf(stderr, "pandoc is not installed: unable to generate HTML indexes\n"); + SYSWARN("pandoc is not installed: unable to generate HTML indexes"); return 0; } @@ -96,44 +98,48 @@ int pandoc_exec(const char *in_file, const char *out_file, const char *css_file, if (!get_pandoc_version(&pandoc_version)) { // < 2.19 if (pandoc_version < 0x02130000) { - strcat(pandoc_versioned_args, "--self-contained "); + strncat(pandoc_versioned_args, "--self-contained ", sizeof(pandoc_versioned_args) - strlen(pandoc_versioned_args) - 1); } else { // >= 2.19 - strcat(pandoc_versioned_args, "--embed-resources "); + strncat(pandoc_versioned_args, "--embed-resources ", sizeof(pandoc_versioned_args) - strlen(pandoc_versioned_args) - 1); } // >= 1.15.0.4 if (pandoc_version >= 0x010f0004) { - strcat(pandoc_versioned_args, "--standalone "); + strncat(pandoc_versioned_args, "--standalone ", sizeof(pandoc_versioned_args) - strlen(pandoc_versioned_args) - 1); } // >= 1.10.0.1 if (pandoc_version >= 0x010a0001) { - strcat(pandoc_versioned_args, "-f gfm+autolink_bare_uris "); + strncat(pandoc_versioned_args, "-f gfm+autolink_bare_uris ", sizeof(pandoc_versioned_args) - strlen(pandoc_versioned_args) - 1); } // > 3.1.9 if (pandoc_version > 0x03010900) { - strcat(pandoc_versioned_args, "-f gfm+alerts "); + strncat(pandoc_versioned_args, "-f gfm+alerts ", sizeof(pandoc_versioned_args) - strlen(pandoc_versioned_args) - 1); } } // Converts a markdown file to html char cmd[STASIS_BUFSIZ] = {0}; - strcpy(cmd, "pandoc "); - strcat(cmd, pandoc_versioned_args); + strncpy(cmd, "pandoc ", sizeof(cmd) - 1); + cmd[sizeof(cmd) - 1] = '\0'; + + strncat(cmd, pandoc_versioned_args, sizeof(cmd) - strlen(cmd) - 1); + cmd[sizeof(cmd) - 1] = '\0'; + if (css_file && strlen(css_file)) { - strcat(cmd, "--css "); - strcat(cmd, css_file); + strncat(cmd, "--css ", sizeof(cmd) - strlen(cmd) - 1); + strncat(cmd, css_file, sizeof(cmd) - strlen(cmd) - 1); } - strcat(cmd, " "); - strcat(cmd, "--metadata title=\""); - strcat(cmd, title); - strcat(cmd, "\" "); - strcat(cmd, "-o "); - strcat(cmd, out_file); - strcat(cmd, " "); - strcat(cmd, in_file); + strncat(cmd, " ", sizeof(cmd) - strlen(cmd) - 1); + strncat(cmd, "--metadata title=\"", sizeof(cmd) - strlen(cmd) - 1); + strncat(cmd, title, sizeof(cmd) - strlen(cmd) - 1); + strncat(cmd, "\" ", sizeof(cmd) - strlen(cmd) - 1); + strncat(cmd, "-o ", sizeof(cmd) - strlen(cmd) - 1); + strncat(cmd, out_file, sizeof(cmd) - strlen(cmd) - 1); + strncat(cmd, " ", sizeof(cmd) - strlen(cmd) - 1); + strncat(cmd, in_file, sizeof(cmd) - strlen(cmd) - 1); if (globals.verbose) { puts(cmd); @@ -152,6 +158,7 @@ int micromamba_configure(const struct Delivery *ctx, struct MicromambaInfo *m) { } m->conda_prefix = globals.conda_install_prefix; m->micromamba_prefix = micromamba_prefix; + m->download_dir = ctx->storage.tmpdir; const size_t pathvar_len = strlen(getenv("PATH")) + strlen(m->micromamba_prefix) + strlen(m->conda_prefix) + 3 + 4 + 1; // ^^^^^^^^^^^^^^^^^^ @@ -160,7 +167,7 @@ int micromamba_configure(const struct Delivery *ctx, struct MicromambaInfo *m) { // 1 = nul terminator char *pathvar = calloc(pathvar_len, sizeof(*pathvar)); if (!pathvar) { - SYSERROR("%s", "Unable to allocate bytes for temporary path string"); + SYSERROR("Unable to allocate bytes for temporary path string"); exit(1); } snprintf(pathvar, pathvar_len, "%s/bin:%s:%s", m->conda_prefix, m->micromamba_prefix, getenv("PATH")); @@ -177,19 +184,19 @@ int micromamba_configure(const struct Delivery *ctx, struct MicromambaInfo *m) { return status; } -int get_latest_rc(struct Delivery ctx[], const size_t nelem) { +int get_latest_rc(struct Delivery **ctx, const size_t nelem) { int result = 0; for (size_t i = 0; i < nelem; i++) { - if (ctx[i].meta.rc > result) { - result = ctx[i].meta.rc; + if (ctx[i]->meta.rc > result) { + result = ctx[i]->meta.rc; } } return result; } int sort_by_latest_rc(const void *a, const void *b) { - const struct Delivery *aa = a; - const struct Delivery *bb = b; + const struct Delivery *aa = *(struct Delivery **) a; + const struct Delivery *bb = *(struct Delivery **) b; if (aa->meta.rc > bb->meta.rc) { return -1; } else if (aa->meta.rc < bb->meta.rc) { @@ -214,24 +221,28 @@ int sort_by_latest_rc(const void *a, const void *b) { } } -struct Delivery *get_latest_deliveries(struct Delivery ctx[], size_t nelem) { +struct Delivery **get_latest_deliveries(struct Delivery **ctx, size_t nelem, size_t *result_nelem) { int latest = 0; size_t n = 0; - struct Delivery *result = calloc(nelem + 1, sizeof(*result)); + struct Delivery **result = calloc(nelem + 1, sizeof(*result)); if (!result) { - fprintf(stderr, "Unable to allocate %zu bytes for result delivery array: %s\n", nelem * sizeof(*result), strerror(errno)); + SYSERROR("Unable to allocate %zu bytes for result delivery array: %s", nelem * sizeof(*result), strerror(errno)); return NULL; } latest = get_latest_rc(ctx, nelem); qsort(ctx, nelem, sizeof(*ctx), sort_by_latest_rc); for (size_t i = 0; i < nelem; i++) { - if (ctx[i].meta.rc == latest) { + if (ctx[i]->meta.rc == latest) { result[n] = ctx[i]; n++; } } + + if (result_nelem) { + *result_nelem = n; + } return result; } @@ -239,7 +250,15 @@ int get_files(struct StrList **out, const char *path, const char *pattern, ...) va_list args; va_start(args, pattern); char userpattern[PATH_MAX] = {0}; - vsprintf(userpattern, pattern, args); + const int len = vsnprintf(userpattern, sizeof(userpattern), pattern, args); + if (len < 0) { + SYSERROR("vsnprintf failed"); + va_end(args); + return -1; + } + if ((size_t) len > sizeof(userpattern)) { + SYSWARN("userpattern truncated!"); + } va_end(args); if (!strlen(userpattern)) { userpattern[0] = '*'; @@ -268,7 +287,7 @@ int get_files(struct StrList **out, const char *path, const char *pattern, ...) } } if (no_match >= strlist_count(list)) { - fprintf(stderr, "no files matching the pattern: %s\n", userpattern); + SYSERROR("no files matching the pattern: %s", userpattern); guard_strlist_free(&list); return -1; } @@ -278,9 +297,12 @@ int get_files(struct StrList **out, const char *path, const char *pattern, ...) struct StrList *get_docker_images(struct Delivery *ctx, char *pattern) { char *tarball = NULL; - asprintf(&tarball, "%s*.tar*", pattern); + if (asprintf(&tarball, "%s*.tar*", pattern) < 0) { + SYSERROR("unable to allocate bytes for tarball pattern: %s", pattern); + return NULL; + } if (!tarball) { - SYSERROR("%s", "Unable to allocate bytes for docker image wildcard pattern"); + SYSERROR("Unable to allocate bytes for docker image wildcard pattern"); return NULL; } tolower_s(tarball); @@ -301,13 +323,22 @@ int load_metadata(struct Delivery *ctx, const char *filename) { return -1; } + // Reserved for future use. + // i.e. adjust values based on the version of the software that produced the input + char *stasis_version = NULL; + char *stasis_version_branch = NULL; + while (fgets(line, sizeof(line) - 1, fp) != NULL) { char **parts = split(line, " ", 1); const char *name = parts[0]; char *value = parts[1]; strip(value); - if (!strcmp(name, "name")) { + if (!strcmp(name, "stasis_version")) { + stasis_version = strdup(value); + } else if (!strcmp(name, "stasis_version_branch")) { + stasis_version_branch = strdup(value); + } else if (!strcmp(name, "name")) { ctx->meta.name = strdup(value); } else if (!strcmp(name, "version")) { ctx->meta.version = strdup(value); @@ -352,6 +383,9 @@ int load_metadata(struct Delivery *ctx, const char *filename) { } guard_array_free(parts); } + + guard_free(stasis_version); + guard_free(stasis_version_branch); fclose(fp); return 0; @@ -361,7 +395,7 @@ int write_manifest(const char *path, char **exclude_path, FILE *fp) { struct dirent *rec = NULL; DIR *dp = opendir(path); if (!dp) { - perror(path); + SYSERROR("unable to open directory: %s, %s", path ? path : "NULL", strerror(errno)); return -1; } while ((rec = readdir(dp)) != NULL) { @@ -373,8 +407,11 @@ int write_manifest(const char *path, char **exclude_path, FILE *fp) { } char filepath[PATH_MAX] = {0}; strncpy(filepath, path, PATH_MAX - 1); - strcat(filepath, "/"); - strcat(filepath, rec->d_name); + filepath[PATH_MAX - 1] = '\0'; + + strncat(filepath, "/", sizeof(filepath) - strlen(filepath) - 1); + strncat(filepath, rec->d_name, sizeof(filepath) - strlen(filepath) - 1); + if (rec->d_type == DT_DIR) { write_manifest(filepath, exclude_path, fp); continue; diff --git a/src/cli/stasis_indexer/include/helpers.h b/src/cli/stasis_indexer/include/helpers.h index 9cefc05..4a51f8f 100644 --- a/src/cli/stasis_indexer/include/helpers.h +++ b/src/cli/stasis_indexer/include/helpers.h @@ -2,6 +2,7 @@ #define HELPERS_H #include "delivery.h" +#include "conda.h" #define ARRAY_COUNT_DYNAMIC(X, COUNTER) \ do { \ @@ -13,12 +14,12 @@ for ((COUNTER) = 0; (X)[COUNTER].MEMBER != NULL; (COUNTER)++) {} \ } while(0) -struct StrList *get_architectures(struct Delivery ctx[], size_t nelem); -struct StrList *get_platforms(struct Delivery ctx[], size_t nelem); +struct StrList *get_architectures(struct Delivery **ctx, size_t nelem); +struct StrList *get_platforms(struct Delivery **ctx, size_t nelem); int get_pandoc_version(size_t *result); int pandoc_exec(const char *in_file, const char *out_file, const char *css_file, const char *title); -int get_latest_rc(struct Delivery ctx[], size_t nelem); -struct Delivery *get_latest_deliveries(struct Delivery ctx[], size_t nelem); +int get_latest_rc(struct Delivery **ctx, size_t nelem); +struct Delivery **get_latest_deliveries(struct Delivery **ctx, size_t nelem, size_t *result_nelem); int get_files(struct StrList **out, const char *path, const char *pattern, ...); struct StrList *get_docker_images(struct Delivery *ctx, char *pattern); int load_metadata(struct Delivery *ctx, const char *filename); diff --git a/src/cli/stasis_indexer/include/junitxml_report.h b/src/cli/stasis_indexer/include/junitxml_report.h index 6d2a248..5747359 100644 --- a/src/cli/stasis_indexer/include/junitxml_report.h +++ b/src/cli/stasis_indexer/include/junitxml_report.h @@ -3,6 +3,6 @@ #include "helpers.h" -int indexer_junitxml_report(struct Delivery ctx[], size_t nelem); +int indexer_junitxml_report(struct Delivery **ctx, size_t nelem); #endif //JUNITXML_REPORT_H diff --git a/src/cli/stasis_indexer/include/readmes.h b/src/cli/stasis_indexer/include/readmes.h index d4fa7ac..e14e681 100644 --- a/src/cli/stasis_indexer/include/readmes.h +++ b/src/cli/stasis_indexer/include/readmes.h @@ -3,6 +3,6 @@ #include "helpers.h" -int indexer_readmes(struct Delivery ctx[], size_t nelem); +int indexer_readmes(struct Delivery **ctx, size_t nelem); #endif //READMES_H diff --git a/src/cli/stasis_indexer/include/website.h b/src/cli/stasis_indexer/include/website.h index e67d58b..83657a1 100644 --- a/src/cli/stasis_indexer/include/website.h +++ b/src/cli/stasis_indexer/include/website.h @@ -3,6 +3,6 @@ #include "helpers.h" -int indexer_make_website(const struct Delivery *ctx); +int indexer_make_website(struct Delivery **ctx); #endif //WEBSITE_H diff --git a/src/cli/stasis_indexer/junitxml_report.c b/src/cli/stasis_indexer/junitxml_report.c index 904a3e5..a7dcd06 100644 --- a/src/cli/stasis_indexer/junitxml_report.c +++ b/src/cli/stasis_indexer/junitxml_report.c @@ -2,6 +2,8 @@ // Created by jhunk on 11/15/24. // +#include <fnmatch.h> + #include "core.h" #include "callbacks.h" #include "junitxml.h" @@ -28,7 +30,17 @@ static int write_report_output(struct Delivery *ctx, FILE *destfp, const char *x } char *bname_tmp = strdup(xmlfilename); + if (!bname_tmp) { + SYSERROR("unable to allocate bytes for temporary basename"); + return -1; + } + char *bname = strdup(path_basename(bname_tmp)); + if (!bname) { + SYSERROR("unable to allocate bytes for basename"); + return -1; + } + if (endswith(bname, ".xml")) { bname[strlen(bname) - 4] = 0; } @@ -36,10 +48,16 @@ static int write_report_output(struct Delivery *ctx, FILE *destfp, const char *x char result_outfile[PATH_MAX] = {0}; char *short_name_pattern = NULL; - asprintf(&short_name_pattern, "-%s", ctx->info.release_name); + if (asprintf(&short_name_pattern, "-%s", ctx->info.release_name) < 0 || !short_name_pattern) { + SYSERROR("unable to allocate bytes for short name pattern"); + guard_free(bname); + return -1; + } char short_name[PATH_MAX] = {0}; strncpy(short_name, bname, sizeof(short_name) - 1); + short_name[sizeof(short_name) - 1] = '\0'; + replace_text(short_name, short_name_pattern, "", 0); replace_text(short_name, "results-", "", 0); guard_free(short_name_pattern); @@ -52,8 +70,7 @@ static int write_report_output(struct Delivery *ctx, FILE *destfp, const char *x testsuite->passed, testsuite->failures, testsuite->skipped, testsuite->errors); - snprintf(result_outfile, sizeof(result_outfile) - strlen(bname) - 3, "%s.md", - bname); + snprintf(result_outfile, sizeof(result_outfile) - strlen(bname), "%s.md", bname); guard_free(bname); FILE *resultfp = fopen(result_outfile, "w+"); @@ -91,44 +108,44 @@ static int write_report_output(struct Delivery *ctx, FILE *destfp, const char *x junitxml_testsuite_free(&testsuite); fclose(resultfp); } else { - fprintf(stderr, "bad test suite: %s: %s\n", strerror(errno), xmlfilename); + SYSWARN("bad test suite: %s: %s", strerror(errno), xmlfilename); } return 0; } -int indexer_junitxml_report(struct Delivery ctx[], const size_t nelem) { +int indexer_junitxml_report(struct Delivery **ctx, const size_t nelem) { char indexfile[PATH_MAX] = {0}; - sprintf(indexfile, "%s/README.md", ctx->storage.results_dir); + snprintf(indexfile, sizeof(indexfile), "%s/README.md", (*ctx)->storage.results_dir); - struct StrList *file_listing = listdir(ctx->storage.results_dir); + struct StrList *file_listing = listdir((*ctx)->storage.results_dir); if (!file_listing) { // no test results to process return 0; } - if (!pushd(ctx->storage.results_dir)) { + if (!pushd((*ctx)->storage.results_dir)) { FILE *indexfp = fopen(indexfile, "w+"); if (!indexfp) { - fprintf(stderr, "Unable to open %s for writing\n", indexfile); + SYSERROR("Unable to open %s for writing", indexfile); return -1; } printf("Index %s opened for writing\n", indexfile); - int current_rc = ctx->meta.rc; + int current_rc = (*ctx)->meta.rc; for (size_t d = 0; d < nelem; d++) { char pattern[PATH_MAX] = {0}; - snprintf(pattern, sizeof(pattern) - 1, "*%s*", ctx[d].info.release_name); + snprintf(pattern, sizeof(pattern), "*%s*", ctx[d]->info.release_name); // if the result directory contains tests for this release name, print them if (!is_file_in_listing(file_listing, pattern)) { // no test results continue; } - if (current_rc > ctx[d].meta.rc) { - current_rc = ctx[d].meta.rc; + if (current_rc > ctx[d]->meta.rc) { + current_rc = ctx[d]->meta.rc; fprintf(indexfp, "\n\n---\n\n"); } - fprintf(indexfp, "### %s\n", ctx[d].info.release_name); + fprintf(indexfp, "### %s\n", ctx[d]->info.release_name); fprintf(indexfp, "\n|Suite|Duration|Total|Pass|Fail|Skip|Error|\n"); fprintf(indexfp, "|:----|:------:|:---:|:--:|:--:|:--:|:---:|\n"); @@ -139,7 +156,7 @@ int indexer_junitxml_report(struct Delivery ctx[], const size_t nelem) { continue; } if (!fnmatch(pattern, filename, 0)) { - if (write_report_output(&ctx[d], indexfp, filename)) { + if (write_report_output(ctx[d], indexfp, filename)) { // warn only SYSERROR("Unable to write xml report file using %s", filename); } @@ -150,7 +167,7 @@ int indexer_junitxml_report(struct Delivery ctx[], const size_t nelem) { fclose(indexfp); popd(); } else { - fprintf(stderr, "Unable to enter delivery directory: %s\n", ctx->storage.delivery_dir); + SYSERROR("Unable to enter delivery directory: %s", (*ctx)->storage.delivery_dir); guard_strlist_free(&file_listing); return -1; } diff --git a/src/cli/stasis_indexer/readmes.c b/src/cli/stasis_indexer/readmes.c index 413a6a3..836df5c 100644 --- a/src/cli/stasis_indexer/readmes.c +++ b/src/cli/stasis_indexer/readmes.c @@ -1,8 +1,9 @@ #include "core.h" #include "readmes.h" -int indexer_readmes(struct Delivery ctx[], const size_t nelem) { - struct Delivery *latest_deliveries = get_latest_deliveries(ctx, nelem); +int indexer_readmes(struct Delivery **ctx, const size_t nelem) { + size_t nelem_real = 0; + struct Delivery **latest_deliveries = get_latest_deliveries(ctx, nelem, &nelem_real); if (!latest_deliveries) { if (errno) { return -1; @@ -11,17 +12,17 @@ int indexer_readmes(struct Delivery ctx[], const size_t nelem) { } char indexfile[PATH_MAX] = {0}; - sprintf(indexfile, "%s/README.md", ctx->storage.delivery_dir); + snprintf(indexfile, sizeof(indexfile), "%s/README.md", (*ctx)->storage.delivery_dir); FILE *indexfp = fopen(indexfile, "w+"); if (!indexfp) { - fprintf(stderr, "Unable to open %s for writing\n", indexfile); + SYSERROR("Unable to open %s for writing", indexfile); return -1; } - struct StrList *archs = get_architectures(latest_deliveries, nelem); - struct StrList *platforms = get_platforms(latest_deliveries, nelem); + struct StrList *archs = get_architectures(latest_deliveries, nelem_real); + struct StrList *platforms = get_platforms(latest_deliveries, nelem_real); - fprintf(indexfp, "# %s-%s\n\n", ctx->meta.name, ctx->meta.version); + fprintf(indexfp, "# %s-%s\n\n", (*ctx)->meta.name, (*ctx)->meta.version); fprintf(indexfp, "## Current Release\n\n"); strlist_sort(platforms, STASIS_SORT_ALPHA); strlist_sort(archs, STASIS_SORT_ALPHA); @@ -31,10 +32,10 @@ int indexer_readmes(struct Delivery ctx[], const size_t nelem) { for (size_t a = 0; a < strlist_count(archs); a++) { char *arch = strlist_item(archs, a); int have_combo = 0; - for (size_t i = 0; i < nelem; i++) { - if (latest_deliveries[i].system.platform) { - if (strstr(latest_deliveries[i].system.platform[DELIVERY_PLATFORM_RELEASE], platform) && - strstr(latest_deliveries[i].system.arch, arch)) { + for (size_t i = 0; i < nelem_real; i++) { + if (latest_deliveries[i]->system.platform) { + if (strstr(latest_deliveries[i]->system.platform[DELIVERY_PLATFORM_RELEASE], platform) && + strstr(latest_deliveries[i]->system.arch, arch)) { have_combo = 1; } } @@ -43,36 +44,37 @@ int indexer_readmes(struct Delivery ctx[], const size_t nelem) { continue; } fprintf(indexfp, "### %s-%s\n\n", platform, arch); - for (size_t i = 0; i < nelem; i++) { + for (size_t i = 0; i < nelem_real; i++) { char link_name[PATH_MAX] = {0}; char readme_name[PATH_MAX] = {0}; char conf_name[PATH_MAX] = {0}; char conf_name_relative[PATH_MAX] = {0}; - if (!latest_deliveries[i].meta.name) { + if (!latest_deliveries[i]->meta.name) { continue; } - sprintf(link_name, "latest-py%s-%s-%s.yml", latest_deliveries[i].meta.python_compact, latest_deliveries[i].system.platform[DELIVERY_PLATFORM_RELEASE], latest_deliveries[i].system.arch); - sprintf(readme_name, "README-py%s-%s-%s.md", latest_deliveries[i].meta.python_compact, latest_deliveries[i].system.platform[DELIVERY_PLATFORM_RELEASE], latest_deliveries[i].system.arch); - sprintf(conf_name, "%s.ini", latest_deliveries[i].info.release_name); - sprintf(conf_name_relative, "../config/%s.ini", latest_deliveries[i].info.release_name); + snprintf(link_name, sizeof(link_name), "latest-py%s-%s-%s.yml", latest_deliveries[i]->meta.python_compact, latest_deliveries[i]->system.platform[DELIVERY_PLATFORM_RELEASE], latest_deliveries[i]->system.arch); + snprintf(readme_name, sizeof(readme_name), "README-py%s-%s-%s.md", latest_deliveries[i]->meta.python_compact, latest_deliveries[i]->system.platform[DELIVERY_PLATFORM_RELEASE], latest_deliveries[i]->system.arch); + snprintf(conf_name, sizeof(conf_name), "%s.ini", latest_deliveries[i]->info.release_name); + snprintf(conf_name_relative, sizeof(conf_name_relative), "../config/%s.ini", latest_deliveries[i]->info.release_name); if (strstr(link_name, platform) && strstr(link_name, arch)) { - fprintf(indexfp, "- Python %s\n", latest_deliveries[i].meta.python); + fprintf(indexfp, "- Python %s\n", latest_deliveries[i]->meta.python); fprintf(indexfp, " - Info: [README](%s)\n", readme_name); fprintf(indexfp, " - Release: [Conda Environment YAML](%s)\n", link_name); fprintf(indexfp, " - Receipt: [STASIS input file](%s)\n", conf_name_relative); char *pattern = NULL; - asprintf(&pattern, "*%s*%s*", - latest_deliveries[i].info.build_number, - strstr(ctx->rules.release_fmt, "%p") ? latest_deliveries[i].meta.python_compact : "" ); - if (!pattern) { - SYSERROR("%s", "Unable to allocate bytes for pattern"); + if (asprintf(&pattern, "*%s*%s*", + latest_deliveries[i]->info.build_number, + strstr((*ctx)->rules.release_fmt, "%p") ? latest_deliveries[i]->meta.python_compact : "" ) < 0) { + SYSERROR("Unable to allocate bytes for pattern"); + fclose(indexfp); return -1; } - struct StrList *docker_images = get_docker_images(&latest_deliveries[i], pattern); + + struct StrList *docker_images = get_docker_images(latest_deliveries[i], pattern); if (docker_images && strlist_count(docker_images) - && !strcmp(latest_deliveries[i].system.platform[DELIVERY_PLATFORM_RELEASE], "linux")) { + && !strcmp(latest_deliveries[i]->system.platform[DELIVERY_PLATFORM_RELEASE], "linux")) { fprintf(indexfp, " - Docker: "); fprintf(indexfp, "[Archive](../packages/docker/%s)\n", path_basename(strlist_item(docker_images, 0))); } @@ -86,9 +88,9 @@ int indexer_readmes(struct Delivery ctx[], const size_t nelem) { } fprintf(indexfp, "## Releases\n"); - int current_rc = ctx->meta.rc; - for (size_t i = 0; ctx[i].meta.name != NULL; i++) { - struct Delivery *current = &ctx[i]; + int current_rc = (*ctx)->meta.rc; + for (size_t i = 0; i < nelem; i++) { + struct Delivery *current = ctx[i]; if (current_rc > current->meta.rc) { current_rc = current->meta.rc; fprintf(indexfp, "\n\n---\n\n"); @@ -99,11 +101,11 @@ int indexer_readmes(struct Delivery ctx[], const size_t nelem) { fprintf(indexfp, "- Receipt: [STASIS input file](../config/%s.ini)\n", current->info.release_name); char *pattern = NULL; - asprintf(&pattern, "*%s*%s*", + if (asprintf(&pattern, "*%s*%s*", current->info.build_number, - strstr(ctx->rules.release_fmt, "%p") ? current->meta.python_compact : "" ); - if (!pattern) { - SYSERROR("%s", "Unable to allocate bytes for pattern"); + strstr((*ctx)->rules.release_fmt, "%p") ? current->meta.python_compact : "" ) < 0) { + SYSERROR("Unable to allocate bytes for pattern"); + fclose(indexfp); return -1; } diff --git a/src/cli/stasis_indexer/stasis_indexer_main.c b/src/cli/stasis_indexer/stasis_indexer_main.c index 279af5a..e87122e 100644 --- a/src/cli/stasis_indexer/stasis_indexer_main.c +++ b/src/cli/stasis_indexer/stasis_indexer_main.c @@ -8,43 +8,49 @@ #include "delivery.h" int indexer_combine_rootdirs(const char *dest, char **rootdirs, const size_t rootdirs_total) { - char cmd[PATH_MAX]; - char destdir_bare[PATH_MAX]; - char destdir_with_output[PATH_MAX]; + char cmd[PATH_MAX] = {0}; + char destdir_bare[PATH_MAX] = {0}; + char destdir_with_output[PATH_MAX] = {0}; char *destdir = destdir_bare; - memset(cmd, 0, sizeof(cmd)); - memset(destdir_bare, 0, sizeof(destdir_bare)); - memset(destdir_with_output, 0, sizeof(destdir_bare)); + strncpy(destdir_bare, dest, sizeof(destdir_bare) - 1); + destdir[sizeof(destdir_bare) - 1] = '\0'; + + strncpy(destdir_with_output, dest, sizeof(destdir_with_output) - 1); + destdir_with_output[sizeof(destdir_with_output) - 1] = '\0'; - strcpy(destdir_bare, dest); - strcpy(destdir_with_output, dest); - strcat(destdir_with_output, "/output"); + strncat(destdir_with_output, "/output", sizeof(destdir_with_output) - strlen(destdir_with_output) - 1); + destdir_with_output[sizeof(destdir_with_output) - 1] = '\0'; if (!access(destdir_with_output, F_OK)) { destdir = destdir_with_output; } - sprintf(cmd, "rsync -ah%s --delete --exclude 'tools/' --exclude 'tmp/' --exclude 'build/' ", globals.verbose ? "v" : "q"); + snprintf(cmd, sizeof(cmd), "rsync -ah%s --delete --exclude 'tools/' --exclude 'tmp/' --exclude 'build/' ", globals.verbose ? "v" : "q"); for (size_t i = 0; i < rootdirs_total; i++) { char srcdir_bare[PATH_MAX] = {0}; char srcdir_with_output[PATH_MAX] = {0}; char *srcdir = srcdir_bare; - strcpy(srcdir_bare, rootdirs[i]); - strcpy(srcdir_with_output, rootdirs[i]); - strcat(srcdir_with_output, "/output"); + strncpy(srcdir_bare, rootdirs[i], sizeof(srcdir_bare) - 1); + srcdir_bare[sizeof(srcdir_bare) - 1] = '\0'; + + strncpy(srcdir_with_output, rootdirs[i], sizeof(srcdir_with_output) - 1); + srcdir_with_output[sizeof(srcdir_with_output) - 1] = '\0'; + + strncat(srcdir_with_output, "/output", sizeof(srcdir_with_output) - strlen(srcdir_with_output) - 1); + srcdir_with_output[sizeof(srcdir_with_output) - 1] = '\0'; if (access(srcdir_bare, F_OK)) { - fprintf(stderr, "%s does not exist\n", srcdir_bare); + SYSWARN("%s does not exist", srcdir_bare); continue; } if (!access(srcdir_with_output, F_OK)) { srcdir = srcdir_with_output; } - snprintf(cmd + strlen(cmd), sizeof(srcdir) - strlen(srcdir) + 4, "'%s'/ ", srcdir); + snprintf(cmd + strlen(cmd), sizeof(cmd) - strlen(cmd) - non_format_len("'%s'/ "), "'%s'/ ", srcdir); } - snprintf(cmd + strlen(cmd), sizeof(cmd) - strlen(destdir) + 1, " %s/", destdir); + snprintf(cmd + strlen(cmd), sizeof(cmd) - strlen(cmd) - non_format_len(" %s/"), " %s/", destdir); if (globals.verbose) { puts(cmd); @@ -67,36 +73,37 @@ int indexer_conda(const struct Delivery *ctx, struct MicromambaInfo m) { return status; } -int indexer_symlinks(struct Delivery *ctx, const size_t nelem) { - struct Delivery *data = NULL; - data = get_latest_deliveries(ctx, nelem); +int indexer_symlinks(struct Delivery **ctx, const size_t nelem) { + struct Delivery **data = NULL; + size_t nelem_real = 0; + data = get_latest_deliveries(ctx, nelem, &nelem_real); //int latest = get_latest_rc(ctx, nelem); - if (!pushd(ctx->storage.delivery_dir)) { - for (size_t i = 0; i < nelem; i++) { + if (!pushd((*ctx)->storage.delivery_dir)) { + for (size_t i = 0; i < nelem_real; i++) { char link_name_spec[PATH_MAX]; char link_name_readme[PATH_MAX]; char file_name_spec[PATH_MAX]; char file_name_readme[PATH_MAX]; - if (!data[i].meta.name) { + if (!data[i]->meta.name) { continue; } - sprintf(link_name_spec, "latest-py%s-%s-%s.yml", data[i].meta.python_compact, data[i].system.platform[DELIVERY_PLATFORM_RELEASE], data[i].system.arch); - sprintf(file_name_spec, "%s.yml", data[i].info.release_name); + snprintf(link_name_spec, sizeof(link_name_spec), "latest-py%s-%s-%s.yml", data[i]->meta.python_compact, data[i]->system.platform[DELIVERY_PLATFORM_RELEASE], data[i]->system.arch); + snprintf(file_name_spec, sizeof(file_name_spec), "%s.yml", data[i]->info.release_name); - sprintf(link_name_readme, "README-py%s-%s-%s.md", data[i].meta.python_compact, data[i].system.platform[DELIVERY_PLATFORM_RELEASE], data[i].system.arch); - sprintf(file_name_readme, "README-%s.md", data[i].info.release_name); + snprintf(link_name_readme, sizeof(link_name_readme), "README-py%s-%s-%s.md", data[i]->meta.python_compact, data[i]->system.platform[DELIVERY_PLATFORM_RELEASE], data[i]->system.arch); + snprintf(file_name_readme, sizeof(file_name_readme), "README-%s.md", data[i]->info.release_name); if (!access(link_name_spec, F_OK)) { if (unlink(link_name_spec)) { - fprintf(stderr, "Unable to remove spec link: %s\n", link_name_spec); + SYSWARN("Unable to remove spec link: %s", link_name_spec); } } if (!access(link_name_readme, F_OK)) { if (unlink(link_name_readme)) { - fprintf(stderr, "Unable to remove readme link: %s\n", link_name_readme); + SYSWARN("Unable to remove readme link: %s", link_name_readme); } } @@ -104,19 +111,19 @@ int indexer_symlinks(struct Delivery *ctx, const size_t nelem) { printf("%s -> %s\n", file_name_spec, link_name_spec); } if (symlink(file_name_spec, link_name_spec)) { - fprintf(stderr, "Unable to link %s as %s\n", file_name_spec, link_name_spec); + SYSWARN("Unable to link %s as %s", file_name_spec, link_name_spec); } if (globals.verbose) { printf("%s -> %s\n", file_name_readme, link_name_readme); } if (symlink(file_name_readme, link_name_readme)) { - fprintf(stderr, "Unable to link %s as %s\n", file_name_readme, link_name_readme); + SYSWARN("Unable to link %s as %s", file_name_readme, link_name_readme); } } popd(); } else { - fprintf(stderr, "Unable to enter delivery directory: %s\n", ctx->storage.delivery_dir); + SYSERROR("Unable to enter delivery directory: %s", (*ctx)->storage.delivery_dir); guard_free(data); return -1; } @@ -130,13 +137,13 @@ void indexer_init_dirs(struct Delivery *ctx, const char *workdir) { path_store(&ctx->storage.root, PATH_MAX, workdir, ""); path_store(&ctx->storage.tmpdir, PATH_MAX, ctx->storage.root, "tmp"); if (delivery_init_tmpdir(ctx)) { - fprintf(stderr, "Failed to configure temporary storage directory\n"); + SYSERROR("Failed to configure temporary storage directory"); exit(1); } char *user_dir = expandpath("~/.stasis/indexer"); if (!user_dir) { - SYSERROR("%s", "expandpath failed"); + SYSERROR("expandpath failed"); } path_store(&ctx->storage.output_dir, PATH_MAX, ctx->storage.root, ""); @@ -154,10 +161,10 @@ void indexer_init_dirs(struct Delivery *ctx, const char *workdir) { char newpath[PATH_MAX] = {0}; if (getenv("PATH")) { - sprintf(newpath, "%s/bin:%s", ctx->storage.tools_dir, getenv("PATH")); + snprintf(newpath, sizeof(newpath), "%s/bin:%s", ctx->storage.tools_dir, getenv("PATH")); setenv("PATH", newpath, 1); } else { - SYSERROR("%s", "environment variable PATH is undefined. Unable to continue."); + SYSERROR("environment variable PATH is undefined. Unable to continue."); exit(1); } } @@ -191,6 +198,7 @@ int main(const int argc, char *argv[]) { break; case 'v': globals.verbose = 1; + LOG_LEVEL++; break; case 'w': do_html = 1; @@ -205,17 +213,25 @@ int main(const int argc, char *argv[]) { if (optind < argc) { rootdirs_total = argc - current_index; rootdirs = calloc(rootdirs_total + 1, sizeof(*rootdirs)); + if (!rootdirs) { + SYSERROR("unable to allocate memory for rootdirs array"); + exit(1); + } int i = 0; while (optind < argc) { if (argv[optind]) { if (access(argv[optind], F_OK) < 0) { - fprintf(stderr, "%s: %s\n", argv[optind], strerror(errno)); + SYSERROR("%s: %s", argv[optind], strerror(errno)); exit(1); } } // use first positional argument rootdirs[i] = realpath(argv[optind], NULL); + if (!rootdirs[i]) { + SYSERROR("Unable to allocate memory for root directory"); + exit(1); + } optind++; break; } @@ -232,7 +248,7 @@ int main(const int argc, char *argv[]) { } if (!rootdirs || !rootdirs_total) { - fprintf(stderr, "You must specify at least one STASIS root directory to index\n"); + SYSERROR("You must specify at least one STASIS root directory to index"); exit(1); } @@ -254,21 +270,28 @@ int main(const int argc, char *argv[]) { } else { strncpy(stasis_sysconfdir_tmp, STASIS_SYSCONFDIR, sizeof(stasis_sysconfdir_tmp) - 1); } + stasis_sysconfdir_tmp[sizeof(stasis_sysconfdir_tmp) - 1] = '\0'; globals.sysconfdir = realpath(stasis_sysconfdir_tmp, NULL); if (!globals.sysconfdir) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Unable to resolve path to configuration directory: %s\n", stasis_sysconfdir_tmp); + SYSERROR("Unable to resolve path to configuration directory: %s", stasis_sysconfdir_tmp); exit(1); } char workdir_template[PATH_MAX] = {0}; const char *system_tmp = getenv("TMPDIR"); if (system_tmp) { - strcat(workdir_template, system_tmp); + strncat(workdir_template, system_tmp, sizeof(workdir_template) - strlen(workdir_template) - 1); } else { - strcat(workdir_template, "/tmp"); + strncat(workdir_template, "/tmp/stasis", sizeof(workdir_template) - strlen(workdir_template) - 1); + } + + if (mkdirs(workdir_template, 0700)) { + SYSERROR("Unable to create directory '%s': %s", workdir_template, strerror(errno)); + exit(1); } - strcat(workdir_template, "/stasis-combine.XXXXXX"); + + strncat(workdir_template, "/stasis-combine.XXXXXX", sizeof(workdir_template) - strlen(workdir_template) - 1); char *workdir = mkdtemp(workdir_template); if (!workdir) { SYSERROR("Unable to create temporary directory: %s", workdir_template); @@ -281,7 +304,17 @@ int main(const int argc, char *argv[]) { struct Delivery ctx = {0}; - printf(BANNER, VERSION, AUTHOR); + char *version = center_text(VERSION, strlen(STASIS_BANNER_HEADER)); + if (!version) { + SYSERROR("%s", "version too long?"); + version = strdup(VERSION); + if (!version) { + SYSERROR("%s", "unable to allocate uncentered fallback version string"); + exit(1); + } + } + printf(BANNER, version, AUTHOR); + guard_free(version); indexer_init_dirs(&ctx, workdir); @@ -289,7 +322,7 @@ int main(const int argc, char *argv[]) { rootdirs_total > 1 ? "Merging" : "Indexing", rootdirs_total > 1 ? "directories" : "directory"); if (indexer_combine_rootdirs(workdir, rootdirs, rootdirs_total)) { - SYSERROR("%s", "Copy operation failed"); + SYSERROR("Copy operation failed"); rmtree(workdir); exit(1); } @@ -302,68 +335,78 @@ int main(const int argc, char *argv[]) { mkdirs(ctx.storage.wheel_artifact_dir, 0755); } + msg(STASIS_MSG_L1, "Configure Micromamba\n"); struct MicromambaInfo m; if (micromamba_configure(&ctx, &m)) { - SYSERROR("%s", "Unable to configure micromamba"); + SYSERROR("Unable to configure micromamba"); exit(1); } msg(STASIS_MSG_L1, "Indexing conda packages\n"); if (indexer_conda(&ctx, m)) { - SYSERROR("%s", "Conda package indexing operation failed"); + SYSERROR("Conda package indexing operation failed"); exit(1); } msg(STASIS_MSG_L1, "Indexing wheel packages\n"); if (indexer_wheels(&ctx)) { - SYSERROR("%s", "Python package indexing operation failed"); + SYSERROR("Python package indexing operation failed"); exit(1); } msg(STASIS_MSG_L1, "Loading metadata\n"); struct StrList *metafiles = NULL; get_files(&metafiles, ctx.storage.meta_dir, "*.stasis"); + if (!metafiles || !strlist_count(metafiles)) { + SYSERROR("%s: No metadata!", ctx.storage.meta_dir); + delivery_free(&ctx); + exit(1); + } strlist_sort(metafiles, STASIS_SORT_LEN_ASCENDING); - struct Delivery *local = calloc(strlist_count(metafiles) + 1, sizeof(*local)); + struct Delivery **local = calloc(strlist_count(metafiles) + 1, sizeof(*local)); if (!local) { - SYSERROR("%s", "Unable to allocate bytes for local delivery context array"); + SYSERROR("Unable to allocate bytes for local delivery context array"); exit(1); } for (size_t i = 0; i < strlist_count(metafiles); i++) { char *item = strlist_item(metafiles, i); // Copy the pre-filled contents of the main delivery context - memcpy(&local[i], &ctx, sizeof(ctx)); + local[i] = delivery_duplicate(&ctx); + if (!local[i]) { + SYSERROR("Unable to duplicate delivery context %zu", i); + exit(1); + } if (globals.verbose) { puts(item); } - load_metadata(&local[i], item); + load_metadata(local[i], item); } qsort(local, strlist_count(metafiles), sizeof(*local), callback_sort_deliveries_cmpfn); msg(STASIS_MSG_L1, "Generating links to latest release iteration\n"); if (indexer_symlinks(local, strlist_count(metafiles))) { - SYSERROR("%s", "Link generation failed"); + SYSERROR("Link generation failed"); exit(1); } msg(STASIS_MSG_L1, "Generating README.md\n"); if (indexer_readmes(local, strlist_count(metafiles))) { - SYSERROR("%s", "README indexing operation failed"); + SYSERROR("README indexing operation failed"); exit(1); } msg(STASIS_MSG_L1, "Indexing test results\n"); if (indexer_junitxml_report(local, strlist_count(metafiles))) { - SYSERROR("%s", "Test result indexing operation failed"); + SYSERROR("Test result indexing operation failed"); exit(1); } if (do_html) { msg(STASIS_MSG_L1, "Generating HTML indexes\n"); if (indexer_make_website(local)) { - SYSERROR("%s", "Site creation failed"); + SYSERROR("Site creation failed"); exit(1); } } @@ -410,7 +453,7 @@ int main(const int argc, char *argv[]) { msg(STASIS_MSG_L1, "Copying indexed delivery to '%s'\n", destdir); char cmd[PATH_MAX] = {0}; - sprintf(cmd, "rsync -ah%s --delete --exclude 'tmp/' --exclude 'tools/' '%s/' '%s/'", globals.verbose ? "v" : "q", workdir, destdir); + snprintf(cmd, sizeof(cmd), "rsync -ah%s --delete --exclude 'tmp/' --exclude 'tools/' '%s/' '%s/'", globals.verbose ? "v" : "q", workdir, destdir); guard_free(destdir); if (globals.verbose) { @@ -418,7 +461,7 @@ int main(const int argc, char *argv[]) { } if (system(cmd)) { - SYSERROR("%s", "Copy operation failed"); + SYSERROR("Copy operation failed"); rmtree(workdir); exit(1); } @@ -430,10 +473,14 @@ int main(const int argc, char *argv[]) { guard_free(destdir); guard_array_free(rootdirs); - guard_strlist_free(&metafiles); guard_free(m.micromamba_prefix); delivery_free(&ctx); + for (size_t i = 0; i < strlist_count(metafiles); i++) { + delivery_free(local[i]); + guard_free(local[i]); + } guard_free(local); + guard_strlist_free(&metafiles); globals_free(); msg(STASIS_MSG_L1, "Done!\n"); diff --git a/src/cli/stasis_indexer/website.c b/src/cli/stasis_indexer/website.c index 55f0c45..07ad6ad 100644 --- a/src/cli/stasis_indexer/website.c +++ b/src/cli/stasis_indexer/website.c @@ -1,19 +1,19 @@ #include "core.h" #include "website.h" -int indexer_make_website(const struct Delivery *ctx) { +int indexer_make_website(struct Delivery **ctx) { char *css_filename = calloc(PATH_MAX, sizeof(*css_filename)); if (!css_filename) { SYSERROR("unable to allocate string for CSS file path: %s", strerror(errno)); return -1; } - sprintf(css_filename, "%s/%s", globals.sysconfdir, "stasis_pandoc.css"); + snprintf(css_filename, PATH_MAX, "%s/%s", globals.sysconfdir, "stasis_pandoc.css"); const int have_css = access(css_filename, F_OK | R_OK) == 0; struct StrList *dirs = strlist_init(); - strlist_append(&dirs, ctx->storage.delivery_dir); - strlist_append(&dirs, ctx->storage.results_dir); + strlist_append(&dirs, (*ctx)->storage.delivery_dir); + strlist_append(&dirs, (*ctx)->storage.results_dir); struct StrList *inputs = NULL; for (size_t i = 0; i < strlist_count(dirs); i++) { @@ -29,33 +29,36 @@ int indexer_make_website(const struct Delivery *ctx) { char *filename = path_basename(strlist_item(inputs, x)); char fullpath_src[PATH_MAX] = {0}; char fullpath_dest[PATH_MAX] = {0}; - sprintf(fullpath_src, "%s/%s", root, filename); + snprintf(fullpath_src, sizeof(fullpath_src), "%s/%s", root, filename); if (access(fullpath_src, F_OK)) { continue; } // Replace *.md extension with *.html. - strcpy(fullpath_dest, fullpath_src); - gen_file_extension_str(fullpath_dest, ".html"); + strncpy(fullpath_dest, fullpath_src, sizeof(fullpath_dest) - 1); + fullpath_dest[sizeof(fullpath_dest) - 1] = '\0'; + + gen_file_extension_str(fullpath_dest, sizeof(fullpath_dest), ".html"); // Convert markdown to html if (pandoc_exec(fullpath_src, fullpath_dest, have_css ? css_filename : NULL, "STASIS")) { - msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "Unable to convert %s\n", fullpath_src); + SYSWARN("Unable to convert %s", fullpath_src); } if (file_replace_text(fullpath_dest, ".md", ".html", 0)) { // inform-only - SYSERROR("%s: failed to rewrite *.md urls with *.html extension", fullpath_dest); + SYSWARN("%s: failed to rewrite *.md urls with *.html extension", fullpath_dest); } // Link the nearest README.html to index.html if (!strcmp(filename, "README.md")) { char link_from[PATH_MAX] = {0}; char link_dest[PATH_MAX] = {0}; - strcpy(link_from, "README.html"); - sprintf(link_dest, "%s/%s", root, "index.html"); + strncpy(link_from, "README.html", sizeof(link_from) - 1); + link_dest[sizeof(link_dest) - 1] = '\0'; + snprintf(link_dest, sizeof(link_dest), "%s/%s", root, "index.html"); if (symlink(link_from, link_dest)) { - SYSERROR("Warning: symlink(%s, %s) failed: %s", link_from, link_dest, strerror(errno)); + SYSWARN("symlink(%s, %s) failed: %s", link_from, link_dest, strerror(errno)); } } } diff --git a/src/lib/core/CMakeLists.txt b/src/lib/core/CMakeLists.txt index e3e3d4b..3a54d94 100644 --- a/src/lib/core/CMakeLists.txt +++ b/src/lib/core/CMakeLists.txt @@ -1,5 +1,7 @@ add_library(stasis_core STATIC globals.c + log.c + timespec.c str.c strlist.c ini.c @@ -10,6 +12,7 @@ add_library(stasis_core STATIC download.c recipe.c relocation.c + wheelinfo.c wheel.c copy.c artifactory.c @@ -21,9 +24,14 @@ add_library(stasis_core STATIC template_func_proto.c envctl.c multiprocessing.c + semaphore.c + version_compare.c ) target_include_directories(stasis_core PRIVATE ${core_INCLUDE} ${delivery_INCLUDE} + ${ZIP_INCLUDEDIR} ${CMAKE_CURRENT_SOURCE_DIR}/include ) +target_link_libraries(stasis_core PRIVATE + ${ZIP_LIBRARIES}) diff --git a/src/lib/core/artifactory.c b/src/lib/core/artifactory.c index eedaf43..d4b48fb 100644 --- a/src/lib/core/artifactory.c +++ b/src/lib/core/artifactory.c @@ -8,43 +8,65 @@ int artifactory_download_cli(char *dest, char *os, char *arch, char *remote_filename) { + SYSDEBUG("ARGS follow"); + SYSDEBUG("dest=%s", dest); + SYSDEBUG("jfrog_artifactory_base_url=%s", jfrog_artifactory_base_url); + SYSDEBUG("jfrog_artifactory_product=%s", jfrog_artifactory_product); + SYSDEBUG("cli_major_ver=%s", cli_major_ver); + SYSDEBUG("version=%s", version); + SYSDEBUG("os=%s", os); + SYSDEBUG("arch=%s", arch); + SYSDEBUG("remote_filename=%s", remote_filename); + char url[PATH_MAX] = {0}; char path[PATH_MAX] = {0}; char os_ident[STASIS_NAME_MAX] = {0}; char arch_ident[STASIS_NAME_MAX] = {0}; // convert platform string to lower-case - strcpy(os_ident, os); + SYSDEBUG("Set os_ident"); + strncpy(os_ident, os, sizeof(os_ident) - 1); + os_ident[sizeof(os_ident) - 1] = '\0'; tolower_s(os_ident); + SYSDEBUG("os_ident=%s", os_ident); // translate OS identifier if (!strcmp(os_ident, "darwin") || startswith(os_ident, "macos")) { - strcpy(os_ident, "mac"); + strncpy(os_ident, "mac", sizeof(os_ident) - 1); + os_ident[sizeof(os_ident) - 1] = '\0'; } else if (!strcmp(os_ident, "linux")) { - strcpy(os_ident, "linux"); + strncpy(os_ident, "linux", sizeof(os_ident) - 1); + os_ident[sizeof(os_ident) - 1] = '\0'; } else { - fprintf(stderr, "%s: unknown operating system: %s\n", __FUNCTION__, os_ident); + SYSERROR("unknown operating system: %s", os_ident); return -1; } // translate ARCH identifier - strcpy(arch_ident, arch); + SYSDEBUG("Set arch_ident"); + strncpy(arch_ident, arch, sizeof(arch_ident) - 1); + arch_ident[sizeof(arch_ident) - 1] = '\0'; + SYSDEBUG("arch_ident=%s", arch_ident); + if (startswith(arch_ident, "i") && endswith(arch_ident, "86")) { - strcpy(arch_ident, "386"); + strncpy(arch_ident, "386", sizeof(arch_ident) - 1); } else if (!strcmp(arch_ident, "amd64") || !strcmp(arch_ident, "x86_64") || !strcmp(arch_ident, "x64")) { if (!strcmp(os_ident, "mac")) { - strcpy(arch_ident, "386"); + strncpy(arch_ident, "386", sizeof(arch_ident) - 1); } else { - strcpy(arch_ident, "amd64"); + strncpy(arch_ident, "amd64", sizeof(arch_ident) - 1); } } else if (!strcmp(arch_ident, "arm64") || !strcmp(arch_ident, "aarch64")) { - strcpy(arch_ident, "arm64"); + strncpy(arch_ident, "arm64", sizeof(arch_ident) - 1); } else { - fprintf(stderr, "%s: unknown architecture: %s\n", __FUNCTION__, arch_ident); + SYSERROR("unknown architecture: %s", arch_ident); return -1; } + arch_ident[sizeof(arch_ident) - 1] = '\0'; - snprintf(url, sizeof(url) - 1, "%s/%s/%s/%s/%s-%s-%s/%s", + + SYSDEBUG("Construct URL"); + snprintf(url, sizeof(url), "%s/%s/%s/%s/%s-%s-%s/%s", jfrog_artifactory_base_url, // https://releases.jfrog.io/artifactory jfrog_artifactory_product, // jfrog-cli cli_major_ver, // v\d+(-jf)? @@ -53,17 +75,26 @@ int artifactory_download_cli(char *dest, os_ident, // ... arch_ident, // jfrog-cli-linux-x86_64 remote_filename); // jf - strcpy(path, dest); + strncpy(path, dest, sizeof(path) - 1); + path[sizeof(path) - 1] = '\0'; if (mkdirs(path, 0755)) { - fprintf(stderr, "%s: %s: %s", __FUNCTION__, path, strerror(errno)); + SYSERROR("%s: %s", path, strerror(errno)); return -1; } - sprintf(path + strlen(path), "/%s", remote_filename); - long fetch_status = download(url, path, NULL); - if (HTTP_ERROR(fetch_status) || fetch_status < 0) { - fprintf(stderr, "%s: download failed: %s\n", __FUNCTION__, url); + SYSDEBUG("Construct path to write data"); + SYSDEBUG("path buffer contents: '%s'", path); + SYSDEBUG("path buffer size: %zu", sizeof(path)); + SYSDEBUG("path strlen: %zu", strlen(path)); + SYSDEBUG("maxlen for snprintf: %zu", sizeof(path) - strlen(path)); + + snprintf(path + strlen(path), sizeof(path) - strlen(path), "/%s", remote_filename); + char *errmsg = NULL; + SYSDEBUG("Downloading..."); + long fetch_status = download(url, path, &errmsg); + if (HTTP_ERROR(fetch_status)) { + SYSERROR("download failed: %s: %s", errmsg, url); return -1; } chmod(path, 0755); @@ -77,7 +108,7 @@ void jfrt_register_opt_str(char *jfrt_val, const char *opt_name, struct StrList // no data return; } - snprintf(data, sizeof(data) - 1, "--%s=\"%s\"", opt_name, jfrt_val); + snprintf(data, sizeof(data), "--%s=\"%s\"", opt_name, jfrt_val); strlist_append(&*opt_map, data); } @@ -88,7 +119,7 @@ void jfrt_register_opt_bool(bool jfrt_val, const char *opt_name, struct StrList // option will not be used return; } - snprintf(data, sizeof(data) - 1, "--%s", opt_name); + snprintf(data, sizeof(data), "--%s", opt_name); strlist_append(&*opt_map, data); } @@ -99,7 +130,7 @@ void jfrt_register_opt_int(int jfrt_val, const char *opt_name, struct StrList ** // option will not be used return; } - snprintf(data, sizeof(data) - 1, "--%s=%d", opt_name, jfrt_val); + snprintf(data, sizeof(data), "--%s=%d", opt_name, jfrt_val); strlist_append(&*opt_map, data); } @@ -110,7 +141,7 @@ void jfrt_register_opt_long(long jfrt_val, const char *opt_name, struct StrList // option will not be used return; } - snprintf(data, sizeof(data) - 1, "--%s=%ld", opt_name, jfrt_val); + snprintf(data, sizeof(data), "--%s=%ld", opt_name, jfrt_val); strlist_append(&*opt_map, data); } @@ -145,8 +176,8 @@ int jfrt_auth_init(struct JFRT_Auth *auth_ctx) { char *client_cert_path = getenv("STASIS_JF_CLIENT_CERT_PATH"); if (!url) { - fprintf(stderr, "Artifactory URL is not configured:\n"); - fprintf(stderr, "please set STASIS_JF_ARTIFACTORY_URL\n"); + SYSERROR("Artifactory URL is not configured:"); + SYSERROR("please set STASIS_JF_ARTIFACTORY_URL"); return -1; } auth_ctx->url = url; @@ -177,11 +208,11 @@ int jfrt_auth_init(struct JFRT_Auth *auth_ctx) { auth_ctx->client_cert_key_path = client_cert_key_path; auth_ctx->client_cert_path = client_cert_path; } else { - fprintf(stderr, "Artifactory authentication is not configured:\n"); - fprintf(stderr, "set STASIS_JF_USER and STASIS_JF_PASSWORD\n"); - fprintf(stderr, "or, set STASIS_JF_ACCESS_TOKEN\n"); - fprintf(stderr, "or, set STASIS_JF_SSH_KEY_PATH and STASIS_JF_SSH_KEY_PASSPHRASE\n"); - fprintf(stderr, "or, set STASIS_JF_CLIENT_CERT_KEY_PATH and STASIS_JF_CLIENT_CERT_PATH\n"); + SYSERROR("Artifactory authentication is not configured:"); + SYSERROR("set STASIS_JF_USER and STASIS_JF_PASSWORD"); + SYSERROR("or, set STASIS_JF_ACCESS_TOKEN"); + SYSERROR("or, set STASIS_JF_SSH_KEY_PATH and STASIS_JF_SSH_KEY_PASSPHRASE"); + SYSERROR("or, set STASIS_JF_CLIENT_CERT_KEY_PATH and STASIS_JF_CLIENT_CERT_PATH"); return -1; } return 0; @@ -229,7 +260,7 @@ int jfrog_cli(struct JFRT_Auth *auth, const char *subsystem, const char *task, c auth->client_cert_path, auth->password, }; - snprintf(cmd, sizeof(cmd) - 1, "jf %s %s %s %s", subsystem, task, auth_args, args ? args : ""); + snprintf(cmd, sizeof(cmd), "jf %s %s %s %s", subsystem, task, auth_args, args ? args : ""); redact_sensitive(redactable, sizeof(redactable) / sizeof (*redactable), cmd, cmd_redacted, sizeof(cmd_redacted) - 1); guard_free(auth_args); @@ -241,8 +272,11 @@ int jfrog_cli(struct JFRT_Auth *auth, const char *subsystem, const char *task, c } if (!globals.verbose) { - strcpy(proc.f_stdout, "/dev/null"); - strcpy(proc.f_stderr, "/dev/null"); + strncpy(proc.f_stdout, "/dev/null", sizeof(proc.f_stdout) - 1); + proc.f_stdout[sizeof(proc.f_stdout) - 1] = '\0'; + + strncpy(proc.f_stderr, "/dev/null", sizeof(proc.f_stderr) - 1); + proc.f_stderr[sizeof(proc.f_stderr) - 1] = '\0'; } return shell(&proc, cmd); } @@ -253,13 +287,13 @@ static int jfrog_cli_rt(struct JFRT_Auth *auth, char *task, char *args) { int jfrog_cli_rt_build_collect_env(struct JFRT_Auth *auth, char *build_name, char *build_number) { char cmd[STASIS_BUFSIZ] = {0}; - snprintf(cmd, sizeof(cmd) - 1, "\"%s\" \"%s\"", build_name, build_number); + snprintf(cmd, sizeof(cmd), "\"%s\" \"%s\"", build_name, build_number); return jfrog_cli(auth, "rt", "build-collect-env", cmd); } int jfrog_cli_rt_build_publish(struct JFRT_Auth *auth, char *build_name, char *build_number) { char cmd[STASIS_BUFSIZ] = {0}; - snprintf(cmd, sizeof(cmd) - 1, "\"%s\" \"%s\"", build_name, build_number); + snprintf(cmd, sizeof(cmd), "\"%s\" \"%s\"", build_name, build_number); return jfrog_cli(auth, "rt", "build-publish", cmd); } @@ -271,7 +305,7 @@ int jfrog_cli_rt_download(struct JFRT_Auth *auth, struct JFRT_Download *ctx, cha char cmd[STASIS_BUFSIZ] = {0}; if (isempty(repo_path)) { - fprintf(stderr, "repo_path argument must be a valid artifactory repository path\n"); + SYSERROR("repo_path argument must be a valid artifactory repository path"); return -1; } @@ -325,7 +359,7 @@ int jfrog_cli_rt_download(struct JFRT_Auth *auth, struct JFRT_Download *ctx, cha return -1; } - snprintf(cmd, sizeof(cmd) - 1, "%s '%s' '%s'", args, repo_path, dest ? dest : ""); + snprintf(cmd, sizeof(cmd), "%s '%s' '%s'", args, repo_path, dest ? dest : ""); guard_free(args); guard_strlist_free(&arg_map); @@ -337,12 +371,12 @@ int jfrog_cli_rt_upload(struct JFRT_Auth *auth, struct JFRT_Upload *ctx, char *s char cmd[STASIS_BUFSIZ] = {0}; if (isempty(src)) { - fprintf(stderr, "src argument must be a valid file system path\n"); + SYSERROR("src argument must be a valid file system path"); return -1; } if (isempty(repo_path)) { - fprintf(stderr, "repo_path argument must be a valid artifactory repository path\n"); + SYSERROR("repo_path argument must be a valid artifactory repository path"); return -1; } @@ -410,12 +444,12 @@ int jfrog_cli_rt_upload(struct JFRT_Auth *auth, struct JFRT_Upload *ctx, char *s if (base) { src = base; } else { - strcat(src, "/"); + strncat(src, "/", sizeof(src) - strlen(src) - 1); } pushd(new_src); } - snprintf(cmd, sizeof(cmd) - 1, "%s '%s' \"%s\"", args, src, repo_path); + snprintf(cmd, sizeof(cmd), "%s '%s' \"%s\"", args, src, repo_path); guard_free(args); guard_strlist_free(&arg_map); @@ -435,7 +469,7 @@ int jfrog_cli_rt_search(struct JFRT_Auth *auth, struct JFRT_Search *ctx, char *r char cmd[STASIS_BUFSIZ] = {0}; if (isempty(repo_path)) { - fprintf(stderr, "repo_path argument must be a valid artifactory repository path\n"); + SYSERROR("repo_path argument must be a valid artifactory repository path"); return -1; } @@ -474,7 +508,7 @@ int jfrog_cli_rt_search(struct JFRT_Auth *auth, struct JFRT_Search *ctx, char *r return -1; } - snprintf(cmd, sizeof(cmd) - 1, "%s '%s/%s'", args, repo_path, pattern ? pattern: ""); + snprintf(cmd, sizeof(cmd), "%s '%s/%s'", args, repo_path, pattern ? pattern: ""); guard_free(args); guard_strlist_free(&arg_map); diff --git a/src/lib/core/conda.c b/src/lib/core/conda.c index c81e6cc..6083023 100644 --- a/src/lib/core/conda.c +++ b/src/lib/core/conda.c @@ -4,36 +4,57 @@ #include "conda.h" -int micromamba(struct MicromambaInfo *info, char *command, ...) { +int micromamba(const struct MicromambaInfo *info, char *command, ...) { struct utsname sys; uname(&sys); tolower_s(sys.sysname); if (!strcmp(sys.sysname, "darwin")) { - strcpy(sys.sysname, "osx"); + strncpy(sys.sysname, "osx", sizeof(sys.sysname) - 1); + sys.sysname[sizeof(sys.sysname) - 1] = '\0'; } if (!strcmp(sys.machine, "x86_64")) { - strcpy(sys.machine, "64"); + strncpy(sys.machine, "64", sizeof(sys.machine) - 1); + sys.machine[sizeof(sys.machine) - 1] = '\0'; } - char url[PATH_MAX]; - sprintf(url, "https://micro.mamba.pm/api/micromamba/%s-%s/latest", sys.sysname, sys.machine); + if (!info->download_dir || isempty(info->download_dir)) { + SYSERROR("micromamba inf->download_dir is NULL, or empty"); + return -1; + } + + if (mkdirs(info->download_dir, 0755) < 0) { + SYSERROR("Unable to create info->download_dir: %s", info->download_dir); + return -1; + } - char installer_path[PATH_MAX]; - sprintf(installer_path, "%s/latest", getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"); + char url[PATH_MAX] = {0}; + snprintf(url, sizeof(url), "https://micro.mamba.pm/api/micromamba/%s-%s/latest", sys.sysname, sys.machine); + + const char installer_name[] = "mm_latest"; + char installer_path[PATH_MAX] = {0}; + snprintf(installer_path, sizeof(installer_path), "%s/%s", info->download_dir, installer_name); if (access(installer_path, F_OK)) { - download(url, installer_path, NULL); + char *errmsg = NULL; + const long http_code = download(url, installer_path, &errmsg); + if (HTTP_ERROR(http_code)) { + SYSERROR("download failed: %ld: %s", http_code, errmsg); + guard_free(errmsg); + return -1; + } } char mmbin[PATH_MAX]; - sprintf(mmbin, "%s/micromamba", info->micromamba_prefix); + snprintf(mmbin, sizeof(mmbin), "%s/micromamba", info->micromamba_prefix); if (access(mmbin, F_OK)) { char untarcmd[PATH_MAX * 2]; mkdirs(info->micromamba_prefix, 0755); - sprintf(untarcmd, "tar -xvf %s -C %s --strip-components=1 bin/micromamba 1>/dev/null", installer_path, info->micromamba_prefix); + snprintf(untarcmd, sizeof(untarcmd), + "tar -xvf %s -C %s --strip-components=1 bin/micromamba 1>/dev/null", + installer_path, info->micromamba_prefix); int untarcmd_status = system(untarcmd); if (untarcmd_status) { return -1; @@ -41,17 +62,41 @@ int micromamba(struct MicromambaInfo *info, char *command, ...) { } char cmd[STASIS_BUFSIZ] = {0}; - sprintf(cmd, "%s -r %s -p %s ", mmbin, info->conda_prefix, info->conda_prefix); va_list args; + int cmd_len = snprintf(cmd, sizeof(cmd), "%s -r %s -p %s ", mmbin, info->conda_prefix, info->conda_prefix); + if (cmd_len < 0) { + SYSERROR("Unable to build argument list for micromamba"); + va_end(args); + return -1; + } + if ((size_t) cmd_len > sizeof(cmd)) { + SYSERROR("micromamba command truncated"); + va_end(args); + return -1; + } + va_start(args, command); - vsprintf(cmd + strlen(cmd), command, args); + cmd_len = vsnprintf(cmd + strlen(cmd), sizeof(cmd) - strlen(cmd), command, args); + if (cmd_len < 0) { + SYSERROR("Unable to append arguments to micromamba command"); + va_end(args); + return -1; + } + if ((size_t) cmd_len > sizeof(cmd)) { + SYSERROR("micromamba command truncated while appending arguments"); + va_end(args); + return -1; + } va_end(args); mkdirs(info->conda_prefix, 0755); char rcpath[PATH_MAX]; - sprintf(rcpath, "%s/.condarc", info->conda_prefix); + snprintf(rcpath, sizeof(rcpath), "%s/.condarc", info->conda_prefix); touch(rcpath); + if (errno == ENOENT) { + errno = 0; + } setenv("CONDARC", rcpath, 1); setenv("MAMBA_ROOT_PREFIX", info->conda_prefix, 1); @@ -62,17 +107,33 @@ int micromamba(struct MicromambaInfo *info, char *command, ...) { } int python_exec(const char *args) { - char command[PATH_MAX] = {0}; - snprintf(command, sizeof(command) - 1, "python %s", args); + const char *command_base = "python "; + + char *command = NULL; + if (asprintf(&command, "%s%s", command_base, args) < 0 || !command) { + SYSERROR("Unable to allocate command string"); + return -1; + } msg(STASIS_MSG_L3, "Executing: %s\n", command); - return system(command); + + const int result = system(command); + guard_free(command); + return result; } int pip_exec(const char *args) { - char command[PATH_MAX] = {0}; - snprintf(command, sizeof(command) - 1, "python -m pip %s", args); + const char *command_base = "python -m pip "; + + char *command = NULL; + if (asprintf(&command, "%s%s", command_base, args) < 0 || !command) { + SYSERROR("Unable to allocate command string"); + return -1; + } msg(STASIS_MSG_L3, "Executing: %s\n", command); - return system(command); + + const int result = system(command); + guard_free(command); + return result; } static const char *PKG_ERROR_STR[] = { @@ -89,7 +150,7 @@ const char *pkg_index_provides_strerror(int code) { return PKG_ERROR_STR[code]; } -int pkg_index_provides(int mode, const char *index, const char *spec) { +int pkg_index_provides(int mode, const char *index, const char *spec, const char *logdir) { char cmd[PATH_MAX] = {0}; char spec_local[255] = {0}; @@ -100,14 +161,22 @@ int pkg_index_provides(int mode, const char *index, const char *spec) { // Normalize the local spec string strncpy(spec_local, spec, sizeof(spec_local) - 1); + spec_local[sizeof(spec_local) - 1] = '\0'; tolower_s(spec_local); lstrip(spec_local); strip(spec_local); - char logfile[] = "/tmp/STASIS-package_exists.XXXXXX"; + if (mkdirs(logdir, 0700) < 0) { + SYSERROR("Unable to create log directory: %s", logdir ? logdir : "NULL"); + return -1; + } + const char logfile_template[] = "STASIS-package_exists.XXXXXX"; + char logfile[PATH_MAX] = {0}; + snprintf(logfile, sizeof(logfile), "%s/%s", logdir, logfile_template); + int logfd = mkstemp(logfile); if (logfd < 0) { - perror(logfile); + SYSERROR("unable to create log file: %s", logfile); remove(logfile); // fail harmlessly if not present return PKG_INDEX_PROVIDES_E_INTERNAL_LOG_HANDLE; } @@ -115,21 +184,25 @@ int pkg_index_provides(int mode, const char *index, const char *spec) { int status = 0; struct Process proc = {0}; proc.redirect_stderr = 1; - strcpy(proc.f_stdout, logfile); + snprintf(proc.f_stdout, sizeof(proc.f_stdout), "%s", logfile); if (mode == PKG_USE_PIP) { // Do an installation in dry-run mode to see if the package exists in the given index. - strncpy(cmd, "python -m pip install --dry-run --no-cache --no-deps ", sizeof(cmd) - 1); + // The --force argument ignores local installation and cache, and actually polls the remote index(es) + strncpy(cmd, "python -m pip install --force --dry-run --no-cache --no-deps ", sizeof(cmd) - 1); + cmd[sizeof(cmd) - 1] = '\0'; if (index) { - snprintf(cmd + strlen(cmd), (sizeof(cmd) - 1) - strlen(cmd), "--index-url='%s' ", index); + snprintf(cmd + strlen(cmd), sizeof(cmd) - strlen(cmd), "--index-url='%s' ", index); } - snprintf(cmd + strlen(cmd), (sizeof(cmd) - 1) - strlen(cmd), "'%s' ", spec_local); + snprintf(cmd + strlen(cmd), sizeof(cmd) - strlen(cmd), "'%s' ", spec_local); } else if (mode == PKG_USE_CONDA) { strncpy(cmd, "mamba search ", sizeof(cmd) - 1); + cmd[sizeof(cmd) - 1] = '\0'; + if (index) { - snprintf(cmd + strlen(cmd), (sizeof(cmd) - 1) - strlen(cmd), "--channel '%s' ", index); + snprintf(cmd + strlen(cmd), sizeof(cmd) - strlen(cmd), "--channel '%s' ", index); } - snprintf(cmd + strlen(cmd), (sizeof(cmd) - 1) - strlen(cmd), "'%s' ", spec_local); + snprintf(cmd + strlen(cmd), sizeof(cmd) - strlen(cmd), "'%s' ", spec_local); } else { return PKG_INDEX_PROVIDES_E_INTERNAL_MODE_UNKNOWN; } @@ -137,22 +210,27 @@ int pkg_index_provides(int mode, const char *index, const char *spec) { // Print errors only when shell() itself throws one // If some day we want to see the errors thrown by pip too, use this // condition instead: (status != 0) + SYSDEBUG("Executing: %s", cmd); status = shell(&proc, cmd); - if (status < 0) { + + SYSDEBUG("Log file: %s", logfile); + if (status != 0) { FILE *fp = fdopen(logfd, "r"); if (!fp) { remove(logfile); return -1; - } else { - char line[BUFSIZ] = {0}; - fflush(stdout); - fflush(stderr); - while (fgets(line, sizeof(line) - 1, fp) != NULL) { - fprintf(stderr, "%s", line); - } - fflush(stderr); - fclose(fp); } + + fflush(stdout); + fflush(stderr); + + char line[STASIS_BUFSIZ] = {0}; + while (fgets(line, sizeof(line) - 1, fp) != NULL) { + SYSDEBUG("%s", strip(line)); + } + + fflush(stderr); + fclose(fp); } remove(logfile); @@ -177,7 +255,6 @@ int pkg_index_provides(int mode, const char *index, const char *spec) { } int conda_exec(const char *args) { - char command[PATH_MAX]; const char *mamba_commands[] = { "build", "install", @@ -192,25 +269,36 @@ int conda_exec(const char *args) { "deactivate", NULL }; - char conda_as[6] = {0}; + char conda_as[10] = {0}; - strcpy(conda_as, "conda"); + strncpy(conda_as, "conda", sizeof(conda_as) - 1); for (size_t i = 0; mamba_commands[i] != NULL; i++) { if (startswith(args, mamba_commands[i])) { - strcpy(conda_as, "mamba"); + strncpy(conda_as, "mamba", sizeof(conda_as) - 1); break; } } + conda_as[sizeof(conda_as) - 1] = '\0'; + + + const char *command_fmt = "%s %s"; + const int len = snprintf(NULL, 0, command_fmt, conda_as, args); + char *command = calloc(len + 1, sizeof(*command)); + if (!command) { + return -1; + } - snprintf(command, sizeof(command) - 1, "%s %s", conda_as, args); + snprintf(command, len + 1, command_fmt, conda_as, args); msg(STASIS_MSG_L3, "Executing: %s\n", command); - return system(command); + const int result = system(command); + guard_free(command); + return result; } static int conda_prepend_bin(const char *root) { char conda_bin[PATH_MAX] = {0}; - snprintf(conda_bin, sizeof(conda_bin) - 1, "%s/bin", root); + snprintf(conda_bin, sizeof(conda_bin), "%s/bin", root); if (env_manipulate_pathstr("PATH", conda_bin, PM_PREPEND | PM_ONCE)) { return -1; } @@ -220,7 +308,7 @@ static int conda_prepend_bin(const char *root) { static int conda_prepend_condabin(const char *root) { char conda_condabin[PATH_MAX] = {0}; - snprintf(conda_condabin, sizeof(conda_condabin) - 1, "%s/condabin", root); + snprintf(conda_condabin, sizeof(conda_condabin), "%s/condabin", root); if (env_manipulate_pathstr("PATH", conda_condabin, PM_PREPEND | PM_ONCE)) { return -1; } @@ -230,7 +318,7 @@ static int conda_prepend_condabin(const char *root) { static int env0_to_runtime(const char *logfile) { FILE *fp = fopen(logfile, "r"); if (!fp) { - perror(logfile); + SYSERROR("unable to open log file for reading: %s, %s", logfile, strerror(errno)); return -1; } @@ -255,13 +343,14 @@ static int env0_to_runtime(const char *logfile) { char **part = split(buf, "=", 1); if (!part) { - perror("unable to split environment variable buffer"); + SYSERROR("unable to split environment variable buffer: %s", strerror(errno)); + fclose(fp); return -1; } if (!part[0]) { - msg(STASIS_MSG_WARN | STASIS_MSG_L1, "Invalid environment variable key ignored: '%s'\n", buf); + SYSWARN("Invalid environment variable key ignored: '%s'", buf); } else if (!part[1]) { - msg(STASIS_MSG_WARN | STASIS_MSG_L1, "Invalid environment variable value ignored: '%s'\n", buf); + SYSWARN("Invalid environment variable value ignored: '%s'", buf); } else { setenv(part[0], part[1], 1); } @@ -280,33 +369,34 @@ int conda_activate(const char *root, const char *env_name) { struct Process proc = {0}; // Where to find conda's init scripts - sprintf(path_conda, "%s%s", root, init_script_conda); - sprintf(path_mamba, "%s%s", root, init_script_mamba); + snprintf(path_conda, sizeof(path_conda), "%s%s", root, init_script_conda); + snprintf(path_mamba, sizeof(path_mamba), "%s%s", root, init_script_mamba); // Set the path to our stdout log // Emulate mktemp()'s behavior. Give us a unique file name, but don't use // the file handle at all. We'll open it as a FILE stream soon enough. - sprintf(logfile, "%s/%s", globals.tmpdir, "shell_XXXXXX"); + snprintf(logfile, sizeof(logfile), "%s/%s", globals.tmpdir, "shell_XXXXXX"); int fd = mkstemp(logfile); if (fd < 0) { - perror(logfile); + SYSERROR("log file creation failed: %s, %s", logfile, strerror(errno)); return -1; } close(fd); // Configure our process for output to a log file - strcpy(proc.f_stdout, logfile); + strncpy(proc.f_stdout, logfile, sizeof(proc.f_stdout) - 1); + proc.f_stdout[sizeof(proc.f_stdout) - 1] = '\0'; // Verify conda's init scripts are available if (access(path_conda, F_OK) < 0) { - perror(path_conda); + SYSERROR("conda is missing: %s, %s", path_conda, strerror(errno)); remove(logfile); return -1; } if (access(path_mamba, F_OK) < 0) { - perror(path_mamba); + SYSERROR("mamba is missing: %s, %s", path_mamba, strerror(errno)); remove(logfile); return -1; } @@ -329,7 +419,7 @@ int conda_activate(const char *root, const char *env_name) { return -1; } - snprintf(command, sizeof(command) - 1, + snprintf(command, sizeof(command), "set -a\n" "source %s\n" "__conda_exe() (\n" @@ -376,15 +466,15 @@ int conda_check_required() { // Construct a "conda list" command that searches for all required packages // using conda's (python's) regex matching - strcat(cmd, "conda list '"); + strncat(cmd, "conda list '", sizeof(cmd) - strlen(cmd) - 1); for (size_t i = 0; conda_minimum_viable_tools[i] != NULL; i++) { - strcat(cmd, "^"); - strcat(cmd, conda_minimum_viable_tools[i]); + strncat(cmd, "^", sizeof(cmd) - strlen(cmd) - 1); + strncat(cmd, conda_minimum_viable_tools[i], sizeof(cmd) - strlen(cmd) - 1); if (conda_minimum_viable_tools[i + 1] != NULL) { - strcat(cmd, "|"); + strncat(cmd, "|", sizeof(cmd) - strlen(cmd) - 1); } } - strcat(cmd, "' | cut -d ' ' -f 1"); + strncat(cmd, "' | cut -d ' ' -f 1", sizeof(cmd) - strlen(cmd) - 1); // Verify all required packages are installed char *cmd_out = shell_output(cmd, &status); @@ -412,7 +502,7 @@ int conda_check_required() { guard_free(cmd_out); guard_strlist_free(&result); } else { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "The base package requirement check could not be performed\n"); + SYSERROR("The base package requirement check could not be performed"); return 2; } return 0; @@ -437,9 +527,11 @@ int conda_setup_headless() { char cmd[PATH_MAX]; size_t total = 0; + const char *cmd_fmt = "'%s'"; if (globals.conda_packages && strlist_count(globals.conda_packages)) { memset(cmd, 0, sizeof(cmd)); - strcpy(cmd, "install "); + strncpy(cmd, "install ", sizeof(cmd) - 1); + cmd[sizeof(cmd) - 1] = '\0'; total = strlist_count(globals.conda_packages); for (size_t i = 0; i < total; i++) { @@ -447,21 +539,23 @@ int conda_setup_headless() { if (isempty(item)) { continue; } - sprintf(cmd + strlen(cmd), "'%s'", item); + + snprintf(cmd + strlen(cmd), sizeof(cmd) - strlen(cmd), cmd_fmt, item); if (i < total - 1) { - strcat(cmd, " "); + strncat(cmd, " ", sizeof(cmd) - strlen(cmd) - 1); } } if (conda_exec(cmd)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Unable to install user-defined base packages (conda)\n"); + SYSERROR("Unable to install user-defined base packages (conda)"); return 1; } } if (globals.pip_packages && strlist_count(globals.pip_packages)) { memset(cmd, 0, sizeof(cmd)); - strcpy(cmd, "install "); + strncpy(cmd, "install ", sizeof(cmd) - 1); + cmd[sizeof(cmd) - 1] = '\0'; total = strlist_count(globals.pip_packages); for (size_t i = 0; i < total; i++) { @@ -469,28 +563,28 @@ int conda_setup_headless() { if (isempty(item)) { continue; } - sprintf(cmd + strlen(cmd), "'%s'", item); + snprintf(cmd + strlen(cmd), sizeof(cmd) - strlen(cmd), cmd_fmt, item); if (i < total - 1) { - strcat(cmd, " "); + strncat(cmd, " ", sizeof(cmd) - strlen(cmd) - 1); } } if (pip_exec(cmd)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Unable to install user-defined base packages (pip)\n"); + SYSERROR("Unable to install user-defined base packages (pip)"); return 1; } } if (conda_check_required()) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Your STASIS configuration lacks the bare" - " minimum software required to build conda packages." - " Please fix it.\n"); + SYSERROR("Your STASIS configuration lacks the bare" + " minimum software required to build conda packages." + " Please fix it."); return 1; } if (globals.always_update_base_environment) { if (conda_exec("update --all")) { - fprintf(stderr, "conda update was unsuccessful\n"); + SYSERROR("conda update was unsuccessful"); return 1; } } @@ -499,21 +593,19 @@ int conda_setup_headless() { } int conda_env_create_from_uri(char *name, char *uri, char *python_version) { - char env_command[PATH_MAX]; char *uri_fs = NULL; - // Convert a bare system path to a file:// path if (!strstr(uri, "://")) { - uri_fs = calloc(strlen(uri) + strlen("file://") + 1, sizeof(*uri_fs)); + uri_fs = calloc(PATH_MAX, sizeof(*uri_fs)); if (!uri_fs) { return -1; } - snprintf(uri_fs, strlen(uri) + strlen("file://") + 1, "%s%s", "file://", uri); + snprintf(uri_fs, PATH_MAX, "%s%s", "file://", uri); } char tempfile[PATH_MAX] = {0}; - sprintf(tempfile, "%s/remote_XXXXXX", globals.tmpdir); + snprintf(tempfile, sizeof(tempfile), "%s/remote_XXXXXX", globals.tmpdir); // Touch a temporary file int fd = mkstemp(tempfile); @@ -521,38 +613,62 @@ int conda_env_create_from_uri(char *name, char *uri, char *python_version) { unlink(tempfile); // We'll create a new file with the same random bits, ending with .yml - strcat(tempfile, ".yml"); + strncat(tempfile, ".yml", sizeof(tempfile) - strlen(tempfile) - 1); char *errmsg = NULL; - download(uri_fs ? uri_fs : uri, tempfile, &errmsg); + const long http_code = download(uri_fs ? uri_fs : uri, tempfile, &errmsg); + if (HTTP_ERROR(http_code)) { + if (errmsg) { + SYSERROR("download failed: %ld: %s", http_code, errmsg); + guard_free(errmsg); + } + guard_free(uri_fs); + return -1; + } guard_free(uri_fs); // Rewrite python version char spec[255] = {0}; - snprintf(spec, sizeof(spec) - 1, "- python=%s\n", python_version); + snprintf(spec, sizeof(spec), "- python=%s\n", python_version); file_replace_text(tempfile, "- python\n", spec, 0); - sprintf(env_command, "env create -n '%s' --file='%s'", name, tempfile); - int status = conda_exec(env_command); + const char *fmt = "env create -n '%s' --file='%s'"; + char *env_command = NULL; + if (asprintf(&env_command, fmt, name, tempfile) < 0 || !env_command) { + SYSERROR("unable to allocate environment command"); + return -1; + } + + const int status = conda_exec(env_command); unlink(tempfile); + guard_free(env_command); return status; } int conda_env_create(char *name, char *python_version, char *packages) { - char env_command[PATH_MAX]; - sprintf(env_command, "create -n %s python=%s %s", name, python_version, packages ? packages : ""); - return conda_exec(env_command); + const char *fmt = "create -n %s python=%s %s"; + const int len = snprintf(NULL, 0, fmt, name, python_version, packages ? packages : ""); + char *env_command = calloc(len + 1, sizeof(*env_command)); + if (!env_command) { + return -1; + } + + snprintf(env_command, len + 1, fmt, name, python_version, packages ? packages : ""); + const int result = conda_exec(env_command); + guard_free(env_command); + + return result; } int conda_env_remove(char *name) { char env_command[PATH_MAX]; - sprintf(env_command, "env remove -n %s", name); + snprintf(env_command, sizeof(env_command), "env remove -n %s", name); return conda_exec(env_command); } int conda_env_export(char *name, char *output_dir, char *output_filename) { char env_command[PATH_MAX]; - sprintf(env_command, "env export -n %s -f %s/%s.yml", name, output_dir, output_filename); + snprintf(env_command, sizeof(env_command), "env export -n %s -f %s/%s.yml", name, output_dir, output_filename); return conda_exec(env_command); } @@ -573,12 +689,12 @@ char *conda_get_active_environment() { int conda_index(const char *path) { char command[PATH_MAX]; - sprintf(command, "index %s", path); + snprintf(command, sizeof(command), "index %s", path); return conda_exec(command); } int conda_env_exists(const char *root, const char *name) { char path[PATH_MAX] = {0}; - snprintf(path, sizeof(path) - 1 - 6, "%s/envs/%s", root, name); + snprintf(path, sizeof(path), "%s/envs/%s", root, name); return access(path, F_OK) == 0; } diff --git a/src/lib/core/copy.c b/src/lib/core/copy.c index 25eede3..7666e67 100644 --- a/src/lib/core/copy.c +++ b/src/lib/core/copy.c @@ -5,7 +5,7 @@ int copy2(const char *src, const char *dest, unsigned int op) { SYSDEBUG("Stat source file: %s", src); if (lstat(src, &src_stat) < 0) { - perror(src); + SYSERROR("unable to stat source file: %s, %s", src, strerror(errno)); return -1; } @@ -14,7 +14,8 @@ int copy2(const char *src, const char *dest, unsigned int op) { } char dname[1024] = {0}; - strcpy(dname, dest); + strncpy(dname, dest, sizeof(dname) - 1); + dname[sizeof(dname) - 1] = '\0'; char *dname_endptr = strrchr(dname, '/'); if (dname_endptr != NULL) { @@ -36,28 +37,29 @@ int copy2(const char *src, const char *dest, unsigned int op) { } } else if (S_ISREG(src_stat.st_mode) && src_stat.st_nlink > 2 && src_stat.st_dev == dnamest.st_dev) { if (link(src, dest) < 0) { - perror(src); + SYSERROR("unable to link: %s, %s", src, strerror(errno)); return -1; } } else if (S_ISFIFO(src_stat.st_mode) || S_ISBLK(src_stat.st_mode) || S_ISCHR(src_stat.st_mode) || S_ISSOCK(src_stat.st_mode)) { if (mknod(dest, src_stat.st_mode, src_stat.st_rdev) < 0) { - perror(src); + SYSERROR("unable to mknod: %s, %s", dest, strerror(errno)); return -1; } } else if (S_ISREG(src_stat.st_mode)) { char buf[STASIS_BUFSIZ] = {0}; size_t bytes_read; - SYSDEBUG("%s", "Opening source file for reading"); + SYSDEBUG("Opening source file for reading"); FILE *fp1 = fopen(src, "rb"); if (!fp1) { - perror(src); + SYSERROR("unable to open source file for reading: %s, %s", src, strerror(errno)); return -1; } - SYSDEBUG("%s", "Opening destination file for writing"); + SYSDEBUG("Opening destination file for writing"); FILE *fp2 = fopen(dest, "w+b"); if (!fp2) { - perror(dest); + SYSERROR("unable to open destination file for writing: %s, %s", dest, strerror(errno)); + fclose(fp1); return -1; } @@ -69,21 +71,21 @@ int copy2(const char *src, const char *dest, unsigned int op) { fclose(fp2); if (bytes_written != (size_t) src_stat.st_size) { - fprintf(stderr, "%s: SHORT WRITE (expected %zu bytes, but wrote %zu bytes)\n", dest, src_stat.st_size, bytes_written); + SYSDEBUG("%s: SHORT WRITE (expected %zu bytes, but wrote %zu bytes)", dest, src_stat.st_size, bytes_written); return -1; } if (op & CT_OWNER && chown(dest, src_stat.st_uid, src_stat.st_gid) < 0) { - perror(dest); + SYSERROR("unable to change owner: %s, %s", dest, strerror(errno)); } if (op & CT_PERM && chmod(dest, src_stat.st_mode) < 0) { - perror(dest); + SYSERROR("unable to chmod: %s, %s", dest, strerror(errno)); } } else { errno = EOPNOTSUPP; return -1; } - SYSDEBUG("%s", "Data copied"); + SYSDEBUG("Data copied"); return 0; } diff --git a/src/lib/core/docker.c b/src/lib/core/docker.c index 4723446..484f476 100644 --- a/src/lib/core/docker.c +++ b/src/lib/core/docker.c @@ -1,17 +1,32 @@ #include "docker.h" -int docker_exec(const char *args, unsigned flags) { +int docker_exec(const char *args, const unsigned flags) { struct Process proc; char cmd[PATH_MAX]; memset(&proc, 0, sizeof(proc)); memset(cmd, 0, sizeof(cmd)); - snprintf(cmd, sizeof(cmd) - 1, "docker %s", args); + snprintf(cmd, sizeof(cmd), "docker %s", args); + + unsigned final_flags = 0; if (flags & STASIS_DOCKER_QUIET) { - strcpy(proc.f_stdout, "/dev/null"); - strcpy(proc.f_stderr, "/dev/null"); + final_flags |= STASIS_DOCKER_QUIET_STDOUT; + final_flags |= STASIS_DOCKER_QUIET_STDERR; } else { + final_flags = flags; + } + + if (final_flags & STASIS_DOCKER_QUIET_STDOUT) { + strncpy(proc.f_stdout, "/dev/null", sizeof(proc.f_stdout) - 1); + proc.f_stdout[sizeof(proc.f_stdout) - 1] = '\0'; + } + if (final_flags & STASIS_DOCKER_QUIET_STDERR) { + strncpy(proc.f_stderr, "/dev/null", sizeof(proc.f_stderr) - 1); + proc.f_stderr[sizeof(proc.f_stderr) - 1] = '\0'; + } + + if (!final_flags) { msg(STASIS_MSG_L2, "Executing: %s\n", cmd); } @@ -19,11 +34,11 @@ int docker_exec(const char *args, unsigned flags) { return proc.returncode; } -int docker_script(const char *image, char *data, unsigned flags) { +int docker_script(const char *image, char *args, char *data, const unsigned flags) { (void)flags; // TODO: placeholder char cmd[PATH_MAX] = {0}; - snprintf(cmd, sizeof(cmd) - 1, "docker run --rm -i %s /bin/sh -", image); + snprintf(cmd, sizeof(cmd), "docker run -i %s \"%s\" /bin/sh -", args ? args : "", image); FILE *outfile = popen(cmd, "w"); if (!outfile) { @@ -34,6 +49,7 @@ int docker_script(const char *image, char *data, unsigned flags) { FILE *infile = fmemopen(data, strlen(data), "r"); if (!infile) { // opening memory file for reading failed + pclose(outfile); return -1; } @@ -55,12 +71,14 @@ int docker_build(const char *dirpath, const char *args, int engine) { memset(cmd, 0, sizeof(cmd)); if (engine & STASIS_DOCKER_BUILD) { - strcpy(build, "build"); + strncpy(build, "build", sizeof(build) - 1); } if (engine & STASIS_DOCKER_BUILD_X) { - strcpy(build, "buildx build"); + strncpy(build, "buildx build", sizeof(build) - 1); } - snprintf(cmd, sizeof(cmd) - 1, "%s %s %s", build, args, dirpath); + build[sizeof(build) - 1] = '\0'; + + snprintf(cmd, sizeof(cmd), "%s %s %s", build, args, dirpath); return docker_exec(cmd, 0); } @@ -70,19 +88,20 @@ int docker_save(const char *image, const char *destdir, const char *compression_ if (compression_program && strlen(compression_program)) { char ext[255] = {0}; if (startswith(compression_program, "zstd")) { - strcpy(ext, "zst"); + strncpy(ext, "zst", sizeof(ext) - 1); } else if (startswith(compression_program, "xz")) { - strcpy(ext, "xz"); + strncpy(ext, "xz", sizeof(ext) - 1); } else if (startswith(compression_program, "gzip")) { - strcpy(ext, "gz"); + strncpy(ext, "gz", sizeof(ext) - 1); } else if (startswith(compression_program, "bzip2")) { - strcpy(ext, "bz2"); + strncpy(ext, "bz2", sizeof(ext) - 1); } else { strncpy(ext, compression_program, sizeof(ext) - 1); } - sprintf(cmd, "save \"%s\" | %s > \"%s/%s.tar.%s\"", image, compression_program, destdir, image, ext); + ext[sizeof(ext) - 1] = '\0'; + snprintf(cmd, sizeof(cmd), "save \"%s\" | %s > \"%s/%s.tar.%s\"", image, compression_program, destdir, image, ext); } else { - sprintf(cmd, "save \"%s\" -o \"%s/%s.tar\"", image, destdir, image); + snprintf(cmd, sizeof(cmd), "save \"%s\" -o \"%s/%s.tar\"", image, destdir, image); } return docker_exec(cmd, 0); @@ -107,8 +126,12 @@ static char *docker_ident() { } memset(&proc, 0, sizeof(proc)); - strcpy(proc.f_stdout, tempfile); - strcpy(proc.f_stderr, "/dev/null"); + strncpy(proc.f_stdout, tempfile, sizeof(proc.f_stdout) - 1); + proc.f_stdout[sizeof(proc.f_stdout) - 1] = '\0'; + + strncpy(proc.f_stderr, "/dev/null", sizeof(proc.f_stderr) - 1); + proc.f_stderr[sizeof(proc.f_stderr) - 1] = '\0'; + shell(&proc, "docker --version"); if (!freopen(tempfile, "r", fp)) { diff --git a/src/lib/core/download.c b/src/lib/core/download.c index c3f8dca..28c5a6f 100644 --- a/src/lib/core/download.c +++ b/src/lib/core/download.c @@ -11,47 +11,111 @@ size_t download_writer(void *fp, size_t size, size_t nmemb, void *stream) { } long download(char *url, const char *filename, char **errmsg) { + SYSDEBUG("ARGS follow"); + SYSDEBUG("url=%s", url); + SYSDEBUG("filename=%s", filename); + SYSDEBUG("errmsg=%s (NULL is OK)", *errmsg); long http_code = -1; - char user_agent[20]; - sprintf(user_agent, "stasis/%s", VERSION); - long timeout = 30L; - char *timeout_str = getenv("STASIS_DOWNLOAD_TIMEOUT"); + char user_agent[STASIS_NAME_MAX] = {0}; + snprintf(user_agent, sizeof(user_agent), "stasis/%s", STASIS_VERSION); - curl_global_init(CURL_GLOBAL_ALL); - CURL *c = curl_easy_init(); - curl_easy_setopt(c, CURLOPT_URL, url); - curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, download_writer); - FILE *fp = fopen(filename, "wb"); - if (!fp) { - return -1; + SYSDEBUG("Setting timeout"); + size_t timeout_default = 30L; + size_t timeout = timeout_default; + const char *timeout_str = getenv("STASIS_DOWNLOAD_TIMEOUT"); + if (timeout_str) { + timeout = strtoul(timeout_str, NULL, 10); + if (timeout == ULONG_MAX && errno == ERANGE) { + SYSERROR("STASIS_DOWNLOAD_TIMEOUT must be a positive integer. Using default (%zu).", timeout); + timeout = timeout_default; + } } - curl_easy_setopt(c, CURLOPT_VERBOSE, 0L); - curl_easy_setopt(c, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(c, CURLOPT_USERAGENT, user_agent); - curl_easy_setopt(c, CURLOPT_NOPROGRESS, 0L); - curl_easy_setopt(c, CURLOPT_WRITEDATA, fp); + SYSDEBUG("Setting max_retries"); + const size_t max_retries_default = 5; + size_t max_retries = max_retries_default; + const char *max_retries_str = getenv("STASIS_DOWNLOAD_RETRY_MAX"); + if (max_retries_str) { + max_retries = strtoul(max_retries_str, NULL, 10); + if (max_retries == ULONG_MAX && errno == ERANGE) { + SYSERROR("STASIS_DOWNLOAD_RETRY_MAX must be a positive integer. Using default (%zu).", max_retries); + max_retries = max_retries_default; + } + } - if (timeout_str) { - timeout = strtol(timeout_str, NULL, 10); + SYSDEBUG("Setting max_retry_seconds"); + const size_t max_retry_seconds_default = 3; + size_t max_retry_seconds = max_retry_seconds_default; + const char *max_retry_seconds_str = getenv("STASIS_DOWNLOAD_RETRY_SECONDS"); + if (max_retry_seconds_str) { + max_retry_seconds = strtoul(max_retry_seconds_str, NULL, 10); + if (max_retry_seconds == ULONG_MAX && errno == ERANGE) { + SYSERROR("STASIS_DOWNLOAD_RETRY_SECONDS must be a positive integer. Using default (%zu).", max_retry_seconds); + max_retry_seconds = max_retry_seconds_default; + } } - curl_easy_setopt(c, CURLOPT_CONNECTTIMEOUT, timeout); - - SYSDEBUG("curl_easy_perform(): \n\turl=%s\n\tfilename=%s\n\tuser agent=%s\n\ttimeout=%zu", url, filename, user_agent, timeout); - CURLcode curl_code = curl_easy_perform(c); - SYSDEBUG("curl status code: %d", curl_code); - if (curl_code != CURLE_OK) { - if (errmsg) { - strcpy(*errmsg, curl_easy_strerror(curl_code)); - } else { - fprintf(stderr, "\nCURL ERROR: %s\n", curl_easy_strerror(curl_code)); + + + SYSDEBUG("Initializing curl context"); + curl_global_init(CURL_GLOBAL_ALL); + CURL *c = curl_easy_init(); + for (size_t retry = 0; retry < max_retries; retry++) { + if (retry) { + SYSWARN("[RETRY %zu/%zu] %s: %s", retry + 1, max_retries, *errmsg, url); } - goto failed; + + SYSDEBUG("Configuring curl"); + curl_easy_setopt(c, CURLOPT_URL, url); + curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, download_writer); + FILE *fp = fopen(filename, "wb"); + if (!fp) { + return -1; + } + + curl_easy_setopt(c, CURLOPT_VERBOSE, 0L); + curl_easy_setopt(c, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(c, CURLOPT_USERAGENT, user_agent); + curl_easy_setopt(c, CURLOPT_NOPROGRESS, 0L); + curl_easy_setopt(c, CURLOPT_WRITEDATA, fp); + curl_easy_setopt(c, CURLOPT_CONNECTTIMEOUT, timeout); + + SYSDEBUG("curl_easy_perform(): \n\turl=%s\n\tfilename=%s\n\tuser agent=%s\n\ttimeout=%zu", url, filename, user_agent, timeout); + const CURLcode curl_code = curl_easy_perform(c); + SYSDEBUG("curl status code: %d", curl_code); + + if (curl_code != CURLE_OK) { + SYSDEBUG("curl failed with code: %s", curl_easy_strerror(curl_code)); + const size_t errmsg_maxlen = 256; + if (!*errmsg) { + SYSDEBUG("allocating memory for error message"); + *errmsg = calloc(errmsg_maxlen, sizeof(char)); + if (!*errmsg) { + SYSERROR("unable to allocate memory for error message"); + goto cleanup; + } + } + snprintf(*errmsg, errmsg_maxlen, "%s", curl_easy_strerror(curl_code)); + curl_easy_reset(c); + fclose(fp); + sleep(max_retry_seconds); + continue; + } + + cleanup: + SYSDEBUG("Cleaning up"); + // Data written. Clean up. + fclose(fp); + + if (CURLE_OK && *errmsg) { + // Retry loop succeeded, no error + *errmsg[0] = '\0'; + } + + curl_easy_getinfo(c, CURLINFO_RESPONSE_CODE, &http_code); + SYSDEBUG("HTTP code: %li", http_code); + + break; } - curl_easy_getinfo(c, CURLINFO_RESPONSE_CODE, &http_code); - failed: - SYSDEBUG("HTTP code: %li", http_code); - fclose(fp); curl_easy_cleanup(c); curl_global_cleanup(); return http_code; diff --git a/src/lib/core/envctl.c b/src/lib/core/envctl.c index 0be3f89..9069a7f 100644 --- a/src/lib/core/envctl.c +++ b/src/lib/core/envctl.c @@ -17,6 +17,8 @@ struct EnvCtl *envctl_init() { } static int callback_builtin_nop(const void *a, const void *b) { + (void) a; // Unused + (void) b; // Unused return STASIS_ENVCTL_RET_SUCCESS; } @@ -51,13 +53,14 @@ size_t envctl_get_index(const struct EnvCtl *envctl, const char *name) { for (size_t i = 0; i < envctl->num_used; i++) { if (!strcmp(envctl->item[i]->name, name)) { // pack state flag, outer (struct) index and inner (name) index - return 1L << 63L | i; + return 1UL << 63UL | i; } } return 0; } void envctl_decode_index(size_t in_i, size_t *state, size_t *out_i, size_t *name_i) { + (void) name_i; *state = ((in_i >> 63L) & 1); *out_i = in_i & 0xffffffffL; } @@ -82,30 +85,35 @@ unsigned envctl_get_flags(const struct EnvCtl *envctl, const char *name) { envctl_decode_index(poll_index, &state, &id, &name_id); if (!state) { return 0; - } else { - fprintf(stderr, "managed environment variable: %s\n", name); } + SYSINFO("managed environment variable: %s", name); return envctl->item[id]->flags; } void envctl_do_required(const struct EnvCtl *envctl, int verbose) { + int failed = 0; for (size_t i = 0; i < envctl->num_used; i++) { - struct EnvCtl_Item *item = envctl->item[i]; + const struct EnvCtl_Item *item = envctl->item[i]; const char *name = item->name; envctl_except_fn *callback = item->callback; if (verbose) { - msg(STASIS_MSG_L2, "Verifying %s\n", name); + msg(STASIS_MSG_L2, "Verifying %s [%s]\n", name, item->flags & STASIS_ENVCTL_REQUIRED ? "required" : "optional"); } - int code = callback((const void *) item, (const void *) name); + const int code = callback((const void *) item, (const void *) name); if (code == STASIS_ENVCTL_RET_IGNORE || code == STASIS_ENVCTL_RET_SUCCESS) { continue; } if (code == STASIS_ENVCTL_RET_FAIL) { - fprintf(stderr, "\n%s must be set. Exiting.\n", name); - exit(1); + SYSERROR("%s%s must be defined.", name, STASIS_COLOR_RESET); + failed++; + continue; } - fprintf(stderr, "\nan unknown envctl callback code occurred: %d\n", code); + SYSWARN("an unknown envctl callback code occurred: %d", code); + } + + if (failed) { + SYSERROR("Environment check failed with %d error(s)", failed); exit(1); } } diff --git a/src/lib/core/environment.c b/src/lib/core/environment.c index f5e8566..4258004 100644 --- a/src/lib/core/environment.c +++ b/src/lib/core/environment.c @@ -70,26 +70,27 @@ void runtime_export(RuntimeEnv *env, char **keys) { NULL, }; - char export_command[7]; // export=6 and setenv=6... convenient + char export_command[10]; // export=6 and setenv=6... convenient char *_sh = getenv("SHELL"); char *sh = path_basename(_sh); if (sh == NULL) { - fprintf(stderr, "echo SHELL environment variable is not defined"); + SYSERROR("echo SHELL environment variable is not defined"); exit(1); } for (size_t i = 0; borne[i] != NULL; i++) { if (strcmp(sh, borne[i]) == 0) { - strcpy(export_command, "export"); + strncpy(export_command, "export", sizeof(export_command) - 1); break; } } for (size_t i = 0; unborne[i] != NULL; i++) { if (strcmp(sh, unborne[i]) == 0) { - strcpy(export_command, "setenv"); + strncpy(export_command, "setenv", sizeof(export_command) - 1); break; } } + export_command[sizeof(export_command) - 1] = '\0'; for (size_t i = 0; i < strlist_count(env); i++) { char output[STASIS_BUFSIZ] = {0}; @@ -106,14 +107,13 @@ void runtime_export(RuntimeEnv *env, char **keys) { if (keys != NULL) { for (size_t j = 0; keys[j] != NULL; j++) { if (strcmp(keys[j], key) == 0) { - //sprintf(output, "%s=\"%s\"\n%s %s", key, value ? value : "", export_command, key); - sprintf(output, "%s %s=\"%s\"", export_command, key, value ? value : ""); + snprintf(output, sizeof(output), "%s %s=\"%s\"", export_command, key, value ? value : ""); puts(output); } } } else { - sprintf(output, "%s %s=\"%s\"", export_command, key, value ? value : ""); + snprintf(output, sizeof(output), "%s %s=\"%s\"", export_command, key, value ? value : ""); puts(output); } guard_free(value); @@ -178,7 +178,7 @@ int runtime_replace(RuntimeEnv **dest, char **src) { } /** - * Determine whether or not a key exists in the runtime environment + * Determine whether a key exists in the runtime environment * * Example: * @@ -245,7 +245,14 @@ char *runtime_get(RuntimeEnv *env, const char *key) { ssize_t key_offset = runtime_contains(env, key); if (key_offset != -1) { char **pair = split(strlist_item(env, key_offset), "=", 0); + if (!pair) { + return NULL; + } result = join(&pair[1], "="); + if (!result) { + guard_array_free(pair); + return NULL; + } guard_array_free(pair); } return result; @@ -285,8 +292,7 @@ char *runtime_expand_var(RuntimeEnv *env, char *input) { // If there's no environment variables to process return the input string if (strchr(input, delim) == NULL) { - //return strdup(input); - return input; + return strdup(input); } expanded = calloc(STASIS_BUFSIZ, sizeof(char)); @@ -336,7 +342,10 @@ char *runtime_expand_var(RuntimeEnv *env, char *input) { if (env) { tmp = runtime_get(env, var); } else { - tmp = getenv(var); + const char *v = getenv(var); + if (v) { + tmp = strdup(v); + } } if (tmp == NULL) { // This mimics shell behavior in general. @@ -348,9 +357,7 @@ char *runtime_expand_var(RuntimeEnv *env, char *input) { } // Append expanded environment variable to output strncat(expanded, tmp, STASIS_BUFSIZ - 1); - if (env) { - guard_free(tmp); - } + guard_free(tmp); } // Nothing to do so append input to output @@ -400,13 +407,28 @@ char *runtime_expand_var(RuntimeEnv *env, char *input) { * @param _value New environment variable value */ void runtime_set(RuntimeEnv *env, const char *_key, char *_value) { + const char *sep = "="; if (_key == NULL) { return; } + const ssize_t key_offset = runtime_contains(env, _key); char *key = strdup(_key); - ssize_t key_offset = runtime_contains(env, key); + if (!key) { + SYSERROR("unable to allocate memory for key"); + exit(1); + } char *value = runtime_expand_var(env, _value); - char *now = join((char *[]) {key, value, NULL}, "="); + if (!value) { + SYSERROR("unable to allocate memory for value"); + exit(1); + } + + lstrip(value); + char *now = join((char *[]) {key, value, NULL}, sep); + if (!now) { + SYSERROR("unable to allocate memory for join"); + exit(1); + } if (key_offset < 0) { strlist_append(&env, now); @@ -415,6 +437,7 @@ void runtime_set(RuntimeEnv *env, const char *_key, char *_value) { } guard_free(now); guard_free(key); + guard_free(value); } /** @@ -424,6 +447,10 @@ void runtime_set(RuntimeEnv *env, const char *_key, char *_value) { void runtime_apply(RuntimeEnv *env) { for (size_t i = 0; i < strlist_count(env); i++) { char **pair = split(strlist_item(env, i), "=", 1); + if (!pair) { + SYSERROR("unable to allocate memory for runtime_apply"); + return; + } setenv(pair[0], pair[1], 1); guard_array_free(pair); } diff --git a/src/lib/core/github.c b/src/lib/core/github.c index c195a28..992ab32 100644 --- a/src/lib/core/github.c +++ b/src/lib/core/github.c @@ -15,7 +15,7 @@ static size_t writer(const void *contents, size_t size, size_t nmemb, void *resu char *ptr = realloc(content->data, content->len + newlen + 1); if (!ptr) { - perror("realloc failed"); + SYSERROR("realloc failed"); return 0; } @@ -58,9 +58,9 @@ int get_github_release_notes(const char *api_token, const char *repo, const char } // Render the header data - sprintf(endpoint_header_auth, endpoint_header_auth_fmt, api_token); - sprintf(endpoint_post_fields, endpoint_post_fields_fmt, tag, target_commitish); - sprintf(endpoint_url, endpoint_url_fmt, repo); + snprintf(endpoint_header_auth, sizeof(endpoint_header_auth), endpoint_header_auth_fmt, api_token); + snprintf(endpoint_post_fields, sizeof(endpoint_post_fields), endpoint_post_fields_fmt, tag, target_commitish); + snprintf(endpoint_url, sizeof(endpoint_url), endpoint_url_fmt, repo); // Begin curl configuration curl_easy_setopt(curl, CURLOPT_URL, endpoint_url); @@ -77,7 +77,7 @@ int get_github_release_notes(const char *api_token, const char *repo, const char // Set the user-agent (github requires one) char user_agent[20] = {0}; - sprintf(user_agent, "stasis/%s", VERSION); + snprintf(user_agent, sizeof(user_agent), "stasis/%s", VERSION); curl_easy_setopt(curl, CURLOPT_USERAGENT, user_agent); // Execute curl request @@ -89,8 +89,7 @@ int get_github_release_notes(const char *api_token, const char *repo, const char curl_easy_cleanup(curl); if(res != CURLE_OK) { - fprintf(stderr, "curl_easy_perform() failed: %s\n", - curl_easy_strerror(res)); + SYSERROR("curl_easy_perform() failed: %s", curl_easy_strerror(res)); return -1; } @@ -109,21 +108,27 @@ int get_github_release_notes(const char *api_token, const char *repo, const char if (last_char == ',') { trim++; } - data_offset[strlen(data_offset) - trim] = 0; + // Truncate the trimmed bytes + memset(&data_offset[strlen(data_offset) - trim], 0, trim); // Extract release notes - *output = strdup(data_offset); + *output = calloc(strlen(data_offset) + 1, sizeof(**output)); + // Copy output (including terminator) + strncpy(*output, data_offset, strlen(data_offset) + 1); } else if ((data_offset = strstr(line, field_message))) { // Skip past the message field data_offset += strlen(field_message); - *(strchr(data_offset, '"')) = 0; - fprintf(stderr, "GitHub API Error: '%s'\n", data_offset); - fprintf(stderr, "URL: %s\n", endpoint_url); - fprintf(stderr, "POST: %s\n", endpoint_post_fields); + char *data_mark = strchr(data_offset, '"'); + if (data_mark) { + *data_mark = '\0'; + } + SYSERROR("GitHub API Error: '%s'", data_offset); + SYSERROR("URL: %s", endpoint_url); + SYSERROR("POST: %s", endpoint_post_fields); guard_free(content.data); return -1; } } else { - fprintf(stderr, "Unknown error\n"); + SYSERROR("Unknown error"); guard_free(content.data); return -1; } diff --git a/src/lib/core/globals.c b/src/lib/core/globals.c index d84e799..b84213e 100644 --- a/src/lib/core/globals.c +++ b/src/lib/core/globals.c @@ -3,10 +3,10 @@ #include "core.h" #include "envctl.h" -const char *VERSION = "1.0.0"; +const char *VERSION = STASIS_VERSION " (" STASIS_VERSION_BRANCH ")"; const char *AUTHOR = "Joseph Hunkeler"; const char *BANNER = - "------------------------------------------------------------------------\n" + STASIS_BANNER_HEADER "\n" #if defined(STASIS_DUMB_TERMINAL) " STASIS \n" #else @@ -18,10 +18,10 @@ const char *BANNER = " |_____/ |_/_/ \\_\\_____/|_____|_____/ \n" "\n" #endif - "------------------------------------------------------------------------\n" + STASIS_BANNER_HEADER "\n" " Delivery Generator \n" - " v%s \n" - "------------------------------------------------------------------------\n" + "%s\n" + STASIS_BANNER_HEADER "\n" "Copyright (C) 2023-2025 %s,\n" "Association of Universities for Research in Astronomy (AURA)\n"; @@ -41,8 +41,10 @@ struct STASIS_GLOBAL globals = { .enable_testing = true, ///< Toggle [test] block "script" execution. "script_setup" always executes. .enable_rewrite_spec_stage_2 = true, ///< Leave template stings in output files .enable_parallel = true, ///< Toggle testing in parallel + .enable_task_logging = true, ///< Toggle logging for multiprocess tasks .parallel_fail_fast = false, ///< Kill ALL multiprocessing tasks immediately on error .pool_status_interval = 30, ///< Report "Task is running" + .task_timeout = 0, ///< Time in seconds before task is terminated }; void globals_free() { @@ -51,6 +53,8 @@ void globals_free() { guard_free(globals.conda_install_prefix); guard_strlist_free(&globals.conda_packages); guard_strlist_free(&globals.pip_packages); + guard_free(globals.wheel_builder); + guard_free(globals.wheel_builder_manylinux_image); guard_free(globals.jfrog.arch); guard_free(globals.jfrog.os); guard_free(globals.jfrog.url); diff --git a/src/lib/core/include/conda.h b/src/lib/core/include/conda.h index ea8613f..cc9426d 100644 --- a/src/lib/core/include/conda.h +++ b/src/lib/core/include/conda.h @@ -29,6 +29,7 @@ struct MicromambaInfo { char *micromamba_prefix; //!< Path to write micromamba binary char *conda_prefix; //!< Path to install conda base tree + char *download_dir; //!< Path to store micromamba installer }; /** @@ -38,7 +39,7 @@ struct MicromambaInfo { * @param ... variadic arguments * @return exit code */ -int micromamba(struct MicromambaInfo *info, char *command, ...); +int micromamba(const struct MicromambaInfo *info, char *command, ...); /** * Execute Python @@ -219,12 +220,13 @@ int conda_index(const char *path); * @param mode USE_CONDA * @param index a file system path or url pointing to a simple index or conda channel * @param spec a pip package specification (e.g. `name==1.2.3`) + * @param logdir the directory to store the output log * @param spec a conda package specification (e.g. `name=1.2.3`) * @return PKG_NOT_FOUND, if not found * @return PKG_FOUND, if found * @return PKG_E_INDEX_PROVIDES_{ERROR}, on error (see conda.h) */ -int pkg_index_provides(int mode, const char *index, const char *spec); +int pkg_index_provides(int mode, const char *index, const char *spec, const char *logdir); const char *pkg_index_provides_strerror(int code); char *conda_get_active_environment(); diff --git a/src/lib/core/include/copy.h b/src/lib/core/include/copy.h index 0f92ddd..1eb5219 100644 --- a/src/lib/core/include/copy.h +++ b/src/lib/core/include/copy.h @@ -1,5 +1,6 @@ //! @file copy.h #ifndef STASIS_COPY_H +#define STASIS_COPY_H #include <stdio.h> #include <stdlib.h> diff --git a/src/lib/core/include/core.h b/src/lib/core/include/core.h index 35a9506..9a2007c 100644 --- a/src/lib/core/include/core.h +++ b/src/lib/core/include/core.h @@ -10,12 +10,14 @@ #include <unistd.h> #include <time.h> #include <sys/statvfs.h> +#include "version.h" +#define STASIS_BANNER_HEADER "------------------------------------------------------------------------" #define STASIS_BUFSIZ 8192 #define STASIS_NAME_MAX 255 #define STASIS_DIRSTACK_MAX 1024 #define STASIS_TIME_STR_MAX 128 -#define HTTP_ERROR(X) X >= 400 +#define HTTP_ERROR(X) (X >= 400 || X < 0) #include "config.h" #include "core_mem.h" @@ -42,6 +44,7 @@ struct STASIS_GLOBAL { bool enable_overwrite; //!< Enable release file clobbering bool enable_rewrite_spec_stage_2; //!< Enable automatic @STR@ replacement in output files bool enable_parallel; //!< Enable testing in parallel + bool enable_task_logging; //!< Enable logging task output to a file long cpu_limit; //!< Limit parallel processing to n cores (default: max - 1) long parallel_fail_fast; //!< Fail immediately on error int pool_status_interval; //!< Report "Task is running" every n seconds @@ -50,6 +53,9 @@ struct STASIS_GLOBAL { char *tmpdir; //!< Path to temporary storage directory char *conda_install_prefix; //!< Path to install conda char *sysconfdir; //!< Path where STASIS reads its configuration files (mission directory, etc) + int task_timeout; ///!< Time in seconds before task is terminated + char *wheel_builder; ///!< Backend to build wheels (build, cibuildwheel, manylinux) + char *wheel_builder_manylinux_image; ///!< Image to use for a Manylinux build struct { char *tox_posargs; char *conda_reactivate; diff --git a/src/lib/core/include/core_mem.h b/src/lib/core/include/core_mem.h index dd79e72..b67130c 100644 --- a/src/lib/core/include/core_mem.h +++ b/src/lib/core/include/core_mem.h @@ -22,4 +22,11 @@ guard_free(ARR); \ } while (0) +#define guard_array_n_free(ARR, LEN) do { \ + for (size_t ARR_I = 0; ARR && ARR_I < LEN ; ARR_I++) { \ + guard_free(ARR[ARR_I]); \ + } \ + guard_free(ARR); \ +} while (0) + #endif //STASIS_CORE_MEM_H diff --git a/src/lib/core/include/core_message.h b/src/lib/core/include/core_message.h index 1ffa846..6ac9cb4 100644 --- a/src/lib/core/include/core_message.h +++ b/src/lib/core/include/core_message.h @@ -2,18 +2,20 @@ #ifndef STASIS_CORE_MESSAGE_H #define STASIS_CORE_MESSAGE_H -#define SYSERROR(MSG, ...) do { \ - fprintf(stderr, "%s:%d:%s():%s - ", path_basename(__FILE__), __LINE__, __FUNCTION__, (errno > 0) ? strerror(errno) : "info"); \ - fprintf(stderr, MSG LINE_SEP, __VA_ARGS__); \ +#define SYSERROR(FMT, ...) do { \ + log_print_error(EXECPOINT, (FMT), ##__VA_ARGS__); \ } while (0) -#ifdef DEBUG -#define SYSDEBUG(MSG, ...) do { \ - fprintf(stderr, "DEBUG: %s:%d:%s(): ", path_basename(__FILE__), __LINE__, __FUNCTION__); \ - fprintf(stderr, MSG LINE_SEP, __VA_ARGS__); \ +#define SYSWARN(FMT, ...) do { \ + log_print_warning(EXECPOINT, (FMT), ##__VA_ARGS__); \ +} while (0) + +#define SYSINFO(FMT, ...) do { \ + log_print_info(EXECPOINT, (FMT), ##__VA_ARGS__); \ +} while (0) + +#define SYSDEBUG(FMT, ...) do { \ + log_print_debug(EXECPOINT, (FMT), ##__VA_ARGS__); \ } while (0) -#else -#define SYSDEBUG(MSG, ...) do {} while (0) -#endif #endif //STASIS_CORE_MESSAGE_H diff --git a/src/lib/core/include/docker.h b/src/lib/core/include/docker.h index 7585d86..dd67f21 100644 --- a/src/lib/core/include/docker.h +++ b/src/lib/core/include/docker.h @@ -6,6 +6,8 @@ //! Flag to squelch output from docker_exec() #define STASIS_DOCKER_QUIET 1 << 1 +#define STASIS_DOCKER_QUIET_STDOUT 1 << 2 +#define STASIS_DOCKER_QUIET_STDERR 1 << 3 //! Flag for older style docker build #define STASIS_DOCKER_BUILD 1 << 1 @@ -83,7 +85,7 @@ int docker_exec(const char *args, unsigned flags); * @return */ int docker_build(const char *dirpath, const char *args, int engine); -int docker_script(const char *image, char *data, unsigned flags); +int docker_script(const char *image, char *args, char *data, unsigned flags); int docker_save(const char *image, const char *destdir, const char *compression_program); void docker_sanitize_tag(char *str); int docker_validate_compression_program(char *prog); diff --git a/src/lib/core/include/log.h b/src/lib/core/include/log.h new file mode 100644 index 0000000..cad954f --- /dev/null +++ b/src/lib/core/include/log.h @@ -0,0 +1,29 @@ +#ifndef STASIS_EXECPOINT_H +#define STASIS_EXECPOINT_H + +#include <stdio.h> + +enum LogLevel { + LOG_LEVEL_WARN = 0, + LOG_LEVEL_INFO, + LOG_LEVEL_DEBUG, +}; +extern enum LogLevel LOG_LEVEL; + +struct ExecPoint { + const int line; // line number + const char *file; // file name of origin + const char *function; // function of origin +}; + +#define EXECPOINT (struct ExecPoint) {.line = __LINE__, .file = __FILE__, .function = __func__} + +void log_print_error(struct ExecPoint ep, const char *fmt, ...); +void log_print_warning(struct ExecPoint ep, const char *fmt, ...); +void log_print_info(struct ExecPoint ep, const char *fmt, ...); +void log_print_debug(struct ExecPoint ep, const char *fmt, ...); +int log_msgv(FILE *stream, struct ExecPoint ep, const char *preface_color, const char *preface, const char *fmt, va_list ap); +int log_msg(FILE *stream, struct ExecPoint ep, const char *preface_color, const char *preface, const char *fmt, ...); +const char *log_get_level_str(void); + +#endif // STASIS_EXECPOINT_H diff --git a/src/lib/core/include/multiprocessing.h b/src/lib/core/include/multiprocessing.h index ff674e9..874777c 100644 --- a/src/lib/core/include/multiprocessing.h +++ b/src/lib/core/include/multiprocessing.h @@ -3,32 +3,36 @@ #define STASIS_MULTIPROCESSING_H #include "core.h" +#include "sem.h" +#include "timespec.h" #include <signal.h> #include <sys/wait.h> -#include <semaphore.h> #include <sys/mman.h> #include <fcntl.h> #include <sys/stat.h> +#include <math.h> + +struct MultiProcessingTimer { + struct timespec t_start; + struct timespec t_stop; + double duration; +}; struct MultiProcessingTask { pid_t pid; ///< Program PID pid_t parent_pid; ///< Program PID (parent process) int status; ///< Child process exit status int signaled_by; ///< Last signal received, if any - time_t _now; ///< Current time - time_t _seconds; ///< Time elapsed since status interval (used by MultiprocessingPool.status_interval) + int timeout; ///< Seconds to elapse before killing the process time_t _startup; ///< Time elapsed since task started - long elapsed; ///< Total time elapsed in seconds char ident[255]; ///< Identity of the pool task char *cmd; ///< Shell command(s) to be executed size_t cmd_len; ///< Length of command string (for mmap/munmap) char working_dir[PATH_MAX]; ///< Path to directory `cmd` should be executed in char log_file[PATH_MAX]; ///< Full path to stdout/stderr log file char parent_script[PATH_MAX]; ///< Path to temporary script executing the task - struct { - struct timespec t_start; - struct timespec t_stop; - } time_data; ///< Wall-time counters + struct MultiProcessingTimer time_data; ///< Wall-time counters + struct MultiProcessingTimer interval_data; ///< Progress report counters }; struct MultiProcessingPool { @@ -38,6 +42,7 @@ struct MultiProcessingPool { char ident[255]; ///< Identity of task pool char log_root[PATH_MAX]; ///< Base directory to store stderr/stdout log files int status_interval; ///< Report a pooled task is "running" every n seconds + struct Semaphore semaphore; }; /// A multiprocessing task's initial state (i.e. "FAIL") diff --git a/src/lib/core/include/sem.h b/src/lib/core/include/sem.h new file mode 100644 index 0000000..b8f9a39 --- /dev/null +++ b/src/lib/core/include/sem.h @@ -0,0 +1,62 @@ +/** +* @file sem.h +*/ +#ifndef STASIS_SEMAPHORE_H +#define STASIS_SEMAPHORE_H + +#include "core.h" +#include <semaphore.h> +#if defined(STASIS_OS_DARWIN) +// Darwin's sem_open() limits the path length to PSEMNAMLEN +// even though it isn't used directly. +#include <sys/posix_sem.h> // PSEMNAMLEN +#endif + +struct Semaphore { + sem_t *sem; + char name[STASIS_NAME_MAX]; +}; + +/** + * Initialize a cross-platform semaphore (Linux/Darwin) + * + * @code c + * #include "sem.h" + * + * int main(int argc, char *argv[]) { + * struct Semaphore s; + * if (semaphore_init(&s, "mysem", 1)) { + * perror("semaphore_init failed"); + * exit(1); + * } + * if (semaphore_wait(&s)) { + * perror("semaphore_wait failed"); + * exit(1); + * } + * + * // + * // Critical section + * // CODE HERE + * // + * + * if (semaphore_post(&s)) { + * perror("semaphore_post failed"); + * exit(1); + * } + * + * semaphore_destroy(&s); + * } + * @endcode + * + * @param s a pointer to `Semaphore` + * @param name of the semaphore + * @param value initial value of the semaphore + * @return -1 on error + * @return 0 on success + */ +int semaphore_init(struct Semaphore *s, const char *name, int value); +int semaphore_wait(struct Semaphore *s); +int semaphore_post(struct Semaphore *s); +void semaphore_destroy(struct Semaphore *s); + +#endif //STASIS_SEMAPHORE_H
\ No newline at end of file diff --git a/src/lib/core/include/str.h b/src/lib/core/include/str.h index bb96db0..3e7c3a4 100644 --- a/src/lib/core/include/str.h +++ b/src/lib/core/include/str.h @@ -9,14 +9,19 @@ #include <string.h> #include <stdarg.h> #include <ctype.h> -#include "relocation.h" #include "core.h" +#include "log.h" +#include "relocation.h" #define STASIS_SORT_ALPHA 1 << 0 #define STASIS_SORT_NUMERIC 1 << 1 #define STASIS_SORT_LEN_ASCENDING 1 << 2 #define STASIS_SORT_LEN_DESCENDING 1 << 3 + +char *strdup_maybe_entry(const char * restrict s, struct ExecPoint ep, int exit_code); +#define strdup_maybe(S) strdup_maybe_entry((S), EXECPOINT, 1) + /** * Determine how many times the character `ch` appears in `sptr` string * @param sptr string to scan @@ -293,18 +298,27 @@ int isdigit_s(const char *s); char *tolower_s(char *s); /** - * Return a copy of the input string with "." characters removed + * Reduce a version string to the major[minor] format used by Python * - * ~~~{.c} - * char *version = strdup("1.2.3"); - * char *version_short = to_short_version(str); - * // version_short is "123" - * free(version_short); + * @code{.c} + * #include <stdio.h> + * #include "str.h" * - * ~~~ + * int main(int argc, char *argv[]) { + * char python_version[] = "3.13.3" + * char *python_short_version = to_short_version(python_version); // "313" + * if (!python_short_version) { + * perror("unable to allocate memory for shortened python version"); + * return 1; + * } + * free(python_short_version); + * return 0; + * } + * @endcode * - * @param s input string - * @return pointer to new string + * @param s python version string + * @return the shortened version string + * @return NULL on error */ char *to_short_version(const char *s); diff --git a/src/lib/core/include/strlist.h b/src/lib/core/include/strlist.h index 18c60eb..f44025c 100644 --- a/src/lib/core/include/strlist.h +++ b/src/lib/core/include/strlist.h @@ -45,7 +45,11 @@ int strlist_append_file(struct StrList *pStrList, char *path, ReaderFn *readerFn void strlist_append_strlist(struct StrList *pStrList1, struct StrList *pStrList2); void strlist_append(struct StrList **pStrList, char *str); void strlist_append_array(struct StrList *pStrList, char **arr); -void strlist_append_tokenize(struct StrList *pStrList, char *str, char *delim); + +int strlist_append_tokenize(struct StrList *pStrList, char *str, char *delim); + +int strlist_append_tokenize_raw(struct StrList *pStrList, char *str, char *delim); +int strlist_appendf(struct StrList **pStrList, const char *fmt, ...); struct StrList *strlist_copy(struct StrList *pStrList); int strlist_cmp(struct StrList *a, struct StrList *b); void strlist_free(struct StrList **pStrList); diff --git a/src/lib/core/include/template.h b/src/lib/core/include/template.h index e3d83fb..436fcc6 100644 --- a/src/lib/core/include/template.h +++ b/src/lib/core/include/template.h @@ -40,7 +40,9 @@ char *tpl_render(char *str); */ int tpl_render_to_file(char *str, const char *filename); -typedef int tplfunc(void *frame, void *data_out); +struct tplfunc_frame; + +typedef int tplfunc(struct tplfunc_frame *frame, void *data_out); struct tplfunc_frame { char *key; ///< Name of the function @@ -68,8 +70,9 @@ struct tplfunc_frame { * @param key function name to expose to "func:" interface * @param tplfunc_ptr pointer to function of type tplfunc * @param argc number of function arguments to accept + * @param data_in pointer to function input data */ -void tpl_register_func(char *key, void *tplfunc_ptr, int argc, void *data_in); +void tpl_register_func(char *key, tplfunc *tplfunc_ptr, int argc, void *data_in); /** * Get the function frame associated with a template function diff --git a/src/lib/core/include/template_func_proto.h b/src/lib/core/include/template_func_proto.h index 286ccfb..0f2ad80 100644 --- a/src/lib/core/include/template_func_proto.h +++ b/src/lib/core/include/template_func_proto.h @@ -4,10 +4,10 @@ #include "template.h" -int get_github_release_notes_tplfunc_entrypoint(void *frame, void *data_out); -int get_github_release_notes_auto_tplfunc_entrypoint(void *frame, void *data_out); -int get_junitxml_file_entrypoint(void *frame, void *data_out); -int get_basetemp_dir_entrypoint(void *frame, void *data_out); -int tox_run_entrypoint(void *frame, void *data_out); +int get_github_release_notes_tplfunc_entrypoint(struct tplfunc_frame *frame, void *data_out); +int get_github_release_notes_auto_tplfunc_entrypoint(struct tplfunc_frame *frame, void *data_out); +int get_junitxml_file_entrypoint(struct tplfunc_frame *frame, void *data_out); +int get_basetemp_dir_entrypoint(struct tplfunc_frame *frame, void *data_out); +int tox_run_entrypoint(struct tplfunc_frame *frame, void *data_out); #endif //TEMPLATE_FUNC_PROTO_H
\ No newline at end of file diff --git a/src/lib/core/include/timespec.h b/src/lib/core/include/timespec.h new file mode 100644 index 0000000..3f4b9a7 --- /dev/null +++ b/src/lib/core/include/timespec.h @@ -0,0 +1,71 @@ +/* Functions for working with timespec structures + * Written by Daniel Collins (2017-2021) + * timespec_mod by Alex Forencich (2019) + * Various contributions by Ingo Albrecht (2021) + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * For more information, please refer to <http://unlicense.org/> +*/ + +#ifndef DAN_TIMESPEC_H +#define DAN_TIMESPEC_H + +#include <stdbool.h> +#include <sys/time.h> +#include <time.h> + +#ifdef __cplusplus +extern "C" { +#endif + +struct timespec timespec_add(struct timespec ts1, struct timespec ts2); +struct timespec timespec_sub(struct timespec ts1, struct timespec ts2); +struct timespec timespec_mod(struct timespec ts1, struct timespec ts2); + +struct timespec timespec_min(struct timespec ts1, struct timespec ts2); +struct timespec timespec_max(struct timespec ts1, struct timespec ts2); +struct timespec timespec_clamp(struct timespec ts1, struct timespec min, struct timespec max); + +int timespec_cmp(struct timespec ts1, struct timespec ts2); +bool timespec_eq(struct timespec ts1, struct timespec ts2); +bool timespec_gt(struct timespec ts1, struct timespec ts2); +bool timespec_ge(struct timespec ts1, struct timespec ts2); +bool timespec_lt(struct timespec ts1, struct timespec ts2); +bool timespec_le(struct timespec ts1, struct timespec ts2); + +struct timespec timespec_from_double(double s); +double timespec_to_double(struct timespec ts); +struct timespec timespec_from_timeval(struct timeval tv); +struct timeval timespec_to_timeval(struct timespec ts); +struct timespec timespec_from_ms(long milliseconds); +long timespec_to_ms(struct timespec ts); + +struct timespec timespec_normalise(struct timespec ts); + +#ifdef __cplusplus +} +#endif + +#endif /* !DAN_TIMESPEC_H */ diff --git a/src/lib/core/include/utils.h b/src/lib/core/include/utils.h index 1906808..98b8ae8 100644 --- a/src/lib/core/include/utils.h +++ b/src/lib/core/include/utils.h @@ -9,6 +9,7 @@ #include <limits.h> #include <errno.h> #include "core.h" +#include "log.h" #include "copy.h" #include "system.h" #include "strlist.h" @@ -27,6 +28,15 @@ #define LINE_SEP "\n" #endif +#if defined(STASIS_OS_LINUX) +#define STASIS_RANDOM_GENERATOR_FILE "/dev/urandom" +#elif defined(STASIS_OS_DARWIN) +#define STASIS_RANDOM_GENERATOR_FILE "/dev/random" +#else +#define STASIS_RANDOM_GENERATOR_FILE NULL +#define NEED_SRAND 1 +#endif + #define STASIS_XML_PRETTY_PRINT_PROG "xmllint" #define STASIS_XML_PRETTY_PRINT_ARGS "--format" @@ -200,17 +210,17 @@ int path_store(char **destptr, size_t maxlen, const char *base, const char *path #define STASIS_COLOR_RESET "" #else //! Set output color to red -#define STASIS_COLOR_RED "\e[1;91m" +#define STASIS_COLOR_RED "\x1b[1;91m" //! Set output color to green -#define STASIS_COLOR_GREEN "\e[1;92m" +#define STASIS_COLOR_GREEN "\x1b[1;92m" //! Set output color to yellow -#define STASIS_COLOR_YELLOW "\e[1;93m" +#define STASIS_COLOR_YELLOW "\x1b[1;93m" //! Set output color to blue -#define STASIS_COLOR_BLUE "\e[1;94m" +#define STASIS_COLOR_BLUE "\x1b[1;94m" //! Set output color to white -#define STASIS_COLOR_WHITE "\e[1;97m" +#define STASIS_COLOR_WHITE "\x1b[1;97m" //! Reset output color to terminal default -#define STASIS_COLOR_RESET "\e[0;37m\e[0m" +#define STASIS_COLOR_RESET "\x1b[0;37m\x1b[0m" #endif #define STASIS_MSG_SUCCESS 0 @@ -284,9 +294,10 @@ int xml_pretty_print_in_place(const char *filename, const char *pretty_print_pro * Applies STASIS fixups to a tox ini config * @param filename path to tox.ini * @param result path to processed configuration + * @param maxlen * @return 0 on success, -1 on error */ -int fix_tox_conf(const char *filename, char **result); +int fix_tox_conf(const char *filename, char **result, size_t maxlen); char *collapse_whitespace(char **s); @@ -411,10 +422,74 @@ int env_manipulate_pathstr(const char *key, char *path, int mode); /** * Append or replace a file extension */ -int gen_file_extension_str(char *filename, const char *extension); +int gen_file_extension_str(char *filename, size_t maxlen, const char *extension); /** * Remove [extra]s from a spec string */ char *remove_extras(char *s); + +void debug_hexdump(char *data, int len); + +/** + * Realloc helper + * + * @code{.c} + * #include <stdio.h> + * #include <stdlib.h> + * #include <string.h> + * #include "utils.h" + * + * int main(int argc, char *argv[]) { + * size_t sz = 10; + * char *data = calloc(sz, sizeof(*data)); + * + * // populate data + * strncat(data, "/path/to/", sz - 1); + * + * // Double the allocation size for data + * if (grow(sz * 2, &sz, &data)) { + * // memory error + * } + * + * // sz is now 20 + * strncat(data, "filename", sz - 1 - strlen(data)); + * + * puts(data); + * // output: "/path/to/filename" + * } + * @endcode + * + * @param size_new increase by `size_new` bytes + * @param size_orig address of variable containing the original allocation size (modified) + * @param data address to write data + * @return 0 on success + * @return -1 on error + */ +int grow(size_t size_new, size_t *size_orig, char **data); + +int in_ascii_range(char c, char lower, char upper); + +#define GIT_HASH_LEN 40 +int is_git_sha(char const *hash); + +int check_python_package_dependencies(const char *srcdir); + +void seconds_to_human_readable(int v, char *result, size_t maxlen); + +#define STR_TO_TIMEOUT_NEGATIVE (-1) +#define STR_TO_TIMEOUT_INVALID_TIME_SCALE (-2) +int str_to_timeout(char *s); + +const char *get_random_generator_file(); +int get_random_bytes(char *result, size_t maxlen); + +/** + * Get length of `s` as if any formatters are not present + * @param s format string + * @return length + */ +int non_format_len(const char *s); + +char *center_text(const char *s, size_t maxwidth); #endif //STASIS_UTILS_H diff --git a/src/lib/core/include/version_compare.h b/src/lib/core/include/version_compare.h new file mode 100644 index 0000000..857de47 --- /dev/null +++ b/src/lib/core/include/version_compare.h @@ -0,0 +1,20 @@ +#ifndef STASIS_VERSION_COMPARE_H +#define STASIS_VERSION_COMPARE_H + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <ctype.h> +#include "str.h" + +#define GT 1 << 1 +#define LT 1 << 2 +#define EQ 1 << 3 +#define NOT 1 << 4 +#define EPOCH_MOD 10000 + +int version_sum(const char *str); +int version_parse_operator(const char *str); +int version_compare(int flags, const char *aa, const char *bb); + +#endif //STASIS_VERSION_COMPARE_H
\ No newline at end of file diff --git a/src/lib/core/include/wheel.h b/src/lib/core/include/wheel.h index 1a689e9..765f2c3 100644 --- a/src/lib/core/include/wheel.h +++ b/src/lib/core/include/wheel.h @@ -1,36 +1,257 @@ -//! @file wheel.h -#ifndef STASIS_WHEEL_H -#define STASIS_WHEEL_H +#ifndef WHEEL_H +#define WHEEL_H -#include <dirent.h> -#include <string.h> #include <stdio.h> -#include "str.h" -#define WHEEL_MATCH_EXACT 0 ///< Match when all patterns are present -#define WHEEL_MATCH_ANY 1 ///< Match when any patterns are present +#include <stdlib.h> +#include <string.h> +#include <fnmatch.h> +#include <zip.h> + +#define WHEEL_MAXELEM 255 + +#define WHEEL_FROM_DIST 0 +#define WHEEL_FROM_METADATA 1 + +enum { + WHEEL_META_METADATA_VERSION=0, + WHEEL_META_NAME, + WHEEL_META_VERSION, + WHEEL_META_AUTHOR, + WHEEL_META_AUTHOR_EMAIL, + WHEEL_META_MAINTAINER, + WHEEL_META_MAINTAINER_EMAIL, + WHEEL_META_SUMMARY, + WHEEL_META_LICENSE, + WHEEL_META_LICENSE_EXPRESSION, + WHEEL_META_LICENSE_FILE, + WHEEL_META_HOME_PAGE, + WHEEL_META_DOWNLOAD_URL, + WHEEL_META_PROJECT_URL, + WHEEL_META_CLASSIFIER, + WHEEL_META_REQUIRES_PYTHON, + WHEEL_META_REQUIRES_EXTERNAL, + WHEEL_META_IMPORT_NAME, + WHEEL_META_IMPORT_NAMESPACE, + WHEEL_META_REQUIRES_DIST, + WHEEL_META_PROVIDES, + WHEEL_META_PROVIDES_DIST, + WHEEL_META_PROVIDES_EXTRA, + WHEEL_META_OBSOLETES, + WHEEL_META_OBSOLETES_DIST, + WHEEL_META_PLATFORM, + WHEEL_META_SUPPORTED_PLATFORM, + WHEEL_META_KEYWORDS, + WHEEL_META_DYNAMIC, + WHEEL_META_DESCRIPTION_CONTENT_TYPE, + WHEEL_META_DESCRIPTION, + WHEEL_META_END_ENUM, // NOP +}; + + +enum { + WHEEL_DIST_VERSION=0, + WHEEL_DIST_GENERATOR, + WHEEL_DIST_ROOT_IS_PURELIB, + WHEEL_DIST_TAG, + WHEEL_DIST_ZIP_SAFE, + WHEEL_DIST_TOP_LEVEL, + WHEEL_DIST_ENTRY_POINT, + WHEEL_DIST_RECORD, + WHEEL_DIST_END_ENUM, // NOP +}; + +struct WheelMetadata_ProvidesExtra { + char *target; + struct StrList *requires_dist; + int count; +}; + +struct WheelMetadata { + char *metadata_version; + char *name; + char *version; + char *summary; + struct StrList *author; + struct StrList *author_email; + struct StrList *maintainer; + struct StrList *maintainer_email; + char *license; + char *license_expression; + char *home_page; + char * download_url; + struct StrList *project_url; + struct StrList *classifier; + struct StrList *requires_python; + struct StrList *requires_external; + char *description_content_type; + struct StrList *license_file; + struct StrList *import_name; + struct StrList *import_namespace; + struct StrList *requires_dist; + struct StrList *provides; + struct StrList *provides_dist; + struct StrList *obsoletes; + struct StrList *obsoletes_dist; + char *description; + struct StrList *platform; + struct StrList *supported_platform; + struct StrList *keywords; + struct StrList *dynamic; + + struct WheelMetadata_ProvidesExtra **provides_extra; +}; + +struct WheelRecord { + char *filename; + char *checksum; + size_t size; +}; + +struct WheelEntryPoint { + char *name; + char *function; + char *type; +}; + +/* +Wheel-Version: 1.0 +Generator: setuptools (75.8.0) +Root-Is-Purelib: false +Tag: cp313-cp313-manylinux_2_17_x86_64 +Tag: cp313-cp313-manylinux2014_x86_64 +*/ struct Wheel { - char *distribution; ///< Package name - char *version; ///< Package version - char *build_tag; ///< Package build tag (optional) - char *python_tag; ///< Package Python tag (pyXY) - char *abi_tag; ///< Package ABI tag (cpXY, abiX, none) - char *platform_tag; ///< Package platform tag (linux_x86_64, any) - char *path_name; ///< Path to package on-disk - char *file_name; ///< Name of package on-disk + char *wheel_version; + char *generator; + char *root_is_pure_lib; + struct StrList *tag; + struct StrList *top_level; + int zip_safe; + struct WheelMetadata *metadata; + struct WheelRecord **record; + size_t num_record; + struct WheelEntryPoint **entry_point; + size_t num_entry_point; +}; + +#define METADATA_MULTILINE_PREFIX " " + + +static inline int consume_append(char **dest, const char *src, const char *accept) { + const char *start = src; + if (!strncmp(src, METADATA_MULTILINE_PREFIX, strlen(METADATA_MULTILINE_PREFIX))) { + start += strlen(METADATA_MULTILINE_PREFIX); + } + + const char *end = strpbrk(start, accept); + size_t cut_len = end ? (size_t)(end - start) : strlen(start); + size_t dest_len = strlen(*dest); + + char *tmp = realloc(*dest, strlen(*dest) + cut_len + 2); + if (!tmp) { + return -1; + } + *dest = tmp; + memcpy(*dest + dest_len, start, cut_len); + + dest_len += cut_len; + (*dest)[dest_len ? dest_len : 0] = '\n'; + (*dest)[dest_len + 1] = '\0'; + return 0; +} + +#define WHEEL_KEY_UNKNOWN (-1) +enum { + WHEELVAL_STR = 0, + WHEELVAL_STRLIST, + WHEELVAL_OBJ_EXTRA, + WHEELVAL_OBJ_RECORD, + WHEELVAL_OBJ_ENTRY_POINT, +}; + +struct WheelValue { + int type; + size_t count; + void *data; }; +enum { + WHEEL_PACKAGE_E_SUCCESS=0, + WHEEL_PACKAGE_E_FILENAME=-1, + WHEEL_PACKAGE_E_ALLOC=-2, + WHEEL_PACKAGE_E_GET=-3, + WHEEL_PACKAGE_E_GET_METADATA=-4, + WHEEL_PACKAGE_E_GET_TOP_LEVEL=-5, + WHEEL_PACKAGE_E_GET_RECORDS=-6, + WHEEL_PACKAGE_E_GET_ENTRY_POINT=-7, +}; + +/** + * Populate a `Wheel` structure using a Python wheel file as input. + * + * @param pkg pointer to a `Wheel` (may be initialized to `NULL`) + * @param filename path to a Python wheel file + * @return a WHEEL_PACKAGE_E_ error code + */ +int wheel_package(struct Wheel **pkg, const char *filename); + +/** + * Frees a `Wheel` structure + * @param pkg pointer to an initialized `Wheel` + */ +void wheel_package_free(struct Wheel **pkg); + + +/** + * Get wheel data by name + * @param pkg pointer to an initialized `Wheel` + * @param from `WHEEL_FROM_DIST`, `WHEEL_FROM_META` + * @param key name of key in DIST or META data + * @return a populated `WheelValue` (stack) + */ +struct WheelValue wheel_get_value_by_name(const struct Wheel *pkg, int from, const char *key); + + /** - * Extract metadata from a Python Wheel file name + * Get wheel data by internal identifier + * @param pkg pointer to an initialized `Wheel` + * @param from `WHEEL_FROM_DIST`, `WHEEL_FROM_META` + * @param id `WHEEL_META_VERSION`, `WHEEL_DIST_VERSION` (see wheel.h) + * @return a populated `WheelValue` (stack) + */ +struct WheelValue wheel_get_value_by_id(const struct Wheel *pkg, int from, ssize_t id); + +/** + * Returns the error code assocated with the `WheelValue`, if possible + * @param val a populated `WheelValue` + * @return error code (see wheel.h) + */ +int wheel_value_error(struct WheelValue const *val); + +/** + * Retreive the key name string for a given id + * @param from `WHEEL_FROM_DIST`, `WHEEL_FROM_META` + * @param id `WHEEL_META_VERSION`, `WHEEL_DIST_VERSION` (see wheel.h) + * @return the key name, or NULL + */ +const char *wheel_get_key_by_id(int from, ssize_t id); + +/** + * Get the contents of a file within a Python wheel + * @param wheelfile path to Python wheel file + * @param filename path to file inside of wheel file archive + * @param contents pointer to store file contents + * @return 0 on success, -1 on error + */ +int wheel_get_file_contents(const char *wheelfile, const char *filename, char **contents); + +/** + * Display the values of a `Wheel` structure in human readable format * - * @param basepath directory containing a wheel file - * @param name of wheel file - * @param to_match a NULL terminated array of patterns (i.e. platform, arch, version, etc) - * @param match_mode WHEEL_MATCH_EXACT - * @param match_mode WHEEL_MATCH ANY - * @return pointer to populated Wheel on success - * @return NULL on error - */ -struct Wheel *get_wheel_info(const char *basepath, const char *name, char *to_match[], unsigned match_mode); -void wheel_free(struct Wheel **wheel); -#endif //STASIS_WHEEL_H + * @param wheel + * @return 0 on success, -1 on error + */ +int wheel_show_info(const struct Wheel *wheel); + +#endif //WHEEL_H diff --git a/src/lib/core/include/wheelinfo.h b/src/lib/core/include/wheelinfo.h new file mode 100644 index 0000000..8009e91 --- /dev/null +++ b/src/lib/core/include/wheelinfo.h @@ -0,0 +1,36 @@ +//! @file wheel.h +#ifndef STASIS_WHEEL_H +#define STASIS_WHEEL_H + +#include <dirent.h> +#include <string.h> +#include <stdio.h> +#include "str.h" +#define WHEEL_MATCH_EXACT 0 ///< Match when all patterns are present +#define WHEEL_MATCH_ANY 1 ///< Match when any patterns are present + +struct WheelInfo { + char *distribution; ///< Package name + char *version; ///< Package version + char *build_tag; ///< Package build tag (optional) + char *python_tag; ///< Package Python tag (pyXY) + char *abi_tag; ///< Package ABI tag (cpXY, abiX, none) + char *platform_tag; ///< Package platform tag (linux_x86_64, any) + char *path_name; ///< Path to package on-disk + char *file_name; ///< Name of package on-disk +}; + +/** + * Extract metadata from a Python Wheel file name + * + * @param basepath directory containing a wheel file + * @param name of wheel file + * @param to_match a NULL terminated array of patterns (i.e. platform, arch, version, etc) + * @param match_mode WHEEL_MATCH_EXACT + * @param match_mode WHEEL_MATCH ANY + * @return pointer to populated Wheel on success + * @return NULL on error + */ +struct WheelInfo *wheelinfo_get(const char *basepath, const char *name, char *to_match[], unsigned match_mode); +void wheelinfo_free(struct WheelInfo **wheel); +#endif //STASIS_WHEEL_H diff --git a/src/lib/core/ini.c b/src/lib/core/ini.c index cf6f670..3c0377f 100644 --- a/src/lib/core/ini.c +++ b/src/lib/core/ini.c @@ -7,12 +7,16 @@ struct INIFILE *ini_init() { struct INIFILE *ini = calloc(1, sizeof(*ini)); + if (!ini) { + return NULL; + } ini->section_count = 0; return ini; } -void ini_section_init(struct INIFILE **ini) { - (*ini)->section = calloc((*ini)->section_count + 1, sizeof(**(*ini)->section)); +struct INISection **ini_section_init(struct INIFILE **ini) { + struct INISection **section = calloc((*ini)->section_count + 1, sizeof(**(*ini)->section)); + return section; } struct INISection *ini_section_search(struct INIFILE **ini, unsigned mode, const char *value) { @@ -177,8 +181,12 @@ int ini_getval(struct INIFILE *ini, char *section_name, char *key, int type, int } break; case INIVAL_TYPE_STR_ARRAY: - strcpy(tbufp, data_copy); + // TODO: data_copy should be at least equal to the length of the data. The use of STASIS_BUFSIZ below is + // the root cause of crashes when stasis reads long arrays. + strncpy(tbufp, data_copy, sizeof(tbuf) - 1); + tbuf[sizeof(tbuf) - 1] = '\0'; guard_free(data_copy); + data_copy = calloc(STASIS_BUFSIZ, sizeof(*data_copy)); if (!data_copy) { return -1; @@ -186,10 +194,11 @@ int ini_getval(struct INIFILE *ini, char *section_name, char *key, int type, int while ((token = strsep(&tbufp, "\n")) != NULL) { //lstrip(token); if (!isempty(token)) { - strcat(data_copy, token); - strcat(data_copy, "\n"); + strncat(data_copy, token, STASIS_BUFSIZ - strlen(data_copy) - 1); + strncat(data_copy, "\n", STASIS_BUFSIZ - strlen(data_copy) - 1); } } + data_copy[STASIS_BUFSIZ - 1] = '\0'; strip(data_copy); result->as_char_p = strdup(data_copy); break; @@ -211,80 +220,82 @@ int ini_getval(struct INIFILE *ini, char *section_name, char *key, int type, int #define getval_returns(t) return result.t #define getval_setup(t, f) \ - union INIVal result; \ + union INIVal result = {0}; \ + do {\ int state_local = 0; \ state_local = ini_getval(ini, section_name, key, t, f, &result); \ if (state != NULL) { \ *state = state_local; \ - } + } \ +} while (0) int ini_getval_int(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { - getval_setup(INIVAL_TYPE_INT, flags) + getval_setup(INIVAL_TYPE_INT, flags); getval_returns(as_int); } unsigned int ini_getval_uint(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { - getval_setup(INIVAL_TYPE_UINT, flags) + getval_setup(INIVAL_TYPE_UINT, flags); getval_returns(as_uint); } long ini_getval_long(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { - getval_setup(INIVAL_TYPE_LONG, flags) + getval_setup(INIVAL_TYPE_LONG, flags); getval_returns(as_long); } unsigned long ini_getval_ulong(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { - getval_setup(INIVAL_TYPE_ULONG, flags) + getval_setup(INIVAL_TYPE_ULONG, flags); getval_returns(as_ulong); } long long ini_getval_llong(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { - getval_setup(INIVAL_TYPE_LLONG, flags) + getval_setup(INIVAL_TYPE_LLONG, flags); getval_returns(as_llong); } unsigned long long ini_getval_ullong(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { - getval_setup(INIVAL_TYPE_ULLONG, flags) + getval_setup(INIVAL_TYPE_ULLONG, flags); getval_returns(as_ullong); } float ini_getval_float(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { - getval_setup(INIVAL_TYPE_FLOAT, flags) + getval_setup(INIVAL_TYPE_FLOAT, flags); getval_returns(as_float); } double ini_getval_double(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { - getval_setup(INIVAL_TYPE_DOUBLE, flags) + getval_setup(INIVAL_TYPE_DOUBLE, flags); getval_returns(as_double); } bool ini_getval_bool(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { - getval_setup(INIVAL_TYPE_BOOL, flags) + getval_setup(INIVAL_TYPE_BOOL, flags); getval_returns(as_bool); } short ini_getval_short(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { - getval_setup(INIVAL_TYPE_SHORT, flags) + getval_setup(INIVAL_TYPE_SHORT, flags); getval_returns(as_short); } unsigned short ini_getval_ushort(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { - getval_setup(INIVAL_TYPE_USHORT, flags) + getval_setup(INIVAL_TYPE_USHORT, flags); getval_returns(as_ushort); } char ini_getval_char(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { - getval_setup(INIVAL_TYPE_CHAR, flags) + getval_setup(INIVAL_TYPE_CHAR, flags); getval_returns(as_char); } unsigned char ini_getval_uchar(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { - getval_setup(INIVAL_TYPE_UCHAR, flags) + getval_setup(INIVAL_TYPE_UCHAR, flags); getval_returns(as_uchar); } char *ini_getval_char_p(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { - getval_setup(INIVAL_TYPE_STR, flags) + getval_setup(INIVAL_TYPE_STR, flags); getval_returns(as_char_p); } @@ -293,7 +304,7 @@ char *ini_getval_str(struct INIFILE *ini, char *section_name, char *key, int fla } char *ini_getval_char_array_p(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { - getval_setup(INIVAL_TYPE_STR_ARRAY, flags) + getval_setup(INIVAL_TYPE_STR_ARRAY, flags); getval_returns(as_char_p); } @@ -302,7 +313,7 @@ char *ini_getval_str_array(struct INIFILE *ini, char *section_name, char *key, i } struct StrList *ini_getval_strlist(struct INIFILE *ini, char *section_name, char *key, char *tok, int flags, int *state) { - getval_setup(INIVAL_TYPE_STR_ARRAY, flags) + getval_setup(INIVAL_TYPE_STR_ARRAY, flags); struct StrList *list = strlist_init(); strlist_append_tokenize(list, result.as_char_p, tok); guard_free(result.as_char_p); @@ -342,6 +353,10 @@ int ini_data_append(struct INIFILE **ini, char *section_name, char *key, char *v section->data_count++; } else { struct INIData *data = ini_data_get(*ini, section_name, key); + if (!data) { + SYSERROR("%s:%s: key does not exist", section_name, key); + return -1; + } size_t value_len_old = strlen(data->value); size_t value_len = strlen(value); size_t value_len_new = value_len_old + value_len; @@ -350,10 +365,10 @@ int ini_data_append(struct INIFILE **ini, char *section_name, char *key, char *v if (!value_tmp) { SYSERROR("Unable to increase data->value size to %zu bytes", value_len_new + 2); return -1; - } else { - data->value = value_tmp; } - strcat(data->value, value); + data->value = value_tmp; + + strncat(data->value, value, value_len_new - strlen(data->value)); } return 0; } @@ -389,20 +404,22 @@ int ini_setval(struct INIFILE **ini, unsigned type, char *section_name, char *ke } int ini_section_create(struct INIFILE **ini, char *key) { - struct INISection **tmp = realloc((*ini)->section, ((*ini)->section_count + 1) * sizeof(**(*ini)->section)); - if (tmp == NULL) { + struct INISection **tmp = realloc((*ini)->section, ((*ini)->section_count + 1) * sizeof (*(*ini)->section)); + if (!tmp) { + ini_free(ini); return 1; - } else { - (*ini)->section = tmp; } + (*ini)->section = tmp; - (*ini)->section[(*ini)->section_count] = calloc(1, sizeof(*(*ini)->section[0])); - if (!(*ini)->section[(*ini)->section_count]) { + struct INISection **section = &(*ini)->section[(*ini)->section_count]; + //[(*ini)->section_count]; + *section = calloc(1, sizeof(*(*ini)->section[0])); + if (!*section) { return -1; } - (*ini)->section[(*ini)->section_count]->key = strdup(key); - if (!(*ini)->section[(*ini)->section_count]->key) { + (*section)->key = strdup(key); + if (!(*section)->key) { return -1; } @@ -417,6 +434,7 @@ int ini_write(struct INIFILE *ini, FILE **stream, unsigned mode) { for (size_t x = 0; x < ini->section_count; x++) { struct INISection *section = ini->section[x]; char *section_name = section->key; + fprintf(*stream, "[%s]" LINE_SEP, section_name); for (size_t y = 0; y < ini->section[x]->data_count; y++) { @@ -425,6 +443,7 @@ int ini_write(struct INIFILE *ini, FILE **stream, unsigned mode) { char *key = data->key; char *value = data->value; unsigned *hint = &data->type_hint; + memset(outvalue, 0, sizeof(outvalue)); if (key && value) { @@ -437,6 +456,9 @@ int ini_write(struct INIFILE *ini, FILE **stream, unsigned mode) { xvalue = ini_getval_str(ini, section_name, key, (int) mode, &err); value = xvalue; } + + const size_t buf_size = sizeof(outvalue); + size_t buf_len = 0; char **parts = split(value, LINE_SEP, 0); for (size_t p = 0; parts && parts[p] != NULL; p++) { char *render = NULL; @@ -447,19 +469,20 @@ int ini_write(struct INIFILE *ini, FILE **stream, unsigned mode) { } if (!render) { - SYSERROR("%s", "rendered string value can never be NULL!\n"); + SYSERROR("rendered string value can never be NULL!"); return -1; } + buf_len = strlen(outvalue); if (*hint == INIVAL_TYPE_STR_ARRAY) { - int leading_space = isspace(*render); + const int leading_space = isspace(*render); if (leading_space) { - sprintf(outvalue + strlen(outvalue), "%s" LINE_SEP, render); + snprintf(outvalue + buf_len, buf_size - buf_len, "%s" LINE_SEP, render); } else { - sprintf(outvalue + strlen(outvalue), " %s" LINE_SEP, render); + snprintf(outvalue + buf_len, buf_size - buf_len, " %s" LINE_SEP, render); } } else { - sprintf(outvalue + strlen(outvalue), "%s", render); + snprintf(outvalue + buf_len, buf_size - buf_len, "%s", render); } if (mode == INI_WRITE_PRESERVE) { guard_free(render); @@ -467,7 +490,12 @@ int ini_write(struct INIFILE *ini, FILE **stream, unsigned mode) { } guard_array_free(parts); strip(outvalue); - strcat(outvalue, LINE_SEP); + + // update length of outvalue + buf_len = strlen(outvalue); + + snprintf(outvalue + buf_len, buf_size - buf_len, "%s", LINE_SEP); + fprintf(*stream, "%s = %s%s", ini->section[x]->data[y]->key, *hint == INIVAL_TYPE_STR_ARRAY ? LINE_SEP : "", outvalue); guard_free(value); } else { @@ -489,13 +517,13 @@ char *unquote(char *s) { } void ini_free(struct INIFILE **ini) { + if (!(*ini)) { + return; + } for (size_t section = 0; section < (*ini)->section_count; section++) { - SYSDEBUG("freeing section: %s", (*ini)->section[section]->key); for (size_t data = 0; data < (*ini)->section[section]->data_count; data++) { if ((*ini)->section[section]->data[data]) { - SYSDEBUG("freeing data key: %s", (*ini)->section[section]->data[data]->key); guard_free((*ini)->section[section]->data[data]->key); - SYSDEBUG("freeing data value: %s", (*ini)->section[section]->data[data]->value); guard_free((*ini)->section[section]->data[data]->value); guard_free((*ini)->section[section]->data[data]); } @@ -518,11 +546,21 @@ struct INIFILE *ini_open(const char *filename) { return NULL; } - ini_section_init(&ini); + ini->section = ini_section_init(&ini); + if (!ini->section) { + ini_free(&ini); + return NULL; + } // Create an implicit section. [default] does not need to be present in the INI config - ini_section_create(&ini, "default"); - strcpy(current_section, "default"); + if (ini_section_create(&ini, "default")) { + SYSERROR("%s", "unable to create default section"); + ini_free(&ini); + ini = NULL; + return NULL; + } + strncpy(current_section, "default", sizeof(current_section) - 1); + current_section[sizeof(current_section) - 1] = '\0'; // Open the configuration file for reading FILE *fp = fopen(filename, "r"); @@ -535,15 +573,15 @@ struct INIFILE *ini_open(const char *filename) { unsigned hint = 0; int multiline_data = 0; int no_data = 0; - char inikey[2][255]; + char inikey[2][255] = {0}; char *key = inikey[0]; char *key_last = inikey[1]; char value[STASIS_BUFSIZ] = {0}; - memset(inikey, 0, sizeof(inikey)); - // Read file for (size_t i = 0; fgets(line, sizeof(line), fp) != NULL; i++) { + const size_t key_last_size = sizeof(inikey[1]); + const size_t key_size = sizeof(inikey[0]); if (no_data && multiline_data) { if (!isempty(line)) { no_data = 0; @@ -552,7 +590,7 @@ struct INIFILE *ini_open(const char *filename) { } memset(value, 0, sizeof(value)); } else { - memset(key, 0, sizeof(inikey[0])); + memset(key, 0, key_size); } // Find pointer to first comment character char *comment = strpbrk(line, ";#"); @@ -576,11 +614,12 @@ struct INIFILE *ini_open(const char *filename) { // Test for section header: [string] if (startswith(line, "[")) { // The previous key is irrelevant now - memset(key_last, 0, sizeof(inikey[1])); + memset(key_last, 0, key_last_size); char *section_name = substring_between(line, "[]"); if (!section_name) { - fprintf(stderr, "error: invalid section syntax, line %zu: '%s'\n", i + 1, line); + SYSERROR("invalid section syntax, line %zu: '%s'", i + 1, line); + ini_free(&ini); return NULL; } @@ -592,11 +631,18 @@ struct INIFILE *ini_open(const char *filename) { // Create new named section strip(section_name); - ini_section_create(&ini, section_name); + if (ini_section_create(&ini, section_name)) { + SYSERROR("unable to create section: %s", section_name); + guard_free(section_name); + ini_free(&ini); + return NULL; + } // Record the name of the section. This is used until another section is found. memset(current_section, 0, sizeof(current_section)); - strcpy(current_section, section_name); + strncpy(current_section, section_name, sizeof(current_section) - 1); + current_section[sizeof(current_section) - 1] = '\0'; + guard_free(section_name); memset(line, 0, sizeof(line)); continue; @@ -607,7 +653,7 @@ struct INIFILE *ini_open(const char *filename) { continue; } - char *operator = strchr(line, '='); + const char *operator = strchr(line, '='); // a value continuation line if (multiline_data && (startswith(line, " ") || startswith(line, "\t"))) { @@ -615,19 +661,26 @@ struct INIFILE *ini_open(const char *filename) { } if (operator) { - size_t key_len = operator - line; - memset(key, 0, sizeof(inikey[0])); + const size_t key_len = operator - line; + memset(key, 0, key_size); + strncpy(key, line, key_len); + key[key_size - 1] = '\0'; lstrip(key); strip(key); - memset(key_last, 0, sizeof(inikey[1])); - strcpy(key_last, key); + + memset(key_last, 0, key_last_size); + strncpy(key_last, key, key_last_size - 1); + key_last[key_last_size - 1] = '\0'; + reading_value = 1; if (strlen(operator) > 1) { - strcpy(value, &operator[1]); + strncpy(value, &operator[1], sizeof(value) - 1); } else { - strcpy(value, ""); + strncpy(value, "", sizeof(value) - 1); } + value[sizeof(value) - 1] = '\0'; + if (isempty(value)) { //printf("%s is probably long raw data\n", key); hint = INIVAL_TYPE_STR_ARRAY; @@ -640,8 +693,10 @@ struct INIFILE *ini_open(const char *filename) { } strip(value); } else { - strcpy(key, key_last); - strcpy(value, line); + strncpy(key, key_last, key_size - 1); + key[key_size - 1] = '\0'; + strncpy(value, line, sizeof(value) - 1); + value[sizeof(value) - 1] = '\0'; } memset(line, 0, sizeof(line)); diff --git a/src/lib/core/junitxml.c b/src/lib/core/junitxml.c index 628f75f..e590cb5 100644 --- a/src/lib/core/junitxml.c +++ b/src/lib/core/junitxml.c @@ -3,17 +3,32 @@ #include "strlist.h" #include "junitxml.h" +static void testcase_failure_free(struct JUNIT_Failure **failure) { + struct JUNIT_Failure *x = (*failure); + guard_free(x->message); + guard_free(x); +} + +static void testcase_error_free(struct JUNIT_Error **error) { + struct JUNIT_Error *x = (*error); + guard_free(x->message); + guard_free(x); +} + +static void testcase_skipped_free(struct JUNIT_Skipped **skipped) { + struct JUNIT_Skipped *x = (*skipped); + guard_free(x->message); + guard_free(x); +} + static void testcase_result_state_free(struct JUNIT_Testcase **testcase) { struct JUNIT_Testcase *tc = (*testcase); if (tc->tc_result_state_type == JUNIT_RESULT_STATE_FAILURE) { - guard_free(tc->result_state.failure->message); - guard_free(tc->result_state.failure); + testcase_failure_free(&tc->result_state.failure); } else if (tc->tc_result_state_type == JUNIT_RESULT_STATE_ERROR) { - guard_free(tc->result_state.error->message); - guard_free(tc->result_state.error); + testcase_error_free(&tc->result_state.error); } else if (tc->tc_result_state_type == JUNIT_RESULT_STATE_SKIPPED) { - guard_free(tc->result_state.skipped->message); - guard_free(tc->result_state.skipped); + testcase_skipped_free(&tc->result_state.skipped); } } @@ -58,10 +73,16 @@ static struct JUNIT_Failure *testcase_failure_from_attributes(struct StrList *at return NULL; } for (size_t x = 0; x < strlist_count(attrs); x += 2) { - char *attr_name = strlist_item(attrs, x); - char *attr_value = strlist_item(attrs, x + 1); - if (!strcmp(attr_name, "message")) { - result->message = strdup(attr_value); + const char *attr_name = strlist_item(attrs, x); + const char *attr_value = strlist_item(attrs, x + 1); + if (attr_name && attr_value) { + if (!strcmp(attr_name, "message")) { + result->message = strdup(attr_value); + if (!result->message) { + SYSERROR("failed to allocate memory for testcase failure message"); + break; + } + } } } return result; @@ -73,10 +94,16 @@ static struct JUNIT_Error *testcase_error_from_attributes(struct StrList *attrs) return NULL; } for (size_t x = 0; x < strlist_count(attrs); x += 2) { - char *attr_name = strlist_item(attrs, x); - char *attr_value = strlist_item(attrs, x + 1); - if (!strcmp(attr_name, "message")) { - result->message = strdup(attr_value); + const char *attr_name = strlist_item(attrs, x); + const char *attr_value = strlist_item(attrs, x + 1); + if (attr_name && attr_value) { + if (!strcmp(attr_name, "message")) { + result->message = strdup(attr_value); + if (!result->message) { + SYSERROR("failed to allocate memory for testcase error message"); + break; + } + } } } return result; @@ -88,10 +115,16 @@ static struct JUNIT_Skipped *testcase_skipped_from_attributes(struct StrList *at return NULL; } for (size_t x = 0; x < strlist_count(attrs); x += 2) { - char *attr_name = strlist_item(attrs, x); - char *attr_value = strlist_item(attrs, x + 1); - if (!strcmp(attr_name, "message")) { - result->message = strdup(attr_value); + const char *attr_name = strlist_item(attrs, x); + const char *attr_value = strlist_item(attrs, x + 1); + if (attr_name && attr_value) { + if (!strcmp(attr_name, "message")) { + result->message = strdup(attr_value); + if (!result->message) { + SYSERROR("failed to allocate memory for testcase skip message"); + break; + } + } } } return result; @@ -100,19 +133,37 @@ static struct JUNIT_Skipped *testcase_skipped_from_attributes(struct StrList *at static struct JUNIT_Testcase *testcase_from_attributes(struct StrList *attrs) { struct JUNIT_Testcase *result = calloc(1, sizeof(*result)); if(!result) { + SYSERROR("failed to allocate memory for testcase"); return NULL; } for (size_t x = 0; x < strlist_count(attrs); x += 2) { char *attr_name = strlist_item(attrs, x); char *attr_value = strlist_item(attrs, x + 1); - if (!strcmp(attr_name, "name")) { - result->name = strdup(attr_value); - } else if (!strcmp(attr_name, "classname")) { - result->classname = strdup(attr_value); - } else if (!strcmp(attr_name, "time")) { - result->time = strtof(attr_value, NULL); - } else if (!strcmp(attr_name, "message")) { - result->message = strdup(attr_value); + if (attr_name && attr_value) { + if (!strcmp(attr_name, "name")) { + result->name = strdup(attr_value); + if (!result->name) { + SYSERROR("failed to allocate memory for testcase name"); + testcase_free(&result); + break; + } + } else if (!strcmp(attr_name, "classname")) { + result->classname = strdup(attr_value); + if (!result->classname) { + SYSERROR("failed to allocate memory for testcase class name"); + testcase_free(&result); + break; + } + } else if (!strcmp(attr_name, "time")) { + result->time = strtof(attr_value, NULL); + } else if (!strcmp(attr_name, "message")) { + result->message = strdup(attr_value); + if (!result->message) { + SYSERROR("failed to allocate memory for testcase message"); + testcase_free(&result); + break; + } + } } } return result; @@ -158,42 +209,78 @@ static int read_xml_data(xmlTextReaderPtr reader, struct JUNIT_Testsuite **tests for (size_t x = 0; x < strlist_count(attrs); x += 2) { char *attr_name = strlist_item(attrs, x); char *attr_value = strlist_item(attrs, x + 1); - if (!strcmp(attr_name, "name")) { - (*testsuite)->name = strdup(attr_value); - } else if (!strcmp(attr_name, "errors")) { - (*testsuite)->errors = (int) strtol(attr_value, NULL, 10); - } else if (!strcmp(attr_name, "failures")) { - (*testsuite)->failures = (int) strtol(attr_value, NULL, 0); - } else if (!strcmp(attr_name, "skipped")) { - (*testsuite)->skipped = (int) strtol(attr_value, NULL, 0); - } else if (!strcmp(attr_name, "tests")) { - (*testsuite)->tests = (int) strtol(attr_value, NULL, 0); - } else if (!strcmp(attr_name, "time")) { - (*testsuite)->time = strtof(attr_value, NULL); - } else if (!strcmp(attr_name, "timestamp")) { - (*testsuite)->timestamp = strdup(attr_value); - } else if (!strcmp(attr_name, "hostname")) { - (*testsuite)->hostname = strdup(attr_value); + if (attr_name && attr_value) { + if (!strcmp(attr_name, "name")) { + (*testsuite)->name = strdup(attr_value); + if (!(*testsuite)->name) { + SYSERROR("%s", "failed to allocate memory for testcase name"); + return -1; + } + } else if (!strcmp(attr_name, "errors")) { + (*testsuite)->errors = (int) strtol(attr_value, NULL, 10); + } else if (!strcmp(attr_name, "failures")) { + (*testsuite)->failures = (int) strtol(attr_value, NULL, 0); + } else if (!strcmp(attr_name, "skipped")) { + (*testsuite)->skipped = (int) strtol(attr_value, NULL, 0); + } else if (!strcmp(attr_name, "tests")) { + (*testsuite)->tests = (int) strtol(attr_value, NULL, 0); + } else if (!strcmp(attr_name, "time")) { + (*testsuite)->time = strtof(attr_value, NULL); + } else if (!strcmp(attr_name, "timestamp")) { + (*testsuite)->timestamp = strdup(attr_value); + if (!(*testsuite)->timestamp) { + SYSERROR("%s", "failed to allocate memory for testcase timestamp"); + return -1; + } + } else if (!strcmp(attr_name, "hostname")) { + (*testsuite)->hostname = strdup(attr_value); + if (!(*testsuite)->hostname) { + SYSERROR("%s", "failed to allocate memory for testcase hostname"); + return -1; + } + } } } } else if (!strcmp(node_name, "testcase")) { struct JUNIT_Testcase *testcase = testcase_from_attributes(attrs); + if (!testcase) { + SYSERROR("%s", "failed to allocate memory for testcase"); + return -1; + } testsuite_append_testcase(testsuite, testcase); } else if (!strcmp(node_name, "failure")) { size_t cur_tc = (*testsuite)->_tc_inuse > 0 ? (*testsuite)->_tc_inuse - 1 : (*testsuite)->_tc_inuse; struct JUNIT_Failure *failure = testcase_failure_from_attributes(attrs); - (*testsuite)->testcase[cur_tc]->tc_result_state_type = JUNIT_RESULT_STATE_FAILURE; - (*testsuite)->testcase[cur_tc]->result_state.failure = failure; + if (!failure) { + SYSERROR("%s", "failed to allocate memory for testcase failure"); + return -1; + } + if ((*testsuite)->testcase && (*testsuite)->testcase[cur_tc]) { + (*testsuite)->testcase[cur_tc]->tc_result_state_type = JUNIT_RESULT_STATE_FAILURE; + (*testsuite)->testcase[cur_tc]->result_state.failure = failure; + } } else if (!strcmp(node_name, "error")) { size_t cur_tc = (*testsuite)->_tc_inuse > 0 ? (*testsuite)->_tc_inuse - 1 : (*testsuite)->_tc_inuse; struct JUNIT_Error *error = testcase_error_from_attributes(attrs); - (*testsuite)->testcase[cur_tc]->tc_result_state_type = JUNIT_RESULT_STATE_ERROR; - (*testsuite)->testcase[cur_tc]->result_state.error = error; + if (!error) { + SYSERROR("%s", "failed to allocate memory for testcase error"); + return -1; + } + if ((*testsuite)->testcase && (*testsuite)->testcase[cur_tc]) { + (*testsuite)->testcase[cur_tc]->tc_result_state_type = JUNIT_RESULT_STATE_ERROR; + (*testsuite)->testcase[cur_tc]->result_state.error = error; + } } else if (!strcmp(node_name, "skipped")) { size_t cur_tc = (*testsuite)->_tc_inuse > 0 ? (*testsuite)->_tc_inuse - 1 : (*testsuite)->_tc_inuse; struct JUNIT_Skipped *skipped = testcase_skipped_from_attributes(attrs); - (*testsuite)->testcase[cur_tc]->tc_result_state_type = JUNIT_RESULT_STATE_SKIPPED; - (*testsuite)->testcase[cur_tc]->result_state.skipped = skipped; + if (!skipped) { + SYSERROR("%s", "failed to allocate memory for testcase skipped"); + return -1; + } + if ((*testsuite)->testcase && (*testsuite)->testcase[cur_tc]) { + (*testsuite)->testcase[cur_tc]->tc_result_state_type = JUNIT_RESULT_STATE_SKIPPED; + (*testsuite)->testcase[cur_tc]->result_state.skipped = skipped; + } } } (*testsuite)->passed = (*testsuite)->tests - (*testsuite)->failures - (*testsuite)->errors - (*testsuite)->skipped; @@ -204,12 +291,17 @@ static int read_xml_data(xmlTextReaderPtr reader, struct JUNIT_Testsuite **tests static int read_xml_file(const char *filename, struct JUNIT_Testsuite **testsuite) { xmlTextReaderPtr reader = xmlReaderForFile(filename, NULL, 0); if (!reader) { + xmlFreeTextReader(reader); return -1; } int result = xmlTextReaderRead(reader); while (result == 1) { - read_xml_data(reader, testsuite); + if (read_xml_data(reader, testsuite) < 0) { + xmlFreeTextReader(reader); + return -1; + } + result = xmlTextReaderRead(reader); } @@ -228,6 +320,10 @@ struct JUNIT_Testsuite *junitxml_testsuite_read(const char *filename) { if (!result) { return NULL; } - read_xml_file(filename, &result); + + if (read_xml_file(filename, &result)) { + junitxml_testsuite_free(&result); + } + return result; }
\ No newline at end of file diff --git a/src/lib/core/log.c b/src/lib/core/log.c new file mode 100644 index 0000000..b1cab4c --- /dev/null +++ b/src/lib/core/log.c @@ -0,0 +1,115 @@ +#include "core.h" +#include "log.h" + +enum LogLevel LOG_LEVEL = LOG_LEVEL_WARN; + +void log_print_error(const struct ExecPoint ep, const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + log_msgv(stderr, ep, STASIS_COLOR_RED, "ERROR", fmt, ap); + va_end(ap); +} + +void log_print_warning(const struct ExecPoint ep, const char *fmt, ...) { + if (LOG_LEVEL < LOG_LEVEL_WARN) { + return; + } + va_list ap; + va_start(ap, fmt); + log_msgv(stderr, ep, STASIS_COLOR_YELLOW, "WARNING", fmt, ap); + va_end(ap); +} + +void log_print_info(const struct ExecPoint ep, const char *fmt, ...) { + if (LOG_LEVEL < LOG_LEVEL_INFO) { + return; + } + va_list ap; + va_start(ap, fmt); + log_msgv(stdout, ep, STASIS_COLOR_WHITE, "INFO", fmt, ap); + va_end(ap); +} + +void log_print_debug(const struct ExecPoint ep, const char *fmt, ...) { + if (LOG_LEVEL < LOG_LEVEL_DEBUG) { + return; + } + va_list ap; + va_start(ap, fmt); + log_msgv(stderr, ep, STASIS_COLOR_BLUE, "DEBUG", fmt, ap); + va_end(ap); +} + +int log_msgv(FILE *stream, const struct ExecPoint ep, const char *preface_color, const char *preface, const char *fmt, va_list ap) { + char header[STASIS_BUFSIZ] = {0}; + const int no_info = LOG_LEVEL < LOG_LEVEL_INFO; + const int some_info = LOG_LEVEL < LOG_LEVEL_DEBUG; + + int len = snprintf(header, sizeof(header), + STASIS_COLOR_RESET + "%s%s: " + STASIS_COLOR_RESET, + preface_color ? preface_color : STASIS_COLOR_RED, + preface ? preface : "UNKNOWN"); + + if (no_info) { + len += snprintf(header + strlen(header), sizeof(header) - len, + STASIS_COLOR_WHITE + "" + STASIS_COLOR_RESET); + } else if (some_info) { + len += snprintf(header + strlen(header), sizeof(header) - len, + STASIS_COLOR_WHITE + "%s(): " + STASIS_COLOR_RESET, + ep.function); + } else { + // everything + len += snprintf(header + strlen(header), sizeof(header) - len, + STASIS_COLOR_WHITE + "%s:%d: %s(): " + STASIS_COLOR_RESET, + path_basename((char *) ep.file), + ep.line, + ep.function); + } + + if (len >= (int) sizeof(header)) { + SYSERROR("header format truncated (%d >= %d)", len, (int) sizeof(header)); + return len; + } + + fprintf(stream, "%s", header); + + va_list ap_out; + va_copy(ap_out, ap); + len = vfprintf(stream, fmt ? fmt : "NO MESSAGE", ap_out); + va_end(ap_out); + if (len < 0) { + SYSERROR("\nvfprintf failed"); + return len; + } + fprintf(stderr, LINE_SEP); + return len; +} + +int log_msg(FILE *stream, const struct ExecPoint ep, const char *preface_color, const char *preface, const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + const int ret = log_msgv(stream, ep, preface_color, preface, fmt, ap); + va_end(ap); + return ret; +} + +const char *log_get_level_str(void) { + switch (LOG_LEVEL) { + case LOG_LEVEL_WARN: + return "WARN"; + case LOG_LEVEL_INFO: + return "INFO"; + case LOG_LEVEL_DEBUG: + default: + // Anything higher is still DEBUG + return "DEBUG"; + } +} diff --git a/src/lib/core/multiprocessing.c b/src/lib/core/multiprocessing.c index 0cf251e..b17bdc1 100644 --- a/src/lib/core/multiprocessing.c +++ b/src/lib/core/multiprocessing.c @@ -4,39 +4,108 @@ /// The sum of all tasks started by mp_task() size_t mp_global_task_count = 0; +static double get_duration(const struct timespec stop, const struct timespec start) { + const struct timespec result = timespec_sub(stop, start); + return timespec_to_double(result); +} + +static double get_task_duration(const struct MultiProcessingTask *task) { + const struct timespec *start = &task->time_data.t_start; + const struct timespec *stop = &task->time_data.t_stop; + return get_duration(*stop, *start); +} + +static double get_task_interval_duration(const struct MultiProcessingTask *task) { + const struct timespec *start = &task->interval_data.t_start; + const struct timespec *stop = &task->interval_data.t_stop; + return get_duration(*stop, *start); +} + +static void update_task_interval_start(struct MultiProcessingTask *task) { + // Record the task stop time + if (clock_gettime(CLOCK_REALTIME, &task->interval_data.t_start) < 0) { + SYSERROR("realtime clock unavailable"); + exit(1); + } +} + +static void update_task_interval_elapsed(struct MultiProcessingTask *task) { + // Record the interval stop time + if (clock_gettime(CLOCK_REALTIME, &task->interval_data.t_stop) < 0) { + SYSERROR("realtime clock unavailable"); + exit(1); + } + task->interval_data.duration = get_task_interval_duration(task); +} + +static void update_task_start(struct MultiProcessingTask *task) { + // Record the task start time + if (clock_gettime(CLOCK_REALTIME, &task->time_data.t_start) < 0) { + SYSERROR("realtime clock unavailable"); + exit(1); + } +} +static void update_task_elapsed(struct MultiProcessingTask *task) { + // Record the task stop time + if (clock_gettime(CLOCK_REALTIME, &task->time_data.t_stop) < 0) { + SYSERROR("realtime clock unavailable"); + exit(1); + } + task->time_data.duration = get_task_duration(task); +} + static struct MultiProcessingTask *mp_pool_next_available(struct MultiProcessingPool *pool) { return &pool->task[pool->num_used]; } int child(struct MultiProcessingPool *pool, struct MultiProcessingTask *task) { + (void) pool; FILE *fp_log = NULL; + semaphore_wait(&pool->semaphore); // The task starts inside the requested working directory if (chdir(task->working_dir)) { perror(task->working_dir); exit(1); } - // Record the task start time - if (clock_gettime(CLOCK_REALTIME, &task->time_data.t_start) < 0) { - perror("clock_gettime"); - exit(1); - } - // Redirect stdout and stderr to the log file fflush(stdout); fflush(stderr); + // Set log file name - sprintf(task->log_file + strlen(task->log_file), "task-%zu-%d.log", mp_global_task_count, task->parent_pid); + if (globals.enable_task_logging) { + snprintf(task->log_file + strlen(task->log_file), sizeof(task->log_file) - strlen(task->log_file), + "task-%zu-%d.log", mp_global_task_count, task->parent_pid); + SYSDEBUG("using log file: %s", task->log_file); + semaphore_post(&pool->semaphore); + } fp_log = freopen(task->log_file, "w+", stdout); if (!fp_log) { - fprintf(stderr, "unable to open '%s' for writing: %s\n", task->log_file, strerror(errno)); + SYSERROR("unable to open '%s' for writing: %s", task->log_file, strerror(errno)); + semaphore_post(&pool->semaphore); return -1; } - dup2(fileno(stdout), fileno(stderr)); + + const int redirect = dup2(STDOUT_FILENO, STDERR_FILENO); + if (redirect < 0) { + SYSERROR("%s", "Unable to redirect stderr to stdout"); + SYSERROR("Unable to redirect stderr to stdout"); + fclose(fp_log); + semaphore_post(&pool->semaphore); + return -1; + } + + // Close child file descriptors + for (int fd = 3; fd < sysconf(_SC_OPEN_MAX); fd++) { + if (fd == redirect) { + continue; + } + close(fd); + } // Generate timestamp for log header - time_t t = time(NULL); + const time_t t = time(NULL); char *timebuf = ctime(&t); if (timebuf) { // strip line feed from timestamp @@ -56,22 +125,30 @@ int child(struct MultiProcessingPool *pool, struct MultiProcessingTask *task) { fflush(stdout); fflush(stderr); char *args[] = {"bash", "--norc", task->parent_script, (char *) NULL}; - return execvp("/bin/bash", args); + semaphore_post(&pool->semaphore); + execvp("bash", args); + SYSERROR("execvp failed (%s)", strerror(errno)); + _exit(127); } int parent(struct MultiProcessingPool *pool, struct MultiProcessingTask *task, pid_t pid, int *child_status) { + // Record the task start time + update_task_start(task); + printf("[%s:%s] Task started (pid: %d)\n", pool->ident, task->ident, pid); // Give the child process access to our PID value task->pid = pid; task->parent_pid = pid; + semaphore_wait(&pool->semaphore); mp_global_task_count++; + semaphore_post(&pool->semaphore); // Check child's status pid_t code = waitpid(pid, child_status, WUNTRACED | WCONTINUED | WNOHANG); if (code < 0) { - perror("waitpid failed"); + SYSERROR("waitpid failed"); return -1; } return 0; @@ -79,24 +156,33 @@ int parent(struct MultiProcessingPool *pool, struct MultiProcessingTask *task, p static int mp_task_fork(struct MultiProcessingPool *pool, struct MultiProcessingTask *task) { SYSDEBUG("Preparing to fork() child task %s:%s", pool->ident, task->ident); + semaphore_wait(&pool->semaphore); pid_t pid = fork(); + int parent_status = 0; int child_status = 0; if (pid == -1) { + SYSERROR("fork failed"); return -1; - } else if (pid == 0) { + } + if (pid == 0) { + semaphore_post(&pool->semaphore); child(pool, task); + } else { + parent_status = parent(pool, task, pid, &child_status); + fflush(stdout); + fflush(stderr); } - return parent(pool, task, pid, &child_status); + return parent_status; } struct MultiProcessingTask *mp_pool_task(struct MultiProcessingPool *pool, const char *ident, char *working_dir, char *cmd) { - SYSDEBUG("%s", "Finding next available slot"); + SYSDEBUG("Finding next available slot"); struct MultiProcessingTask *slot = mp_pool_next_available(pool); if (pool->num_used != pool->num_alloc) { SYSDEBUG("Using slot %zu of %zu", pool->num_used, pool->num_alloc); pool->num_used++; } else { - fprintf(stderr, "Maximum number of tasks reached\n"); + SYSERROR("Maximum number of tasks reached"); return NULL; } @@ -105,18 +191,23 @@ struct MultiProcessingTask *mp_pool_task(struct MultiProcessingPool *pool, const // Set task identifier string memset(slot->ident, 0, sizeof(slot->ident)); - strncpy(slot->ident, ident, sizeof(slot->ident) - 1); + snprintf(slot->ident, sizeof(slot->ident), "%s", ident); // Set log file path memset(slot->log_file, 0, sizeof(*slot->log_file)); - strcat(slot->log_file, pool->log_root); - strcat(slot->log_file, "/"); + if (globals.enable_task_logging) { + snprintf(slot->log_file, sizeof(slot->log_file), "%s", pool->log_root); + // FORTIFY_SOURCE won't leave snprintf alone. The chance for truncation is slim anyway. + strncat(slot->log_file, "/", sizeof(slot->log_file) - strlen(slot->log_file)); + } else { + snprintf(slot->log_file, sizeof(slot->log_file), "/dev/stdout"); + } // Set working directory if (isempty(working_dir)) { - strcpy(slot->working_dir, "."); + snprintf(slot->working_dir, sizeof(slot->working_dir), "."); } else { - strncpy(slot->working_dir, working_dir, PATH_MAX - 1); + snprintf(slot->working_dir, sizeof(slot->working_dir), "%s", working_dir); } // Create a temporary file to act as our intermediate command script @@ -124,7 +215,16 @@ struct MultiProcessingTask *mp_pool_task(struct MultiProcessingPool *pool, const char *t_name = NULL; t_name = xmkstemp(&tp, "w"); - if (!t_name || !tp) { + if (!t_name) { + SYSERROR("Failed to create temporary file name"); + if (tp) { + fclose(tp); + } + return NULL; + } + if (!tp) { + SYSERROR("Failed to create temporary file"); + guard_free(t_name); return NULL; } @@ -135,7 +235,7 @@ struct MultiProcessingTask *mp_pool_task(struct MultiProcessingPool *pool, const // Record the script path memset(slot->parent_script, 0, sizeof(slot->parent_script)); - strncpy(slot->parent_script, t_name, PATH_MAX - 1); + snprintf(slot->parent_script, sizeof(slot->parent_script), "%s", t_name); guard_free(t_name); // Populate the script @@ -146,32 +246,26 @@ struct MultiProcessingTask *mp_pool_task(struct MultiProcessingPool *pool, const // Record the command(s) SYSDEBUG("%s", "mmap() slot command"); - slot->cmd_len = (strlen(cmd) * sizeof(*cmd)) + 1; - slot->cmd = mmap(NULL, slot->cmd_len, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); + slot->cmd_len = strlen(cmd) + 1; + slot->cmd = calloc(slot->cmd_len, sizeof(*slot->cmd)); + if (!slot->cmd) { + SYSERROR("Failed to allocate memory for slot command"); + return NULL; + } memset(slot->cmd, 0, slot->cmd_len); - strncpy(slot->cmd, cmd, slot->cmd_len); + snprintf(slot->cmd, slot->cmd_len, "%s", cmd); - return slot; -} + // Set task timeout + slot->timeout = globals.task_timeout; -static void get_task_duration(struct MultiProcessingTask *task, struct timespec *result) { - // based on the timersub() macro in time.h - // This implementation uses timespec and increases the resolution from microseconds to nanoseconds. - struct timespec *start = &task->time_data.t_start; - struct timespec *stop = &task->time_data.t_stop; - result->tv_sec = (stop->tv_sec - start->tv_sec); - result->tv_nsec = (stop->tv_nsec - start->tv_nsec); - if (result->tv_nsec < 0) { - --result->tv_sec; - result->tv_nsec += 1000000000L; - } + return slot; } void mp_pool_show_summary(struct MultiProcessingPool *pool) { print_banner("=", 79); printf("Pool execution summary for \"%s\"\n", pool->ident); print_banner("=", 79); - printf("STATUS PID DURATION IDENT\n"); + printf("STATUS PID DURATION IDENT\n"); for (size_t i = 0; i < pool->num_used; i++) { struct MultiProcessingTask *task = &pool->task[i]; char status_str[10] = {0}; @@ -179,20 +273,18 @@ void mp_pool_show_summary(struct MultiProcessingPool *pool) { if (task->status == MP_POOL_TASK_STATUS_INITIAL && task->pid == MP_POOL_PID_UNUSED) { // You will only see this label if the task pool is killed by // MP_POOL_FAIL_FAST and tasks are still queued for execution - strcpy(status_str, "HOLD"); + snprintf(status_str, sizeof(status_str), "HOLD"); } else if (!task->status && !task->signaled_by) { - - strcpy(status_str, "DONE"); + snprintf(status_str, sizeof(status_str), "DONE"); } else if (task->signaled_by) { - strcpy(status_str, "TERM"); + snprintf(status_str, sizeof(status_str), "TERM"); } else { - strcpy(status_str, "FAIL"); + snprintf(status_str, sizeof(status_str), "FAIL"); } - struct timespec duration; - get_task_duration(task, &duration); - long diff = duration.tv_sec + duration.tv_nsec / 1000000000L; - printf("%-4s %10d %7lds %-10s\n", status_str, task->parent_pid, diff, task->ident) ; + char duration[255] = {0}; + seconds_to_human_readable(task->time_data.duration, duration, sizeof(duration)); + printf("%-4s %10d %10s %-10s\n", status_str, task->parent_pid, duration, task->ident) ; } puts(""); } @@ -200,14 +292,16 @@ void mp_pool_show_summary(struct MultiProcessingPool *pool) { static int show_log_contents(FILE *stream, struct MultiProcessingTask *task) { FILE *fp = fopen(task->log_file, "r"); if (!fp) { + SYSERROR("Failed to open log file for reading: %s, %s", task->log_file, strerror(errno)); return -1; } - char buf[BUFSIZ] = {0}; + char buf[STASIS_BUFSIZ] = {0}; while ((fgets(buf, sizeof(buf) - 1, fp)) != NULL) { fprintf(stream, "%s", buf); memset(buf, 0, sizeof(buf)); } fprintf(stream, "\n"); + fflush(stream); fclose(fp); return 0; } @@ -223,38 +317,45 @@ int mp_pool_kill(struct MultiProcessingPool *pool, int signum) { if (slot->pid > 0) { int status; printf("Sending signal %d to task '%s' (pid: %d)\n", signum, slot->ident, slot->pid); + semaphore_wait(&pool->semaphore); status = kill(slot->pid, signum); + semaphore_post(&pool->semaphore); if (status && errno != ESRCH) { - fprintf(stderr, "Task '%s' (pid: %d) did not respond: %s\n", slot->ident, slot->pid, strerror(errno)); + SYSERROR("Task '%s' (pid: %d) did not respond: %s", slot->ident, slot->pid, strerror(errno)); } else { // Wait for process to handle the signal, then set the status accordingly if (waitpid(slot->pid, &status, 0) >= 0) { slot->signaled_by = WTERMSIG(status); - // Record the task stop time - if (clock_gettime(CLOCK_REALTIME, &slot->time_data.t_stop) < 0) { - perror("clock_gettime"); - exit(1); - } + semaphore_wait(&pool->semaphore); + update_task_elapsed(slot); + semaphore_post(&pool->semaphore); // We are short-circuiting the normal flow, and the process is now dead, so mark it as such SYSDEBUG("Marking slot %zu: UNUSED", i); slot->pid = MP_POOL_PID_UNUSED; } } } - if (!access(slot->log_file, F_OK)) { - SYSDEBUG("Removing log file: %s", slot->log_file); - remove(slot->log_file); + if (globals.enable_task_logging) { + semaphore_wait(&pool->semaphore); + if (!access(slot->log_file, F_OK)) { + SYSDEBUG("Removing log file: %s", slot->log_file); + remove(slot->log_file); + } + semaphore_post(&pool->semaphore); } + + semaphore_wait(&pool->semaphore); if (!access(slot->parent_script, F_OK)) { SYSDEBUG("Removing runner script: %s", slot->parent_script); remove(slot->parent_script); } + semaphore_post(&pool->semaphore); } return 0; } int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) { - int status = 0; + int child_status = 0; int failures = 0; size_t tasks_complete = 0; size_t lower_i = 0; @@ -268,11 +369,12 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) { } for (size_t i = lower_i; i < upper_i; i++) { + char duration[255] = {0}; struct MultiProcessingTask *slot = &pool->task[i]; if (slot->status == MP_POOL_TASK_STATUS_INITIAL) { slot->_startup = time(NULL); if (mp_task_fork(pool, slot)) { - fprintf(stderr, "%s: mp_task_fork failed\n", slot->ident); + SYSERROR("%s: mp_task_fork failed", slot->ident); kill(0, SIGTERM); } } @@ -281,12 +383,12 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) { if (slot->pid == MP_POOL_PID_UNUSED) { // Child is already used up, skip it hang_check++; - SYSDEBUG("slot %zu: hang_check=%zu", i, hang_check); if (hang_check >= pool->num_used) { // If you join a pool that's already finished it will spin // forever. This protects the program from entering an // infinite loop. - fprintf(stderr, "%s is deadlocked\n", pool->ident); + SYSDEBUG("slot %zu: hang_check=%zu >= pool->num_used=%zu", i, hang_check, pool->num_used); + SYSERROR("%s is deadlocked", pool->ident); failures++; goto pool_deadlocked; } @@ -294,55 +396,73 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) { } // Is the process finished? - pid_t pid = waitpid(slot->pid, &status, WNOHANG | WUNTRACED | WCONTINUED); - int task_ended = WIFEXITED(status); - int task_ended_by_signal = WIFSIGNALED(status); - int task_stopped = WIFSTOPPED(status); - int task_continued = WIFCONTINUED(status); - int status_exit = WEXITSTATUS(status); - int status_signal = WTERMSIG(status); - int status_stopped = WSTOPSIG(status); + const pid_t pid = waitpid(slot->pid, &child_status, WNOHANG | WUNTRACED | WCONTINUED); + + char progress[1024] = {0}; + const double percent = ((double) (tasks_complete + 1) / (double) pool->num_used) * 100; + snprintf(progress, sizeof(progress), "[%s:%s] [%3.1f%%]", pool->ident, slot->ident, percent); + + int task_timed_out = false; + if (slot->timeout) { + task_timed_out = slot->time_data.duration >= (double) slot->timeout; + if (task_timed_out && pid == 0 && slot->pid != 0) { + seconds_to_human_readable(slot->timeout, duration, sizeof(duration)); + printf("%s Task timed out after %s (pid: %d)\n", progress, duration, slot->pid); + if (kill(slot->pid, SIGKILL) == 0) { + child_status = SIGKILL; + } else { + SYSERROR("Timeout reached, however pid %d could not be killed.", slot->pid); + return -1; + } + } + } + + const int task_ended = WIFEXITED(child_status); + const int task_ended_by_signal = WIFSIGNALED(child_status); + const int task_stopped = WIFSTOPPED(child_status); + const int task_continued = WIFCONTINUED(child_status); + const int status_exit = WEXITSTATUS(child_status); + const int status_signal = WTERMSIG(child_status); + const int status_stopped = WSTOPSIG(child_status); // Update status slot->status = status_exit; slot->signaled_by = status_signal; - char progress[1024] = {0}; if (pid > 0) { - double percent = ((double) (tasks_complete + 1) / (double) pool->num_used) * 100; - snprintf(progress, sizeof(progress) - 1, "[%s:%s] [%3.1f%%]", pool->ident, slot->ident, percent); - // The process ended in one the following ways // Note: SIGSTOP nor SIGCONT will not increment the tasks_complete counter if (task_stopped) { printf("%s Task was suspended (%d)\n", progress, status_stopped); continue; - } else if (task_continued) { + } + if (task_continued) { printf("%s Task was resumed\n", progress); continue; - } else if (task_ended_by_signal) { + } + if (task_ended_by_signal) { printf("%s Task ended by signal %d (%s)\n", progress, status_signal, strsignal(status_signal)); tasks_complete++; } else if (task_ended) { printf("%s Task ended (status: %d)\n", progress, status_exit); tasks_complete++; } else { - fprintf(stderr, "%s Task state is unknown (0x%04X)\n", progress, status); - } - - // Show the log (always) - if (show_log_contents(stdout, slot)) { - perror(slot->log_file); + SYSWARN("%s Task state is unknown (0x%04X)", progress, child_status); } - // Record the task stop time - if (clock_gettime(CLOCK_REALTIME, &slot->time_data.t_stop) < 0) { - perror("clock_gettime"); - exit(1); + if (globals.enable_task_logging) { + // Show the log (always) + if (show_log_contents(stdout, slot)) { + SYSWARN("%s Task has no log file", slot->ident); + } } - if (status >> 8 != 0 || (status & 0xff) != 0) { - fprintf(stderr, "%s Task failed after %lus\n", progress, slot->elapsed); + if (child_status >> 8 != 0 || (child_status & 0xff) != 0) { + semaphore_wait(&pool->semaphore); + update_task_elapsed(slot); + semaphore_post(&pool->semaphore); + seconds_to_human_readable(slot->time_data.duration, duration, sizeof(duration)); + fprintf(stderr, "%s Task failed after %s\n", progress, duration); failures++; if (flags & MP_POOL_FAIL_FAST && pool->num_used > 1) { @@ -350,36 +470,49 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) { return -2; } } else { - printf("%s Task finished after %lus\n", progress, slot->elapsed); + seconds_to_human_readable(slot->time_data.duration, duration, sizeof(duration)); + printf("%s Task finished after %s\n", progress, duration); } // Clean up logs and scripts left behind by the task - if (remove(slot->log_file)) { - fprintf(stderr, "%s Unable to remove log file: '%s': %s\n", progress, slot->parent_script, strerror(errno)); + if (globals.enable_task_logging) { + if (remove(slot->log_file)) { + SYSWARN("%s Unable to remove log file: '%s': %s", progress, slot->parent_script, strerror(errno)); + } } if (remove(slot->parent_script)) { - fprintf(stderr, "%s Unable to remove temporary script '%s': %s\n", progress, slot->parent_script, strerror(errno)); + SYSWARN("%s Unable to remove temporary script '%s': %s", progress, slot->parent_script, strerror(errno)); } // Update progress and tell the poller to ignore the PID. The process is gone. slot->pid = MP_POOL_PID_UNUSED; } else if (pid < 0) { - fprintf(stderr, "waitpid failed: %s\n", strerror(errno)); + SYSERROR("waitpid failed: %s", strerror(errno)); return -1; } else { // Track the number of seconds elapsed for each task. // When a task has executed for longer than status_intervals, print a status update - // _seconds represents the time between intervals, not the total runtime of the task - slot->_seconds = time(NULL) - slot->_now; - if (slot->_seconds > pool->status_interval) { - slot->_now = time(NULL); - slot->_seconds = 0; + // interval_elapsed represents the time between intervals, not the total runtime of the task + semaphore_wait(&pool->semaphore); + if (fabs(slot->interval_data.duration) > pool->status_interval) { + slot->interval_data.duration = 0.0; } - if (slot->_seconds == 0) { - printf("[%s:%s] Task is running (pid: %d, elapsed: %lus)\n", pool->ident, slot->ident, slot->parent_pid, slot->elapsed); + if (slot->interval_data.duration == 0.0) { + seconds_to_human_readable(slot->time_data.duration, duration, sizeof(duration)); + printf("[%s:%s] Task is running (pid: %d, elapsed: %s)\n", + pool->ident, slot->ident, slot->parent_pid, duration); + update_task_interval_start(slot); } + + update_task_interval_elapsed(slot); + semaphore_post(&pool->semaphore); + } + + if (!task_ended || !task_ended_by_signal) { + semaphore_wait(&pool->semaphore); + update_task_elapsed(slot); + semaphore_post(&pool->semaphore); } - slot->elapsed = time(NULL) - slot->_startup; } if (tasks_complete == pool->num_used) { @@ -392,11 +525,12 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) { } // Poll again after a short delay - sleep(1); + usleep(100000); } while (1); pool_deadlocked: puts(""); + return failures; } @@ -412,56 +546,77 @@ struct MultiProcessingPool *mp_pool_init(const char *ident, const char *log_root // The pool is shared with children pool = mmap(NULL, sizeof(*pool), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); + if (pool == MAP_FAILED) { + SYSERROR("unable to memory map pool"); + return NULL; + } // Set pool identity string memset(pool->ident, 0, sizeof(pool->ident)); - strncpy(pool->ident, ident, sizeof(pool->ident) - 1); + snprintf(pool->ident, sizeof(pool->ident), "%s", ident); // Set logging base directory memset(pool->log_root, 0, sizeof(pool->log_root)); - strncpy(pool->log_root, log_root, sizeof(pool->log_root) - 1); + snprintf(pool->log_root, sizeof(pool->log_root), "%s", log_root); pool->num_used = 0; pool->num_alloc = MP_POOL_TASK_MAX; // Create the log directory + SYSDEBUG("Creating log directory: %s", pool->log_root); if (mkdirs(log_root, 0700) < 0) { if (errno != EEXIST) { - perror(log_root); + SYSERROR("%s: unable to create directory", log_root); mp_pool_free(&pool); return NULL; } } // Task array is shared with children + SYSDEBUG("Memory mapping pool task array"); pool->task = mmap(NULL, (pool->num_alloc + 1) * sizeof(*pool->task), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); if (pool->task == MAP_FAILED) { - perror("mmap"); + SYSERROR("unable to memory map pool task"); + mp_pool_free(&pool); + return NULL; + } + + SYSDEBUG("initializing pool semaphore"); + char semaphore_name[255] = {0}; + snprintf(semaphore_name, sizeof(semaphore_name), "stasis_mp_%s", ident); + if (semaphore_init(&pool->semaphore, semaphore_name, 1) != 0) { + SYSERROR("unable to initialize pool semaphore"); mp_pool_free(&pool); return NULL; } + pool->status_interval = 3; + return pool; } void mp_pool_free(struct MultiProcessingPool **pool) { - for (size_t i = 0; i < (*pool)->num_alloc; i++) { + SYSDEBUG("freeing pool"); + if (!isempty((*pool)->semaphore.name)) { + semaphore_destroy(&(*pool)->semaphore); } - // Unmap all pool tasks + + // Free all task commands if ((*pool)->task) { - if ((*pool)->task->cmd) { - if (munmap((*pool)->task->cmd, (*pool)->task->cmd_len) < 0) { - perror("munmap"); - } + for (size_t i = 0; i < (*pool)->num_alloc + 1; i++) { + struct MultiProcessingTask *task = &(*pool)->task[i]; + guard_free(task->cmd); } - if (munmap((*pool)->task, sizeof(*(*pool)->task) * (*pool)->num_alloc) < 0) { - perror("munmap"); + // Unmap the task array + if (munmap((*pool)->task, sizeof(*(*pool)->task) * (*pool)->num_alloc + 1) < 0) { + SYSWARN("munmap pool task failed: %s", strerror(errno)); } } // Unmap the pool if ((*pool)) { if (munmap((*pool), sizeof(*(*pool))) < 0) { - perror("munmap"); + SYSWARN("munmap pool failed: %s", strerror(errno)); } (*pool) = NULL; } + SYSDEBUG("pool freed"); }
\ No newline at end of file diff --git a/src/lib/core/recipe.c b/src/lib/core/recipe.c index 99d3fe1..0ee1ef8 100644 --- a/src/lib/core/recipe.c +++ b/src/lib/core/recipe.c @@ -9,19 +9,20 @@ int recipe_clone(char *recipe_dir, char *url, char *gitref, char **result) { memset(destdir, 0, sizeof(destdir)); reponame = path_basename(url); - sprintf(destdir, "%s/%s", recipe_dir, reponame); + snprintf(destdir, sizeof(destdir), "%s/%s", recipe_dir, reponame); if (!*result) { *result = calloc(PATH_MAX, sizeof(*result)); if (!*result) { return -1; } } - strncpy(*result, destdir, PATH_MAX); + strncpy(*result, destdir, PATH_MAX - 1); + (*result)[PATH_MAX - 1] = '\0'; if (!access(destdir, F_OK)) { if (!strcmp(destdir, "/")) { - fprintf(stderr, "STASIS is misconfigured. Please check your output path(s) immediately.\n"); - fprintf(stderr, "recipe_dir = '%s'\nreponame = '%s'\ndestdir = '%s'\n", + SYSERROR("STASIS is misconfigured. Please check your output path(s) immediately."); + SYSERROR("recipe_dir = '%s'\nreponame = '%s'\ndestdir = '%s'", recipe_dir, reponame, destdir); exit(1); } @@ -52,7 +53,7 @@ int recipe_get_type(char *repopath) { for (size_t i = 0; marker[i] != NULL; i++) { char path[PATH_MAX] = {0}; - sprintf(path, "%s/%s", repopath, marker[i]); + snprintf(path, sizeof(path), "%s/%s", repopath, marker[i]); int result = access(path, F_OK); if (!result) { return type[i]; diff --git a/src/lib/core/relocation.c b/src/lib/core/relocation.c index 58b829d..8204101 100644 --- a/src/lib/core/relocation.c +++ b/src/lib/core/relocation.c @@ -8,8 +8,9 @@ * Replace all occurrences of `target` with `replacement` in `original` * * ~~~{.c} - * char *str = calloc(100, sizeof(char)); - * strcpy(str, "This are a test."); + * size_t str_maxlen = 100; + * char *str = calloc(str_maxlen, sizeof(char)); + * strncpy(str, "This are a test.", str_maxlen - 1); * if (replace_text(str, "are", "is")) { * fprintf(stderr, "string replacement failed\n"); * exit(1); @@ -35,7 +36,7 @@ int replace_text(char *original, const char *target, const char *replacement, un if (original_len > sizeof(buffer)) { errno = EINVAL; - SYSERROR("The original string is larger than buffer: %zu > %zu\n", original_len, sizeof(buffer)); + SYSERROR("The original string is larger than buffer: %zu > %zu", original_len, sizeof(buffer)); return -1; } @@ -50,18 +51,18 @@ int replace_text(char *original, const char *target, const char *replacement, un // replacement is shorter than the target if (rep_len < target_len) { // shrink the string - strcat(buffer, replacement); + strncat(buffer, replacement, sizeof(buffer) - strlen(buffer) - 1); memmove(pos, pos + target_len, strlen(pos) - target_len); memset(pos + (strlen(pos) - target_len), 0, target_len); } else { // replacement is longer than the target // write the replacement value to the buffer - strcat(buffer, replacement); + strncat(buffer, replacement, sizeof(buffer) - strlen(buffer) - 1); // target consumed. jump to the end of the substring. pos += target_len; } if (flags & REPLACE_TRUNCATE_AFTER_MATCH) { if (strstr(pos, LINE_SEP)) { - strcat(buffer, LINE_SEP); + strncat(buffer, LINE_SEP, sizeof(buffer) - strlen(buffer) - 1); } break; } @@ -69,7 +70,7 @@ int replace_text(char *original, const char *target, const char *replacement, un if (!((match = strstr(pos, target)))) { // no more matches // append whatever remains to the buffer - strcat(buffer, pos); + strncat(buffer, pos, sizeof(buffer) - strlen(buffer) - 1); // stop break; } @@ -84,7 +85,8 @@ int replace_text(char *original, const char *target, const char *replacement, un memset(original + buffer_len, 0, original_len - buffer_len); } // replace original with contents of buffer - strcpy(original, buffer); + strncpy(original, buffer, buffer_len + 1); + original[buffer_len] = '\0'; return 0; } @@ -110,7 +112,7 @@ int file_replace_text(const char* filename, const char* target, const char* repl FILE *fp = fopen(filename, "r"); if (!fp) { - fprintf(stderr, "unable to open for reading: %s\n", filename); + SYSERROR("unable to open for reading: %s", filename); return -1; } @@ -138,6 +140,7 @@ int file_replace_text(const char* filename, const char* target, const char* repl fp = fopen(filename, "w+"); if (!fp) { SYSERROR("unable to reopen %s for writing", filename); + fclose(tfp); return -1; } diff --git a/src/lib/core/semaphore.c b/src/lib/core/semaphore.c new file mode 100644 index 0000000..7ac1549 --- /dev/null +++ b/src/lib/core/semaphore.c @@ -0,0 +1,81 @@ +/** +* @file semaphore.c +*/ +#include <stdio.h> +#include <fcntl.h> + +#include "core_message.h" +#include "sem.h" +#include "utils.h" + +struct Semaphore *semaphores[1000] = {0}; +bool semaphore_handle_exit_ready = false; + +void semaphore_handle_exit() { + for (size_t i = 0; i < sizeof(semaphores) / sizeof(*semaphores); ++i) { + if (semaphores[i]) { + SYSDEBUG("%s", semaphores[i]->name); + semaphore_destroy(semaphores[i]); + } + } +} + +static void register_semaphore(struct Semaphore *s) { + struct Semaphore **cur = semaphores; + size_t i = 0; + while (i < sizeof(semaphores) / sizeof(*semaphores) && cur != NULL) { + cur++; + i++; + } + cur = &s; +} + +int semaphore_init(struct Semaphore *s, const char *name, const int value) { +#if defined(STASIS_OS_DARWIN) + // see: sem_open(2) + const size_t max_namelen = PSEMNAMLEN; +#else + // see: sem_open(3) + const size_t max_namelen = STASIS_NAME_MAX; +#endif + snprintf(s->name, max_namelen, "/%s", name); + s->sem = sem_open(s->name, O_CREAT, 0644, value); + if (s->sem == SEM_FAILED) { + return -1; + } + SYSDEBUG("%s", s->name); + register_semaphore(s); + if (!semaphore_handle_exit_ready) { + atexit(semaphore_handle_exit); + } + + return 0; +} + +int semaphore_wait(struct Semaphore *s) { + //int sgv_value = 0; + //int sgv_ret = sem_getvalue(s->sem, &sgv_value); + //SYSDEBUG("sem_getvalue() returned %d, value %d", sgv_ret, sgv_value); + const int status = sem_wait(s->sem); + //SYSDEBUG("returning %d", status); + return status; +} + +int semaphore_post(struct Semaphore *s) { + //int sgv_value = 0; + //int sgv_ret = sem_getvalue(s->sem, &sgv_value); + //SYSDEBUG("sem_getvalue() returned %d, value %d", sgv_ret, sgv_value); + const int status = sem_post(s->sem); + //SYSDEBUG("returning %d", status); + return status; +} + +void semaphore_destroy(struct Semaphore *s) { + if (!s) { + SYSDEBUG("would have crashed"); + return; + } + SYSDEBUG("%s", s->name); + sem_close(s->sem); + sem_unlink(s->name); +}
\ No newline at end of file diff --git a/src/lib/core/str.c b/src/lib/core/str.c index 1d0b268..c31ce5e 100644 --- a/src/lib/core/str.c +++ b/src/lib/core/str.c @@ -4,6 +4,22 @@ #include <unistd.h> #include "str.h" + + +char *strdup_maybe_entry(const char * restrict s, const struct ExecPoint ep, const int exit_code) { + // USE MACRO FROM str.h: strdup_maybe() + if (s != NULL) { + char *x = strdup(s); + if (!x) { + // We want to trace the origin of the allocation so SYSERROR can't be used here. + log_print_error(ep, "out of memory"); + exit(exit_code); + } + return x; + } + return NULL; +} + int num_chars(const char *sptr, int ch) { int result = 0; for (int i = 0; sptr[i] != '\0'; i++) { @@ -63,6 +79,7 @@ void strchrdel(char *sptr, const char *chars) { for (size_t i = 0; i < strlen(chars); i++) { char ch[2] = {0}; strncpy(ch, &chars[i], 1); + ch[sizeof(ch) - 1] = '\0'; replace_text(sptr, ch, "", 0); } } @@ -76,7 +93,6 @@ char** split(char *_sptr, const char* delim, size_t max) // Duplicate the input string and save a copy of the pointer to be freed later char *orig = _sptr; char *sptr = strdup(orig); - if (!sptr) { return NULL; } @@ -105,7 +121,7 @@ char** split(char *_sptr, const char* delim, size_t max) // Separate the string into individual parts and store them in the result array char *token = NULL; char *sptr_tmp = sptr; - size_t pos = 0; + ptrdiff_t pos = 0; size_t i; for (i = 0; (token = strsep(&sptr_tmp, delim)) != NULL; i++) { // When max is zero, record all tokens @@ -117,9 +133,12 @@ char** split(char *_sptr, const char* delim, size_t max) } result[i] = calloc(STASIS_BUFSIZ, sizeof(char)); if (!result[i]) { + guard_free(sptr); + guard_array_n_free(result, i); return NULL; } - strcpy(result[i], token); + strncpy(result[i], token, STASIS_BUFSIZ - 1); + result[i][STASIS_BUFSIZ - 1] = '\0'; } // pos is non-zero when maximum split is reached @@ -127,9 +146,12 @@ char** split(char *_sptr, const char* delim, size_t max) // append the remaining string contents to array result[i] = calloc(STASIS_BUFSIZ, sizeof(char)); if (!result[i]) { + guard_free(sptr); + guard_array_n_free(result, i); return NULL; } - strcpy(result[i], &orig[pos]); + strncpy(result[i], &orig[pos], STASIS_BUFSIZ - 1); + result[i][STASIS_BUFSIZ - 1] = '\0'; } guard_free(sptr); @@ -153,9 +175,9 @@ char *join(char **arr, const char *separator) { result = (char *)calloc(total_bytes, sizeof(char)); for (int i = 0; i < records; i++) { - strcat(result, arr[i]); + strncat(result, arr[i], total_bytes - (result ? strlen(result) - 1 : 0)); if (i < (records - 1)) { - strcat(result, separator); + strncat(result, separator, total_bytes - strlen(result) - 1); } } return result; @@ -193,7 +215,8 @@ char *join_ex(char *separator, ...) { // Initialize array argv = calloc(argc + 1, sizeof(char **)); if (argv == NULL) { - perror("join_ex calloc failed"); + SYSERROR("calloc failed: %s", strerror(errno)); + va_end(ap); return NULL; } @@ -207,11 +230,11 @@ char *join_ex(char *separator, ...) { result = calloc(size + 1, sizeof(char)); for (size_t i = 0; i < argc; i++) { // Append argument to string - strcat(result, argv[i]); + strncat(result, argv[i], size - (result ? strlen(result) - 1 : 0)); // no -1 because +1 above // Do not append a trailing separator when we reach the last argument if (i < (argc - 1)) { - strcat(result, separator); + strncat(result, separator, size - strlen(result)); // no -1 because +1 above } guard_free(argv[i]); } @@ -527,7 +550,7 @@ char *normalize_space(char *s) { } if (!(tmp = strdup(s))) { - perror("could not allocate memory for temporary string"); + SYSERROR("could not allocate memory for temporary string"); return NULL; } char *tmp_orig = tmp; @@ -562,7 +585,10 @@ char *normalize_space(char *s) { } // Rewrite the input string - strcpy(result, tmp_orig); + const size_t result_len = strlen(result) + 1; + strncpy(result, tmp_orig, result_len); + result[result_len] = '\0'; + guard_free(tmp_orig); return result; } @@ -581,8 +607,16 @@ char **strdup_array(char **array) { // Create new array result = calloc(elems + 1, sizeof(*result)); + if (!result) { + SYSERROR("could not allocate memory for result array"); + return NULL; + } for (size_t i = 0; i < elems; i++) { result[i] = strdup(array[i]); + if (!result[i]) { + guard_array_free(result); + break; + } } return result; @@ -640,12 +674,35 @@ char *tolower_s(char *s) { } char *to_short_version(const char *s) { - char *result = strdup(s); - if (!result) { - return NULL; + char *result = NULL; + if (num_chars(s, '.') > 1) { + char **version_data = split((char *) s, ".", 1); + if (!version_data) { + goto to_short_version_failed; + } + if (version_data[1]) { + char *dot = strchr(version_data[1], '.'); + if (dot) { + *dot = '\0'; + } + } + result = join(version_data, ""); + if (!result) { + guard_array_free(version_data); + goto to_short_version_failed; + } + guard_array_free(version_data); + } else { + result = strdup(s); + if (!result) { + goto to_short_version_failed; + } + strchrdel(result, "."); } - strchrdel(result, "."); + return result; + to_short_version_failed: + return NULL; } void unindent(char *s) { diff --git a/src/lib/core/strlist.c b/src/lib/core/strlist.c index 5655da9..5cd3f4a 100644 --- a/src/lib/core/strlist.c +++ b/src/lib/core/strlist.c @@ -4,6 +4,10 @@ */ #include "download.h" #include "strlist.h" + +#include <float.h> +#include <math.h> + #include "utils.h" /** @@ -41,13 +45,12 @@ void strlist_append(struct StrList **pStrList, char *str) { tmp = realloc((*pStrList)->data, ((*pStrList)->num_alloc + 1) * sizeof(char *)); if (tmp == NULL) { guard_strlist_free(pStrList); - perror("failed to append to array"); + SYSERROR("failed to append to array: %s", strerror(errno)); exit(1); } (*pStrList)->data = tmp; (*pStrList)->data[(*pStrList)->num_inuse] = strdup(str); (*pStrList)->data[(*pStrList)->num_alloc] = NULL; - strcpy((*pStrList)->data[(*pStrList)->num_inuse], str); (*pStrList)->num_inuse++; (*pStrList)->num_alloc++; } @@ -85,15 +88,20 @@ int strlist_append_file(struct StrList *pStrList, char *_path, ReaderFn *readerF if (is_url) { int fd; char tempfile[PATH_MAX] = {0}; - strcpy(tempfile, "/tmp/.remote_file.XXXXXX"); + strncpy(tempfile, "/tmp/.remote_file.XXXXXX", sizeof(tempfile) - 1); + tempfile[sizeof(tempfile) - 1] = '\0'; + if ((fd = mkstemp(tempfile)) < 0) { retval = -1; goto fatal; } close(fd); filename = strdup(tempfile); - long http_code = download(path, filename, NULL); + char *errmsg = NULL; + long http_code = download(path, filename, &errmsg); if (HTTP_ERROR(http_code)) { + SYSERROR("%s: %s", errmsg, filename); + guard_free(errmsg); retval = -1; goto fatal; } @@ -163,6 +171,10 @@ int strlist_contains(struct StrList *pStrList, const char *value, size_t *index_ for (size_t i = 0; i < strlist_count(pStrList); i++) { const char *item = strlist_item(pStrList, i); + if (!item) { + *index_of = 0; + break; + } if (!strcmp(item, value)) { *index_of = i; return 1; @@ -210,22 +222,84 @@ void strlist_append_strlist(struct StrList *pStrList1, struct StrList *pStrList2 * @param str * @param delim */ - void strlist_append_tokenize(struct StrList *pStrList, char *str, char *delim) { - if (!str || !delim) { - return; - } +int strlist_append_tokenize(struct StrList *pStrList, char *str, char *delim) { + if (!str || !delim) { + return -1; + } - char *tmp = strdup(str); - char **token = split(tmp, delim, 0); - if (token) { - for (size_t i = 0; token[i] != NULL; i++) { - lstrip(token[i]); - strlist_append(&pStrList, token[i]); - } - guard_array_free(token); - } + char *tmp = strdup(str); + if (!tmp) { + return -1; + } + + char **token = split(tmp, delim, 0); + if (!token) { + guard_free(tmp); + return -1; + } + for (size_t i = 0; token[i] != NULL; i++) { + lstrip(token[i]); + strlist_append(&pStrList, token[i]); + } + guard_array_free(token); guard_free(tmp); - } + + return 0; +} + +/** + * Append the contents of a newline delimited string without + * modifying the input `str` + * @param pStrList `StrList` + * @param str + * @param delim + */ +int strlist_append_tokenize_raw(struct StrList *pStrList, char *str, char *delim) { + if (!str || !delim) { + return -1; + } + + char *tmp = strdup(str); + if (!tmp) { + return -1; + } + + char **token = split(tmp, delim, 0); + if (!token) { + guard_free(tmp); + return -1; + } + + for (size_t i = 0; token[i] != NULL; i++) { + strlist_append(&pStrList, token[i]); + } + + guard_array_free(token); + guard_free(tmp); + return 0; +} + +/** + * Append a formatted string + * Behavior is identical to asprintf-family of functions + * @param pStrList `StrList` + * @param fmt printf format string + * @param ... format arguments + * @return same as vasnprintf + */ +int strlist_appendf(struct StrList **pStrList, const char *fmt, ...) { + char *s = NULL; + va_list ap; + va_start(ap, fmt); + const int len = vasprintf(&s, fmt, ap); + va_end(ap); + + if (pStrList && *pStrList && len >= 0) { + strlist_append(pStrList, s); + } + guard_free(s); + return len; +} /** * Produce a new copy of a `StrList` @@ -289,9 +363,10 @@ int strlist_cmp(struct StrList *a, struct StrList *b) { return 1; } - for (size_t i = 0; i < strlist_count(a); i++) { - if (strcmp(strlist_item(a, i), strlist_item(b, i)) != 0) { + const char *item_a = strlist_item(a, i); + const char *item_b = strlist_item(b, i); + if ((!item_a || !item_b) || strcmp(item_a, item_b) != 0) { return 1; } } @@ -367,14 +442,15 @@ void strlist_set(struct StrList **pStrList, size_t index, char *value) { } else { tmp = realloc((*pStrList)->data[index], (strlen(value) + 1) * sizeof(char *)); if (!tmp) { - perror("realloc strlist_set replacement value"); + SYSERROR("strlist_set replacement realloc failed: %s", strerror(errno)); return; - } else if (tmp != (*pStrList)->data[index]) { - (*pStrList)->data[index] = tmp; } + (*pStrList)->data[index] = tmp; - memset((*pStrList)->data[index], '\0', strlen(value) + 1); - strcpy((*pStrList)->data[index], value); + const size_t len = strlen(value) + 1; + memset((*pStrList)->data[index], '\0', len); + strncpy((*pStrList)->data[index], value, len); + (*pStrList)->data[index][len] = '\0'; } } @@ -434,7 +510,13 @@ char strlist_item_as_char(struct StrList *pStrList, size_t index) { char *error_p; strlist_clear_error(); - char result = (char) strtol(strlist_item(pStrList, index), &error_p, 10); + char *item = strlist_item(pStrList, index); + if (!item) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return CHAR_MAX; + } + + char result = (char) strtol(item, &error_p, 10); if (!result && error_p && *error_p != 0) { strlist_set_error(STRLIST_E_INVALID_VALUE); return 0; @@ -453,7 +535,13 @@ unsigned char strlist_item_as_uchar(struct StrList *pStrList, size_t index) { char *error_p; strlist_clear_error(); - unsigned char result = (unsigned char) strtoul(strlist_item(pStrList, index), &error_p, 10); + char *item = strlist_item(pStrList, index); + if (!item) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return UCHAR_MAX; + } + + unsigned char result = (unsigned char) strtoul(item, &error_p, 10); if (!result && error_p && *error_p != 0) { strlist_set_error(STRLIST_E_INVALID_VALUE); return 0; @@ -472,7 +560,12 @@ short strlist_item_as_short(struct StrList *pStrList, size_t index) { char *error_p; strlist_clear_error(); - short result = (short) strtol(strlist_item(pStrList, index), &error_p, 10); + char *item = strlist_item(pStrList, index); + if (!item) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return SHRT_MAX; + } + short result = (short) strtol(item, &error_p, 10); if (!result && error_p && *error_p != 0) { strlist_set_error(STRLIST_E_INVALID_VALUE); return 0; @@ -491,7 +584,13 @@ unsigned short strlist_item_as_ushort(struct StrList *pStrList, size_t index) { char *error_p; strlist_clear_error(); - unsigned short result = (unsigned short) strtoul(strlist_item(pStrList, index), &error_p, 10); + char *item = strlist_item(pStrList, index); + if (!item) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return USHRT_MAX; + } + + unsigned short result = (unsigned short) strtoul(item, &error_p, 10); if (!result && error_p && *error_p != 0) { strlist_set_error(STRLIST_E_INVALID_VALUE); return 0; @@ -510,7 +609,13 @@ int strlist_item_as_int(struct StrList *pStrList, size_t index) { char *error_p; strlist_clear_error(); - int result = (int) strtol(strlist_item(pStrList, index), &error_p, 10); + char *item = strlist_item(pStrList, index); + if (!item) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return INT_MAX; + } + + int result = (int) strtol(item, &error_p, 10); if (!result && error_p && *error_p != 0) { strlist_set_error(STRLIST_E_INVALID_VALUE); return 0; @@ -529,7 +634,13 @@ unsigned int strlist_item_as_uint(struct StrList *pStrList, size_t index) { char *error_p; strlist_clear_error(); - unsigned int result = (unsigned int) strtoul(strlist_item(pStrList, index), &error_p, 10); + char *item = strlist_item(pStrList, index); + if (!item) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return UINT_MAX; + } + + unsigned int result = (unsigned int) strtoul(item, &error_p, 10); if (!result && error_p && *error_p != 0) { strlist_set_error(STRLIST_E_INVALID_VALUE); return 0; @@ -548,7 +659,13 @@ long strlist_item_as_long(struct StrList *pStrList, size_t index) { char *error_p; strlist_clear_error(); - long result = (long) strtol(strlist_item(pStrList, index), &error_p, 10); + char *item = strlist_item(pStrList, index); + if (!item) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return LONG_MAX; + } + + long result = (long) strtol(item, &error_p, 10); if (!result && error_p && *error_p != 0) { strlist_set_error(STRLIST_E_INVALID_VALUE); return 0; @@ -567,7 +684,13 @@ unsigned long strlist_item_as_ulong(struct StrList *pStrList, size_t index) { char *error_p; strlist_clear_error(); - unsigned long result = (unsigned long) strtoul(strlist_item(pStrList, index), &error_p, 10); + char *item = strlist_item(pStrList, index); + if (!item) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return ULONG_MAX; + } + + unsigned long result = (unsigned long) strtoul(item, &error_p, 10); if (!result && error_p && *error_p != 0) { strlist_set_error(STRLIST_E_INVALID_VALUE); return 0; @@ -586,7 +709,13 @@ long long strlist_item_as_long_long(struct StrList *pStrList, size_t index) { char *error_p; strlist_clear_error(); - long long result = (long long) strtoll(strlist_item(pStrList, index), &error_p, 10); + char *item = strlist_item(pStrList, index); + if (!item) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return LONG_LONG_MAX; + } + + long long result = (long long) strtoll(item, &error_p, 10); if (!result && error_p && *error_p != 0) { strlist_set_error(STRLIST_E_INVALID_VALUE); return 0; @@ -605,7 +734,13 @@ unsigned long long strlist_item_as_ulong_long(struct StrList *pStrList, size_t i char *error_p; strlist_clear_error(); - unsigned long long result = (unsigned long long) strtol(strlist_item(pStrList, index), &error_p, 10); + char *item = strlist_item(pStrList, index); + if (!item) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return ULONG_LONG_MAX; + } + + unsigned long long result = (unsigned long long) strtol(item, &error_p, 10); if (!result && error_p && *error_p != 0) { strlist_set_error(STRLIST_E_INVALID_VALUE); return 0; @@ -624,8 +759,14 @@ float strlist_item_as_float(struct StrList *pStrList, size_t index) { char *error_p; strlist_clear_error(); - float result = (float) strtof(strlist_item(pStrList, index), &error_p); - if (!result && error_p && *error_p != 0) { + char *item = strlist_item(pStrList, index); + if (!item) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return HUGE_VALF; + } + + const float result = strtof(item, &error_p); + if ((result == FLT_MIN || result == HUGE_VALF) && errno == ERANGE) { strlist_set_error(STRLIST_E_INVALID_VALUE); return 0; } @@ -643,8 +784,14 @@ double strlist_item_as_double(struct StrList *pStrList, size_t index) { char *error_p; strlist_clear_error(); - double result = (double) strtod(strlist_item(pStrList, index), &error_p); - if (!result && error_p && *error_p != 0) { + char *item = strlist_item(pStrList, index); + if (!item) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return HUGE_VAL; + } + + const double result = strtod(item, &error_p); + if ((result == DBL_MIN || result == HUGE_VAL) && errno == ERANGE) { strlist_set_error(STRLIST_E_INVALID_VALUE); return 0; } @@ -662,8 +809,14 @@ long double strlist_item_as_long_double(struct StrList *pStrList, size_t index) char *error_p; strlist_clear_error(); - long double result = (long double) strtold(strlist_item(pStrList, index), &error_p); - if (!result && error_p && *error_p != 0) { + char *item = strlist_item(pStrList, index); + if (!item) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return LDBL_MAX; + } + + const long double result = strtold(item, &error_p); + if ((result == DBL_MIN || result == HUGE_VALL) && errno == ERANGE) { strlist_set_error(STRLIST_E_INVALID_VALUE); return 0; } @@ -678,7 +831,7 @@ long double strlist_item_as_long_double(struct StrList *pStrList, size_t index) struct StrList *strlist_init() { struct StrList *pStrList = calloc(1, sizeof(struct StrList)); if (pStrList == NULL) { - perror("failed to allocate array"); + SYSERROR("failed to allocate array: %s", strerror(errno)); return NULL; } pStrList->num_inuse = 0; diff --git a/src/lib/core/system.c b/src/lib/core/system.c index 9eff64a..d0956f4 100644 --- a/src/lib/core/system.c +++ b/src/lib/core/system.c @@ -21,7 +21,12 @@ int shell(struct Process *proc, char *args) { FILE *tp = NULL; char *t_name = xmkstemp(&tp, "w"); - if (!t_name || !tp) { + if (!t_name) { + fclose(tp); + return -1; + } + if (!tp) { + guard_free(t_name); return -1; } @@ -36,7 +41,9 @@ int shell(struct Process *proc, char *args) { pid_t pid = fork(); if (pid == -1) { - fprintf(stderr, "fork failed\n"); + SYSERROR("fork failed"); + guard_free(t_name); + fclose(tp); exit(1); } else if (pid == 0) { FILE *fp_out = NULL; @@ -45,7 +52,7 @@ int shell(struct Process *proc, char *args) { if (strlen(proc->f_stdout)) { fp_out = freopen(proc->f_stdout, "w+", stdout); if (!fp_out) { - fprintf(stderr, "Unable to redirect stdout to %s: %s\n", proc->f_stdout, strerror(errno)); + SYSERROR("Unable to redirect stdout to %s: %s", proc->f_stdout, strerror(errno)); exit(1); } } @@ -54,7 +61,10 @@ int shell(struct Process *proc, char *args) { if (!proc->redirect_stderr) { fp_err = freopen(proc->f_stderr, "w+", stderr); if (!fp_err) { - fprintf(stderr, "Unable to redirect stderr to %s: %s\n", proc->f_stdout, strerror(errno)); + SYSERROR("Unable to redirect stderr to %s: %s", proc->f_stdout, strerror(errno)); + if (fp_out) { + fclose(fp_out); + } exit(1); } } @@ -62,12 +72,23 @@ int shell(struct Process *proc, char *args) { if (proc->redirect_stderr) { if (fp_err) { - fclose(fp_err); - fclose(stderr); - } - if (dup2(fileno(stdout), fileno(stderr)) < 0) { - fprintf(stderr, "Unable to redirect stderr to stdout: %s\n", strerror(errno)); - exit(1); + if (dup2(fileno(fp_err), STDERR_FILENO) < 0) { + SYSERROR("Unable to redirect stderr to %s: %s", proc->f_stderr, strerror(errno)); + if (fp_out) { + fclose(fp_out); + } + fclose(fp_err); + _exit(1); + } + } else { + // redirect stderr to stdout + if (dup2(STDOUT_FILENO, STDERR_FILENO) < 0) { + SYSERROR("Unable to redirect stderr to stdout: %s", strerror(errno)); + if (fp_out) { + fclose(fp_out); + } + _exit(1); + } } } @@ -76,13 +97,13 @@ int shell(struct Process *proc, char *args) { if (waitpid(pid, &status, WUNTRACED) > 0) { if (WIFEXITED(status) && WEXITSTATUS(status)) { if (WEXITSTATUS(status) == 127) { - fprintf(stderr, "execv failed\n"); + SYSERROR("execv failed"); } } else if (WIFSIGNALED(status)) { - fprintf(stderr, "signal received: %d\n", WIFSIGNALED(status)); + SYSWARN("signal received: %d", WIFSIGNALED(status)); } } else { - fprintf(stderr, "waitpid() failed\n"); + SYSERROR("waitpid() failed"); } } @@ -156,12 +177,12 @@ char *shell_output(const char *command, int *status) { current_size += initial_size; char *tmp = realloc(result, sizeof(*result) * current_size); if (!tmp) { + guard_free(result); return NULL; - } else if (tmp != result) { - result = tmp; } + result = tmp; } - strcat(result, line); + strncat(result, line, current_size - strlen(result) - 1); memset(line, 0, sizeof(line)); } *status = pclose(pp); diff --git a/src/lib/core/template.c b/src/lib/core/template.c index ba45a5a..8396b1e 100644 --- a/src/lib/core/template.c +++ b/src/lib/core/template.c @@ -20,15 +20,24 @@ struct tplfunc_frame *tpl_pool_func[1024] = {0}; unsigned tpl_pool_func_used = 0; extern void tpl_reset() { - SYSDEBUG("%s", "Resetting template engine"); + SYSDEBUG("Resetting template engine"); tpl_free(); tpl_pool_used = 0; tpl_pool_func_used = 0; } -void tpl_register_func(char *key, void *tplfunc_ptr, int argc, void *data_in) { +void tpl_register_func(char *key, tplfunc *tplfunc_ptr, int argc, void *data_in) { struct tplfunc_frame *frame = calloc(1, sizeof(*frame)); + if (!frame) { + SYSERROR("unable to allocate memory for function frame"); + exit(1); + } + frame->key = strdup(key); + if (!frame->key) { + SYSERROR("unable to allocate memory for function frame key"); + exit(1); + } frame->argc = argc; frame->func = tplfunc_ptr; frame->data_in = data_in; @@ -43,12 +52,12 @@ int tpl_key_exists(char *key) { for (size_t i = 0; i < tpl_pool_used; i++) { if (tpl_pool[i]->key) { if (!strcmp(tpl_pool[i]->key, key)) { - SYSDEBUG("%s", "YES"); + SYSDEBUG("YES"); return true; } } } - SYSDEBUG("%s", "NO"); + SYSDEBUG("NO"); return false; } @@ -67,11 +76,19 @@ void tpl_register(char *key, char **ptr) { } } replacing = 1; - SYSDEBUG("%s", "Item will be replaced"); + SYSDEBUG("Item will be replaced"); } else { - SYSDEBUG("%s", "Creating new item"); + SYSDEBUG("Creating new item"); item = calloc(1, sizeof(*item)); + if (!item) { + SYSERROR("unable to allocate memory for new item"); + exit(1); + } item->key = strdup(key); + if (!key) { + SYSERROR("unable to allocate memory for new key"); + exit(1); + } } if (!item) { @@ -81,7 +98,7 @@ void tpl_register(char *key, char **ptr) { item->ptr = ptr; if (!replacing) { - SYSDEBUG("Registered tpl_item at index %u:\n\tkey=%s\n\tptr=%s", tpl_pool_used, item->key, *item->ptr); + SYSDEBUG("Registered tpl_item at index %u:\n\tkey=%s\n\tptr=%s", tpl_pool_used, item->key, *item->ptr ? *item->ptr : "NULL"); tpl_pool[tpl_pool_used] = item; tpl_pool_used++; } @@ -137,23 +154,6 @@ struct tplfunc_frame *tpl_getfunc(char *key) { return result; } -static int grow(size_t z, size_t *output_bytes, char **output) { - if (z >= *output_bytes) { - size_t new_size = *output_bytes + z + 1; - SYSDEBUG("template output buffer new size: %zu\n", new_size); - - char *tmp = realloc(*output, new_size); - if (!tmp) { - perror("realloc failed"); - return -1; - } else if (tmp != *output) { - *output = tmp; - } - *output_bytes = new_size; - } - return 0; -} - char *tpl_render(char *str) { if (!str) { return NULL; @@ -168,7 +168,7 @@ char *tpl_render(char *str) { output = calloc(output_bytes, sizeof(*output)); if (!output) { - perror("unable to allocate output buffer"); + SYSERROR("unable to allocate output buffer: %s", strerror(errno)); return NULL; } @@ -186,7 +186,7 @@ char *tpl_render(char *str) { } // Read key name - SYSDEBUG("%s", "Reading key"); + SYSDEBUG("Reading key"); size_t key_len = 0; while (isalnum(pos[off]) || pos[off] != '}') { if (isspace(pos[off]) || isblank(pos[off])) { @@ -207,10 +207,10 @@ char *tpl_render(char *str) { int do_func = 0; if (type_stop) { if (!strncmp(key, "env", type_stop - key)) { - SYSDEBUG("%s", "Will render as value of environment variable"); + SYSDEBUG("Will render as value of environment variable"); do_env = 1; } else if (!strncmp(key, "func", type_stop - key)) { - SYSDEBUG("%s", "Will render as output from function"); + SYSDEBUG("Will render as output from function"); do_func = 1; } } @@ -218,7 +218,7 @@ char *tpl_render(char *str) { // Find closing brace b_close = strstr(&pos[off], "}}"); if (!b_close) { - fprintf(stderr, "error while templating '%s'\n\nunbalanced brace at position %zu\n", str, z); + SYSERROR("while templating '%s'\n\nunbalanced brace at position %zu", str, z); guard_free(output); return NULL; } else { @@ -235,10 +235,12 @@ char *tpl_render(char *str) { value = strdup(env_val ? env_val : ""); } else if (do_func) { // {{ func:NAME(a, ...) }} char func_name_temp[STASIS_NAME_MAX] = {0}; - strcpy(func_name_temp, type_stop + 1); + strncpy(func_name_temp, type_stop + 1, sizeof(func_name_temp) - 1); + func_name_temp[sizeof(func_name_temp) - 1] = '\0'; + char *param_begin = strchr(func_name_temp, '('); if (!param_begin) { - fprintf(stderr, "At position %zu in %s\nfunction name must be followed by a '('\n", off, key); + SYSERROR("At position %zu in %s\nfunction name must be followed by a '('", off, key); guard_free(output); return NULL; } @@ -246,7 +248,7 @@ char *tpl_render(char *str) { param_begin++; char *param_end = strrchr(param_begin, ')'); if (!param_end) { - fprintf(stderr, "At position %zu in %s\nfunction arguments must be closed with a ')'\n", off, key); + SYSERROR("At position %zu in %s\nfunction arguments must be closed with a ')'", off, key); guard_free(output); return NULL; } @@ -257,8 +259,13 @@ char *tpl_render(char *str) { for (params_count = 0; params[params_count] != NULL; params_count++) {} struct tplfunc_frame *frame = tpl_getfunc(k); + if (!frame) { + SYSERROR("no function named '%s'", k); + guard_array_n_free(params, (size_t) params_count); + return NULL; + } if (params_count > frame->argc || params_count < frame->argc) { - fprintf(stderr, "At position %zu in %s\nIncorrect number of arguments for function: %s (expected %d, got %d)\n", off, key, frame->key, frame->argc, params_count); + SYSERROR("At position %zu in %s\nIncorrect number of arguments for function: %s (expected %d, got %d)", off, key, frame->key, frame->argc, params_count); value = strdup(""); } else { for (size_t p = 0; p < sizeof(frame->argv) / sizeof(*frame->argv) && params[p] != NULL; p++) { @@ -269,7 +276,7 @@ char *tpl_render(char *str) { char *func_result = NULL; int func_status = 0; if ((func_status = frame->func(frame, &func_result))) { - fprintf(stderr, "%s returned non-zero status: %d\n", frame->key, func_status); + SYSERROR("%s returned non-zero status: %d", frame->key, func_status); } value = strdup(func_result ? func_result : ""); SYSDEBUG("Returned from function: %s (status: %d)\nData OUT\n--------\n'%s'", k, func_status, value); @@ -289,7 +296,7 @@ char *tpl_render(char *str) { // Append replacement value grow(z, &output_bytes, &output); - strcat(output, value); + strncat(output, value, output_bytes - strlen(output) - 1); guard_free(value); output[z] = 0; } @@ -297,8 +304,8 @@ char *tpl_render(char *str) { output[z] = pos[off]; z++; } - SYSDEBUG("template output length: %zu", strlen(output)); - SYSDEBUG("template output bytes: %zu", output_bytes); + //SYSDEBUG("template output length: %zu", strlen(output)); + //SYSDEBUG("template output bytes: %zu", output_bytes); return output; } @@ -320,7 +327,7 @@ int tpl_render_to_file(char *str, const char *filename) { // Write rendered string to file fprintf(fp, "%s", result); fclose(fp); - SYSDEBUG("%s", "Rendered successfully"); + SYSDEBUG("Rendered successfully"); guard_free(result); return 0; diff --git a/src/lib/core/template_func_proto.c b/src/lib/core/template_func_proto.c index 8324389..54a8860 100644 --- a/src/lib/core/template_func_proto.c +++ b/src/lib/core/template_func_proto.c @@ -2,7 +2,7 @@ #include "delivery.h" #include "github.h" -int get_github_release_notes_tplfunc_entrypoint(void *frame, void *data_out) { +int get_github_release_notes_tplfunc_entrypoint(struct tplfunc_frame *frame, void *data_out) { char **output = (char **) data_out; struct tplfunc_frame *f = (struct tplfunc_frame *) frame; char *api_token = getenv("STASIS_GH_TOKEN"); @@ -17,7 +17,7 @@ int get_github_release_notes_tplfunc_entrypoint(void *frame, void *data_out) { return result; } -int get_github_release_notes_auto_tplfunc_entrypoint(void *frame, void *data_out) { +int get_github_release_notes_auto_tplfunc_entrypoint(struct tplfunc_frame *frame, void *data_out) { int result = 0; char **output = (char **) data_out; struct tplfunc_frame *f = (struct tplfunc_frame *) frame; @@ -28,9 +28,9 @@ int get_github_release_notes_auto_tplfunc_entrypoint(void *frame, void *data_out const struct Delivery *ctx = (struct Delivery *) f->data_in; struct StrList *notes_list = strlist_init(); - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(*ctx->tests); i++) { + for (size_t i = 0; i < ctx->tests->num_used; i++) { // Get test context - const struct Test *test = &ctx->tests[i]; + const struct Test *test = ctx->tests->test[i]; if (test->name && test->version && test->repository) { char *repository = strdup(test->repository); char *match = strstr(repository, "spacetelescope/"); @@ -44,7 +44,7 @@ int get_github_release_notes_auto_tplfunc_entrypoint(void *frame, void *data_out // Using HEAD, GitHub returns the previous tag char *note = NULL; char h1_title[NAME_MAX] = {0}; - sprintf(h1_title, "# %s", test->name); + snprintf(h1_title, sizeof(h1_title), "# %s", test->name); strlist_append(¬es_list, h1_title); result += get_github_release_notes(api_token ? api_token : "anonymous", repository, @@ -55,8 +55,8 @@ int get_github_release_notes_auto_tplfunc_entrypoint(void *frame, void *data_out strlist_append(¬es_list, note); guard_free(note); } - guard_free(repository); } + guard_free(repository); } } // Return all notes as a single string @@ -68,7 +68,7 @@ int get_github_release_notes_auto_tplfunc_entrypoint(void *frame, void *data_out return result; } -int get_junitxml_file_entrypoint(void *frame, void *data_out) { +int get_junitxml_file_entrypoint(struct tplfunc_frame *frame, void *data_out) { int result = 0; char **output = (char **) data_out; struct tplfunc_frame *f = (struct tplfunc_frame *) frame; @@ -80,7 +80,9 @@ int get_junitxml_file_entrypoint(void *frame, void *data_out) { return -1; } char nametmp[PATH_MAX] = {0}; - strcpy(nametmp, cwd); + strncpy(nametmp, cwd, sizeof(nametmp) - 1); + nametmp[sizeof(nametmp) - 1] = '\0'; + char *name = path_basename(nametmp); *output = calloc(PATH_MAX, sizeof(**output)); @@ -88,12 +90,12 @@ int get_junitxml_file_entrypoint(void *frame, void *data_out) { SYSERROR("failed to allocate output string: %s", strerror(errno)); return -1; } - sprintf(*output, "%s/results-%s-%s.xml", ctx->storage.results_dir, name, ctx->info.release_name); + snprintf(*output, PATH_MAX, "%s/results-%s-%s.xml", ctx->storage.results_dir, name, ctx->info.release_name); return result; } -int get_basetemp_dir_entrypoint(void *frame, void *data_out) { +int get_basetemp_dir_entrypoint(struct tplfunc_frame *frame, void *data_out) { int result = 0; char **output = (char **) data_out; struct tplfunc_frame *f = (struct tplfunc_frame *) frame; @@ -105,7 +107,8 @@ int get_basetemp_dir_entrypoint(void *frame, void *data_out) { return -1; } char nametmp[PATH_MAX] = {0}; - strcpy(nametmp, cwd); + strncpy(nametmp, cwd, sizeof(nametmp) - 1); + nametmp[sizeof(nametmp) - 1] = '\0'; char *name = path_basename(nametmp); *output = calloc(PATH_MAX, sizeof(**output)); @@ -113,12 +116,12 @@ int get_basetemp_dir_entrypoint(void *frame, void *data_out) { SYSERROR("failed to allocate output string: %s", strerror(errno)); return -1; } - sprintf(*output, "%s/truth-%s-%s", ctx->storage.tmpdir, name, ctx->info.release_name); + snprintf(*output, PATH_MAX, "%s/truth-%s-%s", ctx->storage.tmpdir, name, ctx->info.release_name); return result; } -int tox_run_entrypoint(void *frame, void *data_out) { +int tox_run_entrypoint(struct tplfunc_frame *frame, void *data_out) { char **output = (char **) data_out; struct tplfunc_frame *f = (struct tplfunc_frame *) frame; const struct Delivery *ctx = (const struct Delivery *) f->data_in; @@ -126,7 +129,7 @@ int tox_run_entrypoint(void *frame, void *data_out) { // Apply workaround for tox positional arguments char *toxconf = NULL; if (!access("tox.ini", F_OK)) { - if (!fix_tox_conf("tox.ini", &toxconf)) { + if (!fix_tox_conf("tox.ini", &toxconf, PATH_MAX)) { msg(STASIS_MSG_L3, "Fixing tox positional arguments\n"); *output = calloc(STASIS_BUFSIZ, sizeof(**output)); if (!*output) { diff --git a/src/lib/core/timespec.c b/src/lib/core/timespec.c new file mode 100644 index 0000000..bd33993 --- /dev/null +++ b/src/lib/core/timespec.c @@ -0,0 +1,979 @@ +/* Functions for working with timespec structures + * Written by Daniel Collins (2017-2021) + * timespec_mod by Alex Forencich (2019) + * Various contributions by Ingo Albrecht (2021) + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * For more information, please refer to <http://unlicense.org/> +*/ + +/** \file timespec.c + * \brief Functions for working with timespec structures. + * + * This library aims to provide a comprehensive set of functions with + * well-defined behaviour that handle all edge cases (e.g. negative values) in + * a sensible manner. + * + * Negative values are allowed in the tv_sec and/or tv_usec field of timespec + * structures, tv_usec is always relative to tv_sec, so mixing positive and + * negative values will produce consistent results: + * + * <PRE> + * { tv_sec = 1, tv_nsec = 500000000 } == 1.5 seconds + * { tv_sec = 1, tv_nsec = 0 } == 1.0 seconds + * { tv_sec = 1, tv_nsec = -500000000 } == 0.5 seconds + * { tv_sec = 0, tv_nsec = 500000000 } == 0.5 seconds + * { tv_sec = 0, tv_nsec = 0 } == 0.0 seconds + * { tv_sec = 0, tv_nsec = -500000000 } == -0.5 seconds + * { tv_sec = -1, tv_nsec = 500000000 } == -0.5 seconds + * { tv_sec = -1, tv_nsec = 0 } == -1.0 seconds + * { tv_sec = -1, tv_nsec = -500000000 } == -1.5 seconds + * </PRE> + * + * Furthermore, any timespec structure processed or returned by library functions + * is normalised according to the rules in timespec_normalise(). +*/ + +#include <limits.h> +#include <stdbool.h> +#include <sys/time.h> +#include <time.h> + +#include "timespec.h" + +#define NSEC_PER_SEC 1000000000 + +/** \fn struct timespec timespec_add(struct timespec ts1, struct timespec ts2) + * \brief Returns the result of adding two timespec structures. +*/ +struct timespec timespec_add(struct timespec ts1, struct timespec ts2) +{ + /* Normalise inputs to prevent tv_nsec rollover if whole-second values + * are packed in it. + */ + ts1 = timespec_normalise(ts1); + ts2 = timespec_normalise(ts2); + + ts1.tv_sec += ts2.tv_sec; + ts1.tv_nsec += ts2.tv_nsec; + + return timespec_normalise(ts1); +} + +/** \fn struct timespec timespec_sub(struct timespec ts1, struct timespec ts2) + * \brief Returns the result of subtracting ts2 from ts1. +*/ +struct timespec timespec_sub(struct timespec ts1, struct timespec ts2) +{ + /* Normalise inputs to prevent tv_nsec rollover if whole-second values + * are packed in it. + */ + ts1 = timespec_normalise(ts1); + ts2 = timespec_normalise(ts2); + + ts1.tv_sec -= ts2.tv_sec; + ts1.tv_nsec -= ts2.tv_nsec; + + return timespec_normalise(ts1); +} + +/** \fn struct timespec timespec_mod(struct timespec ts1, struct timespec ts2) + * \brief Returns the remainder left over after dividing ts1 by ts2 (ts1%ts2). +*/ +struct timespec timespec_mod(struct timespec ts1, struct timespec ts2) +{ + int i = 0; + bool neg1 = false; + bool neg2 = false; + + /* Normalise inputs to prevent tv_nsec rollover if whole-second values + * are packed in it. + */ + ts1 = timespec_normalise(ts1); + ts2 = timespec_normalise(ts2); + + /* If ts2 is zero, just return ts1 + */ + if (ts2.tv_sec == 0 && ts2.tv_nsec == 0) + { + return ts1; + } + + /* If inputs are negative, flip and record sign + */ + if (ts1.tv_sec < 0 || ts1.tv_nsec < 0) + { + neg1 = true; + ts1.tv_sec = -ts1.tv_sec; + ts1.tv_nsec = -ts1.tv_nsec; + } + + if (ts2.tv_sec < 0 || ts2.tv_nsec < 0) + { + neg2 = true; + ts2.tv_sec = -ts2.tv_sec; + ts2.tv_nsec = -ts2.tv_nsec; + } + + /* Shift ts2 until it is larger than ts1 or is about to overflow + */ + while ((ts2.tv_sec < (LONG_MAX >> 1)) && timespec_ge(ts1, ts2)) + { + i++; + ts2.tv_nsec <<= 1; + ts2.tv_sec <<= 1; + if (ts2.tv_nsec > NSEC_PER_SEC) + { + ts2.tv_nsec -= NSEC_PER_SEC; + ts2.tv_sec++; + } + } + + /* Division by repeated subtraction + */ + while (i >= 0) + { + if (timespec_ge(ts1, ts2)) + { + ts1 = timespec_sub(ts1, ts2); + } + + if (i == 0) + { + break; + } + + i--; + if (ts2.tv_sec & 1) + { + ts2.tv_nsec += NSEC_PER_SEC; + } + ts2.tv_nsec >>= 1; + ts2.tv_sec >>= 1; + } + + /* If signs differ and result is nonzero, subtract once more to cross zero + */ + if (neg1 ^ neg2 && (ts1.tv_sec != 0 || ts1.tv_nsec != 0)) + { + ts1 = timespec_sub(ts1, ts2); + } + + /* Restore sign + */ + if (neg1) + { + ts1.tv_sec = -ts1.tv_sec; + ts1.tv_nsec = -ts1.tv_nsec; + } + + return ts1; +} + +/** \fn struct timespec timespec_min(struct timespec ts1, struct timespec ts2) + * \brief Return the lesser one of the two given timespec values. +*/ +struct timespec timespec_min(struct timespec ts1, struct timespec ts2) { + if(timespec_le(ts1, ts2)) { + return ts1; + } else { + return ts2; + } +} + +/** \fn struct timespec timespec_max(struct timespec ts1, struct timespec ts2) + * \brief Return the greater one of the two given timespec values. +*/ +struct timespec timespec_max(struct timespec ts1, struct timespec ts2) { + if(timespec_ge(ts1, ts2)) { + return ts1; + } else { + return ts2; + } +} + +/** \fn struct timespec timespec_clamp(struct timespec ts, struct timespec min, struct timespec max) + * \brief Clamp the value of TS between MIN and MAX. +*/ +struct timespec timespec_clamp(struct timespec ts, struct timespec min, struct timespec max) { + if(timespec_gt(ts, max)) { + return max; + } + if(timespec_lt(ts, min)) { + return min; + } + return ts; +} + +/** \fn int timespec_cmp(struct timespec ts1, struct timespec ts2) + * \brief Returns (1, 0, -1) if ts1 is (greater than, equal to, less than) to ts2. +*/ +int timespec_cmp(struct timespec ts1, struct timespec ts2) +{ + ts1 = timespec_normalise(ts1); + ts2 = timespec_normalise(ts2); + + if(ts1.tv_sec == ts2.tv_sec && ts1.tv_nsec == ts2.tv_nsec) + { + return 0; + } + else if((ts1.tv_sec > ts2.tv_sec) + || (ts1.tv_sec == ts2.tv_sec && ts1.tv_nsec > ts2.tv_nsec)) + { + return 1; + } + else { + return -1; + } +} + +/** \fn bool timespec_eq(struct timespec ts1, struct timespec ts2) + * \brief Returns true if the two timespec structures are equal. +*/ +bool timespec_eq(struct timespec ts1, struct timespec ts2) +{ + ts1 = timespec_normalise(ts1); + ts2 = timespec_normalise(ts2); + + return (ts1.tv_sec == ts2.tv_sec && ts1.tv_nsec == ts2.tv_nsec); +} + +/** \fn bool timespec_gt(struct timespec ts1, struct timespec ts2) + * \brief Returns true if ts1 is greater than ts2. +*/ +bool timespec_gt(struct timespec ts1, struct timespec ts2) +{ + ts1 = timespec_normalise(ts1); + ts2 = timespec_normalise(ts2); + + return (ts1.tv_sec > ts2.tv_sec || (ts1.tv_sec == ts2.tv_sec && ts1.tv_nsec > ts2.tv_nsec)); +} + +/** \fn bool timespec_ge(struct timespec ts1, struct timespec ts2) + * \brief Returns true if ts1 is greater than or equal to ts2. +*/ +bool timespec_ge(struct timespec ts1, struct timespec ts2) +{ + ts1 = timespec_normalise(ts1); + ts2 = timespec_normalise(ts2); + + return (ts1.tv_sec > ts2.tv_sec || (ts1.tv_sec == ts2.tv_sec && ts1.tv_nsec >= ts2.tv_nsec)); +} + +/** \fn bool timespec_lt(struct timespec ts1, struct timespec ts2) + * \brief Returns true if ts1 is less than ts2. +*/ +bool timespec_lt(struct timespec ts1, struct timespec ts2) +{ + ts1 = timespec_normalise(ts1); + ts2 = timespec_normalise(ts2); + + return (ts1.tv_sec < ts2.tv_sec || (ts1.tv_sec == ts2.tv_sec && ts1.tv_nsec < ts2.tv_nsec)); +} + +/** \fn bool timespec_le(struct timespec ts1, struct timespec ts2) + * \brief Returns true if ts1 is less than or equal to ts2. +*/ +bool timespec_le(struct timespec ts1, struct timespec ts2) +{ + ts1 = timespec_normalise(ts1); + ts2 = timespec_normalise(ts2); + + return (ts1.tv_sec < ts2.tv_sec || (ts1.tv_sec == ts2.tv_sec && ts1.tv_nsec <= ts2.tv_nsec)); +} + +/** \fn struct timespec timespec_from_double(double s) + * \brief Converts a fractional number of seconds to a timespec. +*/ +struct timespec timespec_from_double(double s) +{ + struct timespec ts = { + .tv_sec = s, + .tv_nsec = (s - (long)(s)) * NSEC_PER_SEC, + }; + + return timespec_normalise(ts); +} + +/** \fn double timespec_to_double(struct timespec ts) + * \brief Converts a timespec to a fractional number of seconds. +*/ +double timespec_to_double(struct timespec ts) +{ + return ((double)(ts.tv_sec) + ((double)(ts.tv_nsec) / NSEC_PER_SEC)); +} + +/** \fn struct timespec timespec_from_timeval(struct timeval tv) + * \brief Converts a timeval to a timespec. +*/ +struct timespec timespec_from_timeval(struct timeval tv) +{ + struct timespec ts = { + .tv_sec = tv.tv_sec, + .tv_nsec = tv.tv_usec * 1000 + }; + + return timespec_normalise(ts); +} + +/** \fn struct timeval timespec_to_timeval(struct timespec ts) + * \brief Converts a timespec to a timeval. +*/ +struct timeval timespec_to_timeval(struct timespec ts) +{ + ts = timespec_normalise(ts); + + struct timeval tv = { + .tv_sec = ts.tv_sec, + .tv_usec = ts.tv_nsec / 1000, + }; + + return tv; +} + +/** \fn struct timespec timespec_from_ms(long milliseconds) + * \brief Converts an integer number of milliseconds to a timespec. +*/ +struct timespec timespec_from_ms(long milliseconds) +{ + struct timespec ts = { + .tv_sec = (milliseconds / 1000), + .tv_nsec = (milliseconds % 1000) * 1000000, + }; + + return timespec_normalise(ts); +} + +/** \fn long timespec_to_ms(struct timespec ts) + * \brief Converts a timespec to an integer number of milliseconds. +*/ +long timespec_to_ms(struct timespec ts) +{ + return (ts.tv_sec * 1000) + (ts.tv_nsec / 1000000); +} + +/** \fn struct timespec timespec_normalise(struct timespec ts) + * \brief Normalises a timespec structure. + * + * Returns a normalised version of a timespec structure, according to the + * following rules: + * + * 1) If tv_nsec is >=1,000,000,00 or <=-1,000,000,000, flatten the surplus + * nanoseconds into the tv_sec field. + * + * 2) If tv_nsec is negative, decrement tv_sec and roll tv_nsec up to represent + * the same value attainable by ADDING nanoseconds to tv_sec. +*/ +struct timespec timespec_normalise(struct timespec ts) +{ + while(ts.tv_nsec >= NSEC_PER_SEC) + { + ++(ts.tv_sec); + ts.tv_nsec -= NSEC_PER_SEC; + } + + while(ts.tv_nsec <= -NSEC_PER_SEC) + { + --(ts.tv_sec); + ts.tv_nsec += NSEC_PER_SEC; + } + + if(ts.tv_nsec < 0) + { + /* Negative nanoseconds isn't valid according to POSIX. + * Decrement tv_sec and roll tv_nsec over. + */ + + --(ts.tv_sec); + ts.tv_nsec = (NSEC_PER_SEC + ts.tv_nsec); + } + + return ts; +} + +#ifdef TEST +#include <stdio.h> + +#define TEST_NORMALISE(ts_sec, ts_nsec, expect_sec, expect_nsec) { \ + struct timespec in = { .tv_sec = ts_sec, .tv_nsec = ts_nsec }; \ + struct timespec got = timespec_normalise(in); \ + if(got.tv_sec != expect_sec || got.tv_nsec != expect_nsec) \ + { \ + printf("%s:%d: timespec_normalise({%ld, %ld}) returned wrong values\n", __FILE__, __LINE__, \ + (long)(ts_sec), (long)(ts_nsec)); \ + printf(" Expected: {%ld, %ld}\n", (long)(expect_sec), (long)(expect_nsec)); \ + printf(" Got: {%ld, %ld}\n", (long)(got.tv_sec), (long)(got.tv_nsec)); \ + ++result; \ + } \ +} + +#define TEST_BINOP(func, ts1_sec, ts1_nsec, ts2_sec, ts2_nsec, expect_sec, expect_nsec) { \ + struct timespec ts1 = { .tv_sec = ts1_sec, .tv_nsec = ts1_nsec }; \ + struct timespec ts2 = { .tv_sec = ts2_sec, .tv_nsec = ts2_nsec }; \ + struct timespec got = func(ts1, ts2); \ + if(got.tv_sec != expect_sec || got.tv_nsec != expect_nsec) \ + { \ + printf(#func "({%ld, %ld}, {%ld, %ld}) returned wrong values\n", \ + (long)(ts1_sec), (long)(ts1_nsec), (long)(ts2_sec), (long)(ts2_nsec)); \ + printf(" Expected: {%ld, %ld}\n", (long)(expect_sec), (long)(expect_nsec)); \ + printf(" Got: {%ld, %ld}\n", (long)(got.tv_sec), (long)(got.tv_nsec)); \ + ++result; \ + } \ +} + +#define TEST_TRINOP(func, ts1_sec, ts1_nsec, ts2_sec, ts2_nsec, ts3_sec, ts3_nsec, expect_sec, expect_nsec) { \ + struct timespec ts1 = { .tv_sec = ts1_sec, .tv_nsec = ts1_nsec }; \ + struct timespec ts2 = { .tv_sec = ts2_sec, .tv_nsec = ts2_nsec }; \ + struct timespec ts3 = { .tv_sec = ts3_sec, .tv_nsec = ts3_nsec }; \ + struct timespec got = func(ts1, ts2, ts3); \ + if(got.tv_sec != expect_sec || got.tv_nsec != expect_nsec) \ + { \ + printf(#func "({%ld, %ld}, {%ld, %ld}, {%ld, %ld}) returned wrong values\n", \ + (long)(ts1_sec), (long)(ts1_nsec), \ + (long)(ts2_sec), (long)(ts2_nsec), \ + (long)(ts3_sec), (long)(ts3_nsec)); \ + printf(" Expected: {%ld, %ld}\n", (long)(expect_sec), (long)(expect_nsec)); \ + printf(" Got: {%ld, %ld}\n", (long)(got.tv_sec), (long)(got.tv_nsec)); \ + ++result; \ + } \ +} + +#define TEST_TEST_FUNC(func, ts1_sec, ts1_nsec, ts2_sec, ts2_nsec, expect) { \ + struct timespec ts1 = { .tv_sec = ts1_sec, .tv_nsec = ts1_nsec }; \ + struct timespec ts2 = { .tv_sec = ts2_sec, .tv_nsec = ts2_nsec }; \ + int got = func(ts1, ts2); \ + if(got != expect) \ + { \ + printf("%s:%d: " #func "({%ld, %ld}, {%ld, %ld}) returned %d, expected %s\n", __FILE__, __LINE__, \ + (long)(ts1_sec), (long)(ts1_nsec), (long)(ts2_sec), (long)(ts2_nsec), \ + got, #expect); \ + ++result; \ + } \ +} + +#define TEST_FROM_DOUBLE(d_secs, expect_sec, expect_nsec) { \ + struct timespec got = timespec_from_double(d_secs); \ + if(got.tv_sec != expect_sec || got.tv_nsec != expect_nsec) \ + { \ + printf("%s:%d: timespec_from_double(%f) returned wrong values\n", __FILE__, __LINE__, (double)(d_secs)); \ + printf(" Expected: {%ld, %ld}\n", (long)(expect_sec), (long)(expect_nsec)); \ + printf(" Got: {%ld, %ld}\n", (long)(got.tv_sec), (long)(got.tv_nsec)); \ + ++result; \ + } \ +} + +#define TEST_TO_DOUBLE(ts_sec, ts_nsec, expect) { \ + struct timespec ts = { .tv_sec = ts_sec, .tv_nsec = ts_nsec }; \ + double got = timespec_to_double(ts); \ + if(got != expect) { \ + printf("%s:%d: timespec_to_double({%ld, %ld}) returned wrong value\n", __FILE__, __LINE__, \ + (long)(ts_sec), (long)(ts_nsec)); \ + printf(" Expected: %f\n", (double)(expect)); \ + printf(" Got: %f\n", got); \ + ++result; \ + } \ +} + +#define TEST_FROM_TIMEVAL(in_sec, in_usec, expect_sec, expect_nsec) { \ + struct timeval tv = { .tv_sec = in_sec, .tv_usec = in_usec }; \ + struct timespec got = timespec_from_timeval(tv); \ + if(got.tv_sec != expect_sec || got.tv_nsec != expect_nsec) \ + { \ + printf("%s:%d: timespec_from_timeval({%ld, %ld}) returned wrong values\n", __FILE__, __LINE__, \ + (long)(in_sec), (long)(in_usec)); \ + printf(" Expected: {%ld, %ld}\n", (long)(expect_sec), (long)(expect_nsec)); \ + printf(" Got: {%ld, %ld}\n", (long)(got.tv_sec), (long)(got.tv_nsec)); \ + ++result; \ + } \ +} + +#define TEST_TO_TIMEVAL(ts_sec, ts_nsec, expect_sec, expect_usec) { \ + struct timespec ts = { .tv_sec = ts_sec, .tv_nsec = ts_nsec }; \ + struct timeval got = timespec_to_timeval(ts); \ + if(got.tv_sec != expect_sec || got.tv_usec != expect_usec) \ + { \ + printf("%s:%d: timespec_to_timeval({%ld, %ld}) returned wrong values\n", __FILE__, __LINE__, \ + (long)(ts_sec), (long)(ts_nsec)); \ + printf(" Expected: {%ld, %ld}\n", (long)(expect_sec), (long)(expect_usec)); \ + printf(" Got: {%ld, %ld}\n", (long)(got.tv_sec), (long)(got.tv_usec)); \ + ++result; \ + } \ +} + +#define TEST_FROM_MS(msecs, expect_sec, expect_nsec) { \ + struct timespec got = timespec_from_ms(msecs); \ + if(got.tv_sec != expect_sec || got.tv_nsec != expect_nsec) \ + { \ + printf("%s:%d: timespec_from_ms(%ld) returned wrong values\n", __FILE__, __LINE__, (long)(msecs)); \ + printf(" Expected: {%ld, %ld}\n", (long)(expect_sec), (long)(expect_nsec)); \ + printf(" Got: {%ld, %ld}\n", (long)(got.tv_sec), (long)(got.tv_nsec)); \ + ++result; \ + } \ +} + +#define TEST_TO_MS(ts_sec, ts_nsec, expect) { \ + struct timespec ts = { .tv_sec = ts_sec, .tv_nsec = ts_nsec }; \ + long got = timespec_to_ms(ts); \ + if(got != expect) { \ + printf("%s:%d: timespec_to_ms({%ld, %ld}) returned wrong value\n", __FILE__, __LINE__, \ + (long)(ts_sec), (long)(ts_nsec)); \ + printf(" Expected: %ld\n", (long)(expect)); \ + printf(" Got: %ld\n", got); \ + ++result; \ + } \ +} + +int main() +{ + int result = 0; + + // timespec_add + + TEST_BINOP(timespec_add, 0,0, 0,0, 0,0); + TEST_BINOP(timespec_add, 0,0, 1,0, 1,0); + TEST_BINOP(timespec_add, 1,0, 0,0, 1,0); + TEST_BINOP(timespec_add, 1,0, 1,0, 2,0); + TEST_BINOP(timespec_add, 1,500000000, 1,0, 2,500000000); + TEST_BINOP(timespec_add, 1,0, 1,500000000, 2,500000000); + TEST_BINOP(timespec_add, 1,500000000, 1,500000000, 3,0); + TEST_BINOP(timespec_add, 1,500000000, 1,499999999, 2,999999999); + TEST_BINOP(timespec_add, 1,500000000, 1,500000000, 3,0); + TEST_BINOP(timespec_add, 1,999999999, 1,999999999, 3,999999998); + TEST_BINOP(timespec_add, 0,500000000, 1,500000000, 2,0); + TEST_BINOP(timespec_add, 1,500000000, 0,500000000, 2,0); + + // timespec_sub + + TEST_BINOP(timespec_sub, 0,0, 0,0, 0,0); + TEST_BINOP(timespec_sub, 1,0, 0,0, 1,0); + TEST_BINOP(timespec_sub, 1,0, 1,0, 0,0); + TEST_BINOP(timespec_sub, 1,500000000, 0,500000000, 1,0); + TEST_BINOP(timespec_sub, 5,500000000, 2,999999999, 2,500000001); + TEST_BINOP(timespec_sub, 0,0, 1,0, -1,0); + TEST_BINOP(timespec_sub, 0,500000000, 1,500000000, -1,0); + TEST_BINOP(timespec_sub, 0,0, 1,500000000, -2,500000000); + TEST_BINOP(timespec_sub, 1,0, 1,500000000, -1,500000000); + TEST_BINOP(timespec_sub, 1,0, 1,499999999, -1,500000001); + + // timespec_mod + + TEST_BINOP(timespec_mod, 0,0, 0,0, 0,0); + TEST_BINOP(timespec_mod, 0,0, 1,0, 0,0); + TEST_BINOP(timespec_mod, 1,0, 0,0, 1,0); + TEST_BINOP(timespec_mod, 1,0, 1,0, 0,0); + TEST_BINOP(timespec_mod, 10,0, 1,0, 0,0); + TEST_BINOP(timespec_mod, 10,0, 3,0, 1,0); + TEST_BINOP(timespec_mod, 10,0, -3,0, -2,0); + TEST_BINOP(timespec_mod, -10,0, 3,0, 2,0); + TEST_BINOP(timespec_mod, -10,0, -3,0, -1,0); + TEST_BINOP(timespec_mod, 10,0, 5,0, 0,0); + TEST_BINOP(timespec_mod, 10,0, -5,0, 0,0); + TEST_BINOP(timespec_mod, -10,0, 5,0, 0,0); + TEST_BINOP(timespec_mod, -10,0, -5,0, 0,0); + TEST_BINOP(timespec_mod, 1,500000000, 0,500000000, 0,0); + TEST_BINOP(timespec_mod, 5,500000000, 2,999999999, 2,500000001); + TEST_BINOP(timespec_mod, 0,500000000, 1,500000000, 0,500000000); + TEST_BINOP(timespec_mod, 0,0, 1,500000000, 0,0); + TEST_BINOP(timespec_mod, 1,0, 1,500000000, 1,0); + TEST_BINOP(timespec_mod, 1,0, 0,1, 0,0); + TEST_BINOP(timespec_mod, 1,123456789, 0,1000, 0,789); + TEST_BINOP(timespec_mod, 1,0, 0,9999999, 0,100); + TEST_BINOP(timespec_mod, 12345,54321, 0,100001, 0,5555); + TEST_BINOP(timespec_mod, LONG_MAX,0, 0,1, 0,0); + TEST_BINOP(timespec_mod, LONG_MAX,0, LONG_MAX,1, LONG_MAX,0); + + // timespec_clamp + + TEST_TRINOP(timespec_clamp, 0,0, 0,0, 0,0, 0,0); + + TEST_TRINOP(timespec_clamp, 1000,0, 2000,0, 3000,0, 2000,0); + TEST_TRINOP(timespec_clamp, 1500,0, 2000,0, 3000,0, 2000,0); + TEST_TRINOP(timespec_clamp, 1999,0, 2000,0, 3000,0, 2000,0); + TEST_TRINOP(timespec_clamp, 2000,0, 2000,0, 3000,0, 2000,0); + TEST_TRINOP(timespec_clamp, 2001,0, 2000,0, 3000,0, 2001,0); + TEST_TRINOP(timespec_clamp, 2250,0, 2000,0, 3000,0, 2250,0); + TEST_TRINOP(timespec_clamp, 2500,0, 2000,0, 3000,0, 2500,0); + TEST_TRINOP(timespec_clamp, 2750,0, 2000,0, 3000,0, 2750,0); + TEST_TRINOP(timespec_clamp, 2999,0, 2000,0, 3000,0, 2999,0); + TEST_TRINOP(timespec_clamp, 3000,0, 2000,0, 3000,0, 3000,0); + TEST_TRINOP(timespec_clamp, 3001,0, 2000,0, 3000,0, 3000,0); + TEST_TRINOP(timespec_clamp, 3500,0, 2000,0, 3000,0, 3000,0); + TEST_TRINOP(timespec_clamp, 4000,0, 2000,0, 3000,0, 3000,0); + + TEST_TRINOP(timespec_clamp, 0,1000, 0,2000, 0,3000, 0,2000); + TEST_TRINOP(timespec_clamp, 0,1500, 0,2000, 0,3000, 0,2000); + TEST_TRINOP(timespec_clamp, 0,1999, 0,2000, 0,3000, 0,2000); + TEST_TRINOP(timespec_clamp, 0,2000, 0,2000, 0,3000, 0,2000); + TEST_TRINOP(timespec_clamp, 0,2001, 0,2000, 0,3000, 0,2001); + TEST_TRINOP(timespec_clamp, 0,2250, 0,2000, 0,3000, 0,2250); + TEST_TRINOP(timespec_clamp, 0,2500, 0,2000, 0,3000, 0,2500); + TEST_TRINOP(timespec_clamp, 0,2750, 0,2000, 0,3000, 0,2750); + TEST_TRINOP(timespec_clamp, 0,2999, 0,2000, 0,3000, 0,2999); + TEST_TRINOP(timespec_clamp, 0,3000, 0,2000, 0,3000, 0,3000); + TEST_TRINOP(timespec_clamp, 0,3001, 0,2000, 0,3000, 0,3000); + TEST_TRINOP(timespec_clamp, 0,3500, 0,2000, 0,3000, 0,3000); + TEST_TRINOP(timespec_clamp, 0,4000, 0,2000, 0,3000, 0,3000); + + TEST_TRINOP(timespec_clamp,0,-1000, 0,-3000, 0,-2000, 0,-2000); + TEST_TRINOP(timespec_clamp,0,-1500, 0,-3000, 0,-2000, 0,-2000); + TEST_TRINOP(timespec_clamp,0,-1999, 0,-3000, 0,-2000, 0,-2000); + TEST_TRINOP(timespec_clamp,0,-3000, 0,-3000, 0,-2000, 0,-3000); + TEST_TRINOP(timespec_clamp,0,-2001, 0,-3000, 0,-2000, 0,-2001); + TEST_TRINOP(timespec_clamp,0,-2250, 0,-3000, 0,-2000, 0,-2250); + TEST_TRINOP(timespec_clamp,0,-2500, 0,-3000, 0,-2000, 0,-2500); + TEST_TRINOP(timespec_clamp,0,-2750, 0,-3000, 0,-2000, 0,-2750); + TEST_TRINOP(timespec_clamp,0,-2999, 0,-3000, 0,-2000, 0,-2999); + TEST_TRINOP(timespec_clamp,0,-2000, 0,-3000, 0,-2000, 0,-2000); + TEST_TRINOP(timespec_clamp,0,-3001, 0,-3000, 0,-2000, 0,-3000); + TEST_TRINOP(timespec_clamp,0,-3500, 0,-3000, 0,-2000, 0,-3000); + TEST_TRINOP(timespec_clamp,0,-2000, 0,-3000, 0,-2000, 0,-2000); + + TEST_TRINOP(timespec_clamp,0,-4000, 0,-3000, 0,3000, 0,-3000); + TEST_TRINOP(timespec_clamp,0,-3001, 0,-3000, 0,3000, 0,-3000); + TEST_TRINOP(timespec_clamp,0,-3000, 0,-3000, 0,3000, 0,-3000); + TEST_TRINOP(timespec_clamp,0,-2999, 0,-3000, 0,3000, 0,-2999); + TEST_TRINOP(timespec_clamp,0,-1500, 0,-3000, 0,3000, 0,-1500); + TEST_TRINOP(timespec_clamp,0, -1, 0,-3000, 0,3000, 0, -1); + TEST_TRINOP(timespec_clamp,0, 0, 0,-3000, 0,3000, 0, 0); + TEST_TRINOP(timespec_clamp,0, 1, 0,-3000, 0,3000, 0, 1); + TEST_TRINOP(timespec_clamp,0, 1500, 0,-3000, 0,3000, 0, 1500); + TEST_TRINOP(timespec_clamp,0, 2999, 0,-3000, 0,3000, 0, 2999); + TEST_TRINOP(timespec_clamp,0, 3000, 0,-3000, 0,3000, 0, 3000); + TEST_TRINOP(timespec_clamp,0, 3001, 0,-3000, 0,3000, 0, 3000); + TEST_TRINOP(timespec_clamp,0, 4000, 0,-3000, 0,3000, 0, 3000); + + // timespec_min + + TEST_BINOP(timespec_min, 0,0, 0,0, 0,0); + TEST_BINOP(timespec_min, 0,0, 1,0, 0,0); + TEST_BINOP(timespec_min, 1,0, 0,0, 0,0); + TEST_BINOP(timespec_min, 1,0, 1,0, 1,0); + TEST_BINOP(timespec_min, 10,0, 1,0, 1,0); + TEST_BINOP(timespec_min, 10,0, 3,0, 3,0); + TEST_BINOP(timespec_min, 10,0, -3,0, -3,0); + TEST_BINOP(timespec_min, -10,0, 3,0, -10,0); + TEST_BINOP(timespec_min, -10,0, -3,0, -10,0); + TEST_BINOP(timespec_min, 10,0, 5,0, 5,0); + TEST_BINOP(timespec_min, 10,0, -5,0, -5,0); + TEST_BINOP(timespec_min, -10,0, 5,0, -10,0); + TEST_BINOP(timespec_min, -10,0, -5,0, -10,0); + TEST_BINOP(timespec_min, 1,500000000, 0,500000000, 0,500000000); + TEST_BINOP(timespec_min, 5,500000000, 2,999999999, 2,999999999); + TEST_BINOP(timespec_min, 0,500000000, 1,500000000, 0,500000000); + TEST_BINOP(timespec_min, 0,0, 1,500000000, 0,0); + TEST_BINOP(timespec_min, 1,0, 1,500000000, 1,0); + TEST_BINOP(timespec_min, 1,0, 0,1, 0,1); + TEST_BINOP(timespec_min, 1,123456789, 0,1000, 0,1000); + TEST_BINOP(timespec_min, 1,0, 0,9999999, 0,9999999); + TEST_BINOP(timespec_min, 12345,54321, 0,100001, 0,100001); + TEST_BINOP(timespec_min, LONG_MIN,0, 0,1, LONG_MIN,0); + TEST_BINOP(timespec_min, LONG_MIN,0, 0,-1, LONG_MIN,0); + TEST_BINOP(timespec_min, LONG_MIN,0, LONG_MAX,0, LONG_MIN,0); + TEST_BINOP(timespec_min, LONG_MIN,0, LONG_MIN,0, LONG_MIN,0); + TEST_BINOP(timespec_min, LONG_MAX,0, 0,1, 0,1); + TEST_BINOP(timespec_min, LONG_MAX,0, 0,-1, 0,-1); + TEST_BINOP(timespec_min, LONG_MAX,0, LONG_MAX,0, LONG_MAX,0); + TEST_BINOP(timespec_min, LONG_MAX,0, LONG_MIN,0, LONG_MIN,0); + + // timespec_max + + TEST_BINOP(timespec_max, 0,0, 0,0, 0,0); + TEST_BINOP(timespec_max, 0,0, 1,0, 1,0); + TEST_BINOP(timespec_max, 1,0, 0,0, 1,0); + TEST_BINOP(timespec_max, 1,0, 1,0, 1,0); + TEST_BINOP(timespec_max, 10,0, 1,0, 10,0); + TEST_BINOP(timespec_max, 10,0, 3,0, 10,0); + TEST_BINOP(timespec_max, 10,0, -3,0, 10,0); + TEST_BINOP(timespec_max, -10,0, 3,0, 3,0); + TEST_BINOP(timespec_max, -10,0, -3,0, -3,0); + TEST_BINOP(timespec_max, 10,0, 5,0, 10,0); + TEST_BINOP(timespec_max, 10,0, -5,0, 10,0); + TEST_BINOP(timespec_max, -10,0, 5,0, 5,0); + TEST_BINOP(timespec_max, -10,0, -5,0, -5,0); + TEST_BINOP(timespec_max, 1,500000000, 0,500000000, 1,500000000); + TEST_BINOP(timespec_max, 5,500000000, 2,999999999, 5,500000000); + TEST_BINOP(timespec_max, 0,500000000, 1,500000000, 1,500000000); + TEST_BINOP(timespec_max, 0,0, 1,500000000, 1,500000000); + TEST_BINOP(timespec_max, 1,0, 1,500000000, 1,500000000); + TEST_BINOP(timespec_max, 1,0, 0,1, 1,0); + TEST_BINOP(timespec_max, 1,123456789, 0,1000, 1,123456789); + TEST_BINOP(timespec_max, 1,0, 0,9999999, 1,0); + TEST_BINOP(timespec_max, 12345,54321, 0,100001, 12345,54321); + TEST_BINOP(timespec_max, LONG_MIN,0, 0,1, 0,1); + TEST_BINOP(timespec_max, LONG_MIN,0, 0,-1, 0,-1); + TEST_BINOP(timespec_max, LONG_MIN,0, LONG_MAX,0, LONG_MAX,0); + TEST_BINOP(timespec_max, LONG_MIN,0, LONG_MIN,0, LONG_MIN,0); + TEST_BINOP(timespec_max, LONG_MAX,0, 0,1, LONG_MAX,0); + TEST_BINOP(timespec_max, LONG_MAX,0, 0,-1, LONG_MAX,0); + TEST_BINOP(timespec_max, LONG_MAX,0, LONG_MAX,0, LONG_MAX,0); + TEST_BINOP(timespec_max, LONG_MAX,0, LONG_MIN,0, LONG_MAX,0); + + // timespec_cmp + + TEST_TEST_FUNC(timespec_cmp, 0,0, 0,0, 0); + TEST_TEST_FUNC(timespec_cmp, 100,0, 100,0, 0); + TEST_TEST_FUNC(timespec_cmp, -100,0, -100,0, 0); + + TEST_TEST_FUNC(timespec_cmp, 1,0, 0,0, 1); + TEST_TEST_FUNC(timespec_cmp, 0,0, 1,0, -1); + TEST_TEST_FUNC(timespec_cmp, 0,1, 0,0, 1); + TEST_TEST_FUNC(timespec_cmp, 0,0, 0,1, -1); + TEST_TEST_FUNC(timespec_cmp, 1,0, 0,100, 1); + TEST_TEST_FUNC(timespec_cmp, 0,100 , 1,0, -1); + + TEST_TEST_FUNC(timespec_cmp, -0,-0, 0,0, 0); + TEST_TEST_FUNC(timespec_cmp, -10,-500000000, -11,500000000, 0); + TEST_TEST_FUNC(timespec_cmp, -10,-500000001, -11,499999999, 0); + TEST_TEST_FUNC(timespec_cmp, -10,-500000001, -11,500000001, -1); + TEST_TEST_FUNC(timespec_cmp, -11,500000001, -10,-500000001, 1); + + // timespec_eq + + TEST_TEST_FUNC(timespec_eq, 0,0, 0,0, true); + TEST_TEST_FUNC(timespec_eq, 100,0, 100,0, true); + TEST_TEST_FUNC(timespec_eq, -200,0, -200,0, true); + TEST_TEST_FUNC(timespec_eq, 0,300, 0,300, true); + TEST_TEST_FUNC(timespec_eq, 0,-400, 0,-400, true); + + TEST_TEST_FUNC(timespec_eq, 100,1, 100,0, false); + TEST_TEST_FUNC(timespec_eq, 101,0, 100,0, false); + TEST_TEST_FUNC(timespec_eq, -100,0, 100,0, false); + TEST_TEST_FUNC(timespec_eq, 0,10, 0,-10, false); + + TEST_TEST_FUNC(timespec_eq, -0,-0, 0,0, true); + TEST_TEST_FUNC(timespec_eq, -10,-500000000, -11,500000000, true); + TEST_TEST_FUNC(timespec_eq, -10,-500000001, -11,499999999, true); + TEST_TEST_FUNC(timespec_eq, -10,-500000001, -11,500000001, false); + + // timespec_gt + + TEST_TEST_FUNC(timespec_gt, 1,0, 0,0, true); + TEST_TEST_FUNC(timespec_gt, 0,0, -1,0, true); + TEST_TEST_FUNC(timespec_gt, 0,1, 0,0, true); + TEST_TEST_FUNC(timespec_gt, 0,0, 0,-1, true); + + TEST_TEST_FUNC(timespec_gt, 1,0, 1,0, false); + TEST_TEST_FUNC(timespec_gt, 1,1, 1,1, false); + TEST_TEST_FUNC(timespec_gt, -1,0, 0,0, false); + TEST_TEST_FUNC(timespec_gt, 0,-1, 0,0, false); + + TEST_TEST_FUNC(timespec_gt, 0,0, -0,-0, false); + TEST_TEST_FUNC(timespec_gt, -10,-500000000, -11,500000000, false); + TEST_TEST_FUNC(timespec_gt, -11,500000000, -10,-500000000, false); + TEST_TEST_FUNC(timespec_gt, -10,-500000001, -11,499999999, false); + TEST_TEST_FUNC(timespec_gt, -11,499999999, -11,499999999, false); + TEST_TEST_FUNC(timespec_gt, -10,-500000001, -11,500000001, false); + TEST_TEST_FUNC(timespec_gt, -11,500000001, -10,-500000001, true); + + // timespec_ge + + TEST_TEST_FUNC(timespec_ge, 1,0, 0,0, true); + TEST_TEST_FUNC(timespec_ge, 0,0, -1,0, true); + TEST_TEST_FUNC(timespec_ge, 0,1, 0,0, true); + TEST_TEST_FUNC(timespec_ge, 0,0, 0,-1, true); + TEST_TEST_FUNC(timespec_ge, 1,0, 1,0, true); + TEST_TEST_FUNC(timespec_ge, 1,1, 1,1, true); + + TEST_TEST_FUNC(timespec_ge, -1,0, 0,0, false); + TEST_TEST_FUNC(timespec_ge, 0,-1, 0,0, false); + + TEST_TEST_FUNC(timespec_ge, 0,0, -0,-0, true); + TEST_TEST_FUNC(timespec_ge, -10,-500000000, -11,500000000, true); + TEST_TEST_FUNC(timespec_ge, -11,500000000, -10,-500000000, true); + TEST_TEST_FUNC(timespec_ge, -10,-500000001, -11,499999999, true); + TEST_TEST_FUNC(timespec_ge, -11,499999999, -11,499999999, true); + TEST_TEST_FUNC(timespec_ge, -10,-500000001, -11,500000001, false); + TEST_TEST_FUNC(timespec_ge, -11,500000001, -10,-500000001, true); + + // timespec_lt + + TEST_TEST_FUNC(timespec_lt, 0,0, 1,0, true); + TEST_TEST_FUNC(timespec_lt, -1,0, 0,0, true); + TEST_TEST_FUNC(timespec_lt, 0,0, 0,1, true); + TEST_TEST_FUNC(timespec_lt, 0,-1, 0,0, true); + + TEST_TEST_FUNC(timespec_lt, 1,0, 1,0, false); + TEST_TEST_FUNC(timespec_lt, 1,1, 1,1, false); + TEST_TEST_FUNC(timespec_lt, 0,0, -1,0, false); + TEST_TEST_FUNC(timespec_lt, 0,0, 0,-1, false); + + TEST_TEST_FUNC(timespec_lt, 0,0, -0,-0, false); + TEST_TEST_FUNC(timespec_lt, -10,-500000000, -11,500000000, false); + TEST_TEST_FUNC(timespec_lt, -11,500000000, -10,-500000000, false); + TEST_TEST_FUNC(timespec_lt, -10,-500000001, -11,499999999, false); + TEST_TEST_FUNC(timespec_lt, -11,499999999, -11,499999999, false); + TEST_TEST_FUNC(timespec_lt, -10,-500000001, -11,500000001, true); + TEST_TEST_FUNC(timespec_lt, -11,500000001, -10,-500000001, false); + + // timespec_le + + TEST_TEST_FUNC(timespec_le, 0,0, 1,0, true); + TEST_TEST_FUNC(timespec_le, -1,0, 0,0, true); + TEST_TEST_FUNC(timespec_le, 0,0, 0,1, true); + TEST_TEST_FUNC(timespec_le, 0,-1, 0,0, true); + TEST_TEST_FUNC(timespec_le, 1,0, 1,0, true); + TEST_TEST_FUNC(timespec_le, 1,1, 1,1, true); + + TEST_TEST_FUNC(timespec_le, 0,0, -1,0, false); + TEST_TEST_FUNC(timespec_le, 0,0, 0,-1, false); + + TEST_TEST_FUNC(timespec_le, 0,0, -0,-0, true); + TEST_TEST_FUNC(timespec_le, -10,-500000000, -11,500000000, true); + TEST_TEST_FUNC(timespec_le, -11,500000000, -10,-500000000, true); + TEST_TEST_FUNC(timespec_le, -10,-500000001, -11,499999999, true); + TEST_TEST_FUNC(timespec_le, -11,499999999, -11,499999999, true); + TEST_TEST_FUNC(timespec_le, -10,-500000001, -11,500000001, true); + TEST_TEST_FUNC(timespec_le, -11,500000001, -10,-500000001, false); + + // timespec_from_double + + TEST_FROM_DOUBLE(0.0, 0,0); + TEST_FROM_DOUBLE(10.0, 10,0); + TEST_FROM_DOUBLE(-10.0, -10,0); + TEST_FROM_DOUBLE(0.5, 0,500000000); + TEST_FROM_DOUBLE(-0.5, -1,500000000); + TEST_FROM_DOUBLE(10.5, 10,500000000); + TEST_FROM_DOUBLE(-10.5, -11,500000000); + + // timespec_to_double + + TEST_TO_DOUBLE(0,0, 0.0); + TEST_TO_DOUBLE(10,0, 10.0); + TEST_TO_DOUBLE(-10,0, -10.0); + TEST_TO_DOUBLE(0,500000000, 0.5); + TEST_TO_DOUBLE(0,-500000000, -0.5); + TEST_TO_DOUBLE(10,500000000, 10.5); + TEST_TO_DOUBLE(10,-500000000, 9.5); + TEST_TO_DOUBLE(-10,500000000, -9.5); + TEST_TO_DOUBLE(-10,-500000000, -10.5); + + // timespec_from_timeval + + TEST_FROM_TIMEVAL(0,0, 0,0); + TEST_FROM_TIMEVAL(1,0, 1,0); + TEST_FROM_TIMEVAL(1000,0, 1000,0); + TEST_FROM_TIMEVAL(0,0, 0,0); + TEST_FROM_TIMEVAL(-1,0, -1,0); + TEST_FROM_TIMEVAL(-1000,0, -1000,0); + + TEST_FROM_TIMEVAL(1,1, 1,1000); + TEST_FROM_TIMEVAL(1,1000, 1,1000000); + TEST_FROM_TIMEVAL(1,-1, 0,999999000); + TEST_FROM_TIMEVAL(1,-1000, 0,999000000); + TEST_FROM_TIMEVAL(-1,-1, -2,999999000); + TEST_FROM_TIMEVAL(-1,-1000, -2,999000000); + + // timespec_to_timeval + + TEST_TO_TIMEVAL(0,0, 0,0); + TEST_TO_TIMEVAL(1,0, 1,0); + TEST_TO_TIMEVAL(10,0, 10,0); + TEST_TO_TIMEVAL(-1,0, -1,0); + TEST_TO_TIMEVAL(-10,0, -10,0); + + TEST_TO_TIMEVAL(1,1, 1,0); + TEST_TO_TIMEVAL(1,999, 1,0); + TEST_TO_TIMEVAL(1,1000, 1,1); + TEST_TO_TIMEVAL(1,1001, 1,1); + TEST_TO_TIMEVAL(1,2000, 1,2); + TEST_TO_TIMEVAL(1,2000000, 1,2000); + + TEST_TO_TIMEVAL(1,-1, 0,999999); + TEST_TO_TIMEVAL(1,-999, 0,999999); + TEST_TO_TIMEVAL(1,-1000, 0,999999); + TEST_TO_TIMEVAL(1,-1001, 0,999998); + TEST_TO_TIMEVAL(1,-2000, 0,999998); + TEST_TO_TIMEVAL(1,-2000000, 0,998000); + + TEST_TO_TIMEVAL(-1,-1, -2,999999); + TEST_TO_TIMEVAL(-1,-999, -2,999999); + TEST_TO_TIMEVAL(-1,-1000, -2,999999); + TEST_TO_TIMEVAL(-1,-1001, -2,999998); + TEST_TO_TIMEVAL(-1,-2000, -2,999998); + TEST_TO_TIMEVAL(-1,-2000000, -2,998000); + + TEST_TO_TIMEVAL(1,1500000000, 2,500000); + TEST_TO_TIMEVAL(1,-1500000000, -1,500000); + TEST_TO_TIMEVAL(-1,-1500000000, -3,500000); + + // timespec_from_ms + + TEST_FROM_MS(0, 0,0); + TEST_FROM_MS(1, 0,1000000); + TEST_FROM_MS(-1, -1,999000000); + TEST_FROM_MS(1500, 1,500000000); + TEST_FROM_MS(-1000, -1,0); + TEST_FROM_MS(-1500, -2,500000000); + + // timespec_to_ms + + TEST_TO_MS(0,0, 0); + TEST_TO_MS(10,0, 10000); + TEST_TO_MS(-10,0, -10000); + TEST_TO_MS(0,500000000, 500); + TEST_TO_MS(0,-500000000, -500); + TEST_TO_MS(10,500000000, 10500); + TEST_TO_MS(10,-500000000, 9500); + TEST_TO_MS(-10,500000000, -9500); + TEST_TO_MS(-10,-500000000, -10500); + + // timespec_normalise + + TEST_NORMALISE(0,0, 0,0); + + TEST_NORMALISE(0,1000000000, 1,0); + TEST_NORMALISE(0,1500000000, 1,500000000); + TEST_NORMALISE(0,-1000000000, -1,0); + TEST_NORMALISE(0,-1500000000, -2,500000000); + + TEST_NORMALISE(5,1000000000, 6,0); + TEST_NORMALISE(5,1500000000, 6,500000000); + TEST_NORMALISE(-5,-1000000000, -6,0); + TEST_NORMALISE(-5,-1500000000, -7,500000000); + + TEST_NORMALISE(0,2000000000, 2,0); + TEST_NORMALISE(0,2100000000, 2,100000000); + TEST_NORMALISE(0,-2000000000, -2,0); + TEST_NORMALISE(0,-2100000000, -3,900000000); + + TEST_NORMALISE(1,-500000001, 0,499999999); + TEST_NORMALISE(1,-500000000, 0,500000000); + TEST_NORMALISE(1,-499999999, 0,500000001); + TEST_NORMALISE(0,-499999999, -1,500000001); + + TEST_NORMALISE(-1,500000000, -1,500000000); + TEST_NORMALISE(-1,499999999, -1,499999999); + + if(result > 0) + { + printf("%d tests failed\n", result); + } + else{ + printf("All tests passed\n"); + } + + return !!result; /* Don't overflow the exit status */ +} +#endif diff --git a/src/lib/core/utils.c b/src/lib/core/utils.c index a248f58..d869143 100644 --- a/src/lib/core/utils.c +++ b/src/lib/core/utils.c @@ -35,9 +35,11 @@ int popd() { int rmtree(char *_path) { int status = 0; char path[PATH_MAX] = {0}; - strncpy(path, _path, sizeof(path) - 1); struct dirent *d_entity; + strncpy(path, _path, sizeof(path) - 1); + path[sizeof(path) - 1] = '\0'; + DIR *dir = opendir(path); if (!dir) { return 1; @@ -45,9 +47,9 @@ int rmtree(char *_path) { while ((d_entity = readdir(dir)) != NULL) { char abspath[PATH_MAX] = {0}; - strcat(abspath, path); - strcat(abspath, DIR_SEP); - strcat(abspath, d_entity->d_name); + strncat(abspath, path, sizeof(abspath) - strlen(abspath) - 1); + strncat(abspath, DIR_SEP, sizeof(abspath) - strlen(abspath) - 1); + strncat(abspath, d_entity->d_name, sizeof(abspath) - strlen(abspath) - 1); if (!strcmp(d_entity->d_name, ".") || !strcmp(d_entity->d_name, "..") || !strcmp(abspath, path)) { continue; @@ -106,6 +108,7 @@ char *expandpath(const char *_path) { char *tmphome; if ((tmphome = getenv(homes[i])) != NULL) { strncpy(home, tmphome, PATH_MAX - 1); + home[PATH_MAX - 1] = '\0'; break; } } @@ -179,7 +182,6 @@ char **file_readlines(const char *filename, size_t start, size_t limit, ReaderFn } if (fp == NULL) { - perror(filename); SYSERROR("failed to open %s for reading", filename); return NULL; } @@ -278,13 +280,14 @@ char *find_program(const char *name) { result[0] = '\0'; while ((path_elem = strsep(&path, PATH_SEP))) { char abspath[PATH_MAX] = {0}; - strcat(abspath, path_elem); - strcat(abspath, DIR_SEP); - strcat(abspath, name); + strncat(abspath, path_elem, sizeof(abspath) - strlen(abspath) - 1); + strncat(abspath, DIR_SEP, sizeof(abspath) - strlen(abspath) - 1); + strncat(abspath, name, sizeof(abspath) - strlen(abspath) - 1); if (access(abspath, F_OK) < 0) { continue; } - strncpy(result, abspath, sizeof(result)); + strncpy(result, abspath, sizeof(result) - 1); + result[sizeof(result) - 1] = '\0'; break; } path = path_orig; @@ -299,7 +302,7 @@ int touch(const char *filename) { FILE *fp = fopen(filename, "w"); if (!fp) { - perror(filename); + SYSERROR("unable to open %s for writing", filename); return 1; } fclose(fp); @@ -316,11 +319,11 @@ int git_clone(struct Process *proc, char *url, char *destdir, char *gitref) { } static char command[PATH_MAX] = {0}; - sprintf(command, "%s clone -c advice.detachedHead=false --recursive %s", program, url); + snprintf(command, sizeof(command), "%s clone -c advice.detachedHead=false --recursive %s", program, url); if (destdir && access(destdir, F_OK) < 0) { // Destination directory does not exist - sprintf(command + strlen(command), " %s", destdir); + snprintf(command + strlen(command), sizeof(command) - strlen(command), " %s", destdir); // Clone the repo result = shell(proc, command); if (result) { @@ -338,7 +341,7 @@ int git_clone(struct Process *proc, char *url, char *destdir, char *gitref) { if (!pushd(chdir_to)) { memset(command, 0, sizeof(command)); - sprintf(command, "%s fetch --all", program); + snprintf(command, sizeof(command), "%s fetch --all", program); result = shell(proc, command); if (result) { goto die_pop; @@ -346,7 +349,7 @@ int git_clone(struct Process *proc, char *url, char *destdir, char *gitref) { if (gitref != NULL) { memset(command, 0, sizeof(command)); - sprintf(command, "%s checkout %s", program, gitref); + snprintf(command, sizeof(command), "%s checkout %s", program, gitref); result = shell(proc, command); if (result) { @@ -376,11 +379,16 @@ char *git_describe(const char *path) { return NULL; } - FILE *pp = popen("git describe --first-parent --always --tags", "r"); + // TODO: Use `-C [path]` if the version of git installed supports it + FILE *pp = popen("git describe --first-parent --long --always --tags", "r"); if (!pp) { return NULL; } - fgets(version, sizeof(version) - 1, pp); + if (fgets(version, sizeof(version) - 1, pp) == NULL) { + pclose(pp); + popd(); + return NULL; + } strip(version); pclose(pp); popd(); @@ -393,7 +401,7 @@ char *git_rev_parse(const char *path, char *args) { memset(version, 0, sizeof(version)); if (isempty(args)) { - fprintf(stderr, "git_rev_parse args cannot be empty\n"); + SYSERROR("git_rev_parse args cannot be empty"); return NULL; } @@ -401,12 +409,17 @@ char *git_rev_parse(const char *path, char *args) { return NULL; } - sprintf(cmd, "git rev-parse %s", args); + // TODO: Use `-C [path]` if the version of git installed supports it + snprintf(cmd, sizeof(cmd), "git rev-parse %s", args); FILE *pp = popen(cmd, "r"); if (!pp) { return NULL; } - fgets(version, sizeof(version) - 1, pp); + if (fgets(version, sizeof(version) - 1, pp) == NULL) { + pclose(pp); + popd(); + return NULL; + } strip(version); pclose(pp); popd(); @@ -414,53 +427,68 @@ char *git_rev_parse(const char *path, char *args) { } void msg(unsigned type, char *fmt, ...) { + va_list args; + va_start(args, fmt); + FILE *stream = NULL; char header[255]; char status[20]; if (type & STASIS_MSG_NOP) { // quiet mode + va_end(args); return; } if (!globals.verbose && type & STASIS_MSG_RESTRICT) { // Verbose mode is not active + va_end(args); return; } memset(header, 0, sizeof(header)); memset(status, 0, sizeof(status)); - va_list args; - va_start(args, fmt); - stream = stdout; fprintf(stream, "%s", STASIS_COLOR_RESET); if (type & STASIS_MSG_ERROR) { // for error output stream = stderr; fprintf(stream, "%s", STASIS_COLOR_RED); - strcpy(status, " ERROR: "); + strncpy(status, " ERROR: ", sizeof(status) - 1); } else if (type & STASIS_MSG_WARN) { stream = stderr; fprintf(stream, "%s", STASIS_COLOR_YELLOW); - strcpy(status, " WARNING: "); + strncpy(status, " WARNING: ", sizeof(status) - 1); } else { fprintf(stream, "%s", STASIS_COLOR_GREEN); - strcpy(status, " "); + strncpy(status, " ", sizeof(status) - 1); } + status[sizeof(status) - 1] = '\0'; if (type & STASIS_MSG_L1) { - sprintf(header, "==>%s" STASIS_COLOR_RESET STASIS_COLOR_WHITE, status); + snprintf(header, sizeof(header), "==>%s" STASIS_COLOR_RESET STASIS_COLOR_WHITE, status); } else if (type & STASIS_MSG_L2) { - sprintf(header, " ->%s" STASIS_COLOR_RESET, status); + snprintf(header, sizeof(header), " ->%s" STASIS_COLOR_RESET, status); } else if (type & STASIS_MSG_L3) { - sprintf(header, STASIS_COLOR_BLUE " ->%s" STASIS_COLOR_RESET, status); + snprintf(header, sizeof(header), STASIS_COLOR_BLUE " ->%s" STASIS_COLOR_RESET, status); } - fprintf(stream, "%s", header); - vfprintf(stream, fmt, args); - fprintf(stream, "%s", STASIS_COLOR_RESET); + if (fprintf(stream, "%s", header) < 0) { + SYSERROR("unable to write message header to stream"); + return; + } + + const int len = vfprintf(stream, fmt, args); + if (len < 0) { + SYSERROR("unable to write message to stream"); + return; + } + + if (fprintf(stream, "%s", STASIS_COLOR_RESET) < 0) { + SYSERROR("unable to write message footer to stream"); + return; + } va_end(args); } @@ -476,37 +504,40 @@ void debug_shell() { char *xmkstemp(FILE **fp, const char *mode) { int fd = -1; - char tmpdir[PATH_MAX]; - char t_name[PATH_MAX * 2]; + char tmpdir[PATH_MAX] = {0}; + char t_name[PATH_MAX * 2] = {0}; - if (globals.tmpdir) { - strcpy(tmpdir, globals.tmpdir); - } else { - strcpy(tmpdir, "/tmp"); + strncpy(tmpdir, globals.tmpdir ? globals.tmpdir : "/tmp/stasis", sizeof(tmpdir) - 1); + tmpdir[sizeof(tmpdir) - 1] = '\0'; + + if (mkdirs(tmpdir, 0700) < 0) { + SYSERROR("unable to create sub-directories in %s", tmpdir); + return NULL; } + memset(t_name, 0, sizeof(t_name)); - sprintf(t_name, "%s/%s", tmpdir, "STASIS.XXXXXX"); + snprintf(t_name, sizeof(t_name), "%s/%s", tmpdir, "STASIS.XXXXXX"); fd = mkstemp(t_name); + if (fd < 0) { + SYSERROR("unable to create temporary file: %s", t_name); + return NULL; + } *fp = fdopen(fd, mode); if (!*fp) { - // unable to open, die - if (fd > 0) - close(fd); - *fp = NULL; + close(fd); return NULL; } char *path = strdup(t_name); if (!path) { // strdup failed, die - if (*fp) { - // close the file handle - fclose(*fp); - *fp = NULL; - } - // fall through. path is NULL. + // close the file handle + fclose(*fp); + *fp = NULL; + return NULL; } + return path; } @@ -574,10 +605,7 @@ int path_store(char **destptr, size_t maxlen, const char *base, const char *path int xml_pretty_print_in_place(const char *filename, const char *pretty_print_prog, const char *pretty_print_args) { int status = 0; - char *tempfile = NULL; char *result = NULL; - FILE *fp = NULL; - FILE *tmpfp = NULL; char cmd[PATH_MAX]; if (!find_program(pretty_print_prog)) { // Pretty printing is optional. 99% chance the XML data will @@ -585,14 +613,21 @@ int xml_pretty_print_in_place(const char *filename, const char *pretty_print_pro return 0; } memset(cmd, 0, sizeof(cmd)); - snprintf(cmd, sizeof(cmd) - 1, "%s %s %s", pretty_print_prog, pretty_print_args, filename); + snprintf(cmd, sizeof(cmd), "%s %s %s", pretty_print_prog, pretty_print_args, filename); result = shell_output(cmd, &status); if (status || !result) { - goto pretty_print_failed; + return status; } - tempfile = xmkstemp(&tmpfp, "w+"); - if (!tmpfp || !tempfile) { + int clean_up_fp = 0; + FILE *tmpfp = NULL; + char *tempfile = xmkstemp(&tmpfp, "w+"); + if (!tempfile || !tmpfp) { + guard_free(tempfile); + if (tmpfp) { + fclose(tmpfp); + } + status = -1; goto pretty_print_failed; } @@ -600,17 +635,20 @@ int xml_pretty_print_in_place(const char *filename, const char *pretty_print_pro fflush(tmpfp); fclose(tmpfp); - fp = fopen(filename, "w+"); + FILE *fp = fopen(filename, "w+"); if (!fp) { + status = -1; goto pretty_print_failed; } + clean_up_fp = 1; if (copy2(tempfile, filename, CT_PERM)) { - goto pretty_print_failed; + SYSWARN("unable to copy: '%s' -> '%s'", tempfile, filename); } if (remove(tempfile)) { goto pretty_print_failed; + SYSWARN("unable to remove temporary file: %s", tempfile); } fclose(fp); @@ -619,38 +657,47 @@ int xml_pretty_print_in_place(const char *filename, const char *pretty_print_pro return 0; pretty_print_failed: - if (fp) { - fclose(fp); - } - if (tmpfp) { - fclose(tmpfp); - } - guard_free(tempfile); - guard_free(result); - return -1; + if (tempfile && remove(tempfile)) { + SYSWARN("unable to remove temporary file: %s", tempfile); + } + guard_free(tempfile); + guard_free(result); + if (clean_up_fp) { + fclose(fp); + } + + return status; } /** * * @param filename /path/to/tox.ini * @param result path of replacement tox.ini configuration + * @param maxlen * @return 0 on success, -1 on error */ -int fix_tox_conf(const char *filename, char **result) { +int fix_tox_conf(const char *filename, char **result, size_t maxlen) { struct INIFILE *toxini; - FILE *fptemp; + FILE *fptemp = NULL; // Create new temporary tox configuration file char *tempfile = xmkstemp(&fptemp, "w+"); if (!tempfile) { + SYSERROR("unable to create temporary file"); + if (fptemp) { + fclose(fptemp); + } return -1; } // If the result pointer is NULL, allocate enough to store a filesystem path if (!*result) { - *result = calloc(PATH_MAX, sizeof(**result)); + *result = calloc(maxlen, sizeof(**result)); if (!*result) { guard_free(tempfile); + if (fptemp) { + fclose(fptemp); + } return -1; } } @@ -660,9 +707,9 @@ int fix_tox_conf(const char *filename, char **result) { if (!toxini) { if (fptemp) { guard_free(result); - guard_free(tempfile); fclose(fptemp); } + guard_free(tempfile); return -1; } @@ -687,10 +734,12 @@ int fix_tox_conf(const char *filename, char **result) { SYSERROR("failed to increase size to +%zu bytes", strlen(value) + strlen(with_posargs) + 1); guard_free(*result); + guard_free(tempfile); + fclose(fptemp); return -1; } value = tmp; - strcat(value, with_posargs); + strncat(value, with_posargs, (strlen(value) + strlen(with_posargs)) - strlen(value) - 1); ini_setval(&toxini, INI_SETVAL_REPLACE, section_name, key, value); } } @@ -705,36 +754,40 @@ int fix_tox_conf(const char *filename, char **result) { fclose(fptemp); // Store path to modified config - strcpy(*result, tempfile); + strncpy(*result, tempfile, maxlen - 1); + (*result)[maxlen - 1] = '\0'; guard_free(tempfile); ini_free(&toxini); return 0; } -static size_t count_blanks(char *s) { - // return the number of leading blanks (tab/space) in a string - size_t blank = 0; - for (size_t i = 0; i < strlen(s); i++) { - if (isblank(s[i])) { - blank++; +/** + * Collapse all whitespace in a string (to single spaces) + * @param s address of string to modify + * @return + */ +char *collapse_whitespace(char **s) { + char *dest = NULL; + char *src = NULL; + int in_ws = 1; + + for (src = dest = *s; *src != '\0'; ++src) { + if (isspace(*src)) { + if (!in_ws) { + *dest++ = ' '; + in_ws = 1; + } } else { - break; + *dest++ = *src; + in_ws = 0; } } - return blank; -} -char *collapse_whitespace(char **s) { - char *x = (*s); - size_t len = strlen(x); - for (size_t i = 0; i < len; i++) { - size_t blank = count_blanks(&x[i]); - if (blank > 1) { - memmove(&x[i], &x[i] + blank, strlen(&x[i])); - } + if (dest > *s && *(dest - 1) == ' ') { + --dest; } - + *dest = '\0'; return *s; } @@ -750,11 +803,12 @@ char *collapse_whitespace(char **s) { int redact_sensitive(const char **to_redact, size_t to_redact_size, char *src, char *dest, size_t maxlen) { const char *redacted = "***REDACTED***"; - char *tmp = calloc(strlen(redacted) + strlen(src) + 1, sizeof(*tmp)); + char *tmp = calloc(maxlen + 1, sizeof(*tmp)); if (!tmp) { return -1; } - strcpy(tmp, src); + strncpy(tmp, src, maxlen); + tmp[maxlen] = '\0'; for (size_t i = 0; i < to_redact_size; i++) { if (to_redact[i] && strstr(tmp, to_redact[i])) { @@ -764,7 +818,8 @@ int redact_sensitive(const char **to_redact, size_t to_redact_size, char *src, c } memset(dest, 0, maxlen); - strncpy(dest, tmp, maxlen - 1); + strncpy(dest, tmp, maxlen); + dest[maxlen] = '\0'; guard_free(tmp); return 0; @@ -793,7 +848,7 @@ struct StrList *listdir(const char *path) { } char *fullpath = join_ex("/", path, rec->d_name, NULL); if (!fullpath) { - SYSERROR("%s", "Unable to allocate bytes to construct full path"); + SYSERROR("Unable to allocate bytes to construct full path"); guard_strlist_free(&node); closedir(dp); return NULL; @@ -817,16 +872,16 @@ long get_cpu_count() { int mkdirs(const char *_path, mode_t mode) { char *token; char pathbuf[PATH_MAX] = {0}; - char *path; - path = pathbuf; - strcpy(path, _path); + strncpy(pathbuf, _path, sizeof(pathbuf) - 1); + pathbuf[sizeof(pathbuf) - 1] = '\0'; + char *path = pathbuf; errno = 0; char result[PATH_MAX] = {0}; int status = 0; while ((token = strsep(&path, "/")) != NULL && !status) { - strcat(result, token); - strcat(result, "/"); + strncat(result, token, sizeof result - strlen(result) - 1); + strncat(result, "/", sizeof result - strlen(result) - 1); status = mkdir(result, mode); if (status && errno == EEXIST) { status = 0; @@ -843,26 +898,32 @@ char *find_version_spec(char *str) { int env_manipulate_pathstr(const char *key, char *path, int mode) { if (isempty(path)) { - SYSERROR("%s", "New PATH element cannot be zero-length or NULL"); + SYSERROR("New PATH element cannot be zero-length or NULL"); return -1; } const char *system_path_old = getenv("PATH"); if (!system_path_old) { - SYSERROR("%s", "Unable to read PATH"); + SYSERROR("Unable to read PATH"); return -1; } char *system_path_new = NULL; if (mode & PM_APPEND) { - asprintf(&system_path_new, "%s%s%s", system_path_old, PATH_SEP, path); + if (asprintf(&system_path_new, "%s%s%s", system_path_old, PATH_SEP, path) < 0 || !system_path_new) { + SYSERROR("%s", "Unable to allocate memory to update PATH"); + return -1; + } } else if (mode & PM_PREPEND) { - asprintf(&system_path_new, "%s%s%s", path, PATH_SEP, system_path_old); + if (asprintf(&system_path_new, "%s%s%s", path, PATH_SEP, system_path_old) < 0 || !system_path_new) { + SYSERROR("%s", "Unable to allocate memory to update PATH"); + return -1; + } } if (!system_path_new) { - SYSERROR("%s", "Unable to allocate memory to update PATH"); + SYSERROR("Unable to allocate memory to update PATH"); return -1; } @@ -882,13 +943,342 @@ int env_manipulate_pathstr(const char *key, char *path, int mode) { return 0; } -int gen_file_extension_str(char *filename, const char *extension) { +int gen_file_extension_str(char *filename, const size_t maxlen, const char *extension) { char *ext_orig = strrchr(filename, '.'); if (!ext_orig) { - strcat(filename, extension); + strncat(filename, extension, maxlen - strlen(filename) - 1); return 0; } return replace_text(ext_orig, ext_orig, extension, 0); } +#define DEBUG_HEXDUMP_FMT_BYTES 6 +#define DEBUG_HEXDUMP_ADDR_MAXLEN 20 +#define DEBUG_HEXDUMP_BYTES_MAXLEN (16 * 3 + 2) +#define DEBUG_HEXDUMP_ASCII_MAXLEN (16 + 1) +#define DEBUG_HEXDUMP_OUTPUT_MAXLEN (DEBUG_HEXDUMP_FMT_BYTES + DEBUG_HEXDUMP_ADDR_MAXLEN + DEBUG_HEXDUMP_BYTES_MAXLEN + DEBUG_HEXDUMP_ASCII_MAXLEN + 1) + +void debug_hexdump(char *data, int len) { + int count = 0; + char addr[DEBUG_HEXDUMP_ADDR_MAXLEN] = {0}; + char bytes[DEBUG_HEXDUMP_BYTES_MAXLEN] = {0}; + char ascii[DEBUG_HEXDUMP_ASCII_MAXLEN] = {0}; + char output[DEBUG_HEXDUMP_OUTPUT_MAXLEN] = {0}; + char *start = data; + char *end = data + len; + + char *pos = start; + while (pos != end) { + if (count == 0) { + snprintf(addr + strlen(addr), sizeof(addr) - strlen(addr), "%p", pos); + } + if (count == 8) { + strncat(bytes, " ", sizeof(bytes) - strlen(bytes) - 1); + } + if (count > 15) { + snprintf(output, sizeof(output), "%s | %s | %s", addr, bytes, ascii); + puts(output); + memset(output, 0, sizeof(output)); + memset(addr, 0, sizeof(addr)); + memset(bytes, 0, sizeof(bytes)); + memset(ascii, 0, sizeof(ascii)); + count = 0; + continue; + } + + snprintf(bytes + strlen(bytes), sizeof(bytes) - strlen(bytes), "%02X ", (unsigned char) *pos); + snprintf(ascii + strlen(ascii), sizeof(ascii) - strlen(ascii), "%c", isprint(*pos) ? *pos : '.'); + + pos++; + count++; + } + + if (count <= 8) { + // Add group padding + strncat(bytes, " ", sizeof(bytes) - strlen(bytes) - 1); + } + const int padding = 16 - count; + for (int i = 0; i < padding; i++) { + strncat(bytes, " ", sizeof(bytes) - strlen(bytes) - 1); + } + snprintf(output, sizeof(output), "%s | %s | %s", addr, bytes, ascii); + puts(output); +} + +int grow(const size_t size_new, size_t *size_orig, char **data) { + if (!*data) { + return 0; + } + if (size_new >= *size_orig) { + const size_t new_size = *size_orig + size_new + 1; + SYSDEBUG("template data buffer new size: %zu", new_size); + + char *tmp = realloc(*data, new_size); + if (!tmp) { + SYSERROR("realloc failed"); + return -1; + } + *data = tmp; + *size_orig = new_size; + } + return 0; +} + +int in_ascii_range(const char c, char lower, char upper) { + if (!(c >= lower && c <= upper)) { + return 0; + } + return 1; +} + +int is_git_sha(char const *hash) { + size_t result = 0; + size_t len = strlen(hash); + + if (len > GIT_HASH_LEN) { + // too long to be a git commit hash + return 0; + } + for (size_t i = 0; i < len; i++) { + if (in_ascii_range(hash[i], 'a', 'f') + || in_ascii_range(hash[i], 'A', 'F') + || in_ascii_range(hash[i], '0', '9')) { + result++; + } + } + if (result < len) { + return 0; + } + return 1; +} + +static int read_vcs_records(const size_t line, char **data) { + (void) line; // unused + const char *vcs_name[] = { + "git", + "svn", + "hg", + "bzr", + }; + for (size_t i = 0; i < sizeof(vcs_name) / sizeof(vcs_name[0]); i++) { + const char *vcs = vcs_name[i]; + char *data_local = strdup(*data); + if (!data_local) { + SYSERROR("out of memory"); + return -1; + } + + // Remove leading/trailing blanks + lstrip(data_local); + strip(data_local); + + // Ignore file comment(s) + if (startswith(data_local, "#") || startswith(data_local, ";")) { + // continue + guard_free(data_local); + return 1; + } + + // Begin matching VCS package syntax + const char *match_vcs = strstr(data_local,vcs); + if (match_vcs) { + const char *match_protocol_sep = strstr(match_vcs, "+"); + if (match_protocol_sep) { + const char *match_protocol = strstr(match_protocol_sep, "://"); + if (match_protocol) { + guard_free(data_local); + // match found + return 0; + } + } + } + guard_free(data_local); + } + + // no match, continue + return 1; +} +int check_python_package_dependencies(const char *srcdir) { + const char *configs[] = { + "pyproject.toml", + "setup.cfg", + "setup.py" + }; + + for (size_t i = 0; i < sizeof(configs) / sizeof(configs[0]); i++) { + char path[PATH_MAX] = {0}; + const char *configfile = configs[i]; + + snprintf(path, sizeof(path), "%s/%s", srcdir, configfile); + if (access(path, F_OK) < 0) { + continue; + } + + //char **data = file_readlines(path, 0, 0, NULL); + struct StrList *data = strlist_init(); + int err = 0; + if ((err = strlist_append_file(data, path, read_vcs_records))) { + guard_strlist_free(&data); + return -1; + } + const size_t count = strlist_count(data); + if (count) { + printf("\nERROR: VCS requirement(s) detected in %s:\n", configfile); + for (size_t j = 0; j < count; j++) { + char *record = strlist_item(data, j); + lstrip(record); + strip(record); + printf("[%zu] %s\n", j, record); + } + guard_strlist_free(&data); + return 1; + } + guard_strlist_free(&data); + } + return 0; +} + +int str_to_timeout(char *s) { + if (!s) { + return 0; // no timeout + } + + char *scale = NULL; + int value = (int) strtol(s, &scale, 10); + if (scale) { + if (*scale == 's') { + value *= 1; // seconds, no-op + } else if (*scale == 'm') { + value *= 60; // minutes + } else if (*scale == 'h') { + value *= 3600; // hours + } else { + return STR_TO_TIMEOUT_INVALID_TIME_SCALE; // invalid time scale + } + } + + if (value < 0) { + return STR_TO_TIMEOUT_NEGATIVE; // cannot be negative + } + return value; +} + +void seconds_to_human_readable(const int v, char *result, const size_t maxlen) { + const int hours = v / 3600; + const int minutes = (v % 3600) / 60; + const int seconds = v % 60; + + memset(result, '\0', maxlen); + if (hours) { + snprintf(result + strlen(result), maxlen, "%dh ", hours); + } + if (hours || minutes) { + snprintf(result + strlen(result), maxlen, "%dm ", minutes); + } + snprintf(result + strlen(result), maxlen, "%ds", seconds); +} + +const char *get_random_generator_file() { + return STASIS_RANDOM_GENERATOR_FILE; +} + +#ifdef NEED_SRAND +static char stasis_srand_initialized = 0; +#endif + +int get_random_bytes(char *result, size_t maxlen) { +#ifdef NEED_SRAND + if (!srand_initialized) { + srand(time(NULL)); + srand_initialized = 1; + } +#endif + size_t bytes = 0; + const char *filename = get_random_generator_file(); + FILE *fp = NULL; + if (filename != NULL) { + fp = fopen(filename, "rb"); + if (!fp) { + SYSERROR("unable to open random generator"); + return -1; + } + } + + do { + int ch = 0; + if (fp) { + ch = fgetc(fp); + } else { + ch = rand() % 255; + } + if (fp && ferror(fp)) { + SYSERROR("unable to read from random generator"); + fclose(fp); + return -1; + } + if (isalnum(ch)) { + result[bytes] = (char) ch; + bytes++; + } + } while (bytes < maxlen); + + if (fp) { + fclose(fp); + } + result[bytes ? bytes - 1 : 0] = '\0'; + return 0; +} + +int non_format_len(const char *s) { + int len = 0; + int until_space = 0; + for (size_t i = 0; i < strlen(s); i++) { + if (until_space && isspace(s[i])) { + until_space = 0; + } + if (until_space && !isspace(s[i])) { + continue; + } + if (s[i] == '%') { + until_space = 1; + continue; + } + len++; + } + return len; +} + +char *center_text(const char *s, const size_t maxwidth) { + if (maxwidth < 2) { + SYSERROR("%s", "maximum width must be greater than 0"); + return NULL; + } + + if (maxwidth % 2 != 0) { + SYSERROR("maximum width (%zu) must be even", maxwidth); + return NULL; + } + + const size_t s_len = strlen(s); + if (s_len + 1 > maxwidth) { + SYSERROR("length of input string (%zu) exceeds maximum width (%zu)", s_len, maxwidth); + return NULL; + } + + char *result = calloc(maxwidth + 1, sizeof(*result)); + if (!result) { + SYSERROR("%s", "unable to allocate bytes for centered text string"); + return NULL; + } + const size_t middle = (maxwidth / 2) - s_len / 2; + size_t i = 0; + for (; i < middle; i++) { + result[i] = ' '; + } + strncpy(&result[i], s, maxwidth - middle - 1); + result[maxwidth] = '\0'; + + return result; +} + diff --git a/src/lib/core/version_compare.c b/src/lib/core/version_compare.c new file mode 100644 index 0000000..4939c8f --- /dev/null +++ b/src/lib/core/version_compare.c @@ -0,0 +1,185 @@ +#include "version_compare.h" + +const struct { + const char *key; + int value; +} WEIGHT[] = { + {.key = "post", 1000}, + {.key = "rc", -1000}, + {.key = "dev", -2000}, +}; + +/** + * Sum each part of a '.'-delimited version string + * @param str version string + * @return sum of each part + * @return -1 on error + */ +int version_sum(const char *str) { + char *end; + + if (!str || isempty((char *) str)) { + return -1; + } + + int result = 0; + int epoch = 0; + char *s = strdup(str); + if (!s) { + return -1; + } + char *ptr = s; + end = ptr; + + // Parsing stops at the first non-alpha, non-'.' character + // Digits are processed until the first invalid character + // I'm torn whether this should be considered an error + int i = 0; + while (end != NULL) { + int tmp_result = 0; + + tmp_result = (int) strtoul(ptr, &end, 10); + + // Circumvent a bug which allows a smaller version to be greater + // than a larger version + // Bug: + // 1.0.3 == 1 + 0 + 3 = 4 + // 2.0.0 == 2 + 0 + 0 = 2 + // Correction: + // ((1 * EPOCH_MOD) + 1).0.3 = 104 + // ((2 * EPOCH_MOD) + 2).0.0 = 202 + if (!i && tmp_result && *end != ':') { + result += tmp_result * EPOCH_MOD; + i++; + } + + ptr = end; + if (*ptr == '.' || *ptr == '-') { + ptr++; + } + else if (!epoch && *ptr == ':') { + epoch = 1; + result += EPOCH_MOD; + ptr++; + } + else if (isalpha(*ptr)) { + int adjusted = 0; + for (size_t w = 0; w < sizeof(WEIGHT) / sizeof(WEIGHT[0]); w++) { + const int has_suffix = strncasecmp(ptr, WEIGHT[w].key, strlen(WEIGHT[w].key)) == 0; + if (has_suffix) { + // skip the suffix + ptr += strlen(WEIGHT[w].key); + // adjust result based on suffix weight + result += WEIGHT[w].value; + adjusted = 1; + break; + } + } + + if (!adjusted) { + result += *ptr - ('a' - 1); + ptr++; + } + } + else { + end = NULL; + } + + if (tmp_result) { + result += tmp_result; + } + } + + free(s); + return result; +} + +/** + * Convert version operator(s) to flags + * @param str input string + * @return operator flags + */ +int version_parse_operator(const char *str) { + const char *valid = "><=!"; + + const char *pos = str; + int result = 0; + + if (isempty((char *) str)) { + return -1; + } + while ((pos = strpbrk(pos, valid)) != NULL) { + switch (*pos) { + case '>': + result |= GT; + break; + case '<': + result |= LT; + break; + case '=': + result |= EQ; + break; + case '!': + result |= NOT; + break; + default: + return -1; + } + pos++; + } + + return result; +} + +static int version_has_epoch(const char *str) { + const char *result = strchr(str, ':'); + return result ? 1 : 0; +} + +/** + * Compare version strings based on flag(s) + * @param flags verison operators + * @param aa version1 + * @param bb version2 + * @return 1 flag operation is true + * @return 0 flag operation is false + */ +int version_compare(const int flags, const char *aa, const char *bb) { + if (!flags || flags < 0) { + return -1; + } + + int result_a = version_sum(aa); + if (result_a < 0) { + return -1; + } + + int result_b = version_sum(bb); + if (result_b < 0) { + return -1; + } + + if (version_has_epoch(aa) && !version_has_epoch(bb)) { + result_a -= EPOCH_MOD; + } + if (!version_has_epoch(aa) && version_has_epoch(bb)) { + result_b -= EPOCH_MOD; + } + + int result = 0; + if (flags & GT && flags & EQ) { + result |= result_a >= result_b; + } else if (flags & LT && flags & EQ) { + result |= result_a <= result_b; + } else if (flags & NOT && flags & EQ) { + result |= result_a != result_b; + } else if (flags & GT) { + result |= result_a > result_b; + } else if (flags & LT) { + result |= result_a < result_b; + } else if (flags & EQ) { + result |= result_a == result_b; + } + + return result; +}
\ No newline at end of file diff --git a/src/lib/core/wheel.c b/src/lib/core/wheel.c index c7e485a..e94060a 100644 --- a/src/lib/core/wheel.c +++ b/src/lib/core/wheel.c @@ -1,126 +1,1400 @@ #include "wheel.h" -struct Wheel *get_wheel_info(const char *basepath, const char *name, char *to_match[], unsigned match_mode) { - struct dirent *rec; - struct Wheel *result = NULL; - char package_path[PATH_MAX]; - char package_name[NAME_MAX]; +#include <ctype.h> - strcpy(package_name, name); - tolower_s(package_name); - sprintf(package_path, "%s/%s", basepath, package_name); +#include "str.h" +#include "strlist.h" - DIR *dp = opendir(package_path); - if (!dp) { - return NULL; +const char *WHEEL_META_KEY[] = { + "Metadata-Version", + "Name", + "Version", + "Author", + "Author-email", + "Maintainer", + "Maintainer-email", + "Summary", + "License", + "License-Expression", + "License-File", + "Home-page", + "Download-URL", + "Project-URL", + "Classifier", + "Requires-Python", + "Requires-External", + "Import-Name", + "Import-Namespace", + "Requires-Dist", + "Provides", + "Provides-Dist", + "Provides-Extra", + "Obsoletes", + "Obsoletes-Dist", + "Platform", + "Supported-Platform", + "Keywords", + "Dynamic", + "Description-Content-Type", + "Description", + NULL, +}; + +const char *WHEEL_DIST_KEY[] = { + "Wheel-Version", + "Generator", + "Root-Is-Purelib", + "Tag", + "Zip-Safe", + "Top-Level", + "Entry-points", + "Record", + NULL, +}; + +static ssize_t wheel_parse_wheel(struct Wheel * pkg, const char * data) { + int read_as = 0; + struct StrList *lines = strlist_init(); + if (!lines) { + return -1; } + strlist_append_tokenize(lines, (char *) data, "\r\n"); - while ((rec = readdir(dp)) != NULL) { - if (!strcmp(rec->d_name, ".") || !strcmp(rec->d_name, "..")) { + for (size_t i = 0; i < strlist_count(lines); i++) { + char *line = strlist_item(lines, i); + if (isempty(line)) { continue; } - char filename[NAME_MAX]; - strcpy(filename, rec->d_name); - char *ext = strstr(filename, ".whl"); - if (ext) { - *ext = '\0'; + + char **pair = split(line, ":", 1); + if (pair) { + char *key = strdup(strip(pair[0])); + if (!key) { + SYSERROR("could not allocate memory for pair wheel key"); + return -1; + } + + char *value = strdup(lstrip(pair[1])); + if (!value) { + SYSERROR("could not allocate memory for wheel value"); + guard_free(key); + return -1; + } + + if (!strcasecmp(key, WHEEL_DIST_KEY[WHEEL_DIST_VERSION])) { + read_as = WHEEL_DIST_VERSION; + } else if (!strcasecmp(key, WHEEL_DIST_KEY[WHEEL_DIST_GENERATOR])) { + read_as = WHEEL_DIST_GENERATOR; + } else if (!strcasecmp(key, WHEEL_DIST_KEY[WHEEL_DIST_ROOT_IS_PURELIB])) { + read_as = WHEEL_DIST_ROOT_IS_PURELIB; + } else if (!strcasecmp(key, WHEEL_DIST_KEY[WHEEL_DIST_TAG])) { + read_as = WHEEL_DIST_TAG; + } + + switch (read_as) { + case WHEEL_DIST_VERSION: { + pkg->wheel_version = strdup(value); + if (!pkg->wheel_version) { + // memory error + guard_free(key); + guard_free(value); + wheel_package_free(&pkg); + return -1; + } + break; + } + case WHEEL_DIST_GENERATOR: { + pkg->generator = strdup(value); + if (!pkg->generator) { + // memory error + guard_free(key); + guard_free(value); + wheel_package_free(&pkg); + return -1; + } + break; + } + case WHEEL_DIST_ROOT_IS_PURELIB: { + pkg->root_is_pure_lib = strdup(value); + if (!pkg->root_is_pure_lib) { + // memory error + guard_free(key); + guard_free(value); + wheel_package_free(&pkg); + return -1; + } + break; + } + case WHEEL_DIST_TAG: { + if (!pkg->tag) { + pkg->tag = strlist_init(); + if (!pkg->tag) { + guard_free(key); + guard_free(value); + wheel_package_free(&pkg); + return -1; + } + } + strlist_append(&pkg->tag, value); + break; + } + default: + SYSWARN("unhandled wheel key on line %zu:\nbuffer contents: '%s'", i, value); + break; + } + guard_free(key); + guard_free(value); + guard_array_n_free(pair, 2); + } + } + guard_strlist_free(&lines); + return data ? (ssize_t) strlen(data) : -1; +} + + +static inline int is_continuation(const char *s) { + return s && (s[0] == ' ' || s[0] == '\t'); +} + +static ssize_t wheel_parse_metadata(struct WheelMetadata * const pkg, const char * const data) { + int read_as = WHEEL_KEY_UNKNOWN; + // triggers + int reading_multiline = 0; + int reading_extra = 0; + size_t provides_extra_i = 0; + int reading_description = 0; + int base_description_len = 1024; + int len_description = 0; + struct WheelMetadata_ProvidesExtra *current_extra = NULL; + + if (!data) { + // data can't be NULL + return -1; + } + + pkg->provides_extra = calloc(WHEEL_MAXELEM + 1, sizeof(*pkg->provides_extra)); + if (!pkg->provides_extra) { + // memory error + return -1; + } + + struct StrList *lines = strlist_init(); + if (!lines) { + // memory error + return -1; + } + + strlist_append_tokenize_raw(lines, (char *) data, "\r\n"); + for (size_t i = 0; i < strlist_count(lines); i++) { + const char *line = strlist_item(lines, i); + char *key = NULL; + char *value = NULL; + + reading_multiline = is_continuation(line); + if (!reading_multiline && line[0] == '\0') { + reading_description = 1; + read_as = WHEEL_META_DESCRIPTION; + } + + char **pair = split((char *) line, ":", 1); + if (!pair) { + // memory error + return -1; + } + + if (!reading_description && !reading_multiline && pair[1]) { + key = strip(strdup(pair[0])); + value = lstrip(strdup(pair[1])); + + if (!key || !value) { + // memory error + return -1; + } + + if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_METADATA_VERSION])) { + read_as = WHEEL_META_METADATA_VERSION; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_NAME])) { + read_as = WHEEL_META_NAME; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_VERSION])) { + read_as = WHEEL_META_VERSION; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_SUMMARY])) { + read_as = WHEEL_META_SUMMARY; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_AUTHOR])) { + read_as = WHEEL_META_AUTHOR; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_AUTHOR_EMAIL])) { + read_as = WHEEL_META_AUTHOR_EMAIL; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_MAINTAINER])) { + read_as = WHEEL_META_MAINTAINER; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_MAINTAINER_EMAIL])) { + read_as = WHEEL_META_MAINTAINER_EMAIL; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_LICENSE])) { + read_as = WHEEL_META_LICENSE; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_LICENSE_EXPRESSION])) { + read_as = WHEEL_META_LICENSE_EXPRESSION; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_HOME_PAGE])) { + read_as = WHEEL_META_HOME_PAGE; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_DOWNLOAD_URL])) { + read_as = WHEEL_META_DOWNLOAD_URL; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_PROJECT_URL])) { + read_as = WHEEL_META_PROJECT_URL; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_CLASSIFIER])) { + read_as = WHEEL_META_CLASSIFIER; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_REQUIRES_PYTHON])) { + read_as = WHEEL_META_REQUIRES_PYTHON; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_REQUIRES_EXTERNAL])) { + read_as = WHEEL_META_REQUIRES_EXTERNAL; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_DESCRIPTION_CONTENT_TYPE])) { + read_as = WHEEL_META_DESCRIPTION_CONTENT_TYPE; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_LICENSE_FILE])) { + read_as = WHEEL_META_LICENSE_FILE; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_REQUIRES_DIST])) { + read_as = WHEEL_META_REQUIRES_DIST; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_PROVIDES])) { + read_as = WHEEL_META_PROVIDES; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_IMPORT_NAME])) { + read_as = WHEEL_META_IMPORT_NAME; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_IMPORT_NAMESPACE])) { + read_as = WHEEL_META_IMPORT_NAMESPACE; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_PROVIDES_DIST])) { + read_as = WHEEL_META_PROVIDES_DIST; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_PROVIDES_EXTRA])) { + read_as = WHEEL_META_PROVIDES_EXTRA; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_PLATFORM])) { + read_as = WHEEL_META_PLATFORM; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_SUPPORTED_PLATFORM])) { + read_as = WHEEL_META_SUPPORTED_PLATFORM; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_KEYWORDS])) { + read_as = WHEEL_META_KEYWORDS; + } else if (!strcasecmp(key,WHEEL_META_KEY[WHEEL_META_DYNAMIC])) { + read_as = WHEEL_META_DYNAMIC; + } else { + read_as = WHEEL_KEY_UNKNOWN; + } } else { - // not a wheel file. nothing to do - continue; + value = strdup(line); + if (!value) { + // memory error + return -1; + } } - size_t match = 0; - size_t pattern_count = 0; - for (; to_match[pattern_count] != NULL; pattern_count++) { - if (strstr(filename, to_match[pattern_count])) { - match++; + switch (read_as) { + case WHEEL_META_METADATA_VERSION: { + pkg->metadata_version = strdup(value); + if (!pkg->metadata_version) { + // memory error + return -1; + } + break; + } + case WHEEL_META_NAME: { + pkg->name = strdup(value); + if (!pkg->name) { + // memory error + return -1; + } + break; + } + case WHEEL_META_VERSION: { + pkg->version = strdup(value); + if (!pkg->version) { + // memory error + return -1; + } + break; + } + case WHEEL_META_SUMMARY: { + pkg->summary = strdup(value); + if (!pkg->summary) { + // memory error + return -1; + } + break; + } + case WHEEL_META_AUTHOR: { + if (!pkg->author) { + pkg->author = strlist_init(); + if (!pkg->author) { + // memory error + return -1; + } + } + strlist_append_tokenize(pkg->author, value, ","); + break; + } + case WHEEL_META_AUTHOR_EMAIL: { + if (!pkg->author_email) { + pkg->author_email = strlist_init(); + if (!pkg->author_email) { + // memory error + return -1; + } + } + strlist_append_tokenize(pkg->author_email, value, ","); + break; + } + case WHEEL_META_MAINTAINER: { + if (!pkg->maintainer) { + pkg->maintainer = strlist_init(); + if (!pkg->maintainer) { + // memory error + return -1; + } + } + strlist_append_tokenize(pkg->maintainer, value, ","); + break; + } + case WHEEL_META_MAINTAINER_EMAIL: { + if (!pkg->maintainer_email) { + pkg->maintainer_email = strlist_init(); + if (!pkg->maintainer_email) { + // memory error + return -1; + } + } + strlist_append_tokenize(pkg->maintainer_email, value, ","); + break; + } + case WHEEL_META_LICENSE: { + if (!reading_multiline) { + pkg->license = strdup(value); + if (!pkg->license) { + // memory error + return -1; + } + } else { + if (pkg->license) { + consume_append(&pkg->license, line, "\r\n"); + } else { + // previously unhandled memory error + return -1; + } + } + break; + } + case WHEEL_META_LICENSE_EXPRESSION: { + pkg->license_expression = strdup(value); + if (!pkg->license_expression) { + // memory error + return -1; + } + break; + } + case WHEEL_META_HOME_PAGE: { + pkg->home_page = strdup(value); + if (!pkg->home_page) { + // memory error + return -1; + } + break; + } + case WHEEL_META_DOWNLOAD_URL: + pkg->download_url = strdup(value); + if (!pkg->download_url) { + // memory error + return -1; + } + break; + case WHEEL_META_PROJECT_URL: { + if (!pkg->project_url) { + pkg->project_url = strlist_init(); + if (!pkg->project_url) { + // memory_error + return -1; + } + } + strlist_append(&pkg->project_url, value); + break; + } + case WHEEL_META_CLASSIFIER: { + if (!pkg->classifier) { + pkg->classifier = strlist_init(); + if (!pkg->classifier) { + // memory error + return -1; + } + } + strlist_append(&pkg->classifier, value); + break; + } + case WHEEL_META_REQUIRES_PYTHON: { + if (!pkg->requires_python) { + pkg->requires_python = strlist_init(); + if (!pkg->requires_python) { + // memory error + return -1; + } + } + strlist_append(&pkg->requires_python, value); + break; + } + case WHEEL_META_REQUIRES_EXTERNAL: { + if (!pkg->requires_external) { + pkg->requires_external = strlist_init(); + if (!pkg->requires_external) { + // memory error + return -1; + } + } + strlist_append(&pkg->requires_external, value); + break; + } + case WHEEL_META_DESCRIPTION_CONTENT_TYPE: { + pkg->description_content_type = strdup(value); + if (!pkg->description_content_type) { + // memory error + return -1; + } + break; + } + case WHEEL_META_LICENSE_FILE: { + if (!pkg->license_file) { + pkg->license_file = strlist_init(); + if (!pkg->license_file) { + // memory error + return -1; + } + } + strlist_append(&pkg->license_file, value); + break; + } + case WHEEL_META_IMPORT_NAME: { + if (!pkg->import_name) { + pkg->import_name = strlist_init(); + if (!pkg->import_name) { + // memory error + return -1; + } + } + strlist_append(&pkg->import_name, value); + break; + } + case WHEEL_META_IMPORT_NAMESPACE: { + if (!pkg->import_namespace) { + pkg->import_namespace = strlist_init(); + if (!pkg->import_namespace) { + // memory error + return -1; + } + } + strlist_append(&pkg->import_namespace, value); + break; + } + case WHEEL_META_REQUIRES_DIST: { + if (!pkg->requires_dist) { + pkg->requires_dist = strlist_init(); + if (!pkg->requires_dist) { + // memory error + return -1; + } + } + if (reading_extra) { + if (strrchr(value, ';')) { + *strrchr(value, ';') = 0; + } + if (!current_extra) { + reading_extra = 0; + break; + } + if (!current_extra->requires_dist) { + current_extra->requires_dist = strlist_init(); + if (!current_extra->requires_dist) { + // memory error + return -1; + } + } + strlist_append(¤t_extra->requires_dist, value); + } else { + strlist_append(&pkg->requires_dist, value); + reading_extra = 0; + } + break; + } + case WHEEL_META_PROVIDES: { + if (!pkg->provides) { + pkg->provides = strlist_init(); + if (!pkg->provides) { + // memory error + return -1; + } + } + strlist_append(&pkg->provides, value); + break; + } + case WHEEL_META_PROVIDES_DIST: { + if (!pkg->provides_dist) { + pkg->provides_dist = strlist_init(); + if (!pkg->provides_dist) { + // memory error + return -1; + } + } + strlist_append(&pkg->provides_dist, value); + break; + } + case WHEEL_META_PROVIDES_EXTRA: + pkg->provides_extra[provides_extra_i] = calloc(1, sizeof(*pkg->provides_extra[0])); + if (!pkg->provides_extra[provides_extra_i]) { + // memory error + return -1; + } + current_extra = pkg->provides_extra[provides_extra_i]; + if (!current_extra) { + SYSERROR("%s", "provides_extra array cannot have a NULL record"); + return -1; + } + current_extra->target = strdup(value); + if (!current_extra->target) { + // memory error + return -1; + } + reading_extra = 1; + provides_extra_i++; + break; + case WHEEL_META_PLATFORM:{ + if (!pkg->platform) { + pkg->platform = strlist_init(); + if (!pkg->platform) { + // memory error + return -1; + } + } + strlist_append(&pkg->platform, value); + break; + } + case WHEEL_META_SUPPORTED_PLATFORM: { + if (!pkg->supported_platform) { + pkg->supported_platform = strlist_init(); + if (!pkg->supported_platform) { + // memory error + return -1; + } + } + strlist_append(&pkg->supported_platform, value); + break; } + case WHEEL_META_KEYWORDS: { + if (!pkg->keywords) { + pkg->keywords = strlist_init(); + if (!pkg->keywords) { + // memory error + return -1; + } + } + strlist_append_tokenize(pkg->keywords, value, ","); + break; + } + case WHEEL_META_DYNAMIC: { + if (!pkg->dynamic) { + pkg->dynamic = strlist_init(); + if (!pkg->dynamic) { + // memory error + return -1; + } + } + strlist_append(&pkg->dynamic, value); + break; + } + case WHEEL_META_DESCRIPTION: { + // reading_description will never be reset to zero + reading_description = 1; + if (!pkg->description) { + pkg->description = malloc((size_t) base_description_len + 1); + if (!pkg->description) { + return -1; + } + len_description = snprintf(pkg->description, (size_t) base_description_len, "%s\n", line); + } else { + const int next_len = snprintf(NULL, 0, "%s\n%s\n", pkg->description, line); + if (next_len + 1 > base_description_len) { + base_description_len *= 2; + char *tmp = realloc(pkg->description, (size_t) base_description_len + 1); + if (!tmp) { + // memory error + guard_free(pkg->description); + return -1; + } + pkg->description = tmp; + } + len_description += snprintf(pkg->description + len_description, (size_t) next_len + 1, "%s\n", line); + } + break; + } + case WHEEL_KEY_UNKNOWN: + default: + SYSWARN("unhandled metadata key on line %zu:\nbuffer contents: '%s'", i, value); + break; + } + guard_free(key); + guard_free(value); + guard_array_free(pair); + if (reading_multiline) { + guard_free(value); } + } + guard_strlist_free(&lines); + + return 0; +} - if (!startswith(rec->d_name, name)) { +int wheel_get_file_contents(const char *wheelfile, const char *filename, char **contents) { + int status = 0; + int err = 0; + struct zip_stat archive_info; + zip_t *archive = zip_open(wheelfile, 0, &err); + if (!archive) { + return status; + } + + zip_stat_init(&archive_info); + for (size_t i = 0; zip_stat_index(archive, i, 0, &archive_info) >= 0; i++) { + char internal_path[1024] = {0}; + snprintf(internal_path, sizeof(internal_path), "%s", filename); + const int match = fnmatch(internal_path, archive_info.name, 0); + if (match == FNM_NOMATCH) { continue; } + if (match < 0) { + goto GWM_FAIL; + } - if (match_mode == WHEEL_MATCH_EXACT && match != pattern_count) { - continue; + zip_file_t *handle = zip_fopen_index(archive, i, 0); + if (!handle) { + goto GWM_FAIL; } - result = calloc(1, sizeof(*result)); - if (!result) { - SYSERROR("Unable to allocate %zu bytes for wheel struct", sizeof(*result)); - closedir(dp); - return NULL; - } - - result->path_name = realpath(package_path, NULL); - if (!result->path_name) { - SYSERROR("Unable to resolve absolute path to %s: %s", filename, strerror(errno)); - wheel_free(&result); - closedir(dp); - return NULL; - } - result->file_name = strdup(rec->d_name); - if (!result->file_name) { - SYSERROR("Unable to allocate bytes for %s: %s", rec->d_name, strerror(errno)); - wheel_free(&result); - closedir(dp); - return NULL; - } - - size_t parts_total; - char **parts = split(filename, "-", 0); - if (!parts) { - // This shouldn't happen unless a wheel file is present in the - // directory with a malformed file name, or we've managed to - // exhaust the system's memory - SYSERROR("%s has no '-' separators! (Delete this file and try again)", filename); - wheel_free(&result); - closedir(dp); - return NULL; - } - - for (parts_total = 0; parts[parts_total] != NULL; parts_total++) {} - if (parts_total == 5) { - // no build tag - result->distribution = strdup(parts[0]); - result->version = strdup(parts[1]); - result->build_tag = NULL; - result->python_tag = strdup(parts[2]); - result->abi_tag = strdup(parts[3]); - result->platform_tag = strdup(parts[4]); - } else if (parts_total == 6) { - // has build tag - result->distribution = strdup(parts[0]); - result->version = strdup(parts[1]); - result->build_tag = strdup(parts[2]); - result->python_tag = strdup(parts[3]); - result->abi_tag = strdup(parts[4]); - result->platform_tag = strdup(parts[5]); - } else { - SYSERROR("Unknown wheel name format: %s. Expected 5 or 6 strings " - "separated by '-', but got %zu instead", filename, parts_total); - guard_array_free(parts); - wheel_free(&result); - closedir(dp); - return NULL; - } - guard_array_free(parts); + *contents = calloc(archive_info.size + 1, sizeof(**contents)); + if (!*contents) { + zip_fclose(handle); + goto GWM_FAIL; + } + + if (zip_fread(handle, *contents, archive_info.size) < 0) { + zip_fclose(handle); + guard_free(*contents); + goto GWM_FAIL; + } + zip_fclose(handle); break; } - closedir(dp); + + goto GWM_END; + GWM_FAIL: + status = -1; + + GWM_END: + zip_close(archive); + return status; +} + +static int wheel_metadata_get(const struct Wheel *pkg, const char *wheel_filename) { + char *data = NULL; + if (wheel_get_file_contents(wheel_filename, "*.dist-info/METADATA", &data)) { + return -1; + } + char *data_orig = data; + const ssize_t result = wheel_parse_metadata(pkg->metadata, data); + guard_free(data_orig); + return (int) result; +} + +static struct WheelValue wheel_data_dump(const struct Wheel *pkg, const ssize_t key) { + struct WheelValue result; + result.type = WHEELVAL_STR; + result.count = 0; + switch (key) { + case WHEEL_DIST_VERSION: + result.data = pkg->wheel_version; + break; + case WHEEL_DIST_GENERATOR: + result.data = pkg->generator; + break; + case WHEEL_DIST_ZIP_SAFE: + result.data = pkg->zip_safe ? "True" : "False"; + break; + case WHEEL_DIST_ROOT_IS_PURELIB: + result.data = pkg->root_is_pure_lib ? "True" : "False"; + break; + case WHEEL_DIST_TOP_LEVEL: + result.type = WHEELVAL_STRLIST; + result.data = pkg->top_level; + break; + case WHEEL_DIST_TAG: + result.type = WHEELVAL_STRLIST; + result.data = pkg->tag; + break; + case WHEEL_DIST_RECORD: + result.type = WHEELVAL_OBJ_RECORD; + result.data = pkg->record; + result.count = pkg->num_record; + break; + case WHEEL_DIST_ENTRY_POINT: + result.type = WHEELVAL_OBJ_ENTRY_POINT; + result.data = pkg->entry_point; + result.count = pkg->num_entry_point; + break; + default: + result.data = NULL; + result.type = WHEEL_KEY_UNKNOWN; + break; + } + + switch (result.type) { + case WHEELVAL_STR: + result.count = result.data != NULL ? strlen(result.data) : 0; + break; + case WHEELVAL_STRLIST: + result.count = result.data != NULL ? strlist_count(result.data) : 0; + break; + default: + break; + } + + return result; +} + +static struct WheelValue wheel_metadata_dump(const struct Wheel *pkg, const ssize_t key) { + const struct WheelMetadata *meta = pkg->metadata; + struct WheelValue result; + result.type = WHEELVAL_STR; + result.count = 0; + switch (key) { + case WHEEL_META_METADATA_VERSION: + result.data = meta->metadata_version; + break; + case WHEEL_META_NAME: + result.data = meta->name; + break; + case WHEEL_META_VERSION: + result.data = meta->version; + break; + case WHEEL_META_SUMMARY: + result.data = meta->summary; + break; + case WHEEL_META_AUTHOR: + result.type = WHEELVAL_STRLIST; + result.data = meta->author; + break; + case WHEEL_META_AUTHOR_EMAIL: + result.type = WHEELVAL_STRLIST; + result.data = meta->author_email; + break; + case WHEEL_META_MAINTAINER: + result.type = WHEELVAL_STRLIST; + result.data = meta->maintainer; + break; + case WHEEL_META_MAINTAINER_EMAIL: + result.type = WHEELVAL_STRLIST; + result.data = meta->maintainer_email; + break; + case WHEEL_META_LICENSE: + result.data = meta->license; + break; + case WHEEL_META_HOME_PAGE: + result.data = meta->home_page; + break; + case WHEEL_META_DOWNLOAD_URL: + result.data = meta->download_url; + break; + case WHEEL_META_PROJECT_URL: + result.type = WHEELVAL_STRLIST; + result.data = meta->project_url; + break; + case WHEEL_META_CLASSIFIER: + result.type = WHEELVAL_STRLIST; + result.data = meta->classifier; + break; + case WHEEL_META_REQUIRES_PYTHON: + result.type = WHEELVAL_STRLIST; + result.data = meta->requires_python; + break; + case WHEEL_META_DESCRIPTION_CONTENT_TYPE: + result.data = meta->description_content_type; + break; + case WHEEL_META_LICENSE_FILE: + result.type = WHEELVAL_STRLIST; + result.data = meta->license_file; + break; + case WHEEL_META_LICENSE_EXPRESSION: + result.data = meta->license_expression; + break; + case WHEEL_META_IMPORT_NAME: + result.type = WHEELVAL_STRLIST; + result.data = meta->import_name; + break; + case WHEEL_META_IMPORT_NAMESPACE: + result.type = WHEELVAL_STRLIST; + result.data = meta->import_namespace; + break; + case WHEEL_META_REQUIRES_DIST: + result.type = WHEELVAL_STRLIST; + result.data = meta->requires_dist; + break; + case WHEEL_META_PROVIDES_DIST: + result.type = WHEELVAL_STRLIST; + result.data = meta->provides_dist; + break; + case WHEEL_META_PROVIDES_EXTRA: + result.type = WHEELVAL_OBJ_EXTRA; + result.data = (struct WheelMetadata_ProvidesExtra *) meta->provides_extra; + result.count = result.data != NULL ? (size_t) ((struct WheelMetadata_ProvidesExtra *) result.data)->count : 0; + break; + case WHEEL_META_OBSOLETES: + result.type = WHEELVAL_STRLIST; + result.data = meta->obsoletes; + break; + case WHEEL_META_OBSOLETES_DIST: + result.type = WHEELVAL_STRLIST; + result.data = meta->obsoletes_dist; + break; + case WHEEL_META_DESCRIPTION: + result.type = WHEELVAL_STR; + result.data = meta->description; + break; + case WHEEL_META_PLATFORM: + result.type = WHEELVAL_STRLIST; + result.data = meta->platform; + break; + case WHEEL_META_SUPPORTED_PLATFORM: + result.type = WHEELVAL_STRLIST; + result.data = meta->supported_platform; + break; + case WHEEL_META_KEYWORDS: + result.type = WHEELVAL_STRLIST; + result.data = meta->keywords; + break; + case WHEEL_META_DYNAMIC: + result.type = WHEELVAL_STRLIST; + result.data = meta->dynamic; + break; + case WHEEL_KEY_UNKNOWN: + default: + result.data = NULL; + break; + } + + switch (result.type) { + case WHEELVAL_STR: + result.count = result.data != NULL ? strlen(result.data) : 0; + break; + case WHEELVAL_STRLIST: + result.count = result.data != NULL ? strlist_count(result.data) : 0; + break; + default: + break; + } + return result; } -void wheel_free(struct Wheel **wheel) { - struct Wheel *w = (*wheel); - guard_free(w->path_name); - guard_free(w->file_name); - guard_free(w->distribution); - guard_free(w->version); - guard_free(w->build_tag); - guard_free(w->python_tag); - guard_free(w->abi_tag); - guard_free(w->python_tag); - guard_free(w->platform_tag); - guard_free(w); +static ssize_t get_key_index(const char **arr, const char *key) { + for (ssize_t i = 0; arr[i] != NULL; i++) { + if (strcmp(arr[i], key) == 0) { + return i; + } + } + return -1; } + +const char *wheel_get_key_by_id(const int from, const ssize_t id) { + if (from == WHEEL_FROM_DIST) { + if (id >= 0 && id < WHEEL_DIST_END_ENUM) { + return WHEEL_DIST_KEY[id]; + } + } + if (from == WHEEL_FROM_METADATA) { + if (id >= 0 && id < WHEEL_META_END_ENUM) { + return WHEEL_META_KEY[id]; + } + } + return NULL; +} + +struct WheelValue wheel_get_value_by_name(const struct Wheel *pkg, const int from, const char *key) { + struct WheelValue result = {0}; + ssize_t id; + + if (from == WHEEL_FROM_DIST) { + id = get_key_index(WHEEL_DIST_KEY, key); + result = wheel_data_dump(pkg, id); + } else if (from == WHEEL_FROM_METADATA) { + id = get_key_index(WHEEL_META_KEY, key); + result = wheel_metadata_dump(pkg, id); + } else { + result.data = NULL; + result.type = WHEEL_KEY_UNKNOWN; + } + + return result; +} + +struct WheelValue wheel_get_value_by_id(const struct Wheel *pkg, const int from, const ssize_t id) { + struct WheelValue result = {0}; + + if (from == WHEEL_FROM_DIST) { + result = wheel_data_dump(pkg, id); + } else if (from == WHEEL_FROM_METADATA) { + result = wheel_metadata_dump(pkg, id); + } else { + result.data = NULL; + result.type = -1; + } + + return result; +} + +void wheel_record_free(struct WheelRecord **record) { + guard_free((*record)->filename); + guard_free((*record)->checksum); + guard_free(*record); +} + +void wheel_entry_point_free(struct WheelEntryPoint **entry) { + guard_free((*entry)->name); + guard_free((*entry)->function); + guard_free((*entry)->type); + guard_free(*entry); +} + +void wheel_metadata_free(struct WheelMetadata *meta) { + if (!meta) { + return; + } + guard_free(meta->license); + guard_free(meta->license_expression); + guard_free(meta->version); + guard_free(meta->name); + guard_free(meta->description); + guard_free(meta->metadata_version); + guard_free(meta->summary); + guard_free(meta->description_content_type); + guard_free(meta->home_page); + guard_free(meta->download_url); + + guard_strlist_free(&meta->author_email); + guard_strlist_free(&meta->author); + guard_strlist_free(&meta->maintainer); + guard_strlist_free(&meta->maintainer_email); + guard_strlist_free(&meta->requires_python); + guard_strlist_free(&meta->requires_external); + guard_strlist_free(&meta->project_url); + guard_strlist_free(&meta->classifier); + guard_strlist_free(&meta->requires_dist); + guard_strlist_free(&meta->keywords); + guard_strlist_free(&meta->license_file); + if (meta->provides_extra) { + for (size_t i = 0; meta->provides_extra[i] != NULL; i++) { + guard_free(meta->provides_extra[i]->target); + guard_strlist_free(&meta->provides_extra[i]->requires_dist); + guard_free(meta->provides_extra[i]); + } + guard_free(meta->provides_extra); + } + + guard_free(meta); +} + +void wheel_package_free(struct Wheel **pkg) { + if (!*pkg) { + return; + } + guard_free((*pkg)->wheel_version); + guard_free((*pkg)->generator); + guard_free((*pkg)->root_is_pure_lib); + wheel_metadata_free((*pkg)->metadata); + + guard_strlist_free(&(*pkg)->tag); + guard_strlist_free(&(*pkg)->top_level); + for (size_t i = 0; (*pkg)->record && (*pkg)->record[i] != NULL; i++) { + wheel_record_free(&(*pkg)->record[i]); + } + guard_array_n_free((*pkg)->record, (*pkg)->num_record); + + for (size_t i = 0; (*pkg)->entry_point && (*pkg)->entry_point[i] != NULL; i++) { + wheel_entry_point_free(&(*pkg)->entry_point[i]); + } + guard_free((*pkg)->entry_point); + guard_free((*pkg)); +} + +int wheel_get_top_level(struct Wheel *pkg, const char *filename) { + char *data = NULL; + if (wheel_get_file_contents(filename, "*.dist-info/top_level.txt", &data)) { + guard_free(data); + return -1; + } + if (!pkg->top_level) { + pkg->top_level = strlist_init(); + } + strlist_append_tokenize(pkg->top_level, data, "\r\n"); + guard_free(data); + return 0; +} + +int wheel_get_zip_safe(struct Wheel *pkg, const char *filename) { + char *data = NULL; + const int exists = wheel_get_file_contents(filename, "*.dist-info/zip-safe", &data) == 0; + guard_free(data); + + pkg->zip_safe = 0; + if (exists) { + pkg->zip_safe = 1; + } + + return 0; +} + +int wheel_get_records(struct Wheel *pkg, const char *filename) { + char *data = NULL; + const int exists = wheel_get_file_contents(filename, "*.dist-info/RECORD", &data) == 0; + + if (!exists) { + guard_free(data); + return 1; + } + + const size_t records_initial_count = 2; + pkg->record = calloc(records_initial_count, sizeof(*pkg->record)); + if (!pkg->record) { + guard_free(data); + return 1; + } + size_t records_count = 0; + + const char *token = NULL; + char *data_orig = data; + while ((token = strsep(&data, "\r\n")) != NULL) { + if (!strlen(token)) { + continue; + } + + pkg->record[records_count] = calloc(1, sizeof(*pkg->record[0])); + if (!pkg->record[records_count]) { + return -1; + } + + struct WheelRecord *record = pkg->record[records_count]; + for (size_t x = 0; x < 3; x++) { + const char *next_comma = strpbrk(token, ","); + if (next_comma) { + if (x == 0) { + record->filename = strndup(token, (size_t) (next_comma - token)); + } else if (x == 1) { + record->checksum = strndup(token, (size_t) (next_comma - token)); + } + token = next_comma + 1; + } else { + record->size = (size_t) strtol(token, NULL, 10); + } + } + records_count++; + + struct WheelRecord **tmp = realloc(pkg->record, (records_count + 2) * sizeof(*pkg->record)); + if (tmp == NULL) { + guard_free(data); + return -1; + } + pkg->record = tmp; + pkg->record[records_count + 1] = NULL; + } + + pkg->num_record = records_count; + guard_free(data_orig); + return 0; +} + +int wheel_get(struct Wheel **pkg, const char *filename) { + char *data = NULL; + if (wheel_get_file_contents(filename, "*.dist-info/WHEEL", &data) < 0) { + return -1; + } + const ssize_t result = wheel_parse_wheel(*pkg, data); + guard_free(data); + return (int) result; +} + +int wheel_get_entry_point(struct Wheel *pkg, const char *filename) { + char *data = NULL; + if (wheel_get_file_contents(filename, "*.dist-info/entry_points.txt", &data) < 0) { + return -1; + } + + struct StrList *lines = strlist_init(); + if (!lines) { + goto GEP_FAIL; + } + strlist_append_tokenize(lines, data, "\r\n"); + + const size_t line_count = strlist_count(lines); + size_t usable_lines = line_count; + for (size_t i = 0; i < line_count; i++) { + const char *item = strlist_item(lines, i); + if (isempty((char *) item) || item[0] == '[') { + usable_lines--; + } + } + pkg->num_entry_point = usable_lines; + + pkg->entry_point = calloc(pkg->num_entry_point + 1, sizeof(*pkg->entry_point)); + if (!pkg->entry_point) { + goto GEP_FAIL; + } + + for (size_t i = 0; i < pkg->num_entry_point; i++) { + pkg->entry_point[i] = calloc(1, sizeof(*pkg->entry_point[0])); + if (!pkg->entry_point[i]) { + goto GEP_FAIL; + } + } + + size_t x = 0; + char section[255] = {0}; + for (size_t i = 0; i < line_count; i++) { + const char *item = strlist_item(lines, i); + if (isempty((char *) item)) { + continue; + } + + if (strpbrk(item, "[")) { + const size_t start = strcspn((char *) item, "[") + 1; + if (start) { + const size_t len = strcspn((char *) item, "]"); + strncpy(section, item + start, len - start); + section[len - start] = '\0'; + continue; + } + } + + pkg->entry_point[x]->type = strdup(section); + if (!pkg->entry_point[x]->type) { + goto GEP_FAIL; + } + + char **pair = split((char *) item, "=", 1); + if (!pair) { + wheel_entry_point_free(&pkg->entry_point[x]); + goto GEP_FAIL; + } + + pkg->entry_point[x]->name = strdup(strip(pair[0])); + if (!pkg->entry_point[x]->name) { + wheel_entry_point_free(&pkg->entry_point[x]); + guard_array_free(pair); + goto GEP_FAIL; + } + + pkg->entry_point[x]->function = strdup(lstrip(pair[1])); + if (!pkg->entry_point[x]->function) { + wheel_entry_point_free(&pkg->entry_point[x]); + guard_array_free(pair); + goto GEP_FAIL; + } + guard_array_free(pair); + x++; + } + pkg->num_entry_point = x; + guard_strlist_free(&lines); + guard_free(data); + return 0; + + GEP_FAIL: + if (pkg->entry_point) { + guard_array_n_free(pkg->entry_point, pkg->num_entry_point); + } + guard_strlist_free(&lines); + guard_free(data); + return -1; +} + +int wheel_value_error(struct WheelValue const *val) { + if (val) { + if (val->type < 0 && val->data == NULL) { + return 1; + } + } + return 0; +} + +int wheel_show_info(const struct Wheel *wheel) { + printf("WHEEL INFO\n\n"); + for (ssize_t i = 0; i < WHEEL_DIST_END_ENUM; i++) { + const char *key = wheel_get_key_by_id(WHEEL_FROM_DIST, i); + if (!key) { + SYSERROR("wheel_get_key_by_id(%zi) failed", i); + return -1; + } + + printf("%s: ", key); + fflush(stdout); + const struct WheelValue dist = wheel_get_value_by_id(wheel, WHEEL_FROM_DIST, i); + if (wheel_value_error(&dist)) { + SYSERROR("wheel_get_value_by_id(%zi) failed", i); + return -1; + } + switch (dist.type) { + case WHEELVAL_STR: { + char *s = dist.data; + if (s != NULL && !isempty(s)) { + printf("%s\n", s); + } else { + printf("[N/A]\n"); + } + break; + } + case WHEELVAL_STRLIST: { + struct StrList *list = dist.data; + if (list) { + printf("\n"); + for (size_t x = 0; x < strlist_count(list); x++) { + const char *item = strlist_item(list, x); + printf(" %s\n", item); + } + } else { + printf("[N/A]\n"); + } + break; + } + case WHEELVAL_OBJ_RECORD: { + struct WheelRecord **record = dist.data; + if (record && *record) { + printf("\n"); + for (size_t x = 0; x < dist.count; x++) { + printf(" [%zu] %s (size: %zu bytes, checksum: %s)\n", x, wheel->record[x]->filename, wheel->record[x]->size, strlen(wheel->record[x]->checksum) ? wheel->record[x]->checksum : "N/A"); + } + } else { + printf("[N/A]\n"); + } + break; + } + case WHEELVAL_OBJ_ENTRY_POINT: { + struct WheelEntryPoint **entry = dist.data; + if (entry && *entry) { + printf("\n"); + for (size_t x = 0; x < dist.count; x++) { + printf(" [%zu] type: %s, name: %s, function: %s\n", x, entry[x]->type, entry[x]->name, entry[x]->function); + } + } else { + printf("[N/A]\n"); + } + break; + } + default: + printf("[no handler]\n"); + break; + } + + } + + printf("\nPACKAGE INFO\n\n"); + for (ssize_t i = 0; i < WHEEL_META_END_ENUM; i++) { + const char *key = wheel_get_key_by_id(WHEEL_FROM_METADATA, i); + if (!key) { + SYSERROR("wheel_get_key_by_id(%zi) failed", i); + return -1; + } + printf("%s: ", key); + fflush(stdout); + + const struct WheelValue pkg = wheel_get_value_by_id(wheel, WHEEL_FROM_METADATA, i); + if (wheel_value_error(&pkg)) { + SYSERROR("wheel_get_value_by_id(%zi) failed", i); + return -1; + } + switch (pkg.type) { + case WHEELVAL_STR: { + char *s = pkg.data; + if (s != NULL && !isempty(s)) { + printf("%s\n", s); + } else { + printf("[N/A]\n"); + } + break; + } + case WHEELVAL_STRLIST: { + struct StrList *list = pkg.data; + if (list) { + printf("\n"); + for (size_t x = 0; x < strlist_count(list); x++) { + const char *item = strlist_item(list, x); + printf(" %s\n", item); + } + } else { + printf("[N/A]\n"); + } + break; + } + case WHEELVAL_OBJ_EXTRA: { + const struct WheelMetadata_ProvidesExtra **extra = pkg.data; + printf("\n"); + if (*extra) { + for (size_t x = 0; extra[x] != NULL; x++) { + printf(" + %s\n", extra[x]->target); + for (size_t z = 0; z < strlist_count(extra[x]->requires_dist); z++) { + const char *item = strlist_item(extra[x]->requires_dist, z); + printf(" `- %s\n", item); + } + } + } else { + printf("[N/A]\n"); + } + break; + } + default: + break; + } + } + return 0; +} + +int wheel_package(struct Wheel **pkg, const char *filename) { + int status = 0; + if (!filename) { + status = WHEEL_PACKAGE_E_FILENAME; + goto fail; + } + if (!*pkg) { + *pkg = calloc(1, sizeof(**pkg)); + if (!*pkg) { + status = WHEEL_PACKAGE_E_ALLOC; + goto fail; + } + + (*pkg)->metadata = calloc(1, sizeof(*(*pkg)->metadata)); + if (!(*pkg)->metadata) { + status = WHEEL_PACKAGE_E_ALLOC; + goto fail; + } + } + if (wheel_get(pkg, filename) < 0) { + status = WHEEL_PACKAGE_E_GET; + goto fail; + } + if (wheel_metadata_get(*pkg, filename) < 0) { + status = WHEEL_PACKAGE_E_GET_METADATA; + goto fail; + } + if (wheel_get_top_level(*pkg, filename) < 0) { + status = WHEEL_PACKAGE_E_GET_TOP_LEVEL; + goto fail; + } + if (wheel_get_records(*pkg, filename) < 0) { + status = WHEEL_PACKAGE_E_GET_RECORDS; + goto fail; + } + if (wheel_get_entry_point(*pkg, filename) < 0) { + status = WHEEL_PACKAGE_E_GET_ENTRY_POINT; + goto fail; + } + + // Optional marker + wheel_get_zip_safe(*pkg, filename); + + status = WHEEL_PACKAGE_E_SUCCESS; + return status; + + fail: + wheel_package_free(pkg); + return status; +} + + diff --git a/src/lib/core/wheelinfo.c b/src/lib/core/wheelinfo.c new file mode 100644 index 0000000..9d8a6af --- /dev/null +++ b/src/lib/core/wheelinfo.c @@ -0,0 +1,133 @@ +#include "wheelinfo.h" + +struct WheelInfo *wheelinfo_get(const char *basepath, const char *name, char *to_match[], unsigned match_mode) { + struct dirent *rec; + struct WheelInfo *result = NULL; + char package_path[PATH_MAX]; + char package_name[NAME_MAX]; + + strncpy(package_name, name, sizeof(package_name) - 1); + package_name[sizeof(package_name) - 1] = '\0'; + tolower_s(package_name); + snprintf(package_path, sizeof(package_path), "%s/%s", basepath, package_name); + + DIR *dp = opendir(package_path); + if (!dp) { + return NULL; + } + + while ((rec = readdir(dp)) != NULL) { + if (!strcmp(rec->d_name, ".") || !strcmp(rec->d_name, "..")) { + continue; + } + + char filename[NAME_MAX]; + strncpy(filename, rec->d_name, sizeof(filename) - 1); + filename[sizeof(filename) - 1] = '\0'; + + char *ext = strstr(filename, ".whl"); + if (ext) { + *ext = '\0'; + } else { + // not a wheel file. nothing to do + continue; + } + + size_t match = 0; + size_t pattern_count = 0; + for (; to_match[pattern_count] != NULL; pattern_count++) { + if (strstr(filename, to_match[pattern_count])) { + match++; + } + } + + if (!startswith(rec->d_name, name)) { + continue; + } + + if (match_mode == WHEEL_MATCH_EXACT && match != pattern_count) { + continue; + } + + result = calloc(1, sizeof(*result)); + if (!result) { + SYSERROR("Unable to allocate %zu bytes for wheel struct", sizeof(*result)); + closedir(dp); + return NULL; + } + + result->path_name = realpath(package_path, NULL); + if (!result->path_name) { + SYSERROR("Unable to resolve absolute path to %s: %s", filename, strerror(errno)); + wheelinfo_free(&result); + closedir(dp); + return NULL; + } + result->file_name = strdup(rec->d_name); + if (!result->file_name) { + SYSERROR("Unable to allocate bytes for %s: %s", rec->d_name, strerror(errno)); + wheelinfo_free(&result); + closedir(dp); + return NULL; + } + + size_t parts_total; + char **parts = split(filename, "-", 0); + if (!parts) { + // This shouldn't happen unless a wheel file is present in the + // directory with a malformed file name, or we've managed to + // exhaust the system's memory + SYSERROR("%s has no '-' separators! (Delete this file and try again)", filename); + wheelinfo_free(&result); + closedir(dp); + return NULL; + } + + for (parts_total = 0; parts[parts_total] != NULL; parts_total++) {} + if (parts_total == 5) { + // no build tag + result->distribution = strdup(parts[0]); + result->version = strdup(parts[1]); + result->build_tag = NULL; + result->python_tag = strdup(parts[2]); + result->abi_tag = strdup(parts[3]); + result->platform_tag = strdup(parts[4]); + } else if (parts_total == 6) { + // has build tag + result->distribution = strdup(parts[0]); + result->version = strdup(parts[1]); + result->build_tag = strdup(parts[2]); + result->python_tag = strdup(parts[3]); + result->abi_tag = strdup(parts[4]); + result->platform_tag = strdup(parts[5]); + } else { + SYSERROR("Unknown wheel name format: %s. Expected 5 or 6 strings " + "separated by '-', but got %zu instead", filename, parts_total); + guard_array_free(parts); + wheelinfo_free(&result); + closedir(dp); + return NULL; + } + guard_array_free(parts); + break; + } + closedir(dp); + return result; +} + +void wheelinfo_free(struct WheelInfo **wheel) { + struct WheelInfo *w = (*wheel); + if (!w) { + return; + } + guard_free(w->path_name); + guard_free(w->file_name); + guard_free(w->distribution); + guard_free(w->version); + guard_free(w->build_tag); + guard_free(w->python_tag); + guard_free(w->abi_tag); + guard_free(w->python_tag); + guard_free(w->platform_tag); + guard_free(w); +} diff --git a/src/lib/delivery/CMakeLists.txt b/src/lib/delivery/CMakeLists.txt index 78ed20f..559b2dc 100644 --- a/src/lib/delivery/CMakeLists.txt +++ b/src/lib/delivery/CMakeLists.txt @@ -1,4 +1,5 @@ add_library(stasis_delivery STATIC + delivery_export.c delivery_postprocess.c delivery_conda.c delivery_docker.c diff --git a/src/lib/delivery/delivery.c b/src/lib/delivery/delivery.c index d480ab4..dc9e2ce 100644 --- a/src/lib/delivery/delivery.c +++ b/src/lib/delivery/delivery.c @@ -1,8 +1,203 @@ #include "delivery.h" +#include "conda.h" + +struct Delivery *delivery_duplicate(struct Delivery *ctx) { + struct Delivery *result = calloc(1, sizeof(*result)); + if (!result) { + return NULL; + } + // Conda + result->conda.conda_packages = strlist_copy(ctx->conda.conda_packages); + result->conda.conda_packages_defer = strlist_copy(ctx->conda.conda_packages_defer); + result->conda.conda_packages_purge = strlist_copy(ctx->conda.conda_packages_purge); + result->conda.pip_packages = strlist_copy(ctx->conda.pip_packages); + result->conda.pip_packages_defer = strlist_copy(ctx->conda.pip_packages_defer); + result->conda.pip_packages_purge = strlist_copy(ctx->conda.pip_packages_purge); + result->conda.wheels_packages = strlist_copy(ctx->conda.wheels_packages); + result->conda.installer_arch = strdup_maybe(ctx->conda.installer_arch); + result->conda.installer_baseurl = strdup_maybe(ctx->conda.installer_baseurl); + result->conda.installer_name = strdup_maybe(ctx->conda.installer_name); + result->conda.installer_path = strdup_maybe(ctx->conda.installer_path); + result->conda.installer_platform = strdup_maybe(ctx->conda.installer_platform); + result->conda.installer_version = strdup_maybe(ctx->conda.installer_version); + result->conda.tool_build_version = strdup_maybe(ctx->conda.tool_build_version); + result->conda.tool_version = strdup_maybe(ctx->conda.tool_version); + + // Info + result->info.build_name = strdup_maybe(ctx->info.build_name); + result->info.build_number = strdup_maybe(ctx->info.build_number); + result->info.release_name = strdup_maybe(ctx->info.release_name); + result->info.time_info = ctx->info.time_info; + result->info.time_now = ctx->info.time_now; + result->info.time_str_epoch = strdup_maybe(ctx->info.time_str_epoch); + + // Meta + result->meta.name = strdup_maybe(ctx->meta.name); + result->meta.based_on = strdup_maybe(ctx->meta.based_on); + result->meta.codename = strdup_maybe(ctx->meta.codename); + result->meta.mission = strdup_maybe(ctx->meta.mission); + result->meta.final = ctx->meta.final; + result->meta.python = strdup_maybe(ctx->meta.python); + result->meta.python_compact = strdup_maybe(ctx->meta.python_compact); + result->meta.rc = ctx->meta.rc; + result->meta.version = strdup_maybe(ctx->meta.version); + + // Rules + result->rules.build_name_fmt = strdup_maybe(ctx->rules.build_name_fmt); + result->rules.build_number_fmt = strdup_maybe(ctx->rules.build_number_fmt); + // Unused member? + result->rules.enable_final = ctx->rules.enable_final; + result->rules.release_fmt = ctx->rules.release_fmt; + // TODO: need content duplication function + memcpy(&result->rules.content, &ctx->rules.content, sizeof(ctx->rules.content)); + + if (ctx->rules._handle) { + SYSDEBUG("duplicating INIFILE handle - BEGIN"); + result->rules._handle = malloc(sizeof(*result->rules._handle)); + if (!result->rules._handle) { + SYSERROR("unable to allocate space for INIFILE handle"); + SYSERROR("%s", "unable to allocate space for INIFILE handle"); + delivery_free(ctx); + return NULL; + } + result->rules._handle->section = malloc(ctx->rules._handle->section_count * sizeof(**ctx->rules._handle->section)); + if (!result->rules._handle->section) { + guard_free(result->rules._handle); + SYSERROR("unable to allocate space for INIFILE section"); + SYSERROR("%s", "unable to allocate space for INIFILE section"); + delivery_free(ctx); + return NULL; + } + memcpy(result->rules._handle, &ctx->rules._handle, sizeof(*ctx->rules._handle)); + SYSDEBUG("duplicating INIFILE handle - END"); + } + + // Runtime + if (ctx->runtime.environ) { + result->runtime.environ = runtime_copy(ctx->runtime.environ->data); + } + + // Storage + result->storage.tools_dir = strdup_maybe(ctx->storage.tools_dir); + result->storage.package_dir = strdup_maybe(ctx->storage.package_dir); + result->storage.results_dir = strdup_maybe(ctx->storage.results_dir); + result->storage.output_dir = strdup_maybe(ctx->storage.output_dir); + result->storage.cfgdump_dir = strdup_maybe(ctx->storage.cfgdump_dir); + result->storage.delivery_dir = strdup_maybe(ctx->storage.delivery_dir); + result->storage.meta_dir = strdup_maybe(ctx->storage.meta_dir); + result->storage.mission_dir = strdup_maybe(ctx->storage.mission_dir); + result->storage.root = strdup_maybe(ctx->storage.root); + result->storage.tmpdir = strdup_maybe(ctx->storage.tmpdir); + result->storage.build_dir = strdup_maybe(ctx->storage.build_dir); + result->storage.build_docker_dir = strdup_maybe(ctx->storage.build_docker_dir); + result->storage.build_recipes_dir = strdup_maybe(ctx->storage.build_recipes_dir); + result->storage.build_sources_dir = strdup_maybe(ctx->storage.build_sources_dir); + result->storage.build_testing_dir = strdup_maybe(ctx->storage.build_testing_dir); + result->storage.conda_artifact_dir = strdup_maybe(ctx->storage.conda_artifact_dir); + result->storage.conda_install_prefix = strdup_maybe(ctx->storage.conda_install_prefix); + result->storage.conda_staging_dir = strdup_maybe(ctx->storage.conda_staging_dir); + result->storage.conda_staging_url = strdup_maybe(ctx->storage.conda_staging_url); + result->storage.docker_artifact_dir = strdup_maybe(ctx->storage.docker_artifact_dir); + result->storage.wheel_artifact_dir = strdup_maybe(ctx->storage.wheel_artifact_dir); + result->storage.wheel_staging_url = strdup_maybe(ctx->storage.wheel_staging_url); + + result->system.arch = strdup_maybe(ctx->system.arch); + if (ctx->system.platform) { + result->system.platform = malloc(DELIVERY_PLATFORM_MAX * sizeof(*result->system.platform)); + if (!result->system.platform) { + SYSERROR("unable to allocate space for system platform array"); + SYSERROR("%s", "unable to allocate space for system platform array"); + delivery_free(ctx); + return NULL; + } + for (size_t i = 0; i < DELIVERY_PLATFORM_MAX; i++) { + result->system.platform[i] = strdup_maybe(ctx->system.platform[i]); + if (!result->system.platform[i]) { + SYSERROR("%s", "unable to allocate record in system platform array"); + guard_array_n_free(result->system.platform, DELIVERY_PLATFORM_MAX); + delivery_free(ctx); + return NULL; + } + } + } + + // Docker + result->deploy.docker.build_args = strlist_copy(ctx->deploy.docker.build_args); + result->deploy.docker.tags = strlist_copy(ctx->deploy.docker.tags); + result->deploy.docker.capabilities = ctx->deploy.docker.capabilities; + result->deploy.docker.dockerfile = strdup_maybe(ctx->deploy.docker.dockerfile); + result->deploy.docker.image_compression = strdup_maybe(ctx->deploy.docker.image_compression); + result->deploy.docker.registry = strdup_maybe(ctx->deploy.docker.registry); + result->deploy.docker.test_script = strdup_maybe(ctx->deploy.docker.test_script); + + // Jfrog + // TODO: break out into a separate a function + for (size_t i = 0; i < sizeof(ctx->deploy.jfrog) / sizeof(ctx->deploy.jfrog[0]); i++) { + result->deploy.jfrog[i].dest = strdup_maybe(ctx->deploy.jfrog[i].dest); + result->deploy.jfrog[i].files = strlist_copy(ctx->deploy.jfrog[i].files); + result->deploy.jfrog[i].repo = strdup_maybe(ctx->deploy.jfrog[i].repo); + result->deploy.jfrog[i].upload_ctx.ant = ctx->deploy.jfrog[i].upload_ctx.ant; + result->deploy.jfrog[i].upload_ctx.archive = ctx->deploy.jfrog[i].upload_ctx.archive; + result->deploy.jfrog[i].upload_ctx.build_name = ctx->deploy.jfrog[i].upload_ctx.build_name; + result->deploy.jfrog[i].upload_ctx.build_number = ctx->deploy.jfrog[i].upload_ctx.build_number; + result->deploy.jfrog[i].upload_ctx.deb = ctx->deploy.jfrog[i].upload_ctx.deb; + result->deploy.jfrog[i].upload_ctx.detailed_summary = ctx->deploy.jfrog[i].upload_ctx.detailed_summary; + result->deploy.jfrog[i].upload_ctx.dry_run = ctx->deploy.jfrog[i].upload_ctx.dry_run; + result->deploy.jfrog[i].upload_ctx.exclusions = strdup_maybe(ctx->deploy.jfrog[i].upload_ctx.exclusions); + result->deploy.jfrog[i].upload_ctx.explode = ctx->deploy.jfrog[i].upload_ctx.explode; + result->deploy.jfrog[i].upload_ctx.fail_no_op = ctx->deploy.jfrog[i].upload_ctx.fail_no_op; + result->deploy.jfrog[i].upload_ctx.flat = ctx->deploy.jfrog[i].upload_ctx.flat; + result->deploy.jfrog[i].upload_ctx.include_dirs = ctx->deploy.jfrog[i].upload_ctx.include_dirs; + result->deploy.jfrog[i].upload_ctx.module = strdup_maybe(ctx->deploy.jfrog[i].upload_ctx.module); + result->deploy.jfrog[i].upload_ctx.project = strdup_maybe(ctx->deploy.jfrog[i].upload_ctx.project); + result->deploy.jfrog[i].upload_ctx.quiet = ctx->deploy.jfrog[i].upload_ctx.quiet; + result->deploy.jfrog[i].upload_ctx.recursive = ctx->deploy.jfrog[i].upload_ctx.recursive; + result->deploy.jfrog[i].upload_ctx.regexp = ctx->deploy.jfrog[i].upload_ctx.regexp; + result->deploy.jfrog[i].upload_ctx.retries = ctx->deploy.jfrog[i].upload_ctx.retries; + result->deploy.jfrog[i].upload_ctx.retry_wait_time = ctx->deploy.jfrog[i].upload_ctx.retry_wait_time; + result->deploy.jfrog[i].upload_ctx.spec = strdup_maybe(ctx->deploy.jfrog[i].upload_ctx.spec); + result->deploy.jfrog[i].upload_ctx.spec_vars = strdup_maybe(ctx->deploy.jfrog[i].upload_ctx.spec_vars); + result->deploy.jfrog[i].upload_ctx.symlinks = ctx->deploy.jfrog[i].upload_ctx.symlinks; + result->deploy.jfrog[i].upload_ctx.sync_deletes = ctx->deploy.jfrog[i].upload_ctx.sync_deletes; + result->deploy.jfrog[i].upload_ctx.target_props = strdup_maybe(ctx->deploy.jfrog[i].upload_ctx.target_props); + result->deploy.jfrog[i].upload_ctx.threads = ctx->deploy.jfrog[i].upload_ctx.threads; + result->deploy.jfrog[i].upload_ctx.workaround_parent_only = ctx->deploy.jfrog[i].upload_ctx.workaround_parent_only; + } + + result->deploy.jfrog_auth.access_token = strdup_maybe(ctx->deploy.jfrog_auth.access_token); + result->deploy.jfrog_auth.client_cert_key_path = strdup_maybe(ctx->deploy.jfrog_auth.client_cert_key_path); + result->deploy.jfrog_auth.client_cert_path = strdup_maybe(ctx->deploy.jfrog_auth.client_cert_path); + result->deploy.jfrog_auth.insecure_tls = ctx->deploy.jfrog_auth.insecure_tls; + result->deploy.jfrog_auth.password = strdup_maybe(ctx->deploy.jfrog_auth.password); + result->deploy.jfrog_auth.server_id = strdup_maybe(ctx->deploy.jfrog_auth.server_id); + result->deploy.jfrog_auth.ssh_key_path = strdup_maybe(ctx->deploy.jfrog_auth.ssh_key_path); + result->deploy.jfrog_auth.ssh_passphrase = strdup_maybe(ctx->deploy.jfrog_auth.ssh_passphrase); + result->deploy.jfrog_auth.url = strdup_maybe(ctx->deploy.jfrog_auth.url); + result->deploy.jfrog_auth.user = strdup_maybe(ctx->deploy.jfrog_auth.user); + + for (size_t i = 0; result->tests && i < result->tests->num_used; i++) { + result->tests->test[i]->disable = ctx->tests->test[i]->disable; + result->tests->test[i]->parallel = ctx->tests->test[i]->parallel; + result->tests->test[i]->build_recipe = strdup_maybe(ctx->tests->test[i]->build_recipe); + result->tests->test[i]->name = strdup_maybe(ctx->tests->test[i]->name); + result->tests->test[i]->version = strdup_maybe(ctx->tests->test[i]->version); + result->tests->test[i]->repository = strdup_maybe(ctx->tests->test[i]->repository); + result->tests->test[i]->repository_info_ref = strdup_maybe(ctx->tests->test[i]->repository_info_ref); + result->tests->test[i]->repository_info_tag = strdup_maybe(ctx->tests->test[i]->repository_info_tag); + result->tests->test[i]->repository_remove_tags = strlist_copy(ctx->tests->test[i]->repository_remove_tags); + if (ctx->tests->test[i]->runtime->environ) { + result->tests->test[i]->runtime->environ = runtime_copy(ctx->tests->test[i]->runtime->environ->data); + } + result->tests->test[i]->script = strdup_maybe(ctx->tests->test[i]->script); + result->tests->test[i]->script_setup = strdup_maybe(ctx->tests->test[i]->script_setup); + } + + return result; +} void delivery_free(struct Delivery *ctx) { guard_free(ctx->system.arch); - guard_array_free(ctx->system.platform); + guard_array_n_free(ctx->system.platform, DELIVERY_PLATFORM_MAX); guard_free(ctx->meta.name); guard_free(ctx->meta.version); guard_free(ctx->meta.codename); @@ -57,23 +252,14 @@ void delivery_free(struct Delivery *ctx) { guard_strlist_free(&ctx->conda.pip_packages_purge); guard_strlist_free(&ctx->conda.wheels_packages); - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { - guard_free(ctx->tests[i].name); - guard_free(ctx->tests[i].version); - guard_free(ctx->tests[i].repository); - guard_free(ctx->tests[i].repository_info_ref); - guard_free(ctx->tests[i].repository_info_tag); - guard_strlist_free(&ctx->tests[i].repository_remove_tags); - guard_free(ctx->tests[i].script); - guard_free(ctx->tests[i].script_setup); - guard_free(ctx->tests[i].build_recipe); - // test-specific runtime variables - guard_runtime_free(ctx->tests[i].runtime.environ); - } + tests_free(&ctx->tests); guard_free(ctx->rules.release_fmt); guard_free(ctx->rules.build_name_fmt); guard_free(ctx->rules.build_number_fmt); + if (ctx->rules._handle) { + ini_free(&ctx->rules._handle); + } guard_free(ctx->deploy.docker.test_script); guard_free(ctx->deploy.docker.registry); @@ -104,8 +290,11 @@ void delivery_free(struct Delivery *ctx) { guard_free(ctx->_stasis_ini_fp.mission_path); } -int delivery_format_str(struct Delivery *ctx, char **dest, const char *fmt) { - size_t fmt_len = strlen(fmt); +int delivery_format_str(struct Delivery *ctx, char **dest, size_t maxlen, const char *fmt) { + const size_t fmt_len = strlen(fmt); + if (maxlen < 1) { + maxlen = 1; + } if (!*dest) { *dest = calloc(STASIS_NAME_MAX, sizeof(**dest)); @@ -119,47 +308,47 @@ int delivery_format_str(struct Delivery *ctx, char **dest, const char *fmt) { i++; switch (fmt[i]) { case 'n': // name - strcat(*dest, ctx->meta.name); + strncat(*dest, ctx->meta.name, maxlen - 1); break; case 'c': // codename - strcat(*dest, ctx->meta.codename); + strncat(*dest, ctx->meta.codename, maxlen - 1); break; case 'm': // mission - strcat(*dest, ctx->meta.mission); + strncat(*dest, ctx->meta.mission, maxlen - 1); break; case 'r': // revision - sprintf(*dest + strlen(*dest), "%d", ctx->meta.rc); + snprintf(*dest + strlen(*dest), maxlen - strlen(*dest), "%d", ctx->meta.rc); break; case 'R': // "final"-aware revision if (ctx->meta.final) - strcat(*dest, "final"); + strncat(*dest, "final", maxlen); else - sprintf(*dest + strlen(*dest), "%d", ctx->meta.rc); + snprintf(*dest + strlen(*dest), maxlen - strlen(*dest), "%d", ctx->meta.rc); break; case 'v': // version - strcat(*dest, ctx->meta.version); + strncat(*dest, ctx->meta.version, maxlen - 1); break; case 'P': // python version - strcat(*dest, ctx->meta.python); + strncat(*dest, ctx->meta.python, maxlen - 1); break; case 'p': // python version major/minor - strcat(*dest, ctx->meta.python_compact); + strncat(*dest, ctx->meta.python_compact, maxlen - 1); break; case 'a': // system architecture name - strcat(*dest, ctx->system.arch); + strncat(*dest, ctx->system.arch, maxlen - 1); break; case 'o': // system platform (OS) name - strcat(*dest, ctx->system.platform[DELIVERY_PLATFORM_RELEASE]); + strncat(*dest, ctx->system.platform[DELIVERY_PLATFORM_RELEASE], maxlen - 1); break; case 't': // unix epoch - sprintf(*dest + strlen(*dest), "%ld", ctx->info.time_now); + snprintf(*dest + strlen(*dest), maxlen - strlen(*dest), "%ld", ctx->info.time_now); break; default: // unknown formatter, write as-is - sprintf(*dest + strlen(*dest), "%c%c", fmt[i - 1], fmt[i]); + snprintf(*dest + strlen(*dest), maxlen - strlen(*dest), "%c%c", fmt[i - 1], fmt[i]); break; } } else { // write non-format text - sprintf(*dest + strlen(*dest), "%c", fmt[i]); + snprintf(*dest + strlen(*dest), maxlen - strlen(*dest), "%c", fmt[i]); } } return 0; @@ -174,15 +363,17 @@ void delivery_defer_packages(struct Delivery *ctx, int type) { if (DEFER_CONDA == type) { dataptr = ctx->conda.conda_packages; deferred = ctx->conda.conda_packages_defer; - strcpy(mode, "conda"); + strncpy(mode, "conda", sizeof(mode) - 1); } else if (DEFER_PIP == type) { dataptr = ctx->conda.pip_packages; deferred = ctx->conda.pip_packages_defer; - strcpy(mode, "pip"); + strncpy(mode, "pip", sizeof(mode) - 1); } else { - SYSERROR("BUG: type %d does not map to a supported package manager!\n", type); + SYSERROR("BUG: type %d does not map to a supported package manager!", type); exit(1); } + mode[sizeof(mode) - 1] = '\0'; + msg(STASIS_MSG_L2, "Filtering %s packages by test definition...\n", mode); struct StrList *filtered = NULL; @@ -199,7 +390,7 @@ void delivery_defer_packages(struct Delivery *ctx, int type) { // Compile a list of packages that are *also* to be tested. char *spec_begin = strpbrk(name, "@~=<>!"); char *spec_end = spec_begin; - char package_name[255] = {0}; + char package_name[STASIS_NAME_MAX] = {0}; if (spec_end) { // A version is present in the package name. Jump past operator(s). @@ -207,25 +398,37 @@ void delivery_defer_packages(struct Delivery *ctx, int type) { spec_end++; } strncpy(package_name, name, spec_begin - name); + package_name[spec_begin - name] = '\0'; } else { strncpy(package_name, name, sizeof(package_name) - 1); + package_name[sizeof(package_name) - 1] = '\0'; } remove_extras(package_name); msg(STASIS_MSG_L3, "package '%s': ", package_name); // When spec is present in name, set tests->version to the version detected in the name - for (size_t x = 0; x < sizeof(ctx->tests) / sizeof(ctx->tests[0]) && ctx->tests[x].name != NULL; x++) { - struct Test *test = &ctx->tests[x]; - char nametmp[1024] = {0}; + for (size_t x = 0; x < ctx->tests->num_used; x++) { + struct Test *test = ctx->tests->test[x]; + char nametmp[STASIS_NAME_MAX] = {0}; strncpy(nametmp, package_name, sizeof(nametmp) - 1); + nametmp[sizeof(nametmp) - 1] = '\0'; + // Is the [test:NAME] in the package name? if (!strcmp(nametmp, test->name)) { // Override test->version when a version is provided by the (pip|conda)_package list item guard_free(test->version); if (spec_begin && spec_end) { - test->version = strdup(spec_end); + char *version_at = strrchr(spec_end, '@'); + if (version_at) { + if (strlen(version_at)) { + version_at++; + } + test->version = strdup(version_at); + } else { + test->version = strdup(spec_end); + } } else { // There are too many possible default branches nowadays: master, main, develop, xyz, etc. // HEAD is a safe bet. @@ -233,6 +436,9 @@ void delivery_defer_packages(struct Delivery *ctx, int type) { } // Is the list item a git+schema:// URL? + // TODO: nametmp is just the name so this will never work. but do we want it to? this looks like + // TODO: an unsafe feature. We shouldn't be able to change what's in the config. we should + // TODO: be getting what we asked for, or exit the program with an error. if (strstr(nametmp, "git+") && strstr(nametmp, "://")) { char *xrepo = strstr(nametmp, "+"); if (xrepo) { @@ -252,13 +458,13 @@ void delivery_defer_packages(struct Delivery *ctx, int type) { int upstream_exists = 0; if (DEFER_PIP == type) { - upstream_exists = pkg_index_provides(PKG_USE_PIP, PYPI_INDEX_DEFAULT, name); + upstream_exists = pkg_index_provides(PKG_USE_PIP, PYPI_INDEX_DEFAULT, name, ctx->storage.tmpdir); } else if (DEFER_CONDA == type) { - upstream_exists = pkg_index_provides(PKG_USE_CONDA, NULL, name); + upstream_exists = pkg_index_provides(PKG_USE_CONDA, NULL, name, ctx->storage.tmpdir); } if (PKG_INDEX_PROVIDES_FAILED(upstream_exists)) { - fprintf(stderr, "%s's existence command failed for '%s': %s\n", + SYSERROR("%s's existence command failed for '%s': %s", mode, name, pkg_index_provides_strerror(upstream_exists)); exit(1); } @@ -283,7 +489,7 @@ void delivery_defer_packages(struct Delivery *ctx, int type) { } if (!strlist_count(deferred)) { - msg(STASIS_MSG_WARN | STASIS_MSG_L2, "No %s packages were filtered by test definitions\n", mode); + SYSWARN("No %s packages were filtered by test definitions", mode); } else { if (DEFER_CONDA == type) { guard_strlist_free(&ctx->conda.conda_packages); diff --git a/src/lib/delivery/delivery_artifactory.c b/src/lib/delivery/delivery_artifactory.c index 97db752..d7a5457 100644 --- a/src/lib/delivery/delivery_artifactory.c +++ b/src/lib/delivery/delivery_artifactory.c @@ -4,8 +4,12 @@ int delivery_init_artifactory(struct Delivery *ctx) { int status = 0; char dest[PATH_MAX] = {0}; char filepath[PATH_MAX] = {0}; - snprintf(dest, sizeof(dest) - 1, "%s/bin", ctx->storage.tools_dir); - snprintf(filepath, sizeof(dest) - 1, "%s/bin/jf", ctx->storage.tools_dir); + + SYSDEBUG("Initializing artifactory tools"); + snprintf(dest, sizeof(dest), "%s/bin", ctx->storage.tools_dir); + SYSDEBUG("dest=%s", dest); + snprintf(filepath, sizeof(dest), "%s/bin/jf", ctx->storage.tools_dir); + SYSDEBUG("filepath=%s", filepath); if (!access(filepath, F_OK)) { // already have it @@ -13,6 +17,7 @@ int delivery_init_artifactory(struct Delivery *ctx) { goto delivery_init_artifactory_envsetup; } + SYSDEBUG("Assign platform"); char *platform = ctx->system.platform[DELIVERY_PLATFORM]; msg(STASIS_MSG_L3, "Downloading %s for %s %s\n", globals.jfrog.remote_filename, platform, ctx->system.arch); if ((status = artifactory_download_cli(dest, @@ -32,7 +37,7 @@ int delivery_init_artifactory(struct Delivery *ctx) { // JFROG_CLI_HOME_DIR is where .jfrog is stored char path[PATH_MAX] = {0}; - snprintf(path, sizeof(path) - 1, "%s/.jfrog", ctx->storage.build_dir); + snprintf(path, sizeof(path), "%s/.jfrog", ctx->storage.build_dir); setenv("JFROG_CLI_HOME_DIR", path, 1); // JFROG_CLI_TEMP_DIR is where the obvious is stored @@ -44,7 +49,7 @@ int delivery_artifact_upload(struct Delivery *ctx) { int status = 0; if (jfrt_auth_init(&ctx->deploy.jfrog_auth)) { - fprintf(stderr, "Failed to initialize Artifactory authentication context\n"); + SYSERROR("Failed to initialize Artifactory authentication context"); return -1; } @@ -55,9 +60,10 @@ int delivery_artifact_upload(struct Delivery *ctx) { jfrt_upload_init(&ctx->deploy.jfrog[i].upload_ctx); if (!globals.jfrog.repo) { - msg(STASIS_MSG_WARN, "Artifactory repository path is not configured!\n"); - fprintf(stderr, "set STASIS_JF_REPO environment variable...\nOr append to configuration file:\n\n"); - fprintf(stderr, "[deploy:artifactory]\nrepo = example/generic/repo/path\n\n"); + SYSWARN("Artifactory repository path is not configured!"); + SYSWARN("set STASIS_JF_REPO environment variable...\nOr append to configuration file:"); + SYSWARN(""); + SYSWARN("[deploy:artifactory]\nrepo = example/generic/repo/path\n"); status++; break; } else if (!ctx->deploy.jfrog[i].repo) { @@ -66,7 +72,7 @@ int delivery_artifact_upload(struct Delivery *ctx) { if (!ctx->deploy.jfrog[i].repo || isempty(ctx->deploy.jfrog[i].repo) || !strlen(ctx->deploy.jfrog[i].repo)) { // Unlikely to trigger if the config parser is working correctly - msg(STASIS_MSG_ERROR, "Artifactory repository path is empty. Cannot continue.\n"); + SYSERROR("Artifactory repository path is empty. Cannot continue."); status++; break; } @@ -76,7 +82,7 @@ int delivery_artifact_upload(struct Delivery *ctx) { ctx->deploy.jfrog[i].upload_ctx.build_number = ctx->info.build_number; if (jfrog_cli_rt_ping(&ctx->deploy.jfrog_auth)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Unable to contact artifactory server: %s\n", ctx->deploy.jfrog_auth.url); + SYSERROR("Unable to contact artifactory server: %s", ctx->deploy.jfrog_auth.url); return -1; } @@ -84,8 +90,8 @@ int delivery_artifact_upload(struct Delivery *ctx) { for (size_t f = 0; f < strlist_count(ctx->deploy.jfrog[i].files); f++) { char dest[PATH_MAX] = {0}; char files[PATH_MAX] = {0}; - snprintf(dest, sizeof(dest) - 1, "%s/%s", ctx->deploy.jfrog[i].repo, ctx->deploy.jfrog[i].dest); - snprintf(files, sizeof(files) - 1, "%s", strlist_item(ctx->deploy.jfrog[i].files, f)); + snprintf(dest, sizeof(dest), "%s/%s", ctx->deploy.jfrog[i].repo, ctx->deploy.jfrog[i].dest); + snprintf(files, sizeof(files), "%s", strlist_item(ctx->deploy.jfrog[i].files, f)); status += jfrog_cli_rt_upload(&ctx->deploy.jfrog_auth, &ctx->deploy.jfrog[i].upload_ctx, files, dest); } } @@ -99,7 +105,7 @@ int delivery_artifact_upload(struct Delivery *ctx) { ctx->deploy.jfrog[0].upload_ctx.build_number); } } else { - msg(STASIS_MSG_WARN | STASIS_MSG_L2, "Artifactory build info upload is disabled by CLI argument\n"); + SYSWARN("Artifactory build info upload is disabled by CLI argument"); } return status; @@ -107,7 +113,7 @@ int delivery_artifact_upload(struct Delivery *ctx) { int delivery_mission_render_files(struct Delivery *ctx) { if (!ctx->storage.mission_dir) { - fprintf(stderr, "Mission directory is not configured. Context not initialized?\n"); + SYSERROR("Mission directory is not configured. Context not initialized?"); return -1; } struct Data { @@ -119,7 +125,7 @@ int delivery_mission_render_files(struct Delivery *ctx) { memset(&data, 0, sizeof(data)); data.src = calloc(PATH_MAX, sizeof(*data.src)); if (!data.src) { - perror("data.src"); + SYSERROR("unable to allocate memory for data.src: %s", strerror(errno)); return -1; } @@ -134,7 +140,7 @@ int delivery_mission_render_files(struct Delivery *ctx) { guard_free(data.src); return 1; } - sprintf(data.src, "%s/%s/%s", ctx->storage.mission_dir, ctx->meta.mission, val.as_char_p); + snprintf(data.src, PATH_MAX, "%s/%s/%s", ctx->storage.mission_dir, ctx->meta.mission, val.as_char_p); msg(STASIS_MSG_L2, "%s\n", data.src); int err = 0; @@ -149,14 +155,14 @@ int delivery_mission_render_files(struct Delivery *ctx) { char *contents = calloc(st.st_size + 1, sizeof(*contents)); if (!contents) { - perror("template file contents"); + SYSERROR("unable to allocate memory for template file contents: %s", strerror(errno)); guard_free(data.dest); continue; } FILE *fp = fopen(data.src, "rb"); if (!fp) { - perror(data.src); + SYSERROR("unable to open source template file: %s", strerror(errno)); guard_free(contents); guard_free(data.dest); continue; @@ -189,13 +195,13 @@ int delivery_series_sync(struct Delivery *ctx) { struct JFRT_Download dl = {0}; if (jfrt_auth_init(&ctx->deploy.jfrog_auth)) { - fprintf(stderr, "Failed to initialize Artifactory authentication context\n"); + SYSERROR("Failed to initialize Artifactory authentication context"); return -1; // error } char *r_fmt = strdup(ctx->rules.release_fmt); if (!r_fmt) { - SYSERROR("%s", "Unable to allocate bytes for release format string"); + SYSERROR("Unable to allocate bytes for release format string"); return -1; } @@ -210,7 +216,7 @@ int delivery_series_sync(struct Delivery *ctx) { } char *release_pattern = NULL; - if (delivery_format_str(ctx, &release_pattern, r_fmt) < 0) { + if (delivery_format_str(ctx, &release_pattern, STASIS_NAME_MAX, r_fmt) < 0) { SYSERROR("Unable to render delivery format string: %s", r_fmt); guard_free(r_fmt); return -1; @@ -223,7 +229,7 @@ int delivery_series_sync(struct Delivery *ctx) { ctx->meta.mission, ctx->info.build_name, release_pattern) < 0) { - SYSERROR("%s", "Unable to allocate bytes for remote directory path"); + SYSERROR("Unable to allocate bytes for remote directory path"); guard_free(release_pattern); return -1; } @@ -231,7 +237,7 @@ int delivery_series_sync(struct Delivery *ctx) { char *dest_dir = NULL; if (asprintf(&dest_dir, "%s/{1}", ctx->storage.output_dir) < 0) { - SYSERROR("%s", "Unable to allocate bytes for destination directory path"); + SYSERROR("Unable to allocate bytes for destination directory path"); return -1; } diff --git a/src/lib/delivery/delivery_build.c b/src/lib/delivery/delivery_build.c index 2d891d2..66f9126 100644 --- a/src/lib/delivery/delivery_build.c +++ b/src/lib/delivery/delivery_build.c @@ -1,43 +1,68 @@ +#include <fnmatch.h> + #include "delivery.h" +#include "conda.h" +#include "recipe.h" int delivery_build_recipes(struct Delivery *ctx) { - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { + for (size_t i = 0; i < ctx->tests->num_used; i++) { char *recipe_dir = NULL; - if (ctx->tests[i].build_recipe) { // build a conda recipe - if (recipe_clone(ctx->storage.build_recipes_dir, ctx->tests[i].build_recipe, NULL, &recipe_dir)) { - fprintf(stderr, "Encountered an issue while cloning recipe for: %s\n", ctx->tests[i].name); + if (ctx->tests->test[i]->build_recipe) { // build a conda recipe + if (recipe_clone(ctx->storage.build_recipes_dir, ctx->tests->test[i]->build_recipe, NULL, &recipe_dir)) { + SYSERROR("Encountered an issue while cloning recipe for: %s", ctx->tests->test[i]->name); return -1; } if (!recipe_dir) { - fprintf(stderr, "BUG: recipe_clone() succeeded but recipe_dir is NULL: %s\n", strerror(errno)); + SYSERROR("BUG: recipe_clone() succeeded but recipe_dir is NULL: %s", strerror(errno)); return -1; } int recipe_type = recipe_get_type(recipe_dir); if(!pushd(recipe_dir)) { if (RECIPE_TYPE_ASTROCONDA == recipe_type) { - pushd(path_basename(ctx->tests[i].repository)); + pushd(path_basename(ctx->tests->test[i]->repository)); } else if (RECIPE_TYPE_CONDA_FORGE == recipe_type) { pushd("recipe"); } - char recipe_version[100]; - char recipe_buildno[100]; + char recipe_version[200]; + char recipe_buildno[200]; char recipe_git_url[PATH_MAX]; char recipe_git_rev[PATH_MAX]; + char tag[100] = {0}; + if (ctx->tests->test[i]->repository_info_tag) { + const int is_long_tag = num_chars(ctx->tests->test[i]->repository_info_tag, '-') > 1; + if (is_long_tag) { + const size_t len = strcspn(ctx->tests->test[i]->repository_info_tag, "-"); + strncpy(tag, ctx->tests->test[i]->repository_info_tag, len); + tag[len] = '\0'; + } else { + strncpy(tag, ctx->tests->test[i]->repository_info_tag, sizeof(tag) - 1); + tag[sizeof(tag) - 1] = '\0'; + } + } else { + strncpy(tag, ctx->tests->test[i]->version, sizeof(tag) - 1); + tag[sizeof(tag) - 1] = '\0'; + } + //sprintf(recipe_version, "{%% set version = GIT_DESCRIBE_TAG ~ \".dev\" ~ GIT_DESCRIBE_NUMBER ~ \"+\" ~ GIT_DESCRIBE_HASH %%}"); - //sprintf(recipe_git_url, " git_url: %s", ctx->tests[i].repository); - //sprintf(recipe_git_rev, " git_rev: %s", ctx->tests[i].version); + //sprintf(recipe_git_url, " git_url: %s", ctx->tests->test[i]->repository); + //sprintf(recipe_git_rev, " git_rev: %s", ctx->tests->test[i]->version); // TODO: Conditionally download archives if github.com is the origin. Else, use raw git_* keys ^^^ - sprintf(recipe_version, "{%% set version = \"%s\" %%}", ctx->tests[i].repository_info_tag ? ctx->tests[i].repository_info_tag : ctx->tests[i].version); - sprintf(recipe_git_url, " url: %s/archive/refs/tags/{{ version }}.tar.gz", ctx->tests[i].repository); - strcpy(recipe_git_rev, ""); - sprintf(recipe_buildno, " number: 0"); + // 03/2026 - How can we know if the repository URL supports archive downloads? + // Perhaps we can key it to the recipe type, because the archive is a requirement imposed + // by conda-forge. Hmm. + + snprintf(recipe_version, sizeof(recipe_version), "{%% set version = \"%s\" %%}", tag); + snprintf(recipe_git_url, sizeof(recipe_git_url), " url: %s/archive/refs/tags/{{ version }}.tar.gz", ctx->tests->test[i]->repository); + strncpy(recipe_git_rev, "", sizeof(recipe_git_rev) - 1); + recipe_git_rev[sizeof(recipe_git_rev) - 1] = '\0'; + snprintf(recipe_buildno, sizeof(recipe_buildno), " number: 0"); unsigned flags = REPLACE_TRUNCATE_AFTER_MATCH; //file_replace_text("meta.yaml", "{% set version = ", recipe_version); if (ctx->meta.final) { // remove this. i.e. statis cannot deploy a release to conda-forge - sprintf(recipe_version, "{%% set version = \"%s\" %%}", ctx->tests[i].version); + snprintf(recipe_version, sizeof(recipe_version), "{%% set version = \"%s\" %%}", ctx->tests->test[i]->version); // TODO: replace sha256 of tagged archive // TODO: leave the recipe unchanged otherwise. in theory this should produce the same conda package hash as conda forge. // For now, remove the sha256 requirement @@ -55,25 +80,28 @@ int delivery_build_recipes(struct Delivery *ctx) { char arch[STASIS_NAME_MAX] = {0}; char platform[STASIS_NAME_MAX] = {0}; - strcpy(platform, ctx->system.platform[DELIVERY_PLATFORM]); + strncpy(platform, ctx->system.platform[DELIVERY_PLATFORM], sizeof(platform) - 1); if (strstr(platform, "Darwin")) { memset(platform, 0, sizeof(platform)); - strcpy(platform, "osx"); + strncpy(platform, "osx", sizeof(platform) - 1); } + platform[sizeof(platform) - 1] = '\0'; tolower_s(platform); + if (strstr(ctx->system.arch, "arm64")) { - strcpy(arch, "arm64"); + strncpy(arch, "arm64", sizeof(arch) - 1); } else if (strstr(ctx->system.arch, "64")) { - strcpy(arch, "64"); + strncpy(arch, "64", sizeof(arch) - 1); } else { - strcat(arch, "32"); // blind guess + strncat(arch, "32", sizeof(arch) - strlen(arch) - 1); // blind guess } + arch[sizeof(arch) - 1] = '\0'; tolower_s(arch); - sprintf(command, "mambabuild --python=%s -m ../.ci_support/%s_%s_.yaml .", + snprintf(command, sizeof(command), "mambabuild --python=%s -m ../.ci_support/%s_%s_.yaml .", ctx->meta.python, platform, arch); } else { - sprintf(command, "mambabuild --python=%s .", ctx->meta.python); + snprintf(command, sizeof(command), "mambabuild --python=%s .", ctx->meta.python); } int status = conda_exec(command); if (status) { @@ -86,7 +114,7 @@ int delivery_build_recipes(struct Delivery *ctx) { } popd(); } else { - fprintf(stderr, "Unable to enter recipe directory %s: %s\n", recipe_dir, strerror(errno)); + SYSERROR("Unable to enter recipe directory %s: %s", recipe_dir, strerror(errno)); guard_free(recipe_dir); return -1; } @@ -112,7 +140,7 @@ int filter_repo_tags(char *repo, struct StrList *patterns) { int match = fnmatch(pattern, tag, 0); if (!match) { char cmd[PATH_MAX] = {0}; - sprintf(cmd, "git tag -d %s", tag); + snprintf(cmd, sizeof(cmd), "git tag -d %s", tag); result += system(cmd); break; } @@ -127,13 +155,238 @@ int filter_repo_tags(char *repo, struct StrList *patterns) { return result; } +static int read_without_line_endings(const size_t line, char ** arg) { + (void) line; + if (*arg) { + strip(*arg); + if (isempty(*arg)) { + return 1; // skip + } + } + return 0; +} + +int manylinux_exec(const char *image, const char *script, const char *copy_to_container_dir, const char *copy_from_container_dir, const char *copy_to_host_dir) { + int result = -1; // fail by default + char *container_name = NULL; + char *source_copy_command = NULL; + char *copy_command = NULL; + char *rm_command = NULL; + char *nop_create_command = NULL; + char *nop_rm_command = NULL; + char *volume_rm_command = NULL; + char *find_command = NULL; + char *wheel_paths_filename = NULL; + char *args = NULL; + + const uid_t uid = geteuid(); + char suffix[7] = {0}; + + // setup + + if (get_random_bytes(suffix, sizeof(suffix))) { + SYSERROR("unable to acquire value from random generator"); + goto manylinux_fail; + } + + if (asprintf(&container_name, "manylinux_build_%d_%zd_%s", uid, time(NULL), suffix) < 0) { + SYSERROR("unable to allocate memory for container name"); + goto manylinux_fail; + } + + if (asprintf(&args, "--name %s -w /build -v %s:/build", container_name, container_name) < 0) { + SYSERROR("unable to allocate memory for docker arguments"); + goto manylinux_fail; + } + + if (!strstr(image, "manylinux")) { + SYSERROR("expected a manylinux image, but got %s", image); + goto manylinux_fail; + } + + if (asprintf(&nop_create_command, "run --name nop_%s -v %s:/build busybox", container_name, container_name) < 0) { + SYSERROR("unable to allocate memory for nop container command"); + goto manylinux_fail; + } + + if (asprintf(&source_copy_command, "cp %s nop_%s:/build", copy_to_container_dir, container_name) < 0) { + SYSERROR("unable to allocate memory for source copy command"); + goto manylinux_fail; + } + + if (asprintf(&nop_rm_command, "rm nop_%s", container_name) < 0) { + SYSERROR("unable to allocate memory for nop container command"); + goto manylinux_fail; + } + + if (asprintf(&wheel_paths_filename, "%s/wheel_paths_%s.txt", globals.tmpdir, container_name) < 0) { + SYSERROR("unable to allocate memory for wheel paths file name"); + goto manylinux_fail; + } + + if (asprintf(&find_command, "run --rm -t -v %s:/build busybox sh -c 'find %s -name \"*.whl\"' > %s", container_name, copy_from_container_dir, wheel_paths_filename) < 0) { + SYSERROR("unable to allocate memory for find command"); + goto manylinux_fail; + } + + // execute + + if (docker_exec(nop_create_command, 0)) { + SYSERROR("docker nop container creation failed"); + goto manylinux_fail; + } + + if (docker_exec(source_copy_command, 0)) { + SYSERROR("docker source copy operation failed"); + goto manylinux_fail; + } + + if (docker_exec(nop_rm_command, STASIS_DOCKER_QUIET)) { + SYSERROR("docker nop container removal failed"); + goto manylinux_fail; + } + + if (docker_script(image, args, (char *) script, 0)) { + SYSERROR("manylinux execution failed"); + goto manylinux_fail; + } + + if (docker_exec(find_command, 0)) { + SYSERROR("docker find command failed"); + goto manylinux_fail; + } + + struct StrList *wheel_paths = strlist_init(); + if (!wheel_paths) { + SYSERROR("wheel_paths not initialized"); + goto manylinux_fail; + } + + if (strlist_append_file(wheel_paths, wheel_paths_filename, read_without_line_endings)) { + SYSERROR("wheel_paths append failed"); + goto manylinux_fail; + } + + for (size_t i = 0; i < strlist_count(wheel_paths); i++) { + const char *item = strlist_item(wheel_paths, i); + if (asprintf(©_command, "cp %s:%s %s", container_name, item, copy_to_host_dir) < 0) { + SYSERROR("unable to allocate memory for docker copy command"); + goto manylinux_fail; + } + + if (docker_exec(copy_command, 0)) { + SYSERROR("docker copy operation failed"); + goto manylinux_fail; + } + guard_free(copy_command); + } + + // Success + result = 0; + + manylinux_fail: + if (wheel_paths_filename) { + remove(wheel_paths_filename); + } + + if (container_name) { + // Keep going on failure unless memory related. + // We don't want build debris everywhere. + if (asprintf(&rm_command, "rm %s", container_name) < 0) { + SYSERROR("unable to allocate memory for rm command"); + goto late_fail; + } + + if (docker_exec(rm_command, STASIS_DOCKER_QUIET)) { + SYSERROR("docker container removal operation failed"); + } + + if (asprintf(&volume_rm_command, "volume rm -f %s", container_name) < 0) { + SYSERROR("unable to allocate memory for docker volume removal command"); + goto late_fail; + } + + if (docker_exec(volume_rm_command, STASIS_DOCKER_QUIET)) { + SYSERROR("docker volume removal operation failed"); + } + } + + late_fail: + guard_free(container_name); + guard_free(args); + guard_free(copy_command); + guard_free(rm_command); + guard_free(volume_rm_command); + guard_free(source_copy_command); + guard_free(nop_create_command); + guard_free(nop_rm_command); + guard_free(find_command); + guard_free(wheel_paths_filename); + guard_strlist_free(&wheel_paths); + return result; +} + +int delivery_build_wheels_manylinux(struct Delivery *ctx, const char *outdir) { + msg(STASIS_MSG_L1, "Building wheels\n"); + + const char *manylinux_image = globals.wheel_builder_manylinux_image; + if (!manylinux_image) { + SYSERROR("manylinux_image not initialized"); + return -1; + } + + int manylinux_build_status = 0; + + msg(STASIS_MSG_L2, "Using: %s\n", manylinux_image); + const struct Meta *meta = &ctx->meta; + const char *script_fmt = + "set -e -x\n" + "git config --global --add safe.directory /build\n" + "python%s -m pip install auditwheel build\n" + "python%s -m build -w .\n" + "auditwheel show --allow-pure-python-wheel dist/*.whl\n" + "auditwheel repair --allow-pure-python-wheel dist/*.whl\n"; + char *script = NULL; + if (asprintf(&script, script_fmt, + meta->python, meta->python) < 0) { + SYSERROR("unable to allocate memory for build script"); + return -1; + } + manylinux_build_status = manylinux_exec( + manylinux_image, + script, + "./", + "/build/wheelhouse", + outdir); + + if (manylinux_build_status) { + SYSERROR("manylinux build failed (%d)", manylinux_build_status); + guard_free(script); + return -1; + } + guard_free(script); + return 0; +} + struct StrList *delivery_build_wheels(struct Delivery *ctx) { + const int on_linux = strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Linux") == 0; + const int docker_usable = ctx->deploy.docker.capabilities.usable; + int use_builder_build = strcmp(globals.wheel_builder, "native") == 0; + const int use_builder_cibuildwheel = strcmp(globals.wheel_builder, "cibuildwheel") == 0 && on_linux && docker_usable; + const int use_builder_manylinux = strcmp(globals.wheel_builder, "manylinux") == 0 && on_linux && docker_usable; + + if (!use_builder_build && !use_builder_cibuildwheel && !use_builder_manylinux) { + SYSWARN("Cannot build wheel for platform using: %s", globals.wheel_builder); + SYSWARN("Falling back to native toolchain.", globals.wheel_builder); + use_builder_build = 1; + } + struct StrList *result = NULL; struct Process proc = {0}; result = strlist_init(); if (!result) { - perror("unable to allocate memory for string list"); + SYSERROR("unable to allocate memory for string list"); return NULL; } @@ -141,57 +394,103 @@ struct StrList *delivery_build_wheels(struct Delivery *ctx) { char name[100] = {0}; char *fullspec = strlist_item(ctx->conda.pip_packages_defer, p); strncpy(name, fullspec, sizeof(name) - 1); + name[sizeof(name) - 1] = '\0'; remove_extras(name); char *spec = find_version_spec(name); if (spec) { *spec = '\0'; } - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { - if ((ctx->tests[i].name && !strcmp(name, ctx->tests[i].name)) && (!ctx->tests[i].build_recipe && ctx->tests[i].repository)) { // build from source + for (size_t i = 0; i < ctx->tests->num_used; i++) { + if ((ctx->tests->test[i]->name && !strcmp(name, ctx->tests->test[i]->name)) && (!ctx->tests->test[i]->build_recipe && ctx->tests->test[i]->repository)) { // build from source char srcdir[PATH_MAX]; char wheeldir[PATH_MAX]; memset(srcdir, 0, sizeof(srcdir)); memset(wheeldir, 0, sizeof(wheeldir)); - sprintf(srcdir, "%s/%s", ctx->storage.build_sources_dir, ctx->tests[i].name); - if (git_clone(&proc, ctx->tests[i].repository, srcdir, ctx->tests[i].version)) { - SYSERROR("Unable to checkout tag '%s' for package '%s' from repository '%s'\n", - ctx->tests[i].version, ctx->tests[i].name, ctx->tests[i].repository); + snprintf(srcdir, sizeof(srcdir), "%s/%s", ctx->storage.build_sources_dir, ctx->tests->test[i]->name); + if (git_clone(&proc, ctx->tests->test[i]->repository, srcdir, ctx->tests->test[i]->version)) { + SYSERROR("Unable to checkout tag '%s' for package '%s' from repository '%s'", + ctx->tests->test[i]->version, ctx->tests->test[i]->name, ctx->tests->test[i]->repository); return NULL; } - if (ctx->tests[i].repository_remove_tags && strlist_count(ctx->tests[i].repository_remove_tags)) { - filter_repo_tags(srcdir, ctx->tests[i].repository_remove_tags); + if (!ctx->tests->test[i]->repository_info_tag) { + ctx->tests->test[i]->repository_info_tag = strdup(git_describe(srcdir)); + } + if (!ctx->tests->test[i]->repository_info_ref) { + ctx->tests->test[i]->repository_info_ref = strdup(git_rev_parse(srcdir, ctx->tests->test[i]->version)); + } + if (ctx->tests->test[i]->repository_remove_tags && strlist_count(ctx->tests->test[i]->repository_remove_tags)) { + filter_repo_tags(srcdir, ctx->tests->test[i]->repository_remove_tags); } if (!pushd(srcdir)) { char dname[NAME_MAX]; char outdir[PATH_MAX]; - char cmd[PATH_MAX * 2]; + char *cmd = NULL; memset(dname, 0, sizeof(dname)); memset(outdir, 0, sizeof(outdir)); - memset(cmd, 0, sizeof(outdir)); - strcpy(dname, ctx->tests[i].name); + const int dep_status = check_python_package_dependencies("."); + if (dep_status) { + SYSERROR("Please replace all occurrences above with standard package specs:\n" + "\n" + " package==x.y.z\n" + " package>=x.y.z\n" + " package<=x.y.z\n" + " ...\n" + "\n"); + COE_CHECK_ABORT(true, "Unreproducible delivery"); + } + + strncpy(dname, ctx->tests->test[i]->name, sizeof(dname) - 1); + dname[sizeof(dname) - 1] = '\0'; tolower_s(dname); - sprintf(outdir, "%s/%s", ctx->storage.wheel_artifact_dir, dname); + snprintf(outdir, sizeof(outdir), "%s/%s", ctx->storage.wheel_artifact_dir, dname); if (mkdirs(outdir, 0755)) { - fprintf(stderr, "failed to create output directory: %s\n", outdir); + SYSERROR("failed to create output directory: %s", outdir); guard_strlist_free(&result); return NULL; } + if (use_builder_manylinux) { + if (delivery_build_wheels_manylinux(ctx, outdir)) { + SYSERROR("failed to generate wheel package for %s-%s", ctx->tests->test[i]->name, + ctx->tests->test[i]->version); + guard_strlist_free(&result); + guard_free(cmd); + return NULL; + } + } else if (use_builder_build || use_builder_cibuildwheel) { + if (use_builder_build) { + if (asprintf(&cmd, "-m build -w -o %s", outdir) < 0) { + SYSERROR("Unable to allocate memory for build command"); + return NULL; + } + } else if (use_builder_cibuildwheel) { + if (asprintf(&cmd, "-m cibuildwheel --output-dir %s --only cp%s-manylinux_%s", + outdir, ctx->meta.python_compact, ctx->system.arch) < 0) { + SYSERROR("Unable to allocate memory for cibuildwheel command"); + return NULL; + } + } - sprintf(cmd, "-m build -w -o %s", outdir); - if (python_exec(cmd)) { - fprintf(stderr, "failed to generate wheel package for %s-%s\n", ctx->tests[i].name, - ctx->tests[i].version); - guard_strlist_free(&result); + if (python_exec(cmd)) { + SYSERROR("failed to generate wheel package for %s-%s", ctx->tests->test[i]->name, + ctx->tests->test[i]->version); + guard_strlist_free(&result); + guard_free(cmd); + return NULL; + } + } else { + SYSERROR("unknown wheel builder backend: %s", globals.wheel_builder); return NULL; } + + guard_free(cmd); popd(); } else { - fprintf(stderr, "Unable to enter source directory %s: %s\n", srcdir, strerror(errno)); + SYSERROR("Unable to enter source directory %s: %s", srcdir, strerror(errno)); guard_strlist_free(&result); return NULL; } @@ -200,4 +499,3 @@ struct StrList *delivery_build_wheels(struct Delivery *ctx) { } return result; } - diff --git a/src/lib/delivery/delivery_conda.c b/src/lib/delivery/delivery_conda.c index 8974ae8..117e6c9 100644 --- a/src/lib/delivery/delivery_conda.c +++ b/src/lib/delivery/delivery_conda.c @@ -1,21 +1,37 @@ #include "delivery.h" +#include "conda.h" -void delivery_get_conda_installer_url(struct Delivery *ctx, char *result) { +void delivery_get_conda_installer_url(struct Delivery *ctx, char *result, size_t maxlen) { + int len = 0; if (ctx->conda.installer_version) { // Use version specified by configuration file - sprintf(result, "%s/%s-%s-%s-%s.sh", ctx->conda.installer_baseurl, + len = snprintf(NULL, 0, "%s/%s-%s-%s-%s.sh", + ctx->conda.installer_baseurl, + ctx->conda.installer_name, + ctx->conda.installer_version, + ctx->conda.installer_platform, + ctx->conda.installer_arch); + + snprintf(result, maxlen - len, "%s/%s-%s-%s-%s.sh", + ctx->conda.installer_baseurl, ctx->conda.installer_name, ctx->conda.installer_version, ctx->conda.installer_platform, ctx->conda.installer_arch); } else { // Use latest installer - sprintf(result, "%s/%s-%s-%s.sh", ctx->conda.installer_baseurl, + len = snprintf(NULL, 0, "%s/%s-%s-%s.sh", + ctx->conda.installer_baseurl, ctx->conda.installer_name, ctx->conda.installer_platform, ctx->conda.installer_arch); - } + snprintf(result, maxlen - len, "%s/%s-%s-%s.sh", + ctx->conda.installer_baseurl, + ctx->conda.installer_name, + ctx->conda.installer_platform, + ctx->conda.installer_arch); + } } int delivery_get_conda_installer(struct Delivery *ctx, char *installer_url) { @@ -23,12 +39,15 @@ int delivery_get_conda_installer(struct Delivery *ctx, char *installer_url) { char *installer = path_basename(installer_url); memset(script_path, 0, sizeof(script_path)); - sprintf(script_path, "%s/%s", ctx->storage.tmpdir, installer); + snprintf(script_path, sizeof(script_path), "%s/%s", ctx->storage.tmpdir, installer); if (access(script_path, F_OK)) { // Script doesn't exist - long fetch_status = download(installer_url, script_path, NULL); + char *errmsg = NULL; + long fetch_status = download(installer_url, script_path, &errmsg); if (HTTP_ERROR(fetch_status) || fetch_status < 0) { // download failed + SYSERROR("download failed: %s: %s", errmsg, installer_url); + guard_free(errmsg); return -1; } } else { @@ -51,31 +70,31 @@ void delivery_install_conda(char *install_script, char *conda_install_dir) { if (!access(conda_install_dir, F_OK)) { // directory exists so remove it if (rmtree(conda_install_dir)) { - perror("unable to remove previous installation"); + SYSERROR("unable to remove previous installation: %s", strerror(errno)); exit(1); } // Proceed with the installation // -b = batch mode (non-interactive) char cmd[PATH_MAX] = {0}; - snprintf(cmd, sizeof(cmd) - 1, "%s %s -b -p %s", + snprintf(cmd, sizeof(cmd), "%s %s -b -p %s", find_program("bash"), install_script, conda_install_dir); if (shell_safe(&proc, cmd)) { - fprintf(stderr, "conda installation failed\n"); + SYSERROR("conda installation failed"); exit(1); } } else { // Proceed with the installation // -b = batch mode (non-interactive) char cmd[PATH_MAX] = {0}; - snprintf(cmd, sizeof(cmd) - 1, "%s %s -b -p %s", + snprintf(cmd, sizeof(cmd), "%s %s -b -p %s", find_program("bash"), install_script, conda_install_dir); if (shell_safe(&proc, cmd)) { - fprintf(stderr, "conda installation failed\n"); + SYSERROR("conda installation failed"); exit(1); } } @@ -86,7 +105,7 @@ void delivery_install_conda(char *install_script, char *conda_install_dir) { void delivery_conda_enable(struct Delivery *ctx, char *conda_install_dir) { if (conda_activate(conda_install_dir, "base")) { - fprintf(stderr, "conda activation failed\n"); + SYSERROR("conda activation failed"); exit(1); } @@ -94,10 +113,10 @@ void delivery_conda_enable(struct Delivery *ctx, char *conda_install_dir) { // way to make sure the file is used. Not setting this variable leads to strange // behavior, especially if a conda environment is already active when STASIS is loaded. char rcpath[PATH_MAX]; - sprintf(rcpath, "%s/%s", conda_install_dir, ".condarc"); + snprintf(rcpath, sizeof(rcpath), "%s/%s", conda_install_dir, ".condarc"); setenv("CONDARC", rcpath, 1); if (runtime_replace(&ctx->runtime.environ, __environ)) { - perror("unable to replace runtime environment after activating conda"); + SYSERROR("unable to replace runtime environment after activating conda"); exit(1); } diff --git a/src/lib/delivery/delivery_docker.c b/src/lib/delivery/delivery_docker.c index 57015ad..79e9729 100644 --- a/src/lib/delivery/delivery_docker.c +++ b/src/lib/delivery/delivery_docker.c @@ -11,15 +11,15 @@ int delivery_docker(struct Delivery *ctx) { size_t total_build_args = strlist_count(ctx->deploy.docker.build_args); if (!has_registry) { - msg(STASIS_MSG_WARN | STASIS_MSG_L2, "No docker registry defined. You will need to manually re-tag the resulting image.\n"); + SYSWARN("No docker registry defined. You will need to manually re-tag the resulting image."); } if (!total_tags) { char default_tag[PATH_MAX]; - msg(STASIS_MSG_WARN | STASIS_MSG_L2, "No docker tags defined by configuration. Generating default tag(s).\n"); + SYSWARN("No docker tags defined by configuration. Generating default tag(s)."); // generate local tag memset(default_tag, 0, sizeof(default_tag)); - sprintf(default_tag, "%s:%s-py%s", ctx->meta.name, ctx->info.build_name, ctx->meta.python_compact); + snprintf(default_tag, sizeof(default_tag), "%s:%s-py%s", ctx->meta.name, ctx->info.build_name, ctx->meta.python_compact); tolower_s(default_tag); // Add tag @@ -29,7 +29,7 @@ int delivery_docker(struct Delivery *ctx) { if (has_registry) { // generate tag for target registry memset(default_tag, 0, sizeof(default_tag)); - sprintf(default_tag, "%s/%s:%s-py%s", ctx->deploy.docker.registry, ctx->meta.name, ctx->info.build_number, ctx->meta.python_compact); + snprintf(default_tag, sizeof(default_tag), "%s/%s:%s-py%s", ctx->deploy.docker.registry, ctx->meta.name, ctx->info.build_number, ctx->meta.python_compact); tolower_s(default_tag); // Add tag @@ -44,9 +44,10 @@ int delivery_docker(struct Delivery *ctx) { // Append image tags to command for (size_t i = 0; i < total_tags; i++) { char *tag_orig = strlist_item(ctx->deploy.docker.tags, i); - strcpy(tag, tag_orig); + strncpy(tag, tag_orig, sizeof(tag) - 1); + tag[sizeof(tag) - 1] = '\0'; docker_sanitize_tag(tag); - sprintf(args + strlen(args), " -t \"%s\" ", tag); + snprintf(args + strlen(args), sizeof(args) - strlen(args), " -t \"%s\" ", tag); } // Append build arguments to command (i.e. --build-arg "key=value" @@ -55,7 +56,7 @@ int delivery_docker(struct Delivery *ctx) { if (!build_arg) { break; } - sprintf(args + strlen(args), " --build-arg \"%s\" ", build_arg); + snprintf(args + strlen(args), sizeof(args) - strlen(args), " --build-arg \"%s\" ", build_arg); } // Build the image @@ -65,34 +66,34 @@ int delivery_docker(struct Delivery *ctx) { memset(delivery_file, 0, sizeof(delivery_file)); memset(dest, 0, sizeof(dest)); - sprintf(delivery_file, "%s/%s.yml", ctx->storage.delivery_dir, ctx->info.release_name); + snprintf(delivery_file, sizeof(delivery_file), "%s/%s.yml", ctx->storage.delivery_dir, ctx->info.release_name); if (access(delivery_file, F_OK) < 0) { - fprintf(stderr, "docker build cannot proceed without delivery file: %s\n", delivery_file); + SYSERROR("docker build cannot proceed without delivery file: %s", delivery_file); return -1; } - sprintf(dest, "%s/%s.yml", ctx->storage.build_docker_dir, ctx->info.release_name); + snprintf(dest, sizeof(dest), "%s/%s.yml", ctx->storage.build_docker_dir, ctx->info.release_name); if (copy2(delivery_file, dest, CT_PERM)) { - fprintf(stderr, "Failed to copy delivery file to %s: %s\n", dest, strerror(errno)); + SYSERROR("Failed to copy delivery file to %s: %s", dest, strerror(errno)); return -1; } memset(dest, 0, sizeof(dest)); - sprintf(dest, "%s/packages", ctx->storage.build_docker_dir); + snprintf(dest, sizeof(dest), "%s/packages", ctx->storage.build_docker_dir); msg(STASIS_MSG_L2, "Copying conda packages\n"); memset(rsync_cmd, 0, sizeof(rsync_cmd)); - sprintf(rsync_cmd, "rsync -avi --progress '%s' '%s'", ctx->storage.conda_artifact_dir, dest); + snprintf(rsync_cmd, sizeof(rsync_cmd), "rsync -avi --progress '%s' '%s'", ctx->storage.conda_artifact_dir, dest); if (system(rsync_cmd)) { - fprintf(stderr, "Failed to copy conda artifacts to docker build directory\n"); + SYSERROR("Failed to copy conda artifacts to docker build directory"); return -1; } msg(STASIS_MSG_L2, "Copying wheel packages\n"); memset(rsync_cmd, 0, sizeof(rsync_cmd)); - sprintf(rsync_cmd, "rsync -avi --progress '%s' '%s'", ctx->storage.wheel_artifact_dir, dest); + snprintf(rsync_cmd, sizeof(rsync_cmd), "rsync -avi --progress '%s' '%s'", ctx->storage.wheel_artifact_dir, dest); if (system(rsync_cmd)) { - fprintf(stderr, "Failed to copy wheel artifacts to docker build directory\n"); + SYSWARN("Failed to copy wheel artifacts to docker build directory. No wheels produced?"); } if (docker_build(ctx->storage.build_docker_dir, args, ctx->deploy.docker.capabilities.build)) { @@ -102,23 +103,24 @@ int delivery_docker(struct Delivery *ctx) { // Test the image // All tags point back to the same image so test the first one we see // regardless of how many are defined - strcpy(tag, strlist_item(ctx->deploy.docker.tags, 0)); + strncpy(tag, strlist_item(ctx->deploy.docker.tags, 0), sizeof(tag) - 1); + tag[sizeof(tag) - 1] = '\0'; docker_sanitize_tag(tag); msg(STASIS_MSG_L2, "Executing image test script for %s\n", tag); if (ctx->deploy.docker.test_script) { if (isempty(ctx->deploy.docker.test_script)) { - msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "Image test script has no content\n"); + SYSWARN("Image test script has no content"); } else { int state; - if ((state = docker_script(tag, ctx->deploy.docker.test_script, 0))) { - msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "Non-zero exit (%d) from test script. %s image archive will not be generated.\n", state >> 8, tag); + if ((state = docker_script(tag, "--rm", ctx->deploy.docker.test_script, 0))) { + SYSERROR("Non-zero exit (%d) from test script. %s image archive will not be generated.", state >> 8, tag); // test failed -- don't save the image return -1; } } } else { - msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "No image test script defined\n"); + SYSWARN("No image test script defined"); } // Test successful, save image diff --git a/src/lib/delivery/delivery_export.c b/src/lib/delivery/delivery_export.c new file mode 100644 index 0000000..0321050 --- /dev/null +++ b/src/lib/delivery/delivery_export.c @@ -0,0 +1,77 @@ +#include "delivery.h" +#include "conda.h" + +static void delivery_export_configuration(const struct Delivery *ctx) { + msg(STASIS_MSG_L2, "Exporting delivery configuration\n"); + + SYSDEBUG("Entering configuration directory: %s", ctx->storage.delivery_dir); + if (!pushd(ctx->storage.cfgdump_dir)) { + char filename[PATH_MAX] = {0}; + SYSDEBUG("Populating filename"); + snprintf(filename, sizeof(filename), "%s.ini", ctx->info.release_name); + SYSDEBUG("filename: %s", filename); + + SYSDEBUG("%s: opening", filename); + FILE *spec = fopen(filename, "w+"); + if (!spec) { + SYSERROR("open failed %s", filename); + exit(1); + } + SYSDEBUG("%s: writing", filename); + ini_write(ctx->_stasis_ini_fp.delivery, &spec, INI_WRITE_RAW); + SYSDEBUG("%s: writing done", filename); + fclose(spec); + SYSDEBUG("%s: closing", filename); + + SYSDEBUG("Zeroing filename"); + memset(filename, 0, sizeof(filename)); + SYSDEBUG("Populating rendered filename"); + snprintf(filename, sizeof(filename), "%s-rendered.ini", ctx->info.release_name); + SYSDEBUG("filename: %s", filename); + + SYSDEBUG("%s: opening", filename); + spec = fopen(filename, "w+"); + if (!spec) { + SYSERROR("open failed %s", filename); + exit(1); + } + SYSDEBUG("%s: writing", filename); + ini_write(ctx->_stasis_ini_fp.delivery, &spec, INI_WRITE_PRESERVE); + SYSDEBUG("%s: writing done", filename); + SYSDEBUG("%s: closing", filename); + fclose(spec); + SYSDEBUG("Returning from %s", ctx->storage.cfgdump_dir); + popd(); + } else { + SYSERROR("Failed to enter directory: %s", ctx->storage.delivery_dir); + exit(1); + } +} + +void delivery_export(const struct Delivery *ctx, char *envs[]) { + delivery_export_configuration(ctx); + + for (size_t i = 0; envs[i] != NULL; i++) { + char *name = envs[i]; + msg(STASIS_MSG_L2, "Exporting %s\n", name); + if (conda_env_export(name, ctx->storage.delivery_dir, name)) { + SYSERROR("export failed %s", name); + exit(1); + } + } +} + +void delivery_rewrite_stage1(struct Delivery *ctx, char *specfile) { + // Rewrite release environment output (i.e. set package origin(s) to point to the deployment server, etc.) + msg(STASIS_MSG_L3, "Rewriting release spec file (stage 1): %s\n", path_basename(specfile)); + delivery_rewrite_spec(ctx, specfile, DELIVERY_REWRITE_SPEC_STAGE_1); + + msg(STASIS_MSG_L1, "Rendering mission templates\n"); + delivery_mission_render_files(ctx); +} + +void delivery_rewrite_stage2(struct Delivery *ctx, char *specfile) { + msg(STASIS_MSG_L3, "Rewriting release spec file (stage 2): %s\n", path_basename(specfile)); + delivery_rewrite_spec(ctx, specfile, DELIVERY_REWRITE_SPEC_STAGE_2); +} + diff --git a/src/lib/delivery/delivery_init.c b/src/lib/delivery/delivery_init.c index 56c591a..5bc326d 100644 --- a/src/lib/delivery/delivery_init.c +++ b/src/lib/delivery/delivery_init.c @@ -1,3 +1,6 @@ +#include <fnmatch.h> +#include <sys/utsname.h> + #include "delivery.h" int has_mount_flags(const char *mount_point, const unsigned long flags) { @@ -11,34 +14,54 @@ int has_mount_flags(const char *mount_point, const unsigned long flags) { int delivery_init_tmpdir(struct Delivery *ctx) { char *tmpdir = NULL; - char *x = NULL; - int unusable = 0; + int unusable = 1; errno = 0; - x = getenv("TMPDIR"); + //int need_setenv = 0; + const char *x = getenv("TMPDIR"); if (x) { guard_free(ctx->storage.tmpdir); tmpdir = strdup(x); + if (!tmpdir) { + // memory error + SYSERROR("unable to allocate tmpdir"); + goto l_delivery_init_tmpdir_fatal; + } } else { - tmpdir = ctx->storage.tmpdir; + tmpdir = strdup("/tmp/stasis"); + if (!tmpdir) { + SYSERROR("unable to allocate tmpdir"); + goto l_delivery_init_tmpdir_fatal; + } + //need_setenv = 1; } - if (!tmpdir) { - // memory error - return -1; + if (!ctx->storage.tmpdir) { + ctx->storage.tmpdir = strdup(tmpdir); + if (!ctx->storage.tmpdir) { + SYSERROR("unable to allocate ctx->storage.tmpdir"); + goto l_delivery_init_tmpdir_fatal; + } + } else { + // we already have a temp directory to use + guard_free(tmpdir); + tmpdir = strdup(ctx->storage.tmpdir); + if (!tmpdir) { + SYSERROR("unable to allocate tmpdir"); + goto l_delivery_init_tmpdir_fatal; + } } - // If the directory doesn't exist, create it if (access(tmpdir, F_OK) < 0) { if (mkdirs(tmpdir, 0755) < 0) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Unable to create temporary storage directory: %s (%s)\n", tmpdir, strerror(errno)); + SYSERROR("Unable to create temporary storage directory: %s (%s)", tmpdir, strerror(errno)); goto l_delivery_init_tmpdir_fatal; } } // If we can't read, write, or execute, then die if (access(tmpdir, R_OK | W_OK | X_OK) < 0) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "%s requires at least 0755 permissions.\n"); + SYSERROR("%s requires at least 0755 permissions."); goto l_delivery_init_tmpdir_fatal; } @@ -50,26 +73,38 @@ int delivery_init_tmpdir(struct Delivery *ctx) { #if defined(STASIS_OS_LINUX) // If we can't execute programs, or write data to the file system at all, then die if ((st.f_flag & ST_NOEXEC) != 0) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "%s is mounted with noexec\n", tmpdir); + SYSERROR("%s is mounted with noexec", tmpdir); goto l_delivery_init_tmpdir_fatal; } #endif if ((st.f_flag & ST_RDONLY) != 0) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "%s is mounted read-only\n", tmpdir); + SYSERROR("%s is mounted read-only", tmpdir); goto l_delivery_init_tmpdir_fatal; } - if (!globals.tmpdir) { + if (!globals.tmpdir || strcmp(globals.tmpdir, ctx->storage.tmpdir) != 0) { globals.tmpdir = strdup(tmpdir); + if (!globals.tmpdir) { + SYSERROR("unable to allocate globals.tmpdir"); + goto l_delivery_init_tmpdir_fatal; + } } if (!ctx->storage.tmpdir) { ctx->storage.tmpdir = strdup(globals.tmpdir); + if (!ctx->storage.tmpdir) { + SYSERROR("unable to allocate globals.tmpdir"); + goto l_delivery_init_tmpdir_fatal; + } } - return unusable; + unusable = 0; + // TODO: Figure out why this breaks EVERYTHING + //if (need_setenv) { + // setenv("TMPDIR", ctx->storage.tmpdir, 1); + //} l_delivery_init_tmpdir_fatal: - unusable = 1; + guard_free(tmpdir); return unusable; } @@ -94,7 +129,7 @@ void delivery_init_dirs_stage1(struct Delivery *ctx) { char *rootdir = getenv("STASIS_ROOT"); if (rootdir) { if (isempty(rootdir)) { - fprintf(stderr, "STASIS_ROOT is set, but empty. Please assign a file system path to this environment variable.\n"); + SYSERROR("STASIS_ROOT is set, but empty. Please assign a file system path to this environment variable."); exit(1); } path_store(&ctx->storage.root, PATH_MAX, rootdir, ctx->info.build_name); @@ -105,9 +140,10 @@ void delivery_init_dirs_stage1(struct Delivery *ctx) { path_store(&ctx->storage.tools_dir, PATH_MAX, ctx->storage.root, "tools"); path_store(&ctx->storage.tmpdir, PATH_MAX, ctx->storage.root, "tmp"); if (delivery_init_tmpdir(ctx)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Set $TMPDIR to a location other than %s\n", globals.tmpdir); - if (globals.tmpdir) + SYSERROR("Set $TMPDIR to a location other than %s", globals.tmpdir); + if (globals.tmpdir) { guard_free(globals.tmpdir); + } exit(1); } @@ -119,7 +155,7 @@ void delivery_init_dirs_stage1(struct Delivery *ctx) { } if (access(ctx->storage.mission_dir, F_OK)) { - msg(STASIS_MSG_L1, "%s: %s\n", ctx->storage.mission_dir, strerror(errno)); + msg(STASIS_MSG_L1, "%s: %s: mission directory does not exist\n", ctx->storage.mission_dir, strerror(errno)); exit(1); } @@ -150,60 +186,77 @@ void delivery_init_dirs_stage1(struct Delivery *ctx) { } int delivery_init_platform(struct Delivery *ctx) { - msg(STASIS_MSG_L2, "Setting architecture\n"); + SYSDEBUG("Setting architecture"); char archsuffix[20]; struct utsname uts; if (uname(&uts)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "uname() failed: %s\n", strerror(errno)); + SYSERROR("uname() failed: %s", strerror(errno)); return -1; } ctx->system.platform = calloc(DELIVERY_PLATFORM_MAX + 1, sizeof(*ctx->system.platform)); if (!ctx->system.platform) { - SYSERROR("Unable to allocate %d records for platform array\n", DELIVERY_PLATFORM_MAX); + SYSERROR("Unable to allocate %d records for platform array", DELIVERY_PLATFORM_MAX); return -1; } for (size_t i = 0; i < DELIVERY_PLATFORM_MAX; i++) { ctx->system.platform[i] = calloc(DELIVERY_PLATFORM_MAXLEN, sizeof(*ctx->system.platform[0])); + if (!ctx->system.platform[i]) { + SYSERROR("Unable to allocate record %zu in platform array", i); + guard_array_n_free(ctx->system.platform, i); + return -1; + } } ctx->system.arch = strdup(uts.machine); if (!ctx->system.arch) { // memory error + guard_array_n_free(ctx->system.platform, DELIVERY_PLATFORM_MAX); + ctx->system.platform = NULL; return -1; } if (!strcmp(ctx->system.arch, "x86_64")) { - strcpy(archsuffix, "64"); + strncpy(archsuffix, "64", sizeof(archsuffix) - 1); } else { - strcpy(archsuffix, ctx->system.arch); + strncpy(archsuffix, ctx->system.arch, sizeof(archsuffix) - 1); } + archsuffix[sizeof(archsuffix) - 1] = '\0'; + + SYSDEBUG("Setting platform"); + strncpy(ctx->system.platform[DELIVERY_PLATFORM], uts.sysname, DELIVERY_PLATFORM_MAXLEN - 1); + ctx->system.platform[DELIVERY_PLATFORM][DELIVERY_PLATFORM_MAXLEN - 1] = '\0'; - msg(STASIS_MSG_L2, "Setting platform\n"); - strcpy(ctx->system.platform[DELIVERY_PLATFORM], uts.sysname); if (!strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Darwin")) { - sprintf(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], "osx-%s", archsuffix); - strcpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], "MacOSX"); - strcpy(ctx->system.platform[DELIVERY_PLATFORM_RELEASE], "macos"); + snprintf(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], DELIVERY_PLATFORM_MAXLEN, "osx-%s", archsuffix); + strncpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], "MacOSX", DELIVERY_PLATFORM_MAXLEN - 1); + ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER][DELIVERY_PLATFORM_MAXLEN - 1] = '\0'; + strncpy(ctx->system.platform[DELIVERY_PLATFORM_RELEASE], "macos", DELIVERY_PLATFORM_MAXLEN - 1); + ctx->system.platform[DELIVERY_PLATFORM_RELEASE][DELIVERY_PLATFORM_MAXLEN - 1] = '\0'; } else if (!strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Linux")) { - sprintf(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], "linux-%s", archsuffix); - strcpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], "Linux"); - strcpy(ctx->system.platform[DELIVERY_PLATFORM_RELEASE], "linux"); + snprintf(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], DELIVERY_PLATFORM_MAXLEN, "linux-%s", archsuffix); + strncpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], "Linux", DELIVERY_PLATFORM_MAXLEN - 1); + ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER][DELIVERY_PLATFORM_MAXLEN - 1] = '\0'; + strncpy(ctx->system.platform[DELIVERY_PLATFORM_RELEASE], "linux", DELIVERY_PLATFORM_MAXLEN - 1); + ctx->system.platform[DELIVERY_PLATFORM_RELEASE][DELIVERY_PLATFORM_MAXLEN - 1] = '\0'; } else { // Not explicitly supported systems - strcpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], ctx->system.platform[DELIVERY_PLATFORM]); - strcpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], ctx->system.platform[DELIVERY_PLATFORM]); - strcpy(ctx->system.platform[DELIVERY_PLATFORM_RELEASE], ctx->system.platform[DELIVERY_PLATFORM]); + strncpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], ctx->system.platform[DELIVERY_PLATFORM], DELIVERY_PLATFORM_MAXLEN - 1); + ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR][DELIVERY_PLATFORM_MAXLEN - 1] = '\0'; + strncpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], ctx->system.platform[DELIVERY_PLATFORM], DELIVERY_PLATFORM_MAXLEN - 1); + ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER][DELIVERY_PLATFORM_MAXLEN - 1] = '\0'; + strncpy(ctx->system.platform[DELIVERY_PLATFORM_RELEASE], ctx->system.platform[DELIVERY_PLATFORM], DELIVERY_PLATFORM_MAXLEN - 1); + ctx->system.platform[DELIVERY_PLATFORM_RELEASE][DELIVERY_PLATFORM_MAXLEN - 1] = '\0'; tolower_s(ctx->system.platform[DELIVERY_PLATFORM_RELEASE]); } long cpu_count = get_cpu_count(); if (!cpu_count) { - fprintf(stderr, "Unable to determine CPU count. Falling back to 1.\n"); + SYSERROR("Unable to determine CPU count. Falling back to 1."); cpu_count = 1; } char ncpus[100] = {0}; - sprintf(ncpus, "%ld", cpu_count); + snprintf(ncpus, sizeof(ncpus), "%ld", cpu_count); // Declare some important bits as environment variables setenv("CPU_COUNT", ncpus, 1); @@ -246,22 +299,33 @@ int delivery_init(struct Delivery *ctx, int render_mode) { } // Configure architecture and platform information - delivery_init_platform(ctx); + if (delivery_init_platform(ctx)) { + // memory error + return -1; + } // Create STASIS directory structure delivery_init_dirs_stage1(ctx); char config_local[PATH_MAX]; - sprintf(config_local, "%s/%s", ctx->storage.tmpdir, "config"); + snprintf(config_local, sizeof(config_local), "%s/%s", ctx->storage.tmpdir, "config"); setenv("XDG_CONFIG_HOME", config_local, 1); + if (mkdirs(config_local, 0755)) { + SYSERROR("%s: unable to create directory", config_local); + // fall through because XDG doesn't _really_ need to be there + } char cache_local[PATH_MAX]; - sprintf(cache_local, "%s/%s", ctx->storage.tmpdir, "cache"); + snprintf(cache_local, sizeof(cache_local), "%s/%s", ctx->storage.tmpdir, "cache"); setenv("XDG_CACHE_HOME", cache_local, 1); + if (mkdirs(cache_local, 0755)) { + SYSERROR("%s: unable to create directory", cache_local); + // fall through because XDG doesn't _really_ need to be there + } // add tools to PATH char pathvar_tmp[STASIS_BUFSIZ]; - sprintf(pathvar_tmp, "%s/bin:%s", ctx->storage.tools_dir, getenv("PATH")); + snprintf(pathvar_tmp, sizeof(pathvar_tmp), "%s/bin:%s", ctx->storage.tools_dir, getenv("PATH")); setenv("PATH", pathvar_tmp, 1); // Prevent git from paginating output @@ -287,18 +351,31 @@ int delivery_init(struct Delivery *ctx, int render_mode) { int bootstrap_build_info(struct Delivery *ctx) { struct Delivery local = {0}; + memcpy(&local.deploy.docker.capabilities, &ctx->deploy.docker.capabilities, sizeof(local.deploy.docker.capabilities)); + + SYSDEBUG("ini_open(%s)", ctx->_stasis_ini_fp.cfg_path); local._stasis_ini_fp.cfg = ini_open(ctx->_stasis_ini_fp.cfg_path); + SYSDEBUG("ini_open(%s)", ctx->_stasis_ini_fp.delivery_path); local._stasis_ini_fp.delivery = ini_open(ctx->_stasis_ini_fp.delivery_path); + if (delivery_init_platform(&local)) { + SYSDEBUG("delivery_init_platform failed"); + delivery_free(&local); return -1; } if (populate_delivery_cfg(&local, INI_READ_RENDER)) { + SYSDEBUG("populate_delivery_cfg failed"); + delivery_free(&local); return -1; } if (populate_delivery_ini(&local, INI_READ_RENDER)) { + SYSDEBUG("populate_delivery_ini failed"); + delivery_free(&local); return -1; } if (populate_info(&local)) { + SYSDEBUG("populate_info failed"); + delivery_free(&local); return -1; } ctx->info.build_name = strdup(local.info.build_name); @@ -308,12 +385,14 @@ int bootstrap_build_info(struct Delivery *ctx) { ctx->info.time_info = malloc(sizeof(*ctx->info.time_info)); if (!ctx->info.time_info) { SYSERROR("Unable to allocate %zu bytes for tm struct: %s", sizeof(*local.info.time_info), strerror(errno)); + delivery_free(&local); return -1; } } memcpy(ctx->info.time_info, local.info.time_info, sizeof(*local.info.time_info)); ctx->info.time_now = local.info.time_now; ctx->info.time_str_epoch = strdup(local.info.time_str_epoch); + SYSDEBUG("delivery_free local resources"); delivery_free(&local); return 0; } @@ -321,11 +400,11 @@ int bootstrap_build_info(struct Delivery *ctx) { int delivery_exists(struct Delivery *ctx) { int release_exists = DELIVERY_NOT_FOUND; char release_pattern[PATH_MAX] = {0}; - sprintf(release_pattern, "*%s*", ctx->info.release_name); + snprintf(release_pattern, sizeof(release_pattern), "*%s*", ctx->info.release_name); if (globals.enable_artifactory) { if (jfrt_auth_init(&ctx->deploy.jfrog_auth)) { - fprintf(stderr, "Failed to initialize Artifactory authentication context\n"); + SYSERROR("Failed to initialize Artifactory authentication context"); return -1; // error } diff --git a/src/lib/delivery/delivery_install.c b/src/lib/delivery/delivery_install.c index 246c604..efdb819 100644 --- a/src/lib/delivery/delivery_install.c +++ b/src/lib/delivery/delivery_install.c @@ -1,8 +1,11 @@ #include "delivery.h" +#include "conda.h" +#include "wheelinfo.h" +#include "version_compare.h" static struct Test *requirement_from_test(struct Delivery *ctx, const char *name) { struct Test *result = NULL; - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { + for (size_t i = 0; i < ctx->tests->num_used; i++) { char *package_name = strdup(name); if (package_name) { char *spec = find_version_spec(package_name); @@ -11,8 +14,8 @@ static struct Test *requirement_from_test(struct Delivery *ctx, const char *name } remove_extras(package_name); - if (ctx->tests[i].name && !strcmp(package_name, ctx->tests[i].name)) { - result = &ctx->tests[i]; + if (ctx->tests->test[i]->name && !strcmp(package_name, ctx->tests->test[i]->name)) { + result = ctx->tests->test[i]; guard_free(package_name); break; } @@ -32,11 +35,13 @@ static char *have_spec_in_config(const struct Delivery *ctx, const char *name) { char package[255] = {0}; if (op) { strncpy(package, config_spec, op - config_spec); + package[op - config_spec] = '\0'; } else { strncpy(package, config_spec, sizeof(package) - 1); + package[sizeof(package) - 1] = '\0'; } remove_extras(package); - if (strncmp(package, name, strlen(package)) == 0) { + if (strncmp(package, name, strlen(name)) == 0) { return config_spec; } } @@ -81,8 +86,10 @@ int delivery_overlay_packages_from_env(struct Delivery *ctx, const char *env_nam char *op = find_version_spec(spec); if (op) { strncpy(spec_name, spec, op - spec); + spec_name[op - spec] = '\0'; } else { strncpy(spec_name, spec, sizeof(spec_name) - 1); + spec_name[sizeof(spec_name) - 1] = '\0'; } struct Test *test_block = requirement_from_test(ctx, spec_name); @@ -102,8 +109,10 @@ int delivery_overlay_packages_from_env(struct Delivery *ctx, const char *env_nam // we only care about packages with specs here. if something else arrives, ignore it if (op) { strncpy(frozen_name, frozen_spec, op - frozen_spec); + frozen_name[op - frozen_spec] = '\0'; } else { strncpy(frozen_name, frozen_spec, sizeof(frozen_name) - 1); + frozen_name[sizeof(frozen_name) - 1] = '\0'; } struct Test *test = requirement_from_test(ctx, frozen_name); if (test && strcmp(test->name, frozen_name) == 0) { @@ -128,6 +137,159 @@ int delivery_overlay_packages_from_env(struct Delivery *ctx, const char *env_nam return 0; } +int delivery_conda_enforce_package_version(struct Delivery *ctx, const char *env_name, const char *name) { + char *spec_installed = NULL; + char *spec_request = NULL; + int status = 0; + + if (isempty((char *) env_name)) { + SYSERROR("environment name cannot be NULL or empty"); + return -1; + } + if (isempty((char *) name)) { + SYSERROR("name cannot be NULL or empty"); + return -1; + } + + int proc_status = 0; + char cmd[PATH_MAX] = {0}; + snprintf(cmd, PATH_MAX, "conda list --name %s", env_name); + + char *output = shell_output(cmd, &proc_status); + if (!output || proc_status) { + SYSERROR("unable to retreive list of installed packages (exit: %d)", proc_status); + guard_free(output); + return -1; + } + + struct StrList *lines = strlist_init(); + if (!lines) { + SYSERROR("unable to allocate memory for installed package list"); + guard_free(output); + status = -1; + goto cleanup; + } + + if (strlist_append_tokenize(lines, output, LINE_SEP)) { + SYSERROR("unable to tokenize installed package list"); + guard_free(output); + strlist_free(&lines); + status = -1; + goto cleanup; + } + + for (size_t i = 0; i < strlist_count(lines); i++) { + char *line = strlist_item(lines, i); + if (!line) { + SYSERROR("line is NULL"); + status = -1; + goto cleanup; + } + if (startswith(line, "#") || isempty(line)) { + continue; + } + collapse_whitespace(&line); + strip(line); + + struct StrList *tokens = strlist_init(); + if (!tokens) { + SYSERROR("unable to allocate memory for tokenized installed package list"); + status = -1; + goto cleanup; + } + + if (strlist_append_tokenize(tokens, line, " ")) { + SYSERROR("unable to tokenize installed package list"); + status = -1; + goto cleanup; + } + + const char *installed_version = strlist_item(tokens, 1); + if (!installed_version) { + SYSERROR("not enough data in line (name and version not found)"); + guard_strlist_free(&tokens); + status = -1; + goto cleanup; + } + + if (strstr(line, name)) { + spec_installed = strdup(installed_version); + if (!spec_installed) { + SYSERROR("unable to allocated memory for installed package version"); + guard_strlist_free(&tokens); + status = -1; + goto cleanup; + } + guard_strlist_free(&tokens); + break; + } + + guard_strlist_free(&tokens); + } + + for (size_t i = 0; i < strlist_count(ctx->conda.conda_packages); i++) { + const char *item = strlist_item(ctx->conda.conda_packages, i); + if (!item) { + SYSERROR("conda_packages list record %zu is NULL", i); + status = -1; + goto cleanup; + } + if (strstr(item, name)) { + const char *spec_tmp = find_version_spec((char *) item); + if (spec_tmp) { + while (!isalnum(*spec_tmp)) { + spec_tmp++; + } + spec_request = strdup(spec_tmp); + } else { + spec_request = strdup(item); + } + + if (!spec_request) { + SYSERROR("unable to allocate memory for conda package spec request"); + status = -1; + goto cleanup; + } + break; + } + } + + const int stop = version_compare(NOT | EQ, spec_request, spec_installed); + if (stop < 0) { + SYSERROR("version comparison failed (spec_request: %s, spec_installed: %s)", spec_request, spec_installed); + status = -1; + goto cleanup; + } + if (stop == 0) { + goto cleanup; + } + + snprintf(cmd, PATH_MAX, "remove --name %s %s", env_name, name); + if (conda_exec(cmd)) { + SYSERROR("unable to remove package %s from %s", name, env_name); + status = -1; + goto cleanup; + } + snprintf(cmd, PATH_MAX, "install --name %s %s=%s", env_name, name, spec_request); + if (conda_exec(cmd)) { + SYSERROR("unable to install package %s into %s", name, env_name); + status = -1; + goto cleanup; + } + + cleanup: + guard_free(spec_request); + guard_free(spec_installed); + strlist_free(&lines); + guard_free(output); + return status; +} + +static int fn_nop(const char *command) { + (void) command; + return 1; +} + int delivery_purge_packages(struct Delivery *ctx, const char *env_name, int use_pkg_manager) { int status = 0; char subcommand[100] = {0}; @@ -145,19 +307,24 @@ int delivery_purge_packages(struct Delivery *ctx, const char *env_name, int use_ case PKG_USE_CONDA: fn = conda_exec; list = ctx->conda.conda_packages_purge; - strcpy(package_manager, "conda"); + strncpy(package_manager, "conda", sizeof(package_manager) - 1); + package_manager[sizeof(package_manager) - 1] = '\0'; // conda is already configured for "always_yes" - strcpy(subcommand, "remove"); + strncpy(subcommand, "remove", sizeof(subcommand) - 1); + subcommand[sizeof(subcommand) - 1] = '\0'; break; case PKG_USE_PIP: fn = pip_exec; list = ctx->conda.pip_packages_purge; - strcpy(package_manager, "pip"); + strncpy(package_manager, "pip", sizeof(package_manager) - 1); + package_manager[sizeof(package_manager) - 1] = '\0'; // avoid user prompt to remove packages - strcpy(subcommand, "uninstall -y"); + strncpy(subcommand, "uninstall -y", sizeof(subcommand) - 1); + subcommand[sizeof(subcommand) - 1] = '\0'; break; default: SYSERROR("Unknown package manager: %d", use_pkg_manager); + fn = fn_nop; status = -1; break; } @@ -166,7 +333,7 @@ int delivery_purge_packages(struct Delivery *ctx, const char *env_name, int use_ char *package = strlist_item(list, i); char *command = NULL; if (asprintf(&command, "%s '%s'", subcommand, package) < 0) { - SYSERROR("%s", "Unable to allocate bytes for removal command"); + SYSERROR("Unable to allocate bytes for removal command"); status = -1; break; } @@ -175,11 +342,12 @@ int delivery_purge_packages(struct Delivery *ctx, const char *env_name, int use_ SYSERROR("%s removal operation failed", package_manager); guard_free(command); status = 1; - break; + goto cleanup; } guard_free(command); } + cleanup: if (current_env) { conda_activate(ctx->storage.conda_install_prefix, current_env); guard_free(current_env); @@ -189,8 +357,7 @@ int delivery_purge_packages(struct Delivery *ctx, const char *env_name, int use_ } int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, char *env_name, int type, struct StrList **manifest) { - char cmd[PATH_MAX]; - char pkgs[STASIS_BUFSIZ]; + char command_base[PATH_MAX]; const char *env_current = getenv("CONDA_DEFAULT_ENV"); if (env_current) { @@ -203,9 +370,8 @@ int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, cha } } - memset(cmd, 0, sizeof(cmd)); - memset(pkgs, 0, sizeof(pkgs)); - strcat(cmd, "install"); + memset(command_base, 0, sizeof(command_base)); + strncat(command_base, "install", sizeof(command_base) - strlen(command_base) - 1); typedef int (*Runner)(const char *); Runner runner = NULL; @@ -215,18 +381,31 @@ int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, cha runner = pip_exec; } + if (!runner) { + SYSERROR("Invalid callback runner of type: %d", type); + return -1; + } + if (INSTALL_PKG_CONDA_DEFERRED & type) { - strcat(cmd, " --use-local"); + strncat(command_base, " --use-local", sizeof(command_base) - strlen(command_base) - 1); + command_base[sizeof(command_base) - 1] = '\0'; } else if (INSTALL_PKG_PIP_DEFERRED & type) { // Don't change the baseline package set unless we're working with a // new build. Release candidates will need to keep packages as stable // as possible between releases. if (!ctx->meta.based_on) { - strcat(cmd, " --upgrade"); + strncat(command_base, " --upgrade", sizeof(command_base) - strlen(command_base) - 1); + command_base[sizeof(command_base) - 1] = '\0'; } + snprintf(command_base + strlen(command_base), sizeof(command_base) - strlen(command_base), " --extra-index-url 'file://%s'", ctx->storage.wheel_artifact_dir); } - sprintf(cmd + strlen(cmd), " --extra-index-url 'file://%s'", ctx->storage.wheel_artifact_dir); + size_t args_alloc_len = STASIS_BUFSIZ; + char *args = calloc(args_alloc_len + 1, sizeof(*args)); + if (!args) { + SYSERROR("Unable to allocate bytes for command arguments"); + return -1; + } for (size_t x = 0; manifest[x] != NULL; x++) { char *name = NULL; @@ -237,79 +416,131 @@ int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, cha continue; } if (INSTALL_PKG_PIP_DEFERRED & type) { + SYSDEBUG("Getting requirements for test: %s", name); struct Test *info = requirement_from_test(ctx, name); if (info) { - if (!strcmp(info->version, "HEAD")) { + if (!strcmp(info->version, "HEAD") || is_git_sha(info->version)) { + SYSDEBUG("Using version: %s", info->version); struct StrList *tag_data = strlist_init(); if (!tag_data) { - SYSERROR("%s", "Unable to allocate memory for tag data\n"); + SYSERROR("Unable to allocate memory for tag data"); + guard_free(args); return -1; } + SYSDEBUG("Tokenizing repository info tag: %s", info->repository_info_tag); strlist_append_tokenize(tag_data, info->repository_info_tag, "-"); - struct Wheel *whl = NULL; + struct WheelInfo *whl = NULL; char *post_commit = NULL; char *hash = NULL; if (strlist_count(tag_data) > 1) { post_commit = strlist_item(tag_data, 1); + SYSDEBUG("post_commit: %s", post_commit); hash = strlist_item(tag_data, 2); + SYSDEBUG("hash: %s", hash); } // We can't match on version here (index 0). The wheel's version is not guaranteed to be // equal to the tag; setuptools_scm auto-increments the value, the user can change it manually, // etc. errno = 0; - whl = get_wheel_info(ctx->storage.wheel_artifact_dir, info->name, + SYSDEBUG("%s", "Getting wheel information"); + whl = wheelinfo_get(ctx->storage.wheel_artifact_dir, info->name, (char *[]) {ctx->meta.python_compact, ctx->system.arch, "none", "any", post_commit, hash, NULL}, WHEEL_MATCH_ANY); if (!whl && errno) { // error - SYSERROR("Unable to read Python wheel info: %s\n", strerror(errno)); + SYSERROR("Unable to read Python wheel info: %s", strerror(errno)); exit(1); } else if (!whl) { // not found - fprintf(stderr, "No wheel packages found that match the description of '%s'", info->name); + SYSERROR("No wheel packages found that match the description of '%s'", info->name); } else { - // found - guard_strlist_free(&tag_data); + // found, replace the original version with newly detected version + SYSDEBUG("Replacing version: %s", whl->version); + guard_free(info->version); info->version = strdup(whl->version); + SYSDEBUG("Version replaced with: %s", whl->version); } - wheel_free(&whl); + guard_strlist_free(&tag_data); + wheelinfo_free(&whl); } char req[255] = {0}; if (!strcmp(name, info->name)) { - strcpy(req, info->name); + strncpy(req, info->name, sizeof(req) - 1); + req[sizeof(req) - 1] = '\0'; } else { - strcpy(req, name); + strncpy(req, name, sizeof(req) - 1); + req[sizeof(req) - 1] = '\0'; char *spec = find_version_spec(req); if (spec) { *spec = 0; } } - snprintf(cmd + strlen(cmd), - sizeof(cmd) - strlen(cmd) - strlen(info->name) - strlen(info->version) + 5, - " '%s==%s'", req, info->version); + const char *fmt_append = "%s '%s==%s'"; + const char *fmt = " '%s==%s'"; + const int required_len = snprintf(NULL, 0, fmt_append, args, req, info->version); + if (required_len > (int) args_alloc_len) { + if (grow(required_len, &args_alloc_len, &args)) { + SYSERROR("Unable to allocate %d bytes for command arguments", required_len); + guard_free(args); + return -1; + } + } + snprintf(args + strlen(args), args_alloc_len - strlen(args), fmt, req, info->version); } else { - fprintf(stderr, "Deferred package '%s' is not present in the tested package list!\n", name); + SYSERROR("Deferred package '%s' is not present in the tested package list!", name); + guard_free(args); return -1; } } else { if (startswith(name, "--") || startswith(name, "-")) { - sprintf(cmd + strlen(cmd), " %s", name); + const char *fmt_append = "%s %s"; + const char *fmt = " %s"; + const int required_len = snprintf(NULL, 0, fmt_append, args, name); + if (required_len > (int) args_alloc_len) { + if (grow(required_len, &args_alloc_len, &args)) { + SYSERROR("Unable to allocate %d bytes for command arguments", required_len); + guard_free(args); + return -1; + } + } + snprintf(args + strlen(args), args_alloc_len - strlen(args), fmt, name); } else { - sprintf(cmd + strlen(cmd), " '%s'", name); + const char *fmt_append = "%s '%s'"; + const char *fmt = " '%s'"; + const int required_len = snprintf(NULL, 0, fmt_append, args, name); + if (required_len > (int) args_alloc_len) { + if (grow(required_len, &args_alloc_len, &args)) { + SYSERROR("Unable to allocate %d bytes for command arguments", required_len); + guard_free(args); + return -1; + } + } + snprintf(args + strlen(args), args_alloc_len - strlen(args), fmt, name); } } } - int status = runner(cmd); + char *command = NULL; + if (asprintf(&command, "%s %s", command_base, args) < 0) { + SYSERROR("Unable to allocate bytes for command"); + guard_free(args); + return -1; + } + + int status = runner(command); + guard_free(args); + guard_free(command); if (status) { + // fail quickly return status; } } + guard_free(args); return 0; } diff --git a/src/lib/delivery/delivery_populate.c b/src/lib/delivery/delivery_populate.c index 84676f1..cfa3da2 100644 --- a/src/lib/delivery/delivery_populate.c +++ b/src/lib/delivery/delivery_populate.c @@ -33,18 +33,18 @@ int populate_info(struct Delivery *ctx) { if (!ctx->info.time_info) { ctx->info.time_info = malloc(sizeof(*ctx->info.time_info)); if (!ctx->info.time_info) { - msg(STASIS_MSG_ERROR, "%s: Unable to allocate memory for time_info\n", strerror(errno)); + SYSERROR("%s: Unable to allocate memory for time_info", strerror(errno)); return -1; } if (!localtime_r(&ctx->info.time_now, ctx->info.time_info)) { - msg(STASIS_MSG_ERROR, "%s: localtime_r failed\n", strerror(errno)); + SYSERROR("%s: localtime_r failed", strerror(errno)); return -1; } } ctx->info.time_str_epoch = calloc(STASIS_TIME_STR_MAX, sizeof(*ctx->info.time_str_epoch)); if (!ctx->info.time_str_epoch) { - msg(STASIS_MSG_ERROR, "%s: Unable to allocate memory for Unix epoch string\n", strerror(errno)); + SYSERROR("%s: Unable to allocate memory for Unix epoch string", strerror(errno)); return -1; } snprintf(ctx->info.time_str_epoch, STASIS_TIME_STR_MAX - 1, "%li", ctx->info.time_now); @@ -55,6 +55,7 @@ int populate_info(struct Delivery *ctx) { int populate_delivery_cfg(struct Delivery *ctx, int render_mode) { struct INIFILE *cfg = ctx->_stasis_ini_fp.cfg; if (!cfg) { + SYSDEBUG("cfg is NULL"); return -1; } int err = 0; @@ -84,6 +85,45 @@ int populate_delivery_cfg(struct Delivery *ctx, int render_mode) { } globals.pip_packages = ini_getval_strlist(cfg, "default", "pip_packages", LINE_SEP, render_mode, &err); + err = 0; + if (!globals.wheel_builder) { + globals.wheel_builder = ini_getval_str(cfg, "default", "wheel_builder", render_mode, &err); + if (err) { + SYSWARN("wheel_builder is undefined. Falling back to system toolchain: 'build'."); + globals.wheel_builder = strdup("build"); + if (!globals.wheel_builder) { + SYSERROR("unable to allocate memory for default wheel_builder value"); + return -1; + } + } + } + + err = 0; + if (!globals.wheel_builder_manylinux_image) { + globals.wheel_builder_manylinux_image = ini_getval_str(cfg, "default", "wheel_builder_manylinux_image", render_mode, &err); + } + + if (err && globals.wheel_builder && strcmp(globals.wheel_builder, "manylinux") == 0) { + SYSERROR("default:wheel_builder is set to 'manylinux', however default:wheel_builder_manylinux_image is not configured"); + return -1; + } + + if (strcmp(globals.wheel_builder, "manylinux") == 0) { + char *manifest_inspect_cmd = NULL; + if (asprintf(&manifest_inspect_cmd, "manifest inspect '%s'", globals.wheel_builder_manylinux_image) < 0) { + SYSERROR("unable to allocate memory for docker command"); + guard_free(manifest_inspect_cmd); + return -1; + } + if (ctx->deploy.docker.capabilities.usable && docker_exec(manifest_inspect_cmd, STASIS_DOCKER_QUIET_STDOUT)) { + SYSERROR("Image provided by default:wheel_builder_manylinux_image does not exist: %s", globals.wheel_builder_manylinux_image); + guard_free(manifest_inspect_cmd); + return -1; + } + guard_free(manifest_inspect_cmd); + } + + if (globals.jfrog.jfrog_artifactory_base_url) { guard_free(globals.jfrog.jfrog_artifactory_base_url); } @@ -153,6 +193,7 @@ static void normalize_ini_list(struct INIFILE **inip, struct StrList **listp, ch (*inip) = ini; (*listp) = list; } + int populate_delivery_ini(struct Delivery *ctx, int render_mode) { struct INIFILE *ini = ctx->_stasis_ini_fp.delivery; struct INIData *rtdata; @@ -162,8 +203,6 @@ int populate_delivery_ini(struct Delivery *ctx, int render_mode) { // keys in the configuration RuntimeEnv *rt = runtime_copy(__environ); while ((rtdata = ini_getall(ini, "runtime")) != NULL) { - char rec[STASIS_BUFSIZ]; - sprintf(rec, "%s=%s", lstrip(strip(rtdata->key)), lstrip(strip(rtdata->value))); runtime_set(rt, rtdata->key, rtdata->value); } runtime_apply(rt); @@ -201,7 +240,9 @@ int populate_delivery_ini(struct Delivery *ctx, int render_mode) { normalize_ini_list(&ini, &ctx->conda.pip_packages_purge, "conda", "pip_packages_purge", render_mode); // Delivery metadata consumed - populate_mission_ini(&ctx, render_mode); + if (populate_mission_ini(&ctx, render_mode)) { + return -1; + } if (ctx->info.release_name) { guard_free(ctx->info.release_name); @@ -215,16 +256,16 @@ int populate_delivery_ini(struct Delivery *ctx, int render_mode) { return -1; } - if (delivery_format_str(ctx, &ctx->info.release_name, ctx->rules.release_fmt)) { - fprintf(stderr, "Failed to generate release name. Format used: %s\n", ctx->rules.release_fmt); + if (delivery_format_str(ctx, &ctx->info.release_name, STASIS_NAME_MAX, ctx->rules.release_fmt)) { + SYSERROR("Failed to generate release name. Format used: %s", ctx->rules.release_fmt); return -1; } if (!ctx->info.build_name) { - delivery_format_str(ctx, &ctx->info.build_name, ctx->rules.build_name_fmt); + delivery_format_str(ctx, &ctx->info.build_name, STASIS_NAME_MAX, ctx->rules.build_name_fmt); } if (!ctx->info.build_number) { - delivery_format_str(ctx, &ctx->info.build_number, ctx->rules.build_number_fmt); + delivery_format_str(ctx, &ctx->info.build_number, STASIS_NAME_MAX, ctx->rules.build_number_fmt); } // Best I can do to make output directories unique. Annoying. @@ -237,11 +278,17 @@ int populate_delivery_ini(struct Delivery *ctx, int render_mode) { ctx->conda.pip_packages_defer = strlist_init(); } - for (size_t z = 0, i = 0; i < ini->section_count; i++) { + ctx->tests = tests_init(TEST_NUM_ALLOC_INITIAL); + for (size_t i = 0; i < ini->section_count; i++) { char *section_name = ini->section[i]->key; if (startswith(section_name, "test:")) { union INIVal val; - struct Test *test = &ctx->tests[z]; + struct Test *test = test_init(); + if (!test) { + SYSERROR("unable to allocate memory for test structure"); + return -1; + } + val.as_char_p = strchr(ini->section[i]->key, ':') + 1; if (val.as_char_p && isempty(val.as_char_p)) { return 1; @@ -259,8 +306,21 @@ int populate_delivery_ini(struct Delivery *ctx, int render_mode) { } test->repository_remove_tags = ini_getval_strlist(ini, section_name, "repository_remove_tags", LINE_SEP, render_mode, &err); test->build_recipe = ini_getval_str(ini, section_name, "build_recipe", render_mode, &err); - test->runtime.environ = ini_getval_strlist(ini, section_name, "runtime", LINE_SEP, render_mode, &err); - z++; + + test->runtime->environ = ini_getval_strlist(ini, section_name, "runtime", LINE_SEP, render_mode, &err); + const char *timeout_str = ini_getval_str(ini, section_name, "timeout", render_mode, &err); + if (timeout_str) { + test->timeout = str_to_timeout((char *) timeout_str); + if (test->timeout == STR_TO_TIMEOUT_INVALID_TIME_SCALE) { + SYSERROR("In 'test:%s', invalid time scale format: %s. Use n[hms].", test->name, timeout_str); + return 1; + } + if (test->timeout == STR_TO_TIMEOUT_NEGATIVE) { + SYSERROR("In 'test:%s', timeout cannot be negative: %s", test->name, timeout_str); + return 1; + } + } + tests_add(ctx->tests, test); } } @@ -309,25 +369,26 @@ int populate_mission_ini(struct Delivery **ctx, int render_mode) { int err = 0; if ((*ctx)->_stasis_ini_fp.mission) { + // mission configurations are optional return 0; } // Now populate the rules char missionfile[PATH_MAX] = {0}; if (getenv("STASIS_SYSCONFDIR")) { - sprintf(missionfile, "%s/%s/%s/%s.ini", + snprintf(missionfile, sizeof(missionfile), "%s/%s/%s/%s.ini", getenv("STASIS_SYSCONFDIR"), "mission", (*ctx)->meta.mission, (*ctx)->meta.mission); } else { - sprintf(missionfile, "%s/%s/%s/%s.ini", + snprintf(missionfile, sizeof(missionfile), "%s/%s/%s/%s.ini", globals.sysconfdir, "mission", (*ctx)->meta.mission, (*ctx)->meta.mission); } - msg(STASIS_MSG_L2, "Reading mission configuration: %s\n", missionfile); + SYSDEBUG("Reading mission configuration: %s", missionfile); (*ctx)->_stasis_ini_fp.mission = ini_open(missionfile); struct INIFILE *ini = (*ctx)->_stasis_ini_fp.mission; if (!ini) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read mission configuration: %s, %s\n", missionfile, strerror(errno)); - exit(1); + SYSERROR("Failed to read mission configuration: %s, %s", missionfile, strerror(errno)); + return -1; } (*ctx)->_stasis_ini_fp.mission_path = strdup(missionfile); @@ -343,7 +404,7 @@ int populate_mission_ini(struct Delivery **ctx, int render_mode) { void validate_delivery_ini(struct INIFILE *ini) { if (!ini) { - SYSERROR("%s", "INIFILE is NULL!"); + SYSERROR("INIFILE is NULL!"); exit(1); } if (ini_section_search(&ini, INI_SEARCH_EXACT, "meta")) { @@ -353,7 +414,7 @@ void validate_delivery_ini(struct INIFILE *ini) { ini_has_key_required(ini, "meta", "mission"); ini_has_key_required(ini, "meta", "python"); } else { - SYSERROR("%s", "[meta] configuration section is required"); + SYSERROR("[meta] configuration section is required"); exit(1); } @@ -363,7 +424,7 @@ void validate_delivery_ini(struct INIFILE *ini) { ini_has_key_required(ini, "conda", "installer_platform"); ini_has_key_required(ini, "conda", "installer_arch"); } else { - SYSERROR("%s", "[conda] configuration section is required"); + SYSERROR("[conda] configuration section is required"); exit(1); } diff --git a/src/lib/delivery/delivery_postprocess.c b/src/lib/delivery/delivery_postprocess.c index 5029e02..63093b3 100644 --- a/src/lib/delivery/delivery_postprocess.c +++ b/src/lib/delivery/delivery_postprocess.c @@ -1,8 +1,12 @@ #include "delivery.h" +#include "log.h" +#include "conda.h" const char *release_header = "# delivery_name: %s\n" "# delivery_fmt: %s\n" + "# stasis_version: %s\n" + "# stasis_branch: %s\n" "# creation_time: %s\n" "# conda_ident: %s\n" "# conda_build_ident: %s\n"; @@ -11,9 +15,11 @@ char *delivery_get_release_header(struct Delivery *ctx) { char output[STASIS_BUFSIZ]; char stamp[100]; strftime(stamp, sizeof(stamp) - 1, "%c", ctx->info.time_info); - sprintf(output, release_header, + snprintf(output, sizeof(output), release_header, ctx->info.release_name, ctx->rules.release_fmt, + STASIS_VERSION, + STASIS_VERSION_BRANCH, stamp, ctx->conda.tool_version, ctx->conda.tool_build_version); @@ -22,14 +28,16 @@ char *delivery_get_release_header(struct Delivery *ctx) { int delivery_dump_metadata(struct Delivery *ctx) { char filename[PATH_MAX]; - sprintf(filename, "%s/meta-%s.stasis", ctx->storage.meta_dir, ctx->info.release_name); + snprintf(filename, sizeof(filename), "%s/meta-%s.stasis", ctx->storage.meta_dir, ctx->info.release_name); FILE *fp = fopen(filename, "w+"); if (!fp) { return -1; } if (globals.verbose) { - printf("%s\n", filename); + msg(STASIS_MSG_L2, "%s\n", filename); } + fprintf(fp, "stasis_version %s\n", STASIS_VERSION); + fprintf(fp, "stasis_version_branch %s\n", STASIS_VERSION_BRANCH); fprintf(fp, "name %s\n", ctx->meta.name); fprintf(fp, "version %s\n", ctx->meta.version); fprintf(fp, "rc %d\n", ctx->meta.rc); @@ -66,16 +74,16 @@ void delivery_rewrite_spec(struct Delivery *ctx, char *filename, unsigned stage) FILE *tp = NULL; if (stage == DELIVERY_REWRITE_SPEC_STAGE_1) { - SYSDEBUG("%s", "Entering stage 1"); + SYSDEBUG("Entering stage 1"); header = delivery_get_release_header(ctx); SYSDEBUG("Release header:\n%s", header); if (!header) { - msg(STASIS_MSG_ERROR, "failed to generate release header string\n", filename); + SYSERROR("failed to generate release header string", filename); exit(1); } tempfile = xmkstemp(&tp, "w+"); if (!tempfile || !tp) { - msg(STASIS_MSG_ERROR, "%s: unable to create temporary file\n", strerror(errno)); + SYSERROR("%s: unable to create temporary file", strerror(errno)); exit(1); } SYSDEBUG("Writing header to temporary file: %s", tempfile); @@ -84,7 +92,7 @@ void delivery_rewrite_spec(struct Delivery *ctx, char *filename, unsigned stage) // Read the original file char **contents = file_readlines(filename, 0, 0, NULL); if (!contents) { - msg(STASIS_MSG_ERROR, "%s: unable to read %s", filename); + SYSERROR("%s: unable to read %s", filename); exit(1); } @@ -128,45 +136,45 @@ void delivery_rewrite_spec(struct Delivery *ctx, char *filename, unsigned stage) // Replace the original file with our temporary data if (copy2(tempfile, filename, CT_PERM) < 0) { - fprintf(stderr, "%s: could not rename '%s' to '%s'\n", strerror(errno), tempfile, filename); + SYSERROR("%s: could not rename '%s' to '%s'", strerror(errno), tempfile, filename); exit(1); } SYSDEBUG("Removing file: %s", tempfile); remove(tempfile); guard_free(tempfile); } else if (globals.enable_rewrite_spec_stage_2 && stage == DELIVERY_REWRITE_SPEC_STAGE_2) { - SYSDEBUG("%s", "Entering stage 2"); + SYSDEBUG("Entering stage 2"); char output[PATH_MAX] = {0}; // Replace "local" channel with the staging URL if (ctx->storage.conda_staging_url) { - SYSDEBUG("%s", "Will replace conda channel with staging area url"); + SYSDEBUG("Will replace conda channel with staging area url"); file_replace_text(filename, "@CONDA_CHANNEL@", ctx->storage.conda_staging_url, 0); } else if (globals.jfrog.repo) { - SYSDEBUG("%s", "Will replace conda channel with artifactory repo packages/conda url"); - sprintf(output, "%s/%s/%s/%s/packages/conda", globals.jfrog.url, globals.jfrog.repo, ctx->meta.mission, ctx->info.build_name); + SYSDEBUG("Will replace conda channel with artifactory repo packages/conda url"); + snprintf(output, sizeof(output), "%s/%s/%s/%s/packages/conda", globals.jfrog.url, globals.jfrog.repo, ctx->meta.mission, ctx->info.build_name); file_replace_text(filename, "@CONDA_CHANNEL@", output, 0); } else { - SYSDEBUG("%s", "Will replace conda channel with local conda artifact directory"); - msg(STASIS_MSG_WARN, "conda_staging_dir is not configured. Using fallback: '%s'\n", ctx->storage.conda_artifact_dir); + SYSDEBUG("Will replace conda channel with local conda artifact directory"); + SYSWARN("conda_staging_dir is not configured. Using fallback: '%s'", ctx->storage.conda_artifact_dir); file_replace_text(filename, "@CONDA_CHANNEL@", ctx->storage.conda_artifact_dir, 0); } if (ctx->storage.wheel_staging_url) { - SYSDEBUG("%s", "Will replace pip arguments with wheel staging url"); - sprintf(output, "--extra-index-url %s/%s/%s/packages/wheels", ctx->storage.wheel_staging_url, ctx->meta.mission, ctx->info.build_name); + SYSDEBUG("Will replace pip arguments with wheel staging url"); + snprintf(output, sizeof(output), "--extra-index-url %s/%s/%s/packages/wheels", ctx->storage.wheel_staging_url, ctx->meta.mission, ctx->info.build_name); file_replace_text(filename, "@PIP_ARGUMENTS@", ctx->storage.wheel_staging_url, 0); } else if (globals.enable_artifactory && globals.jfrog.url && globals.jfrog.repo) { - SYSDEBUG("%s", "Will replace pip arguments with artifactory repo packages/wheel url"); - sprintf(output, "--extra-index-url %s/%s/%s/%s/packages/wheels", globals.jfrog.url, globals.jfrog.repo, ctx->meta.mission, ctx->info.build_name); + SYSDEBUG("Will replace pip arguments with artifactory repo packages/wheel url"); + snprintf(output, sizeof(output), "--extra-index-url %s/%s/%s/%s/packages/wheels", globals.jfrog.url, globals.jfrog.repo, ctx->meta.mission, ctx->info.build_name); file_replace_text(filename, "@PIP_ARGUMENTS@", output, 0); } else { - SYSDEBUG("%s", "Will replace pip arguments with local wheel artifact directory"); - msg(STASIS_MSG_WARN, "wheel_staging_dir is not configured. Using fallback: '%s'\n", ctx->storage.wheel_artifact_dir); - sprintf(output, "--extra-index-url file://%s", ctx->storage.wheel_artifact_dir); + SYSDEBUG("Will replace pip arguments with local wheel artifact directory"); + SYSWARN("wheel_staging_dir is not configured. Using fallback: '%s'", ctx->storage.wheel_artifact_dir); + snprintf(output, sizeof(output), "--extra-index-url file://%s", ctx->storage.wheel_artifact_dir); file_replace_text(filename, "@PIP_ARGUMENTS@", output, 0); } } - SYSDEBUG("%s", "Rewriting finished"); + SYSDEBUG("Rewriting finished"); } int delivery_copy_conda_artifacts(struct Delivery *ctx) { @@ -177,16 +185,15 @@ int delivery_copy_conda_artifacts(struct Delivery *ctx) { memset(conda_build_dir, 0, sizeof(conda_build_dir)); memset(subdir, 0, sizeof(subdir)); - sprintf(conda_build_dir, "%s/%s", ctx->storage.conda_install_prefix, "conda-bld"); + snprintf(conda_build_dir, sizeof(conda_build_dir), "%s/%s", ctx->storage.conda_install_prefix, "conda-bld"); // One must run conda build at least once to create the "conda-bld" directory. // When this directory is missing there can be no build artifacts. if (access(conda_build_dir, F_OK) < 0) { - msg(STASIS_MSG_RESTRICT | STASIS_MSG_WARN | STASIS_MSG_L3, - "Skipped: 'conda build' has never been executed.\n"); + SYSWARN("Skipped: 'conda build' has never been executed."); return 0; } - snprintf(cmd, sizeof(cmd) - 1, "rsync -avi --progress %s/%s %s", + snprintf(cmd, sizeof(cmd), "rsync -avi --progress %s/%s %s", conda_build_dir, ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], ctx->storage.conda_artifact_dir); @@ -200,7 +207,7 @@ int delivery_index_conda_artifacts(struct Delivery *ctx) { int delivery_copy_wheel_artifacts(struct Delivery *ctx) { char cmd[PATH_MAX] = {0}; - snprintf(cmd, sizeof(cmd) - 1, "rsync -avi --progress %s/*/dist/*.whl %s", + snprintf(cmd, sizeof(cmd), "rsync -avi --progress %s/*/dist/*.whl %s", ctx->storage.build_sources_dir, ctx->storage.wheel_artifact_dir); return system(cmd); @@ -217,7 +224,7 @@ int delivery_index_wheel_artifacts(struct Delivery *ctx) { // Generate a "dumb" local pypi index that is compatible with: // pip install --extra-index-url char top_index[PATH_MAX] = {0}; - sprintf(top_index, "%s/index.html", ctx->storage.wheel_artifact_dir); + snprintf(top_index, sizeof(top_index),"%s/index.html", ctx->storage.wheel_artifact_dir); SYSDEBUG("Opening top-level index for writing: %s", top_index); FILE *top_fp = fopen(top_index, "w+"); if (!top_fp) { @@ -232,11 +239,12 @@ int delivery_index_wheel_artifacts(struct Delivery *ctx) { } char bottom_index[PATH_MAX * 2] = {0}; - sprintf(bottom_index, "%s/%s/index.html", ctx->storage.wheel_artifact_dir, rec->d_name); + snprintf(bottom_index, sizeof(bottom_index), "%s/%s/index.html", ctx->storage.wheel_artifact_dir, rec->d_name); SYSDEBUG("Opening bottom-level for writing: %s", bottom_index); FILE *bottom_fp = fopen(bottom_index, "w+"); if (!bottom_fp) { closedir(dp); + fclose(top_fp); return -3; } @@ -248,7 +256,7 @@ int delivery_index_wheel_artifacts(struct Delivery *ctx) { fprintf(top_fp, "<a href=\"%s/\">%s</a><br/>\n", rec->d_name, rec->d_name); char dpath[PATH_MAX * 2] = {0}; - sprintf(dpath, "%s/%s", ctx->storage.wheel_artifact_dir, rec->d_name); + snprintf(dpath, sizeof(dpath), "%s/%s", ctx->storage.wheel_artifact_dir, rec->d_name); struct StrList *packages = listdir(dpath); if (!packages) { closedir(dp); @@ -275,6 +283,6 @@ int delivery_index_wheel_artifacts(struct Delivery *ctx) { } closedir(dp); fclose(top_fp); - SYSDEBUG("%s", "Wheel indexing complete"); + SYSDEBUG("Wheel indexing complete"); return 0; } diff --git a/src/lib/delivery/delivery_show.c b/src/lib/delivery/delivery_show.c index adfa1be..1740688 100644 --- a/src/lib/delivery/delivery_show.c +++ b/src/lib/delivery/delivery_show.c @@ -84,13 +84,13 @@ void delivery_conda_show(struct Delivery *ctx) { void delivery_tests_show(struct Delivery *ctx) { printf("\n====TESTS====\n"); - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { - if (!ctx->tests[i].name) { + for (size_t i = 0; i < ctx->tests->num_used; i++) { + if (!ctx->tests->test[i]->name) { continue; } - printf("%-20s %-20s %s\n", ctx->tests[i].name, - ctx->tests[i].version, - ctx->tests[i].repository); + printf("%-20s %-20s %s\n", ctx->tests->test[i]->name, + ctx->tests->test[i]->version, + ctx->tests->test[i]->repository); } } @@ -108,7 +108,7 @@ void delivery_runtime_show(struct Delivery *ctx) { char *item = strlist_item(rt, i); if (!item) { // not supposed to occur - msg(STASIS_MSG_WARN | STASIS_MSG_L1, "Encountered unexpected NULL at record %zu of %zu of runtime array.\n", i); + SYSWARN("Encountered unexpected NULL in record %zu of %zu in runtime array.", i, total); return; } printf("%s\n", item); diff --git a/src/lib/delivery/delivery_test.c b/src/lib/delivery/delivery_test.c index e80e0ec..f59a62e 100644 --- a/src/lib/delivery/delivery_test.c +++ b/src/lib/delivery/delivery_test.c @@ -1,5 +1,87 @@ #include "delivery.h" +struct Tests *tests_init(const size_t num_tests) { + struct Tests *tests = calloc(1, sizeof(*tests)); + if (!tests) { + return NULL; + } + + tests->test = calloc(num_tests, sizeof(*tests->test)); + if (!tests->test) { + guard_free(tests); + return NULL; + } + tests->num_used = 0; + tests->num_alloc = num_tests; + + return tests; +} + +int tests_add(struct Tests *tests, struct Test *x) { + if (tests->num_used >= tests->num_alloc) { + const size_t old_alloc = tests->num_alloc; + struct Test **tmp = realloc(tests->test, tests->num_alloc++ * sizeof(**tests->test)); + SYSDEBUG("Increasing size of test array: %zu -> %zu", old_alloc, tests->num_alloc); + if (!tmp) { + SYSDEBUG("Failed to allocate %zu bytes for test array", tests->num_alloc * sizeof(**tests->test)); + return -1; + } + tests->test = tmp; + } + + SYSDEBUG("Adding test: '%s'", x->name); + tests->test[tests->num_used++] = x; + return 0; +} + +struct Test *test_init() { + struct Test *result = calloc(1, sizeof(*result)); + if (!result) { + return NULL; + } + + result->runtime = calloc(1, sizeof(*result->runtime)); + if (!result->runtime) { + guard_free(result); + return NULL; + } + + return result; +} + +void test_free(struct Test **x) { + struct Test *test = *x; + if (!test) { + return; + } + guard_free(test->name); + guard_free(test->version); + guard_free(test->repository); + guard_free(test->repository_info_ref); + guard_free(test->repository_info_tag); + guard_strlist_free(&test->repository_remove_tags); + guard_free(test->script); + guard_free(test->script_setup); + guard_free(test->build_recipe); + // test-specific runtime variables + guard_runtime_free(test->runtime->environ); + guard_free(test->runtime); + guard_free(test); +} + +void tests_free(struct Tests **x) { + struct Tests *tests = *x; + if (!tests) { + return; + } + + for (size_t i = 0; i < tests->num_alloc; i++) { + test_free(&tests->test[i]); + } + guard_free(tests->test); + guard_free(tests); +} + void delivery_tests_run(struct Delivery *ctx) { static const int SETUP = 0; static const int PARALLEL = 1; @@ -16,26 +98,26 @@ void delivery_tests_run(struct Delivery *ctx) { // amount of debug information. snprintf(globals.workaround.conda_reactivate, PATH_MAX - 1, "\nset +x; mamba activate ${CONDA_DEFAULT_ENV}; set -x\n"); - if (!ctx->tests[0].name) { - msg(STASIS_MSG_WARN | STASIS_MSG_L2, "no tests are defined!\n"); + if (!ctx->tests || !ctx->tests->num_used) { + SYSWARN("no tests are defined!"); } else { pool[PARALLEL] = mp_pool_init("parallel", ctx->storage.tmpdir); if (!pool[PARALLEL]) { - perror("mp_pool_init/parallel"); + SYSERROR("mp_pool_init/parallel initialization failed"); exit(1); } pool[PARALLEL]->status_interval = globals.pool_status_interval; pool[SERIAL] = mp_pool_init("serial", ctx->storage.tmpdir); if (!pool[SERIAL]) { - perror("mp_pool_init/serial"); + SYSERROR("mp_pool_init/serial initialization failed"); exit(1); } pool[SERIAL]->status_interval = globals.pool_status_interval; pool[SETUP] = mp_pool_init("setup", ctx->storage.tmpdir); if (!pool[SETUP]) { - perror("mp_pool_init/setup"); + SYSERROR("mp_pool_init/setup initialization failed"); exit(1); } pool[SETUP]->status_interval = globals.pool_status_interval; @@ -60,21 +142,20 @@ void delivery_tests_run(struct Delivery *ctx) { // Iterate over our test records, retrieving the source code for each package, and assigning its scripted tasks // to the appropriate processing pool - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { - struct Test *test = &ctx->tests[i]; + for (size_t i = 0; i < ctx->tests->num_used; i++) { + struct Test *test = ctx->tests->test[i]; if (!test->name && !test->repository && !test->script) { // skip unused test records continue; } msg(STASIS_MSG_L2, "Loading tests for %s %s\n", test->name, test->version); if (!test->script || !strlen(test->script)) { - msg(STASIS_MSG_WARN | STASIS_MSG_L3, "Nothing to do. To fix, declare a 'script' in section: [test:%s]\n", - test->name); + SYSWARN("Nothing to do. To fix, declare a 'script' in section: [test:%s]", test->name); continue; } char destdir[PATH_MAX]; - sprintf(destdir, "%s/%s", ctx->storage.build_sources_dir, path_basename(test->repository)); + snprintf(destdir, sizeof(destdir), "%s/%s", ctx->storage.build_sources_dir, path_basename(test->repository)); if (!access(destdir, F_OK)) { msg(STASIS_MSG_L3, "Purging repository %s\n", destdir); @@ -97,6 +178,18 @@ void delivery_tests_run(struct Delivery *ctx) { if (pushd(destdir)) { COE_CHECK_ABORT(1, "Unable to enter repository directory\n"); } else { + const int dep_status = check_python_package_dependencies("."); + if (dep_status) { + SYSERROR("Please replace all occurrences above with standard package specs:\n" + "\n" + " package==x.y.z\n" + " package>=x.y.z\n" + " package<=x.y.z\n" + " ...\n" + "\n"); + COE_CHECK_ABORT(true, "Unreproducible delivery"); + } + char *cmd = calloc(strlen(test->script) + STASIS_BUFSIZ, sizeof(*cmd)); if (!cmd) { SYSERROR("Unable to allocate test script buffer: %s", strerror(errno)); @@ -106,11 +199,12 @@ void delivery_tests_run(struct Delivery *ctx) { msg(STASIS_MSG_L3, "Queuing task for %s\n", test->name); memset(&proc, 0, sizeof(proc)); - strcpy(cmd, test->script); + strncpy(cmd, test->script, strlen(test->script) + STASIS_BUFSIZ - 1); + cmd[strlen(test->script) + STASIS_BUFSIZ - 1] = '\0'; char *cmd_rendered = tpl_render(cmd); if (cmd_rendered) { if (strcmp(cmd_rendered, cmd) != 0) { - strcpy(cmd, cmd_rendered); + strncpy(cmd, cmd_rendered, strlen(test->script) + STASIS_BUFSIZ - 1); cmd[strlen(cmd_rendered) ? strlen(cmd_rendered) - 1 : 0] = 0; } guard_free(cmd_rendered); @@ -135,7 +229,8 @@ void delivery_tests_run(struct Delivery *ctx) { if (!globals.enable_parallel || !test->parallel) { selected = SERIAL; memset(pool_name, 0, sizeof(pool_name)); - strcpy(pool_name, "serial"); + strncpy(pool_name, "serial", sizeof(pool_name) - 1); + pool_name[sizeof(pool_name) - 1] = '\0'; } if (asprintf(&runner_cmd, runner_cmd_fmt, cmd) < 0) { @@ -154,6 +249,12 @@ void delivery_tests_run(struct Delivery *ctx) { } exit(1); } + + // Apply timeout from test block + if (test->timeout) { + task->timeout = test->timeout; + } + guard_free(runner_cmd); guard_free(cmd); popd(); @@ -163,11 +264,11 @@ void delivery_tests_run(struct Delivery *ctx) { // Configure "script_setup" tasks // Directories should exist now, so no need to go through initializing everything all over again. - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { - struct Test *test = &ctx->tests[i]; + for (size_t i = 0; i < ctx->tests->num_used; i++) { + const struct Test *test = ctx->tests->test[i]; if (test->script_setup) { char destdir[PATH_MAX]; - sprintf(destdir, "%s/%s", ctx->storage.build_sources_dir, path_basename(test->repository)); + snprintf(destdir, sizeof(destdir), "%s/%s", ctx->storage.build_sources_dir, path_basename(test->repository)); if (access(destdir, F_OK)) { SYSERROR("%s: %s", destdir, strerror(errno)); exit(1); @@ -181,11 +282,13 @@ void delivery_tests_run(struct Delivery *ctx) { } strncpy(cmd, test->script_setup, cmd_len - 1); + cmd[cmd_len - 1] = '\0'; + char *cmd_rendered = tpl_render(cmd); if (cmd_rendered) { if (strcmp(cmd_rendered, cmd) != 0) { strncpy(cmd, cmd_rendered, cmd_len - 1); - cmd[strlen(cmd_rendered) ? strlen(cmd_rendered) - 1 : 0] = 0; + cmd[strlen(cmd_rendered) ? strlen(cmd_rendered) - 1 : 0] = '\0'; } guard_free(cmd_rendered); } else { @@ -217,7 +320,7 @@ void delivery_tests_run(struct Delivery *ctx) { guard_free(cmd); popd(); } else { - SYSERROR("Failed to change directory: %s\n", destdir); + SYSERROR("Failed to change directory: %s", destdir); exit(1); } } @@ -282,10 +385,10 @@ int delivery_fixup_test_results(struct Delivery *ctx) { continue; } - sprintf(path, "%s/%s", ctx->storage.results_dir, rec->d_name); + snprintf(path, sizeof(path), "%s/%s", ctx->storage.results_dir, rec->d_name); msg(STASIS_MSG_L3, "%s\n", rec->d_name); if (xml_pretty_print_in_place(path, STASIS_XML_PRETTY_PRINT_PROG, STASIS_XML_PRETTY_PRINT_ARGS)) { - msg(STASIS_MSG_L3 | STASIS_MSG_WARN, "Failed to rewrite file '%s'\n", rec->d_name); + SYSWARN("Failed to rewrite file '%s'", rec->d_name); } } diff --git a/src/lib/delivery/include/delivery.h b/src/lib/delivery/include/delivery.h index 26a5499..3103a86 100644 --- a/src/lib/delivery/include/delivery.h +++ b/src/lib/delivery/include/delivery.h @@ -5,20 +5,11 @@ #include <string.h> #include <stdbool.h> -#include <unistd.h> -#include <sys/utsname.h> -#include <fnmatch.h> -#include <sys/statvfs.h> #include "artifactory.h" -#include "conda.h" -#include "copy.h" -#include "core.h" #include "docker.h" #include "environment.h" #include "ini.h" #include "multiprocessing.h" -#include "recipe.h" -#include "wheel.h" #define DELIVERY_PLATFORM_MAX 4 #define DELIVERY_PLATFORM_MAXLEN 65 @@ -44,6 +35,28 @@ struct Content { char *data; }; +//! Number of test records to allocate (grows dynamically) +#define TEST_NUM_ALLOC_INITIAL 10 + +/*! \struct Test + * \brief Test information + */ +struct Test { + char *name; ///< Name of package + char *version; ///< Version of package + char *repository; ///< Git repository of package + char *script_setup; ///< Commands to execute before the main script + char *script; ///< Commands to execute + bool disable; ///< Toggle a test block + bool parallel; ///< Toggle parallel or serial execution + char *build_recipe; ///< Conda recipe to build (optional) + char *repository_info_ref; ///< Git commit hash + char *repository_info_tag; ///< Git tag (first parent) + struct StrList *repository_remove_tags; ///< Git tags to remove (to fix duplicate commit tags) + struct Runtime *runtime; ///< Environment variables specific to the test context + int timeout; ///< Timeout in seconds +}; ///< An array of tests + /*! \struct Delivery * \brief A structure describing a full delivery object */ @@ -64,10 +77,8 @@ struct Delivery { * \brief System information */ struct System { - char *arch; - ///< System CPU architecture ident - char **platform; - ///< System platform name + char *arch; ///< System CPU architecture ident + char **platform; ///< System platform name } system; /*! \struct Storage * \brief Storage paths @@ -155,23 +166,11 @@ struct Delivery { RuntimeEnv *environ; ///< Environment variables } runtime; - /*! \struct Test - * \brief Test information - */ - struct Test { - char *name; ///< Name of package - char *version; ///< Version of package - char *repository; ///< Git repository of package - char *script_setup; ///< Commands to execute before the main script - char *script; ///< Commands to execute - bool disable; ///< Toggle a test block - bool parallel; ///< Toggle parallel or serial execution - char *build_recipe; ///< Conda recipe to build (optional) - char *repository_info_ref; ///< Git commit hash - char *repository_info_tag; ///< Git tag (first parent) - struct StrList *repository_remove_tags; ///< Git tags to remove (to fix duplicate commit tags) - struct Runtime runtime; ///< Environment variables specific to the test context - } tests[1000]; ///< An array of tests + struct Tests { + struct Test **test; + size_t num_used; + size_t num_alloc; + } *tests; struct Deploy { struct JFRT_Auth jfrog_auth; @@ -311,9 +310,10 @@ int delivery_get_conda_installer(struct Delivery *ctx, char *installer_url); * Generate URL based on Delivery context * @param ctx pointer to Delivery context * @param result pointer to char + * @param maxlen * @return in result */ -void delivery_get_conda_installer_url(struct Delivery *ctx, char *result); +void delivery_get_conda_installer_url(struct Delivery *ctx, char *result, size_t maxlen); /** * Install packages based on Delivery context @@ -384,10 +384,11 @@ void delivery_install_conda(char *install_script, char *conda_install_dir); * * @param ctx pointer to Delivery context * @param dest NULL pointer to string, or initialized string + * @param maxlen * @param fmt release format string * @return 0 on success, -1 on error */ -int delivery_format_str(struct Delivery *ctx, char **dest, const char *fmt); +int delivery_format_str(struct Delivery *ctx, char **dest, size_t maxlen, const char *fmt); // helper function int delivery_gather_tool_versions(struct Delivery *ctx); @@ -439,6 +440,16 @@ int delivery_exists(struct Delivery *ctx); int delivery_overlay_packages_from_env(struct Delivery *ctx, const char *env_name); /** + * Conda does not handle version suffixes well, if at all. For example, if pkg-1.2.3rc1 is installed Conda will + * silently ignore a request to install pkg-1.2.3. This function serves as a workaround by comparing the version + * on-disk, and the requested version from the package list, and if the versions are not equal the on-disk package + * is replaced by the one in the package list. + * + * When a package is present in the list without a pinned version it will be reinstalled with whatever is available + */ +int delivery_conda_enforce_package_version(struct Delivery *ctx, const char *env_name, const char *name); + +/** * Retrieve remote deliveries associated with the current version series * @param ctx Delivery context * @return -1 on error @@ -459,4 +470,69 @@ int delivery_series_sync(struct Delivery *ctx); */ int delivery_purge_packages(struct Delivery *ctx, const char *env_name, int use_pkg_manager); +/** + * Export delivery environments + * + * @param ctx Delivery context + * @param envs array of conda environment names + */ +void delivery_export(const struct Delivery *ctx, char *envs[]); + +/** + * STAGE 1: Rewrite delivery-related strings in specfile + * + * @param ctx Delivery context + * @param specfile path to YAML spec file + */ +void delivery_rewrite_stage1(struct Delivery *ctx, char *specfile); + +/** + * STAGE 2: Rewrite delivery-related strings in specfile + * + * @param ctx Delivery context + * @param specfile path to YAML spec file + */ +void delivery_rewrite_stage2(struct Delivery *ctx, char *specfile); + +/** + * Return a copy of a delivery context + * @param ctx Delivery context + * @return a copy + */ +struct Delivery *delivery_duplicate(struct Delivery *ctx); + +/** + * Initialize a `Tests` structure + * @param num_tests number of test records + * @return a an initialized `Tests` structure + */ +struct Tests *tests_init(size_t num_tests); + +/** + * Add a `Test` structure to `Tests` + * @param tests list to add to + * @param x test to add to list + * @return 0=success, -1=error + */ +int tests_add(struct Tests *tests, struct Test *x); + +/** + * Free a `Test` structure + * @param x pointer to `Test` + */ +void test_free(struct Test **x); + +/** + * Free a `Tests` structure + * @param x pointer to `Tests` + */ +void tests_free(struct Tests **x); + +/** + * Initialize a `Test` structure + * @return an initialized `Test` structure + */ +struct Test *test_init(); + + #endif //STASIS_DELIVERY_H @@ -23,6 +23,18 @@ conda_packages = ; (list) Python packages to be installed/overridden in the base environment ;pip_packages = +; (string) Python wheel builder [Linux only] +; DEFAULT: system +; OPTIONS: +; system = Build using local system toolchain +; cibuildwheel = Build using cibuildwheel and docker +; manylinux = Build using manylinux and docker +wheel_builder = manylinux + +; (string) Manylinux image [Linux only] +; When wheel_builder is set to "manylinux", use the following image +wheel_builder_manylinux_image = quay.io/pypa/manylinux2014 + [jfrog_cli_download] url = https://releases.jfrog.io/artifactory product = jfrog-cli diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2b09e9e..26c4250 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -17,7 +17,10 @@ if (BASH_PROGRAM AND TESTS_RT) file(REAL_PATH ${rt_file} rt_name) string(REGEX REPLACE ${ext_pattern} "" rt_name ${rt_file}) add_test (${rt_name} ${BASH_PROGRAM} ${rt_file}) - endforeach() + set_property(TEST ${rt_name} + PROPERTY ENVIRONMENT "GIT_CEILING_DIRECTORIES=${CMAKE_BINARY_DIR}" + ) +endforeach() endif() foreach(source_file ${source_files}) @@ -30,6 +33,9 @@ foreach(source_file ${source_files}) elseif (CMAKE_C_COMPILER_ID STREQUAL "MSVC") target_compile_options(${test_executable} PRIVATE ${win_cflags} ${win_msvc_cflags}) endif() + if (TESTS_VERBOSE) + target_compile_definitions(${test_executable} PRIVATE STASIS_TEST_VERBOSE=1) + endif () target_include_directories(${test_executable} PRIVATE ${core_INCLUDE} ${delivery_INCLUDE} @@ -41,10 +47,12 @@ foreach(source_file ${source_files}) add_test(${test_executable} ${test_executable}) set_tests_properties(${test_executable} PROPERTIES - TIMEOUT 240) + TIMEOUT 600) set_tests_properties(${test_executable} PROPERTIES SKIP_RETURN_CODE 127) set_property(TEST ${test_executable} - PROPERTY ENVIRONMENT "STASIS_SYSCONFDIR=${CMAKE_SOURCE_DIR}") + PROPERTY ENVIRONMENT "STASIS_SYSCONFDIR=${CMAKE_SOURCE_DIR}" + PROPERTY ENVIRONMENT "GIT_CEILING_DIRECTORIES=${CMAKE_BINARY_DIR}" + ) endforeach() diff --git a/tests/include/testing.h b/tests/include/testing.h index ab24115..d11398c 100644 --- a/tests/include/testing.h +++ b/tests/include/testing.h @@ -9,6 +9,15 @@ #define __FILE_NAME__ __FILE__ #endif +#ifdef STASIS_TEST_VERBOSE +#define STASIS_TEST_MSG(MSG, ...) do { \ +fprintf(stderr, "%s:%d:%s(): ", path_basename(__FILE__), __LINE__, __FUNCTION__); \ +fprintf(stderr, MSG LINE_SEP, __VA_ARGS__); \ +} while (0) +#else +#define STASIS_TEST_MSG(MSG, ...) do {} while (0) +#endif + typedef void(STASIS_TEST_FUNC)(); struct stasis_test_result_t { const char *filename; @@ -48,32 +57,39 @@ inline void stasis_testing_record_result_summary() { size_t failed = 0; size_t skipped = 0; size_t passed = 0; - int do_message; + int do_message = 0; + int do_reason = 0; static char status_msg[255] = {0}; for (size_t i = 0; i < stasis_test_results_i; i++) { if (stasis_test_results[i].status && stasis_test_results[i].skip) { strcpy(status_msg, "SKIP"); do_message = 1; + do_reason = 1; skipped++; } else if (!stasis_test_results[i].status) { strcpy(status_msg, "FAIL"); do_message = 1; + do_reason = 1; failed++; } else { +#ifdef STASIS_TEST_VERBOSE + do_message = 1; +#endif strcpy(status_msg, "PASS"); - do_message = 0; passed++; } - fprintf(stdout, "[%s] %s:%d :: %s() => %s", - status_msg, - stasis_test_results[i].filename, - stasis_test_results[i].lineno, - stasis_test_results[i].funcname, - stasis_test_results[i].msg_assertion); if (do_message) { - fprintf(stdout, "\n \\_ %s", stasis_test_results[i].msg_reason); + fprintf(stdout, "[%s] %s:%d :: %s() => %s", + status_msg, + stasis_test_results[i].filename, + stasis_test_results[i].lineno, + stasis_test_results[i].funcname, + stasis_test_results[i].msg_assertion); + if (do_reason) { + fprintf(stdout, "\n \\_ %s", stasis_test_results[i].msg_reason); + } + fprintf(stdout, "\n"); } - fprintf(stdout, "\n"); } fprintf(stdout, "\n[UNIT] %zu tests passed, %zu tests failed, %zu tests skipped out of %zu\n", passed, failed, skipped, stasis_test_results_i); } @@ -106,8 +122,7 @@ inline char *stasis_testing_read_ascii(const char *filename) { } inline int stasis_testing_write_ascii(const char *filename, const char *data) { - FILE *fp; - fp = fopen(filename, "w+"); + FILE *fp = fopen(filename, "w+"); if (!fp) { perror(filename); return -1; @@ -199,6 +214,7 @@ inline void stasis_testing_teardown_workspace() { SYSERROR("%s", "Unable to determine current working directory"); \ exit(1); \ } \ + LOG_LEVEL = LOG_LEVEL_DEBUG; \ atexit(stasis_testing_record_result_summary); \ atexit(stasis_testing_teardown_workspace); \ stasis_testing_setup_workspace(); \ @@ -208,7 +224,7 @@ inline void stasis_testing_teardown_workspace() { #define STASIS_ASSERT(COND, REASON) do { \ stasis_testing_record_result((struct stasis_test_result_t) { \ .filename = __FILE_NAME__, \ - .funcname = __FUNCTION__, \ + .funcname = __func__, \ .lineno = __LINE__, \ .status = (COND), \ .msg_assertion = "ASSERT(" #COND ")", \ @@ -218,7 +234,7 @@ inline void stasis_testing_teardown_workspace() { #define STASIS_ASSERT_FATAL(COND, REASON) do { \ stasis_testing_record_result((struct stasis_test_result_t) { \ .filename = __FILE_NAME__, \ - .funcname = __FUNCTION__, \ + .funcname = __func__, \ .lineno = __LINE__, \ .status = (COND), \ .msg_assertion = "ASSERT FATAL (" #COND ")", \ @@ -232,7 +248,7 @@ inline void stasis_testing_teardown_workspace() { #define STASIS_SKIP_IF(COND, REASON) do { \ stasis_testing_record_result((struct stasis_test_result_t) { \ .filename = __FILE_NAME__, \ - .funcname = __FUNCTION__, \ + .funcname = __func__, \ .lineno = __LINE__, \ .status = true, \ .skip = (COND), \ diff --git a/tests/setup.sh b/tests/setup.sh index 7e38cf9..bce2fbd 100644 --- a/tests/setup.sh +++ b/tests/setup.sh @@ -78,7 +78,7 @@ teardown_workspace() { install_stasis() { pushd "$BUILD_DIR" - if ! cmake -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -DCMAKE_BUILD_TYPE=Debug "${TOPDIR}"/../..; then + if ! cmake -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -DCMAKE_BUILD_TYPE=Debug -DDEBUG_MESSAGES=ON "${TOPDIR}"/../..; then echo "cmake failed" >&2 return 1 fi @@ -109,7 +109,7 @@ STASIS_TEST_RESULT_SKIP=0 run_command() { local logfile="$(mktemp).log" local cmd="${@}" - local lines_on_error=100 + local lines_on_error=1000 /bin/echo "Testing: $cmd " $cmd &>"$logfile" diff --git a/tests/test_artifactory.c b/tests/test_artifactory.c index 4af7eec..7090fa6 100644 --- a/tests/test_artifactory.c +++ b/tests/test_artifactory.c @@ -15,14 +15,14 @@ const char *gbuild_num = "1"; static int jfrog_cli_rt_build_delete(struct JFRT_Auth *auth, char *build_name, char *build_num) { char cmd[STASIS_BUFSIZ]; memset(cmd, 0, sizeof(cmd)); - snprintf(cmd, sizeof(cmd) - 1, "--build \"%s/%s\"", build_name, build_num); + snprintf(cmd, sizeof(cmd), "--build \"%s/%s\"", build_name, build_num); return jfrog_cli(auth, "rt", "delete", cmd); } static int jfrog_cli_rt_delete(struct JFRT_Auth *auth, char *pattern) { char cmd[STASIS_BUFSIZ]; memset(cmd, 0, sizeof(cmd)); - snprintf(cmd, sizeof(cmd) - 1, "\"%s\"", pattern); + snprintf(cmd, sizeof(cmd), "\"%s\"", pattern); return jfrog_cli(auth, "rt", "delete", cmd); } @@ -60,7 +60,7 @@ void test_jfrog_cli_rt_download() { char *filename = "empty_file_upload.txt"; char path[PATH_MAX] = {0}; - sprintf(path, "%s/%s", getenv("STASIS_JF_REPO"), filename); + snprintf(path, sizeof(path), "%s/%s", getenv("STASIS_JF_REPO"), filename); STASIS_ASSERT(jfrog_cli_rt_download(&gauth, &dl, filename, ".") == 0, "jf download failed"); STASIS_ASSERT(jfrog_cli_rt_delete(&gauth, path) == 0, "jf delete test artifact failed"); } @@ -93,15 +93,15 @@ int main(int argc, char *argv[]) { } char path[PATH_MAX] = {0}; - sprintf(path, "%s/bin:%s", ctx.storage.tools_dir, getenv("PATH")); + snprintf(path, sizeof(path), "%s/bin:%s", ctx.storage.tools_dir, getenv("PATH")); setenv("PATH", path, 1); // The default config contains the URL information to download jfrog-cli char cfg_path[PATH_MAX] = {0}; if (strstr(sysconfdir, "..")) { - sprintf(cfg_path, "%s/%s/stasis.ini", basedir, sysconfdir); + snprintf(cfg_path, sizeof(cfg_path), "%s/%s/stasis.ini", basedir, sysconfdir); } else { - sprintf(cfg_path, "%s/stasis.ini", sysconfdir); + snprintf(cfg_path, sizeof(cfg_path), "%s/stasis.ini", sysconfdir); } ctx._stasis_ini_fp.cfg = ini_open(cfg_path); if (!ctx._stasis_ini_fp.cfg) { @@ -115,7 +115,8 @@ int main(int argc, char *argv[]) { // Skip this suite if we're not configured to use it if (jfrt_auth_init(&gauth)) { - SYSERROR("%s", "Not configured to test Artifactory. Skipping."); + SYSERROR("Not configured to test Artifactory. Skipping."); + guard_free(basedir); return STASIS_TEST_SUITE_SKIP; } guard_free(basedir); diff --git a/tests/test_conda.c b/tests/test_conda.c index 4d437e4..bbbef3c 100644 --- a/tests/test_conda.c +++ b/tests/test_conda.c @@ -18,10 +18,10 @@ void test_micromamba() { int result; }; struct testcase tc[] = { - {.mminfo = {.micromamba_prefix = mm_prefix, .conda_prefix = c_prefix}, .cmd = "info", .result = 0}, - {.mminfo = {.micromamba_prefix = mm_prefix, .conda_prefix = c_prefix}, .cmd = "env list", .result = 0}, - {.mminfo = {.micromamba_prefix = mm_prefix, .conda_prefix = c_prefix}, .cmd = "run python -V", .result = 0}, - {.mminfo = {.micromamba_prefix = mm_prefix, .conda_prefix = c_prefix}, .cmd = "no_such_option", .result = 109}, + {.mminfo = {.download_dir = cwd_workspace, .micromamba_prefix = mm_prefix, .conda_prefix = c_prefix}, .cmd = "info", .result = 0}, + {.mminfo = {.download_dir = cwd_workspace, .micromamba_prefix = mm_prefix, .conda_prefix = c_prefix}, .cmd = "env list", .result = 0}, + {.mminfo = {.download_dir = cwd_workspace, .micromamba_prefix = mm_prefix, .conda_prefix = c_prefix}, .cmd = "run python3 -V", .result = 0}, + {.mminfo = {.download_dir = cwd_workspace, .micromamba_prefix = mm_prefix, .conda_prefix = c_prefix}, .cmd = "no_such_option", .result = 109}, }; for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) { @@ -31,7 +31,7 @@ void test_micromamba() { result = result >> 8; } STASIS_ASSERT(result == item->result, "unexpected exit value"); - SYSERROR("micromamba command: '%s' (returned: %d)", item->cmd, result); + SYSDEBUG("micromamba command: '%s' (returned: %d)", item->cmd, result); } } @@ -40,7 +40,7 @@ struct Delivery ctx; void test_conda_installation() { char *install_url = calloc(255, sizeof(install_url)); - delivery_get_conda_installer_url(&ctx, install_url); + delivery_get_conda_installer_url(&ctx, install_url, PATH_MAX); delivery_get_conda_installer(&ctx, install_url); delivery_install_conda(ctx.conda.installer_path, ctx.storage.conda_install_prefix); @@ -92,7 +92,7 @@ void test_conda_exec() { void test_python_exec() { const char *python_system_path = find_program("python3"); char python_path[PATH_MAX]; - sprintf(python_path, "%s/bin/python3", ctx.storage.conda_install_prefix); + snprintf(python_path, sizeof(python_path), "%s/bin/python3", ctx.storage.conda_install_prefix); STASIS_ASSERT(strcmp(python_path, python_system_path ? python_system_path : "/not/found") == 0, "conda is not configured correctly."); STASIS_ASSERT(python_exec("-V") == 0, "python is broken"); @@ -110,13 +110,13 @@ void test_conda_setup_headless() { void test_conda_env_create_from_uri() { const char *url = "https://ssb.stsci.edu/jhunk/stasis_test/test_conda_env_create_from_uri.yml"; - char *name = strdup(__FUNCTION__); + char *name = strdup(__func__); STASIS_ASSERT(conda_env_create_from_uri(name, (char *) url, "3.11") == 0, "creating an environment from a remote source failed"); free(name); } void test_conda_env_create_export_remove() { - char *name = strdup(__FUNCTION__); + char *name = strdup(__func__); STASIS_ASSERT(conda_env_create(name, "3", "fitsverify") == 0, "unable to create a simple environment"); STASIS_ASSERT(conda_env_export(name, ".", name) == 0, "unable to export an environment"); STASIS_ASSERT(conda_env_remove(name) == 0, "unable to remove an environment"); @@ -143,14 +143,14 @@ void test_pip_index_provides() { }; for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) { struct testcase *test = &tc[i]; - int result = pkg_index_provides(PKG_USE_PIP, test->pindex, test->name); + int result = pkg_index_provides(PKG_USE_PIP, test->pindex, test->name, "."); STASIS_ASSERT(result == test->expected, "Unexpected result"); if (PKG_INDEX_PROVIDES_FAILED(result)) { - fprintf(stderr, "error: %s\n", pkg_index_provides_strerror(result)); + SYSERROR("%s", pkg_index_provides_strerror(result)); } else if (result == PKG_NOT_FOUND) { - fprintf(stderr, "package not found: '%s'\n", test->name); + SYSERROR("package not found: '%s'", test->name); } else { - printf("package found: '%s'\n", test->name); + SYSDEBUG("package found: '%s'", test->name); } } } @@ -175,7 +175,7 @@ void test_conda_provides() { for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) { struct testcase *test = &tc[i]; - int result = pkg_index_provides(PKG_USE_CONDA, NULL, test->name); + int result = pkg_index_provides(PKG_USE_CONDA, NULL, test->name, "."); printf("%s returned %d, expecting %d\n", test->name, result, test->expected); STASIS_ASSERT(result == test->expected, "Unexpected result"); } @@ -208,7 +208,7 @@ int main(int argc, char *argv[]) { char ws[] = "workspace_XXXXXX"; if (!mkdtemp(ws)) { - perror("mkdtemp"); + SYSERROR("unable to mkdtemp: %s", strerror(errno)); exit(1); } getcwd(cwd_start, sizeof(cwd_start) - 1); diff --git a/tests/test_docker.c b/tests/test_docker.c index d60522f..b0cf381 100644 --- a/tests/test_docker.c +++ b/tests/test_docker.c @@ -41,7 +41,7 @@ void test_docker_build_and_script_and_save() { if (!pushd("test_docker_build")) { stasis_testing_write_ascii("Dockerfile", dockerfile_contents); STASIS_ASSERT(docker_build(".", "-t test_docker_build", cap_suite.build) == 0, "docker build test failed"); - STASIS_ASSERT(docker_script("test_docker_build", "uname -a", 0) == 0, "simple docker container script execution failed"); + STASIS_ASSERT(docker_script("test_docker_build", "--rm", "uname -a", 0) == 0, "simple docker container script execution failed"); STASIS_ASSERT(docker_save("test_docker_build", ".", STASIS_DOCKER_IMAGE_COMPRESSION) == 0, "saving a simple image failed"); STASIS_ASSERT(docker_exec("load < test_docker_build.tar.*", 0) == 0, "loading a simple image failed"); docker_exec("image rm -f test_docker_build", 0); diff --git a/tests/test_download.c b/tests/test_download.c index 714e614..6ace119 100644 --- a/tests/test_download.c +++ b/tests/test_download.c @@ -21,7 +21,7 @@ void test_download() { for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) { const char *filename = "output.txt"; - char errmsg[BUFSIZ] = {0}; + char errmsg[STASIS_BUFSIZ] = {0}; char *errmsg_p = errmsg; long http_code = download((char *) tc[i].url, filename, &errmsg_p); if (tc[i].errmsg) { @@ -33,7 +33,6 @@ void test_download() { } STASIS_ASSERT(http_code == tc[i].http_code, "expecting non-error HTTP code"); - //char **data = file_readlines(filename, 0, 100, NULL); char *data = stasis_testing_read_ascii(filename); if (http_code >= 0) { STASIS_ASSERT(data != NULL, "data should not be null"); diff --git a/tests/test_environment.c b/tests/test_environment.c index 4f36883..72a938d 100644 --- a/tests/test_environment.c +++ b/tests/test_environment.c @@ -56,9 +56,9 @@ void test_runtime() { STASIS_ASSERT(strcmp(global_custom_value, custom_value) == 0, "local and global CUSTOM_KEY variable are supposed to be identical"); guard_free(custom_value); - char output_truth[BUFSIZ] = {0}; + char output_truth[STASIS_BUFSIZ] = {0}; char *your_path = runtime_get(env, "PATH"); - sprintf(output_truth, "Your PATH is '%s'.", your_path); + snprintf(output_truth, sizeof(output_truth), "Your PATH is '%s'.", your_path); guard_free(your_path); char *output_expanded = runtime_expand_var(env, "Your PATH is '${PATH}'."); diff --git a/tests/test_ini.c b/tests/test_ini.c index e4a7808..3070806 100644 --- a/tests/test_ini.c +++ b/tests/test_ini.c @@ -195,6 +195,23 @@ void test_ini_getval_wrappers() { ini_free(&ini); } +void test_ini_getall() { + const char *filename = "ini_open.ini"; + struct INIFILE *ini = NULL; + const char *data = "[default]\nhello=world!\nthis=is a test\nx=1\ny=0\n"; + + stasis_testing_write_ascii(filename, data); + ini = ini_open(filename); + STASIS_ASSERT_FATAL(ini != NULL, "failed to open ini file"); + + const struct INIData *d = NULL; + while ((d = ini_getall(ini, "default")) != NULL) { + STASIS_ASSERT(d->key != NULL, "INIData key should not be NULL"); + STASIS_ASSERT(d->value != NULL, "INIData key should not be NULL"); + } + ini_free(&ini); +} + int main(int argc, char *argv[]) { STASIS_TEST_BEGIN_MAIN(); STASIS_TEST_FUNC *tests[] = { @@ -204,6 +221,7 @@ int main(int argc, char *argv[]) { test_ini_has_key, test_ini_setval_getval, test_ini_getval_wrappers, + test_ini_getall, }; STASIS_TEST_RUN(tests); STASIS_TEST_END_MAIN(); diff --git a/tests/test_junitxml.c b/tests/test_junitxml.c index 362cb32..0bbbefb 100644 --- a/tests/test_junitxml.c +++ b/tests/test_junitxml.c @@ -4,7 +4,7 @@ void test_junitxml_testsuite_read() { struct JUNIT_Testsuite *testsuite; char datafile[PATH_MAX] = {0}; - snprintf(datafile, sizeof(datafile) - 1, "%s/result.xml", TEST_DATA_DIR); + snprintf(datafile, sizeof(datafile), "%s/result.xml", TEST_DATA_DIR); STASIS_ASSERT_FATAL((testsuite = junitxml_testsuite_read(datafile)) != NULL, "failed to load testsuite data"); STASIS_ASSERT(testsuite->name != NULL, "Test suite must be named"); @@ -48,7 +48,7 @@ void test_junitxml_testsuite_read() { void test_junitxml_testsuite_read_error() { struct JUNIT_Testsuite *testsuite; char datafile[PATH_MAX] = {0}; - snprintf(datafile, sizeof(datafile) - 1, "%s/result_error.xml", TEST_DATA_DIR); + snprintf(datafile, sizeof(datafile), "%s/result_error.xml", TEST_DATA_DIR); STASIS_ASSERT_FATAL((testsuite = junitxml_testsuite_read(datafile)) != NULL, "failed to load testsuite data"); STASIS_ASSERT(testsuite->name != NULL, "test suite must be named"); diff --git a/tests/test_multiprocessing.c b/tests/test_multiprocessing.c index 7c9d695..767a9e0 100644 --- a/tests/test_multiprocessing.c +++ b/tests/test_multiprocessing.c @@ -65,7 +65,7 @@ void test_mp_task() { for (size_t i = 0; i < sizeof(commands) / sizeof(*commands); i++) { struct MultiProcessingTask *task; char task_name[100] = {0}; - sprintf(task_name, "mytask%zu", i); + snprintf(task_name, sizeof(task_name), "mytask%zu", i); STASIS_ASSERT_FATAL((task = mp_pool_task(pool, task_name, NULL, commands[i])) != NULL, "Task should not be NULL"); STASIS_ASSERT(task->pid == MP_POOL_PID_UNUSED, "PID should be non-zero at this point"); STASIS_ASSERT(task->parent_pid == MP_POOL_PID_UNUSED, "Parent PID should be non-zero"); @@ -137,7 +137,7 @@ void test_mp_fail_fast() { for (size_t i = 0; i < sizeof(commands_ff) / sizeof(*commands_ff); i++) { char *command = commands_ff[i]; char taskname[100] = {0}; - snprintf(taskname, sizeof(taskname) - 1, "task_%03zu", i); + snprintf(taskname, sizeof(taskname), "task_%03zu", i); STASIS_ASSERT(mp_pool_task(p, taskname, NULL, (char *) command) != NULL, "Failed to queue task"); } @@ -161,7 +161,7 @@ void test_mp_fail_fast() { if (task->status == 0) result.total_status_success++; if (task->pid == MP_POOL_PID_UNUSED && task->status == MP_POOL_TASK_STATUS_INITIAL) result.total_unused++; } - fprintf(stderr, "total_status_fail = %d\ntotal_status_success = %d\ntotal_signaled = %d\ntotal_unused = %d\n", + STASIS_TEST_MSG("\ntotal_status_fail = %d\ntotal_status_success = %d\ntotal_signaled = %d\ntotal_unused = %d", result.total_status_fail, result.total_status_success, result.total_signaled, result.total_unused); STASIS_ASSERT(result.total_status_fail, "Should have failures"); STASIS_ASSERT(result.total_status_success, "Should have successes"); @@ -171,6 +171,43 @@ void test_mp_fail_fast() { mp_pool_free(&p); } +static void test_mp_timeout() { + struct MultiProcessingPool *p = NULL; + p = mp_pool_init("timeout", "timeoutlogs"); + p->status_interval = 1; + struct MultiProcessingTask *task = mp_pool_task(p, "timeout", NULL, "sleep 5"); + int timeout = 3; + task->timeout = timeout; + mp_pool_join(p, 1, 0); + STASIS_ASSERT((task->time_data.duration >= (double) timeout && task->time_data.duration < (double) timeout + 1), "Timeout occurred out of desired range"); + mp_pool_show_summary(p); + mp_pool_free(&p); +} + +static void test_mp_seconds_to_human_readable() { + const struct testcase { + int seconds; + const char *expected; + } tc[] = { + {.seconds = -1, "-1s"}, + {.seconds = 0, "0s"}, + {.seconds = 10, "10s"}, + {.seconds = 20, "20s"}, + {.seconds = 30, "30s"}, + {.seconds = 60, "1m 0s"}, + {.seconds = 125, "2m 5s"}, + {.seconds = 3600, "1h 0m 0s"}, + {.seconds = 86399, "23h 59m 59s"}, + {.seconds = 86400, "24h 0m 0s"}, + }; + for (size_t i = 0; i < sizeof(tc) / sizeof(tc[0]); i++) { + char result[255] = {0}; + seconds_to_human_readable(tc[i].seconds, result, sizeof(result)); + STASIS_TEST_MSG("seconds=%d, expected: %s, got: %s", tc[i].seconds, tc[i].expected, result); + STASIS_ASSERT(strcmp(result, tc[i].expected) == 0, "bad output"); + } +} + pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; static void *pool_container(void *data) { char *commands_sc[] = { @@ -212,8 +249,13 @@ int main(int argc, char *argv[]) { test_mp_pool_free, test_mp_pool_workflow, test_mp_fail_fast, + test_mp_timeout, + test_mp_seconds_to_human_readable, test_mp_stop_continue }; + + globals.task_timeout = 60; + STASIS_TEST_RUN(tests); STASIS_TEST_END_MAIN(); } diff --git a/tests/test_recipe.c b/tests/test_recipe.c index fc7cc78..3ea21ce 100644 --- a/tests/test_recipe.c +++ b/tests/test_recipe.c @@ -4,7 +4,7 @@ static void make_local_recipe(const char *localdir) { char path[PATH_MAX] = {0}; - sprintf(path, "./%s", localdir); + snprintf(path, sizeof(path), "./%s", localdir); mkdir(path, 0755); if (!pushd(path)) { touch("meta.yaml"); diff --git a/tests/test_relocation.c b/tests/test_relocation.c index a6c33f2..891e346 100644 --- a/tests/test_relocation.c +++ b/tests/test_relocation.c @@ -13,10 +13,14 @@ void test_replace_text() { for (size_t i = 0; i < sizeof(targets) / sizeof(*targets); i += 2) { const char *target = targets[i]; const char *expected = targets[i + 1]; - char input[BUFSIZ] = {0}; - strcpy(input, test_string); + char input[STASIS_BUFSIZ] = {0}; + strncpy(input, test_string, sizeof(input) - 1); + input[sizeof(input) - 1] = '\0'; + printf("input: %s\n", input); + printf("target: %s\n", target); STASIS_ASSERT(replace_text(input, target, "^^^", 0) == 0, "string replacement failed"); + printf("result: %s\n\n", input); STASIS_ASSERT(strcmp(input, expected) == 0, "unexpected replacement"); } @@ -36,16 +40,19 @@ void test_file_replace_text() { STASIS_ASSERT(file_replace_text(filename, target, "^^^", 0) == 0, "string replacement failed"); } else { STASIS_ASSERT(false, "failed to open file for writing"); + fclose(fp); return; } - char input[BUFSIZ] = {0}; + char input[STASIS_BUFSIZ] = {0}; fp = fopen(filename, "r"); if (fp) { fread(input, sizeof(*input), sizeof(input), fp); STASIS_ASSERT(strcmp(input, expected) == 0, "unexpected replacement"); + fclose(fp); } else { STASIS_ASSERT(false, "failed to open file for reading"); + fclose(fp); return; } } diff --git a/tests/test_str.c b/tests/test_str.c index ad0c07a..09d8809 100644 --- a/tests/test_str.c +++ b/tests/test_str.c @@ -6,16 +6,17 @@ void test_to_short_version() { const char *expected; }; - struct testcase tc[] = { - {.data = "1.2.3", .expected = "123"}, + const struct testcase tc[] = { + {.data = "1.2.3", .expected = "12"}, {.data = "py3.12", .expected = "py312"}, - {.data = "generic-1.2.3", .expected = "generic-123"}, + {.data = "generic-1.2.3", .expected = "generic-12"}, {.data = "nothing to do", .expected = "nothing to do"}, }; for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) { char *result = to_short_version(tc[i].data); STASIS_ASSERT_FATAL(result != NULL, "should not be NULL"); + //printf("%s[%zu], result: %s, expected: %s\n", __FUNCTION__, i, result, tc[i].expected); STASIS_ASSERT(strcmp(result, tc[i].expected) == 0, "unexpected result"); guard_free(result); } @@ -36,7 +37,8 @@ void test_tolower_s() { for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) { char input[100] = {0}; - strcpy(input, tc[i].data); + strncpy(input, tc[i].data, sizeof(input) - 1); + input[sizeof(input) - 1] = '\0'; tolower_s(input); STASIS_ASSERT(strcmp(input, tc[i].expected) == 0, "unexpected result"); } @@ -316,9 +318,9 @@ void test_lstrip() { STASIS_ASSERT(lstrip(NULL) == NULL, "incorrect return type"); for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) { char *buf = calloc(255, sizeof(*buf)); - char *result; - strcpy(buf, tc[i].data); - result = lstrip(buf); + strncpy(buf, tc[i].data, 254); + buf[254] = '\0'; + char *result = lstrip(buf); STASIS_ASSERT(strcmp(result ? result : "", tc[i].expected) == 0, "incorrect strip-from-left"); guard_free(buf); } @@ -341,9 +343,9 @@ void test_strip() { STASIS_ASSERT(strip(NULL) == NULL, "incorrect return type"); for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) { char *buf = calloc(255, sizeof(*buf)); - char *result; - strcpy(buf, tc[i].data); - result = strip(buf); + strncpy(buf, tc[i].data, 254); + buf[254] = '\0'; + char *result = strip(buf); STASIS_ASSERT(strcmp(result ? result : "", tc[i].expected) == 0, "incorrect strip-from-right"); guard_free(buf); } diff --git a/tests/test_strlist.c b/tests/test_strlist.c index ce38ff6..38343f4 100644 --- a/tests/test_strlist.c +++ b/tests/test_strlist.c @@ -115,6 +115,7 @@ void test_strlist_append_file() { const char *local_filename = "test_strlist_append_file.txt"; struct testcase tc[] = { + {.origin = "https://this-will-never-work.tld/remote.txt", .expected = (const char *[]){NULL}}, {.origin = "https://ssb.stsci.edu/jhunk/stasis_test/test_strlist_append_file_from_remote.txt", .expected = expected}, {.origin = local_filename, .expected = expected}, }; @@ -141,10 +142,10 @@ void test_strlist_append_file() { const char *left; const char *right; left = strlist_item(list, z); - right = expected[z]; + right = tc[i].expected[z]; STASIS_ASSERT(strcmp(left, right) == 0, "file content is different than expected"); } - STASIS_ASSERT(strcmp_array((const char **) list->data, expected) == 0, "file contents does not match expected values"); + STASIS_ASSERT(strcmp_array((const char **) list->data, tc[i].expected) == 0, "file contents does not match expected values"); guard_strlist_free(&list); } } @@ -199,6 +200,20 @@ void test_strlist_append_tokenize() { guard_strlist_free(&list); } +void test_strlist_appendf() { + const char *fmt = "%c %s %d"; + struct StrList *list; + list = strlist_init(); + const int len = strlist_appendf(NULL, fmt, 'a', "abc", strlen(fmt)); + STASIS_ASSERT(strlist_appendf(&list, fmt, 'a', "abc", strlen(fmt)) == len, "length of formatted string should be 7"); + const char *item = strlist_item(list, 0); + STASIS_ASSERT(item != NULL, "valid pointer expected, item should not be NULL"); + STASIS_ASSERT(strncmp(item, "a", 1) == 0, "first character should be 'a'"); + STASIS_ASSERT(strncmp(item + 2, "abc", 3) == 0, "string should be 'abc'"); + STASIS_ASSERT(strncmp(item + 6, "8", 1) == 0, "length of the raw format should be 8"); + guard_strlist_free(&list); +} + void test_strlist_copy() { struct StrList *list = strlist_init(); struct StrList *list_copy; @@ -627,6 +642,7 @@ void test_strlist_item_as_long_double() { int main(int argc, char *argv[]) { STASIS_TEST_BEGIN_MAIN(); STASIS_TEST_FUNC *tests[] = { + test_strlist_appendf, test_strlist_init, test_strlist_free, test_strlist_append, diff --git a/tests/test_system.c b/tests/test_system.c index 2271e13..cdef618 100644 --- a/tests/test_system.c +++ b/tests/test_system.c @@ -4,7 +4,7 @@ static int ascii_file_contains(const char *filename, const char *value) { int result = -1; char *contents = stasis_testing_read_ascii(filename); if (!contents) { - perror(filename); + SYSERROR("unable to read %s: %s", filename, strerror(errno)); return result; } result = strcmp(contents, value) == 0; @@ -54,7 +54,7 @@ void test_shell_safe_verify_restrictions() { char cmd[PATH_MAX] = {0}; memset(&proc, 0, sizeof(proc)); - sprintf(cmd, "true%c false", invalid_chars[i]); + snprintf(cmd, sizeof(cmd), "true%c false", invalid_chars[i]); shell_safe(&proc, cmd); STASIS_ASSERT(proc.returncode == -1, "expected a negative result due to intentional error"); } diff --git a/tests/test_template.c b/tests/test_template.c index 596c2b7..3efb142 100644 --- a/tests/test_template.c +++ b/tests/test_template.c @@ -10,8 +10,9 @@ static int adder(struct tplfunc_frame *frame, void *result) { int a = (int) strtol(frame->argv[0].t_char_ptr, NULL, 10); int b = (int) strtol(frame->argv[1].t_char_ptr, NULL, 10); char **ptr = (char **) result; - *ptr = calloc(100, sizeof(*ptr)); - sprintf(*ptr, "%d", a + b); + const size_t sz = 100; + *ptr = calloc(sz, sizeof(*ptr)); + snprintf(*ptr, sz, "%d", a + b); return 0; } @@ -19,8 +20,9 @@ static int subtractor(struct tplfunc_frame *frame, void *result) { int a = (int) strtol(frame->argv[0].t_char_ptr, NULL, 10); int b = (int) strtol(frame->argv[1].t_char_ptr, NULL, 10); char **ptr = (char **) result; - *ptr = calloc(100, sizeof(*ptr)); - sprintf(*ptr, "%d", a - b); + const size_t sz = 100; + *ptr = calloc(sz, sizeof(*ptr)); + snprintf(*ptr, sz, "%d", a - b); return 0; } @@ -28,8 +30,9 @@ static int multiplier(struct tplfunc_frame *frame, void *result) { int a = (int) strtol(frame->argv[0].t_char_ptr, NULL, 10); int b = (int) strtol(frame->argv[1].t_char_ptr, NULL, 10); char **ptr = (char **) result; - *ptr = calloc(100, sizeof(*ptr)); - sprintf(*ptr, "%d", a * b); + const size_t sz = 100; + *ptr = calloc(sz, sizeof(*ptr)); + snprintf(*ptr, sz, "%d", a * b); return 0; } @@ -37,8 +40,9 @@ static int divider(struct tplfunc_frame *frame, void *result) { int a = (int) strtol(frame->argv[0].t_char_ptr, NULL, 10); int b = (int) strtol(frame->argv[1].t_char_ptr, NULL, 10); char **ptr = (char **) result; - *ptr = calloc(100, sizeof(*ptr)); - sprintf(*ptr, "%d", a / b); + size_t sz = 100; + *ptr = calloc(sz, sizeof(*ptr)); + snprintf(*ptr, sz, "%d", a / b); return 0; } @@ -57,6 +61,19 @@ void test_tpl_workflow() { STASIS_ASSERT(strcmp(result, "Hello environment!") == 0, "environment variable content mismatch"); guard_free(result); unsetenv("HELLO"); + + const char *message_file = "message.txt"; + char message_fmt[] = "They wanted a {{ hello_message }} " + "So we gave them a {{ hello_message }}"; + const char *message_expected = "They wanted a Hello world! " + "So we gave them a Hello world!"; + const int state = tpl_render_to_file(message_fmt, message_file); + STASIS_ASSERT_FATAL(state == 0, "failed to write rendered string to file"); + char *message_contents = stasis_testing_read_ascii(message_file); + STASIS_ASSERT(strcmp(message_contents, message_expected) == 0, "message in file does not match original message"); + guard_free(message_contents); + remove(message_file); + guard_free(data); } @@ -68,6 +85,8 @@ void test_tpl_register() { STASIS_ASSERT(tpl_pool_used == (used_before_register + 1), "tpl_register did not increment allocation counter"); STASIS_ASSERT(tpl_pool[used_before_register] != NULL, "register did not allocate a tpl_item record in the pool"); + const char *message = tpl_getval("hello_message"); + STASIS_ASSERT(strcmp(message, "Hello world!") == 0, "stored message corrupt"); free(data); } @@ -76,7 +95,7 @@ void test_tpl_register_func() { struct testcase { const char *key; int argc; - void *func; + tplfunc *func; }; struct testcase tc[] = { {.key = "add", .argc = 2, .func = &adder}, @@ -92,25 +111,25 @@ void test_tpl_register_func() { char *result = NULL; result = tpl_render("{{ func:add(0,3) }}"); - STASIS_ASSERT(result != NULL && strcmp(result, "3") == 0, "Answer was not 3"); + STASIS_ASSERT(result != NULL && strcmp(result, "3") == 0, "add: Answer was not 3"); guard_free(result); result = tpl_render("{{ func:add(1,2) }}"); - STASIS_ASSERT(result != NULL && strcmp(result, "3") == 0, "Answer was not 3"); + STASIS_ASSERT(result != NULL && strcmp(result, "3") == 0, "add: Answer was not 3"); guard_free(result); result = tpl_render("{{ func:sub(6,3) }}"); - STASIS_ASSERT(result != NULL && strcmp(result, "3") == 0, "Answer was not 3"); + STASIS_ASSERT(result != NULL && strcmp(result, "3") == 0, "sub: was not 3"); guard_free(result); result = tpl_render("{{ func:sub(4,1) }}"); - STASIS_ASSERT(result != NULL && strcmp(result, "3") == 0, "Answer was not 3"); + STASIS_ASSERT(result != NULL && strcmp(result, "3") == 0, "sub: Answer was not 3"); guard_free(result); result = tpl_render("{{ func:mul(1, 3) }}"); - STASIS_ASSERT(result != NULL && strcmp(result, "3") == 0, "Answer was not 3"); + STASIS_ASSERT(result != NULL && strcmp(result, "3") == 0, "mul: Answer was not 3"); guard_free(result); result = tpl_render("{{ func:div(6,2) }}"); - STASIS_ASSERT(result != NULL && strcmp(result, "3") == 0, "Answer was not 3"); + STASIS_ASSERT(result != NULL && strcmp(result, "3") == 0, "div: Answer was not 3"); guard_free(result); result = tpl_render("{{ func:div(3,1) }}"); - STASIS_ASSERT(result != NULL && strcmp(result, "3") == 0, "Answer was not 3"); + STASIS_ASSERT(result != NULL && strcmp(result, "3") == 0, "div: Answer was not 3"); guard_free(result); } diff --git a/tests/test_tests.c b/tests/test_tests.c new file mode 100644 index 0000000..0f6d7ca --- /dev/null +++ b/tests/test_tests.c @@ -0,0 +1,52 @@ +#include "delivery.h" +#include "testing.h" + +static struct Test *mock_test(const int ident) { + struct Test *test = test_init(); + if (asprintf(&test->name, "test_%d", ident) < 0) { + return NULL; + } + return test; +} + +void test_tests() { + const int initial = TEST_NUM_ALLOC_INITIAL; + const int balloon = initial * 10; + struct Tests *tests = tests_init(initial); + STASIS_ASSERT_FATAL(tests != NULL, "tests structure allocation failed"); + STASIS_ASSERT(tests->num_alloc == (size_t) initial, "incorrect number of records initialized"); + STASIS_ASSERT(tests->num_used == 0, "incorrect number of records used"); + + for (int i = 0; i < balloon; i++) { + struct Test *test = mock_test(i); + if (!test) { + SYSERROR("unable to allocate memory for test %d", i); + return; + } + tests_add(tests, test); + } + + size_t errors = 0; + for (int i = 0; i < initial * 10; i++) { + struct Test *test = tests->test[i]; + if (!test) { + errors++; + continue; + } + if (!test->name) { + errors++; + } + } + STASIS_ASSERT(errors == 0, "no errors should be detected in test->name member"); + + tests_free(&tests); +} + +int main(int argc, char *argv[]) { + STASIS_TEST_BEGIN_MAIN(); + STASIS_TEST_FUNC *tests[] = { + test_tests, + }; + STASIS_TEST_RUN(tests); + STASIS_TEST_END_MAIN(); +}
\ No newline at end of file diff --git a/tests/test_utils.c b/tests/test_utils.c index 0e2eb7b..696e7ff 100644 --- a/tests/test_utils.c +++ b/tests/test_utils.c @@ -65,12 +65,14 @@ void test_fix_tox_conf() { if (fp) { fprintf(fp, "%s", data); fclose(fp); - STASIS_ASSERT(fix_tox_conf(filename, &result) == 0, "fix_tox_conf failed"); + STASIS_ASSERT(fix_tox_conf(filename, &result, PATH_MAX) == 0, "fix_tox_conf failed"); } else { STASIS_ASSERT(false, "writing mock tox.ini failed"); } char **lines = file_readlines(result, 0, 0, NULL); + STASIS_ASSERT_FATAL(lines != NULL, "lines array should not be NULL"); + STASIS_ASSERT_FATAL(result != NULL, "result should not be NULL"); STASIS_ASSERT(strstr_array(lines, expected) != NULL, "{posargs} not found in result"); guard_array_free(lines); @@ -103,7 +105,7 @@ void test_xml_pretty_print_in_place() { } fp = fopen(filename, "r"); - char buf[BUFSIZ] = {0}; + char buf[STASIS_BUFSIZ] = {0}; if (fread(buf, sizeof(*buf), sizeof(buf) - 1, fp) < 1) { STASIS_ASSERT(false, "failed to consume formatted xml file contents"); } @@ -129,7 +131,7 @@ void test_isempty_dir() { STASIS_ASSERT(isempty_dir(dname) > 0, "empty directory went undetected"); char path[PATH_MAX]; - sprintf(path, "%s/file.txt", dname); + snprintf(path, sizeof(path), "%s/file.txt", dname); touch(path); STASIS_ASSERT(isempty_dir(dname) == 0, "populated directory went undetected"); @@ -147,7 +149,9 @@ void test_xmkstemp() { char buf[100] = {0}; tempfp = fopen(tempfile, "r"); - fgets(buf, sizeof(buf) - 1, tempfp); + const char *line = fgets(buf, sizeof(buf) - 1, tempfp); + STASIS_ASSERT_FATAL(line != NULL, "file should contain data written earlier"); + STASIS_ASSERT(strcmp(line, buf) == 0, "file should contain the correct data"); fclose(tempfp); STASIS_ASSERT(strcmp(buf, data) == 0, "data written to temp file is incorrect"); @@ -193,7 +197,7 @@ void test_git_clone_and_describe() { // initialize a bare repo so we can clone it char init_cmd[PATH_MAX]; - sprintf(init_cmd, "git init --bare %s", repo_git); + snprintf(init_cmd, sizeof(init_cmd), "git init --bare %s", repo_git); system(init_cmd); // clone the bare repo @@ -213,17 +217,21 @@ void test_git_clone_and_describe() { // test git_describe is functional char *taginfo_none = git_describe("."); STASIS_ASSERT(taginfo_none != NULL, "should be a git hash, not NULL"); + puts(taginfo_none); + STASIS_ASSERT(is_git_sha(taginfo_none) == true, "not a git hash"); system("git tag -a 1.0.0 -m Mock"); system("git push --tags origin"); - char *taginfo = git_describe("."); + const char *taginfo = git_describe("."); + puts(taginfo); STASIS_ASSERT(taginfo != NULL, "should be 1.0.0, not NULL"); - STASIS_ASSERT(strcmp(taginfo, "1.0.0") == 0, "just-created tag was not described correctly"); + STASIS_ASSERT(startswith(taginfo, "1.0.0") == true, "just-created tag was not described correctly"); chdir(".."); char *taginfo_outer = git_describe(repo); + puts(taginfo_outer); STASIS_ASSERT(taginfo_outer != NULL, "should be 1.0.0, not NULL"); - STASIS_ASSERT(strcmp(taginfo_outer, "1.0.0") == 0, "just-created tag was not described correctly (out-of-dir invocation)"); + STASIS_ASSERT(startswith(taginfo_outer, "1.0.0") == true, "just-created tag was not described correctly (out-of-dir invocation)"); char *taginfo_bad = git_describe("abc1234_not_here_or_there"); STASIS_ASSERT(taginfo_bad == NULL, "a repository that shouldn't exist... exists and has a tag."); @@ -268,11 +276,12 @@ void test_file_readlines() { const char *data = "I am\na file\nwith multiple lines\nsee?\n"; FILE *fp = fopen(filename, "w"); if (!fp) { - perror(filename); + SYSERROR("unable to open file: %s, %s", filename, strerror(errno)); return; } if (fwrite(data, sizeof(*data), strlen(data), fp) != strlen(data)) { perror("short write"); + fclose(fp); return; } fclose(fp); @@ -304,7 +313,8 @@ void test_path_dirname() { const char *input = data[i]; const char *expected = data[i + 1]; char tmp[PATH_MAX] = {0}; - strcpy(tmp, input); + strncpy(tmp, input, sizeof(tmp) - 1); + tmp[sizeof(tmp) - 1] = '\0'; char *result = path_dirname(tmp); STASIS_ASSERT(strcmp(expected, result) == 0, NULL); @@ -325,8 +335,7 @@ void test_path_basename() { } void test_expandpath() { - char *home; - + char *home = NULL; const char *homes[] = { "HOME", "USERPROFILE", @@ -337,10 +346,11 @@ void test_expandpath() { break; } } + STASIS_ASSERT_FATAL(home != NULL, "cannot expand without knowing the user's home directory path"); char path[PATH_MAX] = {0}; - strcat(path, "~"); - strcat(path, DIR_SEP); + strncat(path, "~", sizeof(path) - strlen(path) - 1); + strncat(path, DIR_SEP, sizeof(path) - strlen(path) - 1); char *expanded = expandpath(path); STASIS_ASSERT(startswith(expanded, home) > 0, expanded); @@ -362,10 +372,10 @@ void test_rmtree() { for (size_t i = 0; i < sizeof(tree) / sizeof(*tree); i++) { char path[PATH_MAX]; char mockfile[PATH_MAX + 10]; - sprintf(path, "%s/%s", root, tree[i]); - sprintf(mockfile, "%s/%zu.txt", path, i); + snprintf(path, sizeof(path), "%s/%s", root, tree[i]); + snprintf(mockfile, sizeof(mockfile), "%s/%zu.txt", path, i); if (mkdir(path, 0755)) { - perror(path); + SYSERROR("mkdir failed: %s, %s", path, strerror(errno)); STASIS_ASSERT(false, NULL); } touch(mockfile); diff --git a/tests/test_version_compare.c b/tests/test_version_compare.c new file mode 100644 index 0000000..2a3458f --- /dev/null +++ b/tests/test_version_compare.c @@ -0,0 +1,172 @@ +#include "testing.h" +#include "version_compare.h" + +struct TestCase_version_compare { + char *a, *op, *b; + int expected; +}; + +struct TestCase_version_compare test_cases_version_compare[] = { + {"0", "=", "0", 1}, + {"0", "<", "1",1}, + {"0", "<=", "1",1}, + {"0", ">", "1",0}, + {"0", ">=", "1",0}, + {"0", "!=", "1",1}, + + {"1a", "=", "1b", 0}, + {"1a", "<", "1b", 1}, + {"1a", "<=", "1b", 1}, + {"1a", ">", "1b", 0}, + {"1a", ">=", "1b", 0}, + {"1a", "!=", "1b", 1}, + + {"1.0", "=", "1.0.0", 1}, + {"1.0", "<", "1.0.0", 0}, + {"1.0", "<=", "1.0.0", 1}, + {"1.0", ">", "1.0.0", 0}, + {"1.0", ">=", "1.0.0", 1}, + {"1.0", "!=", "1.0.0", 0}, + + {"1.0rc1", "=", "1.0.0", 0}, + {"1.0rc1", "<", "1.0.0", 1}, + {"1.0rc1", "<=", "1.0.0", 1}, + {"1.0rc1", ">", "1.0.0", 0}, + {"1.0rc1", ">=", "1.0.0", 0}, + {"1.0rc1", "!=", "1.0.0", 1}, + + {"1.0rc1", "=", "1.0.0rc1", 1}, + {"1.0rc1", "<", "1.0.0rc1", 0}, + {"1.0rc1", "<=", "1.0.0rc1", 1}, + {"1.0rc1", ">", "1.0.0rc1", 0}, + {"1.0rc1", ">=", "1.0.0rc1", 1}, + {"1.0rc1", "!=", "1.0.0rc1", 0}, + + {"1.0rc1", "=", "1.0.0dev1", 0}, + {"1.0rc1", "<", "1.0.0dev1", 0}, + {"1.0rc1", "<=", "1.0.0dev1", 0}, + {"1.0rc1", ">", "1.0.0dev1", 1}, + {"1.0rc1", ">=", "1.0.0dev1", 1}, + {"1.0rc1", "!=", "1.0.0dev1", 1}, + + {"1.0post1", "=", "1.0.0dev1", 0}, + {"1.0post1", "<", "1.0.0dev1", 0}, + {"1.0post1", "<=", "1.0.0dev1", 0}, + {"1.0post1", ">", "1.0.0dev1", 1}, + {"1.0post1", ">=", "1.0.0dev1", 1}, + {"1.0post1", "!=", "1.0.0dev1", 1}, + + {"1.0post1", "=", "1.0.0", 0}, + {"1.0post1", "<", "1.0.0", 0}, + {"1.0post1", "<=", "1.0.0", 0}, + {"1.0post1", ">", "1.0.0", 1}, + {"1.0post1", ">=", "1.0.0", 1}, + {"1.0post1", "!=", "1.0.0", 1}, + + {"1.0dev1", "=", "1.0.0dev1", 1}, + {"1.0dev1", "<", "1.0.0dev1", 0}, + {"1.0dev1", "<=", "1.0.0dev1", 1}, + {"1.0dev1", ">", "1.0.0dev1", 0}, + {"1.0dev1", ">=", "1.0.0dev1", 1}, + {"1.0dev1", "!=", "1.0.0dev1", 0}, + + {"1.0a", "=", "1.0.0", 0}, + {"1.0a", "<", "1.0.0", 0}, + {"1.0a", "<=", "1.0.0", 0}, + {"1.0a", ">", "1.0.0", 1}, + {"1.0a", ">=", "1.0.0", 1}, + {"1.0a", "!=", "1.0.0", 1}, + + {"1.0.3", "=", "2.0.0", 0}, + {"1.0.3", "<", "2.0.0", 1}, + {"1.0.3", "<=", "2.0.0", 1}, + {"1.0.3", ">", "2.0.0", 0}, + {"1.0.3", ">=", "2.0.0", 0}, + {"1.0.3", "!=", "2.0.0", 1}, + + {"2022.1", "=", "2022.4", 0}, + {"2022.1", "<", "2022.4", 1}, + {"2022.1", "<=", "2022.4", 1}, + {"2022.1", ">", "2022.4", 0}, + {"2022.1", ">=", "2022.4", 0}, + {"2022.1", "!=", "2022.4", 1}, + + {"1:2022.1", "=", "2022.4", 0}, + {"1:2022.1", "<", "2022.4", 1}, + {"1:2022.1", "<=", "2022.4", 1}, + {"1:2022.1", ">", "2022.4", 0}, + {"1:2022.1", ">=", "2022.4", 0}, + {"1:2022.1", "!=", "2022.4", 1}, + + {"1:2022.1", "=", "2:2022.4", 0}, + {"1:2022.1", "<", "2:2022.4", 1}, + {"1:2022.1", "<=", "2:2022.4", 1}, + {"1:2022.1", ">", "2:2022.4", 0}, + {"1:2022.1", ">=", "2:2022.4", 0}, + {"1:2022.1", "!=", "2:2022.4", 1}, + + {"2:2022.4", "=", "1:2022.1", 0}, + {"2:2022.4", "<", "1:2022.1", 0}, + {"2:2022.4", "<=", "1:2022.1", 0}, + {"2:2022.4", ">", "1:2022.1", 1}, + {"2:2022.4", ">=", "1:2022.1", 1}, + {"2:2022.4", "!=", "1:2022.1", 1}, + + {"2022.1", "=", "2:2022.1", 0}, + {"2022.1", "<", "2:2022.1", 1}, + {"2022.1", "<=", "2:2022.1", 1}, + {"2022.1", ">", "2:2022.1", 0}, + {"2022.1", ">=", "2:2022.1", 0}, + {"2022.1", "!=", "2:2022.1", 1}, + + {"2022.4", "=", "2022.1", 0}, + {"2022.4", "<", "2022.1", 0}, + {"2022.4", "<=", "2022.1", 0}, + {"2022.4", ">", "2022.1", 1}, + {"2022.4", ">=", "2022.1", 1}, + {"2022.4", "!=", "2022.1", 1}, + + // Error cases + {NULL, "", "", -1}, + {"", NULL, "", -1}, + {"", "", NULL, -1}, + {NULL, NULL, NULL, -1}, + {"", "=", "", -1}, + {" ", "=", " ", -1}, + {"a", "", "a", -1}, + {"a", "", "b", -1}, + {"a", "@", "b", -1}, +}; + +void run_cases_version_compare(void) { + const size_t size = sizeof(test_cases_version_compare) / sizeof(test_cases_version_compare[0]); + for (size_t i = 0; i < size; i++) { + int result = 0; + const struct TestCase_version_compare *test = &test_cases_version_compare[i]; + const int op = version_parse_operator(test->op); + result = version_compare(op, test->a, test->b); + STASIS_ASSERT(test->expected == result, "unexpected result"); + + fprintf(stderr, "'%s' '%s' '%s' is %s (%d)", + test->a ? test->a : "NULL", + test->op ? test->op : "NULL", + test->b ? test->b : "NULL", + result == test->expected ? "EXPECTED" : "UNEXPECTED", + result); + if (test->expected != result) { + fprintf(stderr, " [FAILED: got %d, expected %d]\n", result, test->expected); + } else { + puts(""); + } + } +} + +int main(void) { + STASIS_TEST_BEGIN_MAIN(); + STASIS_TEST_FUNC *tests[] = { + run_cases_version_compare, + }; + STASIS_TEST_RUN(tests); + + STASIS_TEST_END_MAIN(); +} diff --git a/tests/test_wheel.c b/tests/test_wheel.c index 6818b22..e486b05 100644 --- a/tests/test_wheel.c +++ b/tests/test_wheel.c @@ -1,91 +1,217 @@ +#include "conda.h" +#include "delivery.h" #include "testing.h" +#include "str.h" #include "wheel.h" -void test_get_wheel_file() { - struct testcase { - const char *filename; - struct Wheel expected; - }; - struct testcase tc[] = { - { - // Test for "build tags" - .filename = "btpackage-1.2.3-mytag-py2.py3-none-any.whl", - .expected = { - .file_name = "btpackage-1.2.3-mytag-py2.py3-none-any.whl", - .version = "1.2.3", - .distribution = "btpackage", - .build_tag = "mytag", - .platform_tag = "any", - .python_tag = "py2.py3", - .abi_tag = "none", - .path_name = ".", - } - }, - { - // Test for universal package format - .filename = "anypackage-1.2.3-py2.py3-none-any.whl", - .expected = { - .file_name = "anypackage-1.2.3-py2.py3-none-any.whl", - .version = "1.2.3", - .distribution = "anypackage", - .build_tag = NULL, - .platform_tag = "any", - .python_tag = "py2.py3", - .abi_tag = "none", - .path_name = ".", - } - }, - { - // Test for binary package format - .filename = "binpackage-1.2.3-cp311-cp311-linux_x86_64.whl", - .expected = { - .file_name = "binpackage-1.2.3-cp311-cp311-linux_x86_64.whl", - .version = "1.2.3", - .distribution = "binpackage", - .build_tag = NULL, - .platform_tag = "linux_x86_64", - .python_tag = "cp311", - .abi_tag = "cp311", - .path_name = ".", - } - }, - }; +char cwd_start[PATH_MAX]; +char cwd_workspace[PATH_MAX]; +int conda_is_installed = 0; +static char conda_prefix[PATH_MAX] = {0}; +struct Delivery ctx; +static const char *testpkg_filename = "testpkg/dist/testpkg-1.0.0-py3-none-any.whl"; + + +static void test_wheel_package() { + const char *filename = testpkg_filename; + struct Wheel *wheel = NULL; + int state = wheel_package(&wheel, filename); + STASIS_ASSERT(state != WHEEL_PACKAGE_E_ALLOC, "Cannot fail to allocate memory for package structure"); + STASIS_ASSERT(state != WHEEL_PACKAGE_E_GET, "Cannot fail to parse wheel"); + STASIS_ASSERT(state != WHEEL_PACKAGE_E_GET_METADATA, "Cannot fail to read wheel metadata"); + STASIS_ASSERT(state != WHEEL_PACKAGE_E_GET_RECORDS, "Cannot fail reading wheel path records"); + STASIS_ASSERT(state != WHEEL_PACKAGE_E_GET_ENTRY_POINT, "Cannot fail reading wheel entry points"); + STASIS_ASSERT(state == WHEEL_PACKAGE_E_SUCCESS, "Wheel file should be usable"); + STASIS_ASSERT(wheel != NULL, "wheel cannot be NULL"); + STASIS_ASSERT(wheel != NULL, "wheel_package failed to initialize wheel struct"); + STASIS_ASSERT(wheel->record != NULL, "Record cannot be NULL"); + STASIS_ASSERT(wheel->num_record > 0, "Record count cannot be zero"); + STASIS_ASSERT(wheel->tag != NULL, "Package tag list cannot be NULL"); + STASIS_ASSERT(wheel->generator != NULL, "Generator field cannot be NULL"); + STASIS_ASSERT(wheel->top_level != NULL, "Top level directory name cannot be NULL"); + STASIS_ASSERT(wheel->wheel_version != NULL, "Wheel version cannot be NULL"); + STASIS_ASSERT(wheel->metadata != NULL, "Metadata cannot be NULL"); + STASIS_ASSERT(wheel->metadata->name != NULL, "Metadata::name cannot be NULL"); + STASIS_ASSERT(wheel->metadata->version != NULL, "Metadata::version cannot be NULL"); + STASIS_ASSERT(wheel->metadata->metadata_version != NULL, "Metadata::version (of metadata) cannot be NULL"); + + // Implied test against key/id getters. If wheel_show_info segfaults, that functionality is broken. + STASIS_ASSERT(wheel_show_info(wheel) == 0, "wheel_show_info should never fail. Enum(s) might be out of sync with META_*_KEYS array(s)"); + + // Get data from DIST + const struct WheelValue dist_version = wheel_get_value_by_name(wheel, WHEEL_FROM_DIST, "Wheel-Version"); + STASIS_ASSERT(dist_version.type == WHEELVAL_STR, "Wheel dist version value must be a string"); + STASIS_ASSERT_FATAL(dist_version.data != NULL, "Wheel dist version value must not be NULL"); + STASIS_ASSERT(dist_version.count != 0, "Wheel value must be populated"); + + // Get data from METADATA + const struct WheelValue meta_name = wheel_get_value_by_name(wheel, WHEEL_FROM_METADATA, "Metadata-Version"); + STASIS_ASSERT(meta_name.type == WHEELVAL_STR, "Wheel metadata version value must be a string"); + STASIS_ASSERT_FATAL(meta_name.data != NULL, "Wheel metadata version value must not be NULL"); + STASIS_ASSERT(meta_name.count != 0, "Wheel metadata version value must be populated"); + + wheel_package_free(&wheel); + STASIS_ASSERT(wheel == NULL, "wheel struct should be NULL after free"); +} + +static void mock_python_package() { + const char *pyproject_toml_data = "[build-system]\n" + "requires = [\"setuptools >= 77.0.3\"]\n" + "build-backend = \"setuptools.build_meta\"\n" + "\n" + "[project]\n" + "name = \"testpkg\"\n" + "version = \"1.0.0\"\n" + "authors = [{name = \"STASIS Team\", email = \"stasis@not-a-real-domain.tld\"}]\n" + "description = \"A STASIS test package\"\n" + "readme = \"README.md\"\n" + "license = \"BSD-3-Clause\"\n" + "classifiers = [\"Programming Language :: Python :: 3\"]\n" + "\n" + "[project.urls]\n" + "Homepage = \"https://not-a-real-address.tld\"\n" + "Documentation = \"https://not-a-real-address.tld/docs\"\n" + "Repository = \"https://not-a-real-address.tld/repo.git\"\n" + "Issues = \"https://not-a-real-address.tld/tracker\"\n" + "Changelog = \"https://not-a-real-address.tld/changes\"\n"; + const char *readme = "# testpkg\n\nThis is a test package, for testing.\n"; - struct Wheel *doesnotexist = get_wheel_info("doesnotexist", "doesnotexist-0.0.1-py2.py3-none-any.whl", (char *[]) {"not", NULL}, WHEEL_MATCH_ANY); - STASIS_ASSERT(doesnotexist == NULL, "returned non-NULL on error"); - - for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) { - struct testcase *test = &tc[i]; - struct Wheel *wheel = get_wheel_info(".", test->expected.distribution, (char *[]) {(char *) test->expected.version, NULL}, WHEEL_MATCH_ANY); - STASIS_ASSERT(wheel != NULL, "result should not be NULL!"); - STASIS_ASSERT(wheel->file_name && strcmp(wheel->file_name, test->expected.file_name) == 0, "mismatched file name"); - STASIS_ASSERT(wheel->version && strcmp(wheel->version, test->expected.version) == 0, "mismatched version"); - STASIS_ASSERT(wheel->distribution && strcmp(wheel->distribution, test->expected.distribution) == 0, "mismatched distribution (package name)"); - STASIS_ASSERT(wheel->platform_tag && strcmp(wheel->platform_tag, test->expected.platform_tag) == 0, "mismatched platform tag ([platform]_[architecture])"); - STASIS_ASSERT(wheel->python_tag && strcmp(wheel->python_tag, test->expected.python_tag) == 0, "mismatched python tag (python version)"); - STASIS_ASSERT(wheel->abi_tag && strcmp(wheel->abi_tag, test->expected.abi_tag) == 0, "mismatched abi tag (python compatibility version)"); - if (wheel->build_tag) { - STASIS_ASSERT(strcmp(wheel->build_tag, test->expected.build_tag) == 0, - "mismatched build tag (optional arbitrary string)"); - } - wheel_free(&wheel); + mkdir("testpkg", 0755); + mkdir("testpkg/src", 0755); + mkdir("testpkg/src/testpkg", 0755); + if (touch("testpkg/src/testpkg/__init__.py")) { + SYSERROR("unable to write __init__.py"); + exit(1); + } + if (touch("testpkg/README.md")) { + SYSERROR("unable to write README.md"); + exit(1); + } + if (stasis_testing_write_ascii("testpkg/pyproject.toml", pyproject_toml_data)) { + SYSERROR("unable to write pyproject.toml"); + exit(1); + } + if (stasis_testing_write_ascii("testpkg/README.md", readme)) { + SYSERROR("unable to write readme"); + exit(1); + } + if (pip_exec("install build")) { + SYSERROR("unable to install build tool using pip"); + exit(1); + } + if (python_exec("-m build -w ./testpkg")) { + SYSERROR("unable build test package"); + exit(1); } } int main(int argc, char *argv[]) { STASIS_TEST_BEGIN_MAIN(); STASIS_TEST_FUNC *tests[] = { - test_get_wheel_file, + test_wheel_package, }; - // Create mock package directories, and files - mkdir("binpackage", 0755); - touch("binpackage/binpackage-1.2.3-cp311-cp311-linux_x86_64.whl"); - mkdir("anypackage", 0755); - touch("anypackage/anypackage-1.2.3-py2.py3-none-any.whl"); - mkdir("btpackage", 0755); - touch("btpackage/btpackage-1.2.3-mytag-py2.py3-none-any.whl"); + char ws[] = "workspace_XXXXXX"; + if (!mkdtemp(ws)) { + SYSERROR("mkdtemp failed: %s, %s", ws, strerror(errno)); + exit(1); + } + getcwd(cwd_start, sizeof(cwd_start) - 1); + mkdir(ws, 0755); + chdir(ws); + getcwd(cwd_workspace, sizeof(cwd_workspace) - 1); + + snprintf(conda_prefix, strlen(cwd_workspace) + strlen("conda") + 2, "%s/conda", cwd_workspace); + + const char *mockinidata = "[meta]\n" + "name = mock\n" + "version = 1.0.0\n" + "rc = 1\n" + "mission = generic\n" + "python = 3.11\n" + "[conda]\n" + "installer_name = Miniforge3\n" + "installer_version = 24.3.0-0\n" + "installer_platform = {{env:STASIS_CONDA_PLATFORM}}\n" + "installer_arch = {{env:STASIS_CONDA_ARCH}}\n" + "installer_baseurl = https://github.com/conda-forge/miniforge/releases/download/24.3.0-0\n"; + stasis_testing_write_ascii("mock.ini", mockinidata); + struct INIFILE *ini = ini_open("mock.ini"); + ctx._stasis_ini_fp.delivery = ini; + ctx._stasis_ini_fp.delivery_path = realpath("mock.ini", NULL); + + const char *sysconfdir = getenv("STASIS_SYSCONFDIR"); + globals.sysconfdir = strdup(sysconfdir ? sysconfdir : STASIS_SYSCONFDIR); + ctx.storage.root = strdup(cwd_workspace); + char *cfgfile = join((char *[]) {globals.sysconfdir, "stasis.ini", NULL}, "/"); + if (!cfgfile) { + SYSERROR("unable to create path to global config"); + exit(1); + } + + ctx._stasis_ini_fp.cfg = ini_open(cfgfile); + if (!ctx._stasis_ini_fp.cfg) { + SYSERROR("unable to open config file, %s", cfgfile); + exit(1); + } + ctx._stasis_ini_fp.cfg_path = realpath(cfgfile, NULL); + if (!ctx._stasis_ini_fp.cfg_path) { + SYSERROR("unable to determine absolute path of config, %s", cfgfile); + exit(1); + } + guard_free(cfgfile); + + setenv("LANG", "C", 1); + if (bootstrap_build_info(&ctx)) { + SYSERROR("bootstrap_build_info failed"); + exit(1); + } + if (delivery_init(&ctx, INI_READ_RENDER)) { + SYSERROR("delivery_init failed"); + exit(1); + } + + char *install_url = calloc(255, sizeof(install_url)); + delivery_get_conda_installer_url(&ctx, install_url, PATH_MAX); + delivery_get_conda_installer(&ctx, install_url); + delivery_install_conda(ctx.conda.installer_path, ctx.storage.conda_install_prefix); + guard_free(install_url); + + if (conda_activate(ctx.storage.conda_install_prefix, "base")) { + SYSERROR("conda_activate failed"); + exit(1); + } + if (conda_exec("install -y boa conda-build")) { + SYSERROR("conda_exec failed"); + exit(1); + } + if (conda_setup_headless()) { + SYSERROR("conda_setup_headless failed"); + exit(1); + } + if (conda_env_create("testpkg", ctx.meta.python, NULL)) { + SYSERROR("conda_env_create failed"); + exit(1); + } + if (conda_activate(ctx.storage.conda_install_prefix, "testpkg")) { + SYSERROR("conda_activate failed"); + exit(1); + } + + mock_python_package(); STASIS_TEST_RUN(tests); + + if (chdir(cwd_start) < 0) { + SYSERROR("chdir failed: %s, %s", cwd_start, strerror(errno)); + exit(1); + } + if (rmtree(cwd_workspace)) { + SYSERROR("rmtree failed: %s, %s", cwd_workspace, strerror(errno)); + } + delivery_free(&ctx); + globals_free(); + STASIS_TEST_END_MAIN(); + }
\ No newline at end of file diff --git a/tests/test_wheelinfo.c b/tests/test_wheelinfo.c new file mode 100644 index 0000000..1abbeac --- /dev/null +++ b/tests/test_wheelinfo.c @@ -0,0 +1,91 @@ +#include "testing.h" +#include "wheelinfo.h" + +void test_wheelinfo_get() { + struct testcase { + const char *filename; + struct WheelInfo expected; + }; + struct testcase tc[] = { + { + // Test for "build tags" + .filename = "btpackage-1.2.3-mytag-py2.py3-none-any.whl", + .expected = { + .file_name = "btpackage-1.2.3-mytag-py2.py3-none-any.whl", + .version = "1.2.3", + .distribution = "btpackage", + .build_tag = "mytag", + .platform_tag = "any", + .python_tag = "py2.py3", + .abi_tag = "none", + .path_name = ".", + } + }, + { + // Test for universal package format + .filename = "anypackage-1.2.3-py2.py3-none-any.whl", + .expected = { + .file_name = "anypackage-1.2.3-py2.py3-none-any.whl", + .version = "1.2.3", + .distribution = "anypackage", + .build_tag = NULL, + .platform_tag = "any", + .python_tag = "py2.py3", + .abi_tag = "none", + .path_name = ".", + } + }, + { + // Test for binary package format + .filename = "binpackage-1.2.3-cp311-cp311-linux_x86_64.whl", + .expected = { + .file_name = "binpackage-1.2.3-cp311-cp311-linux_x86_64.whl", + .version = "1.2.3", + .distribution = "binpackage", + .build_tag = NULL, + .platform_tag = "linux_x86_64", + .python_tag = "cp311", + .abi_tag = "cp311", + .path_name = ".", + } + }, + }; + + struct WheelInfo *doesnotexist = wheelinfo_get("doesnotexist", "doesnotexist-0.0.1-py2.py3-none-any.whl", (char *[]) {"not", NULL}, WHEEL_MATCH_ANY); + STASIS_ASSERT(doesnotexist == NULL, "returned non-NULL on error"); + + for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) { + struct testcase *test = &tc[i]; + struct WheelInfo *wheel = wheelinfo_get(".", test->expected.distribution, (char *[]) {(char *) test->expected.version, NULL}, WHEEL_MATCH_ANY); + STASIS_ASSERT(wheel != NULL, "result should not be NULL!"); + STASIS_ASSERT(wheel->file_name && strcmp(wheel->file_name, test->expected.file_name) == 0, "mismatched file name"); + STASIS_ASSERT(wheel->version && strcmp(wheel->version, test->expected.version) == 0, "mismatched version"); + STASIS_ASSERT(wheel->distribution && strcmp(wheel->distribution, test->expected.distribution) == 0, "mismatched distribution (package name)"); + STASIS_ASSERT(wheel->platform_tag && strcmp(wheel->platform_tag, test->expected.platform_tag) == 0, "mismatched platform tag ([platform]_[architecture])"); + STASIS_ASSERT(wheel->python_tag && strcmp(wheel->python_tag, test->expected.python_tag) == 0, "mismatched python tag (python version)"); + STASIS_ASSERT(wheel->abi_tag && strcmp(wheel->abi_tag, test->expected.abi_tag) == 0, "mismatched abi tag (python compatibility version)"); + if (wheel->build_tag) { + STASIS_ASSERT(strcmp(wheel->build_tag, test->expected.build_tag) == 0, + "mismatched build tag (optional arbitrary string)"); + } + wheelinfo_free(&wheel); + } +} + +int main(int argc, char *argv[]) { + STASIS_TEST_BEGIN_MAIN(); + STASIS_TEST_FUNC *tests[] = { + test_wheelinfo_get, + }; + + // Create mock package directories, and files + mkdir("binpackage", 0755); + touch("binpackage/binpackage-1.2.3-cp311-cp311-linux_x86_64.whl"); + mkdir("anypackage", 0755); + touch("anypackage/anypackage-1.2.3-py2.py3-none-any.whl"); + mkdir("btpackage", 0755); + touch("btpackage/btpackage-1.2.3-mytag-py2.py3-none-any.whl"); + + STASIS_TEST_RUN(tests); + STASIS_TEST_END_MAIN(); +}
\ No newline at end of file |