diff options
Diffstat (limited to 'src/lib/core')
38 files changed, 2571 insertions, 2578 deletions
diff --git a/src/lib/core/CMakeLists.txt b/src/lib/core/CMakeLists.txt index c569187..e3e3d4b 100644 --- a/src/lib/core/CMakeLists.txt +++ b/src/lib/core/CMakeLists.txt @@ -1,5 +1,3 @@ -include_directories(${PROJECT_BINARY_DIR}) - add_library(stasis_core STATIC globals.c str.c @@ -10,17 +8,6 @@ add_library(stasis_core STATIC utils.c system.c download.c - delivery_postprocess.c - delivery_conda.c - delivery_docker.c - delivery_install.c - delivery_artifactory.c - delivery_test.c - delivery_build.c - delivery_show.c - delivery_populate.c - delivery_init.c - delivery.c recipe.c relocation.c wheel.c @@ -35,4 +22,8 @@ add_library(stasis_core STATIC envctl.c multiprocessing.c ) - +target_include_directories(stasis_core PRIVATE + ${core_INCLUDE} + ${delivery_INCLUDE} + ${CMAKE_CURRENT_SOURCE_DIR}/include +) diff --git a/src/lib/core/delivery.c b/src/lib/core/delivery.c deleted file mode 100644 index aa3e51a..0000000 --- a/src/lib/core/delivery.c +++ /dev/null @@ -1,323 +0,0 @@ -#include "delivery.h" - -void delivery_free(struct Delivery *ctx) { - guard_free(ctx->system.arch); - GENERIC_ARRAY_FREE(ctx->system.platform); - guard_free(ctx->meta.name); - guard_free(ctx->meta.version); - guard_free(ctx->meta.codename); - guard_free(ctx->meta.mission); - guard_free(ctx->meta.python); - guard_free(ctx->meta.mission); - guard_free(ctx->meta.python_compact); - guard_free(ctx->meta.based_on); - guard_runtime_free(ctx->runtime.environ); - guard_free(ctx->storage.root); - guard_free(ctx->storage.tmpdir); - guard_free(ctx->storage.delivery_dir); - guard_free(ctx->storage.tools_dir); - guard_free(ctx->storage.package_dir); - guard_free(ctx->storage.results_dir); - guard_free(ctx->storage.output_dir); - guard_free(ctx->storage.conda_install_prefix); - guard_free(ctx->storage.conda_artifact_dir); - guard_free(ctx->storage.conda_staging_dir); - guard_free(ctx->storage.conda_staging_url); - guard_free(ctx->storage.wheel_artifact_dir); - guard_free(ctx->storage.wheel_staging_dir); - guard_free(ctx->storage.wheel_staging_url); - guard_free(ctx->storage.build_dir); - guard_free(ctx->storage.build_recipes_dir); - guard_free(ctx->storage.build_sources_dir); - guard_free(ctx->storage.build_testing_dir); - guard_free(ctx->storage.build_docker_dir); - guard_free(ctx->storage.mission_dir); - guard_free(ctx->storage.docker_artifact_dir); - guard_free(ctx->storage.meta_dir); - guard_free(ctx->storage.package_dir); - guard_free(ctx->storage.cfgdump_dir); - guard_free(ctx->info.time_str_epoch); - guard_free(ctx->info.build_name); - guard_free(ctx->info.build_number); - guard_free(ctx->info.release_name); - guard_free(ctx->conda.installer_baseurl); - guard_free(ctx->conda.installer_name); - guard_free(ctx->conda.installer_version); - guard_free(ctx->conda.installer_platform); - guard_free(ctx->conda.installer_arch); - guard_free(ctx->conda.installer_path); - guard_free(ctx->conda.tool_version); - guard_free(ctx->conda.tool_build_version); - guard_strlist_free(&ctx->conda.conda_packages); - guard_strlist_free(&ctx->conda.conda_packages_defer); - guard_strlist_free(&ctx->conda.pip_packages); - guard_strlist_free(&ctx->conda.pip_packages_defer); - guard_strlist_free(&ctx->conda.wheels_packages); - - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { - guard_free(ctx->tests[i].name); - guard_free(ctx->tests[i].version); - guard_free(ctx->tests[i].repository); - guard_free(ctx->tests[i].repository_info_ref); - guard_free(ctx->tests[i].repository_info_tag); - guard_strlist_free(&ctx->tests[i].repository_remove_tags); - guard_free(ctx->tests[i].script); - guard_free(ctx->tests[i].build_recipe); - // test-specific runtime variables - guard_runtime_free(ctx->tests[i].runtime.environ); - } - - guard_free(ctx->rules.release_fmt); - guard_free(ctx->rules.build_name_fmt); - guard_free(ctx->rules.build_number_fmt); - - guard_free(ctx->deploy.docker.test_script); - guard_free(ctx->deploy.docker.registry); - guard_free(ctx->deploy.docker.image_compression); - guard_strlist_free(&ctx->deploy.docker.tags); - guard_strlist_free(&ctx->deploy.docker.build_args); - - for (size_t i = 0; i < sizeof(ctx->deploy.jfrog) / sizeof(ctx->deploy.jfrog[0]); i++) { - guard_free(ctx->deploy.jfrog[i].repo); - guard_free(ctx->deploy.jfrog[i].dest); - guard_strlist_free(&ctx->deploy.jfrog[i].files); - } - - if (ctx->_stasis_ini_fp.delivery) { - ini_free(&ctx->_stasis_ini_fp.delivery); - } - guard_free(ctx->_stasis_ini_fp.delivery_path); - - if (ctx->_stasis_ini_fp.cfg) { - // optional extras - ini_free(&ctx->_stasis_ini_fp.cfg); - } - guard_free(ctx->_stasis_ini_fp.cfg_path); - - if (ctx->_stasis_ini_fp.mission) { - ini_free(&ctx->_stasis_ini_fp.mission); - } - guard_free(ctx->_stasis_ini_fp.mission_path); -} - -int delivery_format_str(struct Delivery *ctx, char **dest, const char *fmt) { - size_t fmt_len = strlen(fmt); - - if (!*dest) { - *dest = calloc(STASIS_NAME_MAX, sizeof(**dest)); - if (!*dest) { - return -1; - } - } - - for (size_t i = 0; i < fmt_len; i++) { - if (fmt[i] == '%' && strlen(&fmt[i])) { - i++; - switch (fmt[i]) { - case 'n': // name - strcat(*dest, ctx->meta.name); - break; - case 'c': // codename - strcat(*dest, ctx->meta.codename); - break; - case 'm': // mission - strcat(*dest, ctx->meta.mission); - break; - case 'r': // revision - sprintf(*dest + strlen(*dest), "%d", ctx->meta.rc); - break; - case 'R': // "final"-aware revision - if (ctx->meta.final) - strcat(*dest, "final"); - else - sprintf(*dest + strlen(*dest), "%d", ctx->meta.rc); - break; - case 'v': // version - strcat(*dest, ctx->meta.version); - break; - case 'P': // python version - strcat(*dest, ctx->meta.python); - break; - case 'p': // python version major/minor - strcat(*dest, ctx->meta.python_compact); - break; - case 'a': // system architecture name - strcat(*dest, ctx->system.arch); - break; - case 'o': // system platform (OS) name - strcat(*dest, ctx->system.platform[DELIVERY_PLATFORM_RELEASE]); - break; - case 't': // unix epoch - sprintf(*dest + strlen(*dest), "%ld", ctx->info.time_now); - break; - default: // unknown formatter, write as-is - sprintf(*dest + strlen(*dest), "%c%c", fmt[i - 1], fmt[i]); - break; - } - } else { // write non-format text - sprintf(*dest + strlen(*dest), "%c", fmt[i]); - } - } - return 0; -} - -void delivery_defer_packages(struct Delivery *ctx, int type) { - struct StrList *dataptr = NULL; - struct StrList *deferred = NULL; - char *name = NULL; - - char mode[10]; - if (DEFER_CONDA == type) { - dataptr = ctx->conda.conda_packages; - deferred = ctx->conda.conda_packages_defer; - strcpy(mode, "conda"); - } else if (DEFER_PIP == type) { - dataptr = ctx->conda.pip_packages; - deferred = ctx->conda.pip_packages_defer; - strcpy(mode, "pip"); - } else { - SYSERROR("BUG: type %d does not map to a supported package manager!\n", type); - exit(1); - } - msg(STASIS_MSG_L2, "Filtering %s packages by test definition...\n", mode); - - struct StrList *filtered = NULL; - filtered = strlist_init(); - for (size_t i = 0; i < strlist_count(dataptr); i++) { - int build_for_host = 0; - - name = strlist_item(dataptr, i); - if (!strlen(name) || isblank(*name) || isspace(*name)) { - // no data - continue; - } - - // Compile a list of packages that are *also* to be tested. - char *spec_begin = strpbrk(name, "@~=<>!"); - char *spec_end = spec_begin; - char package_name[255] = {0}; - - if (spec_end) { - // A version is present in the package name. Jump past operator(s). - while (*spec_end != '\0' && !isalnum(*spec_end)) { - spec_end++; - } - strncpy(package_name, name, spec_begin - name); - } else { - strncpy(package_name, name, sizeof(package_name) - 1); - } - - char *extra_begin = strchr(package_name, '['); - char *extra_end = NULL; - if (extra_begin) { - extra_end = strchr(extra_begin, ']'); - if (extra_end) { - *extra_begin = '\0'; - } - } - - msg(STASIS_MSG_L3, "package '%s': ", package_name); - - // When spec is present in name, set tests->version to the version detected in the name - for (size_t x = 0; x < sizeof(ctx->tests) / sizeof(ctx->tests[0]) && ctx->tests[x].name != NULL; x++) { - struct Test *test = &ctx->tests[x]; - char nametmp[1024] = {0}; - - strncpy(nametmp, package_name, sizeof(nametmp) - 1); - // Is the [test:NAME] in the package name? - if (!strcmp(nametmp, test->name)) { - // Override test->version when a version is provided by the (pip|conda)_package list item - guard_free(test->version); - if (spec_begin && spec_end) { - test->version = strdup(spec_end); - } else { - // There are too many possible default branches nowadays: master, main, develop, xyz, etc. - // HEAD is a safe bet. - test->version = strdup("HEAD"); - } - - // Is the list item a git+schema:// URL? - if (strstr(nametmp, "git+") && strstr(nametmp, "://")) { - char *xrepo = strstr(nametmp, "+"); - if (xrepo) { - xrepo++; - guard_free(test->repository); - test->repository = strdup(xrepo); - xrepo = NULL; - } - // Extract the name of the package - char *xbasename = path_basename(nametmp); - if (xbasename) { - // Replace the git+schema:// URL with the package name - strlist_set(&dataptr, i, xbasename); - name = strlist_item(dataptr, i); - } - } - - int upstream_exists = 0; - if (DEFER_PIP == type) { - upstream_exists = pkg_index_provides(PKG_USE_PIP, PYPI_INDEX_DEFAULT, name); - } else if (DEFER_CONDA == type) { - upstream_exists = pkg_index_provides(PKG_USE_CONDA, NULL, name); - } - - if (PKG_INDEX_PROVIDES_FAILED(upstream_exists)) { - fprintf(stderr, "%s's existence command failed for '%s': %s\n", - mode, name, pkg_index_provides_strerror(upstream_exists)); - exit(1); - } - - if (upstream_exists == PKG_NOT_FOUND) { - build_for_host = 1; - } else { - build_for_host = 0; - } - - break; - } - } - - if (build_for_host) { - printf("BUILD FOR HOST\n"); - strlist_append(&deferred, name); - } else { - printf("USE EXTERNAL\n"); - strlist_append(&filtered, name); - } - } - - if (!strlist_count(deferred)) { - msg(STASIS_MSG_WARN | STASIS_MSG_L2, "No %s packages were filtered by test definitions\n", mode); - } else { - if (DEFER_CONDA == type) { - strlist_free(&ctx->conda.conda_packages); - ctx->conda.conda_packages = strlist_copy(filtered); - } else if (DEFER_PIP == type) { - strlist_free(&ctx->conda.pip_packages); - ctx->conda.pip_packages = strlist_copy(filtered); - } - } - if (filtered) { - strlist_free(&filtered); - } -} - -int delivery_gather_tool_versions(struct Delivery *ctx) { - int status_tool_version = 0; - int status_tool_build_version = 0; - - // Extract version from tool output - ctx->conda.tool_version = shell_output("conda --version", &status_tool_version); - if (ctx->conda.tool_version) - strip(ctx->conda.tool_version); - - ctx->conda.tool_build_version = shell_output("conda build --version", &status_tool_build_version); - if (ctx->conda.tool_build_version) - strip(ctx->conda.tool_version); - - if (status_tool_version || status_tool_build_version) { - return 1; - } - return 0; -} - diff --git a/src/lib/core/delivery_artifactory.c b/src/lib/core/delivery_artifactory.c deleted file mode 100644 index 9ad5829..0000000 --- a/src/lib/core/delivery_artifactory.c +++ /dev/null @@ -1,204 +0,0 @@ -#include "delivery.h" - -int delivery_init_artifactory(struct Delivery *ctx) { - int status = 0; - char dest[PATH_MAX] = {0}; - char filepath[PATH_MAX] = {0}; - snprintf(dest, sizeof(dest) - 1, "%s/bin", ctx->storage.tools_dir); - snprintf(filepath, sizeof(dest) - 1, "%s/bin/jf", ctx->storage.tools_dir); - - if (!access(filepath, F_OK)) { - // already have it - msg(STASIS_MSG_L3, "Skipped download, %s already exists\n", filepath); - goto delivery_init_artifactory_envsetup; - } - - char *platform = ctx->system.platform[DELIVERY_PLATFORM]; - msg(STASIS_MSG_L3, "Downloading %s for %s %s\n", globals.jfrog.remote_filename, platform, ctx->system.arch); - if ((status = artifactory_download_cli(dest, - globals.jfrog.jfrog_artifactory_base_url, - globals.jfrog.jfrog_artifactory_product, - globals.jfrog.cli_major_ver, - globals.jfrog.version, - platform, - ctx->system.arch, - globals.jfrog.remote_filename))) { - remove(filepath); - } - - delivery_init_artifactory_envsetup: - // CI (ridiculously generic, why?) disables interactive prompts and progress bar output - setenv("CI", "1", 1); - - // JFROG_CLI_HOME_DIR is where .jfrog is stored - char path[PATH_MAX] = {0}; - snprintf(path, sizeof(path) - 1, "%s/.jfrog", ctx->storage.build_dir); - setenv("JFROG_CLI_HOME_DIR", path, 1); - - // JFROG_CLI_TEMP_DIR is where the obvious is stored - setenv("JFROG_CLI_TEMP_DIR", ctx->storage.tmpdir, 1); - return status; -} - -int delivery_artifact_upload(struct Delivery *ctx) { - int status = 0; - - if (jfrt_auth_init(&ctx->deploy.jfrog_auth)) { - fprintf(stderr, "Failed to initialize Artifactory authentication context\n"); - return -1; - } - - for (size_t i = 0; i < sizeof(ctx->deploy.jfrog) / sizeof(*ctx->deploy.jfrog); i++) { - if (!ctx->deploy.jfrog[i].files || !ctx->deploy.jfrog[i].dest) { - break; - } - jfrt_upload_init(&ctx->deploy.jfrog[i].upload_ctx); - - if (!globals.jfrog.repo) { - msg(STASIS_MSG_WARN, "Artifactory repository path is not configured!\n"); - fprintf(stderr, "set STASIS_JF_REPO environment variable...\nOr append to configuration file:\n\n"); - fprintf(stderr, "[deploy:artifactory]\nrepo = example/generic/repo/path\n\n"); - status++; - break; - } else if (!ctx->deploy.jfrog[i].repo) { - ctx->deploy.jfrog[i].repo = strdup(globals.jfrog.repo); - } - - if (!ctx->deploy.jfrog[i].repo || isempty(ctx->deploy.jfrog[i].repo) || !strlen(ctx->deploy.jfrog[i].repo)) { - // Unlikely to trigger if the config parser is working correctly - msg(STASIS_MSG_ERROR, "Artifactory repository path is empty. Cannot continue.\n"); - status++; - break; - } - - ctx->deploy.jfrog[i].upload_ctx.workaround_parent_only = true; - ctx->deploy.jfrog[i].upload_ctx.build_name = ctx->info.build_name; - ctx->deploy.jfrog[i].upload_ctx.build_number = ctx->info.build_number; - - if (jfrog_cli_rt_ping(&ctx->deploy.jfrog_auth)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Unable to contact artifactory server: %s\n", ctx->deploy.jfrog_auth.url); - return -1; - } - - if (strlist_count(ctx->deploy.jfrog[i].files)) { - for (size_t f = 0; f < strlist_count(ctx->deploy.jfrog[i].files); f++) { - char dest[PATH_MAX] = {0}; - char files[PATH_MAX] = {0}; - snprintf(dest, sizeof(dest) - 1, "%s/%s", ctx->deploy.jfrog[i].repo, ctx->deploy.jfrog[i].dest); - snprintf(files, sizeof(files) - 1, "%s", strlist_item(ctx->deploy.jfrog[i].files, f)); - status += jfrog_cli_rt_upload(&ctx->deploy.jfrog_auth, &ctx->deploy.jfrog[i].upload_ctx, files, dest); - } - } - } - - if (globals.enable_artifactory_build_info) { - if (!status && ctx->deploy.jfrog[0].files && ctx->deploy.jfrog[0].dest) { - jfrog_cli_rt_build_collect_env(&ctx->deploy.jfrog_auth, ctx->deploy.jfrog[0].upload_ctx.build_name, - ctx->deploy.jfrog[0].upload_ctx.build_number); - jfrog_cli_rt_build_publish(&ctx->deploy.jfrog_auth, ctx->deploy.jfrog[0].upload_ctx.build_name, - ctx->deploy.jfrog[0].upload_ctx.build_number); - } - } else { - msg(STASIS_MSG_WARN | STASIS_MSG_L2, "Artifactory build info upload is disabled by CLI argument\n"); - } - - return status; -} - -int delivery_mission_render_files(struct Delivery *ctx) { - if (!ctx->storage.mission_dir) { - fprintf(stderr, "Mission directory is not configured. Context not initialized?\n"); - return -1; - } - struct Data { - char *src; - char *dest; - } data; - struct INIFILE *cfg = ctx->_stasis_ini_fp.mission; - - memset(&data, 0, sizeof(data)); - data.src = calloc(PATH_MAX, sizeof(*data.src)); - if (!data.src) { - perror("data.src"); - return -1; - } - - for (size_t i = 0; i < cfg->section_count; i++) { - union INIVal val; - char *section_name = cfg->section[i]->key; - if (!startswith(section_name, "template:")) { - continue; - } - val.as_char_p = strchr(section_name, ':') + 1; - if (val.as_char_p && isempty(val.as_char_p)) { - guard_free(data.src); - return 1; - } - sprintf(data.src, "%s/%s/%s", ctx->storage.mission_dir, ctx->meta.mission, val.as_char_p); - msg(STASIS_MSG_L2, "%s\n", data.src); - - int err = 0; - data.dest = ini_getval_str(cfg, section_name, "destination", INI_READ_RENDER, &err); - - struct stat st; - if (lstat(data.src, &st)) { - perror(data.src); - guard_free(data.dest); - continue; - } - - char *contents = calloc(st.st_size + 1, sizeof(*contents)); - if (!contents) { - perror("template file contents"); - guard_free(data.dest); - continue; - } - - FILE *fp = fopen(data.src, "rb"); - if (!fp) { - perror(data.src); - guard_free(contents); - guard_free(data.dest); - continue; - } - - if (fread(contents, st.st_size, sizeof(*contents), fp) < 1) { - perror("while reading template file"); - guard_free(contents); - guard_free(data.dest); - fclose(fp); - continue; - } - fclose(fp); - - msg(STASIS_MSG_L3, "Writing %s\n", data.dest); - if (tpl_render_to_file(contents, data.dest)) { - guard_free(contents); - guard_free(data.dest); - continue; - } - guard_free(contents); - guard_free(data.dest); - } - - guard_free(data.src); - return 0; -} - -int delivery_series_sync(struct Delivery *ctx) { - struct JFRT_Download dl = {0}; - - char *remote_dir = NULL; - if (asprintf(&remote_dir, "%s/%s/%s/(*)", globals.jfrog.repo, ctx->meta.mission, ctx->info.build_name) < 0) { - SYSERROR("%s", "Unable to allocate bytes for remote directory path"); - return -1; - } - - char *dest_dir = NULL; - if (asprintf(&dest_dir, "%s/{1}", ctx->storage.output_dir) < 0) { - SYSERROR("%s", "Unable to allocate bytes for destination directory path"); - return -1; - } - - return jfrog_cli_rt_download(&ctx->deploy.jfrog_auth, &dl, remote_dir, dest_dir); -} diff --git a/src/lib/core/delivery_build.c b/src/lib/core/delivery_build.c deleted file mode 100644 index fa19f95..0000000 --- a/src/lib/core/delivery_build.c +++ /dev/null @@ -1,198 +0,0 @@ -#include "delivery.h" - -int delivery_build_recipes(struct Delivery *ctx) { - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { - char *recipe_dir = NULL; - if (ctx->tests[i].build_recipe) { // build a conda recipe - if (recipe_clone(ctx->storage.build_recipes_dir, ctx->tests[i].build_recipe, NULL, &recipe_dir)) { - fprintf(stderr, "Encountered an issue while cloning recipe for: %s\n", ctx->tests[i].name); - return -1; - } - if (!recipe_dir) { - fprintf(stderr, "BUG: recipe_clone() succeeded but recipe_dir is NULL: %s\n", strerror(errno)); - return -1; - } - int recipe_type = recipe_get_type(recipe_dir); - if(!pushd(recipe_dir)) { - if (RECIPE_TYPE_ASTROCONDA == recipe_type) { - pushd(path_basename(ctx->tests[i].repository)); - } else if (RECIPE_TYPE_CONDA_FORGE == recipe_type) { - pushd("recipe"); - } - - char recipe_version[100]; - char recipe_buildno[100]; - char recipe_git_url[PATH_MAX]; - char recipe_git_rev[PATH_MAX]; - - //sprintf(recipe_version, "{%% set version = GIT_DESCRIBE_TAG ~ \".dev\" ~ GIT_DESCRIBE_NUMBER ~ \"+\" ~ GIT_DESCRIBE_HASH %%}"); - //sprintf(recipe_git_url, " git_url: %s", ctx->tests[i].repository); - //sprintf(recipe_git_rev, " git_rev: %s", ctx->tests[i].version); - // TODO: Conditionally download archives if github.com is the origin. Else, use raw git_* keys ^^^ - sprintf(recipe_version, "{%% set version = \"%s\" %%}", ctx->tests[i].repository_info_tag ? ctx->tests[i].repository_info_tag : ctx->tests[i].version); - sprintf(recipe_git_url, " url: %s/archive/refs/tags/{{ version }}.tar.gz", ctx->tests[i].repository); - strcpy(recipe_git_rev, ""); - sprintf(recipe_buildno, " number: 0"); - - unsigned flags = REPLACE_TRUNCATE_AFTER_MATCH; - //file_replace_text("meta.yaml", "{% set version = ", recipe_version); - if (ctx->meta.final) { // remove this. i.e. statis cannot deploy a release to conda-forge - sprintf(recipe_version, "{%% set version = \"%s\" %%}", ctx->tests[i].version); - // TODO: replace sha256 of tagged archive - // TODO: leave the recipe unchanged otherwise. in theory this should produce the same conda package hash as conda forge. - // For now, remove the sha256 requirement - file_replace_text("meta.yaml", "sha256:", "\n", flags); - } else { - file_replace_text("meta.yaml", "{% set version = ", recipe_version, flags); - file_replace_text("meta.yaml", " url:", recipe_git_url, flags); - //file_replace_text("meta.yaml", "sha256:", recipe_git_rev); - file_replace_text("meta.yaml", " sha256:", "\n", flags); - file_replace_text("meta.yaml", " number:", recipe_buildno, flags); - } - - char command[PATH_MAX]; - if (RECIPE_TYPE_CONDA_FORGE == recipe_type) { - char arch[STASIS_NAME_MAX] = {0}; - char platform[STASIS_NAME_MAX] = {0}; - - strcpy(platform, ctx->system.platform[DELIVERY_PLATFORM]); - if (strstr(platform, "Darwin")) { - memset(platform, 0, sizeof(platform)); - strcpy(platform, "osx"); - } - tolower_s(platform); - if (strstr(ctx->system.arch, "arm64")) { - strcpy(arch, "arm64"); - } else if (strstr(ctx->system.arch, "64")) { - strcpy(arch, "64"); - } else { - strcat(arch, "32"); // blind guess - } - tolower_s(arch); - - sprintf(command, "mambabuild --python=%s -m ../.ci_support/%s_%s_.yaml .", - ctx->meta.python, platform, arch); - } else { - sprintf(command, "mambabuild --python=%s .", ctx->meta.python); - } - int status = conda_exec(command); - if (status) { - guard_free(recipe_dir); - return -1; - } - - if (RECIPE_TYPE_GENERIC != recipe_type) { - popd(); - } - popd(); - } else { - fprintf(stderr, "Unable to enter recipe directory %s: %s\n", recipe_dir, strerror(errno)); - guard_free(recipe_dir); - return -1; - } - } - guard_free(recipe_dir); - } - return 0; -} - -int filter_repo_tags(char *repo, struct StrList *patterns) { - int result = 0; - - if (!pushd(repo)) { - int list_status = 0; - char *tags_raw = shell_output("git tag -l", &list_status); - struct StrList *tags = strlist_init(); - strlist_append_tokenize(tags, tags_raw, LINE_SEP); - - for (size_t i = 0; tags && i < strlist_count(tags); i++) { - char *tag = strlist_item(tags, i); - for (size_t p = 0; p < strlist_count(patterns); p++) { - char *pattern = strlist_item(patterns, p); - int match = fnmatch(pattern, tag, 0); - if (!match) { - char cmd[PATH_MAX] = {0}; - sprintf(cmd, "git tag -d %s", tag); - result += system(cmd); - break; - } - } - } - guard_strlist_free(&tags); - guard_free(tags_raw); - popd(); - } else { - result = -1; - } - return result; -} - -struct StrList *delivery_build_wheels(struct Delivery *ctx) { - struct StrList *result = NULL; - struct Process proc = {0}; - - result = strlist_init(); - if (!result) { - perror("unable to allocate memory for string list"); - return NULL; - } - - for (size_t p = 0; p < strlist_count(ctx->conda.pip_packages_defer); p++) { - char name[100] = {0}; - char *fullspec = strlist_item(ctx->conda.pip_packages_defer, p); - strncpy(name, fullspec, sizeof(name) - 1); - char *spec = find_version_spec(name); - if (spec) { - *spec = '\0'; - } - - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { - if ((ctx->tests[i].name && !strcmp(name, ctx->tests[i].name)) && (!ctx->tests[i].build_recipe && ctx->tests[i].repository)) { // build from source - char srcdir[PATH_MAX]; - char wheeldir[PATH_MAX]; - memset(srcdir, 0, sizeof(srcdir)); - memset(wheeldir, 0, sizeof(wheeldir)); - - sprintf(srcdir, "%s/%s", ctx->storage.build_sources_dir, ctx->tests[i].name); - git_clone(&proc, ctx->tests[i].repository, srcdir, ctx->tests[i].version); - - if (ctx->tests[i].repository_remove_tags && strlist_count(ctx->tests[i].repository_remove_tags)) { - filter_repo_tags(srcdir, ctx->tests[i].repository_remove_tags); - } - - if (!pushd(srcdir)) { - char dname[NAME_MAX]; - char outdir[PATH_MAX]; - char cmd[PATH_MAX * 2]; - memset(dname, 0, sizeof(dname)); - memset(outdir, 0, sizeof(outdir)); - memset(cmd, 0, sizeof(outdir)); - - strcpy(dname, ctx->tests[i].name); - tolower_s(dname); - sprintf(outdir, "%s/%s", ctx->storage.wheel_artifact_dir, dname); - if (mkdirs(outdir, 0755)) { - fprintf(stderr, "failed to create output directory: %s\n", outdir); - guard_strlist_free(&result); - return NULL; - } - - sprintf(cmd, "-m build -w -o %s", outdir); - if (python_exec(cmd)) { - fprintf(stderr, "failed to generate wheel package for %s-%s\n", ctx->tests[i].name, - ctx->tests[i].version); - guard_strlist_free(&result); - return NULL; - } - popd(); - } else { - fprintf(stderr, "Unable to enter source directory %s: %s\n", srcdir, strerror(errno)); - guard_strlist_free(&result); - return NULL; - } - } - } - } - return result; -} - diff --git a/src/lib/core/delivery_conda.c b/src/lib/core/delivery_conda.c deleted file mode 100644 index 8974ae8..0000000 --- a/src/lib/core/delivery_conda.c +++ /dev/null @@ -1,109 +0,0 @@ -#include "delivery.h" - -void delivery_get_conda_installer_url(struct Delivery *ctx, char *result) { - if (ctx->conda.installer_version) { - // Use version specified by configuration file - sprintf(result, "%s/%s-%s-%s-%s.sh", ctx->conda.installer_baseurl, - ctx->conda.installer_name, - ctx->conda.installer_version, - ctx->conda.installer_platform, - ctx->conda.installer_arch); - } else { - // Use latest installer - sprintf(result, "%s/%s-%s-%s.sh", ctx->conda.installer_baseurl, - ctx->conda.installer_name, - ctx->conda.installer_platform, - ctx->conda.installer_arch); - } - -} - -int delivery_get_conda_installer(struct Delivery *ctx, char *installer_url) { - char script_path[PATH_MAX]; - char *installer = path_basename(installer_url); - - memset(script_path, 0, sizeof(script_path)); - sprintf(script_path, "%s/%s", ctx->storage.tmpdir, installer); - if (access(script_path, F_OK)) { - // Script doesn't exist - long fetch_status = download(installer_url, script_path, NULL); - if (HTTP_ERROR(fetch_status) || fetch_status < 0) { - // download failed - return -1; - } - } else { - msg(STASIS_MSG_RESTRICT | STASIS_MSG_L3, "Skipped, installer already exists\n", script_path); - } - - ctx->conda.installer_path = strdup(script_path); - if (!ctx->conda.installer_path) { - SYSERROR("Unable to duplicate script_path: '%s'", script_path); - return -1; - } - - return 0; -} - -void delivery_install_conda(char *install_script, char *conda_install_dir) { - struct Process proc = {0}; - - if (globals.conda_fresh_start) { - if (!access(conda_install_dir, F_OK)) { - // directory exists so remove it - if (rmtree(conda_install_dir)) { - perror("unable to remove previous installation"); - exit(1); - } - - // Proceed with the installation - // -b = batch mode (non-interactive) - char cmd[PATH_MAX] = {0}; - snprintf(cmd, sizeof(cmd) - 1, "%s %s -b -p %s", - find_program("bash"), - install_script, - conda_install_dir); - if (shell_safe(&proc, cmd)) { - fprintf(stderr, "conda installation failed\n"); - exit(1); - } - } else { - // Proceed with the installation - // -b = batch mode (non-interactive) - char cmd[PATH_MAX] = {0}; - snprintf(cmd, sizeof(cmd) - 1, "%s %s -b -p %s", - find_program("bash"), - install_script, - conda_install_dir); - if (shell_safe(&proc, cmd)) { - fprintf(stderr, "conda installation failed\n"); - exit(1); - } - } - } else { - msg(STASIS_MSG_L3, "Conda removal disabled by configuration\n"); - } -} - -void delivery_conda_enable(struct Delivery *ctx, char *conda_install_dir) { - if (conda_activate(conda_install_dir, "base")) { - fprintf(stderr, "conda activation failed\n"); - exit(1); - } - - // Setting the CONDARC environment variable appears to be the only consistent - // way to make sure the file is used. Not setting this variable leads to strange - // behavior, especially if a conda environment is already active when STASIS is loaded. - char rcpath[PATH_MAX]; - sprintf(rcpath, "%s/%s", conda_install_dir, ".condarc"); - setenv("CONDARC", rcpath, 1); - if (runtime_replace(&ctx->runtime.environ, __environ)) { - perror("unable to replace runtime environment after activating conda"); - exit(1); - } - - if (conda_setup_headless()) { - // no COE check. this call must succeed. - exit(1); - } -} - diff --git a/src/lib/core/delivery_docker.c b/src/lib/core/delivery_docker.c deleted file mode 100644 index 57015ad..0000000 --- a/src/lib/core/delivery_docker.c +++ /dev/null @@ -1,132 +0,0 @@ -#include "delivery.h" - -int delivery_docker(struct Delivery *ctx) { - if (!docker_capable(&ctx->deploy.docker.capabilities)) { - return -1; - } - char tag[STASIS_NAME_MAX]; - char args[PATH_MAX]; - int has_registry = ctx->deploy.docker.registry != NULL; - size_t total_tags = strlist_count(ctx->deploy.docker.tags); - size_t total_build_args = strlist_count(ctx->deploy.docker.build_args); - - if (!has_registry) { - msg(STASIS_MSG_WARN | STASIS_MSG_L2, "No docker registry defined. You will need to manually re-tag the resulting image.\n"); - } - - if (!total_tags) { - char default_tag[PATH_MAX]; - msg(STASIS_MSG_WARN | STASIS_MSG_L2, "No docker tags defined by configuration. Generating default tag(s).\n"); - // generate local tag - memset(default_tag, 0, sizeof(default_tag)); - sprintf(default_tag, "%s:%s-py%s", ctx->meta.name, ctx->info.build_name, ctx->meta.python_compact); - tolower_s(default_tag); - - // Add tag - ctx->deploy.docker.tags = strlist_init(); - strlist_append(&ctx->deploy.docker.tags, default_tag); - - if (has_registry) { - // generate tag for target registry - memset(default_tag, 0, sizeof(default_tag)); - sprintf(default_tag, "%s/%s:%s-py%s", ctx->deploy.docker.registry, ctx->meta.name, ctx->info.build_number, ctx->meta.python_compact); - tolower_s(default_tag); - - // Add tag - strlist_append(&ctx->deploy.docker.tags, default_tag); - } - // regenerate total tag available - total_tags = strlist_count(ctx->deploy.docker.tags); - } - - memset(args, 0, sizeof(args)); - - // Append image tags to command - for (size_t i = 0; i < total_tags; i++) { - char *tag_orig = strlist_item(ctx->deploy.docker.tags, i); - strcpy(tag, tag_orig); - docker_sanitize_tag(tag); - sprintf(args + strlen(args), " -t \"%s\" ", tag); - } - - // Append build arguments to command (i.e. --build-arg "key=value" - for (size_t i = 0; i < total_build_args; i++) { - char *build_arg = strlist_item(ctx->deploy.docker.build_args, i); - if (!build_arg) { - break; - } - sprintf(args + strlen(args), " --build-arg \"%s\" ", build_arg); - } - - // Build the image - char delivery_file[PATH_MAX] = {0}; - char dest[PATH_MAX] = {0}; - char rsync_cmd[PATH_MAX * 2] = {0}; - memset(delivery_file, 0, sizeof(delivery_file)); - memset(dest, 0, sizeof(dest)); - - sprintf(delivery_file, "%s/%s.yml", ctx->storage.delivery_dir, ctx->info.release_name); - if (access(delivery_file, F_OK) < 0) { - fprintf(stderr, "docker build cannot proceed without delivery file: %s\n", delivery_file); - return -1; - } - - sprintf(dest, "%s/%s.yml", ctx->storage.build_docker_dir, ctx->info.release_name); - if (copy2(delivery_file, dest, CT_PERM)) { - fprintf(stderr, "Failed to copy delivery file to %s: %s\n", dest, strerror(errno)); - return -1; - } - - memset(dest, 0, sizeof(dest)); - sprintf(dest, "%s/packages", ctx->storage.build_docker_dir); - - msg(STASIS_MSG_L2, "Copying conda packages\n"); - memset(rsync_cmd, 0, sizeof(rsync_cmd)); - sprintf(rsync_cmd, "rsync -avi --progress '%s' '%s'", ctx->storage.conda_artifact_dir, dest); - if (system(rsync_cmd)) { - fprintf(stderr, "Failed to copy conda artifacts to docker build directory\n"); - return -1; - } - - msg(STASIS_MSG_L2, "Copying wheel packages\n"); - memset(rsync_cmd, 0, sizeof(rsync_cmd)); - sprintf(rsync_cmd, "rsync -avi --progress '%s' '%s'", ctx->storage.wheel_artifact_dir, dest); - if (system(rsync_cmd)) { - fprintf(stderr, "Failed to copy wheel artifacts to docker build directory\n"); - } - - if (docker_build(ctx->storage.build_docker_dir, args, ctx->deploy.docker.capabilities.build)) { - return -1; - } - - // Test the image - // All tags point back to the same image so test the first one we see - // regardless of how many are defined - strcpy(tag, strlist_item(ctx->deploy.docker.tags, 0)); - docker_sanitize_tag(tag); - - msg(STASIS_MSG_L2, "Executing image test script for %s\n", tag); - if (ctx->deploy.docker.test_script) { - if (isempty(ctx->deploy.docker.test_script)) { - msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "Image test script has no content\n"); - } else { - int state; - if ((state = docker_script(tag, ctx->deploy.docker.test_script, 0))) { - msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "Non-zero exit (%d) from test script. %s image archive will not be generated.\n", state >> 8, tag); - // test failed -- don't save the image - return -1; - } - } - } else { - msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "No image test script defined\n"); - } - - // Test successful, save image - if (docker_save(path_basename(tag), ctx->storage.docker_artifact_dir, ctx->deploy.docker.image_compression)) { - // save failed - return -1; - } - - return 0; -} - diff --git a/src/lib/core/delivery_init.c b/src/lib/core/delivery_init.c deleted file mode 100644 index 2fced03..0000000 --- a/src/lib/core/delivery_init.c +++ /dev/null @@ -1,346 +0,0 @@ -#include "delivery.h" - -int has_mount_flags(const char *mount_point, const unsigned long flags) { - struct statvfs st; - if (statvfs(mount_point, &st)) { - SYSERROR("Unable to determine mount-point flags: %s", strerror(errno)); - return -1; - } - return (st.f_flag & flags) != 0; -} - -int delivery_init_tmpdir(struct Delivery *ctx) { - char *tmpdir = NULL; - char *x = NULL; - int unusable = 0; - errno = 0; - - x = getenv("TMPDIR"); - if (x) { - guard_free(ctx->storage.tmpdir); - tmpdir = strdup(x); - } else { - tmpdir = ctx->storage.tmpdir; - } - - if (!tmpdir) { - // memory error - return -1; - } - - // If the directory doesn't exist, create it - if (access(tmpdir, F_OK) < 0) { - if (mkdirs(tmpdir, 0755) < 0) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Unable to create temporary storage directory: %s (%s)\n", tmpdir, strerror(errno)); - goto l_delivery_init_tmpdir_fatal; - } - } - - // If we can't read, write, or execute, then die - if (access(tmpdir, R_OK | W_OK | X_OK) < 0) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "%s requires at least 0755 permissions.\n"); - goto l_delivery_init_tmpdir_fatal; - } - - struct statvfs st; - if (statvfs(tmpdir, &st) < 0) { - goto l_delivery_init_tmpdir_fatal; - } - -#if defined(STASIS_OS_LINUX) - // If we can't execute programs, or write data to the file system at all, then die - if ((st.f_flag & ST_NOEXEC) != 0) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "%s is mounted with noexec\n", tmpdir); - goto l_delivery_init_tmpdir_fatal; - } -#endif - if ((st.f_flag & ST_RDONLY) != 0) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "%s is mounted read-only\n", tmpdir); - goto l_delivery_init_tmpdir_fatal; - } - - if (!globals.tmpdir) { - globals.tmpdir = strdup(tmpdir); - } - - if (!ctx->storage.tmpdir) { - ctx->storage.tmpdir = strdup(globals.tmpdir); - } - return unusable; - - l_delivery_init_tmpdir_fatal: - unusable = 1; - return unusable; -} - -void delivery_init_dirs_stage2(struct Delivery *ctx) { - path_store(&ctx->storage.build_recipes_dir, PATH_MAX, ctx->storage.build_dir, "recipes"); - path_store(&ctx->storage.build_sources_dir, PATH_MAX, ctx->storage.build_dir, "sources"); - path_store(&ctx->storage.build_testing_dir, PATH_MAX, ctx->storage.build_dir, "testing"); - path_store(&ctx->storage.build_docker_dir, PATH_MAX, ctx->storage.build_dir, "docker"); - - path_store(&ctx->storage.delivery_dir, PATH_MAX, ctx->storage.output_dir, "delivery"); - path_store(&ctx->storage.results_dir, PATH_MAX, ctx->storage.output_dir, "results"); - path_store(&ctx->storage.package_dir, PATH_MAX, ctx->storage.output_dir, "packages"); - path_store(&ctx->storage.cfgdump_dir, PATH_MAX, ctx->storage.output_dir, "config"); - path_store(&ctx->storage.meta_dir, PATH_MAX, ctx->storage.output_dir, "meta"); - - path_store(&ctx->storage.conda_artifact_dir, PATH_MAX, ctx->storage.package_dir, "conda"); - path_store(&ctx->storage.wheel_artifact_dir, PATH_MAX, ctx->storage.package_dir, "wheels"); - path_store(&ctx->storage.docker_artifact_dir, PATH_MAX, ctx->storage.package_dir, "docker"); -} - -void delivery_init_dirs_stage1(struct Delivery *ctx) { - char *rootdir = getenv("STASIS_ROOT"); - if (rootdir) { - if (isempty(rootdir)) { - fprintf(stderr, "STASIS_ROOT is set, but empty. Please assign a file system path to this environment variable.\n"); - exit(1); - } - path_store(&ctx->storage.root, PATH_MAX, rootdir, ctx->info.build_name); - } else { - // use "stasis" in current working directory - path_store(&ctx->storage.root, PATH_MAX, "stasis", ctx->info.build_name); - } - path_store(&ctx->storage.tools_dir, PATH_MAX, ctx->storage.root, "tools"); - path_store(&ctx->storage.tmpdir, PATH_MAX, ctx->storage.root, "tmp"); - if (delivery_init_tmpdir(ctx)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Set $TMPDIR to a location other than %s\n", globals.tmpdir); - if (globals.tmpdir) - guard_free(globals.tmpdir); - exit(1); - } - - path_store(&ctx->storage.build_dir, PATH_MAX, ctx->storage.root, "build"); - path_store(&ctx->storage.output_dir, PATH_MAX, ctx->storage.root, "output"); - - if (!ctx->storage.mission_dir) { - path_store(&ctx->storage.mission_dir, PATH_MAX, globals.sysconfdir, "mission"); - } - - if (access(ctx->storage.mission_dir, F_OK)) { - msg(STASIS_MSG_L1, "%s: %s\n", ctx->storage.mission_dir, strerror(errno)); - exit(1); - } - - // Override installation prefix using global configuration key - if (globals.conda_install_prefix && strlen(globals.conda_install_prefix)) { - // user wants a specific path - globals.conda_fresh_start = false; - /* - if (mkdirs(globals.conda_install_prefix, 0755)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Unable to create directory: %s: %s\n", - strerror(errno), globals.conda_install_prefix); - exit(1); - } - */ - /* - ctx->storage.conda_install_prefix = realpath(globals.conda_install_prefix, NULL); - if (!ctx->storage.conda_install_prefix) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "realpath(): Conda installation prefix reassignment failed\n"); - exit(1); - } - ctx->storage.conda_install_prefix = strdup(globals.conda_install_prefix); - */ - path_store(&ctx->storage.conda_install_prefix, PATH_MAX, globals.conda_install_prefix, "conda"); - } else { - // install conda under the STASIS tree - path_store(&ctx->storage.conda_install_prefix, PATH_MAX, ctx->storage.tools_dir, "conda"); - } -} - -int delivery_init_platform(struct Delivery *ctx) { - msg(STASIS_MSG_L2, "Setting architecture\n"); - char archsuffix[20]; - struct utsname uts; - if (uname(&uts)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "uname() failed: %s\n", strerror(errno)); - return -1; - } - - ctx->system.platform = calloc(DELIVERY_PLATFORM_MAX + 1, sizeof(*ctx->system.platform)); - if (!ctx->system.platform) { - SYSERROR("Unable to allocate %d records for platform array\n", DELIVERY_PLATFORM_MAX); - return -1; - } - for (size_t i = 0; i < DELIVERY_PLATFORM_MAX; i++) { - ctx->system.platform[i] = calloc(DELIVERY_PLATFORM_MAXLEN, sizeof(*ctx->system.platform[0])); - } - - ctx->system.arch = strdup(uts.machine); - if (!ctx->system.arch) { - // memory error - return -1; - } - - if (!strcmp(ctx->system.arch, "x86_64")) { - strcpy(archsuffix, "64"); - } else { - strcpy(archsuffix, ctx->system.arch); - } - - msg(STASIS_MSG_L2, "Setting platform\n"); - strcpy(ctx->system.platform[DELIVERY_PLATFORM], uts.sysname); - if (!strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Darwin")) { - sprintf(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], "osx-%s", archsuffix); - strcpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], "MacOSX"); - strcpy(ctx->system.platform[DELIVERY_PLATFORM_RELEASE], "macos"); - } else if (!strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Linux")) { - sprintf(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], "linux-%s", archsuffix); - strcpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], "Linux"); - strcpy(ctx->system.platform[DELIVERY_PLATFORM_RELEASE], "linux"); - } else { - // Not explicitly supported systems - strcpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], ctx->system.platform[DELIVERY_PLATFORM]); - strcpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], ctx->system.platform[DELIVERY_PLATFORM]); - strcpy(ctx->system.platform[DELIVERY_PLATFORM_RELEASE], ctx->system.platform[DELIVERY_PLATFORM]); - tolower_s(ctx->system.platform[DELIVERY_PLATFORM_RELEASE]); - } - - long cpu_count = get_cpu_count(); - if (!cpu_count) { - fprintf(stderr, "Unable to determine CPU count. Falling back to 1.\n"); - cpu_count = 1; - } - char ncpus[100] = {0}; - sprintf(ncpus, "%ld", cpu_count); - - // Declare some important bits as environment variables - setenv("CPU_COUNT", ncpus, 1); - setenv("STASIS_CPU_COUNT", ncpus, 1); - setenv("STASIS_ARCH", ctx->system.arch, 1); - setenv("STASIS_PLATFORM", ctx->system.platform[DELIVERY_PLATFORM], 1); - setenv("STASIS_CONDA_ARCH", ctx->system.arch, 1); - setenv("STASIS_CONDA_PLATFORM", ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], 1); - setenv("STASIS_CONDA_PLATFORM_SUBDIR", ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], 1); - - // Register template variables - // These were moved out of main() because we can't take the address of system.platform[x] - // _before_ the array has been initialized. - tpl_register("system.arch", &ctx->system.arch); - tpl_register("system.platform", &ctx->system.platform[DELIVERY_PLATFORM_RELEASE]); - - return 0; -} - -int delivery_init(struct Delivery *ctx, int render_mode) { - populate_info(ctx); - populate_delivery_cfg(ctx, INI_READ_RENDER); - - // Set artifactory URL via environment variable if possible - char *jfurl = getenv("STASIS_JF_ARTIFACTORY_URL"); - if (jfurl) { - if (globals.jfrog.url) { - guard_free(globals.jfrog.url); - } - globals.jfrog.url = strdup(jfurl); - } - - // Set artifactory repository via environment if possible - char *jfrepo = getenv("STASIS_JF_REPO"); - if (jfrepo) { - if (globals.jfrog.repo) { - guard_free(globals.jfrog.repo); - } - globals.jfrog.repo = strdup(jfrepo); - } - - // Configure architecture and platform information - delivery_init_platform(ctx); - - // Create STASIS directory structure - delivery_init_dirs_stage1(ctx); - - char config_local[PATH_MAX]; - sprintf(config_local, "%s/%s", ctx->storage.tmpdir, "config"); - setenv("XDG_CONFIG_HOME", config_local, 1); - - char cache_local[PATH_MAX]; - sprintf(cache_local, "%s/%s", ctx->storage.tmpdir, "cache"); - setenv("XDG_CACHE_HOME", cache_local, 1); - - // add tools to PATH - char pathvar_tmp[STASIS_BUFSIZ]; - sprintf(pathvar_tmp, "%s/bin:%s", ctx->storage.tools_dir, getenv("PATH")); - setenv("PATH", pathvar_tmp, 1); - - // Prevent git from paginating output - setenv("GIT_PAGER", "", 1); - - populate_delivery_ini(ctx, render_mode); - - if (ctx->deploy.docker.tags) { - for (size_t i = 0; i < strlist_count(ctx->deploy.docker.tags); i++) { - char *item = strlist_item(ctx->deploy.docker.tags, i); - tolower_s(item); - } - } - - if (ctx->deploy.docker.image_compression) { - if (docker_validate_compression_program(ctx->deploy.docker.image_compression)) { - SYSERROR("[deploy:docker].image_compression - invalid command / program is not installed: %s", ctx->deploy.docker.image_compression); - return -1; - } - } - return 0; -} - -int bootstrap_build_info(struct Delivery *ctx) { - struct Delivery local = {0}; - local._stasis_ini_fp.cfg = ini_open(ctx->_stasis_ini_fp.cfg_path); - local._stasis_ini_fp.delivery = ini_open(ctx->_stasis_ini_fp.delivery_path); - delivery_init_platform(&local); - populate_delivery_cfg(&local, INI_READ_RENDER); - populate_delivery_ini(&local, INI_READ_RENDER); - populate_info(&local); - ctx->info.build_name = strdup(local.info.build_name); - ctx->info.build_number = strdup(local.info.build_number); - ctx->info.release_name = strdup(local.info.release_name); - ctx->info.time_info = malloc(sizeof(*ctx->info.time_info)); - if (!ctx->info.time_info) { - SYSERROR("Unable to allocate %zu bytes for tm struct: %s", sizeof(*local.info.time_info), strerror(errno)); - return -1; - } - memcpy(ctx->info.time_info, local.info.time_info, sizeof(*local.info.time_info)); - ctx->info.time_now = local.info.time_now; - ctx->info.time_str_epoch = strdup(local.info.time_str_epoch); - delivery_free(&local); - return 0; -} - -int delivery_exists(struct Delivery *ctx) { - int release_exists = DELIVERY_NOT_FOUND; - char release_pattern[PATH_MAX] = {0}; - sprintf(release_pattern, "*%s*", ctx->info.release_name); - - if (globals.enable_artifactory) { - if (jfrt_auth_init(&ctx->deploy.jfrog_auth)) { - fprintf(stderr, "Failed to initialize Artifactory authentication context\n"); - return -1; // error - } - - struct JFRT_Search search = {.fail_no_op = true}; - // release_exists error states: - // `jf rt search --fail_no_op` returns 2 on failure - // otherwise, search returns an empty list "[]" and returns 0 - const int match = jfrog_cli_rt_search(&ctx->deploy.jfrog_auth, &search, globals.jfrog.repo, release_pattern); - if (!match) { - release_exists = DELIVERY_FOUND; - } - } else { - struct StrList *files = listdir(ctx->storage.delivery_dir); - const size_t files_count = strlist_count(files); - - for (size_t i = 0; i < files_count; i++) { - char *filename = strlist_item(files, i); - const int match = fnmatch(release_pattern, filename, FNM_PATHNAME); - if (match == 0) { - release_exists = DELIVERY_FOUND; - break; - } - } - guard_strlist_free(&files); - } - - return release_exists; -} diff --git a/src/lib/core/delivery_install.c b/src/lib/core/delivery_install.c deleted file mode 100644 index a348346..0000000 --- a/src/lib/core/delivery_install.c +++ /dev/null @@ -1,236 +0,0 @@ -#include "delivery.h" - -static struct Test *requirement_from_test(struct Delivery *ctx, const char *name) { - struct Test *result = NULL; - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { - char *package_name = strdup(name); - if (package_name) { - char *spec = find_version_spec(package_name); - if (spec) { - *spec = '\0'; - } - - if (ctx->tests[i].name && !strcmp(package_name, ctx->tests[i].name)) { - result = &ctx->tests[i]; - break; - } - guard_free(package_name); - } else { - SYSERROR("unable to allocate memory for package name: %s", name); - return NULL; - } - } - return result; -} - -static char *have_spec_in_config(const struct Delivery *ctx, const char *name) { - for (size_t x = 0; x < strlist_count(ctx->conda.pip_packages); x++) { - char *config_spec = strlist_item(ctx->conda.pip_packages, x); - char *op = find_version_spec(config_spec); - char package[255] = {0}; - if (op) { - strncpy(package, config_spec, op - config_spec); - } else { - strncpy(package, config_spec, sizeof(package) - 1); - } - if (strncmp(package, name, strlen(package)) == 0) { - return config_spec; - } - } - return NULL; -} - -int delivery_overlay_packages_from_env(struct Delivery *ctx, const char *env_name) { - char *current_env = conda_get_active_environment(); - int need_restore = current_env && strcmp(env_name, current_env) != 0; - - conda_activate(ctx->storage.conda_install_prefix, env_name); - // Retrieve a listing of python packages installed under "env_name" - int freeze_status = 0; - char *freeze_output = shell_output("python -m pip freeze", &freeze_status); - if (freeze_status) { - guard_free(freeze_output); - guard_free(current_env); - return -1; - } - - if (need_restore) { - // Restore the original conda environment - conda_activate(ctx->storage.conda_install_prefix, current_env); - } - guard_free(current_env); - - struct StrList *frozen_list = strlist_init(); - strlist_append_tokenize(frozen_list, freeze_output, LINE_SEP); - guard_free(freeze_output); - - struct StrList *new_list = strlist_init(); - - // - consume package specs that have no test blocks. - // - these will be third-party packages like numpy, scipy, etc. - // - and they need to be present at the head of the list so they - // get installed first. - for (size_t i = 0; i < strlist_count(ctx->conda.pip_packages); i++) { - char *spec = strlist_item(ctx->conda.pip_packages, i); - char spec_name[255] = {0}; - char *op = find_version_spec(spec); - if (op) { - strncpy(spec_name, spec, op - spec); - } else { - strncpy(spec_name, spec, sizeof(spec_name) - 1); - } - struct Test *test_block = requirement_from_test(ctx, spec_name); - if (!test_block) { - msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "from config without test: %s\n", spec); - strlist_append(&new_list, spec); - } - } - - // now consume packages that have a test block - // if the ini provides a spec, override the environment's version. - // otherwise, use the spec derived from the environment - for (size_t i = 0; i < strlist_count(frozen_list); i++) { - char *frozen_spec = strlist_item(frozen_list, i); - char frozen_name[255] = {0}; - char *op = find_version_spec(frozen_spec); - // we only care about packages with specs here. if something else arrives, ignore it - if (op) { - strncpy(frozen_name, frozen_spec, op - frozen_spec); - } else { - strncpy(frozen_name, frozen_spec, sizeof(frozen_name) - 1); - } - struct Test *test = requirement_from_test(ctx, frozen_name); - if (test && strcmp(test->name, frozen_name) == 0) { - char *config_spec = have_spec_in_config(ctx, frozen_name); - if (config_spec) { - msg(STASIS_MSG_L2, "from config: %s\n", config_spec); - strlist_append(&new_list, config_spec); - } else { - msg(STASIS_MSG_L2, "from environment: %s\n", frozen_spec); - strlist_append(&new_list, frozen_spec); - } - } - } - - // Replace the package manifest as needed - if (strlist_count(new_list)) { - guard_strlist_free(&ctx->conda.pip_packages); - ctx->conda.pip_packages = strlist_copy(new_list); - } - guard_strlist_free(&new_list); - guard_strlist_free(&frozen_list); - return 0; -} - -int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, char *env_name, int type, struct StrList **manifest) { - char cmd[PATH_MAX]; - char pkgs[STASIS_BUFSIZ]; - const char *env_current = getenv("CONDA_DEFAULT_ENV"); - - if (env_current) { - // The requested environment is not the current environment - if (strcmp(env_current, env_name) != 0) { - // Activate the requested environment - printf("Activating: %s\n", env_name); - conda_activate(conda_install_dir, env_name); - runtime_replace(&ctx->runtime.environ, __environ); - } - } - - memset(cmd, 0, sizeof(cmd)); - memset(pkgs, 0, sizeof(pkgs)); - strcat(cmd, "install"); - - typedef int (*Runner)(const char *); - Runner runner = NULL; - if (INSTALL_PKG_CONDA & type) { - runner = conda_exec; - } else if (INSTALL_PKG_PIP & type) { - runner = pip_exec; - } - - if (INSTALL_PKG_CONDA_DEFERRED & type) { - strcat(cmd, " --use-local"); - } else if (INSTALL_PKG_PIP_DEFERRED & type) { - // Don't change the baseline package set unless we're working with a - // new build. Release candidates will need to keep packages as stable - // as possible between releases. - if (!ctx->meta.based_on) { - strcat(cmd, " --upgrade"); - } - sprintf(cmd + strlen(cmd), " --extra-index-url 'file://%s'", ctx->storage.wheel_artifact_dir); - } - - for (size_t x = 0; manifest[x] != NULL; x++) { - char *name = NULL; - for (size_t p = 0; p < strlist_count(manifest[x]); p++) { - name = strlist_item(manifest[x], p); - strip(name); - if (!strlen(name)) { - continue; - } - if (INSTALL_PKG_PIP_DEFERRED & type) { - struct Test *info = requirement_from_test(ctx, name); - if (info) { - if (!strcmp(info->version, "HEAD")) { - struct StrList *tag_data = strlist_init(); - if (!tag_data) { - SYSERROR("%s", "Unable to allocate memory for tag data\n"); - return -1; - } - strlist_append_tokenize(tag_data, info->repository_info_tag, "-"); - - struct Wheel *whl = NULL; - char *post_commit = NULL; - char *hash = NULL; - if (strlist_count(tag_data) > 1) { - post_commit = strlist_item(tag_data, 1); - hash = strlist_item(tag_data, 2); - } - - // We can't match on version here (index 0). The wheel's version is not guaranteed to be - // equal to the tag; setuptools_scm auto-increments the value, the user can change it manually, - // etc. - errno = 0; - whl = get_wheel_info(ctx->storage.wheel_artifact_dir, info->name, - (char *[]) {ctx->meta.python_compact, ctx->system.arch, - "none", "any", - post_commit, hash, - NULL}, WHEEL_MATCH_ANY); - if (!whl && errno) { - // error - SYSERROR("Unable to read Python wheel info: %s\n", strerror(errno)); - exit(1); - } else if (!whl) { - // not found - fprintf(stderr, "No wheel packages found that match the description of '%s'", info->name); - } else { - // found - guard_strlist_free(&tag_data); - info->version = strdup(whl->version); - } - wheel_free(&whl); - } - snprintf(cmd + strlen(cmd), - sizeof(cmd) - strlen(cmd) - strlen(info->name) - strlen(info->version) + 5, - " '%s==%s'", info->name, info->version); - } else { - fprintf(stderr, "Deferred package '%s' is not present in the tested package list!\n", name); - return -1; - } - } else { - if (startswith(name, "--") || startswith(name, "-")) { - sprintf(cmd + strlen(cmd), " %s", name); - } else { - sprintf(cmd + strlen(cmd), " '%s'", name); - } - } - } - int status = runner(cmd); - if (status) { - return status; - } - } - return 0; -} - diff --git a/src/lib/core/delivery_populate.c b/src/lib/core/delivery_populate.c deleted file mode 100644 index c699545..0000000 --- a/src/lib/core/delivery_populate.c +++ /dev/null @@ -1,346 +0,0 @@ -#include "delivery.h" - -static void ini_has_key_required(struct INIFILE *ini, const char *section_name, char *key) { - int status = ini_has_key(ini, section_name, key); - if (!status) { - SYSERROR("%s:%s key is required but not defined", section_name, key); - exit(1); - } -} - -static void conv_str(char **x, union INIVal val) { - if (*x) { - guard_free(*x); - } - if (val.as_char_p) { - char *tplop = tpl_render(val.as_char_p); - if (tplop) { - *x = tplop; - } else { - *x = NULL; - } - } else { - *x = NULL; - } -} - - - -int populate_info(struct Delivery *ctx) { - if (!ctx->info.time_str_epoch) { - // Record timestamp used for release - time(&ctx->info.time_now); - ctx->info.time_info = localtime(&ctx->info.time_now); - - ctx->info.time_str_epoch = calloc(STASIS_TIME_STR_MAX, sizeof(*ctx->info.time_str_epoch)); - if (!ctx->info.time_str_epoch) { - msg(STASIS_MSG_ERROR, "Unable to allocate memory for Unix epoch string\n"); - return -1; - } - snprintf(ctx->info.time_str_epoch, STASIS_TIME_STR_MAX - 1, "%li", ctx->info.time_now); - } - return 0; -} - -int populate_delivery_cfg(struct Delivery *ctx, int render_mode) { - struct INIFILE *cfg = ctx->_stasis_ini_fp.cfg; - if (!cfg) { - return -1; - } - int err = 0; - ctx->storage.conda_staging_dir = ini_getval_str(cfg, "default", "conda_staging_dir", render_mode, &err); - ctx->storage.conda_staging_url = ini_getval_str(cfg, "default", "conda_staging_url", render_mode, &err); - ctx->storage.wheel_staging_dir = ini_getval_str(cfg, "default", "wheel_staging_dir", render_mode, &err); - ctx->storage.wheel_staging_url = ini_getval_str(cfg, "default", "wheel_staging_url", render_mode, &err); - globals.conda_fresh_start = ini_getval_bool(cfg, "default", "conda_fresh_start", render_mode, &err); - if (!globals.continue_on_error) { - globals.continue_on_error = ini_getval_bool(cfg, "default", "continue_on_error", render_mode, &err); - } - if (!globals.always_update_base_environment) { - globals.always_update_base_environment = ini_getval_bool(cfg, "default", "always_update_base_environment", render_mode, &err); - } - globals.conda_install_prefix = ini_getval_str(cfg, "default", "conda_install_prefix", render_mode, &err); - globals.conda_packages = ini_getval_strlist(cfg, "default", "conda_packages", LINE_SEP, render_mode, &err); - globals.pip_packages = ini_getval_strlist(cfg, "default", "pip_packages", LINE_SEP, render_mode, &err); - - globals.jfrog.jfrog_artifactory_base_url = ini_getval_str(cfg, "jfrog_cli_download", "url", render_mode, &err); - globals.jfrog.jfrog_artifactory_product = ini_getval_str(cfg, "jfrog_cli_download", "product", render_mode, &err); - globals.jfrog.cli_major_ver = ini_getval_str(cfg, "jfrog_cli_download", "version_series", render_mode, &err); - globals.jfrog.version = ini_getval_str(cfg, "jfrog_cli_download", "version", render_mode, &err); - globals.jfrog.remote_filename = ini_getval_str(cfg, "jfrog_cli_download", "filename", render_mode, &err); - globals.jfrog.url = ini_getval_str(cfg, "deploy:artifactory", "url", render_mode, &err); - globals.jfrog.repo = ini_getval_str(cfg, "deploy:artifactory", "repo", render_mode, &err); - - return 0; -} - -int populate_delivery_ini(struct Delivery *ctx, int render_mode) { - struct INIFILE *ini = ctx->_stasis_ini_fp.delivery; - struct INIData *rtdata; - - validate_delivery_ini(ini); - // Populate runtime variables first they may be interpreted by other - // keys in the configuration - RuntimeEnv *rt = runtime_copy(__environ); - while ((rtdata = ini_getall(ini, "runtime")) != NULL) { - char rec[STASIS_BUFSIZ]; - sprintf(rec, "%s=%s", lstrip(strip(rtdata->key)), lstrip(strip(rtdata->value))); - runtime_set(rt, rtdata->key, rtdata->value); - } - runtime_apply(rt); - ctx->runtime.environ = rt; - - int err = 0; - ctx->meta.mission = ini_getval_str(ini, "meta", "mission", render_mode, &err); - - if (!strcasecmp(ctx->meta.mission, "hst")) { - ctx->meta.codename = ini_getval_str(ini, "meta", "codename", render_mode, &err); - } else { - ctx->meta.codename = NULL; - } - - ctx->meta.version = ini_getval_str(ini, "meta", "version", render_mode, &err); - ctx->meta.name = ini_getval_str(ini, "meta", "name", render_mode, &err); - ctx->meta.rc = ini_getval_int(ini, "meta", "rc", render_mode, &err); - ctx->meta.final = ini_getval_bool(ini, "meta", "final", render_mode, &err); - ctx->meta.based_on = ini_getval_str(ini, "meta", "based_on", render_mode, &err); - - if (!ctx->meta.python) { - ctx->meta.python = ini_getval_str(ini, "meta", "python", render_mode, &err); - guard_free(ctx->meta.python_compact); - ctx->meta.python_compact = to_short_version(ctx->meta.python); - } else { - ini_setval(&ini, INI_SETVAL_REPLACE, "meta", "python", ctx->meta.python); - } - - ctx->conda.installer_name = ini_getval_str(ini, "conda", "installer_name", render_mode, &err); - ctx->conda.installer_version = ini_getval_str(ini, "conda", "installer_version", render_mode, &err); - ctx->conda.installer_platform = ini_getval_str(ini, "conda", "installer_platform", render_mode, &err); - ctx->conda.installer_arch = ini_getval_str(ini, "conda", "installer_arch", render_mode, &err); - ctx->conda.installer_baseurl = ini_getval_str(ini, "conda", "installer_baseurl", render_mode, &err); - ctx->conda.conda_packages = ini_getval_strlist(ini, "conda", "conda_packages", " "LINE_SEP, render_mode, &err); - - if (ctx->conda.conda_packages->data && ctx->conda.conda_packages->data[0] && strpbrk(ctx->conda.conda_packages->data[0], " \t")) { - normalize_space(ctx->conda.conda_packages->data[0]); - replace_text(ctx->conda.conda_packages->data[0], " ", LINE_SEP, 0); - char *pip_packages_replacement = join(ctx->conda.conda_packages->data, LINE_SEP); - ini_setval(&ini, INI_SETVAL_REPLACE, "conda", "conda_packages", pip_packages_replacement); - guard_free(pip_packages_replacement); - guard_strlist_free(&ctx->conda.conda_packages); - ctx->conda.conda_packages = ini_getval_strlist(ini, "conda", "conda_packages", LINE_SEP, render_mode, &err); - } - - for (size_t i = 0; i < strlist_count(ctx->conda.conda_packages); i++) { - char *pkg = strlist_item(ctx->conda.conda_packages, i); - if (strpbrk(pkg, ";#") || isempty(pkg)) { - strlist_remove(ctx->conda.conda_packages, i); - } - } - - ctx->conda.pip_packages = ini_getval_strlist(ini, "conda", "pip_packages", LINE_SEP, render_mode, &err); - if (ctx->conda.pip_packages->data && ctx->conda.pip_packages->data[0] && strpbrk(ctx->conda.pip_packages->data[0], " \t")) { - normalize_space(ctx->conda.pip_packages->data[0]); - replace_text(ctx->conda.pip_packages->data[0], " ", LINE_SEP, 0); - char *pip_packages_replacement = join(ctx->conda.pip_packages->data, LINE_SEP); - ini_setval(&ini, INI_SETVAL_REPLACE, "conda", "pip_packages", pip_packages_replacement); - guard_free(pip_packages_replacement); - guard_strlist_free(&ctx->conda.pip_packages); - ctx->conda.pip_packages = ini_getval_strlist(ini, "conda", "pip_packages", LINE_SEP, render_mode, &err); - } - - for (size_t i = 0; i < strlist_count(ctx->conda.pip_packages); i++) { - char *pkg = strlist_item(ctx->conda.pip_packages, i); - if (strpbrk(pkg, ";#") || isempty(pkg)) { - strlist_remove(ctx->conda.pip_packages, i); - } - } - - // Delivery metadata consumed - populate_mission_ini(&ctx, render_mode); - - if (ctx->info.release_name) { - guard_free(ctx->info.release_name); - guard_free(ctx->info.build_name); - guard_free(ctx->info.build_number); - } - - if (delivery_format_str(ctx, &ctx->info.release_name, ctx->rules.release_fmt)) { - fprintf(stderr, "Failed to generate release name. Format used: %s\n", ctx->rules.release_fmt); - return -1; - } - - if (!ctx->info.build_name) { - delivery_format_str(ctx, &ctx->info.build_name, ctx->rules.build_name_fmt); - } - if (!ctx->info.build_number) { - delivery_format_str(ctx, &ctx->info.build_number, ctx->rules.build_number_fmt); - } - - // Best I can do to make output directories unique. Annoying. - delivery_init_dirs_stage2(ctx); - - if (!ctx->conda.conda_packages_defer) { - ctx->conda.conda_packages_defer = strlist_init(); - } - if (!ctx->conda.pip_packages_defer) { - ctx->conda.pip_packages_defer = strlist_init(); - } - - for (size_t z = 0, i = 0; i < ini->section_count; i++) { - char *section_name = ini->section[i]->key; - if (startswith(section_name, "test:")) { - union INIVal val; - struct Test *test = &ctx->tests[z]; - val.as_char_p = strchr(ini->section[i]->key, ':') + 1; - if (val.as_char_p && isempty(val.as_char_p)) { - return 1; - } - conv_str(&test->name, val); - - test->version = ini_getval_str(ini, section_name, "version", render_mode, &err); - test->repository = ini_getval_str(ini, section_name, "repository", render_mode, &err); - test->script_setup = ini_getval_str(ini, section_name, "script_setup", INI_READ_RAW, &err); - test->script = ini_getval_str(ini, section_name, "script", INI_READ_RAW, &err); - test->disable = ini_getval_bool(ini, section_name, "disable", render_mode, &err); - test->parallel = ini_getval_bool(ini, section_name, "parallel", render_mode, &err); - if (err) { - test->parallel = true; - } - test->repository_remove_tags = ini_getval_strlist(ini, section_name, "repository_remove_tags", LINE_SEP, render_mode, &err); - test->build_recipe = ini_getval_str(ini, section_name, "build_recipe", render_mode, &err); - test->runtime.environ = ini_getval_strlist(ini, section_name, "runtime", LINE_SEP, render_mode, &err); - z++; - } - } - - for (size_t z = 0, i = 0; i < ini->section_count; i++) { - char *section_name = ini->section[i]->key; - struct Deploy *deploy = &ctx->deploy; - if (startswith(section_name, "deploy:artifactory")) { - struct JFrog *jfrog = &deploy->jfrog[z]; - // Artifactory base configuration - - jfrog->upload_ctx.workaround_parent_only = ini_getval_bool(ini, section_name, "workaround_parent_only", render_mode, &err); - jfrog->upload_ctx.exclusions = ini_getval_str(ini, section_name, "exclusions", render_mode, &err); - jfrog->upload_ctx.explode = ini_getval_bool(ini, section_name, "explode", render_mode, &err); - jfrog->upload_ctx.recursive = ini_getval_bool(ini, section_name, "recursive", render_mode, &err); - jfrog->upload_ctx.retries = ini_getval_int(ini, section_name, "retries", render_mode, &err); - jfrog->upload_ctx.retry_wait_time = ini_getval_int(ini, section_name, "retry_wait_time", render_mode, &err); - jfrog->upload_ctx.detailed_summary = ini_getval_bool(ini, section_name, "detailed_summary", render_mode, &err); - jfrog->upload_ctx.quiet = ini_getval_bool(ini, section_name, "quiet", render_mode, &err); - jfrog->upload_ctx.regexp = ini_getval_bool(ini, section_name, "regexp", render_mode, &err); - jfrog->upload_ctx.spec = ini_getval_str(ini, section_name, "spec", render_mode, &err); - jfrog->upload_ctx.flat = ini_getval_bool(ini, section_name, "flat", render_mode, &err); - jfrog->repo = ini_getval_str(ini, section_name, "repo", render_mode, &err); - jfrog->dest = ini_getval_str(ini, section_name, "dest", render_mode, &err); - jfrog->files = ini_getval_strlist(ini, section_name, "files", LINE_SEP, render_mode, &err); - z++; - } - } - - for (size_t i = 0; i < ini->section_count; i++) { - char *section_name = ini->section[i]->key; - struct Deploy *deploy = &ctx->deploy; - if (startswith(ini->section[i]->key, "deploy:docker")) { - struct Docker *docker = &deploy->docker; - - docker->registry = ini_getval_str(ini, section_name, "registry", render_mode, &err); - docker->image_compression = ini_getval_str(ini, section_name, "image_compression", render_mode, &err); - docker->test_script = ini_getval_str(ini, section_name, "test_script", render_mode, &err); - docker->build_args = ini_getval_strlist(ini, section_name, "build_args", LINE_SEP, render_mode, &err); - docker->tags = ini_getval_strlist(ini, section_name, "tags", LINE_SEP, render_mode, &err); - } - } - return 0; -} - -int populate_mission_ini(struct Delivery **ctx, int render_mode) { - int err = 0; - - if ((*ctx)->_stasis_ini_fp.mission) { - return 0; - } - - // Now populate the rules - char missionfile[PATH_MAX] = {0}; - if (getenv("STASIS_SYSCONFDIR")) { - sprintf(missionfile, "%s/%s/%s/%s.ini", - getenv("STASIS_SYSCONFDIR"), "mission", (*ctx)->meta.mission, (*ctx)->meta.mission); - } else { - sprintf(missionfile, "%s/%s/%s/%s.ini", - globals.sysconfdir, "mission", (*ctx)->meta.mission, (*ctx)->meta.mission); - } - - msg(STASIS_MSG_L2, "Reading mission configuration: %s\n", missionfile); - (*ctx)->_stasis_ini_fp.mission = ini_open(missionfile); - struct INIFILE *ini = (*ctx)->_stasis_ini_fp.mission; - if (!ini) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read mission configuration: %s, %s\n", missionfile, strerror(errno)); - exit(1); - } - (*ctx)->_stasis_ini_fp.mission_path = strdup(missionfile); - - (*ctx)->rules.release_fmt = ini_getval_str(ini, "meta", "release_fmt", render_mode, &err); - - // Used for setting artifactory build info - (*ctx)->rules.build_name_fmt = ini_getval_str(ini, "meta", "build_name_fmt", render_mode, &err); - - // Used for setting artifactory build info - (*ctx)->rules.build_number_fmt = ini_getval_str(ini, "meta", "build_number_fmt", render_mode, &err); - return 0; -} - -void validate_delivery_ini(struct INIFILE *ini) { - if (!ini) { - SYSERROR("%s", "INIFILE is NULL!"); - exit(1); - } - if (ini_section_search(&ini, INI_SEARCH_EXACT, "meta")) { - ini_has_key_required(ini, "meta", "name"); - ini_has_key_required(ini, "meta", "version"); - ini_has_key_required(ini, "meta", "rc"); - ini_has_key_required(ini, "meta", "mission"); - ini_has_key_required(ini, "meta", "python"); - } else { - SYSERROR("%s", "[meta] configuration section is required"); - exit(1); - } - - if (ini_section_search(&ini, INI_SEARCH_EXACT, "conda")) { - ini_has_key_required(ini, "conda", "installer_name"); - ini_has_key_required(ini, "conda", "installer_version"); - ini_has_key_required(ini, "conda", "installer_platform"); - ini_has_key_required(ini, "conda", "installer_arch"); - } else { - SYSERROR("%s", "[conda] configuration section is required"); - exit(1); - } - - for (size_t i = 0; i < ini->section_count; i++) { - struct INISection *section = ini->section[i]; - if (section && startswith(section->key, "test:")) { - char *name = strstr(section->key, ":"); - if (name && strlen(name) > 1) { - name = &name[1]; - } - //ini_has_key_required(ini, section->key, "version"); - //ini_has_key_required(ini, section->key, "repository"); - if (globals.enable_testing) { - ini_has_key_required(ini, section->key, "script"); - } - } - } - - if (ini_section_search(&ini, INI_SEARCH_EXACT, "deploy:docker")) { - // yeah? - } - - for (size_t i = 0; i < ini->section_count; i++) { - struct INISection *section = ini->section[i]; - if (section && startswith(section->key, "deploy:artifactory")) { - ini_has_key_required(ini, section->key, "files"); - ini_has_key_required(ini, section->key, "dest"); - } - } -} - diff --git a/src/lib/core/delivery_postprocess.c b/src/lib/core/delivery_postprocess.c deleted file mode 100644 index 40ac43f..0000000 --- a/src/lib/core/delivery_postprocess.c +++ /dev/null @@ -1,258 +0,0 @@ -#include "delivery.h" - - -const char *release_header = "# delivery_name: %s\n" - "# delivery_fmt: %s\n" - "# creation_time: %s\n" - "# conda_ident: %s\n" - "# conda_build_ident: %s\n"; - -char *delivery_get_release_header(struct Delivery *ctx) { - char output[STASIS_BUFSIZ]; - char stamp[100]; - strftime(stamp, sizeof(stamp) - 1, "%c", ctx->info.time_info); - sprintf(output, release_header, - ctx->info.release_name, - ctx->rules.release_fmt, - stamp, - ctx->conda.tool_version, - ctx->conda.tool_build_version); - return strdup(output); -} - -int delivery_dump_metadata(struct Delivery *ctx) { - char filename[PATH_MAX]; - sprintf(filename, "%s/meta-%s.stasis", ctx->storage.meta_dir, ctx->info.release_name); - FILE *fp = fopen(filename, "w+"); - if (!fp) { - return -1; - } - if (globals.verbose) { - printf("%s\n", filename); - } - fprintf(fp, "name %s\n", ctx->meta.name); - fprintf(fp, "version %s\n", ctx->meta.version); - fprintf(fp, "rc %d\n", ctx->meta.rc); - fprintf(fp, "python %s\n", ctx->meta.python); - fprintf(fp, "python_compact %s\n", ctx->meta.python_compact); - fprintf(fp, "mission %s\n", ctx->meta.mission); - fprintf(fp, "codename %s\n", ctx->meta.codename ? ctx->meta.codename : ""); - fprintf(fp, "platform %s %s %s %s\n", - ctx->system.platform[DELIVERY_PLATFORM], - ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], - ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], - ctx->system.platform[DELIVERY_PLATFORM_RELEASE]); - fprintf(fp, "arch %s\n", ctx->system.arch); - fprintf(fp, "time %s\n", ctx->info.time_str_epoch); - fprintf(fp, "release_fmt %s\n", ctx->rules.release_fmt); - fprintf(fp, "release_name %s\n", ctx->info.release_name); - fprintf(fp, "build_name_fmt %s\n", ctx->rules.build_name_fmt); - fprintf(fp, "build_name %s\n", ctx->info.build_name); - fprintf(fp, "build_number_fmt %s\n", ctx->rules.build_number_fmt); - fprintf(fp, "build_number %s\n", ctx->info.build_number); - fprintf(fp, "conda_installer_baseurl %s\n", ctx->conda.installer_baseurl); - fprintf(fp, "conda_installer_name %s\n", ctx->conda.installer_name); - fprintf(fp, "conda_installer_version %s\n", ctx->conda.installer_version); - fprintf(fp, "conda_installer_platform %s\n", ctx->conda.installer_platform); - fprintf(fp, "conda_installer_arch %s\n", ctx->conda.installer_arch); - - fclose(fp); - return 0; -} - -void delivery_rewrite_spec(struct Delivery *ctx, char *filename, unsigned stage) { - char *header = NULL; - char *tempfile = NULL; - FILE *tp = NULL; - - if (stage == DELIVERY_REWRITE_SPEC_STAGE_1) { - header = delivery_get_release_header(ctx); - if (!header) { - msg(STASIS_MSG_ERROR, "failed to generate release header string\n", filename); - exit(1); - } - tempfile = xmkstemp(&tp, "w+"); - if (!tempfile || !tp) { - msg(STASIS_MSG_ERROR, "%s: unable to create temporary file\n", strerror(errno)); - exit(1); - } - fprintf(tp, "%s", header); - - // Read the original file - char **contents = file_readlines(filename, 0, 0, NULL); - if (!contents) { - msg(STASIS_MSG_ERROR, "%s: unable to read %s", filename); - exit(1); - } - - // Write temporary data - for (size_t i = 0; contents[i] != NULL; i++) { - if (startswith(contents[i], "channels:")) { - // Allow for additional conda channel injection - if (ctx->conda.conda_packages_defer && strlist_count(ctx->conda.conda_packages_defer)) { - fprintf(tp, "%s - @CONDA_CHANNEL@\n", contents[i]); - continue; - } - } else if (strstr(contents[i], "- pip:")) { - if (ctx->conda.pip_packages_defer && strlist_count(ctx->conda.pip_packages_defer)) { - // Allow for additional pip argument injection - fprintf(tp, "%s - @PIP_ARGUMENTS@\n", contents[i]); - continue; - } - } else if (startswith(contents[i], "prefix:")) { - // Remove the prefix key - if (strstr(contents[i], "/") || strstr(contents[i], "\\")) { - // path is on the same line as the key - continue; - } else { - // path is on the next line? - if (contents[i + 1] && (strstr(contents[i + 1], "/") || strstr(contents[i + 1], "\\"))) { - i++; - } - continue; - } - } - fprintf(tp, "%s", contents[i]); - } - GENERIC_ARRAY_FREE(contents); - guard_free(header); - fflush(tp); - fclose(tp); - - // Replace the original file with our temporary data - if (copy2(tempfile, filename, CT_PERM) < 0) { - fprintf(stderr, "%s: could not rename '%s' to '%s'\n", strerror(errno), tempfile, filename); - exit(1); - } - remove(tempfile); - guard_free(tempfile); - } else if (globals.enable_rewrite_spec_stage_2 && stage == DELIVERY_REWRITE_SPEC_STAGE_2) { - char output[PATH_MAX] = {0}; - // Replace "local" channel with the staging URL - if (ctx->storage.conda_staging_url) { - file_replace_text(filename, "@CONDA_CHANNEL@", ctx->storage.conda_staging_url, 0); - } else if (globals.jfrog.repo) { - sprintf(output, "%s/%s/%s/%s/packages/conda", globals.jfrog.url, globals.jfrog.repo, ctx->meta.mission, ctx->info.build_name); - file_replace_text(filename, "@CONDA_CHANNEL@", output, 0); - } else { - msg(STASIS_MSG_WARN, "conda_staging_dir is not configured. Using fallback: '%s'\n", ctx->storage.conda_artifact_dir); - file_replace_text(filename, "@CONDA_CHANNEL@", ctx->storage.conda_artifact_dir, 0); - } - - if (ctx->storage.wheel_staging_url) { - file_replace_text(filename, "@PIP_ARGUMENTS@", ctx->storage.wheel_staging_url, 0); - } else if (globals.enable_artifactory && globals.jfrog.url && globals.jfrog.repo) { - sprintf(output, "--extra-index-url %s/%s/%s/%s/packages/wheels", globals.jfrog.url, globals.jfrog.repo, ctx->meta.mission, ctx->info.build_name); - file_replace_text(filename, "@PIP_ARGUMENTS@", output, 0); - } else { - msg(STASIS_MSG_WARN, "wheel_staging_dir is not configured. Using fallback: '%s'\n", ctx->storage.wheel_artifact_dir); - sprintf(output, "--extra-index-url file://%s", ctx->storage.wheel_artifact_dir); - file_replace_text(filename, "@PIP_ARGUMENTS@", output, 0); - } - } -} - -int delivery_copy_conda_artifacts(struct Delivery *ctx) { - char cmd[STASIS_BUFSIZ]; - char conda_build_dir[PATH_MAX]; - char subdir[PATH_MAX]; - memset(cmd, 0, sizeof(cmd)); - memset(conda_build_dir, 0, sizeof(conda_build_dir)); - memset(subdir, 0, sizeof(subdir)); - - sprintf(conda_build_dir, "%s/%s", ctx->storage.conda_install_prefix, "conda-bld"); - // One must run conda build at least once to create the "conda-bld" directory. - // When this directory is missing there can be no build artifacts. - if (access(conda_build_dir, F_OK) < 0) { - msg(STASIS_MSG_RESTRICT | STASIS_MSG_WARN | STASIS_MSG_L3, - "Skipped: 'conda build' has never been executed.\n"); - return 0; - } - - snprintf(cmd, sizeof(cmd) - 1, "rsync -avi --progress %s/%s %s", - conda_build_dir, - ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], - ctx->storage.conda_artifact_dir); - - return system(cmd); -} - -int delivery_index_conda_artifacts(struct Delivery *ctx) { - return conda_index(ctx->storage.conda_artifact_dir); -} - -int delivery_copy_wheel_artifacts(struct Delivery *ctx) { - char cmd[PATH_MAX] = {0}; - snprintf(cmd, sizeof(cmd) - 1, "rsync -avi --progress %s/*/dist/*.whl %s", - ctx->storage.build_sources_dir, - ctx->storage.wheel_artifact_dir); - return system(cmd); -} - -int delivery_index_wheel_artifacts(struct Delivery *ctx) { - struct dirent *rec; - - DIR *dp = opendir(ctx->storage.wheel_artifact_dir); - if (!dp) { - return -1; - } - - // Generate a "dumb" local pypi index that is compatible with: - // pip install --extra-index-url - char top_index[PATH_MAX] = {0}; - sprintf(top_index, "%s/index.html", ctx->storage.wheel_artifact_dir); - FILE *top_fp = fopen(top_index, "w+"); - if (!top_fp) { - closedir(dp); - return -2; - } - - while ((rec = readdir(dp)) != NULL) { - // skip directories - if (DT_REG == rec->d_type || !strcmp(rec->d_name, "..") || !strcmp(rec->d_name, ".")) { - continue; - } - - char bottom_index[PATH_MAX * 2] = {0}; - sprintf(bottom_index, "%s/%s/index.html", ctx->storage.wheel_artifact_dir, rec->d_name); - FILE *bottom_fp = fopen(bottom_index, "w+"); - if (!bottom_fp) { - closedir(dp); - return -3; - } - - if (globals.verbose) { - printf("+ %s\n", rec->d_name); - } - // Add record to top level index - fprintf(top_fp, "<a href=\"%s/\">%s</a><br/>\n", rec->d_name, rec->d_name); - - char dpath[PATH_MAX * 2] = {0}; - sprintf(dpath, "%s/%s", ctx->storage.wheel_artifact_dir, rec->d_name); - struct StrList *packages = listdir(dpath); - if (!packages) { - closedir(dp); - fclose(top_fp); - fclose(bottom_fp); - return -4; - } - - for (size_t i = 0; i < strlist_count(packages); i++) { - char *package = strlist_item(packages, i); - if (!endswith(package, ".whl")) { - continue; - } - if (globals.verbose) { - printf("`- %s\n", package); - } - // Write record to bottom level index - fprintf(bottom_fp, "<a href=\"%s\">%s</a><br/>\n", package, package); - } - fclose(bottom_fp); - - guard_strlist_free(&packages); - } - closedir(dp); - fclose(top_fp); - return 0; -} diff --git a/src/lib/core/delivery_show.c b/src/lib/core/delivery_show.c deleted file mode 100644 index adfa1be..0000000 --- a/src/lib/core/delivery_show.c +++ /dev/null @@ -1,117 +0,0 @@ -#include "delivery.h" - -void delivery_debug_show(struct Delivery *ctx) { - printf("\n====DEBUG====\n"); - printf("%-20s %-10s\n", "System configuration directory:", globals.sysconfdir); - printf("%-20s %-10s\n", "Mission directory:", ctx->storage.mission_dir); - printf("%-20s %-10s\n", "Testing enabled:", globals.enable_testing ? "Yes" : "No"); - printf("%-20s %-10s\n", "Docker image builds enabled:", globals.enable_docker ? "Yes" : "No"); - printf("%-20s %-10s\n", "Artifact uploading enabled:", globals.enable_artifactory ? "Yes" : "No"); -} - -void delivery_meta_show(struct Delivery *ctx) { - if (globals.verbose) { - delivery_debug_show(ctx); - } - - printf("\n====DELIVERY====\n"); - printf("%-20s %-10s\n", "Target Python:", ctx->meta.python); - printf("%-20s %-10s\n", "Name:", ctx->meta.name); - printf("%-20s %-10s\n", "Mission:", ctx->meta.mission); - if (ctx->meta.codename) { - printf("%-20s %-10s\n", "Codename:", ctx->meta.codename); - } - if (ctx->meta.version) { - printf("%-20s %-10s\n", "Version", ctx->meta.version); - } - if (!ctx->meta.final) { - printf("%-20s %-10d\n", "RC Level:", ctx->meta.rc); - } - printf("%-20s %-10s\n", "Final Release:", ctx->meta.final ? "Yes" : "No"); - printf("%-20s %-10s\n", "Based On:", ctx->meta.based_on ? ctx->meta.based_on : "New"); -} - -void delivery_conda_show(struct Delivery *ctx) { - printf("\n====CONDA====\n"); - printf("%-20s %-10s\n", "Prefix:", ctx->storage.conda_install_prefix); - - puts("Native Packages:"); - if (strlist_count(ctx->conda.conda_packages) || strlist_count(ctx->conda.conda_packages_defer)) { - struct StrList *list_conda = strlist_init(); - if (strlist_count(ctx->conda.conda_packages)) { - strlist_append_strlist(list_conda, ctx->conda.conda_packages); - } - if (strlist_count(ctx->conda.conda_packages_defer)) { - strlist_append_strlist(list_conda, ctx->conda.conda_packages_defer); - } - strlist_sort(list_conda, STASIS_SORT_ALPHA); - - for (size_t i = 0; i < strlist_count(list_conda); i++) { - char *token = strlist_item(list_conda, i); - if (isempty(token) || isblank(*token) || startswith(token, "-")) { - continue; - } - printf("%21s%s\n", "", token); - } - guard_strlist_free(&list_conda); - } else { - printf("%21s%s\n", "", "N/A"); - } - - puts("Python Packages:"); - if (strlist_count(ctx->conda.pip_packages) || strlist_count(ctx->conda.pip_packages_defer)) { - struct StrList *list_python = strlist_init(); - if (strlist_count(ctx->conda.pip_packages)) { - strlist_append_strlist(list_python, ctx->conda.pip_packages); - } - if (strlist_count(ctx->conda.pip_packages_defer)) { - strlist_append_strlist(list_python, ctx->conda.pip_packages_defer); - } - strlist_sort(list_python, STASIS_SORT_ALPHA); - - for (size_t i = 0; i < strlist_count(list_python); i++) { - char *token = strlist_item(list_python, i); - if (isempty(token) || isblank(*token) || startswith(token, "-")) { - continue; - } - printf("%21s%s\n", "", token); - } - guard_strlist_free(&list_python); - } else { - printf("%21s%s\n", "", "N/A"); - } -} - -void delivery_tests_show(struct Delivery *ctx) { - printf("\n====TESTS====\n"); - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { - if (!ctx->tests[i].name) { - continue; - } - printf("%-20s %-20s %s\n", ctx->tests[i].name, - ctx->tests[i].version, - ctx->tests[i].repository); - } -} - -void delivery_runtime_show(struct Delivery *ctx) { - printf("\n====RUNTIME====\n"); - struct StrList *rt = NULL; - rt = strlist_copy(ctx->runtime.environ); - if (!rt) { - // no data - return; - } - strlist_sort(rt, STASIS_SORT_ALPHA); - size_t total = strlist_count(rt); - for (size_t i = 0; i < total; i++) { - char *item = strlist_item(rt, i); - if (!item) { - // not supposed to occur - msg(STASIS_MSG_WARN | STASIS_MSG_L1, "Encountered unexpected NULL at record %zu of %zu of runtime array.\n", i); - return; - } - printf("%s\n", item); - } -} - diff --git a/src/lib/core/delivery_test.c b/src/lib/core/delivery_test.c deleted file mode 100644 index e80e0ec..0000000 --- a/src/lib/core/delivery_test.c +++ /dev/null @@ -1,295 +0,0 @@ -#include "delivery.h" - -void delivery_tests_run(struct Delivery *ctx) { - static const int SETUP = 0; - static const int PARALLEL = 1; - static const int SERIAL = 2; - struct MultiProcessingPool *pool[3]; - struct Process proc = {0}; - - if (!globals.workaround.conda_reactivate) { - globals.workaround.conda_reactivate = calloc(PATH_MAX, sizeof(*globals.workaround.conda_reactivate)); - } else { - memset(globals.workaround.conda_reactivate, 0, PATH_MAX); - } - // Test blocks always run with xtrace enabled. Disable, and reenable it. Conda's wrappers produce an incredible - // amount of debug information. - snprintf(globals.workaround.conda_reactivate, PATH_MAX - 1, "\nset +x; mamba activate ${CONDA_DEFAULT_ENV}; set -x\n"); - - if (!ctx->tests[0].name) { - msg(STASIS_MSG_WARN | STASIS_MSG_L2, "no tests are defined!\n"); - } else { - pool[PARALLEL] = mp_pool_init("parallel", ctx->storage.tmpdir); - if (!pool[PARALLEL]) { - perror("mp_pool_init/parallel"); - exit(1); - } - pool[PARALLEL]->status_interval = globals.pool_status_interval; - - pool[SERIAL] = mp_pool_init("serial", ctx->storage.tmpdir); - if (!pool[SERIAL]) { - perror("mp_pool_init/serial"); - exit(1); - } - pool[SERIAL]->status_interval = globals.pool_status_interval; - - pool[SETUP] = mp_pool_init("setup", ctx->storage.tmpdir); - if (!pool[SETUP]) { - perror("mp_pool_init/setup"); - exit(1); - } - pool[SETUP]->status_interval = globals.pool_status_interval; - - // Test block scripts shall exit non-zero on error. - // This will fail a test block immediately if "string" is not found in file.txt: - // grep string file.txt - // - // And this is how to avoid that scenario: - // #1: - // if ! grep string file.txt; then - // # handle error - // fi - // - // #2: - // grep string file.txt || handle error - // - // #3: - // # Use ':' as a NO-OP if/when the result doesn't matter - // grep string file.txt || : - const char *runner_cmd_fmt = "set -e -x\n%s\n"; - - // Iterate over our test records, retrieving the source code for each package, and assigning its scripted tasks - // to the appropriate processing pool - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { - struct Test *test = &ctx->tests[i]; - if (!test->name && !test->repository && !test->script) { - // skip unused test records - continue; - } - msg(STASIS_MSG_L2, "Loading tests for %s %s\n", test->name, test->version); - if (!test->script || !strlen(test->script)) { - msg(STASIS_MSG_WARN | STASIS_MSG_L3, "Nothing to do. To fix, declare a 'script' in section: [test:%s]\n", - test->name); - continue; - } - - char destdir[PATH_MAX]; - sprintf(destdir, "%s/%s", ctx->storage.build_sources_dir, path_basename(test->repository)); - - if (!access(destdir, F_OK)) { - msg(STASIS_MSG_L3, "Purging repository %s\n", destdir); - if (rmtree(destdir)) { - COE_CHECK_ABORT(1, "Unable to remove repository\n"); - } - } - msg(STASIS_MSG_L3, "Cloning repository %s\n", test->repository); - if (!git_clone(&proc, test->repository, destdir, test->version)) { - test->repository_info_tag = strdup(git_describe(destdir)); - test->repository_info_ref = strdup(git_rev_parse(destdir, "HEAD")); - } else { - COE_CHECK_ABORT(1, "Unable to clone repository\n"); - } - - if (test->repository_remove_tags && strlist_count(test->repository_remove_tags)) { - filter_repo_tags(destdir, test->repository_remove_tags); - } - - if (pushd(destdir)) { - COE_CHECK_ABORT(1, "Unable to enter repository directory\n"); - } else { - char *cmd = calloc(strlen(test->script) + STASIS_BUFSIZ, sizeof(*cmd)); - if (!cmd) { - SYSERROR("Unable to allocate test script buffer: %s", strerror(errno)); - exit(1); - } - - msg(STASIS_MSG_L3, "Queuing task for %s\n", test->name); - memset(&proc, 0, sizeof(proc)); - - strcpy(cmd, test->script); - char *cmd_rendered = tpl_render(cmd); - if (cmd_rendered) { - if (strcmp(cmd_rendered, cmd) != 0) { - strcpy(cmd, cmd_rendered); - cmd[strlen(cmd_rendered) ? strlen(cmd_rendered) - 1 : 0] = 0; - } - guard_free(cmd_rendered); - } else { - SYSERROR("An error occurred while rendering the following:\n%s", cmd); - exit(1); - } - // Move indents - // HEREDOCs will not work otherwise - unindent(cmd); - - if (test->disable) { - msg(STASIS_MSG_L2, "Script execution disabled by configuration\n", test->name); - guard_free(cmd); - continue; - } - - char *runner_cmd = NULL; - char pool_name[100] = "parallel"; - struct MultiProcessingTask *task = NULL; - int selected = PARALLEL; - if (!globals.enable_parallel || !test->parallel) { - selected = SERIAL; - memset(pool_name, 0, sizeof(pool_name)); - strcpy(pool_name, "serial"); - } - - if (asprintf(&runner_cmd, runner_cmd_fmt, cmd) < 0) { - SYSERROR("Unable to allocate memory for runner command: %s", strerror(errno)); - exit(1); - } - task = mp_pool_task(pool[selected], test->name, destdir, runner_cmd); - if (!task) { - SYSERROR("Failed to add task to %s pool: %s", pool_name, runner_cmd); - popd(); - if (!globals.continue_on_error) { - guard_free(runner_cmd); - tpl_free(); - delivery_free(ctx); - globals_free(); - } - exit(1); - } - guard_free(runner_cmd); - guard_free(cmd); - popd(); - - } - } - - // Configure "script_setup" tasks - // Directories should exist now, so no need to go through initializing everything all over again. - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { - struct Test *test = &ctx->tests[i]; - if (test->script_setup) { - char destdir[PATH_MAX]; - sprintf(destdir, "%s/%s", ctx->storage.build_sources_dir, path_basename(test->repository)); - if (access(destdir, F_OK)) { - SYSERROR("%s: %s", destdir, strerror(errno)); - exit(1); - } - if (!pushd(destdir)) { - const size_t cmd_len = strlen(test->script_setup) + STASIS_BUFSIZ; - char *cmd = calloc(cmd_len, sizeof(*cmd)); - if (!cmd) { - SYSERROR("Unable to allocate test script_setup buffer: %s", strerror(errno)); - exit(1); - } - - strncpy(cmd, test->script_setup, cmd_len - 1); - char *cmd_rendered = tpl_render(cmd); - if (cmd_rendered) { - if (strcmp(cmd_rendered, cmd) != 0) { - strncpy(cmd, cmd_rendered, cmd_len - 1); - cmd[strlen(cmd_rendered) ? strlen(cmd_rendered) - 1 : 0] = 0; - } - guard_free(cmd_rendered); - } else { - SYSERROR("An error occurred while rendering the following:\n%s", cmd); - exit(1); - } - unindent(cmd); - - struct MultiProcessingTask *task = NULL; - char *runner_cmd = NULL; - if (asprintf(&runner_cmd, runner_cmd_fmt, cmd) < 0) { - SYSERROR("Unable to allocate memory for runner command: %s", strerror(errno)); - exit(1); - } - - task = mp_pool_task(pool[SETUP], test->name, destdir, runner_cmd); - if (!task) { - SYSERROR("Failed to add task %s to setup pool: %s", test->name, runner_cmd); - popd(); - if (!globals.continue_on_error) { - guard_free(runner_cmd); - tpl_free(); - delivery_free(ctx); - globals_free(); - } - exit(1); - } - guard_free(runner_cmd); - guard_free(cmd); - popd(); - } else { - SYSERROR("Failed to change directory: %s\n", destdir); - exit(1); - } - } - } - - size_t opt_flags = 0; - if (globals.parallel_fail_fast) { - opt_flags |= MP_POOL_FAIL_FAST; - } - - // Execute all queued tasks - for (size_t p = 0; p < sizeof(pool) / sizeof(*pool); p++) { - long jobs = globals.cpu_limit; - - if (!pool[p]->num_used) { - // Skip empty pool - continue; - } - - // Setup tasks run sequentially - if (p == (size_t) SETUP || p == (size_t) SERIAL) { - jobs = 1; - } - - // Run tasks in the pool - // 1. Setup (builds) - // 2. Parallel (fast jobs) - // 3. Serial (long jobs) - int pool_status = mp_pool_join(pool[p], jobs, opt_flags); - - // On error show a summary of the current pool, and die - if (pool_status != 0) { - mp_pool_show_summary(pool[p]); - COE_CHECK_ABORT(true, "Task failure"); - } - } - - // All tasks were successful - for (size_t p = 0; p < sizeof(pool) / sizeof(*pool); p++) { - if (pool[p]->num_used) { - // Only show pools that actually had jobs to run - mp_pool_show_summary(pool[p]); - } - mp_pool_free(&pool[p]); - } - } -} - -int delivery_fixup_test_results(struct Delivery *ctx) { - struct dirent *rec; - - DIR *dp = opendir(ctx->storage.results_dir); - if (!dp) { - perror(ctx->storage.results_dir); - return -1; - } - - while ((rec = readdir(dp)) != NULL) { - char path[PATH_MAX] = {0}; - - if (!strcmp(rec->d_name, ".") || !strcmp(rec->d_name, "..") || !endswith(rec->d_name, ".xml")) { - continue; - } - - sprintf(path, "%s/%s", ctx->storage.results_dir, rec->d_name); - msg(STASIS_MSG_L3, "%s\n", rec->d_name); - if (xml_pretty_print_in_place(path, STASIS_XML_PRETTY_PRINT_PROG, STASIS_XML_PRETTY_PRINT_ARGS)) { - msg(STASIS_MSG_L3 | STASIS_MSG_WARN, "Failed to rewrite file '%s'\n", rec->d_name); - } - } - - closedir(dp); - return 0; -} - diff --git a/src/lib/core/include/artifactory.h b/src/lib/core/include/artifactory.h new file mode 100644 index 0000000..e580886 --- /dev/null +++ b/src/lib/core/include/artifactory.h @@ -0,0 +1,362 @@ +//! @file artifactory.h +#ifndef STASIS_ARTIFACTORY_H +#define STASIS_ARTIFACTORY_H + +#include <stdio.h> +#include <stdlib.h> +#include "core.h" +#include "download.h" + +//! JFrog Artifactory Authentication struct +struct JFRT_Auth { + bool insecure_tls; //!< Disable TLS + char *access_token; //!< Generated access token + char *password; //!< Password + char *client_cert_key_path; //!< Path to where SSL key is stored + char *client_cert_path; //!< Path to where SSL cert is stored + char *ssh_key_path; //!< Path to SSH private key + char *ssh_passphrase; //!< Passphrase for SSH private key + char *user; //!< Account to authenticate as + char *server_id; //!< Artifactory server identification (unused) + char *url; //!< Artifactory server address +}; + +//! JFrog Artifactory Upload struct +struct JFRT_Upload { + bool quiet; //!< Enable quiet mode + char *project; //!< Destination project name + bool ant; //!< Enable Ant style regex + bool archive; //!< Generate a ZIP archive of the uploaded file(s) + char *build_name; //!< Build name + char *build_number; //!< Build number + bool deb; //!< Is Debian package? + bool detailed_summary; //!< Enable upload summary + bool dry_run; //!< Enable dry run (no-op) + char *exclusions; //!< Exclude patterns (separated by semicolons) + bool explode; //!< If uploaded file is an archive, extract it at the destination + bool fail_no_op; //!< Exit 2 when no file are affected + bool flat; //!< Upload with exact file system structure + bool include_dirs; //!< Enable to upload empty directories + char *module; //!< Build-info module name (optional) + bool recursive; //!< Upload files recursively + bool regexp; //!< Use regular expressions instead of wildcards + int retries; //!< Number of retries before giving up + int retry_wait_time; //!< Seconds between retries + char *spec; //!< Path to JSON upload spec + char *spec_vars; + bool symlinks; //!< Preserve symbolic links + bool sync_deletes; //!< Destination is replaced by uploaded files + char *target_props; //!< Properties (separated by semicolons) + int threads; //!< Thread count + bool workaround_parent_only; //!< Change directory to local parent directory before uploading files +}; + +struct JFRT_Download { + char *archive_entries; + char *build; + char *build_name; + char *build_number; + char *bundle; + bool detailed_summary; + bool dry_run; + char *exclude_artifacts; + char *exclude_props; + char *exclusions; + bool explode; + bool fail_no_op; + bool flat; + char *gpg_key; + char *include_deps; + char *include_dirs; + int limit; + int min_split; + char *module; + int offset; + char *project; + char *props; + bool quiet; + bool recursive; + int retries; + int retry_wait_time; + bool skip_checksum; + char *sort_by; + char *sort_order; + char *spec; + char *spec_vars; + int split_count; + bool sync_deletes; + int threads; + bool validate_symlinks; +}; + +struct JFRT_Search { + char *bundle; + bool count; + char *sort_by; + char *sort_order; + int limit; + int offset; + char *spec; + char *spec_vars; + char *props; + bool recursive; + char *build; + bool fail_no_op; + char *exclusions; + char *exclude_artifacts; + char *exclude_patterns; + char *exclude_props; + char *archive_entries; + char *include; + char *include_deps; + char *include_dirs; + char *project; + char *transitive; +}; + +/** + * Download the JFrog CLI tool from jfrog.com + * ```c + * if (artifactory_download_cli(".", + * "https://releases.jfrog.io/artifactory", + * "jfrog-cli", + * "v2-jf", + * "[RELEASE]", + * "Linux", + * "x86_64", + * "jf") { + * remove("./jf"); + * fprintf(stderr, "Failed to download JFrog CLI\n"); + * exit(1); + * } + * + * ``` + * + * @param dest Directory path + * @param jfrog_artifactory_base_url jfrog.com base URL + * @param jfrog_artifactory_product jfrog.com project (jfrog-cli) + * @param cli_major_ver Version series (v1, v2-jf, vX-jf) + * @param version Version to download. "[RELEASE]" will download the latest version available + * @param os Operating system name + * @param arch System CPU architecture + * @param remote_filename File to download (jf) + * @return + */ +int artifactory_download_cli(char *dest, + char *jfrog_artifactory_base_url, + char *jfrog_artifactory_product, + char *cli_major_ver, + char *version, + char *os, + char *arch, + char *remote_filename); + +/** + * JFrog CLI binding. Executes the "jf" tool with arguments. + * + * ```c + * struct JFRT_Auth auth_ctx; + * auth_ctx.user = strdup("myuser"); + * auth_ctx.password = strdup("mypassword"); + * auth_ctx.url = strdup("https://myserver.tld/artifactory"); + * jfrt_auth_init(&auth_ctx); + * + * if (jfrog_cli(&auth_ctx, "rt", "ping", NULL) { + * fprintf(stderr, "Failed to ping artifactory server: %s\n", auth_ctx.url); + * exit(1); + * } + * ``` + * + * @param auth JFRT_Auth structure + * @param subsystem "jf" tool subsystem (i.e. "rt") + * @param task "jf" tool task "upload", "download", etc + * @param args Command line arguments to pass to "jf" tool + * @return exit code from "jf" + */ +int jfrog_cli(struct JFRT_Auth *auth, const char *subsystem, const char *task, char *args); + +/** + * Issue an Artifactory server ping + * + * ```c + * struct JFRT_Auth auth_ctx; + * auth_ctx.user = strdup("myuser"); + * auth_ctx.password = strdup("mypassword"); + * auth_ctx.url = strdup("https://myserver.tld/artifactory"); + * jfrt_auth_init(&auth_ctx); + * + * if (jfrog_cli_ping(&auth_ctx)) { + * fprintf(stderr, "Failed to ping artifactory server: %s\n", auth_ctx.url); + * exit(1); + * } + * ``` + * + * @param auth JFRT_Auth structure + * @return exit code from "jf" + */ +int jfrog_cli_rt_ping(struct JFRT_Auth *auth); + +/** + * Upload files to an Artifactory repository + * + * ```c + * struct JFRT_Auth auth_ctx; + * auth_ctx.user = strdup("myuser"); + * auth_ctx.password = strdup("mypassword"); + * auth_ctx.url = strdup("https://myserver.tld/artifactory"); + * jfrt_auth_init(&auth_ctx); + * + * struct JFRT_Upload upload_ctx; + * jfrt_upload_init(&upload_ctx); + * + * if (jfrt_cli_rt_upload(&auth_ctx, &upload_ctx, + * "local/files_*.ext", "repo_name/ext_files/")) { + * fprintf(stderr, "Upload failed\n"); + * exit(1); + * } + * ``` + * + * @param auth JFRT_Auth structure + * @param ctx JFRT_Upload structure + * @param src local pattern to upload + * @param repo_path remote Artifactory destination path + * @return exit code from "jf" + */ +int jfrog_cli_rt_upload(struct JFRT_Auth *auth, struct JFRT_Upload *ctx, char *src, char *repo_path); + +/** + * Download a file from an Artifactory repository + * + * ```c + * struct JFRT_Auth auth_ctx; + * auth_ctx.user = strdup("myuser"); + * auth_ctx.password = strdup("mypassword"); + * auth_ctx.url = strdup("https://myserver.tld/artifactory"); + * jfrt_auth_init(&auth_ctx); + * + * struct JFRT_Download download_ctx; + * memset(download_ctx, 0, sizeof(download_ctx)); + * + * if (jfrt_cli_rt_download(&auth_ctx, &download_ctx, + * "repo_name/ext_files/", "local/files_*.ext")) { + * fprintf(stderr, "Upload failed\n"); + * exit(1); + * } + * ``` + * + * @param auth JFRT_Auth structure + * @param ctx JFRT_Download structure + * @param repo_path Remote repository w/ file pattern + * @param dest Local destination path + * @return exit code from "jf" + */ +int jfrog_cli_rt_download(struct JFRT_Auth *auth, struct JFRT_Download *ctx, char *repo_path, char *dest); + +/** + * Search for files in an Artifactory repository + * + * @param auth JFRT_Auth structure + * @param ctx JFRT_Search structure + * @param repo_path Remote repository w/ file pattern + * @param dest Local destination path + * @return exit code from "jf" + */ +int jfrog_cli_rt_search(struct JFRT_Auth *auth, struct JFRT_Search *ctx, char *repo_path, char *pattern); + +/** + * Collect runtime data for Artifactory build object. + * + * ```c + * struct JFRT_Auth auth_ctx; + * auth_ctx.user = strdup("myuser"); + * auth_ctx.password = strdup("mypassword"); + * auth_ctx.url = strdup("https://myserver.tld/artifactory"); + * jfrt_auth_init(&auth_ctx); + * + * if (jfrog_cli_rt_build_collect_env(&auth_ctx, "mybuildname", "1.2.3+gabcdef")) { + * fprintf(stderr, "Failed to collect runtime data for Artifactory build object\n"); + * exit(1); + * } + * ``` + * + * @param auth JFRT_Auth structure + * @param build_name Artifactory build name + * @param build_number Artifactory build number + * @return exit code from "jf" + */ +int jfrog_cli_rt_build_collect_env(struct JFRT_Auth *auth, char *build_name, char *build_number); + +/** + * Publish build object to Artifactory server + * + * ```c + * struct JFRT_Auth auth_ctx; + * auth_ctx.user = strdup("myuser"); + * auth_ctx.password = strdup("mypassword"); + * auth_ctx.url = strdup("https://myserver.tld/artifactory"); + * jfrt_auth_init(&auth_ctx); + * + * if (jfrog_cli_rt_build_collect_env(&auth_ctx, "mybuildname", "1.2.3+gabcdef")) { + * fprintf(stderr, "Failed to collect runtime data for Artifactory build object\n"); + * exit(1); + * } + * + * if (jfrog_cli_rt_build_publish(&auth_ctx, "mybuildname", "1.2.3+gabcdef")) { + * fprintf(stderr, "Failed to publish Artifactory build object\n"); + * exit(1); + * } + * ``` + * + * @param auth JFRT_Auth structure + * @param build_name Artifactory build name + * @param build_number Artifactory build number + * @return exit code from "jf" + */ +int jfrog_cli_rt_build_publish(struct JFRT_Auth *auth, char *build_name, char *build_number); + +/** + * Configure JFrog CLI authentication according to STASIS specs + * + * This function will use the STASIS_JF_* environment variables to configure the authentication + * context. With this in mind, if an STASIS_JF_* environment variable is not defined, the original value of + * the structure member will be used instead. + * + * Use STASIS_JF_* variables to configure context + * + * ```c + * struct JFRT_Auth auth_ctx; + * jfrt_auth_init(&ctx); + * ``` + * + * Use your own input, but let the environment take over when variables are defined + * + * ```c + * struct JFRT_Auth auth_ctx; + * auth_ctx.user = strdup("myuser"); + * auth_ctx.password = strdup("mypassword"); + * auth_ctx.url = strdup("https://myserver.tld/artifactory"); + * jfrt_auth_init(&auth_ctx); + * ``` + * + * Use your own input without STASIS's help. Purely an illustrative example. + * + * ```c + * struct JFRT_Auth auth_ctx; + * memset(auth_ctx, 0, sizeof(auth_ctx)); + * auth_ctx.user = strdup("myuser"); + * auth_ctx.password = strdup("mypassword"); + * auth_ctx.url = strdup("https://myserver.tld/artifactory"); + * ``` + * + * @param auth_ctx + * @return + */ +int jfrt_auth_init(struct JFRT_Auth *auth_ctx); + +/** + * Zero-out and apply likely defaults to a JFRT_Upload structure + * @param ctx JFRT_Upload structure + */ +void jfrt_upload_init(struct JFRT_Upload *ctx); + +#endif //STASIS_ARTIFACTORY_H
\ No newline at end of file diff --git a/src/lib/core/include/conda.h b/src/lib/core/include/conda.h new file mode 100644 index 0000000..b8d0caa --- /dev/null +++ b/src/lib/core/include/conda.h @@ -0,0 +1,234 @@ +//! @file conda.h +#ifndef STASIS_CONDA_H +#define STASIS_CONDA_H + +#include <stdio.h> +#include <string.h> +#include <sys/utsname.h> +#include "core.h" +#include "download.h" + +#define CONDA_INSTALL_PREFIX "conda" +#define PYPI_INDEX_DEFAULT "https://pypi.org/simple" + +#define PKG_USE_PIP 0 +#define PKG_USE_CONDA 1 + +#define PKG_NOT_FOUND 0 +#define PKG_FOUND 1 + +#define PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET (-10) +#define PKG_E_SUCCESS (PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET + 0) +#define PKG_INDEX_PROVIDES_E_INTERNAL_MODE_UNKNOWN (PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET + 1) +#define PKG_INDEX_PROVIDES_E_INTERNAL_LOG_HANDLE (PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET + 2) +#define PKG_INDEX_PROVIDES_E_MANAGER_RUNTIME (PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET + 3) +#define PKG_INDEX_PROVIDES_E_MANAGER_SIGNALED (PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET + 4) +#define PKG_INDEX_PROVIDES_E_MANAGER_EXEC (PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET + 5) +#define PKG_INDEX_PROVIDES_FAILED(ECODE) (ECODE <= PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET) + +struct MicromambaInfo { + char *micromamba_prefix; //!< Path to write micromamba binary + char *conda_prefix; //!< Path to install conda base tree +}; + +/** + * Execute micromamba + * @param info MicromambaInfo data structure (must be populated before use) + * @param command printf-style formatter string + * @param ... variadic arguments + * @return exit code + */ +int micromamba(struct MicromambaInfo *info, char *command, ...); + +/** + * Execute Python + * Python interpreter is determined by PATH + * + * ```c + * if (python_exec("-c 'printf(\"Hello world\")'")) { + * fprintf(stderr, "Hello world failed\n"); + * exit(1); + * } + * ``` + * + * @param args arguments to pass to interpreter + * @return exit code from python interpreter + */ +int python_exec(const char *args); + +/** + * Execute Pip + * Pip is determined by PATH + * + * ```c + * if (pip_exec("freeze")) { + * fprintf(stderr, "pip freeze failed\n"); + * exit(1); + * } + * ``` + * + * @param args arguments to pass to Pip + * @return exit code from Pip + */ +int pip_exec(const char *args); + +/** + * Execute conda (or if possible, mamba) + * Conda/Mamba is determined by PATH + * + * ```c + * if (conda_exec("env list")) { + * fprintf(stderr, "Failed to list conda environments\n"); + * exit(1); + * } + * ``` + * + * @param args arguments to pass to Conda + * @return exit code from Conda + */ +int conda_exec(const char *args); + +/** + * Configure the runtime environment to use Conda/Mamba + * + * ```c + * if (conda_activate("/path/to/conda/installation", "base")) { + * fprintf(stderr, "Failed to activate conda's base environment\n"); + * exit(1); + * } + * ``` + * + * @param root directory where conda is installed + * @param env_name the conda environment to activate + * @return 0 on success, -1 on error + */ +int conda_activate(const char *root, const char *env_name); + +/** + * Configure the active conda installation for headless operation + */ +int conda_setup_headless(); + +/** + * Creates a Conda environment from a YAML config + * + * ```c + * if (conda_env_create_from_uri("myenv", "https://myserver.tld/environment.yml")) { + * fprintf(stderr, "Environment creation failed\n"); + * exit(1); + * } + * ``` + * + * @param name Name of new environment to create + * @param uri /path/to/environment.yml + * @param uri file:///path/to/environment.yml + * @param uri http://myserver.tld/environment.yml + * @param uri https://myserver.tld/environment.yml + * @param uri ftp://myserver.tld/environment.yml + * @return exit code from "conda" + */ +int conda_env_create_from_uri(char *name, char *uri); + +/** + * Create a Conda environment using generic package specs + * + * ```c + * // Create a basic environment without any conda packages + * if (conda_env_create("myenv", "3.11", NULL)) { + * fprintf(stderr, "Environment creation failed\n"); + * exit(1); + * } + * + * // Create a basic environment and install conda packages + * if (conda_env_create("myenv", "3.11", "hstcal fitsverify")) { + * fprintf(stderr, "Environment creation failed\n"); + * exit(1); + * } + * ``` + * + * @param name Environment name + * @param python_version Desired version of Python + * @param packages Packages to install (or NULL) + * @return exit code from "conda" + */ +int conda_env_create(char *name, char *python_version, char *packages); + +/** + * Remove a Conda environment + * + * ```c + * if (conda_env_remove("myenv")) { + * fprintf(stderr, "Unable to remove conda environment\n"); + * exit(1); + * } + * ``` + * + * @param name Environment name + * @return exit code from "conda" + */ +int conda_env_remove(char *name); + +/** + * Export a Conda environment in YAML format + * + * ```c + * if (conda_env_export("myenv", "./", "myenv.yml")) { + * fprintf(stderr, "Unable to export environment\n"); + * exit(1); + * } + * ``` + * + * @param name Environment name to export + * @param output_dir Destination directory + * @param output_filename Destination file name + * @return exit code from "conda" + */ +int conda_env_export(char *name, char *output_dir, char *output_filename); + +/** + * Run "conda index" on a local conda channel + * + * ```c + * if (conda_index("/path/to/channel")) { + * fprintf(stderr, "Unable to index requested path\n"); + * exit(1); + * } + * ``` + * + * @param path Top-level directory of conda channel + * @return exit code from "conda" + */ +int conda_index(const char *path); + +/** + * Determine whether a package index contains a package + * + * ```c + * int result = pkg_index_provides(USE_PIP, NULL, "numpy>1.26"); + * if (PKG_INDEX_PROVIDES_FAILED(result)) { + * fprintf(stderr, "failed: %s\n", pkg_index_provides_strerror(result)); + * exit(1); + * } else if (result == PKG_NOT_FOUND) { + * // package does not exist upstream + * } else { + * // package exists upstream + * } + * ``` + * + * @param mode USE_PIP + * @param mode USE_CONDA + * @param index a file system path or url pointing to a simple index or conda channel + * @param spec a pip package specification (e.g. `name==1.2.3`) + * @param spec a conda package specification (e.g. `name=1.2.3`) + * @return PKG_NOT_FOUND, if not found + * @return PKG_FOUND, if found + * @return PKG_E_INDEX_PROVIDES_{ERROR}, on error (see conda.h) + */ +int pkg_index_provides(int mode, const char *index, const char *spec); +const char *pkg_index_provides_strerror(int code); + +char *conda_get_active_environment(); + +int conda_env_exists(const char *root, const char *name); + +#endif //STASIS_CONDA_H diff --git a/src/lib/core/include/copy.h b/src/lib/core/include/copy.h new file mode 100644 index 0000000..0f92ddd --- /dev/null +++ b/src/lib/core/include/copy.h @@ -0,0 +1,35 @@ +//! @file copy.h +#ifndef STASIS_COPY_H + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <dirent.h> +#include <errno.h> +#include <sys/stat.h> +#include <unistd.h> +#include "core.h" + +#define CT_OWNER 1 << 1 +#define CT_PERM 1 << 2 + +/** + * Copy a single file + * + * ```c + * if (copy2("/source/path/example.txt", "/destination/path/example.txt", CT_PERM | CT_OWNER)) { + * fprintf(stderr, "Unable to copy file\n"); + * exit(1); + * } + * ``` + * + * + * @param src source file path + * @param dest destination file path + * @param op CT_OWNER (preserve ownership) + * @param op CT_PERM (preserve permission bits) + * @return 0 on success, -1 on error + */ +int copy2(const char *src, const char *dest, unsigned op); + +#endif // STASIS_COPY_H
\ No newline at end of file diff --git a/src/lib/core/include/core.h b/src/lib/core/include/core.h new file mode 100644 index 0000000..362ac8d --- /dev/null +++ b/src/lib/core/include/core.h @@ -0,0 +1,85 @@ +//! @file core.h +#ifndef STASIS_CORE_H +#define STASIS_CORE_H + +#include <stdio.h> +#include <stdlib.h> +#include <stdbool.h> +#include <string.h> +#include <limits.h> +#include <unistd.h> +#include <time.h> +#include <sys/statvfs.h> + +#define SYSERROR(MSG, ...) do { \ + fprintf(stderr, "%s:%s:%d:%s - ", path_basename(__FILE__), __FUNCTION__, __LINE__, (errno > 0) ? strerror(errno) : "info"); \ + fprintf(stderr, MSG LINE_SEP, __VA_ARGS__); \ +} while (0) +#define STASIS_BUFSIZ 8192 +#define STASIS_NAME_MAX 255 +#define STASIS_DIRSTACK_MAX 1024 +#define STASIS_TIME_STR_MAX 128 +#define HTTP_ERROR(X) X >= 400 + +#include "config.h" +#include "core_mem.h" + +#define COE_CHECK_ABORT(COND, MSG) \ + do {\ + if (!globals.continue_on_error && COND) { \ + msg(STASIS_MSG_ERROR, MSG ": Aborting execution (--continue-on-error/-C is not enabled)\n"); \ + exit(1); \ + } \ + } while (0) + +struct STASIS_GLOBAL { + bool verbose; //!< Enable verbose output + bool always_update_base_environment; //!< Update base environment immediately after activation + bool continue_on_error; //!< Do not stop on test failures + bool conda_fresh_start; //!< Always install a new copy of Conda + bool enable_docker; //!< Enable docker image builds + bool enable_artifactory; //!< Enable artifactory uploads + bool enable_artifactory_build_info; //!< Enable build info (best disabled for pure test runs) + bool enable_artifactory_upload; //!< Enable artifactory file upload (dry-run when false) + bool enable_testing; //!< Enable package testing + bool enable_overwrite; //!< Enable release file clobbering + bool enable_rewrite_spec_stage_2; //!< Enable automatic @STR@ replacement in output files + bool enable_parallel; //!< Enable testing in parallel + long cpu_limit; //!< Limit parallel processing to n cores (default: max - 1) + long parallel_fail_fast; //!< Fail immediately on error + int pool_status_interval; //!< Report "Task is running" every n seconds + struct StrList *conda_packages; //!< Conda packages to install after initial activation + struct StrList *pip_packages; //!< Pip packages to install after initial activation + char *tmpdir; //!< Path to temporary storage directory + char *conda_install_prefix; //!< Path to install conda + char *sysconfdir; //!< Path where STASIS reads its configuration files (mission directory, etc) + struct { + char *tox_posargs; + char *conda_reactivate; + } workaround; + struct Jfrog { + char *jfrog_artifactory_base_url; + char *jfrog_artifactory_product; + char *cli_major_ver; + char *version; + char *os; + char *arch; + char *remote_filename; + char *repo; + char *url; + } jfrog; + struct EnvCtl *envctl; +}; +extern struct STASIS_GLOBAL globals; + +extern const char *VERSION; +extern const char *AUTHOR; +extern const char *BANNER; + + +/** + * Free memory allocated in global configuration structure + */ +void globals_free(); + +#endif //STASIS_CORE_H diff --git a/src/lib/core/include/core_mem.h b/src/lib/core/include/core_mem.h new file mode 100644 index 0000000..bd50e9d --- /dev/null +++ b/src/lib/core/include/core_mem.h @@ -0,0 +1,18 @@ +//! @file core_mem.h +#ifndef STASIS_CORE_MEM_H +#define STASIS_CORE_MEM_H + +#include "environment.h" +#include "strlist.h" + +#define guard_runtime_free(X) do { if (X) { runtime_free(X); X = NULL; } } while (0) +#define guard_strlist_free(X) do { if ((*X)) { strlist_free(X); (*X) = NULL; } } while (0) +#define guard_free(X) do { if (X) { free(X); X = NULL; } } while (0) +#define GENERIC_ARRAY_FREE(ARR) do { \ + for (size_t ARR_I = 0; ARR && ARR[ARR_I] != NULL; ARR_I++) { \ + guard_free(ARR[ARR_I]); \ + } \ + guard_free(ARR); \ +} while (0) + +#endif //STASIS_CORE_MEM_H diff --git a/src/lib/core/include/docker.h b/src/lib/core/include/docker.h new file mode 100644 index 0000000..7585d86 --- /dev/null +++ b/src/lib/core/include/docker.h @@ -0,0 +1,92 @@ +//! @file docker.h +#ifndef STASIS_DOCKER_H +#define STASIS_DOCKER_H + +#include "core.h" + +//! Flag to squelch output from docker_exec() +#define STASIS_DOCKER_QUIET 1 << 1 + +//! Flag for older style docker build +#define STASIS_DOCKER_BUILD 1 << 1 +//! Flag for docker buildx +#define STASIS_DOCKER_BUILD_X 1 << 2 + +//! Compress "docker save"ed images with a compression program +#define STASIS_DOCKER_IMAGE_COMPRESSION "zstd" + +struct DockerCapabilities { + int podman; //!< Is "docker" really podman? + int build; //!< Is a build plugin available? + int available; //!< Is a "docker" program available? + int usable; //!< Is docker in a usable state for the current user? +}; + +/** + * Determine the state of docker on the system + * + * ```c + * struct DockerCapabilities docker_is; + * if (!docker_capable(&docker_is)) { + * fprintf(stderr, "%s is %savailable, and %susable\n", + * docker_is.podman ? "Podman" : "Docker", + * docker_is.available ? "" : "not ", + * docker_is.usable ? "" : "not "); + * exit(1); + * } + * ``` + * + * @param result DockerCapabilities struct + * @return 1 on success, 0 on error + */ +int docker_capable(struct DockerCapabilities *result); + +/** + * Execute a docker command + * + * Use the `STASIS_DOCKER_QUIET` flag to suppress all output from stdout and stderr. + * + * ```c + * if (docker_exec("run --rm -t ubuntu:latest /bin/bash -c 'echo Hello world'", 0)) { + * fprintf(stderr, "Docker hello world failed\n"); + * exit(1); + * } + * ``` + * + * @param args arguments to pass to docker + * @param flags + * @return exit code from "docker" + */ +int docker_exec(const char *args, unsigned flags); + +/** + * Build a docker image + * + * ```c + * struct DockerCapabilities docker_is; + * docker_capable(&docker_is); + * + * if (docker_is.usable) { + * printf("Building docker image\n"); + * if (docker_build("path/to/Dockerfile/dir")) { + * fprintf("Docker build failed\n"); + * exit(1); + * } + * } else { + * fprintf(stderr, "No usable docker installation available\n"); + * } + * ``` + * + * @param dirpath + * @param args + * @param engine + * @return + */ +int docker_build(const char *dirpath, const char *args, int engine); +int docker_script(const char *image, char *data, unsigned flags); +int docker_save(const char *image, const char *destdir, const char *compression_program); +void docker_sanitize_tag(char *str); +int docker_validate_compression_program(char *prog); + + +#endif //STASIS_DOCKER_H diff --git a/src/lib/core/include/download.h b/src/lib/core/include/download.h new file mode 100644 index 0000000..0b6311e --- /dev/null +++ b/src/lib/core/include/download.h @@ -0,0 +1,12 @@ +//! @file download.h +#ifndef STASIS_DOWNLOAD_H +#define STASIS_DOWNLOAD_H + +#include <stdlib.h> +#include <string.h> +#include <curl/curl.h> + +size_t download_writer(void *fp, size_t size, size_t nmemb, void *stream); +long download(char *url, const char *filename, char **errmsg); + +#endif //STASIS_DOWNLOAD_H diff --git a/src/lib/core/include/envctl.h b/src/lib/core/include/envctl.h new file mode 100644 index 0000000..659cae3 --- /dev/null +++ b/src/lib/core/include/envctl.h @@ -0,0 +1,39 @@ +//! @file envctl.h +#ifndef STASIS_ENVCTL_H +#define STASIS_ENVCTL_H + +#include <stdlib.h> +#include "core.h" + +#define STASIS_ENVCTL_PASSTHRU 0 +#define STASIS_ENVCTL_REQUIRED 1 << 1 +#define STASIS_ENVCTL_REDACT 1 << 2 +#define STASIS_ENVCTL_DEFAULT_ALLOC 100 + +#define STASIS_ENVCTL_RET_FAIL (-1) +#define STASIS_ENVCTL_RET_SUCCESS 1 +#define STASIS_ENVCTL_RET_IGNORE 2 +typedef int (envctl_except_fn)(const void *, const void *); + +struct EnvCtl_Item { + unsigned flags; //<! One or more STASIS_ENVCTL_* flags + const char *name; //<! Environment variable name + envctl_except_fn *callback; +}; + +struct EnvCtl { + size_t num_alloc; + size_t num_used; + struct EnvCtl_Item **item; +}; + +struct EnvCtl *envctl_init(); +int envctl_register(struct EnvCtl **envctl, unsigned flags, envctl_except_fn *callback, const char *name); +unsigned envctl_get_flags(const struct EnvCtl *envctl, const char *name); +unsigned envctl_check_required(unsigned flags); +unsigned envctl_check_redact(unsigned flags); +int envctl_check_present(const struct EnvCtl_Item *item, const char *name); +void envctl_do_required(const struct EnvCtl *envctl, int verbose); +void envctl_free(struct EnvCtl **envctl); + +#endif // STASIS_ENVCTL_H
\ No newline at end of file diff --git a/src/lib/core/include/environment.h b/src/lib/core/include/environment.h new file mode 100644 index 0000000..34bc600 --- /dev/null +++ b/src/lib/core/include/environment.h @@ -0,0 +1,23 @@ +/** + * @file environment.h + */ +#ifndef STASIS_ENVIRONMENT_H +#define STASIS_ENVIRONMENT_H + +#include <stdio.h> +#include <stdlib.h> +#include <dirent.h> +#include "environment.h" + +typedef struct StrList RuntimeEnv; + +ssize_t runtime_contains(RuntimeEnv *env, const char *key); +RuntimeEnv *runtime_copy(char **env); +int runtime_replace(RuntimeEnv **dest, char **src); +char *runtime_get(RuntimeEnv *env, const char *key); +void runtime_set(RuntimeEnv *env, const char *_key, char *_value); +char *runtime_expand_var(RuntimeEnv *env, char *input); +void runtime_export(RuntimeEnv *env, char **keys); +void runtime_apply(RuntimeEnv *env); +void runtime_free(RuntimeEnv *env); +#endif //STASIS_ENVIRONMENT_H diff --git a/src/lib/core/include/github.h b/src/lib/core/include/github.h new file mode 100644 index 0000000..f9b47a3 --- /dev/null +++ b/src/lib/core/include/github.h @@ -0,0 +1,11 @@ +//! @file github.h +#ifndef STASIS_GITHUB_H +#define STASIS_GITHUB_H + +#include <curl/curl.h> + +#define STASIS_GITHUB_API_VERSION "2022-11-28" + +int get_github_release_notes(const char *api_token, const char *repo, const char *tag, const char *target_commitish, char **output); + +#endif //STASIS_GITHUB_H
\ No newline at end of file diff --git a/src/lib/core/include/ini.h b/src/lib/core/include/ini.h new file mode 100644 index 0000000..557f157 --- /dev/null +++ b/src/lib/core/include/ini.h @@ -0,0 +1,260 @@ +/// @file ini.h + +#ifndef STASIS_INI_H +#define STASIS_INI_H +#include <stdio.h> +#include <stddef.h> +#include <stdbool.h> +#include "template.h" + +#define INI_WRITE_RAW 0 ///< Dump INI data. Contents are not modified. +#define INI_WRITE_PRESERVE 1 ///< Dump INI data. Template strings are +#define INI_READ_RAW 0 ///< Dump INI data. Contents are not modified. +#define INI_READ_RENDER 1 ///< Dump INI data. Template strings are +#define INI_SETVAL_APPEND 0 +#define INI_SETVAL_REPLACE 1 +#define INI_SEARCH_EXACT 0 +#define INI_SEARCH_BEGINS 1 +#define INI_SEARCH_SUBSTR 2 + ///< expanded to preserve runtime state. + +#define INIVAL_TYPE_CHAR 1 ///< Byte +#define INIVAL_TYPE_UCHAR 2 ///< Unsigned byte +#define INIVAL_TYPE_SHORT 3 ///< Short integer +#define INIVAL_TYPE_USHORT 4 ///< Unsigned short integer +#define INIVAL_TYPE_INT 5 ///< Integer +#define INIVAL_TYPE_UINT 6 ///< Unsigned integer +#define INIVAL_TYPE_LONG 7 ///< Long integer +#define INIVAL_TYPE_ULONG 8 ///< Unsigned long integer +#define INIVAL_TYPE_LLONG 9 ///< Long long integer +#define INIVAL_TYPE_ULLONG 10 ///< Unsigned long long integer +#define INIVAL_TYPE_DOUBLE 11 ///< Double precision float +#define INIVAL_TYPE_FLOAT 12 ///< Single precision float +#define INIVAL_TYPE_STR 13 ///< String +#define INIVAL_TYPE_STR_ARRAY 14 ///< String Array +#define INIVAL_TYPE_BOOL 15 ///< Boolean + +#define INIVAL_TO_LIST 1 << 1 + +/*! \union INIVal + * \brief Consolidate possible value types + */ +union INIVal { + char as_char; ///< Byte + unsigned char as_uchar; ///< Unsigned byte + short as_short; ///< Short integer + unsigned short as_ushort; ///< Unsigned short integer + int as_int; ///< Integer + unsigned as_uint; ///< Unsigned integer + long as_long; ///< Long integer + unsigned long as_ulong; ///< Unsigned long integer + long long as_llong; ///< Long long integer + unsigned long long as_ullong; ///< Unsigned long long integer + double as_double; ///< Double precision float + float as_float; ///< Single precision float + char *as_char_p; ///< String + char **as_char_array_p; ///< String Array + bool as_bool; ///< Boolean +}; + + +/*! \struct INIData + * \brief A structure to describe an INI data record + */ +struct INIData { + char *key; ///< INI variable name + char *value; ///< INI variable value + unsigned type_hint; +}; + +/*! \struct INISection + * \brief A structure to describe an INI section + */ +struct INISection { + size_t data_count; ///< Total INIData records + char *key; ///< INI section name + struct INIData **data; ///< Array of INIData records +}; + +/*! \struct INIFILE + * \brief A structure to describe an INI configuration file + */ +struct INIFILE { + size_t section_count; ///< Total INISection records + struct INISection **section; ///< Array of INISection records +}; + +/** + * Open and parse and INI configuration file + * + * ~~~.c + * #include "ini.h" + * int main(int argc, char *argv[]) { + * const char *filename = "example.ini" + * struct INIFILE *ini; + * ini = ini_open(filename); + * if (!ini) { + * perror(filename); + * exit(1); + * } + * } + * ~~~ + * + * @param filename path to INI file + * @return pointer to INIFILE + */ +struct INIFILE *ini_open(const char *filename); + +/** + * + * @param ini + * @param value + * @return + */ +struct INISection *ini_section_search(struct INIFILE **ini, unsigned mode, const char *value); + +/** + * + * @param ini + * @param key + * @return + */ +int ini_section_create(struct INIFILE **ini, char *key); + +/** + * + * @param ini + * @param section + * @param key + * @return + */ +int ini_has_key(struct INIFILE *ini, const char *section, const char *key); + +/** + * Assign value to a section key + * @param ini + * @param type INI_SETVAL_APPEND or INI_SETVAL_REPLACE + * @param section_name + * @param key + * @param value + * @return + */ +int ini_setval(struct INIFILE **ini, unsigned type, char *section_name, char *key, char *value); + +/** + * Retrieve all data records in an INI section + * + * `example.ini` + * ~~~.ini + * [example] + * key_1 = a string + * key_2 = 100 + * ~~~ + * + * `example.c` + * ~~~.c + * #include "ini.h" + * int main(int argc, char *argv[]) { + * const char *filename = "example.ini" + * struct INIData *data; + * struct INIFILE *ini; + * ini = ini_open(filename); + * if (!ini) { + * perror(filename); + * exit(1); + * } + * // Read all records in "example" section + * for (size_t i = 0; ((data = ini_getall(&ini, "example") != NULL); i++) { + * printf("key=%s, value=%s\n", data->key, data->value); + * } + * } + * ~~~ + * + * @param ini pointer to INIFILE + * @param section_name to read + * @return pointer to INIData + */ +struct INIData *ini_getall(struct INIFILE *ini, char *section_name); + +/** + * Retrieve a single record from a section key + * + * `example.ini` + * ~~~.ini + * [example] + * key_1 = a string + * key_2 = 100 + * ~~~ + * + * `example.c` + * ~~~.c + * #include "ini.h" + * int main(int argc, char *argv[]) { + * const char *filename = "example.ini" + * union INIVal *data; + * struct INIFILE *ini; + * ini = ini_open(filename); + * if (!ini) { + * perror(filename); + * exit(1); + * } + * data = ini_getval(&ini, "example", "key_1", INIVAL_TYPE_STR); + * puts(data.as_char_p); + * data = ini_getval(&ini, "example", "key_2", INIVAL_TYPE_INT); + * printf("%d\n", data.as_int); + * } + * ~~~ + * + * @param ini pointer to INIFILE + * @param section_name to read + * @param key to return + * @param type INIVAL_TYPE_INT + * @param type INIVAL_TYPE_UINT + * @param type INIVAL_TYPE_LONG + * @param type INIVAL_TYPE_ULONG + * @param type INIVAL_TYPE_LLONG + * @param type INIVAL_TYPE_ULLONG + * @param type INIVAL_TYPE_DOUBLE + * @param type INIVAL_TYPE_FLOAT + * @param type INIVAL_TYPE_STR + * @param type INIVAL_TYPE_STR_ARRAY + * @param type INIVAL_TYPE_BOOL + * @param result pointer to INIVal + * @return 0 on success + * @return Non-zero on error + */ +int ini_getval(struct INIFILE *ini, char *section_name, char *key, int type, int flags, union INIVal *result); + +/** + * Write INIFILE sections and data to a file stream + * @param ini pointer to INIFILE + * @param file pointer to address of file stream + * @return 0 on success, -1 on error + */ +int ini_write(struct INIFILE *ini, FILE **stream, unsigned mode); + +/** + * Free memory allocated by ini_open() + * @param ini + */ +void ini_free(struct INIFILE **ini); + +int ini_getval_int(struct INIFILE *ini, char *section_name, char *key, int flags, int *state); +unsigned int ini_getval_uint(struct INIFILE *ini, char *section_name, char *key, int flags, int *state); +long ini_getval_long(struct INIFILE *ini, char *section_name, char *key, int flags, int *state); +unsigned long ini_getval_ulong(struct INIFILE *ini, char *section_name, char *key, int flags, int *state); +long long ini_getval_llong(struct INIFILE *ini, char *section_name, char *key, int flags, int *state); +unsigned long long ini_getval_ullong(struct INIFILE *ini, char *section_name, char *key, int flags, int *state); +float ini_getval_float(struct INIFILE *ini, char *section_name, char *key, int flags, int *state); +double ini_getval_double(struct INIFILE *ini, char *section_name, char *key, int flags, int *state); +bool ini_getval_bool(struct INIFILE *ini, char *section_name, char *key, int flags, int *state); +short ini_getval_short(struct INIFILE *ini, char *section_name, char *key, int flags, int *state); +unsigned short ini_getval_ushort(struct INIFILE *ini, char *section_name, char *key, int flags, int *state); +char ini_getval_char(struct INIFILE *ini, char *section_name, char *key, int flags, int *state); +unsigned char ini_getval_uchar(struct INIFILE *ini, char *section_name, char *key, int flags, int *state); +char *ini_getval_char_p(struct INIFILE *ini, char *section_name, char *key, int flags, int *state); +char *ini_getval_str(struct INIFILE *ini, char *section_name, char *key, int flags, int *state); +char *ini_getval_char_array_p(struct INIFILE *ini, char *section_name, char *key, int flags, int *state); +char *ini_getval_str_array(struct INIFILE *ini, char *section_name, char *key, int flags, int *state); +struct StrList *ini_getval_strlist(struct INIFILE *ini, char *section_name, char *key, char *tok, int flags, int *state); +#endif //STASIS_INI_H diff --git a/src/lib/core/include/junitxml.h b/src/lib/core/include/junitxml.h new file mode 100644 index 0000000..777ee27 --- /dev/null +++ b/src/lib/core/include/junitxml.h @@ -0,0 +1,135 @@ +/// @file junitxml.h +#ifndef STASIS_JUNITXML_H +#define STASIS_JUNITXML_H +#include <libxml/xmlreader.h> + +#define JUNIT_RESULT_STATE_NONE 0 +#define JUNIT_RESULT_STATE_FAILURE 1 +#define JUNIT_RESULT_STATE_SKIPPED 2 +#define JUNIT_RESULT_STATE_ERROR 3 + +/** + * Represents a failed test case + */ +struct JUNIT_Failure { + /// Error text + char *message; +}; + +/** + * Represents a test case error + */ +struct JUNIT_Error { + /// Error text + char *message; +}; + +/** + * Represents a skipped test case + */ +struct JUNIT_Skipped { + /// Type of skip event + char *type; + /// Reason text + char *message; +}; + +/** + * Represents a junit test case + */ +struct JUNIT_Testcase { + /// Class name + char *classname; + /// Name of test + char *name; + /// Test duration in fractional seconds + float time; + /// Standard output message + char *message; + /// Result type + int tc_result_state_type; + /// Type container for result (there can only be one) + union tc_state_ptr { + struct JUNIT_Failure *failure; + struct JUNIT_Skipped *skipped; + struct JUNIT_Error *error; + } result_state; ///< Result data +}; + +/** + * Represents a junit test suite + */ +struct JUNIT_Testsuite { + /// Test suite name + char *name; + /// Total number of test terminated due to an error + int errors; + /// Total number of failed tests + int failures; + /// Total number of skipped tests + int skipped; + /// Total number of tests + int tests; + /// Total duration in fractional seconds + float time; + /// Timestamp + char *timestamp; + /// Test runner host name + char *hostname; + /// Array of test cases + struct JUNIT_Testcase **testcase; + /// Total number of test cases in use + size_t _tc_inuse; + /// Total number of test cases allocated + size_t _tc_alloc; +}; + +/** + * Extract information from a junit XML file + * + * ~~~{.c} + * struct JUNIT_Testsuite *testsuite; + * const char *filename = "/path/to/result.xml"; + * + * testsuite = junitxml_testsuite_read(filename); + * if (testsuite) { + * // Did any test cases fail? + * if (testsuite->failures) { + * printf("Test suite '%s' has %d failure(s)\n", testsuite->name, testsuite->failures + * // Scan test cases for failure data + * for (size_t i = 0; i < testsuite->_tc_inuse; i++) { + * // Check result state (one of) + * // JUNIT_RESULT_STATE_FAILURE + * // JUNIT_RESULT_STATE_ERROR + * // JUNIT_RESULT_STATE_SKIPPED + * struct JUNIT_Testcase *testcase = testsuite->testcase[i]; + * if (testcase->tc_result_state_type) { + * if (testcase->tc_result_state_type == JUNIT_RESULT_STATE_FAILURE) { + * // Display information from failed test case + * printf("[FAILED] %s::%s\nOutput:\n%s\n", + * testcase->classname, + * testcase->name, + * testcase->result_state.failure->message); + * } + * } + * } + * } + * // Release test suite resources + * junitxml_testsuite_free(&testsuite); + * } else { + * // handle error + * } + * ~~~ + * + * @param filename path to junit XML file + * @return pointer to JUNIT_Testsuite + */ +struct JUNIT_Testsuite *junitxml_testsuite_read(const char *filename); + +/** + * Free memory allocated by junitxml_testsuite_read + * @param testsuite pointer to JUNIT_Testsuite + */ +void junitxml_testsuite_free(struct JUNIT_Testsuite **testsuite); + +#endif //STASIS_JUNITXML_H diff --git a/src/lib/core/include/multiprocessing.h b/src/lib/core/include/multiprocessing.h new file mode 100644 index 0000000..ec7c1ad --- /dev/null +++ b/src/lib/core/include/multiprocessing.h @@ -0,0 +1,134 @@ +/// @file multiprocessing.h +#ifndef STASIS_MULTIPROCESSING_H +#define STASIS_MULTIPROCESSING_H + +#include "core.h" +#include <signal.h> +#include <sys/wait.h> +#include <semaphore.h> +#include <sys/mman.h> +#include <fcntl.h> +#include <sys/stat.h> + +struct MultiProcessingTask { + pid_t pid; ///< Program PID + pid_t parent_pid; ///< Program PID (parent process) + int status; ///< Child process exit status + int signaled_by; ///< Last signal received, if any + time_t _now; ///< Current time + time_t _seconds; ///< Time elapsed (used by MultiprocessingPool.status_interval) + char ident[255]; ///< Identity of the pool task + char *cmd; ///< Shell command(s) to be executed + size_t cmd_len; ///< Length of command string (for mmap/munmap) + char working_dir[PATH_MAX]; ///< Path to directory `cmd` should be executed in + char log_file[PATH_MAX]; ///< Full path to stdout/stderr log file + char parent_script[PATH_MAX]; ///< Path to temporary script executing the task + struct { + struct timespec t_start; + struct timespec t_stop; + } time_data; ///< Wall-time counters +}; + +struct MultiProcessingPool { + struct MultiProcessingTask *task; ///< Array of tasks to execute + size_t num_used; ///< Number of tasks populated in the task array + size_t num_alloc; ///< Number of tasks allocated by the task array + char ident[255]; ///< Identity of task pool + char log_root[PATH_MAX]; ///< Base directory to store stderr/stdout log files + int status_interval; ///< Report a pooled task is "running" every n seconds +}; + +/// A multiprocessing task's initial state (i.e. "FAIL") +#define MP_POOL_TASK_STATUS_INITIAL (-1) + +/// Maximum number of multiprocessing tasks STASIS can execute +#define MP_POOL_TASK_MAX 1000 + +/// Value signifies a process is unused or finished executing +#define MP_POOL_PID_UNUSED 0 + +/// Option flags for mp_pool_join() +#define MP_POOL_FAIL_FAST 1 << 1 + +/** + * Create a multiprocessing pool + * + * ```c + * #include "multiprocessing.h" + * #include "utils.h" // for get_cpu_count() + * + * int main(int argc, char *argv[]) { + * struct MultiProcessingPool *mp; + * mp = mp_pool_init("mypool", "/tmp/mypool_logs"); + * if (mp) { + * char *commands[] = { + * "/bin/echo hello world", + * "/bin/echo world hello", + * NULL + * } + * for (size_t i = 0; commands[i] != NULL); i++) { + * struct MultiProcessingTask *task; + * char task_name[100]; + * + * sprintf(task_name, "mytask%zu", i); + * task = mp_task(mp, task_name, commands[i]); + * if (!task) { + * // handle task creation error + * } + * } + * if (mp_pool_join(mp, get_cpu_count(), MP_POOL_FAIL_FAST)) { + * // handle pool execution error + * } + * mp_pool_free(&mp); + * } else { + * // handle pool initialization error + * } + * } + * ``` + * + * @param ident a name to identify the pool + * @param log_root the path to store program output + * @return pointer to initialized MultiProcessingPool + * @return NULL on error + */ +struct MultiProcessingPool *mp_pool_init(const char *ident, const char *log_root); + +/** + * Create a multiprocessing pool task + * + * @param pool a pointer to MultiProcessingPool + * @param ident a name to identify the task + * @param cmd a command to execute + * @return pointer to MultiProcessingTask structure + * @return NULL on error + */ +struct MultiProcessingTask *mp_pool_task(struct MultiProcessingPool *pool, const char *ident, char *working_dir, char *cmd); + +/** + * Execute all tasks in a pool + * + * @param pool a pointer to MultiProcessingPool + * @param jobs the number of processes to spawn at once (for serial execution use `1`) + * @param flags option to be OR'd (MP_POOL_FAIL_FAST) + * @return 0 on success + * @return >0 on failure + * @return <0 on error + */ +int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags); + +/** + * Show summary of pool tasks + * + * @param pool a pointer to MultiProcessingPool + */ +void mp_pool_show_summary(struct MultiProcessingPool *pool); + +/** + * Release resources allocated by mp_pool_init() + * + * @param a pointer to MultiProcessingPool + */ +void mp_pool_free(struct MultiProcessingPool **pool); + + +#endif //STASIS_MULTIPROCESSING_H diff --git a/src/lib/core/include/os_darwin.h b/src/lib/core/include/os_darwin.h new file mode 100644 index 0000000..e8513ff --- /dev/null +++ b/src/lib/core/include/os_darwin.h @@ -0,0 +1,26 @@ +#ifndef STASIS_OS_DARWIN_H +#define STASIS_OS_DARWIN_H + +#include <sys/mount.h> + +#ifndef __DARWIN_64_BIT_INO_T +#define statvfs statfs + +#ifndef ST_RDONLY +#define ST_RDONLY MNT_RDONLY +#endif + +#define ST_NOEXEC MNT_NOEXEC +#define f_flag f_flags +#endif // __DARWIN_64_BIT_INO_T + +#include <limits.h> + +#ifndef PATH_MAX +#include <sys/syslimits.h> +#endif + +extern char **environ; +#define __environ environ + +#endif diff --git a/src/lib/core/include/os_linux.h b/src/lib/core/include/os_linux.h new file mode 100644 index 0000000..d418090 --- /dev/null +++ b/src/lib/core/include/os_linux.h @@ -0,0 +1,10 @@ +#ifndef STASIS_OS_LINUX_H +#define STASIS_OS_LINUX_H + +#include <limits.h> + +#ifndef PATH_MAX +#include <linux/limits.h> +#endif + +#endif diff --git a/src/lib/core/include/package.h b/src/lib/core/include/package.h new file mode 100644 index 0000000..eff1874 --- /dev/null +++ b/src/lib/core/include/package.h @@ -0,0 +1,30 @@ +#ifndef STASIS_PACKAGE_H +#define STASIS_PACKAGE_H + +struct Package { + struct { + const char *name; + const char *version_spec; + const char *version; + } meta; + struct { + const char *uri; + unsigned handler; + } source; + struct { + struct Test *test; + size_t pass; + size_t fail; + size_t skip; + }; + unsigned state; +}; + +struct Package *stasis_package_init(void); +void stasis_package_set_name(struct Package *pkg, const char *name); +void stasis_package_set_version(struct Package *pkg, const char *version); +void stasis_package_set_version_spec(struct Package *pkg, const char *version_spec); +void stasis_package_set_uri(struct Package *pkg, const char *uri); +void stasis_package_set_handler(struct Package *pkg, unsigned handler); + +#endif //STASIS_PACKAGE_H diff --git a/src/lib/core/include/recipe.h b/src/lib/core/include/recipe.h new file mode 100644 index 0000000..4dea248 --- /dev/null +++ b/src/lib/core/include/recipe.h @@ -0,0 +1,72 @@ +//! @file recipe.h +#ifndef STASIS_RECIPE_H +#define STASIS_RECIPE_H + +#include "str.h" +#include "utils.h" + +//! Unable to determine recipe repo type +#define RECIPE_TYPE_UNKNOWN 0 +//! Recipe repo is from conda-forge +#define RECIPE_TYPE_CONDA_FORGE 1 +//! Recipe repo is from astroconda +#define RECIPE_TYPE_ASTROCONDA 2 +//! Recipe repo provides the required build configurations but doesn't match conda-forge or astroconda's signature +#define RECIPE_TYPE_GENERIC 3 + +/** + * Download a Conda package recipe + * + * ```c + * char *recipe = NULL; + * + * if (recipe_clone("base/dir", "https://github.com/example/repo", "branch", &recipe)) { + * fprintf(stderr, "Failed to clone conda recipe\n"); + * exit(1); + * } else { + * chdir(recipe); + * } + * ``` + * + * @param recipe_dir path to store repository + * @param url remote address of git repository + * @param gitref branch/tag/commit + * @param result absolute path to downloaded repository + * @return exit code from "git", -1 on error + */ +int recipe_clone(char *recipe_dir, char *url, char *gitref, char **result); + +/** + * Determine the layout/type of repository path + * + * ```c + * if (recipe_clone("base/dir", "https://github.com/example/repo", "branch", &recipe)) { + * fprintf(stderr, "Failed to clone conda recipe\n"); + * exit(1); + * } + * + * int recipe_type; + * recipe_type = recipe_get_type(recipe); + * switch (recipe_type) { + * case RECIPE_TYPE_CONDA_FORGE: + * // do something specific for conda-forge directory structure + * break; + * case RECIPE_TYPE_ASTROCONDA: + * // do something specific for astroconda directory structure + * break; + * case RECIPE_TYPE_GENERIC: + * // do something specific for a directory containing a meta.yaml config + * break; + * case RECIPE_TYPE_UNKNOWN: + * default: + * // the structure is foreign or the path doesn't contain a conda recipe + * break; + * } + * ``` + * + * @param repopath path to git repository containing conda recipe(s) + * @return One of RECIPE_TYPE_UNKNOWN, RECIPE_TYPE_CONDA_FORGE, RECIPE_TYPE_ASTROCONDA, RECIPE_TYPE_GENERIC + */ +int recipe_get_type(char *repopath); + +#endif //STASIS_RECIPE_H diff --git a/src/lib/core/include/relocation.h b/src/lib/core/include/relocation.h new file mode 100644 index 0000000..9a1f0f4 --- /dev/null +++ b/src/lib/core/include/relocation.h @@ -0,0 +1,24 @@ +/** + * @file relocation.h + */ +#ifndef STASIS_RELOCATION_H +#define STASIS_RELOCATION_H + +#include "config.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#if defined(STASIS_OS_DARWIN) +#include <limits.h> +# else +#include <linux/limits.h> +#endif +#include <unistd.h> + +#define REPLACE_TRUNCATE_AFTER_MATCH 1 + +int replace_text(char *original, const char *target, const char *replacement, unsigned flags); +int file_replace_text(const char* filename, const char* target, const char* replacement, unsigned flags); + +#endif //STASIS_RELOCATION_H diff --git a/src/lib/core/include/rules.h b/src/lib/core/include/rules.h new file mode 100644 index 0000000..666d331 --- /dev/null +++ b/src/lib/core/include/rules.h @@ -0,0 +1,11 @@ +// +// Created by jhunk on 12/18/23. +// + +#ifndef STASIS_RULES_H +#define STASIS_RULES_H + +#include "core.h" + + +#endif //STASIS_RULES_H diff --git a/src/lib/core/include/str.h b/src/lib/core/include/str.h new file mode 100644 index 0000000..bb96db0 --- /dev/null +++ b/src/lib/core/include/str.h @@ -0,0 +1,313 @@ +/** + * @file str.h + */ +#ifndef STASIS_STR_H +#define STASIS_STR_H + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <stdarg.h> +#include <ctype.h> +#include "relocation.h" +#include "core.h" + +#define STASIS_SORT_ALPHA 1 << 0 +#define STASIS_SORT_NUMERIC 1 << 1 +#define STASIS_SORT_LEN_ASCENDING 1 << 2 +#define STASIS_SORT_LEN_DESCENDING 1 << 3 + +/** + * Determine how many times the character `ch` appears in `sptr` string + * @param sptr string to scan + * @param ch character to find + * @return count of characters found + */ +int num_chars(const char *sptr, int ch); + +/** + * Scan for `pattern` string at the beginning of `sptr` + * + * @param sptr string to scan + * @param pattern string to search for + * @return 1 = found, 0 = not found / error + */ +int startswith(const char *sptr, const char *pattern); + +/** + * Scan for `pattern` string at the end of `sptr` + * + * @param sptr string to scan + * @param pattern string to search for + * @return 1 = found, 0 = not found / error + */ +int endswith(const char *sptr, const char *pattern); + +/** + * Deletes any characters matching `chars` from `sptr` string + * + * @param sptr string to be modified in-place + * @param chars a string containing characters (e.g. " \n" would delete whitespace and line feeds) + */ +void strchrdel(char *sptr, const char *chars); + +/** + * Split a string by every delimiter in `delim` string. + * + * Callee should free memory using `GENERIC_ARRAY_FREE()` + * + * @param sptr string to split + * @param delim characters to split on + * @return success=parts of string, failure=NULL + */ +char** split(char *sptr, const char* delim, size_t max); + +/** + * Create new a string from an array of strings + * + * ~~~{.c} + * char *array[] = { + * "this", + * "is", + * "a", + * "test", + * NULL, + * } + * + * char *test = join(array, " "); // "this is a test" + * char *test2 = join(array, "_"); // "this_is_a_test" + * char *test3 = join(array, ", "); // "this, is, a, test" + * + * free(test); + * free(test2); + * free(test3); + * ~~~ + * + * @param arr + * @param separator characters to insert between elements in string + * @return new joined string + */ +char *join(char **arr, const char *separator); + +/** + * Join two or more strings by a `separator` string + * @param separator + * @param ... + * @return string + */ +char *join_ex(char *separator, ...); + +/** + * Extract the string encapsulated by characters listed in `delims` + * + * ~~~{.c} + * char *str = "this is [some data] in a string"; + * char *data = substring_between(string, "[]"); + * // data = "some data"; + * ~~~ + * + * @param sptr string to parse + * @param delims two characters surrounding a string + * @return success=text between delimiters, failure=NULL + */ +char *substring_between(char *sptr, const char *delims); + +/** + * Sort an array of strings + * @param arr a NULL terminated array of strings + * @param sort_mode + * - STASIS_SORT_LEN_DESCENDING + * - STASIS_SORT_LEN_ASCENDING + * - STASIS_SORT_ALPHA + * - STASIS_SORT_NUMERIC + */ +void strsort(char **arr, unsigned int sort_mode); + +/** + * Determine whether the input character is a relational operator + * Note: `~` is non-standard + * @param ch + * @return 0=no, 1=yes + */ +int isrelational(char ch); + +/** + * Print characters in `s`, `len` times + * @param s + * @param len + */ +void print_banner(const char *s, int len); + +/** + * Search for string in an array of strings + * @param arr array of strings + * @param str string to search for + * @return yes=`pointer to string`, no=`NULL`, failure=`NULL` + */ +char *strstr_array(char **arr, const char *str); + +/** + * Remove duplicate strings from an array of strings + * @param arr + * @return success=array of unique strings, failure=NULL + */ +char **strdeldup(char **arr); + +/** Remove leading whitespace from a string + * + * ~~~{.c} + * char input[100]; + * + * strcpy(input, " I had leading spaces"); + * lstrip(input); + * // input is now "I had leading spaces" + * ~~~ + * @param sptr pointer to string + * @return pointer to first non-whitespace character in string + */ +char *lstrip(char *sptr); + +/** + * Strips trailing whitespace from a given string + * + * ~~~{.c} + * char input[100]; + * + * strcpy(input, "I had trailing spaces "); + * strip(input); + * // input is now "I had trailing spaces" + * ~~~ + * + * @param sptr input string + * @return truncated string + */ +char *strip(char *sptr); + +/** + * Check if a given string is "visibly" empty + * + * ~~~{.c} + * char visibly[100]; + * + * strcpy(visibly, "\t \t\n"); + * if (isempty(visibly)) { + * printf("string is 'empty'\n"); + * } else { + * printf("string is not 'empty'\n"); + * } + * ~~~ + * + * @param sptr pointer to string + * @return 0=not empty, 1=empty + */ +int isempty(char *sptr); + +/** + * Determine if a string is encapsulated by quotes + * @param sptr pointer to string + * @return 0=not quoted, 1=quoted + */ +int isquoted(char *sptr); + +/** + * Collapse whitespace in `s`. The string is modified in place. + * @param s + * @return pointer to `s` + */ +char *normalize_space(char *s); + +/** + * Duplicate an array of strings + * + * ~~~{.c} + * char **array_orig = calloc(10, sizeof(*orig)); + * orig[0] = strdup("one"); + * orig[1] = strdup("two"); + * orig[2] = strdup("three"); + * // ... + * char **array_orig_copy = strdup_array(orig); + * + * for (size_t i = 0; array_orig_copy[i] != NULL; i++) { + * printf("array_orig[%zu] = '%s'\narray_orig_copy[%zu] = '%s'\n\n", + * i, array_orig[i], + * i, array_orig_copy[i]); + * free(array_orig_copy[i]); + * free(array_orig[i]); + * } + * free(array_orig_copy); + * free(array_orig); + * + * ~~~ + * + * @param array + * @return + */ +char **strdup_array(char **array); + +/** + * Compare an array of strings + * + * ~~~{.c} + * const char *a[] = { + * "I", + * "like", + * "computers." + * }; + * const char *b[] = { + * "I", + * "like", + * "cars." + * }; + * if (!strcmp_array(a, b)) { + * printf("a and b are not equal\n"); + * } else { + * printf("a and b are equal\n"); + * } + * ~~~ + * + * @param a pointer to array + * @param b poitner to array + * @return 0 on identical, non-zero for different + */ +int strcmp_array(const char **a, const char **b); + +/** + * Determine whether a string is comprised of digits + * @param s + * @return 0=no, 1=yes + */ +int isdigit_s(const char *s); + +/** + * Convert input string to lowercase + * + * ~~~{.c} + * char *str = strdup("HELLO WORLD!"); + * tolower_s(str); + * // str is "hello world!" + * ~~~ + * + * @param s input string + * @return pointer to input string + */ +char *tolower_s(char *s); + +/** + * Return a copy of the input string with "." characters removed + * + * ~~~{.c} + * char *version = strdup("1.2.3"); + * char *version_short = to_short_version(str); + * // version_short is "123" + * free(version_short); + * + * ~~~ + * + * @param s input string + * @return pointer to new string + */ +char *to_short_version(const char *s); + +void unindent(char *s); + +#endif //STASIS_STR_H diff --git a/src/lib/core/include/strlist.h b/src/lib/core/include/strlist.h new file mode 100644 index 0000000..cdbfc01 --- /dev/null +++ b/src/lib/core/include/strlist.h @@ -0,0 +1,60 @@ +/** + * String array convenience functions + * @file strlist.h + */ +#ifndef STASIS_STRLIST_H +#define STASIS_STRLIST_H + +typedef int (ReaderFn)(size_t line, char **); + +#include <stdlib.h> +#include "core.h" +#include "utils.h" +#include "str.h" + + +struct StrList { + size_t num_alloc; + size_t num_inuse; + char **data; +}; + +struct StrList *strlist_init(); +void strlist_remove(struct StrList *pStrList, size_t index); +long double strlist_item_as_long_double(struct StrList *pStrList, size_t index); +double strlist_item_as_double(struct StrList *pStrList, size_t index); +float strlist_item_as_float(struct StrList *pStrList, size_t index); +unsigned long long strlist_item_as_ulong_long(struct StrList *pStrList, size_t index); +long long strlist_item_as_long_long(struct StrList *pStrList, size_t index); +unsigned long strlist_item_as_ulong(struct StrList *pStrList, size_t index); +long strlist_item_as_long(struct StrList *pStrList, size_t index); +unsigned int strlist_item_as_uint(struct StrList *pStrList, size_t index); +int strlist_item_as_int(struct StrList *pStrList, size_t index); +unsigned short strlist_item_as_ushort(struct StrList *pStrList, size_t index); +short strlist_item_as_short(struct StrList *pStrList, size_t index); +unsigned char strlist_item_as_uchar(struct StrList *pStrList, size_t index); +char strlist_item_as_char(struct StrList *pStrList, size_t index); +char *strlist_item_as_str(struct StrList *pStrList, size_t index); +char *strlist_item(struct StrList *pStrList, size_t index); +void strlist_set(struct StrList **pStrList, size_t index, char *value); +size_t strlist_count(struct StrList *pStrList); +void strlist_reverse(struct StrList *pStrList); +void strlist_sort(struct StrList *pStrList, unsigned int mode); +int strlist_append_file(struct StrList *pStrList, char *path, ReaderFn *readerFn); +void strlist_append_strlist(struct StrList *pStrList1, struct StrList *pStrList2); +void strlist_append(struct StrList **pStrList, char *str); +void strlist_append_array(struct StrList *pStrList, char **arr); +void strlist_append_tokenize(struct StrList *pStrList, char *str, char *delim); +struct StrList *strlist_copy(struct StrList *pStrList); +int strlist_cmp(struct StrList *a, struct StrList *b); +void strlist_free(struct StrList **pStrList); + +#define STRLIST_E_SUCCESS 0 +#define STRLIST_E_OUT_OF_RANGE 1 +#define STRLIST_E_INVALID_VALUE 2 +#define STRLIST_E_UNKNOWN 3 +extern int strlist_errno; +const char *strlist_get_error(int flag); + + +#endif //STASIS_STRLIST_H diff --git a/src/lib/core/include/system.h b/src/lib/core/include/system.h new file mode 100644 index 0000000..7019b92 --- /dev/null +++ b/src/lib/core/include/system.h @@ -0,0 +1,34 @@ +/** + * System functions + * @file system.h + */ +#ifndef STASIS_SYSTEM_H +#define STASIS_SYSTEM_H + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <errno.h> +#include <unistd.h> +#include <limits.h> +#include <sys/wait.h> +#include <sys/stat.h> + +#define STASIS_SHELL_SAFE_RESTRICT ";&|()" + +struct Process { + // Write stdout stream to file + char f_stdout[PATH_MAX]; + // Write stderr stream to file + char f_stderr[PATH_MAX]; + // Combine stderr and stdout (into stdout stream) + int redirect_stderr; + // Exit code from program + int returncode; +}; + +int shell(struct Process *proc, char *args); +int shell_safe(struct Process *proc, char *args); +char *shell_output(const char *command, int *status); + +#endif //STASIS_SYSTEM_H diff --git a/src/lib/core/include/template.h b/src/lib/core/include/template.h new file mode 100644 index 0000000..e3d83fb --- /dev/null +++ b/src/lib/core/include/template.h @@ -0,0 +1,81 @@ +//! @file template.h +#ifndef STASIS_TEMPLATE_H +#define STASIS_TEMPLATE_H + +#include "core.h" + +/** + * Map a text value to a pointer in memory + * + * @param key in-text variable name + * @param ptr pointer to string + */ +void tpl_register(char *key, char **ptr); + +/** + * Free the template engine + */ +void tpl_free(); + +/** + * Retrieve the value of a key mapped by the template engine + * @param key string registered by `tpl_register` + * @return a pointer to value, or NULL if the key is not present + */ +char *tpl_getval(char *key); + +/** + * Replaces occurrences of all registered key value pairs in `str` + * @param str the text data to render + * @return a rendered copy of `str`, or NULL. + * The caller is responsible for free()ing memory allocated by this function + */ +char *tpl_render(char *str); + +/** + * Write tpl_render() output to a file + * @param str the text to render + * @param filename the output file name + * @return 0 on success, <0 on error + */ +int tpl_render_to_file(char *str, const char *filename); + +typedef int tplfunc(void *frame, void *data_out); + +struct tplfunc_frame { + char *key; ///< Name of the function + tplfunc *func; ///< Pointer to the function + void *data_in; ///< Pointer to internal data (can be NULL) + int argc; ///< Maximum number of arguments to accept + union { + char **t_char_refptr; ///< &pointer + char *t_char_ptr; ///< pointer + void *t_void_ptr; ///< pointer to void + int *t_int_ptr; ///< pointer to int + unsigned *t_uint_ptr; ///< pointer to unsigned int + float *t_float_ptr; ///< pointer to float + double *t_double_ptr; ///< pointer to double + char t_char; ///< type of char + int t_int; ///< type of int + unsigned t_uint; ///< type of unsigned int + float t_float; ///< type of float + double t_double; ///< type of double + } argv[10]; // accept up to 10 arguments +}; + +/** + * Register a template function + * @param key function name to expose to "func:" interface + * @param tplfunc_ptr pointer to function of type tplfunc + * @param argc number of function arguments to accept + */ +void tpl_register_func(char *key, void *tplfunc_ptr, int argc, void *data_in); + +/** + * Get the function frame associated with a template function + * @param key function name + * @return tplfunc_frame structure + */ +struct tplfunc_frame *tpl_getfunc(char *key); + +#endif //STASIS_TEMPLATE_H diff --git a/src/lib/core/include/template_func_proto.h b/src/lib/core/include/template_func_proto.h new file mode 100644 index 0000000..286ccfb --- /dev/null +++ b/src/lib/core/include/template_func_proto.h @@ -0,0 +1,13 @@ +//! @file template_func_proto.h +#ifndef TEMPLATE_FUNC_PROTO_H +#define TEMPLATE_FUNC_PROTO_H + +#include "template.h" + +int get_github_release_notes_tplfunc_entrypoint(void *frame, void *data_out); +int get_github_release_notes_auto_tplfunc_entrypoint(void *frame, void *data_out); +int get_junitxml_file_entrypoint(void *frame, void *data_out); +int get_basetemp_dir_entrypoint(void *frame, void *data_out); +int tox_run_entrypoint(void *frame, void *data_out); + +#endif //TEMPLATE_FUNC_PROTO_H
\ No newline at end of file diff --git a/src/lib/core/include/utils.h b/src/lib/core/include/utils.h new file mode 100644 index 0000000..87f28cc --- /dev/null +++ b/src/lib/core/include/utils.h @@ -0,0 +1,416 @@ +//! @file utils.h +#ifndef STASIS_UTILS_H +#define STASIS_UTILS_H +#include <stdio.h> +#include <stdlib.h> +#include <dirent.h> +#include <string.h> +#include <unistd.h> +#include <limits.h> +#include <errno.h> +#include "core.h" +#include "copy.h" +#include "system.h" +#include "strlist.h" +#include "utils.h" +#include "ini.h" + +#if defined(STASIS_OS_WINDOWS) +#define PATH_ENV_VAR "path" +#define DIR_SEP "\\" +#define PATH_SEP ";" +#define LINE_SEP "\r\n" +#else +#define PATH_ENV_VAR "PATH" +#define DIR_SEP "/" +#define PATH_SEP ":" +#define LINE_SEP "\n" +#endif + +#define STASIS_XML_PRETTY_PRINT_PROG "xmllint" +#define STASIS_XML_PRETTY_PRINT_ARGS "--format" + +/** + * Change directory. Push path on directory stack. + * + * ```c + * pushd("/somepath"); + * + * FILE fp = fopen("somefile", "w"); // i.e. /somepath/somefile + * fprintf(fp, "Hello world.\n"); + * fclose(fp); + * + * popd(); + * ``` + * + * @param path of directory + * @return 0 on success, -1 on error + */ +int pushd(const char *path); + +/** + * Return from directory. Pop last path from directory stack. + * + * @see pushd + * @return 0 on success, -1 if stack is empty + */ +int popd(void); + +/** + * Expand "~" to the user's home directory + * + * ```c + * char *home = expandpath("~"); // == /home/username + * char *config = expandpath("~/.config"); // == /home/username/.config + * char *nope = expandpath("/tmp/test"); // == /tmp/test + * char *nada = expandpath("/~/broken"); // == /~/broken + * + * free(home); + * free(config); + * free(nope); + * free(nada); + * ``` + * + * @param _path (Must start with a `~`) + * @return success=expanded path or original path, failure=NULL + */ +char *expandpath(const char *_path); + +/** + * Remove a directory tree recursively + * + * ```c + * mkdirs("a/b/c"); + * rmtree("a"); + * // a/b/c is removed + * ``` + * + * @param _path + * @return 0 on success, -1 on error + */ +int rmtree(char *_path); + + +char **file_readlines(const char *filename, size_t start, size_t limit, ReaderFn *readerFn); + +/** + * Strip directory from file name + * Note: Caller is responsible for freeing memory + * + * @param _path + * @return success=file name, failure=NULL + */ +char *path_basename(char *path); + +/** + * Return parent directory of file, or the parent of a directory + * + * @param path + * @return success=directory, failure=empty string + */ +char *path_dirname(char *path); + +/** + * Scan PATH directories for a named program + * @param name program name + * @return path to program, or NULL on error + */ +char *find_program(const char *name); + +/** + * Create an empty file, or update modified timestamp on an existing file + * @param filename file to touch + * @return 0 on success, 1 on error + */ +int touch(const char *filename); + +/** + * Clone a git repository + * + * ```c + * struct Process proc; + * memset(proc, 0, sizeof(proc)); + * + * if (git_clone(&proc, "https://github.com/myuser/myrepo", "./repos", "unstable_branch")) { + * fprintf(stderr, "Failed to clone repository\n"); + * exit(1); + * } + * + * if (pushd("./repos/myrepo")) { + * fprintf(stderr, "Unable to enter repository directory\n"); + * } else { + * // do something with repository + * popd(); + * } + * ``` + * + * @see pushd + * + * @param proc Process struct + * @param url URL (or file system path) of repoistory to clone + * @param destdir destination directory + * @param gitref commit/branch/tag of checkout (NULL will use HEAD of default branch for repo) + * @return exit code from "git" + */ +int git_clone(struct Process *proc, char *url, char *destdir, char *gitref); + +/** + * Git describe wrapper + * @param path to repository + * @return output from "git describe", or NULL on error + */ +char *git_describe(const char *path); + +/** + * Git rev-parse wrapper + * @param path to repository + * @param args to pass to git rev-parse + * @return output from "git rev-parse", or NULL on error + */ +char *git_rev_parse(const char *path, char *args); + +/** + * Helper function to initialize simple STASIS internal path strings + * + * ```c + * char *mypath = NULL; + * + * if (path_store(&mypath, PATH_MAX, "/some", "path")) { + * fprintf(stderr, "Unable to allocate memory for path elements\n"); + * exit(1); + * } + * // mypath is allocated to size PATH_MAX and contains the string: /some/path + * // base+path will truncate at maxlen - 1 + * ``` + * + * @param destptr address of destination string pointer + * @param maxlen maximum length of the path + * @param base path + * @param path to append to base + * @return 0 on success, -1 on error + */ +int path_store(char **destptr, size_t maxlen, const char *base, const char *path); + +#if defined(STASIS_DUMB_TERMINAL) +#define STASIS_COLOR_RED "" +#define STASIS_COLOR_GREEN "" +#define STASIS_COLOR_YELLOW "" +#define STASIS_COLOR_BLUE "" +#define STASIS_COLOR_WHITE "" +#define STASIS_COLOR_RESET "" +#else +//! Set output color to red +#define STASIS_COLOR_RED "\e[1;91m" +//! Set output color to green +#define STASIS_COLOR_GREEN "\e[1;92m" +//! Set output color to yellow +#define STASIS_COLOR_YELLOW "\e[1;93m" +//! Set output color to blue +#define STASIS_COLOR_BLUE "\e[1;94m" +//! Set output color to white +#define STASIS_COLOR_WHITE "\e[1;97m" +//! Reset output color to terminal default +#define STASIS_COLOR_RESET "\e[0;37m\e[0m" +#endif + +#define STASIS_MSG_SUCCESS 0 +//! Suppress printing of the message text +#define STASIS_MSG_NOP 1 << 0 +//! The message is an error +#define STASIS_MSG_ERROR 1 << 1 +//! The message is a warning +#define STASIS_MSG_WARN 1 << 2 +//! The message will be indented once +#define STASIS_MSG_L1 1 << 3 +//! The message will be indented twice +#define STASIS_MSG_L2 1 << 4 +//! The message will be indented thrice +#define STASIS_MSG_L3 1 << 5 +//! The message will only be printed in verbose mode +#define STASIS_MSG_RESTRICT 1 << 6 + +void msg(unsigned type, char *fmt, ...); + +// Enter an interactive shell that ends the program on-exit +void debug_shell(); + +/** + * Creates a temporary file returning an open file pointer via @a fp, and the + * path to the file. The caller is responsible for closing @a fp and + * free()ing the returned file path. + * + * ```c + * FILE *fp = NULL; + * char *tempfile = xmkstemp(&fp, "r+"); + * if (!fp || !tempfile) { + * fprintf(stderr, "Failed to generate temporary file for read/write\n"); + * exit(1); + * } + * ``` + * + * @param fp pointer to FILE (to be initialized) + * @param mode fopen() style file mode string + * @return system path to the temporary file + * @return NULL on failure + */ +char *xmkstemp(FILE **fp, const char *mode); + +/** + * Is the path an empty directory structure? + * + * ```c + * if (isempty_dir("/some/path")) { + * fprintf(stderr, "The directory is is empty!\n"); + * } else { + * printf("The directory contains dirs/files\n"); + * } + * ``` + * + * @param path directory + * @return 0 = no, 1 = yes + */ +int isempty_dir(const char *path); + +/** + * Rewrite an XML file with a pretty printer command + * @param filename path to modify + * @param pretty_print_prog program to call + * @param pretty_print_args arguments to pass to program + * @return 0 on success, -1 on error + */ +int xml_pretty_print_in_place(const char *filename, const char *pretty_print_prog, const char *pretty_print_args); + +/** + * Applies STASIS fixups to a tox ini config + * @param filename path to tox.ini + * @param result path to processed configuration + * @return 0 on success, -1 on error + */ +int fix_tox_conf(const char *filename, char **result); + +char *collapse_whitespace(char **s); + +/** + * Write ***REDACTED*** in dest for each occurrence of to_redacted token present in src + * + * ```c + * char command[PATH_MAX] = {0}; + * char command_redacted[PATH_MAX] = {0}; + * const char *password = "abc123"; + * const char *host = "myhostname"; + * const char *to_redact_case1[] = {password, host, NULL}; + * const char *to_redact_case2[] = {password, "--host", NULL}; + * const char *to_redact_case3[] = {password, "--host", host, NULL}; + * + * sprintf(command, "echo %s | program --host=%s -", password, host); + * + * // CASE 1 + * redact_sensitive(to_redact_case1, command, command_redacted, sizeof(command_redacted) - 1); + * printf("executing: %s\n", command_redacted); + * // User sees: + * // executing: echo ***REDACTED*** | program --host=***REDACTED*** - + * system(command); + * + * // CASE 2 remove an entire argument + * redact_sensitive(to_redact_case2, command, command_redacted, sizeof(command_redacted) - 1); + * printf("executing: %s\n", command_redacted); + * // User sees: + * // executing: echo ***REDACTED*** | program ***REDACTED*** - + * system(command); + * + * // CASE 3 remove it all (noisy) + * redact_sensitive(to_redact_case3, command, command_redacted, sizeof(command_redacted) - 1); + * printf("executing: %s\n", command_redacted); + * // User sees: + * // executing: echo ***REDACTED*** | program ***REDACTED***=***REDACTED*** - + * system(command); + * ``` + * + * @param to_redact array of tokens to redact + * @param src input string + * @param dest output string + * @param maxlen maximum length of dest byte array + * @return 0 on success, -1 on error + */ +int redact_sensitive(const char **to_redact, size_t to_redact_size, char *src, char *dest, size_t maxlen); + +/** + * Given a directory path, return a list of files + * + * ~~~{.c} + * struct StrList *files; + * + * basepath = "."; + * files = listdir(basepath); + * for (size_t i = 0; i < strlist_count(files); i++) { + * char *filename = strlist_item(files, i); + * printf("%s/%s\n", basepath, filename); + * } + * guard_strlist_free(&files); + * ~~~ + * + * @param path of a directory + * @return a StrList containing file names + */ +struct StrList *listdir(const char *path); + +/** + * Get CPU count + * @return CPU count on success, zero on error + */ +long get_cpu_count(); + +/** + * Create all leafs in directory path + * @param _path directory path to create + * @param mode mode_t permissions + * @return + */ +int mkdirs(const char *_path, mode_t mode); + +/** + * Return pointer to a (possible) version specifier + * + * ```c + * char s[] = "abc==1.2.3"; + * char *spec_begin = find_version_spec(s); + * // spec_begin is "==1.2.3" + * + * char package_name[255]; + * char s[] = "abc"; + * char *spec_pos = find_version_spec(s); + * if (spec_pos) { + * strncpy(package_name, spec_pos - s); + * // use spec + * } else { + * // spec not found + * } + * + * @param str a pointer to a buffer containing a package spec (i.e. abc==1.2.3, abc>=1.2.3, abc) + * @return a pointer to the first occurrence of a version spec character + * @return NULL if not found + */ +char *find_version_spec(char *package_name); + +// mode flags for env_manipulate_pathstr +#define PM_APPEND 1 << 0 +#define PM_PREPEND 1 << 1 +#define PM_ONCE 1 << 2 + +/** +* Add paths to the head or tail of an environment variable. +* +* @param key environment variable to manipulate +* @param path to insert (does not need to exist) +* @param mode PM_APPEND `$path:$PATH` +* @param mode PM_PREPEND `$PATH:path` +* @param mode PM_ONCE do not manipulate if `path` is present in PATH variable +*/ +int env_manipulate_pathstr(const char *key, char *path, int mode); + +/** +* Append or replace a file extension +*/ +int gen_file_extension_str(char *filename, const char *extension); + +#endif //STASIS_UTILS_H diff --git a/src/lib/core/include/wheel.h b/src/lib/core/include/wheel.h new file mode 100644 index 0000000..1a689e9 --- /dev/null +++ b/src/lib/core/include/wheel.h @@ -0,0 +1,36 @@ +//! @file wheel.h +#ifndef STASIS_WHEEL_H +#define STASIS_WHEEL_H + +#include <dirent.h> +#include <string.h> +#include <stdio.h> +#include "str.h" +#define WHEEL_MATCH_EXACT 0 ///< Match when all patterns are present +#define WHEEL_MATCH_ANY 1 ///< Match when any patterns are present + +struct Wheel { + char *distribution; ///< Package name + char *version; ///< Package version + char *build_tag; ///< Package build tag (optional) + char *python_tag; ///< Package Python tag (pyXY) + char *abi_tag; ///< Package ABI tag (cpXY, abiX, none) + char *platform_tag; ///< Package platform tag (linux_x86_64, any) + char *path_name; ///< Path to package on-disk + char *file_name; ///< Name of package on-disk +}; + +/** + * Extract metadata from a Python Wheel file name + * + * @param basepath directory containing a wheel file + * @param name of wheel file + * @param to_match a NULL terminated array of patterns (i.e. platform, arch, version, etc) + * @param match_mode WHEEL_MATCH_EXACT + * @param match_mode WHEEL_MATCH ANY + * @return pointer to populated Wheel on success + * @return NULL on error + */ +struct Wheel *get_wheel_info(const char *basepath, const char *name, char *to_match[], unsigned match_mode); +void wheel_free(struct Wheel **wheel); +#endif //STASIS_WHEEL_H |