From 5a9688e9e78a25a42bddfc4388fb4ce3311ded74 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Mon, 14 Oct 2024 09:32:03 -0400 Subject: Refactor directory structure * Move core library sources into src/lib/core * Move command-line programs into src/cli --- src/CMakeLists.txt | 47 +- src/artifactory.c | 496 ----------------- src/cli/CMakeLists.txt | 2 + src/cli/stasis/CMakeLists.txt | 7 + src/cli/stasis/stasis_main.c | 797 +++++++++++++++++++++++++++ src/cli/stasis_indexer/CMakeLists.txt | 6 + src/cli/stasis_indexer/stasis_indexer.c | 949 ++++++++++++++++++++++++++++++++ src/conda.c | 466 ---------------- src/copy.c | 86 --- src/delivery.c | 317 ----------- src/delivery_artifactory.c | 192 ------- src/delivery_build.c | 191 ------- src/delivery_conda.c | 110 ---- src/delivery_docker.c | 132 ----- src/delivery_init.c | 345 ------------ src/delivery_install.c | 224 -------- src/delivery_populate.c | 348 ------------ src/delivery_postprocess.c | 266 --------- src/delivery_show.c | 117 ---- src/delivery_test.c | 295 ---------- src/docker.c | 205 ------- src/download.c | 61 -- src/envctl.c | 125 ----- src/environment.c | 443 --------------- src/github.c | 133 ----- src/globals.c | 65 --- src/ini.c | 678 ----------------------- src/junitxml.c | 240 -------- src/lib/CMakeLists.txt | 1 + src/lib/core/CMakeLists.txt | 38 ++ src/lib/core/artifactory.c | 496 +++++++++++++++++ src/lib/core/conda.c | 465 ++++++++++++++++ src/lib/core/copy.c | 86 +++ src/lib/core/delivery.c | 317 +++++++++++ src/lib/core/delivery_artifactory.c | 192 +++++++ src/lib/core/delivery_build.c | 190 +++++++ src/lib/core/delivery_conda.c | 110 ++++ src/lib/core/delivery_docker.c | 132 +++++ src/lib/core/delivery_init.c | 345 ++++++++++++ src/lib/core/delivery_install.c | 224 ++++++++ src/lib/core/delivery_populate.c | 348 ++++++++++++ src/lib/core/delivery_postprocess.c | 266 +++++++++ src/lib/core/delivery_show.c | 117 ++++ src/lib/core/delivery_test.c | 295 ++++++++++ src/lib/core/docker.c | 204 +++++++ src/lib/core/download.c | 59 ++ src/lib/core/envctl.c | 124 +++++ src/lib/core/environment.c | 443 +++++++++++++++ src/lib/core/github.c | 134 +++++ src/lib/core/globals.c | 66 +++ src/lib/core/ini.c | 678 +++++++++++++++++++++++ src/lib/core/junitxml.c | 240 ++++++++ src/lib/core/multiprocessing.c | 449 +++++++++++++++ src/lib/core/recipe.c | 64 +++ src/lib/core/relocation.c | 155 ++++++ src/lib/core/rules.c | 5 + src/lib/core/str.c | 654 ++++++++++++++++++++++ src/lib/core/strlist.c | 659 ++++++++++++++++++++++ src/lib/core/system.c | 173 ++++++ src/lib/core/template.c | 318 +++++++++++ src/lib/core/template_func_proto.c | 160 ++++++ src/lib/core/utils.c | 820 +++++++++++++++++++++++++++ src/lib/core/wheel.c | 126 +++++ src/multiprocessing.c | 448 --------------- src/recipe.c | 64 --- src/relocation.c | 155 ------ src/rules.c | 5 - src/stasis_indexer.c | 948 ------------------------------- src/stasis_main.c | 794 -------------------------- src/str.c | 654 ---------------------- src/strlist.c | 658 ---------------------- src/system.c | 173 ------ src/template.c | 318 ----------- src/template_func_proto.c | 158 ------ src/utils.c | 819 --------------------------- src/wheel.c | 126 ----- 76 files changed, 10916 insertions(+), 10900 deletions(-) delete mode 100644 src/artifactory.c create mode 100644 src/cli/CMakeLists.txt create mode 100644 src/cli/stasis/CMakeLists.txt create mode 100644 src/cli/stasis/stasis_main.c create mode 100644 src/cli/stasis_indexer/CMakeLists.txt create mode 100644 src/cli/stasis_indexer/stasis_indexer.c delete mode 100644 src/conda.c delete mode 100644 src/copy.c delete mode 100644 src/delivery.c delete mode 100644 src/delivery_artifactory.c delete mode 100644 src/delivery_build.c delete mode 100644 src/delivery_conda.c delete mode 100644 src/delivery_docker.c delete mode 100644 src/delivery_init.c delete mode 100644 src/delivery_install.c delete mode 100644 src/delivery_populate.c delete mode 100644 src/delivery_postprocess.c delete mode 100644 src/delivery_show.c delete mode 100644 src/delivery_test.c delete mode 100644 src/docker.c delete mode 100644 src/download.c delete mode 100644 src/envctl.c delete mode 100644 src/environment.c delete mode 100644 src/github.c delete mode 100644 src/globals.c delete mode 100644 src/ini.c delete mode 100644 src/junitxml.c create mode 100644 src/lib/CMakeLists.txt create mode 100644 src/lib/core/CMakeLists.txt create mode 100644 src/lib/core/artifactory.c create mode 100644 src/lib/core/conda.c create mode 100644 src/lib/core/copy.c create mode 100644 src/lib/core/delivery.c create mode 100644 src/lib/core/delivery_artifactory.c create mode 100644 src/lib/core/delivery_build.c create mode 100644 src/lib/core/delivery_conda.c create mode 100644 src/lib/core/delivery_docker.c create mode 100644 src/lib/core/delivery_init.c create mode 100644 src/lib/core/delivery_install.c create mode 100644 src/lib/core/delivery_populate.c create mode 100644 src/lib/core/delivery_postprocess.c create mode 100644 src/lib/core/delivery_show.c create mode 100644 src/lib/core/delivery_test.c create mode 100644 src/lib/core/docker.c create mode 100644 src/lib/core/download.c create mode 100644 src/lib/core/envctl.c create mode 100644 src/lib/core/environment.c create mode 100644 src/lib/core/github.c create mode 100644 src/lib/core/globals.c create mode 100644 src/lib/core/ini.c create mode 100644 src/lib/core/junitxml.c create mode 100644 src/lib/core/multiprocessing.c create mode 100644 src/lib/core/recipe.c create mode 100644 src/lib/core/relocation.c create mode 100644 src/lib/core/rules.c create mode 100644 src/lib/core/str.c create mode 100644 src/lib/core/strlist.c create mode 100644 src/lib/core/system.c create mode 100644 src/lib/core/template.c create mode 100644 src/lib/core/template_func_proto.c create mode 100644 src/lib/core/utils.c create mode 100644 src/lib/core/wheel.c delete mode 100644 src/multiprocessing.c delete mode 100644 src/recipe.c delete mode 100644 src/relocation.c delete mode 100644 src/rules.c delete mode 100644 src/stasis_indexer.c delete mode 100644 src/stasis_main.c delete mode 100644 src/str.c delete mode 100644 src/strlist.c delete mode 100644 src/system.c delete mode 100644 src/template.c delete mode 100644 src/template_func_proto.c delete mode 100644 src/utils.c delete mode 100644 src/wheel.c (limited to 'src') diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5c5bc6e..bfee276 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -2,49 +2,6 @@ include_directories(${CMAKE_BINARY_DIR}/include) include_directories(${CMAKE_SOURCE_DIR}/include) include_directories(${PROJECT_BINARY_DIR}) -add_library(stasis_core STATIC - globals.c - str.c - strlist.c - ini.c - conda.c - environment.c - utils.c - system.c - download.c - delivery_postprocess.c - delivery_conda.c - delivery_docker.c - delivery_install.c - delivery_artifactory.c - delivery_test.c - delivery_build.c - delivery_show.c - delivery_populate.c - delivery_init.c - delivery.c - recipe.c - relocation.c - wheel.c - copy.c - artifactory.c - template.c - rules.c - docker.c - junitxml.c - github.c - template_func_proto.c - envctl.c - multiprocessing.c -) +add_subdirectory(lib) +add_subdirectory(cli) -add_executable(stasis - stasis_main.c -) -target_link_libraries(stasis PRIVATE stasis_core) -target_link_libraries(stasis PUBLIC LibXml2::LibXml2) -add_executable(stasis_indexer - stasis_indexer.c -) -target_link_libraries(stasis_indexer PRIVATE stasis_core) -install(TARGETS stasis stasis_indexer RUNTIME) diff --git a/src/artifactory.c b/src/artifactory.c deleted file mode 100644 index 6c4079a..0000000 --- a/src/artifactory.c +++ /dev/null @@ -1,496 +0,0 @@ -#include "core.h" - -extern struct STASIS_GLOBAL globals; - -int artifactory_download_cli(char *dest, - char *jfrog_artifactory_base_url, - char *jfrog_artifactory_product, - char *cli_major_ver, - char *version, - char *os, - char *arch, - char *remote_filename) { - char url[PATH_MAX] = {0}; - char path[PATH_MAX] = {0}; - char os_ident[STASIS_NAME_MAX] = {0}; - char arch_ident[STASIS_NAME_MAX] = {0}; - - // convert platform string to lower-case - strcpy(os_ident, os); - tolower_s(os_ident); - - // translate OS identifier - if (!strcmp(os_ident, "darwin") || startswith(os_ident, "macos")) { - strcpy(os_ident, "mac"); - } else if (!strcmp(os_ident, "linux")) { - strcpy(os_ident, "linux"); - } else { - fprintf(stderr, "%s: unknown operating system: %s\n", __FUNCTION__, os_ident); - return -1; - } - - // translate ARCH identifier - strcpy(arch_ident, arch); - if (startswith(arch_ident, "i") && endswith(arch_ident, "86")) { - strcpy(arch_ident, "386"); - } else if (!strcmp(arch_ident, "amd64") || !strcmp(arch_ident, "x86_64") || !strcmp(arch_ident, "x64")) { - if (!strcmp(os_ident, "mac")) { - strcpy(arch_ident, "386"); - } else { - strcpy(arch_ident, "amd64"); - } - } else if (!strcmp(arch_ident, "arm64") || !strcmp(arch_ident, "aarch64")) { - strcpy(arch_ident, "arm64"); - } else { - fprintf(stderr, "%s: unknown architecture: %s\n", __FUNCTION__, arch_ident); - return -1; - } - - snprintf(url, sizeof(url) - 1, "%s/%s/%s/%s/%s-%s-%s/%s", - jfrog_artifactory_base_url, // https://releases.jfrog.io/artifactory - jfrog_artifactory_product, // jfrog-cli - cli_major_ver, // v\d+(-jf)? - version, // 1.2.3 - jfrog_artifactory_product, // ... - os_ident, // ... - arch_ident, // jfrog-cli-linux-x86_64 - remote_filename); // jf - strcpy(path, dest); - - if (mkdirs(path, 0755)) { - fprintf(stderr, "%s: %s: %s", __FUNCTION__, path, strerror(errno)); - return -1; - } - - sprintf(path + strlen(path), "/%s", remote_filename); - long fetch_status = download(url, path, NULL); - if (HTTP_ERROR(fetch_status) || fetch_status < 0) { - fprintf(stderr, "%s: download failed: %s\n", __FUNCTION__, url); - return -1; - } - chmod(path, 0755); - return 0; -} - -void jfrt_register_opt_str(char *jfrt_val, const char *opt_name, struct StrList **opt_map) { - char data[STASIS_BUFSIZ]; - memset(data, 0, sizeof(data)); - - if (jfrt_val == NULL) { - // no data - return; - } - snprintf(data, sizeof(data) - 1, "--%s=\"%s\"", opt_name, jfrt_val); - strlist_append(&*opt_map, data); -} - -void jfrt_register_opt_bool(bool jfrt_val, const char *opt_name, struct StrList **opt_map) { - char data[STASIS_BUFSIZ]; - memset(data, 0, sizeof(data)); - - if (jfrt_val == false) { - // option will not be used - return; - } - snprintf(data, sizeof(data) - 1, "--%s", opt_name); - strlist_append(&*opt_map, data); -} - -void jfrt_register_opt_int(int jfrt_val, const char *opt_name, struct StrList **opt_map) { - char data[STASIS_BUFSIZ]; - memset(data, 0, sizeof(data)); - - if (jfrt_val == 0) { - // option will not be used - return; - } - snprintf(data, sizeof(data) - 1, "--%s=%d", opt_name, jfrt_val); - strlist_append(&*opt_map, data); -} - -void jfrt_register_opt_long(long jfrt_val, const char *opt_name, struct StrList **opt_map) { - char data[STASIS_BUFSIZ]; - memset(data, 0, sizeof(data)); - - if (jfrt_val == 0) { - // option will not be used - return; - } - snprintf(data, sizeof(data) - 1, "--%s=%ld", opt_name, jfrt_val); - strlist_append(&*opt_map, data); -} - -void jfrt_upload_init(struct JFRT_Upload *ctx) { - memset(ctx, 0, sizeof(*ctx)); - ctx->recursive = true; - ctx->threads = 3; - ctx->retries = 3; -} - -static int auth_required(const char *cmd) { - const char *modes[] = { - "build-collect-env", - NULL, - }; - for (size_t i = 0; modes[i] != NULL; i++) { - if (!startswith(cmd, modes[i])) { - return 1; - } - } - return 0; -} - -int jfrt_auth_init(struct JFRT_Auth *auth_ctx) { - char *url = getenv("STASIS_JF_ARTIFACTORY_URL"); - char *user = getenv("STASIS_JF_USER"); - char *access_token = getenv("STASIS_JF_ACCESS_TOKEN"); - char *password = getenv("STASIS_JF_PASSWORD"); - char *ssh_key_path = getenv("STASIS_JF_SSH_KEY_PATH"); - char *ssh_passphrase = getenv("STASIS_JF_SSH_PASSPHRASE"); - char *client_cert_key_path = getenv("STASIS_JF_CLIENT_CERT_KEY_PATH"); - char *client_cert_path = getenv("STASIS_JF_CLIENT_CERT_PATH"); - - if (!url) { - fprintf(stderr, "Artifactory URL is not configured:\n"); - fprintf(stderr, "please set STASIS_JF_ARTIFACTORY_URL\n"); - return -1; - } - auth_ctx->url = url; - - if (access_token) { - auth_ctx->user = NULL; - auth_ctx->access_token = access_token; - auth_ctx->password = NULL; - auth_ctx->ssh_key_path = NULL; - } else if (user && password) { - auth_ctx->user = user; - auth_ctx->password = password; - auth_ctx->access_token = NULL; - auth_ctx->ssh_key_path = NULL; - } else if (ssh_key_path) { - auth_ctx->user = NULL; - auth_ctx->ssh_key_path = ssh_key_path; - if (ssh_passphrase) { - auth_ctx->ssh_passphrase = ssh_passphrase; - } - auth_ctx->password = NULL; - auth_ctx->access_token = NULL; - } else if (client_cert_key_path && client_cert_path) { - auth_ctx->user = NULL; - auth_ctx->password = NULL; - auth_ctx->access_token = NULL; - auth_ctx->ssh_key_path = NULL; - auth_ctx->client_cert_key_path = client_cert_key_path; - auth_ctx->client_cert_path = client_cert_path; - } else { - fprintf(stderr, "Artifactory authentication is not configured:\n"); - fprintf(stderr, "set STASIS_JF_USER and STASIS_JF_PASSWORD\n"); - fprintf(stderr, "or, set STASIS_JF_ACCESS_TOKEN\n"); - fprintf(stderr, "or, set STASIS_JF_SSH_KEY_PATH and STASIS_JF_SSH_KEY_PASSPHRASE\n"); - fprintf(stderr, "or, set STASIS_JF_CLIENT_CERT_KEY_PATH and STASIS_JF_CLIENT_CERT_PATH\n"); - return -1; - } - return 0; -} - -int jfrog_cli(struct JFRT_Auth *auth, const char *subsystem, const char *task, char *args) { - struct Process proc; - char cmd[STASIS_BUFSIZ]; - char cmd_redacted[STASIS_BUFSIZ]; - int status; - - memset(&proc, 0, sizeof(proc)); - memset(cmd, 0, sizeof(cmd)); - memset(cmd_redacted, 0, sizeof(cmd_redacted)); - - struct StrList *arg_map = strlist_init(); - if (!arg_map) { - return -1; - } - - char *auth_args = NULL; - if (auth_required(task)) { - // String options - jfrt_register_opt_str(auth->url, "url", &arg_map); - jfrt_register_opt_str(auth->user, "user", &arg_map); - jfrt_register_opt_str(auth->access_token, "access-token", &arg_map); - jfrt_register_opt_str(auth->password, "password", &arg_map); - jfrt_register_opt_str(auth->ssh_key_path, "ssh-key-path", &arg_map); - jfrt_register_opt_str(auth->ssh_passphrase, "ssh-passphrase", &arg_map); - jfrt_register_opt_str(auth->client_cert_key_path, "client-cert-key-path", &arg_map); - jfrt_register_opt_str(auth->client_cert_path, "client-cert-path", &arg_map); - jfrt_register_opt_bool(auth->insecure_tls, "insecure-tls", &arg_map); - jfrt_register_opt_str(auth->server_id, "server-id", &arg_map); - } - - auth_args = join(arg_map->data, " "); - if (!auth_args) { - return -1; - } - - const char *redactable[] = { - auth->access_token, - auth->ssh_key_path, - auth->ssh_passphrase, - auth->client_cert_key_path, - auth->client_cert_path, - auth->password, - }; - snprintf(cmd, sizeof(cmd) - 1, "jf %s %s %s %s", subsystem, task, auth_args, args ? args : ""); - redact_sensitive(redactable, sizeof(redactable) / sizeof (*redactable), cmd, cmd_redacted, sizeof(cmd_redacted) - 1); - - guard_free(auth_args); - guard_strlist_free(&arg_map); - - // Pings are noisy. Squelch them. - if (task && !strstr(task, "ping")) { - msg(STASIS_MSG_L2, "Executing: %s\n", cmd_redacted); - } - - if (!globals.verbose) { - strcpy(proc.f_stdout, "/dev/null"); - strcpy(proc.f_stderr, "/dev/null"); - } - status = shell(&proc, cmd); - return status; -} - -static int jfrog_cli_rt(struct JFRT_Auth *auth, char *task, char *args) { - return jfrog_cli(auth, "rt", task, args); -} - -int jfrog_cli_rt_build_collect_env(struct JFRT_Auth *auth, char *build_name, char *build_number) { - char cmd[STASIS_BUFSIZ]; - memset(cmd, 0, sizeof(cmd)); - snprintf(cmd, sizeof(cmd) - 1, "\"%s\" \"%s\"", build_name, build_number); - return jfrog_cli(auth, "rt", "build-collect-env", cmd); -} - -int jfrog_cli_rt_build_publish(struct JFRT_Auth *auth, char *build_name, char *build_number) { - char cmd[STASIS_BUFSIZ]; - memset(cmd, 0, sizeof(cmd)); - snprintf(cmd, sizeof(cmd) - 1, "\"%s\" \"%s\"", build_name, build_number); - return jfrog_cli(auth, "rt", "build-publish", cmd); -} - -int jfrog_cli_rt_ping(struct JFRT_Auth *auth) { - return jfrog_cli_rt(auth, "ping", NULL); -} - -int jfrog_cli_rt_download(struct JFRT_Auth *auth, struct JFRT_Download *ctx, char *repo_path, char *dest) { - char cmd[STASIS_BUFSIZ]; - memset(cmd, 0, sizeof(cmd)); - - if (isempty(repo_path)) { - fprintf(stderr, "repo_path argument must be a valid artifactory repository path\n"); - return -1; - } - - // dest is an optional argument, therefore may be NULL or an empty string - - struct StrList *arg_map = strlist_init(); - if (!arg_map) { - return -1; - } - - jfrt_register_opt_str(ctx->archive_entries, "archive-entries", &arg_map); - jfrt_register_opt_str(ctx->build, "build", &arg_map); - jfrt_register_opt_str(ctx->build_name, "build-name", &arg_map); - jfrt_register_opt_str(ctx->build_number, "build-number", &arg_map); - jfrt_register_opt_str(ctx->bundle, "bundle", &arg_map); - jfrt_register_opt_str(ctx->exclude_artifacts, "exclude-artifacts", &arg_map); - jfrt_register_opt_str(ctx->exclude_props, "exclude-props", &arg_map); - jfrt_register_opt_str(ctx->exclusions, "exclusions", &arg_map); - jfrt_register_opt_str(ctx->gpg_key, "gpg-key", &arg_map); - jfrt_register_opt_str(ctx->include_deps, "include-deps", &arg_map); - jfrt_register_opt_str(ctx->include_dirs, "include-dirs", &arg_map); - jfrt_register_opt_str(ctx->module, "module", &arg_map); - jfrt_register_opt_str(ctx->project, "project", &arg_map); - jfrt_register_opt_str(ctx->props, "props", &arg_map); - jfrt_register_opt_str(ctx->sort_by, "sort-by", &arg_map); - jfrt_register_opt_str(ctx->sort_order, "sort-order", &arg_map); - jfrt_register_opt_str(ctx->spec, "spec", &arg_map); - jfrt_register_opt_str(ctx->spec_vars, "spec-vars", &arg_map); - - jfrt_register_opt_bool(ctx->detailed_summary, "detailed-summary", &arg_map); - jfrt_register_opt_bool(ctx->dry_run, "dry-run", &arg_map); - jfrt_register_opt_bool(ctx->explode, "explode", &arg_map); - jfrt_register_opt_bool(ctx->fail_no_op, "fail-no-op", &arg_map); - jfrt_register_opt_bool(ctx->flat, "flat", &arg_map); - jfrt_register_opt_bool(ctx->quiet, "quiet", &arg_map); - jfrt_register_opt_bool(ctx->recursive, "recursive", &arg_map); - jfrt_register_opt_bool(ctx->retries, "retries", &arg_map); - jfrt_register_opt_bool(ctx->retry_wait_time, "retry-wait-time", &arg_map); - jfrt_register_opt_bool(ctx->skip_checksum, "skip-checksum", &arg_map); - - jfrt_register_opt_int(ctx->limit, "limit", &arg_map); - jfrt_register_opt_int(ctx->min_split, "min-split", &arg_map); - jfrt_register_opt_int(ctx->offset, "offset", &arg_map); - jfrt_register_opt_int(ctx->split_count, "split-count", &arg_map); - jfrt_register_opt_int(ctx->sync_deletes, "sync-deletes", &arg_map); - jfrt_register_opt_int(ctx->threads, "threads", &arg_map); - jfrt_register_opt_int(ctx->validate_symlinks, "validate-symlinks", &arg_map); - - char *args = join(arg_map->data, " "); - if (!args) { - return -1; - } - - snprintf(cmd, sizeof(cmd) - 1, "%s '%s' %s", args, repo_path, dest ? dest : ""); - guard_free(args); - guard_strlist_free(&arg_map); - - int status = jfrog_cli_rt(auth, "download", cmd); - return status; -} - -int jfrog_cli_rt_upload(struct JFRT_Auth *auth, struct JFRT_Upload *ctx, char *src, char *repo_path) { - char cmd[STASIS_BUFSIZ]; - memset(cmd, 0, sizeof(cmd)); - - if (isempty(src)) { - fprintf(stderr, "src argument must be a valid file system path\n"); - return -1; - } - - if (isempty(repo_path)) { - fprintf(stderr, "repo_path argument must be a valid artifactory repository path\n"); - return -1; - } - - struct StrList *arg_map = strlist_init(); - if (!arg_map) { - return -1; - } - - // String options - jfrt_register_opt_str(ctx->build_name, "build-name", &arg_map); - jfrt_register_opt_str(ctx->build_number, "build-number", &arg_map); - jfrt_register_opt_str(ctx->exclusions, "exclusions", &arg_map); - jfrt_register_opt_str(ctx->module, "module", &arg_map); - jfrt_register_opt_str(ctx->spec, "spec", &arg_map); - jfrt_register_opt_str(ctx->spec_vars, "spec-vars", &arg_map); - jfrt_register_opt_str(ctx->project, "project", &arg_map); - jfrt_register_opt_str(ctx->target_props, "target-props", &arg_map); - - // Boolean options - jfrt_register_opt_bool(ctx->quiet, "quiet", &arg_map); - jfrt_register_opt_bool(ctx->ant, "ant", &arg_map); - jfrt_register_opt_bool(ctx->archive, "archive", &arg_map); - jfrt_register_opt_bool(ctx->deb, "deb", &arg_map); - jfrt_register_opt_bool(ctx->detailed_summary, "detailed-summary", &arg_map); - jfrt_register_opt_bool(ctx->dry_run, "dry-run", &arg_map); - jfrt_register_opt_bool(ctx->explode, "explode", &arg_map); - jfrt_register_opt_bool(ctx->fail_no_op, "fail-no-op", &arg_map); - jfrt_register_opt_bool(ctx->flat, "flat", &arg_map); - jfrt_register_opt_bool(ctx->include_dirs, "include-dirs", &arg_map); - jfrt_register_opt_bool(ctx->recursive, "recursive", &arg_map); - jfrt_register_opt_bool(ctx->symlinks, "symlinks", &arg_map); - jfrt_register_opt_bool(ctx->sync_deletes, "sync-deletes", &arg_map); - jfrt_register_opt_bool(ctx->regexp, "regexp", &arg_map); - - // Integer options - jfrt_register_opt_int(ctx->retries, "retries", &arg_map); - jfrt_register_opt_int(ctx->retry_wait_time, "retry-wait-time", &arg_map); - jfrt_register_opt_int(ctx->threads, "threads", &arg_map); - - char *args = join(arg_map->data, " "); - if (!args) { - return -1; - } - - char *new_src = NULL; - char *base = NULL; - if (ctx->workaround_parent_only) { - struct StrList *components = strlist_init(); - - strlist_append_tokenize(components, src, "/"); - int max_components = (int) strlist_count(components); - for (int i = 0; i < max_components; i++) { - if (strstr(components->data[i], "*")) { - max_components = i; - break; - } - } - base = join(&components->data[max_components], "/"); - guard_free(components->data[max_components]); - new_src = join(components->data, "/"); - guard_strlist_free(&components); - } - - if (new_src) { - if (base) { - src = base; - } else { - strcat(src, "/"); - } - pushd(new_src); - } - - snprintf(cmd, sizeof(cmd) - 1, "%s '%s' \"%s\"", args, src, repo_path); - guard_free(args); - guard_strlist_free(&arg_map); - - int status = jfrog_cli_rt(auth, "upload", cmd); - if (new_src) { - popd(); - guard_free(new_src); - } - if (base) { - guard_free(base); - } - - return status; -} - -int jfrog_cli_rt_search(struct JFRT_Auth *auth, struct JFRT_Search *ctx, char *repo_path, char *pattern) { - char cmd[STASIS_BUFSIZ]; - memset(cmd, 0, sizeof(cmd)); - - if (isempty(repo_path)) { - fprintf(stderr, "repo_path argument must be a valid artifactory repository path\n"); - return -1; - } - - struct StrList *arg_map = strlist_init(); - if (!arg_map) { - return -1; - } - - jfrt_register_opt_str(ctx->archive_entries, "archive-entries", &arg_map); - jfrt_register_opt_str(ctx->build, "build", &arg_map); - jfrt_register_opt_str(ctx->bundle, "bundle", &arg_map); - jfrt_register_opt_str(ctx->exclusions, "exclusions", &arg_map); - jfrt_register_opt_str(ctx->exclude_patterns, "exclude-patterns", &arg_map); - jfrt_register_opt_str(ctx->exclude_artifacts, "exclude-artifacts", &arg_map); - jfrt_register_opt_str(ctx->exclude_props, "exclude-props", &arg_map); - jfrt_register_opt_str(ctx->include, "include", &arg_map); - jfrt_register_opt_str(ctx->include_deps, "include_deps", &arg_map); - jfrt_register_opt_str(ctx->include_dirs, "include_dirs", &arg_map); - jfrt_register_opt_str(ctx->project, "project", &arg_map); - jfrt_register_opt_str(ctx->props, "props", &arg_map); - jfrt_register_opt_str(ctx->sort_by, "sort-by", &arg_map); - jfrt_register_opt_str(ctx->sort_order, "sort-order", &arg_map); - jfrt_register_opt_str(ctx->spec, "spec", &arg_map); - jfrt_register_opt_str(ctx->spec_vars, "spec-vars", &arg_map); - - jfrt_register_opt_bool(ctx->count, "count", &arg_map); - jfrt_register_opt_bool(ctx->fail_no_op, "fail-no-op", &arg_map); - jfrt_register_opt_bool(ctx->recursive, "recursive", &arg_map); - jfrt_register_opt_bool(ctx->transitive, "transitive", &arg_map); - - jfrt_register_opt_int(ctx->limit, "limit", &arg_map); - jfrt_register_opt_int(ctx->offset, "offset", &arg_map); - - char *args = join(arg_map->data, " "); - if (!args) { - return -1; - } - - snprintf(cmd, sizeof(cmd) - 1, "%s '%s/%s'", args, repo_path, pattern ? pattern: ""); - guard_free(args); - guard_strlist_free(&arg_map); - - int status = jfrog_cli_rt(auth, "search", cmd); - return status; -} diff --git a/src/cli/CMakeLists.txt b/src/cli/CMakeLists.txt new file mode 100644 index 0000000..92a21b7 --- /dev/null +++ b/src/cli/CMakeLists.txt @@ -0,0 +1,2 @@ +add_subdirectory(stasis) +add_subdirectory(stasis_indexer) \ No newline at end of file diff --git a/src/cli/stasis/CMakeLists.txt b/src/cli/stasis/CMakeLists.txt new file mode 100644 index 0000000..3766f6d --- /dev/null +++ b/src/cli/stasis/CMakeLists.txt @@ -0,0 +1,7 @@ +add_executable(stasis + stasis_main.c +) +target_link_libraries(stasis PRIVATE stasis_core) +target_link_libraries(stasis PUBLIC LibXml2::LibXml2) + +install(TARGETS stasis RUNTIME) diff --git a/src/cli/stasis/stasis_main.c b/src/cli/stasis/stasis_main.c new file mode 100644 index 0000000..75482c5 --- /dev/null +++ b/src/cli/stasis/stasis_main.c @@ -0,0 +1,797 @@ +#include +#include +#include +#include +#include +#include "core.h" +#include "envctl.h" +#include "delivery.h" +#include "template_func_proto.h" + +#define OPT_ALWAYS_UPDATE_BASE 1000 +#define OPT_NO_DOCKER 1001 +#define OPT_NO_ARTIFACTORY 1002 +#define OPT_NO_ARTIFACTORY_BUILD_INFO 1003 +#define OPT_NO_TESTING 1004 +#define OPT_OVERWRITE 1005 +#define OPT_NO_REWRITE_SPEC_STAGE_2 1006 +#define OPT_FAIL_FAST 1007 +#define OPT_NO_PARALLEL 1008 +#define OPT_POOL_STATUS_INTERVAL 1009 + +static struct option long_options[] = { + {"help", no_argument, 0, 'h'}, + {"version", no_argument, 0, 'V'}, + {"continue-on-error", no_argument, 0, 'C'}, + {"config", required_argument, 0, 'c'}, + {"cpu-limit", required_argument, 0, 'l'}, + {"pool-status-interval", required_argument, 0, OPT_POOL_STATUS_INTERVAL}, + {"python", required_argument, 0, 'p'}, + {"verbose", no_argument, 0, 'v'}, + {"unbuffered", no_argument, 0, 'U'}, + {"update-base", no_argument, 0, OPT_ALWAYS_UPDATE_BASE}, + {"fail-fast", no_argument, 0, OPT_FAIL_FAST}, + {"overwrite", no_argument, 0, OPT_OVERWRITE}, + {"no-docker", no_argument, 0, OPT_NO_DOCKER}, + {"no-artifactory", no_argument, 0, OPT_NO_ARTIFACTORY}, + {"no-artifactory-build-info", no_argument, 0, OPT_NO_ARTIFACTORY_BUILD_INFO}, + {"no-testing", no_argument, 0, OPT_NO_TESTING}, + {"no-parallel", no_argument, 0, OPT_NO_PARALLEL}, + {"no-rewrite", no_argument, 0, OPT_NO_REWRITE_SPEC_STAGE_2}, + {0, 0, 0, 0}, +}; + +const char *long_options_help[] = { + "Display this usage statement", + "Display program version", + "Allow tests to fail", + "Read configuration file", + "Number of processes to spawn concurrently (default: cpus - 1)", + "Report task status every n seconds (default: 30)", + "Override version of Python in configuration", + "Increase output verbosity", + "Disable line buffering", + "Update conda installation prior to STASIS environment creation", + "On error, immediately terminate all tasks", + "Overwrite an existing release", + "Do not build docker images", + "Do not upload artifacts to Artifactory", + "Do not upload build info objects to Artifactory", + "Do not execute test scripts", + "Do not execute tests in parallel", + "Do not rewrite paths and URLs in output files", + NULL, +}; + +static int get_option_max_width(struct option option[]) { + int i = 0; + int max = 0; + const int indent = 4; + while (option[i].name != 0) { + int len = (int) strlen(option[i].name); + if (option[i].has_arg) { + len += indent; + } + if (len > max) { + max = len; + } + i++; + } + return max; +} + +static void usage(char *progname) { + printf("usage: %s ", progname); + printf("[-"); + for (int x = 0; long_options[x].val != 0; x++) { + if (long_options[x].has_arg == no_argument && long_options[x].val <= 'z') { + putchar(long_options[x].val); + } + } + printf("] {DELIVERY_FILE}\n"); + + int width = get_option_max_width(long_options); + for (int x = 0; long_options[x].name != 0; x++) { + char tmp[STASIS_NAME_MAX] = {0}; + char output[sizeof(tmp)] = {0}; + char opt_long[50] = {0}; // --? [ARG]? + char opt_short[50] = {0}; // -? [ARG]? + + strcat(opt_long, "--"); + strcat(opt_long, long_options[x].name); + if (long_options[x].has_arg) { + strcat(opt_long, " ARG"); + } + + if (long_options[x].val <= 'z') { + strcat(opt_short, "-"); + opt_short[1] = (char) long_options[x].val; + if (long_options[x].has_arg) { + strcat(opt_short, " ARG"); + } + } else { + strcat(opt_short, " "); + } + + sprintf(tmp, " %%-%ds\t%%s\t\t%%s", width + 4); + sprintf(output, tmp, opt_long, opt_short, long_options_help[x]); + puts(output); + } +} + +static int callback_except_jf(const void *a, const void *b) { + const struct EnvCtl_Item *item = a; + const char *name = b; + + if (!globals.enable_artifactory) { + return STASIS_ENVCTL_RET_IGNORE; + } + + if (envctl_check_required(item->flags)) { + const char *content = getenv(name); + if (!content || isempty((char *) content)) { + return STASIS_ENVCTL_RET_FAIL; + } + } + + return STASIS_ENVCTL_RET_SUCCESS; +} + +static int callback_except_gh(const void *a, const void *b) { + const struct EnvCtl_Item *item = a; + const char *name = b; + //printf("GH exception check: %s\n", name); + if (envctl_check_required(item->flags) && envctl_check_present(item, name)) { + return STASIS_ENVCTL_RET_SUCCESS; + } + + return STASIS_ENVCTL_RET_FAIL; +} + +static void check_system_env_requirements() { + msg(STASIS_MSG_L1, "Checking environment\n"); + globals.envctl = envctl_init(); + envctl_register(&globals.envctl, STASIS_ENVCTL_PASSTHRU, NULL, "TMPDIR"); + envctl_register(&globals.envctl, STASIS_ENVCTL_PASSTHRU, NULL, "STASIS_ROOT"); + envctl_register(&globals.envctl, STASIS_ENVCTL_PASSTHRU, NULL, "STASIS_SYSCONFDIR"); + envctl_register(&globals.envctl, STASIS_ENVCTL_PASSTHRU, NULL, "STASIS_CPU_COUNT"); + envctl_register(&globals.envctl, STASIS_ENVCTL_REQUIRED | STASIS_ENVCTL_REDACT, callback_except_gh, "STASIS_GH_TOKEN"); + envctl_register(&globals.envctl, STASIS_ENVCTL_REQUIRED, callback_except_jf, "STASIS_JF_ARTIFACTORY_URL"); + envctl_register(&globals.envctl, STASIS_ENVCTL_REDACT, NULL, "STASIS_JF_ACCESS_TOKEN"); + envctl_register(&globals.envctl, STASIS_ENVCTL_PASSTHRU, NULL, "STASIS_JF_USER"); + envctl_register(&globals.envctl, STASIS_ENVCTL_REDACT, NULL, "STASIS_JF_PASSWORD"); + envctl_register(&globals.envctl, STASIS_ENVCTL_REDACT, NULL, "STASIS_JF_SSH_KEY_PATH"); + envctl_register(&globals.envctl, STASIS_ENVCTL_REDACT, NULL, "STASIS_JF_SSH_PASSPHRASE"); + envctl_register(&globals.envctl, STASIS_ENVCTL_REDACT, NULL, "STASIS_JF_CLIENT_CERT_CERT_PATH"); + envctl_register(&globals.envctl, STASIS_ENVCTL_REDACT, NULL, "STASIS_JF_CLIENT_CERT_KEY_PATH"); + envctl_register(&globals.envctl, STASIS_ENVCTL_REQUIRED, callback_except_jf, "STASIS_JF_REPO"); + envctl_do_required(globals.envctl, globals.verbose); +} + +static void check_system_requirements(struct Delivery *ctx) { + const char *tools_required[] = { + "rsync", + NULL, + }; + + msg(STASIS_MSG_L1, "Checking system requirements\n"); + for (size_t i = 0; tools_required[i] != NULL; i++) { + if (!find_program(tools_required[i])) { + msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "'%s' must be installed.\n", tools_required[i]); + exit(1); + } + } + + if (!globals.tmpdir && !ctx->storage.tmpdir) { + delivery_init_tmpdir(ctx); + } + + struct DockerCapabilities dcap; + if (!docker_capable(&dcap)) { + msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "Docker is broken\n"); + msg(STASIS_MSG_L3, "Available: %s\n", dcap.available ? "Yes" : "No"); + msg(STASIS_MSG_L3, "Usable: %s\n", dcap.usable ? "Yes" : "No"); + msg(STASIS_MSG_L3, "Podman [Docker Emulation]: %s\n", dcap.podman ? "Yes" : "No"); + msg(STASIS_MSG_L3, "Build plugin(s): "); + if (dcap.usable) { + if (dcap.build & STASIS_DOCKER_BUILD) { + printf("build "); + } + if (dcap.build & STASIS_DOCKER_BUILD_X) { + printf("buildx "); + } + puts(""); + } else { + printf("N/A\n"); + } + + // disable docker builds + globals.enable_docker = false; + } +} + +static void check_requirements(struct Delivery *ctx) { + check_system_requirements(ctx); + check_system_env_requirements(); +} + +int main(int argc, char *argv[]) { + struct Delivery ctx; + struct Process proc = { + .f_stdout = "", + .f_stderr = "", + .redirect_stderr = 0, + }; + char env_name[STASIS_NAME_MAX] = {0}; + char env_name_testing[STASIS_NAME_MAX] = {0}; + char *delivery_input = NULL; + char *config_input = NULL; + char installer_url[PATH_MAX]; + char python_override_version[STASIS_NAME_MAX]; + int user_disabled_docker = false; + globals.cpu_limit = get_cpu_count(); + if (globals.cpu_limit > 1) { + globals.cpu_limit--; // max - 1 + } + + memset(env_name, 0, sizeof(env_name)); + memset(env_name_testing, 0, sizeof(env_name_testing)); + memset(installer_url, 0, sizeof(installer_url)); + memset(python_override_version, 0, sizeof(python_override_version)); + memset(&proc, 0, sizeof(proc)); + memset(&ctx, 0, sizeof(ctx)); + + int c; + int option_index = 0; + while ((c = getopt_long(argc, argv, "hVCc:p:vU", long_options, &option_index)) != -1) { + switch (c) { + case 'h': + usage(path_basename(argv[0])); + exit(0); + case 'V': + puts(VERSION); + exit(0); + case 'c': + config_input = strdup(optarg); + break; + case 'C': + globals.continue_on_error = true; + break; + case 'p': + strcpy(python_override_version, optarg); + break; + case 'l': + globals.cpu_limit = strtol(optarg, NULL, 10); + if (globals.cpu_limit <= 1) { + globals.cpu_limit = 1; + globals.enable_parallel = false; // No point + } + break; + case OPT_ALWAYS_UPDATE_BASE: + globals.always_update_base_environment = true; + break; + case OPT_FAIL_FAST: + globals.parallel_fail_fast = true; + break; + case OPT_POOL_STATUS_INTERVAL: + globals.pool_status_interval = (int) strtol(optarg, NULL, 10); + if (globals.pool_status_interval < 1) { + globals.pool_status_interval = 1; + } else if (globals.pool_status_interval > 60 * 10) { + // Possible poor choice alert + fprintf(stderr, "Caution: Excessive pausing between status updates may cause third-party CI/CD" + " jobs to fail if the stdout/stderr streams are idle for too long!\n"); + } + break; + case 'U': + setenv("PYTHONUNBUFFERED", "1", 1); + fflush(stdout); + fflush(stderr); + setvbuf(stdout, NULL, _IONBF, 0); + setvbuf(stderr, NULL, _IONBF, 0); + break; + case 'v': + globals.verbose = true; + break; + case OPT_OVERWRITE: + globals.enable_overwrite = true; + break; + case OPT_NO_DOCKER: + globals.enable_docker = false; + user_disabled_docker = true; + break; + case OPT_NO_ARTIFACTORY: + globals.enable_artifactory = false; + break; + case OPT_NO_ARTIFACTORY_BUILD_INFO: + globals.enable_artifactory_build_info = false; + break; + case OPT_NO_TESTING: + globals.enable_testing = false; + break; + case OPT_NO_REWRITE_SPEC_STAGE_2: + globals.enable_rewrite_spec_stage_2 = false; + break; + case OPT_NO_PARALLEL: + globals.enable_parallel = false; + break; + case '?': + default: + exit(1); + } + } + + if (optind < argc) { + while (optind < argc) { + // use first positional argument + delivery_input = argv[optind++]; + break; + } + } + + if (!delivery_input) { + fprintf(stderr, "error: a DELIVERY_FILE is required\n"); + usage(path_basename(argv[0])); + exit(1); + } + + printf(BANNER, VERSION, AUTHOR); + + msg(STASIS_MSG_L1, "Setup\n"); + + // Expose variables for use with the template engine + // NOTE: These pointers are populated by delivery_init() so please avoid using + // tpl_render() until then. + tpl_register("meta.name", &ctx.meta.name); + tpl_register("meta.version", &ctx.meta.version); + tpl_register("meta.codename", &ctx.meta.codename); + tpl_register("meta.mission", &ctx.meta.mission); + tpl_register("meta.python", &ctx.meta.python); + tpl_register("meta.python_compact", &ctx.meta.python_compact); + tpl_register("info.time_str_epoch", &ctx.info.time_str_epoch); + tpl_register("info.release_name", &ctx.info.release_name); + tpl_register("info.build_name", &ctx.info.build_name); + tpl_register("info.build_number", &ctx.info.build_number); + tpl_register("storage.tmpdir", &ctx.storage.tmpdir); + tpl_register("storage.output_dir", &ctx.storage.output_dir); + tpl_register("storage.delivery_dir", &ctx.storage.delivery_dir); + tpl_register("storage.conda_artifact_dir", &ctx.storage.conda_artifact_dir); + tpl_register("storage.wheel_artifact_dir", &ctx.storage.wheel_artifact_dir); + tpl_register("storage.build_sources_dir", &ctx.storage.build_sources_dir); + tpl_register("storage.build_docker_dir", &ctx.storage.build_docker_dir); + tpl_register("storage.results_dir", &ctx.storage.results_dir); + tpl_register("storage.tools_dir", &ctx.storage.tools_dir); + tpl_register("conda.installer_baseurl", &ctx.conda.installer_baseurl); + tpl_register("conda.installer_name", &ctx.conda.installer_name); + tpl_register("conda.installer_version", &ctx.conda.installer_version); + tpl_register("conda.installer_arch", &ctx.conda.installer_arch); + tpl_register("conda.installer_platform", &ctx.conda.installer_platform); + tpl_register("deploy.jfrog.repo", &globals.jfrog.repo); + tpl_register("deploy.jfrog.url", &globals.jfrog.url); + tpl_register("deploy.docker.registry", &ctx.deploy.docker.registry); + tpl_register("workaround.conda_reactivate", &globals.workaround.conda_reactivate); + + // Expose function(s) to the template engine + // Prototypes can be found in template_func_proto.h + tpl_register_func("get_github_release_notes", &get_github_release_notes_tplfunc_entrypoint, 3, NULL); + tpl_register_func("get_github_release_notes_auto", &get_github_release_notes_auto_tplfunc_entrypoint, 1, &ctx); + tpl_register_func("junitxml_file", &get_junitxml_file_entrypoint, 1, &ctx); + tpl_register_func("basetemp_dir", &get_basetemp_dir_entrypoint, 1, &ctx); + tpl_register_func("tox_run", &tox_run_entrypoint, 2, &ctx); + + // Set up PREFIX/etc directory information + // The user may manipulate the base directory path with STASIS_SYSCONFDIR + // environment variable + char stasis_sysconfdir_tmp[PATH_MAX]; + if (getenv("STASIS_SYSCONFDIR")) { + strncpy(stasis_sysconfdir_tmp, getenv("STASIS_SYSCONFDIR"), sizeof(stasis_sysconfdir_tmp) - 1); + } else { + strncpy(stasis_sysconfdir_tmp, STASIS_SYSCONFDIR, sizeof(stasis_sysconfdir_tmp) - 1); + } + + globals.sysconfdir = realpath(stasis_sysconfdir_tmp, NULL); + if (!globals.sysconfdir) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Unable to resolve path to configuration directory: %s\n", stasis_sysconfdir_tmp); + exit(1); + } + + // Override Python version from command-line, if any + if (strlen(python_override_version)) { + guard_free(ctx.meta.python); + ctx.meta.python = strdup(python_override_version); + guard_free(ctx.meta.python_compact); + ctx.meta.python_compact = to_short_version(ctx.meta.python); + } + + if (!config_input) { + // no configuration passed by argument. use basic config. + char cfgfile[PATH_MAX * 2]; + sprintf(cfgfile, "%s/%s", globals.sysconfdir, "stasis.ini"); + if (!access(cfgfile, F_OK | R_OK)) { + config_input = strdup(cfgfile); + } else { + msg(STASIS_MSG_WARN, "STASIS global configuration is not readable, or does not exist: %s", cfgfile); + } + } + + if (config_input) { + msg(STASIS_MSG_L2, "Reading STASIS global configuration: %s\n", config_input); + ctx._stasis_ini_fp.cfg = ini_open(config_input); + if (!ctx._stasis_ini_fp.cfg) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read config file: %s, %s\n", delivery_input, strerror(errno)); + exit(1); + } + ctx._stasis_ini_fp.cfg_path = strdup(config_input); + guard_free(config_input); + } + + msg(STASIS_MSG_L2, "Reading STASIS delivery configuration: %s\n", delivery_input); + ctx._stasis_ini_fp.delivery = ini_open(delivery_input); + if (!ctx._stasis_ini_fp.delivery) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read delivery file: %s, %s\n", delivery_input, strerror(errno)); + exit(1); + } + ctx._stasis_ini_fp.delivery_path = strdup(delivery_input); + + msg(STASIS_MSG_L2, "Bootstrapping delivery context\n"); + if (bootstrap_build_info(&ctx)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to bootstrap delivery context\n"); + exit(1); + } + + msg(STASIS_MSG_L2, "Initializing delivery context\n"); + if (delivery_init(&ctx, INI_READ_RENDER)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to initialize delivery context\n"); + exit(1); + } + check_requirements(&ctx); + + msg(STASIS_MSG_L2, "Configuring JFrog CLI\n"); + if (delivery_init_artifactory(&ctx)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "JFrog CLI configuration failed\n"); + exit(1); + } + + runtime_apply(ctx.runtime.environ); + strcpy(env_name, ctx.info.release_name); + strcpy(env_name_testing, env_name); + strcat(env_name_testing, "-test"); + + // Safety gate: Avoid clobbering a delivered release unless the user wants that behavior + msg(STASIS_MSG_L1, "Checking release history\n"); + if (delivery_exists(&ctx)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Refusing to overwrite release: %s\nUse --overwrite to enable release clobbering.\n", ctx.info.release_name); + exit(1); + } + + msg(STASIS_MSG_L1, "Conda setup\n"); + delivery_get_conda_installer_url(&ctx, installer_url); + msg(STASIS_MSG_L2, "Downloading: %s\n", installer_url); + if (delivery_get_conda_installer(&ctx, installer_url)) { + msg(STASIS_MSG_ERROR, "download failed: %s\n", installer_url); + exit(1); + } + + // Unlikely to occur: this should help prevent rmtree() from destroying your entire filesystem + // if path is "/" then, die + // or if empty string, die + if (!strcmp(ctx.storage.conda_install_prefix, DIR_SEP) || !strlen(ctx.storage.conda_install_prefix)) { + fprintf(stderr, "error: ctx.storage.conda_install_prefix is malformed!\n"); + exit(1); + } + + msg(STASIS_MSG_L2, "Installing: %s\n", ctx.conda.installer_name); + delivery_install_conda(ctx.conda.installer_path, ctx.storage.conda_install_prefix); + + msg(STASIS_MSG_L2, "Configuring: %s\n", ctx.storage.conda_install_prefix); + delivery_conda_enable(&ctx, ctx.storage.conda_install_prefix); + + char *pathvar = NULL; + pathvar = getenv("PATH"); + if (!pathvar) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "PATH variable is not set. Cannot continue.\n"); + exit(1); + } else { + char pathvar_tmp[STASIS_BUFSIZ]; + sprintf(pathvar_tmp, "%s/bin:%s", ctx.storage.conda_install_prefix, pathvar); + setenv("PATH", pathvar_tmp, 1); + pathvar = NULL; + } + + + // + // Implied environment creation modes/actions + // + // 1. No base environment config + // 1a. Caller is warned + // 1b. Caller has full control over all packages + // 2. Default base environment (etc/stasis/mission/[name]/base.yml) + // 2a. Depends on packages defined by base.yml + // 2b. Caller may issue a reduced package set in the INI config + // 2c. Caller must be vigilant to avoid incompatible packages (base.yml + // *should* have no version constraints) + // 3. External base environment (based_on=schema://[release_name].yml) + // 3a. Depends on a previous release or arbitrary yaml configuration + // 3b. Bugs, conflicts, and dependency resolution issues are inherited and + // must be handled in the INI config + msg(STASIS_MSG_L1, "Creating release environment(s)\n"); + + char *mission_base = NULL; + if (isempty(ctx.meta.based_on)) { + guard_free(ctx.meta.based_on); + char *mission_base_orig = NULL; + + if (asprintf(&mission_base_orig, "%s/%s/base.yml", ctx.storage.mission_dir, ctx.meta.mission) < 0) { + SYSERROR("Unable to allocate bytes for %s/%s/base.yml path\n", ctx.storage.mission_dir, ctx.meta.mission); + exit(1); + } + + if (access(mission_base_orig, F_OK) < 0) { + msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "Mission does not provide a base.yml configuration: %s (%s)\n", + ctx.meta.mission, ctx.storage.mission_dir); + } else { + msg(STASIS_MSG_L2, "Using base environment configuration: %s\n", mission_base_orig); + if (asprintf(&mission_base, "%s/%s-base.yml", ctx.storage.tmpdir, ctx.info.release_name) < 0) { + SYSERROR("%s", "Unable to allocate bytes for temporary base.yml configuration"); + remove(mission_base); + exit(1); + } + copy2(mission_base_orig, mission_base, CT_OWNER | CT_PERM); + char spec[255] = {0}; + snprintf(spec, sizeof(spec) - 1, "- python=%s\n", ctx.meta.python); + file_replace_text(mission_base, "- python\n", spec, 0); + ctx.meta.based_on = mission_base; + } + guard_free(mission_base_orig); + } + + if (!isempty(ctx.meta.based_on)) { + if (conda_env_remove(env_name)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to remove release environment: %s\n", env_name); + exit(1); + } + + msg(STASIS_MSG_L2, "Based on: %s\n", ctx.meta.based_on); + if (conda_env_create_from_uri(env_name, ctx.meta.based_on)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "unable to install release environment using configuration file\n"); + exit(1); + } + + if (conda_env_remove(env_name_testing)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to remove testing environment %s\n", env_name_testing); + exit(1); + } + if (conda_env_create_from_uri(env_name_testing, ctx.meta.based_on)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "unable to install testing environment using configuration file\n"); + exit(1); + } + } else { + if (conda_env_create(env_name, ctx.meta.python, NULL)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to create release environment\n"); + exit(1); + } + if (conda_env_create(env_name_testing, ctx.meta.python, NULL)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to create testing environment\n"); + exit(1); + } + } + // The base environment configuration not used past this point + remove(mission_base); + + // Activate test environment + msg(STASIS_MSG_L1, "Activating test environment\n"); + if (conda_activate(ctx.storage.conda_install_prefix, env_name_testing)) { + fprintf(stderr, "failed to activate test environment\n"); + exit(1); + } + + delivery_gather_tool_versions(&ctx); + if (!ctx.conda.tool_version) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Could not determine conda version\n"); + exit(1); + } + if (!ctx.conda.tool_build_version) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Could not determine conda-build version\n"); + exit(1); + } + + if (pip_exec("install build")) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "'build' tool installation failed\n"); + exit(1); + } + + if (!isempty(ctx.meta.based_on)) { + msg(STASIS_MSG_L1, "Generating package overlay from environment: %s\n", env_name); + if (delivery_overlay_packages_from_env(&ctx, env_name)) { + msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "%s", "Failed to generate package overlay. Resulting environment integrity cannot be guaranteed.\n"); + exit(1); + } + } + + msg(STASIS_MSG_L1, "Filter deliverable packages\n"); + delivery_defer_packages(&ctx, DEFER_CONDA); + delivery_defer_packages(&ctx, DEFER_PIP); + + msg(STASIS_MSG_L1, "Overview\n"); + delivery_meta_show(&ctx); + delivery_conda_show(&ctx); + if (globals.verbose) { + //delivery_runtime_show(&ctx); + } + + // Execute configuration-defined tests + if (globals.enable_testing) { + delivery_tests_show(&ctx); + + msg(STASIS_MSG_L1, "Begin test execution\n"); + delivery_tests_run(&ctx); + msg(STASIS_MSG_L2, "Rewriting test results\n"); + delivery_fixup_test_results(&ctx); + } else { + msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Test execution is disabled\n"); + } + + if (ctx.conda.conda_packages_defer && strlist_count(ctx.conda.conda_packages_defer)) { + msg(STASIS_MSG_L2, "Building Conda recipe(s)\n"); + if (delivery_build_recipes(&ctx)) { + exit(1); + } + msg(STASIS_MSG_L3, "Copying artifacts\n"); + if (delivery_copy_conda_artifacts(&ctx)) { + exit(1); + } + msg(STASIS_MSG_L3, "Indexing artifacts\n"); + if (delivery_index_conda_artifacts(&ctx)) { + exit(1); + } + } + + if (strlist_count(ctx.conda.pip_packages_defer)) { + if (!(ctx.conda.wheels_packages = delivery_build_wheels(&ctx))) { + exit(1); + } + if (delivery_index_wheel_artifacts(&ctx)) { + exit(1); + } + + } + + // Populate the release environment + msg(STASIS_MSG_L1, "Populating release environment\n"); + msg(STASIS_MSG_L2, "Installing conda packages\n"); + if (strlist_count(ctx.conda.conda_packages)) { + if (delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_CONDA, (struct StrList *[]) {ctx.conda.conda_packages, NULL})) { + exit(1); + } + } + if (strlist_count(ctx.conda.conda_packages_defer)) { + msg(STASIS_MSG_L3, "Installing deferred conda packages\n"); + if (delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_CONDA | INSTALL_PKG_CONDA_DEFERRED, (struct StrList *[]) {ctx.conda.conda_packages_defer, NULL})) { + exit(1); + } + } else { + msg(STASIS_MSG_L3, "No deferred conda packages\n"); + } + + msg(STASIS_MSG_L2, "Installing pip packages\n"); + if (strlist_count(ctx.conda.pip_packages)) { + if (delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_PIP, (struct StrList *[]) {ctx.conda.pip_packages, NULL})) { + exit(1); + } + } + + if (strlist_count(ctx.conda.pip_packages_defer)) { + msg(STASIS_MSG_L3, "Installing deferred pip packages\n"); + if (delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_PIP | INSTALL_PKG_PIP_DEFERRED, (struct StrList *[]) {ctx.conda.pip_packages_defer, NULL})) { + exit(1); + } + } else { + msg(STASIS_MSG_L3, "No deferred pip packages\n"); + } + + conda_exec("list"); + + msg(STASIS_MSG_L1, "Creating release\n"); + msg(STASIS_MSG_L2, "Exporting delivery configuration\n"); + if (!pushd(ctx.storage.cfgdump_dir)) { + char filename[PATH_MAX] = {0}; + sprintf(filename, "%s.ini", ctx.info.release_name); + FILE *spec = fopen(filename, "w+"); + if (!spec) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", filename); + exit(1); + } + ini_write(ctx._stasis_ini_fp.delivery, &spec, INI_WRITE_RAW); + fclose(spec); + + memset(filename, 0, sizeof(filename)); + sprintf(filename, "%s-rendered.ini", ctx.info.release_name); + spec = fopen(filename, "w+"); + if (!spec) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", filename); + exit(1); + } + ini_write(ctx._stasis_ini_fp.delivery, &spec, INI_WRITE_PRESERVE); + fclose(spec); + popd(); + } else { + SYSERROR("Failed to enter directory: %s", ctx.storage.delivery_dir); + exit(1); + } + + msg(STASIS_MSG_L2, "Exporting %s\n", env_name_testing); + if (conda_env_export(env_name_testing, ctx.storage.delivery_dir, env_name_testing)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", env_name_testing); + exit(1); + } + + msg(STASIS_MSG_L2, "Exporting %s\n", env_name); + if (conda_env_export(env_name, ctx.storage.delivery_dir, env_name)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", env_name); + exit(1); + } + + // Rewrite release environment output (i.e. set package origin(s) to point to the deployment server, etc.) + char specfile[PATH_MAX]; + sprintf(specfile, "%s/%s.yml", ctx.storage.delivery_dir, env_name); + msg(STASIS_MSG_L3, "Rewriting release spec file (stage 1): %s\n", path_basename(specfile)); + delivery_rewrite_spec(&ctx, specfile, DELIVERY_REWRITE_SPEC_STAGE_1); + + msg(STASIS_MSG_L1, "Rendering mission templates\n"); + delivery_mission_render_files(&ctx); + + int want_docker = ini_section_search(&ctx._stasis_ini_fp.delivery, INI_SEARCH_BEGINS, "deploy:docker") ? true : false; + int want_artifactory = ini_section_search(&ctx._stasis_ini_fp.delivery, INI_SEARCH_BEGINS, "deploy:artifactory") ? true : false; + + if (want_docker) { + if (user_disabled_docker) { + msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Docker image building is disabled by CLI argument\n"); + } else { + char dockerfile[PATH_MAX] = {0}; + sprintf(dockerfile, "%s/%s", ctx.storage.build_docker_dir, "Dockerfile"); + if (globals.enable_docker) { + if (!access(dockerfile, F_OK)) { + msg(STASIS_MSG_L1, "Building Docker image\n"); + if (delivery_docker(&ctx)) { + msg(STASIS_MSG_L1 | STASIS_MSG_ERROR, "Failed to build docker image!\n"); + COE_CHECK_ABORT(1, "Failed to build docker image"); + } + } else { + msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Docker image building is disabled. No Dockerfile found in %s\n", ctx.storage.build_docker_dir); + } + } else { + msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Docker image building is disabled. System configuration error\n"); + } + } + } else { + msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Docker image building is disabled. deploy:docker is not configured\n"); + } + + msg(STASIS_MSG_L3, "Rewriting release spec file (stage 2): %s\n", path_basename(specfile)); + delivery_rewrite_spec(&ctx, specfile, DELIVERY_REWRITE_SPEC_STAGE_2); + + msg(STASIS_MSG_L1, "Dumping metadata\n"); + if (delivery_dump_metadata(&ctx)) { + msg(STASIS_MSG_L1 | STASIS_MSG_ERROR, "Metadata dump failed\n"); + } + + if (want_artifactory) { + if (globals.enable_artifactory) { + msg(STASIS_MSG_L1, "Uploading artifacts\n"); + delivery_artifact_upload(&ctx); + } else { + msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Artifactory upload is disabled by CLI argument\n"); + } + } else { + msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Artifactory upload is disabled. deploy:artifactory is not configured\n"); + } + + msg(STASIS_MSG_L1, "Cleaning up\n"); + delivery_free(&ctx); + globals_free(); + tpl_free(); + + msg(STASIS_MSG_L1, "Done!\n"); + return 0; +} + diff --git a/src/cli/stasis_indexer/CMakeLists.txt b/src/cli/stasis_indexer/CMakeLists.txt new file mode 100644 index 0000000..eae1394 --- /dev/null +++ b/src/cli/stasis_indexer/CMakeLists.txt @@ -0,0 +1,6 @@ +add_executable(stasis_indexer + stasis_indexer.c +) +target_link_libraries(stasis_indexer PRIVATE stasis_core) + +install(TARGETS stasis_indexer RUNTIME) diff --git a/src/cli/stasis_indexer/stasis_indexer.c b/src/cli/stasis_indexer/stasis_indexer.c new file mode 100644 index 0000000..bd59920 --- /dev/null +++ b/src/cli/stasis_indexer/stasis_indexer.c @@ -0,0 +1,949 @@ +#include +#include +#include "delivery.h" +#include "junitxml.h" + +static struct option long_options[] = { + {"help", no_argument, 0, 'h'}, + {"destdir", required_argument, 0, 'd'}, + {"verbose", no_argument, 0, 'v'}, + {"unbuffered", no_argument, 0, 'U'}, + {"web", no_argument, 0, 'w'}, + {0, 0, 0, 0}, +}; + +const char *long_options_help[] = { + "Display this usage statement", + "Destination directory", + "Increase output verbosity", + "Disable line buffering", + "Generate HTML indexes (requires pandoc)", + NULL, +}; + +static void usage(char *name) { + int maxopts = sizeof(long_options) / sizeof(long_options[0]); + unsigned char *opts = calloc(maxopts + 1, sizeof(char)); + for (int i = 0; i < maxopts; i++) { + opts[i] = long_options[i].val; + } + printf("usage: %s [-%s] {{STASIS_ROOT}...}\n", name, opts); + guard_free(opts); + + for (int i = 0; i < maxopts - 1; i++) { + char line[255]; + sprintf(line, " --%s -%c %-20s", long_options[i].name, long_options[i].val, long_options_help[i]); + puts(line); + } + +} + +int indexer_combine_rootdirs(const char *dest, char **rootdirs, const size_t rootdirs_total) { + char cmd[PATH_MAX]; + char destdir_bare[PATH_MAX]; + char destdir_with_output[PATH_MAX]; + char *destdir = destdir_bare; + + memset(cmd, 0, sizeof(cmd)); + memset(destdir_bare, 0, sizeof(destdir_bare)); + memset(destdir_with_output, 0, sizeof(destdir_bare)); + + strcpy(destdir_bare, dest); + strcpy(destdir_with_output, dest); + strcat(destdir_with_output, "/output"); + + if (!access(destdir_with_output, F_OK)) { + destdir = destdir_with_output; + } + + sprintf(cmd, "rsync -ah%s --delete --exclude 'tools/' --exclude 'tmp/' --exclude 'build/' ", globals.verbose ? "v" : "q"); + for (size_t i = 0; i < rootdirs_total; i++) { + char srcdir_bare[PATH_MAX] = {0}; + char srcdir_with_output[PATH_MAX] = {0}; + char *srcdir = srcdir_bare; + strcpy(srcdir_bare, rootdirs[i]); + strcpy(srcdir_with_output, rootdirs[i]); + strcat(srcdir_with_output, "/output"); + + if (access(srcdir_bare, F_OK)) { + fprintf(stderr, "%s does not exist\n", srcdir_bare); + continue; + } + + if (!access(srcdir_with_output, F_OK)) { + srcdir = srcdir_with_output; + } + snprintf(cmd + strlen(cmd), sizeof(srcdir) - strlen(srcdir) + 4, "'%s'/ ", srcdir); + } + snprintf(cmd + strlen(cmd), sizeof(cmd) - strlen(destdir) + 1, " %s/", destdir); + + if (globals.verbose) { + puts(cmd); + } + + if (system(cmd)) { + return -1; + } + return 0; +} + +int indexer_wheels(struct Delivery *ctx) { + return delivery_index_wheel_artifacts(ctx); +} + +int indexer_load_metadata(struct Delivery *ctx, const char *filename) { + char line[STASIS_NAME_MAX] = {0}; + FILE *fp; + + fp = fopen(filename, "r"); + if (!fp) { + return -1; + } + + while (fgets(line, sizeof(line) - 1, fp) != NULL) { + char **parts = split(line, " ", 1); + char *name = parts[0]; + char *value = parts[1]; + strip(value); + if (!strcmp(name, "name")) { + ctx->meta.name = strdup(value); + } else if (!strcmp(name, "version")) { + ctx->meta.version = strdup(value); + } else if (!strcmp(name, "rc")) { + ctx->meta.rc = (int) strtol(value, NULL, 10); + } else if (!strcmp(name, "python")) { + ctx->meta.python = strdup(value); + } else if (!strcmp(name, "python_compact")) { + ctx->meta.python_compact = strdup(value); + } else if (!strcmp(name, "mission")) { + ctx->meta.mission = strdup(value); + } else if (!strcmp(name, "codename")) { + ctx->meta.codename = strdup(value); + } else if (!strcmp(name, "platform")) { + ctx->system.platform = split(value, " ", 0); + } else if (!strcmp(name, "arch")) { + ctx->system.arch = strdup(value); + } else if (!strcmp(name, "time")) { + ctx->info.time_str_epoch = strdup(value); + } else if (!strcmp(name, "release_fmt")) { + ctx->rules.release_fmt = strdup(value); + } else if (!strcmp(name, "release_name")) { + ctx->info.release_name = strdup(value); + } else if (!strcmp(name, "build_name_fmt")) { + ctx->rules.build_name_fmt = strdup(value); + } else if (!strcmp(name, "build_name")) { + ctx->info.build_name = strdup(value); + } else if (!strcmp(name, "build_number_fmt")) { + ctx->rules.build_number_fmt = strdup(value); + } else if (!strcmp(name, "build_number")) { + ctx->info.build_number = strdup(value); + } else if (!strcmp(name, "conda_installer_baseurl")) { + ctx->conda.installer_baseurl = strdup(value); + } else if (!strcmp(name, "conda_installer_name")) { + ctx->conda.installer_name = strdup(value); + } else if (!strcmp(name, "conda_installer_version")) { + ctx->conda.installer_version = strdup(value); + } else if (!strcmp(name, "conda_installer_platform")) { + ctx->conda.installer_platform = strdup(value); + } else if (!strcmp(name, "conda_installer_arch")) { + ctx->conda.installer_arch = strdup(value); + } + GENERIC_ARRAY_FREE(parts); + } + fclose(fp); + + return 0; +} + +int indexer_get_files(struct StrList **out, const char *path, const char *pattern, ...) { + va_list args; + va_start(args, pattern); + char userpattern[PATH_MAX] = {0}; + vsprintf(userpattern, pattern, args); + va_end(args); + struct StrList *list = listdir(path); + if (!list) { + return -1; + } + + if (!(*out)) { + (*out) = strlist_init(); + if (!(*out)) { + guard_strlist_free(&list); + return -1; + } + } + + size_t no_match = 0; + for (size_t i = 0; i < strlist_count(list); i++) { + char *item = strlist_item(list, i); + if (fnmatch(userpattern, item, 0)) { + no_match++; + continue; + } else { + strlist_append(&(*out), item); + } + } + if (no_match >= strlist_count(list)) { + fprintf(stderr, "no files matching the pattern: %s\n", userpattern); + guard_strlist_free(&list); + return -1; + } + guard_strlist_free(&list); + return 0; +} + +int get_latest_rc(struct Delivery ctx[], size_t nelem) { + int result = 0; + for (size_t i = 0; i < nelem; i++) { + if (ctx[i].meta.rc > result) { + result = ctx[i].meta.rc; + } + } + return result; +} + +struct Delivery **get_latest_deliveries(struct Delivery ctx[], size_t nelem) { + struct Delivery **result = NULL; + int latest = 0; + size_t n = 0; + + result = calloc(nelem + 1, sizeof(result)); + if (!result) { + fprintf(stderr, "Unable to allocate %zu bytes for result delivery array: %s\n", nelem * sizeof(result), strerror(errno)); + return NULL; + } + + latest = get_latest_rc(ctx, nelem); + for (size_t i = 0; i < nelem; i++) { + if (ctx[i].meta.rc == latest) { + result[n] = &ctx[i]; + n++; + } + } + + return result; +} + +int get_pandoc_version(size_t *result) { + *result = 0; + int state = 0; + char *version_str = shell_output("pandoc --version", &state); + if (state || !version_str) { + // an error occurred + return -1; + } + + // Verify that we're looking at pandoc + if (strlen(version_str) > 7 && !strncmp(version_str, "pandoc ", 7)) { + // we have pandoc + char *v_begin = &version_str[7]; + if (!v_begin) { + SYSERROR("unexpected pandoc output: %s", version_str); + return -1; + } + char *v_end = strchr(version_str, '\n'); + if (v_end) { + *v_end = 0; + } + + char **parts = split(v_begin, ".", 0); + if (!parts) { + SYSERROR("unable to split pandoc version string, '%s': %s", version_str, strerror(errno)); + return -1; + } + + size_t parts_total; + for (parts_total = 0; parts[parts_total] != NULL; parts_total++); + + // generate the version as an integer + // note: pandoc version scheme never exceeds four elements (or bytes in this case) + for (size_t i = 0; i < 4; i++) { + unsigned char tmp = 0; + if (i < parts_total) { + // only process version elements we have. the rest will be zeros. + tmp = strtoul(parts[i], NULL, 10); + } + // pack version element into result + *result = (*result << 8) | tmp; + } + } else { + // invalid version string + return 1; + } + + return 0; +} + +int indexer_make_website(struct Delivery *ctx) { + char cmd[PATH_MAX]; + const char *pattern = "*.md"; + + if (!find_program("pandoc")) { + fprintf(stderr, "pandoc is not installed: unable to generate HTML indexes\n"); + return 0; + } + + char *css_filename = calloc(PATH_MAX, sizeof(*css_filename)); + if (!css_filename) { + SYSERROR("unable to allocate string for CSS file path: %s", strerror(errno)); + return -1; + } + + sprintf(css_filename, "%s/%s", globals.sysconfdir, "stasis_pandoc.css"); + int have_css = access(css_filename, F_OK | R_OK) == 0; + + char pandoc_versioned_args[255] = {0}; + size_t pandoc_version = 0; + + if (!get_pandoc_version(&pandoc_version)) { + // < 2.19 + if (pandoc_version < 0x02130000) { + strcat(pandoc_versioned_args, "--self-contained "); + } else { + // >= 2.19 + strcat(pandoc_versioned_args, "--embed-resources "); + } + + // >= 1.15.0.4 + if (pandoc_version >= 0x010f0004) { + strcat(pandoc_versioned_args, "--standalone "); + } + + // >= 1.10.0.1 + if (pandoc_version >= 0x010a0001) { + strcat(pandoc_versioned_args, "-f gfm+autolink_bare_uris "); + } + + // > 3.1.9 + if (pandoc_version > 0x03010900) { + strcat(pandoc_versioned_args, "-f gfm+alerts "); + } + } + + struct StrList *dirs = strlist_init(); + strlist_append(&dirs, ctx->storage.delivery_dir); + strlist_append(&dirs, ctx->storage.results_dir); + + struct StrList *inputs = NULL; + for (size_t i = 0; i < strlist_count(dirs); i++) { + if (indexer_get_files(&inputs, ctx->storage.delivery_dir, pattern)) { + SYSERROR("%s does not contain files with pattern: %s", ctx->storage.delivery_dir, pattern); + guard_strlist_free(&inputs); + continue; + } + char *root = strlist_item(dirs, i); + for (size_t x = 0; x < strlist_count(inputs); x++) { + char *filename = strlist_item(inputs, x); + char fullpath_src[PATH_MAX] = {0}; + char fullpath_dest[PATH_MAX] = {0}; + sprintf(fullpath_src, "%s/%s", root, filename); + if (access(fullpath_src, F_OK)) { + continue; + } + + // Replace *.md extension with *.html. + strcpy(fullpath_dest, fullpath_src); + char *ext = strrchr(fullpath_dest, '.'); + if (ext) { + *ext = '\0'; + } + strcat(fullpath_dest, ".html"); + + // Converts a markdown file to html + strcpy(cmd, "pandoc "); + strcat(cmd, pandoc_versioned_args); + if (have_css) { + strcat(cmd, "--css "); + strcat(cmd, css_filename); + } + strcat(cmd, " "); + strcat(cmd, "--metadata title=\"STASIS\" "); + strcat(cmd, "-o "); + strcat(cmd, fullpath_dest); + strcat(cmd, " "); + strcat(cmd, fullpath_src); + if (globals.verbose) { + puts(cmd); + } + // This might be negative when killed by a signal. + // Otherwise, the return code is not critical to us. + if (system(cmd) < 0) { + guard_free(css_filename); + guard_strlist_free(&dirs); + return 1; + } + if (file_replace_text(fullpath_dest, ".md", ".html", 0)) { + // inform-only + SYSERROR("%s: failed to rewrite *.md urls with *.html extension", fullpath_dest); + } + + // Link the nearest README.html to index.html + if (!strcmp(filename, "README.md")) { + char link_from[PATH_MAX] = {0}; + char link_dest[PATH_MAX] = {0}; + strcpy(link_from, "README.html"); + sprintf(link_dest, "%s/%s", root, "index.html"); + if (symlink(link_from, link_dest)) { + SYSERROR("Warning: symlink(%s, %s) failed: %s", link_from, link_dest, strerror(errno)); + } + } + } + guard_strlist_free(&inputs); + } + guard_free(css_filename); + guard_strlist_free(&dirs); + + return 0; +} + +static int micromamba_configure(const struct Delivery *ctx, struct MicromambaInfo *m) { + int status = 0; + char *micromamba_prefix = NULL; + if (asprintf(µmamba_prefix, "%s/bin", ctx->storage.tools_dir) < 0) { + return -1; + } + m->conda_prefix = globals.conda_install_prefix; + m->micromamba_prefix = micromamba_prefix; + + size_t pathvar_len = (strlen(getenv("PATH")) + strlen(m->micromamba_prefix) + strlen(m->conda_prefix)) + 3 + 4 + 1; + // ^^^^^^^^^^^^^^^^^^ + // 3 = separators + // 4 = chars (/bin) + // 1 = nul terminator + char *pathvar = calloc(pathvar_len, sizeof(*pathvar)); + if (!pathvar) { + SYSERROR("%s", "Unable to allocate bytes for temporary path string"); + exit(1); + } + snprintf(pathvar, pathvar_len, "%s/bin:%s:%s", m->conda_prefix, m->micromamba_prefix, getenv("PATH")); + setenv("PATH", pathvar, 1); + guard_free(pathvar); + + status += micromamba(m, "config prepend --env channels conda-forge"); + if (!globals.verbose) { + status += micromamba(m, "config set --env quiet true"); + } + status += micromamba(m, "config set --env always_yes true"); + status += micromamba(m, "install conda-build pandoc"); + + return status; +} + +int indexer_conda(struct Delivery *ctx, struct MicromambaInfo m) { + int status = 0; + + status += micromamba(&m, "run conda index %s", ctx->storage.conda_artifact_dir); + return status; +} + +static struct StrList *get_architectures(struct Delivery ctx[], size_t nelem) { + struct StrList *architectures = strlist_init(); + for (size_t i = 0; i < nelem; i++) { + if (!strstr_array(architectures->data, ctx[i].system.arch)) { + strlist_append(&architectures, ctx[i].system.arch); + } + } + return architectures; +} + +static struct StrList *get_platforms(struct Delivery ctx[], size_t nelem) { + struct StrList *platforms = strlist_init(); + for (size_t i = 0; i < nelem; i++) { + if (!strstr_array(platforms->data, ctx[i].system.platform[DELIVERY_PLATFORM_RELEASE])) { + strlist_append(&platforms, ctx[i].system.platform[DELIVERY_PLATFORM_RELEASE]); + } + } + return platforms; +} + +int indexer_symlinks(struct Delivery ctx[], size_t nelem) { + struct Delivery **data = NULL; + data = get_latest_deliveries(ctx, nelem); + //int latest = get_latest_rc(ctx, nelem); + + if (!pushd(ctx->storage.delivery_dir)) { + for (size_t i = 0; i < nelem; i++) { + char link_name_spec[PATH_MAX]; + char link_name_readme[PATH_MAX]; + + char file_name_spec[PATH_MAX]; + char file_name_readme[PATH_MAX]; + + if (!data[i]) { + continue; + } + sprintf(link_name_spec, "latest-py%s-%s-%s.yml", data[i]->meta.python_compact, data[i]->system.platform[DELIVERY_PLATFORM_RELEASE], data[i]->system.arch); + sprintf(file_name_spec, "%s.yml", data[i]->info.release_name); + + sprintf(link_name_readme, "README-py%s-%s-%s.md", data[i]->meta.python_compact, data[i]->system.platform[DELIVERY_PLATFORM_RELEASE], data[i]->system.arch); + sprintf(file_name_readme, "README-%s.md", data[i]->info.release_name); + + if (!access(link_name_spec, F_OK)) { + if (unlink(link_name_spec)) { + fprintf(stderr, "Unable to remove spec link: %s\n", link_name_spec); + } + } + if (!access(link_name_readme, F_OK)) { + if (unlink(link_name_readme)) { + fprintf(stderr, "Unable to remove readme link: %s\n", link_name_readme); + } + } + + if (globals.verbose) { + printf("%s -> %s\n", file_name_spec, link_name_spec); + } + if (symlink(file_name_spec, link_name_spec)) { + fprintf(stderr, "Unable to link %s as %s\n", file_name_spec, link_name_spec); + } + + if (globals.verbose) { + printf("%s -> %s\n", file_name_readme, link_name_readme); + } + if (symlink(file_name_readme, link_name_readme)) { + fprintf(stderr, "Unable to link %s as %s\n", file_name_readme, link_name_readme); + } + } + popd(); + } else { + fprintf(stderr, "Unable to enter delivery directory: %s\n", ctx->storage.delivery_dir); + guard_free(data); + return -1; + } + + // "latest" is an array of pointers to ctx[]. Do not free the contents of the array. + guard_free(data); + return 0; +} + +int indexer_readmes(struct Delivery ctx[], size_t nelem) { + struct Delivery **latest = NULL; + latest = get_latest_deliveries(ctx, nelem); + + char indexfile[PATH_MAX] = {0}; + sprintf(indexfile, "%s/README.md", ctx->storage.delivery_dir); + + if (!pushd(ctx->storage.delivery_dir)) { + FILE *indexfp; + indexfp = fopen(indexfile, "w+"); + if (!indexfp) { + fprintf(stderr, "Unable to open %s for writing\n", indexfile); + return -1; + } + struct StrList *archs = get_architectures(*latest, nelem); + struct StrList *platforms = get_platforms(*latest, nelem); + + fprintf(indexfp, "# %s-%s\n\n", ctx->meta.name, ctx->meta.version); + fprintf(indexfp, "## Current Release\n\n"); + for (size_t p = 0; p < strlist_count(platforms); p++) { + char *platform = strlist_item(platforms, p); + for (size_t a = 0; a < strlist_count(archs); a++) { + char *arch = strlist_item(archs, a); + int have_combo = 0; + for (size_t i = 0; i < nelem; i++) { + if (latest[i] && latest[i]->system.platform) { + if (strstr(latest[i]->system.platform[DELIVERY_PLATFORM_RELEASE], platform) && + strstr(latest[i]->system.arch, arch)) { + have_combo = 1; + } + } + } + if (!have_combo) { + continue; + } + fprintf(indexfp, "### %s-%s\n\n", platform, arch); + + fprintf(indexfp, "|Release|Info|Receipt|\n"); + fprintf(indexfp, "|:----:|:----:|:----:|\n"); + for (size_t i = 0; i < nelem; i++) { + char link_name[PATH_MAX]; + char readme_name[PATH_MAX]; + char conf_name[PATH_MAX]; + char conf_name_relative[PATH_MAX]; + if (!latest[i]) { + continue; + } + sprintf(link_name, "latest-py%s-%s-%s.yml", latest[i]->meta.python_compact, latest[i]->system.platform[DELIVERY_PLATFORM_RELEASE], latest[i]->system.arch); + sprintf(readme_name, "README-py%s-%s-%s.md", latest[i]->meta.python_compact, latest[i]->system.platform[DELIVERY_PLATFORM_RELEASE], latest[i]->system.arch); + sprintf(conf_name, "%s.ini", latest[i]->info.release_name); + sprintf(conf_name_relative, "../config/%s-rendered.ini", latest[i]->info.release_name); + if (strstr(link_name, platform) && strstr(link_name, arch)) { + fprintf(indexfp, "|[%s](%s)|[%s](%s)|[%s](%s)|\n", link_name, link_name, readme_name, readme_name, conf_name, conf_name_relative); + } + } + fprintf(indexfp, "\n"); + } + fprintf(indexfp, "\n"); + } + guard_strlist_free(&archs); + guard_strlist_free(&platforms); + fclose(indexfp); + popd(); + } else { + fprintf(stderr, "Unable to enter delivery directory: %s\n", ctx->storage.delivery_dir); + guard_free(latest); + return -1; + } + + // "latest" is an array of pointers to ctxs[]. Do not free the contents of the array. + guard_free(latest); + return 0; +} + +int indexer_junitxml_report(struct Delivery ctx[], size_t nelem) { + struct Delivery **latest = NULL; + latest = get_latest_deliveries(ctx, nelem); + + char indexfile[PATH_MAX] = {0}; + sprintf(indexfile, "%s/README.md", ctx->storage.results_dir); + + struct StrList *file_listing = listdir(ctx->storage.results_dir); + if (!file_listing) { + // no test results to process + return 0; + } + + if (!pushd(ctx->storage.results_dir)) { + FILE *indexfp; + indexfp = fopen(indexfile, "w+"); + if (!indexfp) { + fprintf(stderr, "Unable to open %s for writing\n", indexfile); + return -1; + } + struct StrList *archs = get_architectures(*latest, nelem); + struct StrList *platforms = get_platforms(*latest, nelem); + + fprintf(indexfp, "# %s-%s Test Report\n\n", ctx->meta.name, ctx->meta.version); + fprintf(indexfp, "## Current Release\n\n"); + for (size_t p = 0; p < strlist_count(platforms); p++) { + char *platform = strlist_item(platforms, p); + for (size_t a = 0; a < strlist_count(archs); a++) { + char *arch = strlist_item(archs, a); + int have_combo = 0; + for (size_t i = 0; i < nelem; i++) { + if (latest[i] && latest[i]->system.platform) { + if (strstr(latest[i]->system.platform[DELIVERY_PLATFORM_RELEASE], platform) && + strstr(latest[i]->system.arch, arch)) { + have_combo = 1; + break; + } + } + } + if (!have_combo) { + continue; + } + fprintf(indexfp, "### %s-%s\n\n", platform, arch); + + fprintf(indexfp, "|Suite|Duration|Fail |Skip |Error |\n"); + fprintf(indexfp, "|:----|:------:|:------:|:---:|:----:|\n"); + for (size_t f = 0; f < strlist_count(file_listing); f++) { + char *filename = strlist_item(file_listing, f); + if (!endswith(filename, ".xml")) { + continue; + } + + if (strstr(filename, platform) && strstr(filename, arch)) { + struct JUNIT_Testsuite *testsuite = junitxml_testsuite_read(filename); + if (testsuite) { + if (globals.verbose) { + printf("%s: duration: %0.4f, failed: %d, skipped: %d, errors: %d\n", filename, testsuite->time, testsuite->failures, testsuite->skipped, testsuite->errors); + } + fprintf(indexfp, "|[%s](%s)|%0.4f|%d|%d|%d|\n", filename, filename, testsuite->time, testsuite->failures, testsuite->skipped, testsuite->errors); + /* + * TODO: Display failure/skip/error output. + * + for (size_t i = 0; i < testsuite->_tc_inuse; i++) { + if (testsuite->testcase[i]->tc_result_state_type) { + printf("testcase: %s :: %s\n", testsuite->testcase[i]->classname, testsuite->testcase[i]->name); + if (testsuite->testcase[i]->tc_result_state_type == JUNIT_RESULT_STATE_FAILURE) { + printf("failure: %s\n", testsuite->testcase[i]->result_state.failure->message); + } else if (testsuite->testcase[i]->tc_result_state_type == JUNIT_RESULT_STATE_SKIPPED) { + printf("skipped: %s\n", testsuite->testcase[i]->result_state.skipped->message); + } + } + } + */ + junitxml_testsuite_free(&testsuite); + } else { + fprintf(stderr, "bad test suite: %s: %s\n", strerror(errno), filename); + continue; + } + } + } + fprintf(indexfp, "\n"); + } + fprintf(indexfp, "\n"); + } + guard_strlist_free(&archs); + guard_strlist_free(&platforms); + fclose(indexfp); + popd(); + } else { + fprintf(stderr, "Unable to enter delivery directory: %s\n", ctx->storage.delivery_dir); + guard_free(latest); + return -1; + } + + // "latest" is an array of pointers to ctxs[]. Do not free the contents of the array. + guard_free(latest); + return 0; +} + +void indexer_init_dirs(struct Delivery *ctx, const char *workdir) { + path_store(&ctx->storage.root, PATH_MAX, workdir, ""); + path_store(&ctx->storage.tmpdir, PATH_MAX, ctx->storage.root, "tmp"); + if (delivery_init_tmpdir(ctx)) { + fprintf(stderr, "Failed to configure temporary storage directory\n"); + exit(1); + } + path_store(&ctx->storage.output_dir, PATH_MAX, ctx->storage.root, ""); + path_store(&ctx->storage.tools_dir, PATH_MAX, ctx->storage.output_dir, "tools"); + path_store(&globals.conda_install_prefix, PATH_MAX, ctx->storage.tools_dir, "conda"); + path_store(&ctx->storage.cfgdump_dir, PATH_MAX, ctx->storage.output_dir, "config"); + path_store(&ctx->storage.meta_dir, PATH_MAX, ctx->storage.output_dir, "meta"); + path_store(&ctx->storage.delivery_dir, PATH_MAX, ctx->storage.output_dir, "delivery"); + path_store(&ctx->storage.package_dir, PATH_MAX, ctx->storage.output_dir, "packages"); + path_store(&ctx->storage.results_dir, PATH_MAX, ctx->storage.output_dir, "results"); + path_store(&ctx->storage.wheel_artifact_dir, PATH_MAX, ctx->storage.package_dir, "wheels"); + path_store(&ctx->storage.conda_artifact_dir, PATH_MAX, ctx->storage.package_dir, "conda"); + + char newpath[PATH_MAX] = {0}; + if (getenv("PATH")) { + sprintf(newpath, "%s/bin:%s", ctx->storage.tools_dir, getenv("PATH")); + setenv("PATH", newpath, 1); + } else { + SYSERROR("%s", "environment variable PATH is undefined. Unable to continue."); + exit(1); + } +} + +int main(int argc, char *argv[]) { + size_t rootdirs_total = 0; + char *destdir = NULL; + char **rootdirs = NULL; + int do_html = 0; + int c = 0; + int option_index = 0; + while ((c = getopt_long(argc, argv, "hd:vUw", long_options, &option_index)) != -1) { + switch (c) { + case 'h': + usage(path_basename(argv[0])); + exit(0); + case 'd': + if (mkdir(optarg, 0755)) { + if (errno != 0 && errno != EEXIST) { + SYSERROR("Unable to create destination directory, '%s': %s", optarg, strerror(errno)); + exit(1); + } + } + destdir = realpath(optarg, NULL); + break; + case 'U': + fflush(stdout); + fflush(stderr); + setvbuf(stdout, NULL, _IONBF, 0); + setvbuf(stderr, NULL, _IONBF, 0); + break; + case 'v': + globals.verbose = 1; + break; + case 'w': + do_html = 1; + break; + case '?': + default: + exit(1); + } + } + + int current_index = optind; + if (optind < argc) { + rootdirs_total = argc - current_index; + rootdirs = calloc(rootdirs_total + 1, sizeof(**rootdirs)); + + int i = 0; + while (optind < argc) { + if (argv[optind]) { + if (access(argv[optind], F_OK) < 0) { + fprintf(stderr, "%s: %s\n", argv[optind], strerror(errno)); + exit(1); + } + } + // use first positional argument + rootdirs[i] = realpath(argv[optind], NULL); + optind++; + break; + } + } + + if (isempty(destdir)) { + if (mkdir("output", 0755)) { + if (errno != 0 && errno != EEXIST) { + SYSERROR("Unable to create destination directory, '%s': %s", "output", strerror(errno)); + exit(1); + } + } + destdir = realpath("output", NULL); + } + + if (!rootdirs || !rootdirs_total) { + fprintf(stderr, "You must specify at least one STASIS root directory to index\n"); + exit(1); + } else { + for (size_t i = 0; i < rootdirs_total; i++) { + if (isempty(rootdirs[i]) || !strcmp(rootdirs[i], "/") || !strcmp(rootdirs[i], "\\")) { + SYSERROR("Unsafe directory: %s", rootdirs[i]); + exit(1); + } else if (access(rootdirs[i], F_OK)) { + SYSERROR("%s: %s", rootdirs[i], strerror(errno)); + exit(1); + } + } + } + + char stasis_sysconfdir_tmp[PATH_MAX]; + if (getenv("STASIS_SYSCONFDIR")) { + strncpy(stasis_sysconfdir_tmp, getenv("STASIS_SYSCONFDIR"), sizeof(stasis_sysconfdir_tmp) - 1); + } else { + strncpy(stasis_sysconfdir_tmp, STASIS_SYSCONFDIR, sizeof(stasis_sysconfdir_tmp) - 1); + } + + globals.sysconfdir = realpath(stasis_sysconfdir_tmp, NULL); + if (!globals.sysconfdir) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Unable to resolve path to configuration directory: %s\n", stasis_sysconfdir_tmp); + exit(1); + } + + char *workdir; + char workdir_template[PATH_MAX] = {0}; + char *system_tmp = getenv("TMPDIR"); + if (system_tmp) { + strcat(workdir_template, system_tmp); + } else { + strcat(workdir_template, "/tmp"); + } + strcat(workdir_template, "/stasis-combine.XXXXXX"); + workdir = mkdtemp(workdir_template); + if (!workdir) { + SYSERROR("Unable to create temporary directory: %s", workdir_template); + exit(1); + } else if (isempty(workdir) || !strcmp(workdir, "/") || !strcmp(workdir, "\\")) { + SYSERROR("Unsafe directory: %s", workdir); + exit(1); + } + + struct Delivery ctx; + memset(&ctx, 0, sizeof(ctx)); + + printf(BANNER, VERSION, AUTHOR); + + indexer_init_dirs(&ctx, workdir); + + msg(STASIS_MSG_L1, "%s delivery root %s\n", + rootdirs_total > 1 ? "Merging" : "Indexing", + rootdirs_total > 1 ? "directories" : "directory"); + if (indexer_combine_rootdirs(workdir, rootdirs, rootdirs_total)) { + SYSERROR("%s", "Copy operation failed"); + rmtree(workdir); + exit(1); + } + + if (access(ctx.storage.conda_artifact_dir, F_OK)) { + mkdirs(ctx.storage.conda_artifact_dir, 0755); + } + + if (access(ctx.storage.wheel_artifact_dir, F_OK)) { + mkdirs(ctx.storage.wheel_artifact_dir, 0755); + } + + struct MicromambaInfo m; + if (micromamba_configure(&ctx, &m)) { + SYSERROR("%s", "Unable to configure micromamba"); + exit(1); + } + + msg(STASIS_MSG_L1, "Indexing conda packages\n"); + if (indexer_conda(&ctx, m)) { + SYSERROR("%s", "Conda package indexing operation failed"); + exit(1); + } + + msg(STASIS_MSG_L1, "Indexing wheel packages\n"); + if (indexer_wheels(&ctx)) { + SYSERROR("%s", "Python package indexing operation failed"); + exit(1); + } + + msg(STASIS_MSG_L1, "Loading metadata\n"); + struct StrList *metafiles = NULL; + indexer_get_files(&metafiles, ctx.storage.meta_dir, "*.stasis"); + strlist_sort(metafiles, STASIS_SORT_LEN_ASCENDING); + struct Delivery local[strlist_count(metafiles)]; + + for (size_t i = 0; i < strlist_count(metafiles); i++) { + char *item = strlist_item(metafiles, i); + memset(&local[i], 0, sizeof(ctx)); + memcpy(&local[i], &ctx, sizeof(ctx)); + char path[PATH_MAX]; + sprintf(path, "%s/%s", ctx.storage.meta_dir, item); + if (globals.verbose) { + puts(path); + } + indexer_load_metadata(&local[i], path); + } + + msg(STASIS_MSG_L1, "Generating links to latest release iteration\n"); + if (indexer_symlinks(local, strlist_count(metafiles))) { + SYSERROR("%s", "Link generation failed"); + exit(1); + } + + msg(STASIS_MSG_L1, "Generating README.md\n"); + if (indexer_readmes(local, strlist_count(metafiles))) { + SYSERROR("%s", "README indexing operation failed"); + exit(1); + } + + msg(STASIS_MSG_L1, "Indexing test results\n"); + if (indexer_junitxml_report(local, strlist_count(metafiles))) { + SYSERROR("%s", "Test result indexing operation failed"); + exit(1); + } + + if (do_html) { + msg(STASIS_MSG_L1, "Generating HTML indexes\n"); + if (indexer_make_website(local)) { + SYSERROR("%s", "Site creation failed"); + exit(1); + } + } + + msg(STASIS_MSG_L1, "Copying indexed delivery to '%s'\n", destdir); + char cmd[PATH_MAX]; + memset(cmd, 0, sizeof(cmd)); + sprintf(cmd, "rsync -ah%s --delete --exclude 'tmp/' --exclude 'tools/' '%s/' '%s/'", globals.verbose ? "v" : "q", workdir, destdir); + guard_free(destdir); + + if (globals.verbose) { + puts(cmd); + } + + if (system(cmd)) { + SYSERROR("%s", "Copy operation failed"); + rmtree(workdir); + exit(1); + } + + msg(STASIS_MSG_L1, "Removing work directory: %s\n", workdir); + if (rmtree(workdir)) { + SYSERROR("Failed to remove work directory: %s", strerror(errno)); + } + + guard_free(destdir); + GENERIC_ARRAY_FREE(rootdirs); + guard_strlist_free(&metafiles); + delivery_free(&ctx); + globals_free(); + msg(STASIS_MSG_L1, "Done!\n"); + return 0; +} diff --git a/src/conda.c b/src/conda.c deleted file mode 100644 index e60abc7..0000000 --- a/src/conda.c +++ /dev/null @@ -1,466 +0,0 @@ -// -// Created by jhunk on 5/14/23. -// - -#include -#include "conda.h" - -int micromamba(struct MicromambaInfo *info, char *command, ...) { - struct utsname sys; - uname(&sys); - - tolower_s(sys.sysname); - if (!strcmp(sys.sysname, "darwin")) { - strcpy(sys.sysname, "osx"); - } - - if (!strcmp(sys.machine, "x86_64")) { - strcpy(sys.machine, "64"); - } - - char url[PATH_MAX]; - sprintf(url, "https://micro.mamba.pm/api/micromamba/%s-%s/latest", sys.sysname, sys.machine); - - char installer_path[PATH_MAX]; - sprintf(installer_path, "%s/latest", getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"); - - if (access(installer_path, F_OK)) { - download(url, installer_path, NULL); - } - - char mmbin[PATH_MAX]; - sprintf(mmbin, "%s/micromamba", info->micromamba_prefix); - - if (access(mmbin, F_OK)) { - char untarcmd[PATH_MAX * 2]; - mkdirs(info->micromamba_prefix, 0755); - sprintf(untarcmd, "tar -xvf %s -C %s --strip-components=1 bin/micromamba 1>/dev/null", installer_path, info->micromamba_prefix); - int untarcmd_status = system(untarcmd); - if (untarcmd_status) { - return -1; - } - } - - char cmd[STASIS_BUFSIZ]; - memset(cmd, 0, sizeof(cmd)); - sprintf(cmd, "%s -r %s -p %s ", mmbin, info->conda_prefix, info->conda_prefix); - va_list args; - va_start(args, command); - vsprintf(cmd + strlen(cmd), command, args); - va_end(args); - - mkdirs(info->conda_prefix, 0755); - - char rcpath[PATH_MAX]; - sprintf(rcpath, "%s/.condarc", info->conda_prefix); - touch(rcpath); - - setenv("CONDARC", rcpath, 1); - setenv("MAMBA_ROOT_PREFIX", info->conda_prefix, 1); - int status = system(cmd); - unsetenv("MAMBA_ROOT_PREFIX"); - - return status; -} - -int python_exec(const char *args) { - char command[PATH_MAX]; - memset(command, 0, sizeof(command)); - snprintf(command, sizeof(command) - 1, "python %s", args); - msg(STASIS_MSG_L3, "Executing: %s\n", command); - return system(command); -} - -int pip_exec(const char *args) { - char command[PATH_MAX]; - memset(command, 0, sizeof(command)); - snprintf(command, sizeof(command) - 1, "python -m pip %s", args); - msg(STASIS_MSG_L3, "Executing: %s\n", command); - return system(command); -} - -int pip_index_provides(const char *index_url, const char *spec) { - char cmd[PATH_MAX] = {0}; - char spec_local[255] = {0}; - - if (isempty((char *) spec)) { - // NULL or zero-length; no package spec means there's nothing to do. - return -1; - } - - // Normalize the local spec string - strncpy(spec_local, spec, sizeof(spec_local) - 1); - tolower_s(spec_local); - lstrip(spec_local); - strip(spec_local); - - char logfile[] = "/tmp/STASIS-package_exists.XXXXXX"; - int logfd = mkstemp(logfile); - if (logfd < 0) { - perror(logfile); - remove(logfile); // fail harmlessly if not present - return -1; - } - - - int status = 0; - struct Process proc; - memset(&proc, 0, sizeof(proc)); - proc.redirect_stderr = 1; - strcpy(proc.f_stdout, logfile); - - // Do an installation in dry-run mode to see if the package exists in the given index. - snprintf(cmd, sizeof(cmd) - 1, "python -m pip install --dry-run --no-deps --index-url=%s %s", index_url, spec_local); - status = shell(&proc, cmd); - - // Print errors only when shell() itself throws one - // If some day we want to see the errors thrown by pip too, use this condition instead: (status != 0) - if (status < 0) { - FILE *fp = fdopen(logfd, "r"); - if (!fp) { - remove(logfile); - return -1; - } else { - char line[BUFSIZ] = {0}; - fflush(stdout); - fflush(stderr); - while (fgets(line, sizeof(line) - 1, fp) != NULL) { - fprintf(stderr, "%s", line); - } - fflush(stderr); - fclose(fp); - } - } - remove(logfile); - return proc.returncode == 0; -} - -int conda_exec(const char *args) { - char command[PATH_MAX]; - const char *mamba_commands[] = { - "build", - "install", - "update", - "create", - "list", - "search", - "run", - "info", - "clean", - "activate", - "deactivate", - NULL - }; - char conda_as[6]; - memset(conda_as, 0, sizeof(conda_as)); - - strcpy(conda_as, "conda"); - for (size_t i = 0; mamba_commands[i] != NULL; i++) { - if (startswith(args, mamba_commands[i])) { - strcpy(conda_as, "mamba"); - break; - } - } - - snprintf(command, sizeof(command) - 1, "%s %s", conda_as, args); - msg(STASIS_MSG_L3, "Executing: %s\n", command); - return system(command); -} - -int conda_activate(const char *root, const char *env_name) { - int fd = -1; - FILE *fp = NULL; - const char *init_script_conda = "/etc/profile.d/conda.sh"; - const char *init_script_mamba = "/etc/profile.d/mamba.sh"; - char path_conda[PATH_MAX] = {0}; - char path_mamba[PATH_MAX] = {0}; - char logfile[PATH_MAX] = {0}; - struct Process proc; - memset(&proc, 0, sizeof(proc)); - - // Where to find conda's init scripts - sprintf(path_conda, "%s%s", root, init_script_conda); - sprintf(path_mamba, "%s%s", root, init_script_mamba); - - // Set the path to our stdout log - // Emulate mktemp()'s behavior. Give us a unique file name, but don't use - // the file handle at all. We'll open it as a FILE stream soon enough. - sprintf(logfile, "%s/%s", globals.tmpdir, "shell_XXXXXX"); - fd = mkstemp(logfile); - if (fd < 0) { - perror(logfile); - return -1; - } - close(fd); - - // Configure our process for output to a log file - strcpy(proc.f_stdout, logfile); - - // Verify conda's init scripts are available - if (access(path_conda, F_OK) < 0) { - perror(path_conda); - remove(logfile); - return -1; - } - - if (access(path_mamba, F_OK) < 0) { - perror(path_mamba); - remove(logfile); - return -1; - } - - // Fully activate conda and record its effect on the runtime environment - char command[PATH_MAX * 3]; - snprintf(command, sizeof(command) - 1, "set -a; source %s; source %s; conda activate %s &>/dev/null; env -0", path_conda, path_mamba, env_name); - int retval = shell(&proc, command); - if (retval) { - // it didn't work; drop out for cleanup - remove(logfile); - return retval; - } - - // Parse the log file: - // 1. Extract the environment keys and values from the sub-shell - // 2. Apply it to STASIS's runtime environment - // 3. Now we're ready to execute conda commands anywhere - fp = fopen(proc.f_stdout, "r"); - if (!fp) { - perror(logfile); - return -1; - } - - while (!feof(fp)) { - char buf[STASIS_BUFSIZ] = {0}; - int ch = 0; - size_t z = 0; - // We are ingesting output from "env -0" and can't use fgets() - // Copy each character into the buffer until we encounter '\0' or EOF - while (z < sizeof(buf) && (ch = (int) fgetc(fp)) != 0) { - if (ch == EOF) { - break; - } - buf[z] = (char) ch; - z++; - } - buf[strlen(buf)] = 0; - - if (!strlen(buf)) { - continue; - } - - char **part = split(buf, "=", 1); - if (!part) { - perror("unable to split environment variable buffer"); - return -1; - } - if (!part[0]) { - msg(STASIS_MSG_WARN | STASIS_MSG_L1, "Invalid environment variable key ignored: '%s'\n", buf); - } else if (!part[1]) { - msg(STASIS_MSG_WARN | STASIS_MSG_L1, "Invalid environment variable value ignored: '%s'\n", buf); - } else { - setenv(part[0], part[1], 1); - } - GENERIC_ARRAY_FREE(part); - } - fclose(fp); - remove(logfile); - return 0; -} - -int conda_check_required() { - int status = 0; - struct StrList *result = NULL; - char cmd[PATH_MAX] = {0}; - const char *conda_minimum_viable_tools[] = { - "boa", - "conda-build", - "conda-verify", - NULL - }; - - // Construct a "conda list" command that searches for all required packages - // using conda's (python's) regex matching - strcat(cmd, "conda list '"); - for (size_t i = 0; conda_minimum_viable_tools[i] != NULL; i++) { - strcat(cmd, "^"); - strcat(cmd, conda_minimum_viable_tools[i]); - if (conda_minimum_viable_tools[i + 1] != NULL) { - strcat(cmd, "|"); - } - } - strcat(cmd, "' | cut -d ' ' -f 1"); - - // Verify all required packages are installed - char *cmd_out = shell_output(cmd, &status); - if (cmd_out) { - size_t found = 0; - result = strlist_init(); - strlist_append_tokenize(result, cmd_out, "\n"); - for (size_t i = 0; i < strlist_count(result); i++) { - char *item = strlist_item(result, i); - if (isempty(item) || startswith(item, "#")) { - continue; - } - - for (size_t x = 0; conda_minimum_viable_tools[x] != NULL; x++) { - if (!strcmp(item, conda_minimum_viable_tools[x])) { - found++; - } - } - } - if (found < (sizeof(conda_minimum_viable_tools) / sizeof(*conda_minimum_viable_tools)) - 1) { - guard_free(cmd_out); - guard_strlist_free(&result); - return 1; - } - guard_free(cmd_out); - guard_strlist_free(&result); - } else { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "The base package requirement check could not be performed\n"); - return 2; - } - return 0; -} - -int conda_setup_headless() { - if (globals.verbose) { - conda_exec("config --system --set quiet false"); - } else { - // Not verbose, so squelch conda's noise - conda_exec("config --system --set quiet true"); - } - - // Configure conda for headless CI - conda_exec("config --system --set auto_update_conda false"); // never update conda automatically - conda_exec("config --system --set notify_outdated_conda false"); // never notify about outdated conda version - conda_exec("config --system --set always_yes true"); // never prompt for input - conda_exec("config --system --set safety_checks disabled"); // speedup - conda_exec("config --system --set rollback_enabled false"); // speedup - conda_exec("config --system --set report_errors false"); // disable data sharing - conda_exec("config --system --set solver libmamba"); // use a real solver - - char cmd[PATH_MAX]; - size_t total = 0; - if (globals.conda_packages && strlist_count(globals.conda_packages)) { - memset(cmd, 0, sizeof(cmd)); - strcpy(cmd, "install "); - - total = strlist_count(globals.conda_packages); - for (size_t i = 0; i < total; i++) { - char *item = strlist_item(globals.conda_packages, i); - if (isempty(item)) { - continue; - } - sprintf(cmd + strlen(cmd), "'%s'", item); - if (i < total - 1) { - strcat(cmd, " "); - } - } - - if (conda_exec(cmd)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Unable to install user-defined base packages (conda)\n"); - return 1; - } - } - - if (globals.pip_packages && strlist_count(globals.pip_packages)) { - memset(cmd, 0, sizeof(cmd)); - strcpy(cmd, "install "); - - total = strlist_count(globals.pip_packages); - for (size_t i = 0; i < total; i++) { - char *item = strlist_item(globals.pip_packages, i); - if (isempty(item)) { - continue; - } - sprintf(cmd + strlen(cmd), "'%s'", item); - if (i < total - 1) { - strcat(cmd, " "); - } - } - - if (pip_exec(cmd)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Unable to install user-defined base packages (pip)\n"); - return 1; - } - } - - if (conda_check_required()) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Your STASIS configuration lacks the bare" - " minimum software required to build conda packages." - " Please fix it.\n"); - return 1; - } - - if (globals.always_update_base_environment) { - if (conda_exec("update --all")) { - fprintf(stderr, "conda update was unsuccessful\n"); - return 1; - } - } - - return 0; -} - -int conda_env_create_from_uri(char *name, char *uri) { - char env_command[PATH_MAX]; - sprintf(env_command, "env create -n %s -f %s", name, uri); - return conda_exec(env_command); -} - -int conda_env_create(char *name, char *python_version, char *packages) { - char env_command[PATH_MAX]; - sprintf(env_command, "create -n %s python=%s %s", name, python_version, packages ? packages : ""); - return conda_exec(env_command); -} - -int conda_env_remove(char *name) { - char env_command[PATH_MAX]; - sprintf(env_command, "env remove -n %s", name); - return conda_exec(env_command); -} - -int conda_env_export(char *name, char *output_dir, char *output_filename) { - char env_command[PATH_MAX]; - sprintf(env_command, "env export -n %s -f %s/%s.yml", name, output_dir, output_filename); - return conda_exec(env_command); -} - -char *conda_get_active_environment() { - const char *name = getenv("CONDA_DEFAULT_ENV"); - if (!name) { - return NULL; - } - - char *result = NULL; - result = strdup(name); - if (!result) { - return NULL; - } - - return result; -} - -int conda_provides(const char *spec) { - struct Process proc; - memset(&proc, 0, sizeof(proc)); - strcpy(proc.f_stdout, "/dev/null"); - strcpy(proc.f_stderr, "/dev/null"); - - // It's worth noting the departure from using conda_exec() here: - // conda_exec() expects the program output to be visible to the user. - // For this operation we only need the exit value. - char cmd[PATH_MAX] = {0}; - snprintf(cmd, sizeof(cmd) - 1, "mamba search --use-index-cache %s", spec); - if (shell(&proc, cmd) < 0) { - fprintf(stderr, "shell: %s", strerror(errno)); - return -1; - } - return proc.returncode == 0; -} - -int conda_index(const char *path) { - char command[PATH_MAX]; - sprintf(command, "index %s", path); - return conda_exec(command); -} diff --git a/src/copy.c b/src/copy.c deleted file mode 100644 index f69a756..0000000 --- a/src/copy.c +++ /dev/null @@ -1,86 +0,0 @@ -#include "copy.h" - -int copy2(const char *src, const char *dest, unsigned int op) { - size_t bytes_read; - size_t bytes_written; - char buf[STASIS_BUFSIZ]; - struct stat src_stat, dnamest; - FILE *fp1, *fp2; - - if (lstat(src, &src_stat) < 0) { - perror(src); - return -1; - } - - if (access(dest, F_OK) == 0) { - unlink(dest); - } - - char dname[1024] = {0}; - strcpy(dname, dest); - char *dname_endptr; - - dname_endptr = strrchr(dname, '/'); - if (dname_endptr != NULL) { - *dname_endptr = '\0'; - } - - stat(dname, &dnamest); - if (S_ISLNK(src_stat.st_mode)) { - char lpath[1024] = {0}; - if (readlink(src, lpath, sizeof(lpath)) < 0) { - perror(src); - return -1; - } - if (symlink(lpath, dest) < 0) { - // silent - return -1; - } - } else if (S_ISREG(src_stat.st_mode) && src_stat.st_nlink > 2 && src_stat.st_dev == dnamest.st_dev) { - if (link(src, dest) < 0) { - perror(src); - return -1; - } - } else if (S_ISFIFO(src_stat.st_mode) || S_ISBLK(src_stat.st_mode) || S_ISCHR(src_stat.st_mode) || S_ISSOCK(src_stat.st_mode)) { - if (mknod(dest, src_stat.st_mode, src_stat.st_rdev) < 0) { - perror(src); - return -1; - } - } else if (S_ISREG(src_stat.st_mode)) { - fp1 = fopen(src, "rb"); - if (!fp1) { - perror(src); - return -1; - } - - fp2 = fopen(dest, "w+b"); - if (!fp2) { - perror(dest); - return -1; - } - - bytes_written = 0; - while ((bytes_read = fread(buf, sizeof(char), sizeof(buf), fp1)) != 0) { - bytes_written += fwrite(buf, sizeof(char), bytes_read, fp2); - } - fclose(fp1); - fclose(fp2); - - if (bytes_written != (size_t) src_stat.st_size) { - fprintf(stderr, "%s: SHORT WRITE (expected %zu bytes, but wrote %zu bytes)\n", dest, src_stat.st_size, bytes_written); - return -1; - } - - if (op & CT_OWNER && chown(dest, src_stat.st_uid, src_stat.st_gid) < 0) { - perror(dest); - } - - if (op & CT_PERM && chmod(dest, src_stat.st_mode) < 0) { - perror(dest); - } - } else { - errno = EOPNOTSUPP; - return -1; - } - return 0; -} diff --git a/src/delivery.c b/src/delivery.c deleted file mode 100644 index 07e04c8..0000000 --- a/src/delivery.c +++ /dev/null @@ -1,317 +0,0 @@ -#include "core.h" - -void delivery_free(struct Delivery *ctx) { - guard_free(ctx->system.arch); - GENERIC_ARRAY_FREE(ctx->system.platform); - guard_free(ctx->meta.name); - guard_free(ctx->meta.version); - guard_free(ctx->meta.codename); - guard_free(ctx->meta.mission); - guard_free(ctx->meta.python); - guard_free(ctx->meta.mission); - guard_free(ctx->meta.python_compact); - guard_free(ctx->meta.based_on); - guard_runtime_free(ctx->runtime.environ); - guard_free(ctx->storage.root); - guard_free(ctx->storage.tmpdir); - guard_free(ctx->storage.delivery_dir); - guard_free(ctx->storage.tools_dir); - guard_free(ctx->storage.package_dir); - guard_free(ctx->storage.results_dir); - guard_free(ctx->storage.output_dir); - guard_free(ctx->storage.conda_install_prefix); - guard_free(ctx->storage.conda_artifact_dir); - guard_free(ctx->storage.conda_staging_dir); - guard_free(ctx->storage.conda_staging_url); - guard_free(ctx->storage.wheel_artifact_dir); - guard_free(ctx->storage.wheel_staging_dir); - guard_free(ctx->storage.wheel_staging_url); - guard_free(ctx->storage.build_dir); - guard_free(ctx->storage.build_recipes_dir); - guard_free(ctx->storage.build_sources_dir); - guard_free(ctx->storage.build_testing_dir); - guard_free(ctx->storage.build_docker_dir); - guard_free(ctx->storage.mission_dir); - guard_free(ctx->storage.docker_artifact_dir); - guard_free(ctx->storage.meta_dir); - guard_free(ctx->storage.package_dir); - guard_free(ctx->storage.cfgdump_dir); - guard_free(ctx->info.time_str_epoch); - guard_free(ctx->info.build_name); - guard_free(ctx->info.build_number); - guard_free(ctx->info.release_name); - guard_free(ctx->conda.installer_baseurl); - guard_free(ctx->conda.installer_name); - guard_free(ctx->conda.installer_version); - guard_free(ctx->conda.installer_platform); - guard_free(ctx->conda.installer_arch); - guard_free(ctx->conda.installer_path); - guard_free(ctx->conda.tool_version); - guard_free(ctx->conda.tool_build_version); - guard_strlist_free(&ctx->conda.conda_packages); - guard_strlist_free(&ctx->conda.conda_packages_defer); - guard_strlist_free(&ctx->conda.pip_packages); - guard_strlist_free(&ctx->conda.pip_packages_defer); - guard_strlist_free(&ctx->conda.wheels_packages); - - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { - guard_free(ctx->tests[i].name); - guard_free(ctx->tests[i].version); - guard_free(ctx->tests[i].repository); - guard_free(ctx->tests[i].repository_info_ref); - guard_free(ctx->tests[i].repository_info_tag); - guard_strlist_free(&ctx->tests[i].repository_remove_tags); - guard_free(ctx->tests[i].script); - guard_free(ctx->tests[i].build_recipe); - // test-specific runtime variables - guard_runtime_free(ctx->tests[i].runtime.environ); - } - - guard_free(ctx->rules.release_fmt); - guard_free(ctx->rules.build_name_fmt); - guard_free(ctx->rules.build_number_fmt); - - guard_free(ctx->deploy.docker.test_script); - guard_free(ctx->deploy.docker.registry); - guard_free(ctx->deploy.docker.image_compression); - guard_strlist_free(&ctx->deploy.docker.tags); - guard_strlist_free(&ctx->deploy.docker.build_args); - - for (size_t i = 0; i < sizeof(ctx->deploy.jfrog) / sizeof(ctx->deploy.jfrog[0]); i++) { - guard_free(ctx->deploy.jfrog[i].repo); - guard_free(ctx->deploy.jfrog[i].dest); - guard_strlist_free(&ctx->deploy.jfrog[i].files); - } - - if (ctx->_stasis_ini_fp.delivery) { - ini_free(&ctx->_stasis_ini_fp.delivery); - } - guard_free(ctx->_stasis_ini_fp.delivery_path); - - if (ctx->_stasis_ini_fp.cfg) { - // optional extras - ini_free(&ctx->_stasis_ini_fp.cfg); - } - guard_free(ctx->_stasis_ini_fp.cfg_path); - - if (ctx->_stasis_ini_fp.mission) { - ini_free(&ctx->_stasis_ini_fp.mission); - } - guard_free(ctx->_stasis_ini_fp.mission_path); -} - -int delivery_format_str(struct Delivery *ctx, char **dest, const char *fmt) { - size_t fmt_len = strlen(fmt); - - if (!*dest) { - *dest = calloc(STASIS_NAME_MAX, sizeof(**dest)); - if (!*dest) { - return -1; - } - } - - for (size_t i = 0; i < fmt_len; i++) { - if (fmt[i] == '%' && strlen(&fmt[i])) { - i++; - switch (fmt[i]) { - case 'n': // name - strcat(*dest, ctx->meta.name); - break; - case 'c': // codename - strcat(*dest, ctx->meta.codename); - break; - case 'm': // mission - strcat(*dest, ctx->meta.mission); - break; - case 'r': // revision - sprintf(*dest + strlen(*dest), "%d", ctx->meta.rc); - break; - case 'R': // "final"-aware revision - if (ctx->meta.final) - strcat(*dest, "final"); - else - sprintf(*dest + strlen(*dest), "%d", ctx->meta.rc); - break; - case 'v': // version - strcat(*dest, ctx->meta.version); - break; - case 'P': // python version - strcat(*dest, ctx->meta.python); - break; - case 'p': // python version major/minor - strcat(*dest, ctx->meta.python_compact); - break; - case 'a': // system architecture name - strcat(*dest, ctx->system.arch); - break; - case 'o': // system platform (OS) name - strcat(*dest, ctx->system.platform[DELIVERY_PLATFORM_RELEASE]); - break; - case 't': // unix epoch - sprintf(*dest + strlen(*dest), "%ld", ctx->info.time_now); - break; - default: // unknown formatter, write as-is - sprintf(*dest + strlen(*dest), "%c%c", fmt[i - 1], fmt[i]); - break; - } - } else { // write non-format text - sprintf(*dest + strlen(*dest), "%c", fmt[i]); - } - } - return 0; -} - -void delivery_defer_packages(struct Delivery *ctx, int type) { - struct StrList *dataptr = NULL; - struct StrList *deferred = NULL; - char *name = NULL; - char cmd[PATH_MAX]; - - memset(cmd, 0, sizeof(cmd)); - - char mode[10]; - if (DEFER_CONDA == type) { - dataptr = ctx->conda.conda_packages; - deferred = ctx->conda.conda_packages_defer; - strcpy(mode, "conda"); - } else if (DEFER_PIP == type) { - dataptr = ctx->conda.pip_packages; - deferred = ctx->conda.pip_packages_defer; - strcpy(mode, "pip"); - } else { - SYSERROR("BUG: type %d does not map to a supported package manager!\n", type); - exit(1); - } - msg(STASIS_MSG_L2, "Filtering %s packages by test definition...\n", mode); - - struct StrList *filtered = NULL; - filtered = strlist_init(); - for (size_t i = 0; i < strlist_count(dataptr); i++) { - int build_for_host = 0; - - name = strlist_item(dataptr, i); - if (!strlen(name) || isblank(*name) || isspace(*name)) { - // no data - continue; - } - - // Compile a list of packages that are *also* to be tested. - char *spec_begin = strpbrk(name, "@~=<>!"); - char *spec_end = spec_begin; - char package_name[255] = {0}; - - if (spec_end) { - // A version is present in the package name. Jump past operator(s). - while (*spec_end != '\0' && !isalnum(*spec_end)) { - spec_end++; - } - strncpy(package_name, name, spec_begin - name); - } else { - strncpy(package_name, name, sizeof(package_name) - 1); - } - - msg(STASIS_MSG_L3, "package '%s': ", package_name); - - // When spec is present in name, set tests->version to the version detected in the name - for (size_t x = 0; x < sizeof(ctx->tests) / sizeof(ctx->tests[0]) && ctx->tests[x].name != NULL; x++) { - struct Test *test = &ctx->tests[x]; - char nametmp[1024] = {0}; - - if (spec_end != NULL && spec_begin != NULL) { - strncpy(nametmp, name, spec_begin - name); - } else { - strcpy(nametmp, name); - } - // Is the [test:NAME] in the package name? - if (!strcmp(nametmp, test->name)) { - // Override test->version when a version is provided by the (pip|conda)_package list item - guard_free(test->version); - if (spec_begin && spec_end) { - test->version = strdup(spec_end); - } else { - // There are too many possible default branches nowadays: master, main, develop, xyz, etc. - // HEAD is a safe bet. - test->version = strdup("HEAD"); - } - - // Is the list item a git+schema:// URL? - if (strstr(nametmp, "git+") && strstr(nametmp, "://")) { - char *xrepo = strstr(nametmp, "+"); - if (xrepo) { - xrepo++; - guard_free(test->repository); - test->repository = strdup(xrepo); - xrepo = NULL; - } - // Extract the name of the package - char *xbasename = path_basename(nametmp); - if (xbasename) { - // Replace the git+schema:// URL with the package name - strlist_set(&dataptr, i, xbasename); - name = strlist_item(dataptr, i); - } - } - - int upstream_exists = 0; - if (DEFER_PIP == type) { - upstream_exists = pip_index_provides(PYPI_INDEX_DEFAULT, name); - } else if (DEFER_CONDA == type) { - upstream_exists = conda_provides(name); - } else { - fprintf(stderr, "\nUnknown package type: %d\n", type); - exit(1); - } - - if (upstream_exists < 0) { - fprintf(stderr, "%s's existence command failed for '%s'\n" - "(This may be due to a network/firewall issue!)\n", mode, name); - exit(1); - } - if (!upstream_exists) { - build_for_host = 1; - } else { - build_for_host = 0; - } - - break; - } - } - - if (build_for_host) { - printf("BUILD FOR HOST\n"); - strlist_append(&deferred, name); - } else { - printf("USE EXTERNAL\n"); - strlist_append(&filtered, name); - } - } - - if (!strlist_count(deferred)) { - msg(STASIS_MSG_WARN | STASIS_MSG_L2, "No %s packages were filtered by test definitions\n", mode); - } else { - if (DEFER_CONDA == type) { - strlist_free(&ctx->conda.conda_packages); - ctx->conda.conda_packages = strlist_copy(filtered); - } else if (DEFER_PIP == type) { - strlist_free(&ctx->conda.pip_packages); - ctx->conda.pip_packages = strlist_copy(filtered); - } - } - if (filtered) { - strlist_free(&filtered); - } -} - -void delivery_gather_tool_versions(struct Delivery *ctx) { - int status = 0; - - // Extract version from tool output - ctx->conda.tool_version = shell_output("conda --version", &status); - if (ctx->conda.tool_version) - strip(ctx->conda.tool_version); - - ctx->conda.tool_build_version = shell_output("conda build --version", &status); - if (ctx->conda.tool_build_version) - strip(ctx->conda.tool_version); -} - diff --git a/src/delivery_artifactory.c b/src/delivery_artifactory.c deleted file mode 100644 index 27f4823..0000000 --- a/src/delivery_artifactory.c +++ /dev/null @@ -1,192 +0,0 @@ -#include "delivery.h" - -int delivery_init_artifactory(struct Delivery *ctx) { - int status = 0; - char dest[PATH_MAX] = {0}; - char filepath[PATH_MAX] = {0}; - snprintf(dest, sizeof(dest) - 1, "%s/bin", ctx->storage.tools_dir); - snprintf(filepath, sizeof(dest) - 1, "%s/bin/jf", ctx->storage.tools_dir); - - if (!access(filepath, F_OK)) { - // already have it - msg(STASIS_MSG_L3, "Skipped download, %s already exists\n", filepath); - goto delivery_init_artifactory_envsetup; - } - - char *platform = ctx->system.platform[DELIVERY_PLATFORM]; - msg(STASIS_MSG_L3, "Downloading %s for %s %s\n", globals.jfrog.remote_filename, platform, ctx->system.arch); - if ((status = artifactory_download_cli(dest, - globals.jfrog.jfrog_artifactory_base_url, - globals.jfrog.jfrog_artifactory_product, - globals.jfrog.cli_major_ver, - globals.jfrog.version, - platform, - ctx->system.arch, - globals.jfrog.remote_filename))) { - remove(filepath); - } - - delivery_init_artifactory_envsetup: - // CI (ridiculously generic, why?) disables interactive prompts and progress bar output - setenv("CI", "1", 1); - - // JFROG_CLI_HOME_DIR is where .jfrog is stored - char path[PATH_MAX] = {0}; - snprintf(path, sizeof(path) - 1, "%s/.jfrog", ctx->storage.build_dir); - setenv("JFROG_CLI_HOME_DIR", path, 1); - - // JFROG_CLI_TEMP_DIR is where the obvious is stored - setenv("JFROG_CLI_TEMP_DIR", ctx->storage.tmpdir, 1); - return status; -} - -int delivery_artifact_upload(struct Delivery *ctx) { - int status = 0; - - if (jfrt_auth_init(&ctx->deploy.jfrog_auth)) { - fprintf(stderr, "Failed to initialize Artifactory authentication context\n"); - return -1; - } - - for (size_t i = 0; i < sizeof(ctx->deploy.jfrog) / sizeof(*ctx->deploy.jfrog); i++) { - if (!ctx->deploy.jfrog[i].files || !ctx->deploy.jfrog[i].dest) { - break; - } - jfrt_upload_init(&ctx->deploy.jfrog[i].upload_ctx); - - if (!globals.jfrog.repo) { - msg(STASIS_MSG_WARN, "Artifactory repository path is not configured!\n"); - fprintf(stderr, "set STASIS_JF_REPO environment variable...\nOr append to configuration file:\n\n"); - fprintf(stderr, "[deploy:artifactory]\nrepo = example/generic/repo/path\n\n"); - status++; - break; - } else if (!ctx->deploy.jfrog[i].repo) { - ctx->deploy.jfrog[i].repo = strdup(globals.jfrog.repo); - } - - if (!ctx->deploy.jfrog[i].repo || isempty(ctx->deploy.jfrog[i].repo) || !strlen(ctx->deploy.jfrog[i].repo)) { - // Unlikely to trigger if the config parser is working correctly - msg(STASIS_MSG_ERROR, "Artifactory repository path is empty. Cannot continue.\n"); - status++; - break; - } - - ctx->deploy.jfrog[i].upload_ctx.workaround_parent_only = true; - ctx->deploy.jfrog[i].upload_ctx.build_name = ctx->info.build_name; - ctx->deploy.jfrog[i].upload_ctx.build_number = ctx->info.build_number; - - char files[PATH_MAX]; - char dest[PATH_MAX]; // repo + remote dir - - if (jfrog_cli_rt_ping(&ctx->deploy.jfrog_auth)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Unable to contact artifactory server: %s\n", ctx->deploy.jfrog_auth.url); - return -1; - } - - if (strlist_count(ctx->deploy.jfrog[i].files)) { - for (size_t f = 0; f < strlist_count(ctx->deploy.jfrog[i].files); f++) { - memset(dest, 0, sizeof(dest)); - memset(files, 0, sizeof(files)); - snprintf(dest, sizeof(dest) - 1, "%s/%s", ctx->deploy.jfrog[i].repo, ctx->deploy.jfrog[i].dest); - snprintf(files, sizeof(files) - 1, "%s", strlist_item(ctx->deploy.jfrog[i].files, f)); - status += jfrog_cli_rt_upload(&ctx->deploy.jfrog_auth, &ctx->deploy.jfrog[i].upload_ctx, files, dest); - } - } - } - - if (globals.enable_artifactory_build_info) { - if (!status && ctx->deploy.jfrog[0].files && ctx->deploy.jfrog[0].dest) { - jfrog_cli_rt_build_collect_env(&ctx->deploy.jfrog_auth, ctx->deploy.jfrog[0].upload_ctx.build_name, - ctx->deploy.jfrog[0].upload_ctx.build_number); - jfrog_cli_rt_build_publish(&ctx->deploy.jfrog_auth, ctx->deploy.jfrog[0].upload_ctx.build_name, - ctx->deploy.jfrog[0].upload_ctx.build_number); - } - } else { - msg(STASIS_MSG_WARN | STASIS_MSG_L2, "Artifactory build info upload is disabled by CLI argument\n"); - } - - return status; -} - -int delivery_mission_render_files(struct Delivery *ctx) { - if (!ctx->storage.mission_dir) { - fprintf(stderr, "Mission directory is not configured. Context not initialized?\n"); - return -1; - } - struct Data { - char *src; - char *dest; - } data; - struct INIFILE *cfg = ctx->_stasis_ini_fp.mission; - union INIVal val; - - memset(&data, 0, sizeof(data)); - data.src = calloc(PATH_MAX, sizeof(*data.src)); - if (!data.src) { - perror("data.src"); - return -1; - } - - for (size_t i = 0; i < cfg->section_count; i++) { - char *section_name = cfg->section[i]->key; - if (!startswith(section_name, "template:")) { - continue; - } - val.as_char_p = strchr(section_name, ':') + 1; - if (val.as_char_p && isempty(val.as_char_p)) { - guard_free(data.src); - return 1; - } - sprintf(data.src, "%s/%s/%s", ctx->storage.mission_dir, ctx->meta.mission, val.as_char_p); - msg(STASIS_MSG_L2, "%s\n", data.src); - - int err = 0; - data.dest = ini_getval_str(cfg, section_name, "destination", INI_READ_RENDER, &err); - - char *contents; - struct stat st; - if (lstat(data.src, &st)) { - perror(data.src); - guard_free(data.dest); - continue; - } - - contents = calloc(st.st_size + 1, sizeof(*contents)); - if (!contents) { - perror("template file contents"); - guard_free(data.dest); - continue; - } - - FILE *fp; - fp = fopen(data.src, "rb"); - if (!fp) { - perror(data.src); - guard_free(contents); - guard_free(data.dest); - continue; - } - - if (fread(contents, st.st_size, sizeof(*contents), fp) < 1) { - perror("while reading template file"); - guard_free(contents); - guard_free(data.dest); - fclose(fp); - continue; - } - fclose(fp); - - msg(STASIS_MSG_L3, "Writing %s\n", data.dest); - if (tpl_render_to_file(contents, data.dest)) { - guard_free(contents); - guard_free(data.dest); - continue; - } - guard_free(contents); - guard_free(data.dest); - } - - guard_free(data.src); - return 0; -} - diff --git a/src/delivery_build.c b/src/delivery_build.c deleted file mode 100644 index 3777a4c..0000000 --- a/src/delivery_build.c +++ /dev/null @@ -1,191 +0,0 @@ -#include -#include "delivery.h" - -int delivery_build_recipes(struct Delivery *ctx) { - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { - char *recipe_dir = NULL; - if (ctx->tests[i].build_recipe) { // build a conda recipe - int recipe_type; - int status; - if (recipe_clone(ctx->storage.build_recipes_dir, ctx->tests[i].build_recipe, NULL, &recipe_dir)) { - fprintf(stderr, "Encountered an issue while cloning recipe for: %s\n", ctx->tests[i].name); - return -1; - } - if (!recipe_dir) { - fprintf(stderr, "BUG: recipe_clone() succeeded but recipe_dir is NULL: %s\n", strerror(errno)); - return -1; - } - recipe_type = recipe_get_type(recipe_dir); - if(!pushd(recipe_dir)) { - if (RECIPE_TYPE_ASTROCONDA == recipe_type) { - pushd(path_basename(ctx->tests[i].repository)); - } else if (RECIPE_TYPE_CONDA_FORGE == recipe_type) { - pushd("recipe"); - } - - char recipe_version[100]; - char recipe_buildno[100]; - char recipe_git_url[PATH_MAX]; - char recipe_git_rev[PATH_MAX]; - - //sprintf(recipe_version, "{%% set version = GIT_DESCRIBE_TAG ~ \".dev\" ~ GIT_DESCRIBE_NUMBER ~ \"+\" ~ GIT_DESCRIBE_HASH %%}"); - //sprintf(recipe_git_url, " git_url: %s", ctx->tests[i].repository); - //sprintf(recipe_git_rev, " git_rev: %s", ctx->tests[i].version); - // TODO: Conditionally download archives if github.com is the origin. Else, use raw git_* keys ^^^ - sprintf(recipe_version, "{%% set version = \"%s\" %%}", ctx->tests[i].repository_info_tag ? ctx->tests[i].repository_info_tag : ctx->tests[i].version); - sprintf(recipe_git_url, " url: %s/archive/refs/tags/{{ version }}.tar.gz", ctx->tests[i].repository); - strcpy(recipe_git_rev, ""); - sprintf(recipe_buildno, " number: 0"); - - unsigned flags = REPLACE_TRUNCATE_AFTER_MATCH; - //file_replace_text("meta.yaml", "{% set version = ", recipe_version); - if (ctx->meta.final) { // remove this. i.e. statis cannot deploy a release to conda-forge - sprintf(recipe_version, "{%% set version = \"%s\" %%}", ctx->tests[i].version); - // TODO: replace sha256 of tagged archive - // TODO: leave the recipe unchanged otherwise. in theory this should produce the same conda package hash as conda forge. - // For now, remove the sha256 requirement - file_replace_text("meta.yaml", "sha256:", "\n", flags); - } else { - file_replace_text("meta.yaml", "{% set version = ", recipe_version, flags); - file_replace_text("meta.yaml", " url:", recipe_git_url, flags); - //file_replace_text("meta.yaml", "sha256:", recipe_git_rev); - file_replace_text("meta.yaml", " sha256:", "\n", flags); - file_replace_text("meta.yaml", " number:", recipe_buildno, flags); - } - - char command[PATH_MAX]; - if (RECIPE_TYPE_CONDA_FORGE == recipe_type) { - char arch[STASIS_NAME_MAX] = {0}; - char platform[STASIS_NAME_MAX] = {0}; - - strcpy(platform, ctx->system.platform[DELIVERY_PLATFORM]); - if (strstr(platform, "Darwin")) { - memset(platform, 0, sizeof(platform)); - strcpy(platform, "osx"); - } - tolower_s(platform); - if (strstr(ctx->system.arch, "arm64")) { - strcpy(arch, "arm64"); - } else if (strstr(ctx->system.arch, "64")) { - strcpy(arch, "64"); - } else { - strcat(arch, "32"); // blind guess - } - tolower_s(arch); - - sprintf(command, "mambabuild --python=%s -m ../.ci_support/%s_%s_.yaml .", - ctx->meta.python, platform, arch); - } else { - sprintf(command, "mambabuild --python=%s .", ctx->meta.python); - } - status = conda_exec(command); - if (status) { - guard_free(recipe_dir); - return -1; - } - - if (RECIPE_TYPE_GENERIC != recipe_type) { - popd(); - } - popd(); - } else { - fprintf(stderr, "Unable to enter recipe directory %s: %s\n", recipe_dir, strerror(errno)); - guard_free(recipe_dir); - return -1; - } - } - guard_free(recipe_dir); - } - return 0; -} - -int filter_repo_tags(char *repo, struct StrList *patterns) { - int result = 0; - - if (!pushd(repo)) { - int list_status = 0; - char *tags_raw = shell_output("git tag -l", &list_status); - struct StrList *tags = strlist_init(); - strlist_append_tokenize(tags, tags_raw, LINE_SEP); - - for (size_t i = 0; tags && i < strlist_count(tags); i++) { - char *tag = strlist_item(tags, i); - for (size_t p = 0; p < strlist_count(patterns); p++) { - char *pattern = strlist_item(patterns, p); - int match = fnmatch(pattern, tag, 0); - if (!match) { - char cmd[PATH_MAX] = {0}; - sprintf(cmd, "git tag -d %s", tag); - result += system(cmd); - break; - } - } - } - guard_strlist_free(&tags); - guard_free(tags_raw); - popd(); - } else { - result = -1; - } - return result; -} - -struct StrList *delivery_build_wheels(struct Delivery *ctx) { - struct StrList *result = NULL; - struct Process proc; - memset(&proc, 0, sizeof(proc)); - - result = strlist_init(); - if (!result) { - perror("unable to allocate memory for string list"); - return NULL; - } - - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { - if (!ctx->tests[i].build_recipe && ctx->tests[i].repository) { // build from source - char srcdir[PATH_MAX]; - char wheeldir[PATH_MAX]; - memset(srcdir, 0, sizeof(srcdir)); - memset(wheeldir, 0, sizeof(wheeldir)); - - sprintf(srcdir, "%s/%s", ctx->storage.build_sources_dir, ctx->tests[i].name); - git_clone(&proc, ctx->tests[i].repository, srcdir, ctx->tests[i].version); - - if (ctx->tests[i].repository_remove_tags && strlist_count(ctx->tests[i].repository_remove_tags)) { - filter_repo_tags(srcdir, ctx->tests[i].repository_remove_tags); - } - - if (!pushd(srcdir)) { - char dname[NAME_MAX]; - char outdir[PATH_MAX]; - char cmd[PATH_MAX * 2]; - memset(dname, 0, sizeof(dname)); - memset(outdir, 0, sizeof(outdir)); - memset(cmd, 0, sizeof(outdir)); - - strcpy(dname, ctx->tests[i].name); - tolower_s(dname); - sprintf(outdir, "%s/%s", ctx->storage.wheel_artifact_dir, dname); - if (mkdirs(outdir, 0755)) { - fprintf(stderr, "failed to create output directory: %s\n", outdir); - guard_strlist_free(&result); - return NULL; - } - - sprintf(cmd, "-m build -w -o %s", outdir); - if (python_exec(cmd)) { - fprintf(stderr, "failed to generate wheel package for %s-%s\n", ctx->tests[i].name, ctx->tests[i].version); - guard_strlist_free(&result); - return NULL; - } - popd(); - } else { - fprintf(stderr, "Unable to enter source directory %s: %s\n", srcdir, strerror(errno)); - guard_strlist_free(&result); - return NULL; - } - } - } - return result; -} - diff --git a/src/delivery_conda.c b/src/delivery_conda.c deleted file mode 100644 index 93a06fc..0000000 --- a/src/delivery_conda.c +++ /dev/null @@ -1,110 +0,0 @@ -#include "delivery.h" - -void delivery_get_conda_installer_url(struct Delivery *ctx, char *result) { - if (ctx->conda.installer_version) { - // Use version specified by configuration file - sprintf(result, "%s/%s-%s-%s-%s.sh", ctx->conda.installer_baseurl, - ctx->conda.installer_name, - ctx->conda.installer_version, - ctx->conda.installer_platform, - ctx->conda.installer_arch); - } else { - // Use latest installer - sprintf(result, "%s/%s-%s-%s.sh", ctx->conda.installer_baseurl, - ctx->conda.installer_name, - ctx->conda.installer_platform, - ctx->conda.installer_arch); - } - -} - -int delivery_get_conda_installer(struct Delivery *ctx, char *installer_url) { - char script_path[PATH_MAX]; - char *installer = path_basename(installer_url); - - memset(script_path, 0, sizeof(script_path)); - sprintf(script_path, "%s/%s", ctx->storage.tmpdir, installer); - if (access(script_path, F_OK)) { - // Script doesn't exist - long fetch_status = download(installer_url, script_path, NULL); - if (HTTP_ERROR(fetch_status) || fetch_status < 0) { - // download failed - return -1; - } - } else { - msg(STASIS_MSG_RESTRICT | STASIS_MSG_L3, "Skipped, installer already exists\n", script_path); - } - - ctx->conda.installer_path = strdup(script_path); - if (!ctx->conda.installer_path) { - SYSERROR("Unable to duplicate script_path: '%s'", script_path); - return -1; - } - - return 0; -} - -void delivery_install_conda(char *install_script, char *conda_install_dir) { - struct Process proc; - memset(&proc, 0, sizeof(proc)); - - if (globals.conda_fresh_start) { - if (!access(conda_install_dir, F_OK)) { - // directory exists so remove it - if (rmtree(conda_install_dir)) { - perror("unable to remove previous installation"); - exit(1); - } - - // Proceed with the installation - // -b = batch mode (non-interactive) - char cmd[PATH_MAX] = {0}; - snprintf(cmd, sizeof(cmd) - 1, "%s %s -b -p %s", - find_program("bash"), - install_script, - conda_install_dir); - if (shell_safe(&proc, cmd)) { - fprintf(stderr, "conda installation failed\n"); - exit(1); - } - } else { - // Proceed with the installation - // -b = batch mode (non-interactive) - char cmd[PATH_MAX] = {0}; - snprintf(cmd, sizeof(cmd) - 1, "%s %s -b -p %s", - find_program("bash"), - install_script, - conda_install_dir); - if (shell_safe(&proc, cmd)) { - fprintf(stderr, "conda installation failed\n"); - exit(1); - } - } - } else { - msg(STASIS_MSG_L3, "Conda removal disabled by configuration\n"); - } -} - -void delivery_conda_enable(struct Delivery *ctx, char *conda_install_dir) { - if (conda_activate(conda_install_dir, "base")) { - fprintf(stderr, "conda activation failed\n"); - exit(1); - } - - // Setting the CONDARC environment variable appears to be the only consistent - // way to make sure the file is used. Not setting this variable leads to strange - // behavior, especially if a conda environment is already active when STASIS is loaded. - char rcpath[PATH_MAX]; - sprintf(rcpath, "%s/%s", conda_install_dir, ".condarc"); - setenv("CONDARC", rcpath, 1); - if (runtime_replace(&ctx->runtime.environ, __environ)) { - perror("unable to replace runtime environment after activating conda"); - exit(1); - } - - if (conda_setup_headless()) { - // no COE check. this call must succeed. - exit(1); - } -} - diff --git a/src/delivery_docker.c b/src/delivery_docker.c deleted file mode 100644 index e1d7f60..0000000 --- a/src/delivery_docker.c +++ /dev/null @@ -1,132 +0,0 @@ -#include "delivery.h" - -int delivery_docker(struct Delivery *ctx) { - if (!docker_capable(&ctx->deploy.docker.capabilities)) { - return -1; - } - char tag[STASIS_NAME_MAX]; - char args[PATH_MAX]; - int has_registry = ctx->deploy.docker.registry != NULL; - size_t total_tags = strlist_count(ctx->deploy.docker.tags); - size_t total_build_args = strlist_count(ctx->deploy.docker.build_args); - - if (!has_registry) { - msg(STASIS_MSG_WARN | STASIS_MSG_L2, "No docker registry defined. You will need to manually retag the resulting image.\n"); - } - - if (!total_tags) { - char default_tag[PATH_MAX]; - msg(STASIS_MSG_WARN | STASIS_MSG_L2, "No docker tags defined by configuration. Generating default tag(s).\n"); - // generate local tag - memset(default_tag, 0, sizeof(default_tag)); - sprintf(default_tag, "%s:%s-py%s", ctx->meta.name, ctx->info.build_name, ctx->meta.python_compact); - tolower_s(default_tag); - - // Add tag - ctx->deploy.docker.tags = strlist_init(); - strlist_append(&ctx->deploy.docker.tags, default_tag); - - if (has_registry) { - // generate tag for target registry - memset(default_tag, 0, sizeof(default_tag)); - sprintf(default_tag, "%s/%s:%s-py%s", ctx->deploy.docker.registry, ctx->meta.name, ctx->info.build_number, ctx->meta.python_compact); - tolower_s(default_tag); - - // Add tag - strlist_append(&ctx->deploy.docker.tags, default_tag); - } - // regenerate total tag available - total_tags = strlist_count(ctx->deploy.docker.tags); - } - - memset(args, 0, sizeof(args)); - - // Append image tags to command - for (size_t i = 0; i < total_tags; i++) { - char *tag_orig = strlist_item(ctx->deploy.docker.tags, i); - strcpy(tag, tag_orig); - docker_sanitize_tag(tag); - sprintf(args + strlen(args), " -t \"%s\" ", tag); - } - - // Append build arguments to command (i.e. --build-arg "key=value" - for (size_t i = 0; i < total_build_args; i++) { - char *build_arg = strlist_item(ctx->deploy.docker.build_args, i); - if (!build_arg) { - break; - } - sprintf(args + strlen(args), " --build-arg \"%s\" ", build_arg); - } - - // Build the image - char delivery_file[PATH_MAX]; - char dest[PATH_MAX]; - char rsync_cmd[PATH_MAX * 2]; - memset(delivery_file, 0, sizeof(delivery_file)); - memset(dest, 0, sizeof(dest)); - - sprintf(delivery_file, "%s/%s.yml", ctx->storage.delivery_dir, ctx->info.release_name); - if (access(delivery_file, F_OK) < 0) { - fprintf(stderr, "docker build cannot proceed without delivery file: %s\n", delivery_file); - return -1; - } - - sprintf(dest, "%s/%s.yml", ctx->storage.build_docker_dir, ctx->info.release_name); - if (copy2(delivery_file, dest, CT_PERM)) { - fprintf(stderr, "Failed to copy delivery file to %s: %s\n", dest, strerror(errno)); - return -1; - } - - memset(dest, 0, sizeof(dest)); - sprintf(dest, "%s/packages", ctx->storage.build_docker_dir); - - msg(STASIS_MSG_L2, "Copying conda packages\n"); - memset(rsync_cmd, 0, sizeof(rsync_cmd)); - sprintf(rsync_cmd, "rsync -avi --progress '%s' '%s'", ctx->storage.conda_artifact_dir, dest); - if (system(rsync_cmd)) { - fprintf(stderr, "Failed to copy conda artifacts to docker build directory\n"); - return -1; - } - - msg(STASIS_MSG_L2, "Copying wheel packages\n"); - memset(rsync_cmd, 0, sizeof(rsync_cmd)); - sprintf(rsync_cmd, "rsync -avi --progress '%s' '%s'", ctx->storage.wheel_artifact_dir, dest); - if (system(rsync_cmd)) { - fprintf(stderr, "Failed to copy wheel artifactory to docker build directory\n"); - } - - if (docker_build(ctx->storage.build_docker_dir, args, ctx->deploy.docker.capabilities.build)) { - return -1; - } - - // Test the image - // All tags point back to the same image so test the first one we see - // regardless of how many are defined - strcpy(tag, strlist_item(ctx->deploy.docker.tags, 0)); - docker_sanitize_tag(tag); - - msg(STASIS_MSG_L2, "Executing image test script for %s\n", tag); - if (ctx->deploy.docker.test_script) { - if (isempty(ctx->deploy.docker.test_script)) { - msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "Image test script has no content\n"); - } else { - int state; - if ((state = docker_script(tag, ctx->deploy.docker.test_script, 0))) { - msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "Non-zero exit (%d) from test script. %s image archive will not be generated.\n", state >> 8, tag); - // test failed -- don't save the image - return -1; - } - } - } else { - msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "No image test script defined\n"); - } - - // Test successful, save image - if (docker_save(path_basename(tag), ctx->storage.docker_artifact_dir, ctx->deploy.docker.image_compression)) { - // save failed - return -1; - } - - return 0; -} - diff --git a/src/delivery_init.c b/src/delivery_init.c deleted file mode 100644 index e914f99..0000000 --- a/src/delivery_init.c +++ /dev/null @@ -1,345 +0,0 @@ -#include "delivery.h" - -int has_mount_flags(const char *mount_point, const unsigned long flags) { - struct statvfs st; - if (statvfs(mount_point, &st)) { - SYSERROR("Unable to determine mount-point flags: %s", strerror(errno)); - return -1; - } - return (st.f_flag & flags) != 0; -} - -int delivery_init_tmpdir(struct Delivery *ctx) { - char *tmpdir = NULL; - char *x = NULL; - int unusable = 0; - errno = 0; - - x = getenv("TMPDIR"); - if (x) { - guard_free(ctx->storage.tmpdir); - tmpdir = strdup(x); - } else { - tmpdir = ctx->storage.tmpdir; - } - - if (!tmpdir) { - // memory error - return -1; - } - - // If the directory doesn't exist, create it - if (access(tmpdir, F_OK) < 0) { - if (mkdirs(tmpdir, 0755) < 0) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Unable to create temporary storage directory: %s (%s)\n", tmpdir, strerror(errno)); - goto l_delivery_init_tmpdir_fatal; - } - } - - // If we can't read, write, or execute, then die - if (access(tmpdir, R_OK | W_OK | X_OK) < 0) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "%s requires at least 0755 permissions.\n"); - goto l_delivery_init_tmpdir_fatal; - } - - struct statvfs st; - if (statvfs(tmpdir, &st) < 0) { - goto l_delivery_init_tmpdir_fatal; - } - -#if defined(STASIS_OS_LINUX) - // If we can't execute programs, or write data to the file system at all, then die - if ((st.f_flag & ST_NOEXEC) != 0) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "%s is mounted with noexec\n", tmpdir); - goto l_delivery_init_tmpdir_fatal; - } -#endif - if ((st.f_flag & ST_RDONLY) != 0) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "%s is mounted read-only\n", tmpdir); - goto l_delivery_init_tmpdir_fatal; - } - - if (!globals.tmpdir) { - globals.tmpdir = strdup(tmpdir); - } - - if (!ctx->storage.tmpdir) { - ctx->storage.tmpdir = strdup(globals.tmpdir); - } - return unusable; - - l_delivery_init_tmpdir_fatal: - unusable = 1; - return unusable; -} - -void delivery_init_dirs_stage2(struct Delivery *ctx) { - path_store(&ctx->storage.build_recipes_dir, PATH_MAX, ctx->storage.build_dir, "recipes"); - path_store(&ctx->storage.build_sources_dir, PATH_MAX, ctx->storage.build_dir, "sources"); - path_store(&ctx->storage.build_testing_dir, PATH_MAX, ctx->storage.build_dir, "testing"); - path_store(&ctx->storage.build_docker_dir, PATH_MAX, ctx->storage.build_dir, "docker"); - - path_store(&ctx->storage.delivery_dir, PATH_MAX, ctx->storage.output_dir, "delivery"); - path_store(&ctx->storage.results_dir, PATH_MAX, ctx->storage.output_dir, "results"); - path_store(&ctx->storage.package_dir, PATH_MAX, ctx->storage.output_dir, "packages"); - path_store(&ctx->storage.cfgdump_dir, PATH_MAX, ctx->storage.output_dir, "config"); - path_store(&ctx->storage.meta_dir, PATH_MAX, ctx->storage.output_dir, "meta"); - - path_store(&ctx->storage.conda_artifact_dir, PATH_MAX, ctx->storage.package_dir, "conda"); - path_store(&ctx->storage.wheel_artifact_dir, PATH_MAX, ctx->storage.package_dir, "wheels"); - path_store(&ctx->storage.docker_artifact_dir, PATH_MAX, ctx->storage.package_dir, "docker"); -} - -void delivery_init_dirs_stage1(struct Delivery *ctx) { - char *rootdir = getenv("STASIS_ROOT"); - if (rootdir) { - if (isempty(rootdir)) { - fprintf(stderr, "STASIS_ROOT is set, but empty. Please assign a file system path to this environment variable.\n"); - exit(1); - } - path_store(&ctx->storage.root, PATH_MAX, rootdir, ctx->info.build_name); - } else { - // use "stasis" in current working directory - path_store(&ctx->storage.root, PATH_MAX, "stasis", ctx->info.build_name); - } - path_store(&ctx->storage.tools_dir, PATH_MAX, ctx->storage.root, "tools"); - path_store(&ctx->storage.tmpdir, PATH_MAX, ctx->storage.root, "tmp"); - if (delivery_init_tmpdir(ctx)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Set $TMPDIR to a location other than %s\n", globals.tmpdir); - if (globals.tmpdir) - guard_free(globals.tmpdir); - exit(1); - } - - path_store(&ctx->storage.build_dir, PATH_MAX, ctx->storage.root, "build"); - path_store(&ctx->storage.output_dir, PATH_MAX, ctx->storage.root, "output"); - - if (!ctx->storage.mission_dir) { - path_store(&ctx->storage.mission_dir, PATH_MAX, globals.sysconfdir, "mission"); - } - - if (access(ctx->storage.mission_dir, F_OK)) { - msg(STASIS_MSG_L1, "%s: %s\n", ctx->storage.mission_dir, strerror(errno)); - exit(1); - } - - // Override installation prefix using global configuration key - if (globals.conda_install_prefix && strlen(globals.conda_install_prefix)) { - // user wants a specific path - globals.conda_fresh_start = false; - /* - if (mkdirs(globals.conda_install_prefix, 0755)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Unable to create directory: %s: %s\n", - strerror(errno), globals.conda_install_prefix); - exit(1); - } - */ - /* - ctx->storage.conda_install_prefix = realpath(globals.conda_install_prefix, NULL); - if (!ctx->storage.conda_install_prefix) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "realpath(): Conda installation prefix reassignment failed\n"); - exit(1); - } - ctx->storage.conda_install_prefix = strdup(globals.conda_install_prefix); - */ - path_store(&ctx->storage.conda_install_prefix, PATH_MAX, globals.conda_install_prefix, "conda"); - } else { - // install conda under the STASIS tree - path_store(&ctx->storage.conda_install_prefix, PATH_MAX, ctx->storage.tools_dir, "conda"); - } -} - -int delivery_init_platform(struct Delivery *ctx) { - msg(STASIS_MSG_L2, "Setting architecture\n"); - char archsuffix[20]; - struct utsname uts; - if (uname(&uts)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "uname() failed: %s\n", strerror(errno)); - return -1; - } - - ctx->system.platform = calloc(DELIVERY_PLATFORM_MAX + 1, sizeof(*ctx->system.platform)); - if (!ctx->system.platform) { - SYSERROR("Unable to allocate %d records for platform array\n", DELIVERY_PLATFORM_MAX); - return -1; - } - for (size_t i = 0; i < DELIVERY_PLATFORM_MAX; i++) { - ctx->system.platform[i] = calloc(DELIVERY_PLATFORM_MAXLEN, sizeof(*ctx->system.platform[0])); - } - - ctx->system.arch = strdup(uts.machine); - if (!ctx->system.arch) { - // memory error - return -1; - } - - if (!strcmp(ctx->system.arch, "x86_64")) { - strcpy(archsuffix, "64"); - } else { - strcpy(archsuffix, ctx->system.arch); - } - - msg(STASIS_MSG_L2, "Setting platform\n"); - strcpy(ctx->system.platform[DELIVERY_PLATFORM], uts.sysname); - if (!strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Darwin")) { - sprintf(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], "osx-%s", archsuffix); - strcpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], "MacOSX"); - strcpy(ctx->system.platform[DELIVERY_PLATFORM_RELEASE], "macos"); - } else if (!strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Linux")) { - sprintf(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], "linux-%s", archsuffix); - strcpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], "Linux"); - strcpy(ctx->system.platform[DELIVERY_PLATFORM_RELEASE], "linux"); - } else { - // Not explicitly supported systems - strcpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], ctx->system.platform[DELIVERY_PLATFORM]); - strcpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], ctx->system.platform[DELIVERY_PLATFORM]); - strcpy(ctx->system.platform[DELIVERY_PLATFORM_RELEASE], ctx->system.platform[DELIVERY_PLATFORM]); - tolower_s(ctx->system.platform[DELIVERY_PLATFORM_RELEASE]); - } - - long cpu_count = get_cpu_count(); - if (!cpu_count) { - fprintf(stderr, "Unable to determine CPU count. Falling back to 1.\n"); - cpu_count = 1; - } - char ncpus[100] = {0}; - sprintf(ncpus, "%ld", cpu_count); - - // Declare some important bits as environment variables - setenv("CPU_COUNT", ncpus, 1); - setenv("STASIS_CPU_COUNT", ncpus, 1); - setenv("STASIS_ARCH", ctx->system.arch, 1); - setenv("STASIS_PLATFORM", ctx->system.platform[DELIVERY_PLATFORM], 1); - setenv("STASIS_CONDA_ARCH", ctx->system.arch, 1); - setenv("STASIS_CONDA_PLATFORM", ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], 1); - setenv("STASIS_CONDA_PLATFORM_SUBDIR", ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], 1); - - // Register template variables - // These were moved out of main() because we can't take the address of system.platform[x] - // _before_ the array has been initialized. - tpl_register("system.arch", &ctx->system.arch); - tpl_register("system.platform", &ctx->system.platform[DELIVERY_PLATFORM_RELEASE]); - - return 0; -} - -int delivery_init(struct Delivery *ctx, int render_mode) { - populate_info(ctx); - populate_delivery_cfg(ctx, INI_READ_RENDER); - - // Set artifactory URL via environment variable if possible - char *jfurl = getenv("STASIS_JF_ARTIFACTORY_URL"); - if (jfurl) { - if (globals.jfrog.url) { - guard_free(globals.jfrog.url); - } - globals.jfrog.url = strdup(jfurl); - } - - // Set artifactory repository via environment if possible - char *jfrepo = getenv("STASIS_JF_REPO"); - if (jfrepo) { - if (globals.jfrog.repo) { - guard_free(globals.jfrog.repo); - } - globals.jfrog.repo = strdup(jfrepo); - } - - // Configure architecture and platform information - delivery_init_platform(ctx); - - // Create STASIS directory structure - delivery_init_dirs_stage1(ctx); - - char config_local[PATH_MAX]; - sprintf(config_local, "%s/%s", ctx->storage.tmpdir, "config"); - setenv("XDG_CONFIG_HOME", config_local, 1); - - char cache_local[PATH_MAX]; - sprintf(cache_local, "%s/%s", ctx->storage.tmpdir, "cache"); - setenv("XDG_CACHE_HOME", ctx->storage.tmpdir, 1); - - // add tools to PATH - char pathvar_tmp[STASIS_BUFSIZ]; - sprintf(pathvar_tmp, "%s/bin:%s", ctx->storage.tools_dir, getenv("PATH")); - setenv("PATH", pathvar_tmp, 1); - - // Prevent git from paginating output - setenv("GIT_PAGER", "", 1); - - populate_delivery_ini(ctx, render_mode); - - if (ctx->deploy.docker.tags) { - for (size_t i = 0; i < strlist_count(ctx->deploy.docker.tags); i++) { - char *item = strlist_item(ctx->deploy.docker.tags, i); - tolower_s(item); - } - } - - if (ctx->deploy.docker.image_compression) { - if (docker_validate_compression_program(ctx->deploy.docker.image_compression)) { - SYSERROR("[deploy:docker].image_compression - invalid command / program is not installed: %s", ctx->deploy.docker.image_compression); - return -1; - } - } - return 0; -} - -int bootstrap_build_info(struct Delivery *ctx) { - struct Delivery local; - memset(&local, 0, sizeof(local)); - local._stasis_ini_fp.cfg = ini_open(ctx->_stasis_ini_fp.cfg_path); - local._stasis_ini_fp.delivery = ini_open(ctx->_stasis_ini_fp.delivery_path); - delivery_init_platform(&local); - populate_delivery_cfg(&local, INI_READ_RENDER); - populate_delivery_ini(&local, INI_READ_RENDER); - populate_info(&local); - ctx->info.build_name = strdup(local.info.build_name); - ctx->info.build_number = strdup(local.info.build_number); - ctx->info.release_name = strdup(local.info.release_name); - ctx->info.time_info = malloc(sizeof(*ctx->info.time_info)); - if (!ctx->info.time_info) { - SYSERROR("Unable to allocate %zu bytes for tm struct: %s", sizeof(*local.info.time_info), strerror(errno)); - return -1; - } - memcpy(ctx->info.time_info, local.info.time_info, sizeof(*local.info.time_info)); - ctx->info.time_now = local.info.time_now; - ctx->info.time_str_epoch = strdup(local.info.time_str_epoch); - delivery_free(&local); - return 0; -} - -int delivery_exists(struct Delivery *ctx) { - int release_exists = 0; - char release_pattern[PATH_MAX] = {0}; - sprintf(release_pattern, "*%s*", ctx->info.release_name); - - if (globals.enable_artifactory) { - if (jfrt_auth_init(&ctx->deploy.jfrog_auth)) { - fprintf(stderr, "Failed to initialize Artifactory authentication context\n"); - return -1; // error - } - - struct JFRT_Search search = {.fail_no_op = true}; - release_exists = jfrog_cli_rt_search(&ctx->deploy.jfrog_auth, &search, globals.jfrog.repo, release_pattern); - if (release_exists != 2) { - if (!globals.enable_overwrite && !release_exists) { - // --fail_no_op returns 2 on failure - // without: it returns an empty list "[]" and exit code 0 - return 1; // found - } - } - } else { - struct StrList *files = listdir(ctx->storage.delivery_dir); - for (size_t i = 0; i < strlist_count(files); i++) { - char *filename = strlist_item(files, i); - release_exists = fnmatch(release_pattern, filename, FNM_PATHNAME); - if (!globals.enable_overwrite && !release_exists) { - guard_strlist_free(&files); - return 1; // found - } - } - guard_strlist_free(&files); - } - return 0; // not found -} diff --git a/src/delivery_install.c b/src/delivery_install.c deleted file mode 100644 index 76c3f4a..0000000 --- a/src/delivery_install.c +++ /dev/null @@ -1,224 +0,0 @@ -#include "delivery.h" - -static struct Test *requirement_from_test(struct Delivery *ctx, const char *name) { - struct Test *result = NULL; - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { - if (ctx->tests[i].name && !strcmp(name, ctx->tests[i].name)) { - result = &ctx->tests[i]; - break; - } - } - return result; -} - -static char *have_spec_in_config(struct Delivery *ctx, const char *name) { - for (size_t x = 0; x < strlist_count(ctx->conda.pip_packages); x++) { - char *config_spec = strlist_item(ctx->conda.pip_packages, x); - char *op = find_version_spec(config_spec); - char package[255] = {0}; - if (op) { - strncpy(package, config_spec, op - config_spec); - } else { - strncpy(package, config_spec, sizeof(package) - 1); - } - if (strncmp(package, name, strlen(package)) == 0) { - return config_spec; - } - } - return NULL; -} - -int delivery_overlay_packages_from_env(struct Delivery *ctx, const char *env_name) { - char *current_env = conda_get_active_environment(); - int need_restore = current_env && strcmp(env_name, current_env) != 0; - - conda_activate(ctx->storage.conda_install_prefix, env_name); - // Retrieve a listing of python packages installed under "env_name" - int freeze_status = 0; - char *freeze_output = shell_output("python -m pip freeze", &freeze_status); - if (freeze_status) { - guard_free(freeze_output); - guard_free(current_env); - return -1; - } - - if (need_restore) { - // Restore the original conda environment - conda_activate(ctx->storage.conda_install_prefix, current_env); - } - guard_free(current_env); - - struct StrList *frozen_list = strlist_init(); - strlist_append_tokenize(frozen_list, freeze_output, LINE_SEP); - guard_free(freeze_output); - - struct StrList *new_list = strlist_init(); - - // - consume package specs that have no test blocks. - // - these will be third-party packages like numpy, scipy, etc. - // - and they need to be present at the head of the list so they - // get installed first. - for (size_t i = 0; i < strlist_count(ctx->conda.pip_packages); i++) { - char *spec = strlist_item(ctx->conda.pip_packages, i); - char spec_name[255] = {0}; - char *op = find_version_spec(spec); - if (op) { - strncpy(spec_name, spec, op - spec); - } else { - strncpy(spec_name, spec, sizeof(spec_name) - 1); - } - struct Test *test_block = requirement_from_test(ctx, spec_name); - if (!test_block) { - msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "from config without test: %s\n", spec); - strlist_append(&new_list, spec); - } - } - - // now consume packages that have a test block - // if the ini provides a spec, override the environment's version. - // otherwise, use the spec derived from the environment - for (size_t i = 0; i < strlist_count(frozen_list); i++) { - char *frozen_spec = strlist_item(frozen_list, i); - char frozen_name[255] = {0}; - char *op = find_version_spec(frozen_spec); - // we only care about packages with specs here. if something else arrives, ignore it - if (op) { - strncpy(frozen_name, frozen_spec, op - frozen_spec); - } else { - strncpy(frozen_name, frozen_spec, sizeof(frozen_name) - 1); - } - struct Test *test = requirement_from_test(ctx, frozen_name); - if (test && strcmp(test->name, frozen_name) == 0) { - char *config_spec = have_spec_in_config(ctx, frozen_name); - if (config_spec) { - msg(STASIS_MSG_L2, "from config: %s\n", config_spec); - strlist_append(&new_list, config_spec); - } else { - msg(STASIS_MSG_L2, "from environment: %s\n", frozen_spec); - strlist_append(&new_list, frozen_spec); - } - } - } - - // Replace the package manifest as needed - if (strlist_count(new_list)) { - guard_strlist_free(&ctx->conda.pip_packages); - ctx->conda.pip_packages = strlist_copy(new_list); - } - guard_strlist_free(&new_list); - guard_strlist_free(&frozen_list); - return 0; -} - -int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, char *env_name, int type, struct StrList **manifest) { - char cmd[PATH_MAX]; - char pkgs[STASIS_BUFSIZ]; - char *env_current = getenv("CONDA_DEFAULT_ENV"); - - if (env_current) { - // The requested environment is not the current environment - if (strcmp(env_current, env_name) != 0) { - // Activate the requested environment - printf("Activating: %s\n", env_name); - conda_activate(conda_install_dir, env_name); - runtime_replace(&ctx->runtime.environ, __environ); - } - } - - memset(cmd, 0, sizeof(cmd)); - memset(pkgs, 0, sizeof(pkgs)); - strcat(cmd, "install"); - - typedef int (*Runner)(const char *); - Runner runner = NULL; - if (INSTALL_PKG_CONDA & type) { - runner = conda_exec; - } else if (INSTALL_PKG_PIP & type) { - runner = pip_exec; - } - - if (INSTALL_PKG_CONDA_DEFERRED & type) { - strcat(cmd, " --use-local"); - } else if (INSTALL_PKG_PIP_DEFERRED & type) { - // Don't change the baseline package set unless we're working with a - // new build. Release candidates will need to keep packages as stable - // as possible between releases. - if (!ctx->meta.based_on) { - strcat(cmd, " --upgrade"); - } - sprintf(cmd + strlen(cmd), " --extra-index-url 'file://%s'", ctx->storage.wheel_artifact_dir); - } - - for (size_t x = 0; manifest[x] != NULL; x++) { - char *name = NULL; - for (size_t p = 0; p < strlist_count(manifest[x]); p++) { - name = strlist_item(manifest[x], p); - strip(name); - if (!strlen(name)) { - continue; - } - if (INSTALL_PKG_PIP_DEFERRED & type) { - struct Test *info = requirement_from_test(ctx, name); - if (info) { - if (!strcmp(info->version, "HEAD")) { - struct StrList *tag_data = strlist_init(); - if (!tag_data) { - SYSERROR("%s", "Unable to allocate memory for tag data\n"); - return -1; - } - strlist_append_tokenize(tag_data, info->repository_info_tag, "-"); - - struct Wheel *whl = NULL; - char *post_commit = NULL; - char *hash = NULL; - if (strlist_count(tag_data) > 1) { - post_commit = strlist_item(tag_data, 1); - hash = strlist_item(tag_data, 2); - } - - // We can't match on version here (index 0). The wheel's version is not guaranteed to be - // equal to the tag; setuptools_scm auto-increments the value, the user can change it manually, - // etc. - errno = 0; - whl = get_wheel_info(ctx->storage.wheel_artifact_dir, info->name, - (char *[]) {ctx->meta.python_compact, ctx->system.arch, - "none", "any", - post_commit, hash, - NULL}, WHEEL_MATCH_ANY); - if (!whl && errno) { - // error - SYSERROR("Unable to read Python wheel info: %s\n", strerror(errno)); - exit(1); - } else if (!whl) { - // not found - fprintf(stderr, "No wheel packages found that match the description of '%s'", info->name); - } else { - // found - guard_strlist_free(&tag_data); - info->version = strdup(whl->version); - } - wheel_free(&whl); - } - snprintf(cmd + strlen(cmd), - sizeof(cmd) - strlen(cmd) - strlen(info->name) - strlen(info->version) + 5, - " '%s==%s'", info->name, info->version); - } else { - fprintf(stderr, "Deferred package '%s' is not present in the tested package list!\n", name); - return -1; - } - } else { - if (startswith(name, "--") || startswith(name, "-")) { - sprintf(cmd + strlen(cmd), " %s", name); - } else { - sprintf(cmd + strlen(cmd), " '%s'", name); - } - } - } - int status = runner(cmd); - if (status) { - return status; - } - } - return 0; -} - diff --git a/src/delivery_populate.c b/src/delivery_populate.c deleted file mode 100644 index b37f677..0000000 --- a/src/delivery_populate.c +++ /dev/null @@ -1,348 +0,0 @@ -#include "delivery.h" - -static void ini_has_key_required(struct INIFILE *ini, const char *section_name, char *key) { - int status = ini_has_key(ini, section_name, key); - if (!status) { - SYSERROR("%s:%s key is required but not defined", section_name, key); - exit(1); - } -} - -static void conv_str(char **x, union INIVal val) { - if (*x) { - guard_free(*x); - } - if (val.as_char_p) { - char *tplop = tpl_render(val.as_char_p); - if (tplop) { - *x = tplop; - } else { - *x = NULL; - } - } else { - *x = NULL; - } -} - - - -int populate_info(struct Delivery *ctx) { - if (!ctx->info.time_str_epoch) { - // Record timestamp used for release - time(&ctx->info.time_now); - ctx->info.time_info = localtime(&ctx->info.time_now); - - ctx->info.time_str_epoch = calloc(STASIS_TIME_STR_MAX, sizeof(*ctx->info.time_str_epoch)); - if (!ctx->info.time_str_epoch) { - msg(STASIS_MSG_ERROR, "Unable to allocate memory for Unix epoch string\n"); - return -1; - } - snprintf(ctx->info.time_str_epoch, STASIS_TIME_STR_MAX - 1, "%li", ctx->info.time_now); - } - return 0; -} - -int populate_delivery_cfg(struct Delivery *ctx, int render_mode) { - struct INIFILE *cfg = ctx->_stasis_ini_fp.cfg; - if (!cfg) { - return -1; - } - int err = 0; - ctx->storage.conda_staging_dir = ini_getval_str(cfg, "default", "conda_staging_dir", render_mode, &err); - ctx->storage.conda_staging_url = ini_getval_str(cfg, "default", "conda_staging_url", render_mode, &err); - ctx->storage.wheel_staging_dir = ini_getval_str(cfg, "default", "wheel_staging_dir", render_mode, &err); - ctx->storage.wheel_staging_url = ini_getval_str(cfg, "default", "wheel_staging_url", render_mode, &err); - globals.conda_fresh_start = ini_getval_bool(cfg, "default", "conda_fresh_start", render_mode, &err); - if (!globals.continue_on_error) { - globals.continue_on_error = ini_getval_bool(cfg, "default", "continue_on_error", render_mode, &err); - } - if (!globals.always_update_base_environment) { - globals.always_update_base_environment = ini_getval_bool(cfg, "default", "always_update_base_environment", render_mode, &err); - } - globals.conda_install_prefix = ini_getval_str(cfg, "default", "conda_install_prefix", render_mode, &err); - globals.conda_packages = ini_getval_strlist(cfg, "default", "conda_packages", LINE_SEP, render_mode, &err); - globals.pip_packages = ini_getval_strlist(cfg, "default", "pip_packages", LINE_SEP, render_mode, &err); - - globals.jfrog.jfrog_artifactory_base_url = ini_getval_str(cfg, "jfrog_cli_download", "url", render_mode, &err); - globals.jfrog.jfrog_artifactory_product = ini_getval_str(cfg, "jfrog_cli_download", "product", render_mode, &err); - globals.jfrog.cli_major_ver = ini_getval_str(cfg, "jfrog_cli_download", "version_series", render_mode, &err); - globals.jfrog.version = ini_getval_str(cfg, "jfrog_cli_download", "version", render_mode, &err); - globals.jfrog.remote_filename = ini_getval_str(cfg, "jfrog_cli_download", "filename", render_mode, &err); - globals.jfrog.url = ini_getval_str(cfg, "deploy:artifactory", "url", render_mode, &err); - globals.jfrog.repo = ini_getval_str(cfg, "deploy:artifactory", "repo", render_mode, &err); - - return 0; -} - -int populate_delivery_ini(struct Delivery *ctx, int render_mode) { - union INIVal val; - struct INIFILE *ini = ctx->_stasis_ini_fp.delivery; - struct INIData *rtdata; - RuntimeEnv *rt; - - validate_delivery_ini(ini); - // Populate runtime variables first they may be interpreted by other - // keys in the configuration - rt = runtime_copy(__environ); - while ((rtdata = ini_getall(ini, "runtime")) != NULL) { - char rec[STASIS_BUFSIZ]; - sprintf(rec, "%s=%s", lstrip(strip(rtdata->key)), lstrip(strip(rtdata->value))); - runtime_set(rt, rtdata->key, rtdata->value); - } - runtime_apply(rt); - ctx->runtime.environ = rt; - - int err = 0; - ctx->meta.mission = ini_getval_str(ini, "meta", "mission", render_mode, &err); - - if (!strcasecmp(ctx->meta.mission, "hst")) { - ctx->meta.codename = ini_getval_str(ini, "meta", "codename", render_mode, &err); - } else { - ctx->meta.codename = NULL; - } - - ctx->meta.version = ini_getval_str(ini, "meta", "version", render_mode, &err); - ctx->meta.name = ini_getval_str(ini, "meta", "name", render_mode, &err); - ctx->meta.rc = ini_getval_int(ini, "meta", "rc", render_mode, &err); - ctx->meta.final = ini_getval_bool(ini, "meta", "final", render_mode, &err); - ctx->meta.based_on = ini_getval_str(ini, "meta", "based_on", render_mode, &err); - - if (!ctx->meta.python) { - ctx->meta.python = ini_getval_str(ini, "meta", "python", render_mode, &err); - guard_free(ctx->meta.python_compact); - ctx->meta.python_compact = to_short_version(ctx->meta.python); - } else { - ini_setval(&ini, INI_SETVAL_REPLACE, "meta", "python", ctx->meta.python); - } - - ctx->conda.installer_name = ini_getval_str(ini, "conda", "installer_name", render_mode, &err); - ctx->conda.installer_version = ini_getval_str(ini, "conda", "installer_version", render_mode, &err); - ctx->conda.installer_platform = ini_getval_str(ini, "conda", "installer_platform", render_mode, &err); - ctx->conda.installer_arch = ini_getval_str(ini, "conda", "installer_arch", render_mode, &err); - ctx->conda.installer_baseurl = ini_getval_str(ini, "conda", "installer_baseurl", render_mode, &err); - ctx->conda.conda_packages = ini_getval_strlist(ini, "conda", "conda_packages", " "LINE_SEP, render_mode, &err); - - if (ctx->conda.conda_packages->data && ctx->conda.conda_packages->data[0] && strpbrk(ctx->conda.conda_packages->data[0], " \t")) { - normalize_space(ctx->conda.conda_packages->data[0]); - replace_text(ctx->conda.conda_packages->data[0], " ", LINE_SEP, 0); - char *pip_packages_replacement = join(ctx->conda.conda_packages->data, LINE_SEP); - ini_setval(&ini, INI_SETVAL_REPLACE, "conda", "conda_packages", pip_packages_replacement); - guard_free(pip_packages_replacement); - guard_strlist_free(&ctx->conda.conda_packages); - ctx->conda.conda_packages = ini_getval_strlist(ini, "conda", "conda_packages", LINE_SEP, render_mode, &err); - } - - for (size_t i = 0; i < strlist_count(ctx->conda.conda_packages); i++) { - char *pkg = strlist_item(ctx->conda.conda_packages, i); - if (strpbrk(pkg, ";#") || isempty(pkg)) { - strlist_remove(ctx->conda.conda_packages, i); - } - } - - ctx->conda.pip_packages = ini_getval_strlist(ini, "conda", "pip_packages", LINE_SEP, render_mode, &err); - if (ctx->conda.pip_packages->data && ctx->conda.pip_packages->data[0] && strpbrk(ctx->conda.pip_packages->data[0], " \t")) { - normalize_space(ctx->conda.pip_packages->data[0]); - replace_text(ctx->conda.pip_packages->data[0], " ", LINE_SEP, 0); - char *pip_packages_replacement = join(ctx->conda.pip_packages->data, LINE_SEP); - ini_setval(&ini, INI_SETVAL_REPLACE, "conda", "pip_packages", pip_packages_replacement); - guard_free(pip_packages_replacement); - guard_strlist_free(&ctx->conda.pip_packages); - ctx->conda.pip_packages = ini_getval_strlist(ini, "conda", "pip_packages", LINE_SEP, render_mode, &err); - } - - for (size_t i = 0; i < strlist_count(ctx->conda.pip_packages); i++) { - char *pkg = strlist_item(ctx->conda.pip_packages, i); - if (strpbrk(pkg, ";#") || isempty(pkg)) { - strlist_remove(ctx->conda.pip_packages, i); - } - } - - // Delivery metadata consumed - populate_mission_ini(&ctx, render_mode); - - if (ctx->info.release_name) { - guard_free(ctx->info.release_name); - guard_free(ctx->info.build_name); - guard_free(ctx->info.build_number); - } - - if (delivery_format_str(ctx, &ctx->info.release_name, ctx->rules.release_fmt)) { - fprintf(stderr, "Failed to generate release name. Format used: %s\n", ctx->rules.release_fmt); - return -1; - } - - if (!ctx->info.build_name) { - delivery_format_str(ctx, &ctx->info.build_name, ctx->rules.build_name_fmt); - } - if (!ctx->info.build_number) { - delivery_format_str(ctx, &ctx->info.build_number, ctx->rules.build_number_fmt); - } - - // Best I can do to make output directories unique. Annoying. - delivery_init_dirs_stage2(ctx); - - if (!ctx->conda.conda_packages_defer) { - ctx->conda.conda_packages_defer = strlist_init(); - } - if (!ctx->conda.pip_packages_defer) { - ctx->conda.pip_packages_defer = strlist_init(); - } - - for (size_t z = 0, i = 0; i < ini->section_count; i++) { - char *section_name = ini->section[i]->key; - if (startswith(section_name, "test:")) { - struct Test *test = &ctx->tests[z]; - val.as_char_p = strchr(ini->section[i]->key, ':') + 1; - if (val.as_char_p && isempty(val.as_char_p)) { - return 1; - } - conv_str(&test->name, val); - - test->version = ini_getval_str(ini, section_name, "version", render_mode, &err); - test->repository = ini_getval_str(ini, section_name, "repository", render_mode, &err); - test->script_setup = ini_getval_str(ini, section_name, "script_setup", INI_READ_RAW, &err); - test->script = ini_getval_str(ini, section_name, "script", INI_READ_RAW, &err); - test->disable = ini_getval_bool(ini, section_name, "disable", render_mode, &err); - test->parallel = ini_getval_bool(ini, section_name, "parallel", render_mode, &err); - if (err) { - test->parallel = true; - } - test->repository_remove_tags = ini_getval_strlist(ini, section_name, "repository_remove_tags", LINE_SEP, render_mode, &err); - test->build_recipe = ini_getval_str(ini, section_name, "build_recipe", render_mode, &err); - test->runtime.environ = ini_getval_strlist(ini, section_name, "runtime", LINE_SEP, render_mode, &err); - z++; - } - } - - for (size_t z = 0, i = 0; i < ini->section_count; i++) { - char *section_name = ini->section[i]->key; - struct Deploy *deploy = &ctx->deploy; - if (startswith(section_name, "deploy:artifactory")) { - struct JFrog *jfrog = &deploy->jfrog[z]; - // Artifactory base configuration - - jfrog->upload_ctx.workaround_parent_only = ini_getval_bool(ini, section_name, "workaround_parent_only", render_mode, &err); - jfrog->upload_ctx.exclusions = ini_getval_str(ini, section_name, "exclusions", render_mode, &err); - jfrog->upload_ctx.explode = ini_getval_bool(ini, section_name, "explode", render_mode, &err); - jfrog->upload_ctx.recursive = ini_getval_bool(ini, section_name, "recursive", render_mode, &err); - jfrog->upload_ctx.retries = ini_getval_int(ini, section_name, "retries", render_mode, &err); - jfrog->upload_ctx.retry_wait_time = ini_getval_int(ini, section_name, "retry_wait_time", render_mode, &err); - jfrog->upload_ctx.detailed_summary = ini_getval_bool(ini, section_name, "detailed_summary", render_mode, &err); - jfrog->upload_ctx.quiet = ini_getval_bool(ini, section_name, "quiet", render_mode, &err); - jfrog->upload_ctx.regexp = ini_getval_bool(ini, section_name, "regexp", render_mode, &err); - jfrog->upload_ctx.spec = ini_getval_str(ini, section_name, "spec", render_mode, &err); - jfrog->upload_ctx.flat = ini_getval_bool(ini, section_name, "flat", render_mode, &err); - jfrog->repo = ini_getval_str(ini, section_name, "repo", render_mode, &err); - jfrog->dest = ini_getval_str(ini, section_name, "dest", render_mode, &err); - jfrog->files = ini_getval_strlist(ini, section_name, "files", LINE_SEP, render_mode, &err); - z++; - } - } - - for (size_t i = 0; i < ini->section_count; i++) { - char *section_name = ini->section[i]->key; - struct Deploy *deploy = &ctx->deploy; - if (startswith(ini->section[i]->key, "deploy:docker")) { - struct Docker *docker = &deploy->docker; - - docker->registry = ini_getval_str(ini, section_name, "registry", render_mode, &err); - docker->image_compression = ini_getval_str(ini, section_name, "image_compression", render_mode, &err); - docker->test_script = ini_getval_str(ini, section_name, "test_script", render_mode, &err); - docker->build_args = ini_getval_strlist(ini, section_name, "build_args", LINE_SEP, render_mode, &err); - docker->tags = ini_getval_strlist(ini, section_name, "tags", LINE_SEP, render_mode, &err); - } - } - return 0; -} - -int populate_mission_ini(struct Delivery **ctx, int render_mode) { - int err = 0; - struct INIFILE *ini; - - if ((*ctx)->_stasis_ini_fp.mission) { - return 0; - } - - // Now populate the rules - char missionfile[PATH_MAX] = {0}; - if (getenv("STASIS_SYSCONFDIR")) { - sprintf(missionfile, "%s/%s/%s/%s.ini", - getenv("STASIS_SYSCONFDIR"), "mission", (*ctx)->meta.mission, (*ctx)->meta.mission); - } else { - sprintf(missionfile, "%s/%s/%s/%s.ini", - globals.sysconfdir, "mission", (*ctx)->meta.mission, (*ctx)->meta.mission); - } - - msg(STASIS_MSG_L2, "Reading mission configuration: %s\n", missionfile); - (*ctx)->_stasis_ini_fp.mission = ini_open(missionfile); - ini = (*ctx)->_stasis_ini_fp.mission; - if (!ini) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read mission configuration: %s, %s\n", missionfile, strerror(errno)); - exit(1); - } - (*ctx)->_stasis_ini_fp.mission_path = strdup(missionfile); - - (*ctx)->rules.release_fmt = ini_getval_str(ini, "meta", "release_fmt", render_mode, &err); - - // Used for setting artifactory build info - (*ctx)->rules.build_name_fmt = ini_getval_str(ini, "meta", "build_name_fmt", render_mode, &err); - - // Used for setting artifactory build info - (*ctx)->rules.build_number_fmt = ini_getval_str(ini, "meta", "build_number_fmt", render_mode, &err); - return 0; -} - -void validate_delivery_ini(struct INIFILE *ini) { - if (!ini) { - SYSERROR("%s", "INIFILE is NULL!"); - exit(1); - } - if (ini_section_search(&ini, INI_SEARCH_EXACT, "meta")) { - ini_has_key_required(ini, "meta", "name"); - ini_has_key_required(ini, "meta", "version"); - ini_has_key_required(ini, "meta", "rc"); - ini_has_key_required(ini, "meta", "mission"); - ini_has_key_required(ini, "meta", "python"); - } else { - SYSERROR("%s", "[meta] configuration section is required"); - exit(1); - } - - if (ini_section_search(&ini, INI_SEARCH_EXACT, "conda")) { - ini_has_key_required(ini, "conda", "installer_name"); - ini_has_key_required(ini, "conda", "installer_version"); - ini_has_key_required(ini, "conda", "installer_platform"); - ini_has_key_required(ini, "conda", "installer_arch"); - } else { - SYSERROR("%s", "[conda] configuration section is required"); - exit(1); - } - - for (size_t i = 0; i < ini->section_count; i++) { - struct INISection *section = ini->section[i]; - if (section && startswith(section->key, "test:")) { - char *name = strstr(section->key, ":"); - if (name && strlen(name) > 1) { - name = &name[1]; - } - //ini_has_key_required(ini, section->key, "version"); - //ini_has_key_required(ini, section->key, "repository"); - if (globals.enable_testing) { - ini_has_key_required(ini, section->key, "script"); - } - } - } - - if (ini_section_search(&ini, INI_SEARCH_EXACT, "deploy:docker")) { - // yeah? - } - - for (size_t i = 0; i < ini->section_count; i++) { - struct INISection *section = ini->section[i]; - if (section && startswith(section->key, "deploy:artifactory")) { - ini_has_key_required(ini, section->key, "files"); - ini_has_key_required(ini, section->key, "dest"); - } - } -} - diff --git a/src/delivery_postprocess.c b/src/delivery_postprocess.c deleted file mode 100644 index 1a902e3..0000000 --- a/src/delivery_postprocess.c +++ /dev/null @@ -1,266 +0,0 @@ -#include "delivery.h" - - -const char *release_header = "# delivery_name: %s\n" - "# delivery_fmt: %s\n" - "# creation_time: %s\n" - "# conda_ident: %s\n" - "# conda_build_ident: %s\n"; - -char *delivery_get_release_header(struct Delivery *ctx) { - char output[STASIS_BUFSIZ]; - char stamp[100]; - strftime(stamp, sizeof(stamp) - 1, "%c", ctx->info.time_info); - sprintf(output, release_header, - ctx->info.release_name, - ctx->rules.release_fmt, - stamp, - ctx->conda.tool_version, - ctx->conda.tool_build_version); - return strdup(output); -} - -int delivery_dump_metadata(struct Delivery *ctx) { - FILE *fp; - char filename[PATH_MAX]; - sprintf(filename, "%s/meta-%s.stasis", ctx->storage.meta_dir, ctx->info.release_name); - fp = fopen(filename, "w+"); - if (!fp) { - return -1; - } - if (globals.verbose) { - printf("%s\n", filename); - } - fprintf(fp, "name %s\n", ctx->meta.name); - fprintf(fp, "version %s\n", ctx->meta.version); - fprintf(fp, "rc %d\n", ctx->meta.rc); - fprintf(fp, "python %s\n", ctx->meta.python); - fprintf(fp, "python_compact %s\n", ctx->meta.python_compact); - fprintf(fp, "mission %s\n", ctx->meta.mission); - fprintf(fp, "codename %s\n", ctx->meta.codename ? ctx->meta.codename : ""); - fprintf(fp, "platform %s %s %s %s\n", - ctx->system.platform[DELIVERY_PLATFORM], - ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], - ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], - ctx->system.platform[DELIVERY_PLATFORM_RELEASE]); - fprintf(fp, "arch %s\n", ctx->system.arch); - fprintf(fp, "time %s\n", ctx->info.time_str_epoch); - fprintf(fp, "release_fmt %s\n", ctx->rules.release_fmt); - fprintf(fp, "release_name %s\n", ctx->info.release_name); - fprintf(fp, "build_name_fmt %s\n", ctx->rules.build_name_fmt); - fprintf(fp, "build_name %s\n", ctx->info.build_name); - fprintf(fp, "build_number_fmt %s\n", ctx->rules.build_number_fmt); - fprintf(fp, "build_number %s\n", ctx->info.build_number); - fprintf(fp, "conda_installer_baseurl %s\n", ctx->conda.installer_baseurl); - fprintf(fp, "conda_installer_name %s\n", ctx->conda.installer_name); - fprintf(fp, "conda_installer_version %s\n", ctx->conda.installer_version); - fprintf(fp, "conda_installer_platform %s\n", ctx->conda.installer_platform); - fprintf(fp, "conda_installer_arch %s\n", ctx->conda.installer_arch); - - fclose(fp); - return 0; -} - -void delivery_rewrite_spec(struct Delivery *ctx, char *filename, unsigned stage) { - char output[PATH_MAX]; - char *header = NULL; - char *tempfile = NULL; - FILE *tp = NULL; - - if (stage == DELIVERY_REWRITE_SPEC_STAGE_1) { - header = delivery_get_release_header(ctx); - if (!header) { - msg(STASIS_MSG_ERROR, "failed to generate release header string\n", filename); - exit(1); - } - tempfile = xmkstemp(&tp, "w+"); - if (!tempfile || !tp) { - msg(STASIS_MSG_ERROR, "%s: unable to create temporary file\n", strerror(errno)); - exit(1); - } - fprintf(tp, "%s", header); - - // Read the original file - char **contents = file_readlines(filename, 0, 0, NULL); - if (!contents) { - msg(STASIS_MSG_ERROR, "%s: unable to read %s", filename); - exit(1); - } - - // Write temporary data - for (size_t i = 0; contents[i] != NULL; i++) { - if (startswith(contents[i], "channels:")) { - // Allow for additional conda channel injection - if (ctx->conda.conda_packages_defer && strlist_count(ctx->conda.conda_packages_defer)) { - fprintf(tp, "%s - @CONDA_CHANNEL@\n", contents[i]); - continue; - } - } else if (strstr(contents[i], "- pip:")) { - if (ctx->conda.pip_packages_defer && strlist_count(ctx->conda.pip_packages_defer)) { - // Allow for additional pip argument injection - fprintf(tp, "%s - @PIP_ARGUMENTS@\n", contents[i]); - continue; - } - } else if (startswith(contents[i], "prefix:")) { - // Remove the prefix key - if (strstr(contents[i], "/") || strstr(contents[i], "\\")) { - // path is on the same line as the key - continue; - } else { - // path is on the next line? - if (contents[i + 1] && (strstr(contents[i + 1], "/") || strstr(contents[i + 1], "\\"))) { - i++; - } - continue; - } - } - fprintf(tp, "%s", contents[i]); - } - GENERIC_ARRAY_FREE(contents); - guard_free(header); - fflush(tp); - fclose(tp); - - // Replace the original file with our temporary data - if (copy2(tempfile, filename, CT_PERM) < 0) { - fprintf(stderr, "%s: could not rename '%s' to '%s'\n", strerror(errno), tempfile, filename); - exit(1); - } - remove(tempfile); - guard_free(tempfile); - } else if (globals.enable_rewrite_spec_stage_2 && stage == DELIVERY_REWRITE_SPEC_STAGE_2) { - // Replace "local" channel with the staging URL - if (ctx->storage.conda_staging_url) { - file_replace_text(filename, "@CONDA_CHANNEL@", ctx->storage.conda_staging_url, 0); - } else if (globals.jfrog.repo) { - sprintf(output, "%s/%s/%s/%s/packages/conda", globals.jfrog.url, globals.jfrog.repo, ctx->meta.mission, ctx->info.build_name); - file_replace_text(filename, "@CONDA_CHANNEL@", output, 0); - } else { - msg(STASIS_MSG_WARN, "conda_staging_dir is not configured. Using fallback: '%s'\n", ctx->storage.conda_artifact_dir); - file_replace_text(filename, "@CONDA_CHANNEL@", ctx->storage.conda_artifact_dir, 0); - } - - if (ctx->storage.wheel_staging_url) { - file_replace_text(filename, "@PIP_ARGUMENTS@", ctx->storage.wheel_staging_url, 0); - } else if (globals.enable_artifactory && globals.jfrog.url && globals.jfrog.repo) { - sprintf(output, "--extra-index-url %s/%s/%s/%s/packages/wheels", globals.jfrog.url, globals.jfrog.repo, ctx->meta.mission, ctx->info.build_name); - file_replace_text(filename, "@PIP_ARGUMENTS@", output, 0); - } else { - msg(STASIS_MSG_WARN, "wheel_staging_dir is not configured. Using fallback: '%s'\n", ctx->storage.wheel_artifact_dir); - sprintf(output, "--extra-index-url file://%s", ctx->storage.wheel_artifact_dir); - file_replace_text(filename, "@PIP_ARGUMENTS@", output, 0); - } - } -} - -int delivery_copy_conda_artifacts(struct Delivery *ctx) { - char cmd[STASIS_BUFSIZ]; - char conda_build_dir[PATH_MAX]; - char subdir[PATH_MAX]; - memset(cmd, 0, sizeof(cmd)); - memset(conda_build_dir, 0, sizeof(conda_build_dir)); - memset(subdir, 0, sizeof(subdir)); - - sprintf(conda_build_dir, "%s/%s", ctx->storage.conda_install_prefix, "conda-bld"); - // One must run conda build at least once to create the "conda-bld" directory. - // When this directory is missing there can be no build artifacts. - if (access(conda_build_dir, F_OK) < 0) { - msg(STASIS_MSG_RESTRICT | STASIS_MSG_WARN | STASIS_MSG_L3, - "Skipped: 'conda build' has never been executed.\n"); - return 0; - } - - snprintf(cmd, sizeof(cmd) - 1, "rsync -avi --progress %s/%s %s", - conda_build_dir, - ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], - ctx->storage.conda_artifact_dir); - - return system(cmd); -} - -int delivery_index_conda_artifacts(struct Delivery *ctx) { - return conda_index(ctx->storage.conda_artifact_dir); -} - -int delivery_copy_wheel_artifacts(struct Delivery *ctx) { - char cmd[PATH_MAX]; - memset(cmd, 0, sizeof(cmd)); - snprintf(cmd, sizeof(cmd) - 1, "rsync -avi --progress %s/*/dist/*.whl %s", - ctx->storage.build_sources_dir, - ctx->storage.wheel_artifact_dir); - return system(cmd); -} - -int delivery_index_wheel_artifacts(struct Delivery *ctx) { - struct dirent *rec; - DIR *dp; - FILE *top_fp; - - dp = opendir(ctx->storage.wheel_artifact_dir); - if (!dp) { - return -1; - } - - // Generate a "dumb" local pypi index that is compatible with: - // pip install --extra-index-url - char top_index[PATH_MAX]; - memset(top_index, 0, sizeof(top_index)); - sprintf(top_index, "%s/index.html", ctx->storage.wheel_artifact_dir); - top_fp = fopen(top_index, "w+"); - if (!top_fp) { - closedir(dp); - return -2; - } - - while ((rec = readdir(dp)) != NULL) { - // skip directories - if (DT_REG == rec->d_type || !strcmp(rec->d_name, "..") || !strcmp(rec->d_name, ".")) { - continue; - } - - FILE *bottom_fp; - char bottom_index[PATH_MAX * 2]; - memset(bottom_index, 0, sizeof(bottom_index)); - sprintf(bottom_index, "%s/%s/index.html", ctx->storage.wheel_artifact_dir, rec->d_name); - bottom_fp = fopen(bottom_index, "w+"); - if (!bottom_fp) { - closedir(dp); - return -3; - } - - if (globals.verbose) { - printf("+ %s\n", rec->d_name); - } - // Add record to top level index - fprintf(top_fp, "%s
\n", rec->d_name, rec->d_name); - - char dpath[PATH_MAX * 2]; - memset(dpath, 0, sizeof(dpath)); - sprintf(dpath, "%s/%s", ctx->storage.wheel_artifact_dir, rec->d_name); - struct StrList *packages = listdir(dpath); - if (!packages) { - closedir(dp); - fclose(top_fp); - fclose(bottom_fp); - return -4; - } - - for (size_t i = 0; i < strlist_count(packages); i++) { - char *package = strlist_item(packages, i); - if (!endswith(package, ".whl")) { - continue; - } - if (globals.verbose) { - printf("`- %s\n", package); - } - // Write record to bottom level index - fprintf(bottom_fp, "%s
\n", package, package); - } - fclose(bottom_fp); - - guard_strlist_free(&packages); - } - closedir(dp); - fclose(top_fp); - return 0; -} diff --git a/src/delivery_show.c b/src/delivery_show.c deleted file mode 100644 index adfa1be..0000000 --- a/src/delivery_show.c +++ /dev/null @@ -1,117 +0,0 @@ -#include "delivery.h" - -void delivery_debug_show(struct Delivery *ctx) { - printf("\n====DEBUG====\n"); - printf("%-20s %-10s\n", "System configuration directory:", globals.sysconfdir); - printf("%-20s %-10s\n", "Mission directory:", ctx->storage.mission_dir); - printf("%-20s %-10s\n", "Testing enabled:", globals.enable_testing ? "Yes" : "No"); - printf("%-20s %-10s\n", "Docker image builds enabled:", globals.enable_docker ? "Yes" : "No"); - printf("%-20s %-10s\n", "Artifact uploading enabled:", globals.enable_artifactory ? "Yes" : "No"); -} - -void delivery_meta_show(struct Delivery *ctx) { - if (globals.verbose) { - delivery_debug_show(ctx); - } - - printf("\n====DELIVERY====\n"); - printf("%-20s %-10s\n", "Target Python:", ctx->meta.python); - printf("%-20s %-10s\n", "Name:", ctx->meta.name); - printf("%-20s %-10s\n", "Mission:", ctx->meta.mission); - if (ctx->meta.codename) { - printf("%-20s %-10s\n", "Codename:", ctx->meta.codename); - } - if (ctx->meta.version) { - printf("%-20s %-10s\n", "Version", ctx->meta.version); - } - if (!ctx->meta.final) { - printf("%-20s %-10d\n", "RC Level:", ctx->meta.rc); - } - printf("%-20s %-10s\n", "Final Release:", ctx->meta.final ? "Yes" : "No"); - printf("%-20s %-10s\n", "Based On:", ctx->meta.based_on ? ctx->meta.based_on : "New"); -} - -void delivery_conda_show(struct Delivery *ctx) { - printf("\n====CONDA====\n"); - printf("%-20s %-10s\n", "Prefix:", ctx->storage.conda_install_prefix); - - puts("Native Packages:"); - if (strlist_count(ctx->conda.conda_packages) || strlist_count(ctx->conda.conda_packages_defer)) { - struct StrList *list_conda = strlist_init(); - if (strlist_count(ctx->conda.conda_packages)) { - strlist_append_strlist(list_conda, ctx->conda.conda_packages); - } - if (strlist_count(ctx->conda.conda_packages_defer)) { - strlist_append_strlist(list_conda, ctx->conda.conda_packages_defer); - } - strlist_sort(list_conda, STASIS_SORT_ALPHA); - - for (size_t i = 0; i < strlist_count(list_conda); i++) { - char *token = strlist_item(list_conda, i); - if (isempty(token) || isblank(*token) || startswith(token, "-")) { - continue; - } - printf("%21s%s\n", "", token); - } - guard_strlist_free(&list_conda); - } else { - printf("%21s%s\n", "", "N/A"); - } - - puts("Python Packages:"); - if (strlist_count(ctx->conda.pip_packages) || strlist_count(ctx->conda.pip_packages_defer)) { - struct StrList *list_python = strlist_init(); - if (strlist_count(ctx->conda.pip_packages)) { - strlist_append_strlist(list_python, ctx->conda.pip_packages); - } - if (strlist_count(ctx->conda.pip_packages_defer)) { - strlist_append_strlist(list_python, ctx->conda.pip_packages_defer); - } - strlist_sort(list_python, STASIS_SORT_ALPHA); - - for (size_t i = 0; i < strlist_count(list_python); i++) { - char *token = strlist_item(list_python, i); - if (isempty(token) || isblank(*token) || startswith(token, "-")) { - continue; - } - printf("%21s%s\n", "", token); - } - guard_strlist_free(&list_python); - } else { - printf("%21s%s\n", "", "N/A"); - } -} - -void delivery_tests_show(struct Delivery *ctx) { - printf("\n====TESTS====\n"); - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { - if (!ctx->tests[i].name) { - continue; - } - printf("%-20s %-20s %s\n", ctx->tests[i].name, - ctx->tests[i].version, - ctx->tests[i].repository); - } -} - -void delivery_runtime_show(struct Delivery *ctx) { - printf("\n====RUNTIME====\n"); - struct StrList *rt = NULL; - rt = strlist_copy(ctx->runtime.environ); - if (!rt) { - // no data - return; - } - strlist_sort(rt, STASIS_SORT_ALPHA); - size_t total = strlist_count(rt); - for (size_t i = 0; i < total; i++) { - char *item = strlist_item(rt, i); - if (!item) { - // not supposed to occur - msg(STASIS_MSG_WARN | STASIS_MSG_L1, "Encountered unexpected NULL at record %zu of %zu of runtime array.\n", i); - return; - } - printf("%s\n", item); - } -} - diff --git a/src/delivery_test.c b/src/delivery_test.c deleted file mode 100644 index cb78f64..0000000 --- a/src/delivery_test.c +++ /dev/null @@ -1,295 +0,0 @@ -#include "delivery.h" - -void delivery_tests_run(struct Delivery *ctx) { - static const int SETUP = 0; - static const int PARALLEL = 1; - static const int SERIAL = 2; - struct MultiProcessingPool *pool[3]; - struct Process proc; - memset(&proc, 0, sizeof(proc)); - - if (!globals.workaround.conda_reactivate) { - globals.workaround.conda_reactivate = calloc(PATH_MAX, sizeof(*globals.workaround.conda_reactivate)); - } else { - memset(globals.workaround.conda_reactivate, 0, PATH_MAX); - } - // Test blocks always run with xtrace enabled. Disable, and reenable it. Conda's wrappers produce an incredible - // amount of debug information. - snprintf(globals.workaround.conda_reactivate, PATH_MAX - 1, "\nset +x; mamba activate ${CONDA_DEFAULT_ENV}; set -x\n"); - - if (!ctx->tests[0].name) { - msg(STASIS_MSG_WARN | STASIS_MSG_L2, "no tests are defined!\n"); - } else { - pool[PARALLEL] = mp_pool_init("parallel", ctx->storage.tmpdir); - if (!pool[PARALLEL]) { - perror("mp_pool_init/parallel"); - exit(1); - } - pool[PARALLEL]->status_interval = globals.pool_status_interval; - - pool[SERIAL] = mp_pool_init("serial", ctx->storage.tmpdir); - if (!pool[SERIAL]) { - perror("mp_pool_init/serial"); - exit(1); - } - pool[SERIAL]->status_interval = globals.pool_status_interval; - - pool[SETUP] = mp_pool_init("setup", ctx->storage.tmpdir); - if (!pool[SETUP]) { - perror("mp_pool_init/setup"); - exit(1); - } - pool[SETUP]->status_interval = globals.pool_status_interval; - - // Test block scripts shall exit non-zero on error. - // This will fail a test block immediately if "string" is not found in file.txt: - // grep string file.txt - // - // And this is how to avoid that scenario: - // #1: - // if ! grep string file.txt; then - // # handle error - // fi - // - // #2: - // grep string file.txt || handle error - // - // #3: - // # Use ':' as a NO-OP if/when the result doesn't matter - // grep string file.txt || : - const char *runner_cmd_fmt = "set -e -x\n%s\n"; - - // Iterate over our test records, retrieving the source code for each package, and assigning its scripted tasks - // to the appropriate processing pool - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { - struct Test *test = &ctx->tests[i]; - if (!test->name && !test->repository && !test->script) { - // skip unused test records - continue; - } - msg(STASIS_MSG_L2, "Loading tests for %s %s\n", test->name, test->version); - if (!test->script || !strlen(test->script)) { - msg(STASIS_MSG_WARN | STASIS_MSG_L3, "Nothing to do. To fix, declare a 'script' in section: [test:%s]\n", - test->name); - continue; - } - - char destdir[PATH_MAX]; - sprintf(destdir, "%s/%s", ctx->storage.build_sources_dir, path_basename(test->repository)); - - if (!access(destdir, F_OK)) { - msg(STASIS_MSG_L3, "Purging repository %s\n", destdir); - if (rmtree(destdir)) { - COE_CHECK_ABORT(1, "Unable to remove repository\n"); - } - } - msg(STASIS_MSG_L3, "Cloning repository %s\n", test->repository); - if (!git_clone(&proc, test->repository, destdir, test->version)) { - test->repository_info_tag = strdup(git_describe(destdir)); - test->repository_info_ref = strdup(git_rev_parse(destdir, "HEAD")); - } else { - COE_CHECK_ABORT(1, "Unable to clone repository\n"); - } - - if (test->repository_remove_tags && strlist_count(test->repository_remove_tags)) { - filter_repo_tags(destdir, test->repository_remove_tags); - } - - if (pushd(destdir)) { - COE_CHECK_ABORT(1, "Unable to enter repository directory\n"); - } else { - char *cmd = calloc(strlen(test->script) + STASIS_BUFSIZ, sizeof(*cmd)); - if (!cmd) { - SYSERROR("Unable to allocate test script buffer: %s", strerror(errno)); - exit(1); - } - - msg(STASIS_MSG_L3, "Queuing task for %s\n", test->name); - memset(&proc, 0, sizeof(proc)); - - strcpy(cmd, test->script); - char *cmd_rendered = tpl_render(cmd); - if (cmd_rendered) { - if (strcmp(cmd_rendered, cmd) != 0) { - strcpy(cmd, cmd_rendered); - cmd[strlen(cmd_rendered) ? strlen(cmd_rendered) - 1 : 0] = 0; - } - guard_free(cmd_rendered); - } else { - SYSERROR("An error occurred while rendering the following:\n%s", cmd); - exit(1); - } - - if (test->disable) { - msg(STASIS_MSG_L2, "Script execution disabled by configuration\n", test->name); - guard_free(cmd); - continue; - } - - char *runner_cmd = NULL; - char pool_name[100] = "parallel"; - struct MultiProcessingTask *task = NULL; - int selected = PARALLEL; - if (!globals.enable_parallel || !test->parallel) { - selected = SERIAL; - memset(pool_name, 0, sizeof(pool_name)); - strcpy(pool_name, "serial"); - } - - if (asprintf(&runner_cmd, runner_cmd_fmt, cmd) < 0) { - SYSERROR("Unable to allocate memory for runner command: %s", strerror(errno)); - exit(1); - } - task = mp_pool_task(pool[selected], test->name, destdir, runner_cmd); - if (!task) { - SYSERROR("Failed to add task to %s pool: %s", pool_name, runner_cmd); - popd(); - if (!globals.continue_on_error) { - guard_free(runner_cmd); - tpl_free(); - delivery_free(ctx); - globals_free(); - } - exit(1); - } - guard_free(runner_cmd); - guard_free(cmd); - popd(); - - } - } - - // Configure "script_setup" tasks - // Directories should exist now, so no need to go through initializing everything all over again. - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { - struct Test *test = &ctx->tests[i]; - if (test->script_setup) { - char destdir[PATH_MAX]; - sprintf(destdir, "%s/%s", ctx->storage.build_sources_dir, path_basename(test->repository)); - if (access(destdir, F_OK)) { - SYSERROR("%s: %s", destdir, strerror(errno)); - exit(1); - } - if (!pushd(destdir)) { - const size_t cmd_len = strlen(test->script_setup) + STASIS_BUFSIZ; - char *cmd = calloc(cmd_len, sizeof(*cmd)); - if (!cmd) { - SYSERROR("Unable to allocate test script_setup buffer: %s", strerror(errno)); - exit(1); - } - - strncpy(cmd, test->script_setup, cmd_len - 1); - char *cmd_rendered = tpl_render(cmd); - if (cmd_rendered) { - if (strcmp(cmd_rendered, cmd) != 0) { - strncpy(cmd, cmd_rendered, cmd_len - 1); - cmd[strlen(cmd_rendered) ? strlen(cmd_rendered) - 1 : 0] = 0; - } - guard_free(cmd_rendered); - } else { - SYSERROR("An error occurred while rendering the following:\n%s", cmd); - exit(1); - } - - struct MultiProcessingTask *task = NULL; - char *runner_cmd = NULL; - if (asprintf(&runner_cmd, runner_cmd_fmt, cmd) < 0) { - SYSERROR("Unable to allocate memory for runner command: %s", strerror(errno)); - exit(1); - } - - task = mp_pool_task(pool[SETUP], test->name, destdir, runner_cmd); - if (!task) { - SYSERROR("Failed to add task %s to setup pool: %s", test->name, runner_cmd); - popd(); - if (!globals.continue_on_error) { - guard_free(runner_cmd); - tpl_free(); - delivery_free(ctx); - globals_free(); - } - exit(1); - } - guard_free(runner_cmd); - guard_free(cmd); - popd(); - } else { - SYSERROR("Failed to change directory: %s\n", destdir); - exit(1); - } - } - } - - size_t opt_flags = 0; - if (globals.parallel_fail_fast) { - opt_flags |= MP_POOL_FAIL_FAST; - } - - // Execute all queued tasks - for (size_t p = 0; p < sizeof(pool) / sizeof(*pool); p++) { - int pool_status; - long jobs = globals.cpu_limit; - - if (!pool[p]->num_used) { - // Skip empty pool - continue; - } - - // Setup tasks run sequentially - if (p == (size_t) SETUP || p == (size_t) SERIAL) { - jobs = 1; - } - - // Run tasks in the pool - // 1. Setup (builds) - // 2. Parallel (fast jobs) - // 3. Serial (long jobs) - pool_status = mp_pool_join(pool[p], jobs, opt_flags); - - // On error show a summary of the current pool, and die - if (pool_status != 0) { - mp_pool_show_summary(pool[p]); - COE_CHECK_ABORT(true, "Task failure"); - } - } - - // All tasks were successful - for (size_t p = 0; p < sizeof(pool) / sizeof(*pool); p++) { - if (pool[p]->num_used) { - // Only show pools that actually had jobs to run - mp_pool_show_summary(pool[p]); - } - mp_pool_free(&pool[p]); - } - } -} - -int delivery_fixup_test_results(struct Delivery *ctx) { - struct dirent *rec; - DIR *dp; - - dp = opendir(ctx->storage.results_dir); - if (!dp) { - perror(ctx->storage.results_dir); - return -1; - } - - while ((rec = readdir(dp)) != NULL) { - char path[PATH_MAX]; - memset(path, 0, sizeof(path)); - - if (!strcmp(rec->d_name, ".") || !strcmp(rec->d_name, "..") || !endswith(rec->d_name, ".xml")) { - continue; - } - - sprintf(path, "%s/%s", ctx->storage.results_dir, rec->d_name); - msg(STASIS_MSG_L3, "%s\n", rec->d_name); - if (xml_pretty_print_in_place(path, STASIS_XML_PRETTY_PRINT_PROG, STASIS_XML_PRETTY_PRINT_ARGS)) { - msg(STASIS_MSG_L3 | STASIS_MSG_WARN, "Failed to rewrite file '%s'\n", rec->d_name); - } - } - - closedir(dp); - return 0; -} - diff --git a/src/docker.c b/src/docker.c deleted file mode 100644 index 2d5e2cc..0000000 --- a/src/docker.c +++ /dev/null @@ -1,205 +0,0 @@ -#include "core.h" -#include "docker.h" - - -int docker_exec(const char *args, unsigned flags) { - struct Process proc; - char cmd[PATH_MAX]; - - memset(&proc, 0, sizeof(proc)); - memset(cmd, 0, sizeof(cmd)); - snprintf(cmd, sizeof(cmd) - 1, "docker %s", args); - if (flags & STASIS_DOCKER_QUIET) { - strcpy(proc.f_stdout, "/dev/null"); - strcpy(proc.f_stderr, "/dev/null"); - } else { - msg(STASIS_MSG_L2, "Executing: %s\n", cmd); - } - - shell(&proc, cmd); - return proc.returncode; -} - -int docker_script(const char *image, char *data, unsigned flags) { - (void)flags; // TODO: placeholder - FILE *infile; - FILE *outfile; - char cmd[PATH_MAX]; - char buffer[STASIS_BUFSIZ]; - - memset(cmd, 0, sizeof(cmd)); - snprintf(cmd, sizeof(cmd) - 1, "docker run --rm -i %s /bin/sh -", image); - - outfile = popen(cmd, "w"); - if (!outfile) { - // opening command pipe for writing failed - return -1; - } - - infile = fmemopen(data, strlen(data), "r"); - if (!infile) { - // opening memory file for reading failed - return -1; - } - - do { - memset(buffer, 0, sizeof(buffer)); - if (fgets(buffer, sizeof(buffer) - 1, infile) != NULL) { - fputs(buffer, outfile); - } - } while (!feof(infile)); - - fclose(infile); - return pclose(outfile); -} - -int docker_build(const char *dirpath, const char *args, int engine) { - char cmd[PATH_MAX]; - char build[15]; - - memset(build, 0, sizeof(build)); - memset(cmd, 0, sizeof(cmd)); - - if (engine & STASIS_DOCKER_BUILD) { - strcpy(build, "build"); - } - if (engine & STASIS_DOCKER_BUILD_X) { - strcpy(build, "buildx build"); - } - snprintf(cmd, sizeof(cmd) - 1, "%s %s %s", build, args, dirpath); - return docker_exec(cmd, 0); -} - -int docker_save(const char *image, const char *destdir, const char *compression_program) { - char cmd[PATH_MAX]; - - memset(cmd, 0, sizeof(cmd)); - - if (compression_program && strlen(compression_program)) { - char ext[255]; - memset(ext, 0, sizeof(ext)); - if (startswith(compression_program, "zstd")) { - strcpy(ext, "zst"); - } else if (startswith(compression_program, "xz")) { - strcpy(ext, "xz"); - } else if (startswith(compression_program, "gzip")) { - strcpy(ext, "gz"); - } else if (startswith(compression_program, "bzip2")) { - strcpy(ext, "bz2"); - } else { - strncpy(ext, compression_program, sizeof(ext) - 1); - } - sprintf(cmd, "save \"%s\" | %s > \"%s/%s.tar.%s\"", image, compression_program, destdir, image, ext); - } else { - sprintf(cmd, "save \"%s\" -o \"%s/%s.tar\"", image, destdir, image); - - } - return docker_exec(cmd, 0); -} - -static int docker_exists() { - if (find_program("docker")) { - return true; - } - return false; -} - -static char *docker_ident() { - FILE *fp = NULL; - char *tempfile = NULL; - char line[PATH_MAX]; - struct Process proc; - - tempfile = xmkstemp(&fp, "w+"); - if (!fp || !tempfile) { - return NULL; - } - - memset(&proc, 0, sizeof(proc)); - strcpy(proc.f_stdout, tempfile); - strcpy(proc.f_stderr, "/dev/null"); - shell(&proc, "docker --version"); - - if (!freopen(tempfile, "r", fp)) { - remove(tempfile); - guard_free(tempfile); - return NULL; - } - - if (!fgets(line, sizeof(line) - 1, fp)) { - fclose(fp); - remove(tempfile); - guard_free(tempfile); - return NULL; - } - - fclose(fp); - remove(tempfile); - guard_free(tempfile); - - return strdup(line); -} - -int docker_capable(struct DockerCapabilities *result) { - char *version = NULL; - memset(result, 0, sizeof(*result)); - - if (!docker_exists()) { - // docker isn't available - return false; - } - result->available = true; - - if (docker_exec("ps", STASIS_DOCKER_QUIET)) { - // user cannot connect to the socket - return false; - } - - version = docker_ident(); - if (version && startswith(version, "podman")) { - result->podman = true; - } - guard_free(version); - - if (!docker_exec("buildx build --help", STASIS_DOCKER_QUIET)) { - result->build |= STASIS_DOCKER_BUILD_X; - } - if (!docker_exec("build --help", STASIS_DOCKER_QUIET)) { - result->build |= STASIS_DOCKER_BUILD; - } - if (!result->build) { - // can't use docker without a build plugin - return false; - } - result->usable = true; - return true; -} - -void docker_sanitize_tag(char *str) { - char *pos = str; - while (*pos != 0) { - if (!isalnum(*pos)) { - if (*pos != '.' && *pos != ':' && *pos != '/') { - *pos = '-'; - } - } - pos++; - } -} - -int docker_validate_compression_program(char *prog) { - int result = -1; - char **parts = NULL; - if (!prog) { - goto invalid; - } - parts = split(prog, " ", 1); - if (!parts) { - goto invalid; - } - result = find_program(parts[0]) ? 0 : -1; - - invalid: - GENERIC_ARRAY_FREE(parts); - return result; -} diff --git a/src/download.c b/src/download.c deleted file mode 100644 index f83adda..0000000 --- a/src/download.c +++ /dev/null @@ -1,61 +0,0 @@ -// -// Created by jhunk on 10/5/23. -// - -#include -#include -#include "download.h" - -size_t download_writer(void *fp, size_t size, size_t nmemb, void *stream) { - size_t bytes = fwrite(fp, size, nmemb, (FILE *) stream); - return bytes; -} - -long download(char *url, const char *filename, char **errmsg) { - extern char *VERSION; - CURL *c; - CURLcode curl_code; - long http_code = -1; - FILE *fp; - char user_agent[20]; - sprintf(user_agent, "stasis/%s", VERSION); - long timeout = 30L; - char *timeout_str = getenv("STASIS_DOWNLOAD_TIMEOUT"); - - curl_global_init(CURL_GLOBAL_ALL); - c = curl_easy_init(); - curl_easy_setopt(c, CURLOPT_URL, url); - curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, download_writer); - fp = fopen(filename, "wb"); - if (!fp) { - return -1; - } - - curl_easy_setopt(c, CURLOPT_VERBOSE, 0L); - curl_easy_setopt(c, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(c, CURLOPT_USERAGENT, user_agent); - curl_easy_setopt(c, CURLOPT_NOPROGRESS, 0L); - curl_easy_setopt(c, CURLOPT_WRITEDATA, fp); - - if (timeout_str) { - timeout = strtol(timeout_str, NULL, 10); - } - curl_easy_setopt(c, CURLOPT_CONNECTTIMEOUT, timeout); - - curl_code = curl_easy_perform(c); - if (curl_code != CURLE_OK) { - if (errmsg) { - strcpy(*errmsg, curl_easy_strerror(curl_code)); - } else { - fprintf(stderr, "\nCURL ERROR: %s\n", curl_easy_strerror(curl_code)); - } - goto failed; - } - curl_easy_getinfo(c, CURLINFO_RESPONSE_CODE, &http_code); - - failed: - fclose(fp); - curl_easy_cleanup(c); - curl_global_cleanup(); - return http_code; -} \ No newline at end of file diff --git a/src/envctl.c b/src/envctl.c deleted file mode 100644 index 78dd760..0000000 --- a/src/envctl.c +++ /dev/null @@ -1,125 +0,0 @@ -#include "envctl.h" -#include "core.h" - -struct EnvCtl *envctl_init() { - struct EnvCtl *result; - - result = calloc(1, sizeof(*result)); - if (!result) { - return NULL; - } - - result->num_alloc = STASIS_ENVCTL_DEFAULT_ALLOC; - result->item = calloc(result->num_alloc + 1, sizeof(result->item)); - if (!result->item) { - guard_free(result); - return NULL; - } - - return result; -} - -static int callback_builtin_nop(const void *a, const void *b) { - return STASIS_ENVCTL_RET_SUCCESS; -} - -int envctl_register(struct EnvCtl **envctl, unsigned flags, envctl_except_fn *callback, const char *name) { - if ((*envctl)->num_used == (*envctl)->num_alloc) { - (*envctl)->num_alloc += STASIS_ENVCTL_DEFAULT_ALLOC; - struct EnvCtl_Item **tmp = realloc((*envctl)->item, (*envctl)->num_alloc + 1 * sizeof((*envctl)->item)); - if (!tmp) { - return 1; - } else { - (*envctl)->item = tmp; - } - } - - struct EnvCtl_Item **item = (*envctl)->item; - item[(*envctl)->num_used] = calloc(1, sizeof(*item[0])); - if (!item[(*envctl)->num_used]) { - return 1; - } - if (!callback) { - callback = &callback_builtin_nop; - } - item[(*envctl)->num_used]->callback = callback; - item[(*envctl)->num_used]->name = name; - item[(*envctl)->num_used]->flags = flags; - - (*envctl)->num_used++; - return 0; -} - -size_t envctl_get_index(const struct EnvCtl *envctl, const char *name) { - for (size_t i = 0; i < envctl->num_used; i++) { - if (!strcmp(envctl->item[i]->name, name)) { - // pack state flag, outer (struct) index and inner (name) index - return 1L << 63L | i; - } - } - return 0; -} - -void envctl_decode_index(size_t in_i, size_t *state, size_t *out_i, size_t *name_i) { - *state = ((in_i >> 63L) & 1); - *out_i = in_i & 0xffffffffL; -} - -unsigned envctl_check_required(unsigned flags) { - return flags & STASIS_ENVCTL_REQUIRED; -} - -unsigned envctl_check_redact(unsigned flags) { - return flags & STASIS_ENVCTL_REDACT; -} - -int envctl_check_present(const struct EnvCtl_Item *item, const char *name) { - return ((!strcmp(item->name, name)) && getenv(name)) ? 1 : 0; -} - -unsigned envctl_get_flags(const struct EnvCtl *envctl, const char *name) { - size_t poll_index = envctl_get_index(envctl, name); - size_t id = 0; - size_t name_id = 0; - size_t state = 0; - envctl_decode_index(poll_index, &state, &id, &name_id); - if (!state) { - return 0; - } else { - fprintf(stderr, "managed environment variable: %s\n", name); - } - return envctl->item[id]->flags; -} - -void envctl_do_required(const struct EnvCtl *envctl, int verbose) { - for (size_t i = 0; i < envctl->num_used; i++) { - struct EnvCtl_Item *item = envctl->item[i]; - const char *name = item->name; - envctl_except_fn *callback = item->callback; - - if (verbose) { - msg(STASIS_MSG_L2, "Verifying %s\n", name); - } - int code = callback((const void *) item, (const void *) name); - if (code == STASIS_ENVCTL_RET_IGNORE || code == STASIS_ENVCTL_RET_SUCCESS) { - continue; - } else if (code == STASIS_ENVCTL_RET_FAIL) { - fprintf(stderr, "\n%s must be set. Exiting.\n", name); - exit(1); - } else { - fprintf(stderr, "\nan unknown envctl callback code occurred: %d\n", code); - exit(1); - } - } -} - -void envctl_free(struct EnvCtl **envctl) { - if (!envctl) { - return; - } - for (size_t i = 0; i < (*envctl)->num_used; i++) { - guard_free((*envctl)->item[i]); - } - guard_free((*envctl)->item); - guard_free(*envctl); -} \ No newline at end of file diff --git a/src/environment.c b/src/environment.c deleted file mode 100644 index 580062c..0000000 --- a/src/environment.c +++ /dev/null @@ -1,443 +0,0 @@ -/** - * @file environment.c - */ -#include "environment.h" -#include "utils.h" -#include "strlist.h" - -//extern char **__environ; - -/** - * Print a shell-specific listing of environment variables to `stdout` - * - * Example: - * ~~~{.c} - * int main(int argc, char *argv[], char *arge[]) { - * RuntimeEnv *rt = runtime_copy(arge); - * runtime_export(rt, NULL); - * runtime_free(rt); - * return 0; - * } - * ~~~ - * - * Usage: - * ~~~{.sh} - * $ gcc program.c - * $ ./a.out - * PATH="/thing/stuff/bin:/example/please/bin" - * SHELL="/your/shell" - * CC="/your/compiler" - * ...=... - * - * # You can also use this to modify the shell environment - * # (use `runtime_set` to manipulate the output) - * $ source $(./a.out) - * ~~~ - * - * Example of exporting specific keys from the environment: - * - * ~~~{.c} - * int main(int argc, char *argv[], char *arge[]) { - * RuntimeEnv *rt = runtime_copy(arge); - * - * // inline declaration - * runtime_export(rt, (char *[]) {"PATH", "LS_COLORS", NULL}); - * - * // standard declaration - * char *keys_to_export[] = { - * "PATH", "LS_COLORS", NULL - * } - * runtime_export(rt, keys_to_export); - * - * runtime_free(rt); - * return 0; - * } - * ~~~ - * - * @param env `RuntimeEnv` structure - * @param keys Array of keys to export. A value of `NULL` exports all environment keys - */ -void runtime_export(RuntimeEnv *env, char **keys) { - char *borne[] = { - "bash", - "dash", - "zsh", - NULL, - }; - char *unborne[] = { - "csh" - "tcsh", - NULL, - }; - - char output[STASIS_BUFSIZ]; - char export_command[7]; // export=6 and setenv=6... convenient - char *_sh = getenv("SHELL"); - char *sh = path_basename(_sh); - if (sh == NULL) { - fprintf(stderr, "echo SHELL environment variable is not defined"); - exit(1); - } - - for (size_t i = 0; borne[i] != NULL; i++) { - if (strcmp(sh, borne[i]) == 0) { - strcpy(export_command, "export"); - break; - } - } - for (size_t i = 0; unborne[i] != NULL; i++) { - if (strcmp(sh, unborne[i]) == 0) { - strcpy(export_command, "setenv"); - break; - } - } - - for (size_t i = 0; i < strlist_count(env); i++) { - char **pair = split(strlist_item(env, i), "=", 0); - char *key = pair[0]; - char *value = NULL; - - // We split a potentially large string by "=" so: - // Recombine elements pair[1..N] into a single string by "=" - if (pair[1] != NULL) { - value = join(&pair[1], "="); - } - - if (keys != NULL) { - for (size_t j = 0; keys[j] != NULL; j++) { - if (strcmp(keys[j], key) == 0) { - //sprintf(output, "%s=\"%s\"\n%s %s", key, value ? value : "", export_command, key); - sprintf(output, "%s %s=\"%s\"", export_command, key, value ? value : ""); - puts(output); - } - } - } - else { - sprintf(output, "%s %s=\"%s\"", export_command, key, value ? value : ""); - puts(output); - } - guard_free(value); - GENERIC_ARRAY_FREE(pair); - } -} - -/** - * Populate a `RuntimeEnv` structure - * - * Example: - * - * ~~~{.c} - * int main(int argc, char *argv[], char *arge[]) { - * RuntimeEnv *rt = NULL; - * // Example 1: Copy the shell environment - * rt = runtime_copy(arge); - * // Example 2: Create your own environment - * rt = runtime_copy((char *[]) {"SHELL=/bin/bash", "PATH=/opt/secure:/bin:/usr/bin"}) - * - * runtime_free(rt); - * return 0; - * } - * ~~~ - * - * @param env Array of strings in `var=value` format - * @return `RuntimeEnv` structure - */ -RuntimeEnv *runtime_copy(char **env) { - RuntimeEnv *rt = NULL; - size_t env_count; - for (env_count = 0; env[env_count] != NULL; env_count++); - - rt = strlist_init(); - for (size_t i = 0; i < env_count; i++) { - strlist_append(&rt, env[i]); - } - return rt; -} - -/** - * Replace the contents of `dest` with `src` - * @param dest pointer of type `RuntimeEnv` - * @param src pointer to environment array - * @return 0 on success, <0 on error - */ -int runtime_replace(RuntimeEnv **dest, char **src) { - RuntimeEnv *rt_tmp = runtime_copy(src); - if (!rt_tmp) { - return -1; - } - runtime_free((*dest)); - - (*dest) = runtime_copy(rt_tmp->data); - if (!(*dest)) { - return -1; - } - runtime_free(rt_tmp); - - runtime_apply((*dest)); - return 0; -} - -/** - * Determine whether or not a key exists in the runtime environment - * - * Example: - * - * ~~~{.c} - * int main(int argc, char *argv[], char *arge[]) { - * RuntimeEnv *rt = runtime_copy(arge); - * if (runtime_contains(rt, "PATH") { - * // $PATH is present - * } - * else { - * // $PATH is NOT present - * } - * - * runtime_free(rt); - * return 0; - * } - * ~~~ - * - * @param env `RuntimeEnv` structure - * @param key Environment variable string - * @return -1=no, positive_value=yes - */ -ssize_t runtime_contains(RuntimeEnv *env, const char *key) { - ssize_t result = -1; - for (ssize_t i = 0; i < (ssize_t) strlist_count(env); i++) { - char **pair = split(strlist_item(env, i), "=", 0); - if (pair == NULL) { - break; - } - if (strcmp(pair[0], key) == 0) { - result = i; - GENERIC_ARRAY_FREE(pair); - break; - } - GENERIC_ARRAY_FREE(pair); - } - return result; -} - -/** - * Retrieve the value of a runtime environment variable - * - * Example: - * - * ~~~{.c} - * int main(int argc, char *argv[], char *arge[]) { - * RuntimeEnv *rt = runtime_copy(arge); - * char *path = runtime_get("PATH"); - * if (path == NULL) { - * // handle error - * } - * - * runtime_free(rt); - * return 0; - * } - * ~~~ - * - * @param env `RuntimeEnv` structure - * @param key Environment variable string - * @return success=string, failure=`NULL` - */ -char *runtime_get(RuntimeEnv *env, const char *key) { - char *result = NULL; - ssize_t key_offset = runtime_contains(env, key); - if (key_offset != -1) { - char **pair = split(strlist_item(env, key_offset), "=", 0); - result = join(&pair[1], "="); - GENERIC_ARRAY_FREE(pair); - } - return result; -} - -/** - * Parse an input string and expand any environment variable(s) found - * - * Example: - * - * ~~~{.c} - * int main(int argc, char *argv[], char *arge[]) { - * RuntimeEnv *rt = runtime_copy(arge); - * char *secure_path = runtime_expand_var(rt, "/opt/secure:$PATH:/aux/bin"); - * if (secure_path == NULL) { - * // handle error - * } - * // secure_path = "/opt/secure:/your/original/path/here:/aux/bin"; - * - * runtime_free(rt); - * return 0; - * } - * ~~~ - * - * @param env `RuntimeEnv` structure - * @param input String to parse - * @return success=expanded string, failure=`NULL` - */ -char *runtime_expand_var(RuntimeEnv *env, char *input) { - const char delim = '$'; - const char *delim_literal = "$$"; - char *expanded = NULL; - - // Input is invalid - if (!input) { - return NULL; - } - - // If there's no environment variables to process return the input string - if (strchr(input, delim) == NULL) { - //return strdup(input); - return input; - } - - expanded = calloc(STASIS_BUFSIZ, sizeof(char)); - if (expanded == NULL) { - SYSERROR("could not allocate %d bytes for runtime_expand_var buffer", STASIS_BUFSIZ); - return NULL; - } - - // Parse the input string - size_t i; - for (i = 0; i < strlen(input); i++) { - char var[MAXNAMLEN]; // environment variable name - memset(var, '\0', MAXNAMLEN); // zero out name - - // Handle literal statement "$$var" - // Value becomes "$var" (unexpanded) - if (strncmp(&input[i], delim_literal, strlen(delim_literal)) == 0) { - strncat(expanded, &delim, 2); - i += strlen(delim_literal); - // Ignore opening brace - if (input[i] == '{') { - i++; - } - } - - // Handle variable when encountering a single $ - // Value expands from "$var" to "environment value of var" - if (input[i] == delim) { - // Ignore opening brace - if (input[i+1] == '{') { - i++; - } - char *tmp = NULL; - i++; - - // Construct environment variable name from input - // "$ var" == no - // "$-*)!@ == no - // "$var" == yes - for (size_t c = 0; isalnum(input[i]) || input[i] == '_'; c++, i++) { - // Ignore closing brace - if (input[i] == '}') { - i++; - } - var[c] = input[i]; - } - - if (env) { - tmp = runtime_get(env, var); - } else { - tmp = getenv(var); - } - if (tmp == NULL) { - // This mimics shell behavior in general. - // Prevent appending whitespace when an environment variable does not exist - if (i > 0) { - i--; - } - continue; - } - // Append expanded environment variable to output - strncat(expanded, tmp, STASIS_BUFSIZ - 1); - if (env) { - guard_free(tmp); - } - } - - // Nothing to do so append input to output - if (input[i] == '}') { - // Unless we ended on a closing brace - continue; - } - strncat(expanded, &input[i], 1); - } - - return expanded; -} - -/** - * Set a runtime environment variable. - * - * - * Note: `_value` is passed through `runtime_expand_var` to provide shell expansion - * - * - * Example: - * - * ~~~{.c} - * int main(int argc, char *argv[], char *arge[]) { - * RuntimeEnv *rt = runtime_copy(arge); - * - * runtime_set(rt, "new_var", "1"); - * char *new_var = runtime_get("new_var"); - * // new_var = 1; - * - * char *path = runtime_get("PATH"); - * // path = /your/path:/here - * - * runtime_set(rt, "PATH", "/opt/secure:$PATH"); - * char *secure_path = runtime_get("PATH"); - * // secure_path = /opt/secure:/your/path:/here - * // NOTE: path and secure_path are COPIES, unlike `getenv()` and `setenv()` that reuse their pointers in `environ` - * - * runtime_free(rt); - * return 0; - * } - * ~~~ - * - * - * @param env `RuntimeEnv` structure - * @param _key Environment variable to set - * @param _value New environment variable value - */ -void runtime_set(RuntimeEnv *env, const char *_key, char *_value) { - if (_key == NULL) { - return; - } - char *key = strdup(_key); - ssize_t key_offset = runtime_contains(env, key); - char *value = runtime_expand_var(env, _value); - char *now = join((char *[]) {key, value, NULL}, "="); - - if (key_offset < 0) { - strlist_append(&env, now); - } else { - strlist_set(&env, key_offset, now); - } - guard_free(now); - guard_free(key); -} - -/** - * Update the global `environ` array with data from `RuntimeEnv` - * @param env `RuntimeEnv` structure - */ -void runtime_apply(RuntimeEnv *env) { - for (size_t i = 0; i < strlist_count(env); i++) { - char **pair = split(strlist_item(env, i), "=", 1); - setenv(pair[0], pair[1], 1); - GENERIC_ARRAY_FREE(pair); - } -} - -/** - * Free `RuntimeEnv` allocated by `runtime_copy` - * @param env `RuntimeEnv` structure - */ -void runtime_free(RuntimeEnv *env) { - if (env == NULL) { - return; - } - strlist_free(&env); -} diff --git a/src/github.c b/src/github.c deleted file mode 100644 index 36e2e7c..0000000 --- a/src/github.c +++ /dev/null @@ -1,133 +0,0 @@ -#include -#include -#include -#include "core.h" - -struct GHContent { - char *data; - size_t len; -}; - -static size_t writer(void *contents, size_t size, size_t nmemb, void *result) { - const size_t newlen = size * nmemb; - struct GHContent *content = (struct GHContent *) result; - - char *ptr = realloc(content->data, content->len + newlen + 1); - if (!ptr) { - perror("realloc failed"); - return 0; - } - - content->data = ptr; - memcpy(&(content->data[content->len]), contents, newlen); - content->len += newlen; - content->data[content->len] = 0; - - return newlen; -} - -static char *unescape_lf(char *value) { - char *seq = strstr(value, "\\n"); - while (seq != NULL) { - size_t cur_len = strlen(seq); - memmove(seq, seq + 1, strlen(seq) - 1); - *seq = '\n'; - if (strlen(seq) && cur_len) { - seq[cur_len - 1] = 0; - } - seq = strstr(value, "\\n"); - } - return value; -} - -int get_github_release_notes(const char *api_token, const char *repo, const char *tag, const char *target_commitish, char **output) { - const char *field_body = "\"body\":\""; - const char *field_message = "\"message\":\""; - const char *endpoint_header_auth_fmt = "Authorization: Bearer %s"; - const char *endpoint_header_api_version = "X-GitHub-Api-Version: " STASIS_GITHUB_API_VERSION; - const char *endpoint_post_fields_fmt = "{\"tag_name\":\"%s\", \"target_commitish\":\"%s\"}"; - const char *endpoint_url_fmt = "https://api.github.com/repos/%s/releases/generate-notes"; - char endpoint_header_auth[PATH_MAX] = {0}; - char endpoint_post_fields[PATH_MAX] = {0}; - char endpoint_url[PATH_MAX] = {0}; - struct curl_slist *list = NULL; - struct GHContent content; - - CURL *curl = curl_easy_init(); - if (!curl) { - return -1; - } - - // Render the header data - sprintf(endpoint_header_auth, endpoint_header_auth_fmt, api_token); - sprintf(endpoint_post_fields, endpoint_post_fields_fmt, tag, target_commitish); - sprintf(endpoint_url, endpoint_url_fmt, repo); - - // Begin curl configuration - curl_easy_setopt(curl, CURLOPT_URL, endpoint_url); - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, endpoint_post_fields); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writer); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) &content); - - // Append headers to the request - list = curl_slist_append(list, "Accept: application/vnd.github+json"); - list = curl_slist_append(list, endpoint_header_auth); - list = curl_slist_append(list, endpoint_header_api_version); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list); - - // Set the user-agent (github requires one) - char user_agent[20] = {0}; - sprintf(user_agent, "stasis/%s", VERSION); - curl_easy_setopt(curl, CURLOPT_USERAGENT, user_agent); - - // Execute curl request - memset(&content, 0, sizeof(content)); - CURLcode res; - res = curl_easy_perform(curl); - - // Clean up - curl_slist_free_all(list); - curl_easy_cleanup(curl); - - if(res != CURLE_OK) { - fprintf(stderr, "curl_easy_perform() failed: %s\n", - curl_easy_strerror(res)); - return -1; - } - - // Replace all "\\n" literals with new line characters - char *line = unescape_lf(content.data); - if (line) { - char *data_offset = NULL; - if ((data_offset = strstr(line, field_body))) { - // Skip past the body field - data_offset += strlen(field_body); - // Remove quotation mark (and trailing comma if it exists) - int trim = 2; - char last_char = data_offset[strlen(data_offset) - trim]; - if (last_char == ',') { - trim++; - } - data_offset[strlen(data_offset) - trim] = 0; - // Extract release notes - *output = strdup(data_offset); - } else if ((data_offset = strstr(line, field_message))) { - // Skip past the message field - data_offset += strlen(field_message); - *(strchr(data_offset, '"')) = 0; - fprintf(stderr, "GitHub API Error: '%s'\n", data_offset); - fprintf(stderr, "URL: %s\n", endpoint_url); - fprintf(stderr, "POST: %s\n", endpoint_post_fields); - guard_free(content.data); - return -1; - } - } else { - fprintf(stderr, "Unknown error\n"); - guard_free(content.data); - return -1; - } - - guard_free(content.data); - return 0; -} \ No newline at end of file diff --git a/src/globals.c b/src/globals.c deleted file mode 100644 index 1b682cb..0000000 --- a/src/globals.c +++ /dev/null @@ -1,65 +0,0 @@ -#include -#include -#include "core.h" - -const char *VERSION = "1.0.0"; -const char *AUTHOR = "Joseph Hunkeler"; -const char *BANNER = - "------------------------------------------------------------------------\n" -#if defined(STASIS_DUMB_TERMINAL) - " STASIS \n" -#else - " _____ _______ _____ _____ _____ \n" - " / ____|__ __|/\\ / ____|_ _|/ ____| \n" - " | (___ | | / \\ | (___ | | | (___ \n" - " \\___ \\ | | / /\\ \\ \\___ \\ | | \\___ \\ \n" - " ____) | | |/ ____ \\ ____) |_| |_ ____) | \n" - " |_____/ |_/_/ \\_\\_____/|_____|_____/ \n" - "\n" -#endif - "------------------------------------------------------------------------\n" - " Delivery Generator \n" - " v%s \n" - "------------------------------------------------------------------------\n" - "Copyright (C) 2023-2024 %s,\n" - "Association of Universities for Research in Astronomy (AURA)\n"; - -struct STASIS_GLOBAL globals = { - .verbose = false, ///< Toggle verbose mode - .continue_on_error = false, ///< Do not stop program on error - .always_update_base_environment = false, ///< Run "conda update --all" after installing Conda - .conda_fresh_start = true, ///< Remove/reinstall Conda at startup - .conda_install_prefix = NULL, ///< Path to install Conda - .conda_packages = NULL, ///< Conda packages to install - .pip_packages = NULL, ///< Python packages to install - .tmpdir = NULL, ///< Path to store temporary data - .enable_docker = true, ///< Toggle docker usage - .enable_artifactory = true, ///< Toggle artifactory server usage - .enable_artifactory_build_info = true, ///< Toggle build-info uploads - .enable_testing = true, ///< Toggle [test] block "script" execution. "script_setup" always executes. - .enable_rewrite_spec_stage_2 = true, ///< Leave template stings in output files - .enable_parallel = true, ///< Toggle testing in parallel - .parallel_fail_fast = false, ///< Kill ALL multiprocessing tasks immediately on error - .pool_status_interval = 30, ///< Report "Task is running" -}; - -void globals_free() { - guard_free(globals.tmpdir); - guard_free(globals.sysconfdir); - guard_free(globals.conda_install_prefix); - guard_strlist_free(&globals.conda_packages); - guard_strlist_free(&globals.pip_packages); - guard_free(globals.jfrog.arch); - guard_free(globals.jfrog.os); - guard_free(globals.jfrog.url); - guard_free(globals.jfrog.repo); - guard_free(globals.jfrog.version); - guard_free(globals.jfrog.cli_major_ver); - guard_free(globals.jfrog.jfrog_artifactory_base_url); - guard_free(globals.jfrog.jfrog_artifactory_product); - guard_free(globals.jfrog.remote_filename); - guard_free(globals.workaround.conda_reactivate); - if (globals.envctl) { - envctl_free(&globals.envctl); - } -} diff --git a/src/ini.c b/src/ini.c deleted file mode 100644 index d44e1cc..0000000 --- a/src/ini.c +++ /dev/null @@ -1,678 +0,0 @@ -#include -#include -#include -#include -#include "core.h" -#include "ini.h" - -struct INIFILE *ini_init() { - struct INIFILE *ini; - ini = calloc(1, sizeof(*ini)); - ini->section_count = 0; - return ini; -} - -void ini_section_init(struct INIFILE **ini) { - (*ini)->section = calloc((*ini)->section_count + 1, sizeof(**(*ini)->section)); -} - -struct INISection *ini_section_search(struct INIFILE **ini, unsigned mode, const char *value) { - struct INISection *result = NULL; - for (size_t i = 0; i < (*ini)->section_count; i++) { - if ((*ini)->section[i]->key != NULL) { - if (mode == INI_SEARCH_EXACT) { - if (!strcmp((*ini)->section[i]->key, value)) { - result = (*ini)->section[i]; - break; - } - } else if (mode == INI_SEARCH_BEGINS) { - if (startswith((*ini)->section[i]->key, value)) { - result = (*ini)->section[i]; - break; - } - } else if (mode == INI_SEARCH_SUBSTR) { - if (strstr((*ini)->section[i]->key, value)) { - result = (*ini)->section[i]; - break; - } - } - } - } - return result; -} - -int ini_data_init(struct INIFILE **ini, char *section_name) { - struct INISection *section = ini_section_search(ini, INI_SEARCH_EXACT, section_name); - if (section == NULL) { - return 1; - } - section->data = calloc(section->data_count + 1, sizeof(**section->data)); - return 0; -} - -int ini_has_key(struct INIFILE *ini, const char *section_name, const char *key) { - if (!ini || !section_name || !key) { - return 0; - } - struct INISection *section = ini_section_search(&ini, INI_SEARCH_EXACT, section_name); - if (!section) { - return 0; - } - for (size_t i = 0; i < section->data_count; i++) { - const struct INIData *data = section->data[i]; - if (data && data->key) { - if (!strcmp(data->key, key)) { - return 1; - } - } - } - return 0; -} - -struct INIData *ini_data_get(struct INIFILE *ini, char *section_name, char *key) { - struct INISection *section = NULL; - - section = ini_section_search(&ini, INI_SEARCH_EXACT, section_name); - if (!section) { - return NULL; - } - - for (size_t i = 0; i < section->data_count; i++) { - if (section->data[i]->key != NULL) { - if (!strcmp(section->data[i]->key, key)) { - return section->data[i]; - } - } - } - - return NULL; -} - -struct INIData *ini_getall(struct INIFILE *ini, char *section_name) { - struct INISection *section = NULL; - struct INIData *result = NULL; - static size_t i = 0; - - section = ini_section_search(&ini, INI_SEARCH_EXACT, section_name); - if (!section) { - return NULL; - } - if (i == section->data_count) { - i = 0; - return NULL; - } - if (section->data_count) { - result = section->data[i]; - i++; - } else { - result = NULL; - i = 0; - } - - return result; -} - -int ini_getval(struct INIFILE *ini, char *section_name, char *key, int type, int flags, union INIVal *result) { - char *token = NULL; - char tbuf[STASIS_BUFSIZ]; - char *tbufp = tbuf; - struct INIData *data; - data = ini_data_get(ini, section_name, key); - if (!data) { - result->as_char_p = NULL; - return -1; - } - - char *data_copy = strdup(data->value); - if (flags == INI_READ_RENDER) { - char *render = tpl_render(data_copy); - if (render && strcmp(render, data_copy) != 0) { - guard_free(data_copy); - data_copy = render; - } else { - guard_free(render); - } - } - lstrip(data_copy); - - switch (type) { - case INIVAL_TYPE_CHAR: - result->as_char = (char) strtol(data_copy, NULL, 10); - break; - case INIVAL_TYPE_UCHAR: - result->as_uchar = (unsigned char) strtoul(data_copy, NULL, 10); - break; - case INIVAL_TYPE_SHORT: - result->as_short = (short) strtol(data_copy, NULL, 10); - break; - case INIVAL_TYPE_USHORT: - result->as_ushort = (unsigned short) strtoul(data_copy, NULL, 10); - break; - case INIVAL_TYPE_INT: - result->as_int = (int) strtol(data_copy, NULL, 10); - break; - case INIVAL_TYPE_UINT: - result->as_uint = (unsigned int) strtoul(data_copy, NULL, 10); - break; - case INIVAL_TYPE_LONG: - result->as_long = (long) strtol(data_copy, NULL, 10); - break; - case INIVAL_TYPE_ULONG: - result->as_ulong = (unsigned long) strtoul(data_copy, NULL, 10); - break; - case INIVAL_TYPE_LLONG: - result->as_llong = (long long) strtoll(data_copy, NULL, 10); - break; - case INIVAL_TYPE_ULLONG: - result->as_ullong = (unsigned long long) strtoull(data_copy, NULL, 10); - break; - case INIVAL_TYPE_DOUBLE: - result->as_double = (double) strtod(data_copy, NULL); - break; - case INIVAL_TYPE_FLOAT: - result->as_float = strtof(data_copy, NULL); - break; - case INIVAL_TYPE_STR: - result->as_char_p = strdup(data_copy); - if (!result->as_char_p) { - return -1; - } - break; - case INIVAL_TYPE_STR_ARRAY: - strcpy(tbufp, data_copy); - guard_free(data_copy); - data_copy = calloc(STASIS_BUFSIZ, sizeof(*data_copy)); - if (!data_copy) { - return -1; - } - while ((token = strsep(&tbufp, "\n")) != NULL) { - //lstrip(token); - if (!isempty(token)) { - strcat(data_copy, token); - strcat(data_copy, "\n"); - } - } - strip(data_copy); - result->as_char_p = strdup(data_copy); - break; - case INIVAL_TYPE_BOOL: - result->as_bool = false; - if ((!strcmp(data_copy, "true") || !strcmp(data_copy, "True")) || - (!strcmp(data_copy, "yes") || !strcmp(data_copy, "Yes")) || - strtol(data_copy, NULL, 10)) { - result->as_bool = true; - } - break; - default: - memset(result, 0, sizeof(*result)); - break; - } - guard_free(data_copy); - return 0; -} - -#define getval_returns(t) return result.t -#define getval_setup(t, f) \ - union INIVal result; \ - int state_local = 0; \ - state_local = ini_getval(ini, section_name, key, t, f, &result); \ - if (state != NULL) { \ - *state = state_local; \ - } - -int ini_getval_int(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { - getval_setup(INIVAL_TYPE_INT, flags) - getval_returns(as_int); -} - -unsigned int ini_getval_uint(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { - getval_setup(INIVAL_TYPE_UINT, flags) - getval_returns(as_uint); -} - -long ini_getval_long(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { - getval_setup(INIVAL_TYPE_LONG, flags) - getval_returns(as_long); -} - -unsigned long ini_getval_ulong(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { - getval_setup(INIVAL_TYPE_ULONG, flags) - getval_returns(as_ulong); -} - -long long ini_getval_llong(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { - getval_setup(INIVAL_TYPE_LLONG, flags) - getval_returns(as_llong); -} - -unsigned long long ini_getval_ullong(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { - getval_setup(INIVAL_TYPE_ULLONG, flags) - getval_returns(as_ullong); -} - -float ini_getval_float(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { - getval_setup(INIVAL_TYPE_FLOAT, flags) - getval_returns(as_float); -} - -double ini_getval_double(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { - getval_setup(INIVAL_TYPE_DOUBLE, flags) - getval_returns(as_double); -} - -bool ini_getval_bool(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { - getval_setup(INIVAL_TYPE_BOOL, flags) - getval_returns(as_bool); -} - -short ini_getval_short(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { - getval_setup(INIVAL_TYPE_SHORT, flags) - getval_returns(as_short); -} - -unsigned short ini_getval_ushort(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { - getval_setup(INIVAL_TYPE_USHORT, flags) - getval_returns(as_ushort); -} - -char ini_getval_char(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { - getval_setup(INIVAL_TYPE_CHAR, flags) - getval_returns(as_char); -} - -unsigned char ini_getval_uchar(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { - getval_setup(INIVAL_TYPE_UCHAR, flags) - getval_returns(as_uchar); -} - -char *ini_getval_char_p(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { - getval_setup(INIVAL_TYPE_STR, flags) - getval_returns(as_char_p); -} - -char *ini_getval_str(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { - return ini_getval_char_p(ini, section_name, key, flags, state); -} - -char *ini_getval_char_array_p(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { - getval_setup(INIVAL_TYPE_STR_ARRAY, flags) - getval_returns(as_char_p); -} - -char *ini_getval_str_array(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { - return ini_getval_char_array_p(ini, section_name, key, flags, state); -} - -struct StrList *ini_getval_strlist(struct INIFILE *ini, char *section_name, char *key, char *tok, int flags, int *state) { - getval_setup(INIVAL_TYPE_STR_ARRAY, flags) - struct StrList *list; - list = strlist_init(); - strlist_append_tokenize(list, result.as_char_p, tok); - guard_free(result.as_char_p); - return list; -} - -int ini_data_append(struct INIFILE **ini, char *section_name, char *key, char *value, unsigned int hint) { - struct INISection *section = ini_section_search(ini, INI_SEARCH_EXACT, section_name); - if (section == NULL) { - return 1; - } - - struct INIData **tmp = realloc(section->data, (section->data_count + 1) * sizeof(**section->data)); - if (tmp == NULL) { - return 1; - } else { - section->data = tmp; - } - if (!ini_data_get((*ini), section_name, key)) { - struct INIData **data = section->data; - data[section->data_count] = calloc(1, sizeof(*data[0])); - if (!data[section->data_count]) { - SYSERROR("Unable to allocate %zu bytes for section data", sizeof(*data[0])); - return -1; - } - data[section->data_count]->type_hint = hint; - data[section->data_count]->key = key ? strdup(key) : strdup(""); - if (!data[section->data_count]->key) { - SYSERROR("Unable to allocate data key%s", ""); - return -1; - } - data[section->data_count]->value = strdup(value); - if (!data[section->data_count]->value) { - SYSERROR("Unable to allocate data value%s", ""); - return -1; - } - section->data_count++; - } else { - struct INIData *data = ini_data_get(*ini, section_name, key); - size_t value_len_old = strlen(data->value); - size_t value_len = strlen(value); - size_t value_len_new = value_len_old + value_len; - char *value_tmp = NULL; - value_tmp = realloc(data->value, value_len_new + 2); - if (!value_tmp) { - SYSERROR("Unable to increase data->value size to %zu bytes", value_len_new + 2); - return -1; - } else { - data->value = value_tmp; - } - strcat(data->value, value); - } - return 0; -} - -int ini_setval(struct INIFILE **ini, unsigned type, char *section_name, char *key, char *value) { - struct INISection *section = ini_section_search(ini, INI_SEARCH_EXACT, section_name); - if (section == NULL) { - // no section - return -1; - } - if (ini_has_key(*ini, section_name, key)) { - if (!type) { - if (ini_data_append(ini, section_name, key, value, 0)) { - // append failed - return -1; - } - } else { - struct INIData *data = ini_data_get(*ini, section_name, key); - if (data) { - guard_free(data->value); - data->value = strdup(value); - if (!data->value) { - // allocation failed - return -1; - } - } else { - // getting data failed - return -1; - } - } - } - return 0; -} - -int ini_section_create(struct INIFILE **ini, char *key) { - struct INISection **tmp = realloc((*ini)->section, ((*ini)->section_count + 1) * sizeof(**(*ini)->section)); - if (tmp == NULL) { - return 1; - } else { - (*ini)->section = tmp; - } - - (*ini)->section[(*ini)->section_count] = calloc(1, sizeof(*(*ini)->section[0])); - if (!(*ini)->section[(*ini)->section_count]) { - return -1; - } - - (*ini)->section[(*ini)->section_count]->key = strdup(key); - if (!(*ini)->section[(*ini)->section_count]->key) { - return -1; - } - - (*ini)->section_count++; - return 0; -} - -int ini_write(struct INIFILE *ini, FILE **stream, unsigned mode) { - if (!*stream) { - return -1; - } - for (size_t x = 0; x < ini->section_count; x++) { - struct INISection *section = ini->section[x]; - char *section_name = section->key; - fprintf(*stream, "[%s]" LINE_SEP, section_name); - - for (size_t y = 0; y < ini->section[x]->data_count; y++) { - struct INIData *data = section->data[y]; - char outvalue[STASIS_BUFSIZ]; - char *key = data->key; - char *value = data->value; - unsigned *hint = &data->type_hint; - memset(outvalue, 0, sizeof(outvalue)); - - if (key && value) { - int err = 0; - char *xvalue = NULL; - if (*hint == INIVAL_TYPE_STR_ARRAY) { - xvalue = ini_getval_str_array(ini, section_name, key, (int) mode, &err); - value = xvalue; - } else { - xvalue = ini_getval_str(ini, section_name, key, (int) mode, &err); - value = xvalue; - } - char **parts = split(value, LINE_SEP, 0); - size_t parts_total = 0; - for (; parts && parts[parts_total] != NULL; parts_total++); - for (size_t p = 0; parts && parts[p] != NULL; p++) { - char *render = NULL; - if (mode == INI_WRITE_PRESERVE) { - render = tpl_render(parts[p]); - } else { - render = parts[p]; - } - - if (!render) { - SYSERROR("%s", "rendered string value can never be NULL!\n"); - return -1; - } - - if (*hint == INIVAL_TYPE_STR_ARRAY) { - int leading_space = isspace(*render); - if (leading_space) { - sprintf(outvalue + strlen(outvalue), "%s" LINE_SEP, render); - } else { - sprintf(outvalue + strlen(outvalue), " %s" LINE_SEP, render); - } - } else { - sprintf(outvalue + strlen(outvalue), "%s", render); - } - if (mode == INI_WRITE_PRESERVE) { - guard_free(render); - } - } - GENERIC_ARRAY_FREE(parts); - strip(outvalue); - strcat(outvalue, LINE_SEP); - fprintf(*stream, "%s = %s%s", ini->section[x]->data[y]->key, *hint == INIVAL_TYPE_STR_ARRAY ? LINE_SEP : "", outvalue); - guard_free(value); - } else { - fprintf(*stream, "%s = %s", ini->section[x]->data[y]->key, ini->section[x]->data[y]->value); - } - } - fprintf(*stream, LINE_SEP); - } - return 0; -} - -char *unquote(char *s) { - if ((startswith(s, "'") && endswith(s, "'")) - || (startswith(s, "\"") && endswith(s, "\""))) { - memmove(s, s + 1, strlen(s)); - s[strlen(s) - 1] = '\0'; - } - return s; -} - -void ini_free(struct INIFILE **ini) { - for (size_t section = 0; section < (*ini)->section_count; section++) { -#ifdef DEBUG - SYSERROR("freeing section: %s", (*ini)->section[section]->key); -#endif - for (size_t data = 0; data < (*ini)->section[section]->data_count; data++) { - if ((*ini)->section[section]->data[data]) { -#ifdef DEBUG - SYSERROR("freeing data key: %s", (*ini)->section[section]->data[data]->key); -#endif - guard_free((*ini)->section[section]->data[data]->key); -#ifdef DEBUG - SYSERROR("freeing data value: %s", (*ini)->section[section]->data[data]->value); -#endif - guard_free((*ini)->section[section]->data[data]->value); - guard_free((*ini)->section[section]->data[data]); - } - } - guard_free((*ini)->section[section]->data); - guard_free((*ini)->section[section]->key); - guard_free((*ini)->section[section]); - } - guard_free((*ini)->section); - guard_free((*ini)); -} - -struct INIFILE *ini_open(const char *filename) { - FILE *fp; - char line[STASIS_BUFSIZ] = {0}; - char current_section[STASIS_BUFSIZ] = {0}; - char reading_value = 0; - - struct INIFILE *ini = ini_init(); - if (ini == NULL) { - return NULL; - } - - ini_section_init(&ini); - - // Create an implicit section. [default] does not need to be present in the INI config - ini_section_create(&ini, "default"); - strcpy(current_section, "default"); - - // Open the configuration file for reading - fp = fopen(filename, "r"); - if (!fp) { - ini_free(&ini); - ini = NULL; - return NULL; - } - - unsigned hint = 0; - int multiline_data = 0; - int no_data = 0; - char inikey[2][255]; - char *key = inikey[0]; - char *key_last = inikey[1]; - char value[STASIS_BUFSIZ]; - - memset(value, 0, sizeof(value)); - memset(inikey, 0, sizeof(inikey)); - - // Read file - for (size_t i = 0; fgets(line, sizeof(line), fp) != NULL; i++) { - if (no_data && multiline_data) { - if (!isempty(line)) { - no_data = 0; - } else { - multiline_data = 0; - } - memset(value, 0, sizeof(value)); - } else { - memset(key, 0, sizeof(inikey[0])); - } - // Find pointer to first comment character - char *comment = strpbrk(line, ";#"); - if (comment) { - if (!reading_value || line - comment == 0) { - // Remove comment from line (standalone and inline comments) - if (!((comment - line > 0 && (*(comment - 1) == '\\')) || (*comment - 1) == '#')) { - *comment = '\0'; - } else { - // Handle escaped comment characters. Remove the escape character '\' - memmove(comment - 1, comment, strlen(comment)); - if (strlen(comment)) { - comment[strlen(comment) - 1] = '\0'; - } else { - comment[0] = '\0'; - } - } - } - } - - // Test for section header: [string] - if (startswith(line, "[")) { - // The previous key is irrelevant now - memset(key_last, 0, sizeof(inikey[1])); - - char *section_name = substring_between(line, "[]"); - if (!section_name) { - fprintf(stderr, "error: invalid section syntax, line %zu: '%s'\n", i + 1, line); - return NULL; - } - - // Ignore default section because we already have an implicit one - if (!strncmp(section_name, "default", strlen("default"))) { - guard_free(section_name); - continue; - } - - // Create new named section - strip(section_name); - ini_section_create(&ini, section_name); - - // Record the name of the section. This is used until another section is found. - memset(current_section, 0, sizeof(current_section)); - strcpy(current_section, section_name); - guard_free(section_name); - memset(line, 0, sizeof(line)); - continue; - } - - // no data, skip - if (!reading_value && isempty(line)) { - continue; - } - - char *operator = strchr(line, '='); - - // a value continuation line - if (multiline_data && (startswith(line, " ") || startswith(line, "\t"))) { - operator = NULL; - } - - if (operator) { - size_t key_len = operator - line; - memset(key, 0, sizeof(inikey[0])); - strncpy(key, line, key_len); - lstrip(key); - strip(key); - memset(key_last, 0, sizeof(inikey[1])); - strcpy(key_last, key); - reading_value = 1; - if (strlen(operator) > 1) { - strcpy(value, &operator[1]); - } else { - strcpy(value, ""); - } - if (isempty(value)) { - //printf("%s is probably long raw data\n", key); - hint = INIVAL_TYPE_STR_ARRAY; - multiline_data = 1; - no_data = 1; - } else { - //printf("%s is probably short data\n", key); - hint = INIVAL_TYPE_STR; - multiline_data = 0; - } - strip(value); - } else { - strcpy(key, key_last); - strcpy(value, line); - } - memset(line, 0, sizeof(line)); - - // Store key value pair in section's data array - if (strlen(key)) { - lstrip(key); - strip(key); - unquote(value); - if (!multiline_data) { - reading_value = 0; - ini_data_append(&ini, current_section, key, value, hint); - continue; - } - ini_data_append(&ini, current_section, key, value, hint); - reading_value = 1; - } - } - fclose(fp); - - return ini; -} \ No newline at end of file diff --git a/src/junitxml.c b/src/junitxml.c deleted file mode 100644 index c7d0834..0000000 --- a/src/junitxml.c +++ /dev/null @@ -1,240 +0,0 @@ -#include -#include -#include "strlist.h" -#include "junitxml.h" - -static void testcase_result_state_free(struct JUNIT_Testcase **testcase) { - struct JUNIT_Testcase *tc = (*testcase); - if (tc->tc_result_state_type == JUNIT_RESULT_STATE_FAILURE) { - guard_free(tc->result_state.failure->message); - guard_free(tc->result_state.failure); - } else if (tc->tc_result_state_type == JUNIT_RESULT_STATE_SKIPPED) { - guard_free(tc->result_state.skipped->message); - guard_free(tc->result_state.skipped); - } -} - -static void testcase_free(struct JUNIT_Testcase **testcase) { - struct JUNIT_Testcase *tc = (*testcase); - guard_free(tc->name); - guard_free(tc->message); - guard_free(tc->classname); - testcase_result_state_free(&tc); - guard_free(tc); -} - -void junitxml_testsuite_free(struct JUNIT_Testsuite **testsuite) { - struct JUNIT_Testsuite *suite = (*testsuite); - guard_free(suite->name); - guard_free(suite->hostname); - guard_free(suite->timestamp); - for (size_t i = 0; i < suite->_tc_alloc; i++) { - testcase_free(&suite->testcase[i]); - } - guard_free(suite); -} - -static int testsuite_append_testcase(struct JUNIT_Testsuite **testsuite, struct JUNIT_Testcase *testcase) { - struct JUNIT_Testsuite *suite = (*testsuite); - struct JUNIT_Testcase **tmp = realloc(suite->testcase, (suite->_tc_alloc + 1 ) * sizeof(*testcase)); - if (tmp == NULL) { - return -1; - } else { - suite->testcase = tmp; - } - suite->testcase[suite->_tc_inuse] = testcase; - suite->_tc_inuse++; - suite->_tc_alloc++; - return 0; -} - -static struct JUNIT_Failure *testcase_failure_from_attributes(struct StrList *attrs) { - struct JUNIT_Failure *result; - - result = calloc(1, sizeof(*result)); - if(!result) { - return NULL; - } - for (size_t x = 0; x < strlist_count(attrs); x += 2) { - char *attr_name = strlist_item(attrs, x); - char *attr_value = strlist_item(attrs, x + 1); - if (!strcmp(attr_name, "message")) { - result->message = strdup(attr_value); - } - } - return result; -} - -static struct JUNIT_Error *testcase_error_from_attributes(struct StrList *attrs) { - struct JUNIT_Error *result; - - result = calloc(1, sizeof(*result)); - if(!result) { - return NULL; - } - for (size_t x = 0; x < strlist_count(attrs); x += 2) { - char *attr_name = strlist_item(attrs, x); - char *attr_value = strlist_item(attrs, x + 1); - if (!strcmp(attr_name, "message")) { - result->message = strdup(attr_value); - } - } - return result; -} - -static struct JUNIT_Skipped *testcase_skipped_from_attributes(struct StrList *attrs) { - struct JUNIT_Skipped *result; - - result = calloc(1, sizeof(*result)); - if(!result) { - return NULL; - } - for (size_t x = 0; x < strlist_count(attrs); x += 2) { - char *attr_name = strlist_item(attrs, x); - char *attr_value = strlist_item(attrs, x + 1); - if (!strcmp(attr_name, "message")) { - result->message = strdup(attr_value); - } - } - return result; -} - -static struct JUNIT_Testcase *testcase_from_attributes(struct StrList *attrs) { - struct JUNIT_Testcase *result; - - result = calloc(1, sizeof(*result)); - if(!result) { - return NULL; - } - for (size_t x = 0; x < strlist_count(attrs); x += 2) { - char *attr_name = strlist_item(attrs, x); - char *attr_value = strlist_item(attrs, x + 1); - if (!strcmp(attr_name, "name")) { - result->name = strdup(attr_value); - } else if (!strcmp(attr_name, "classname")) { - result->classname = strdup(attr_value); - } else if (!strcmp(attr_name, "time")) { - result->time = strtof(attr_value, NULL); - } else if (!strcmp(attr_name, "message")) { - result->message = strdup(attr_value); - } - } - return result; -} - -static struct StrList *attributes_to_strlist(xmlTextReaderPtr reader) { - struct StrList *list; - xmlNodePtr node = xmlTextReaderCurrentNode(reader); - if (!node) { - return NULL; - } - - list = strlist_init(); - if (xmlTextReaderNodeType(reader) == 1 && node->properties) { - xmlAttr *attr = node->properties; - while (attr && attr->name && attr->children) { - char *attr_name = (char *) attr->name; - char *attr_value = (char *) xmlNodeListGetString(node->doc, attr->children, 1); - strlist_append(&list, attr_name ? attr_name : ""); - strlist_append(&list, attr_value ? attr_value : ""); - xmlFree((xmlChar *) attr_value); - attr = attr->next; - } - } - return list; -} - -static int read_xml_data(xmlTextReaderPtr reader, struct JUNIT_Testsuite **testsuite) { - const xmlChar *name; - //const xmlChar *value; - - name = xmlTextReaderConstName(reader); - if (!name) { - // name could not be converted to string - name = BAD_CAST "--"; - } - //value = xmlTextReaderConstValue(reader); - const char *node_name = (char *) name; - //const char *node_value = (char *) value; - - struct StrList *attrs = attributes_to_strlist(reader); - if (attrs && strlist_count(attrs)) { - if (!strcmp(node_name, "testsuite")) { - for (size_t x = 0; x < strlist_count(attrs); x += 2) { - char *attr_name = strlist_item(attrs, x); - char *attr_value = strlist_item(attrs, x + 1); - if (!strcmp(attr_name, "name")) { - (*testsuite)->name = strdup(attr_value); - } else if (!strcmp(attr_name, "errors")) { - (*testsuite)->errors = (int) strtol(attr_value, NULL, 10); - } else if (!strcmp(attr_name, "failures")) { - (*testsuite)->failures = (int) strtol(attr_value, NULL, 0); - } else if (!strcmp(attr_name, "skipped")) { - (*testsuite)->skipped = (int) strtol(attr_value, NULL, 0); - } else if (!strcmp(attr_name, "tests")) { - (*testsuite)->tests = (int) strtol(attr_value, NULL, 0); - } else if (!strcmp(attr_name, "time")) { - (*testsuite)->time = strtof(attr_value, NULL); - } else if (!strcmp(attr_name, "timestamp")) { - (*testsuite)->timestamp = strdup(attr_value); - } else if (!strcmp(attr_name, "hostname")) { - (*testsuite)->hostname = strdup(attr_value); - } - } - } else if (!strcmp(node_name, "testcase")) { - struct JUNIT_Testcase *testcase = testcase_from_attributes(attrs); - testsuite_append_testcase(testsuite, testcase); - } else if (!strcmp(node_name, "failure")) { - size_t cur_tc = (*testsuite)->_tc_inuse > 0 ? (*testsuite)->_tc_inuse - 1 : (*testsuite)->_tc_inuse; - struct JUNIT_Failure *failure = testcase_failure_from_attributes(attrs); - (*testsuite)->testcase[cur_tc]->tc_result_state_type = JUNIT_RESULT_STATE_FAILURE; - (*testsuite)->testcase[cur_tc]->result_state.failure = failure; - } else if (!strcmp(node_name, "error")) { - size_t cur_tc = (*testsuite)->_tc_inuse > 0 ? (*testsuite)->_tc_inuse - 1 : (*testsuite)->_tc_inuse; - struct JUNIT_Error *error = testcase_error_from_attributes(attrs); - (*testsuite)->testcase[cur_tc]->tc_result_state_type = JUNIT_RESULT_STATE_ERROR; - (*testsuite)->testcase[cur_tc]->result_state.error = error; - } else if (!strcmp(node_name, "skipped")) { - size_t cur_tc = (*testsuite)->_tc_inuse > 0 ? (*testsuite)->_tc_inuse - 1 : (*testsuite)->_tc_inuse; - struct JUNIT_Skipped *skipped = testcase_skipped_from_attributes(attrs); - (*testsuite)->testcase[cur_tc]->tc_result_state_type = JUNIT_RESULT_STATE_SKIPPED; - (*testsuite)->testcase[cur_tc]->result_state.skipped = skipped; - } - } - guard_strlist_free(&attrs); - return 0; -} - -static int read_xml_file(const char *filename, struct JUNIT_Testsuite **testsuite) { - xmlTextReaderPtr reader; - int result; - - reader = xmlReaderForFile(filename, NULL, 0); - if (!reader) { - return -1; - } - - result = xmlTextReaderRead(reader); - while (result == 1) { - read_xml_data(reader, testsuite); - result = xmlTextReaderRead(reader); - } - - xmlFreeTextReader(reader); - return 0; -} - -struct JUNIT_Testsuite *junitxml_testsuite_read(const char *filename) { - struct JUNIT_Testsuite *result; - - if (access(filename, F_OK)) { - return NULL; - } - - result = calloc(1, sizeof(*result)); - if (!result) { - return NULL; - } - read_xml_file(filename, &result); - return result; -} \ No newline at end of file diff --git a/src/lib/CMakeLists.txt b/src/lib/CMakeLists.txt new file mode 100644 index 0000000..82bfe4a --- /dev/null +++ b/src/lib/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(core) \ No newline at end of file diff --git a/src/lib/core/CMakeLists.txt b/src/lib/core/CMakeLists.txt new file mode 100644 index 0000000..c569187 --- /dev/null +++ b/src/lib/core/CMakeLists.txt @@ -0,0 +1,38 @@ +include_directories(${PROJECT_BINARY_DIR}) + +add_library(stasis_core STATIC + globals.c + str.c + strlist.c + ini.c + conda.c + environment.c + utils.c + system.c + download.c + delivery_postprocess.c + delivery_conda.c + delivery_docker.c + delivery_install.c + delivery_artifactory.c + delivery_test.c + delivery_build.c + delivery_show.c + delivery_populate.c + delivery_init.c + delivery.c + recipe.c + relocation.c + wheel.c + copy.c + artifactory.c + template.c + rules.c + docker.c + junitxml.c + github.c + template_func_proto.c + envctl.c + multiprocessing.c +) + diff --git a/src/lib/core/artifactory.c b/src/lib/core/artifactory.c new file mode 100644 index 0000000..6b9635d --- /dev/null +++ b/src/lib/core/artifactory.c @@ -0,0 +1,496 @@ +#include "artifactory.h" + +extern struct STASIS_GLOBAL globals; + +int artifactory_download_cli(char *dest, + char *jfrog_artifactory_base_url, + char *jfrog_artifactory_product, + char *cli_major_ver, + char *version, + char *os, + char *arch, + char *remote_filename) { + char url[PATH_MAX] = {0}; + char path[PATH_MAX] = {0}; + char os_ident[STASIS_NAME_MAX] = {0}; + char arch_ident[STASIS_NAME_MAX] = {0}; + + // convert platform string to lower-case + strcpy(os_ident, os); + tolower_s(os_ident); + + // translate OS identifier + if (!strcmp(os_ident, "darwin") || startswith(os_ident, "macos")) { + strcpy(os_ident, "mac"); + } else if (!strcmp(os_ident, "linux")) { + strcpy(os_ident, "linux"); + } else { + fprintf(stderr, "%s: unknown operating system: %s\n", __FUNCTION__, os_ident); + return -1; + } + + // translate ARCH identifier + strcpy(arch_ident, arch); + if (startswith(arch_ident, "i") && endswith(arch_ident, "86")) { + strcpy(arch_ident, "386"); + } else if (!strcmp(arch_ident, "amd64") || !strcmp(arch_ident, "x86_64") || !strcmp(arch_ident, "x64")) { + if (!strcmp(os_ident, "mac")) { + strcpy(arch_ident, "386"); + } else { + strcpy(arch_ident, "amd64"); + } + } else if (!strcmp(arch_ident, "arm64") || !strcmp(arch_ident, "aarch64")) { + strcpy(arch_ident, "arm64"); + } else { + fprintf(stderr, "%s: unknown architecture: %s\n", __FUNCTION__, arch_ident); + return -1; + } + + snprintf(url, sizeof(url) - 1, "%s/%s/%s/%s/%s-%s-%s/%s", + jfrog_artifactory_base_url, // https://releases.jfrog.io/artifactory + jfrog_artifactory_product, // jfrog-cli + cli_major_ver, // v\d+(-jf)? + version, // 1.2.3 + jfrog_artifactory_product, // ... + os_ident, // ... + arch_ident, // jfrog-cli-linux-x86_64 + remote_filename); // jf + strcpy(path, dest); + + if (mkdirs(path, 0755)) { + fprintf(stderr, "%s: %s: %s", __FUNCTION__, path, strerror(errno)); + return -1; + } + + sprintf(path + strlen(path), "/%s", remote_filename); + long fetch_status = download(url, path, NULL); + if (HTTP_ERROR(fetch_status) || fetch_status < 0) { + fprintf(stderr, "%s: download failed: %s\n", __FUNCTION__, url); + return -1; + } + chmod(path, 0755); + return 0; +} + +void jfrt_register_opt_str(char *jfrt_val, const char *opt_name, struct StrList **opt_map) { + char data[STASIS_BUFSIZ]; + memset(data, 0, sizeof(data)); + + if (jfrt_val == NULL) { + // no data + return; + } + snprintf(data, sizeof(data) - 1, "--%s=\"%s\"", opt_name, jfrt_val); + strlist_append(&*opt_map, data); +} + +void jfrt_register_opt_bool(bool jfrt_val, const char *opt_name, struct StrList **opt_map) { + char data[STASIS_BUFSIZ]; + memset(data, 0, sizeof(data)); + + if (jfrt_val == false) { + // option will not be used + return; + } + snprintf(data, sizeof(data) - 1, "--%s", opt_name); + strlist_append(&*opt_map, data); +} + +void jfrt_register_opt_int(int jfrt_val, const char *opt_name, struct StrList **opt_map) { + char data[STASIS_BUFSIZ]; + memset(data, 0, sizeof(data)); + + if (jfrt_val == 0) { + // option will not be used + return; + } + snprintf(data, sizeof(data) - 1, "--%s=%d", opt_name, jfrt_val); + strlist_append(&*opt_map, data); +} + +void jfrt_register_opt_long(long jfrt_val, const char *opt_name, struct StrList **opt_map) { + char data[STASIS_BUFSIZ]; + memset(data, 0, sizeof(data)); + + if (jfrt_val == 0) { + // option will not be used + return; + } + snprintf(data, sizeof(data) - 1, "--%s=%ld", opt_name, jfrt_val); + strlist_append(&*opt_map, data); +} + +void jfrt_upload_init(struct JFRT_Upload *ctx) { + memset(ctx, 0, sizeof(*ctx)); + ctx->recursive = true; + ctx->threads = 3; + ctx->retries = 3; +} + +static int auth_required(const char *cmd) { + const char *modes[] = { + "build-collect-env", + NULL, + }; + for (size_t i = 0; modes[i] != NULL; i++) { + if (!startswith(cmd, modes[i])) { + return 1; + } + } + return 0; +} + +int jfrt_auth_init(struct JFRT_Auth *auth_ctx) { + char *url = getenv("STASIS_JF_ARTIFACTORY_URL"); + char *user = getenv("STASIS_JF_USER"); + char *access_token = getenv("STASIS_JF_ACCESS_TOKEN"); + char *password = getenv("STASIS_JF_PASSWORD"); + char *ssh_key_path = getenv("STASIS_JF_SSH_KEY_PATH"); + char *ssh_passphrase = getenv("STASIS_JF_SSH_PASSPHRASE"); + char *client_cert_key_path = getenv("STASIS_JF_CLIENT_CERT_KEY_PATH"); + char *client_cert_path = getenv("STASIS_JF_CLIENT_CERT_PATH"); + + if (!url) { + fprintf(stderr, "Artifactory URL is not configured:\n"); + fprintf(stderr, "please set STASIS_JF_ARTIFACTORY_URL\n"); + return -1; + } + auth_ctx->url = url; + + if (access_token) { + auth_ctx->user = NULL; + auth_ctx->access_token = access_token; + auth_ctx->password = NULL; + auth_ctx->ssh_key_path = NULL; + } else if (user && password) { + auth_ctx->user = user; + auth_ctx->password = password; + auth_ctx->access_token = NULL; + auth_ctx->ssh_key_path = NULL; + } else if (ssh_key_path) { + auth_ctx->user = NULL; + auth_ctx->ssh_key_path = ssh_key_path; + if (ssh_passphrase) { + auth_ctx->ssh_passphrase = ssh_passphrase; + } + auth_ctx->password = NULL; + auth_ctx->access_token = NULL; + } else if (client_cert_key_path && client_cert_path) { + auth_ctx->user = NULL; + auth_ctx->password = NULL; + auth_ctx->access_token = NULL; + auth_ctx->ssh_key_path = NULL; + auth_ctx->client_cert_key_path = client_cert_key_path; + auth_ctx->client_cert_path = client_cert_path; + } else { + fprintf(stderr, "Artifactory authentication is not configured:\n"); + fprintf(stderr, "set STASIS_JF_USER and STASIS_JF_PASSWORD\n"); + fprintf(stderr, "or, set STASIS_JF_ACCESS_TOKEN\n"); + fprintf(stderr, "or, set STASIS_JF_SSH_KEY_PATH and STASIS_JF_SSH_KEY_PASSPHRASE\n"); + fprintf(stderr, "or, set STASIS_JF_CLIENT_CERT_KEY_PATH and STASIS_JF_CLIENT_CERT_PATH\n"); + return -1; + } + return 0; +} + +int jfrog_cli(struct JFRT_Auth *auth, const char *subsystem, const char *task, char *args) { + struct Process proc; + char cmd[STASIS_BUFSIZ]; + char cmd_redacted[STASIS_BUFSIZ]; + int status; + + memset(&proc, 0, sizeof(proc)); + memset(cmd, 0, sizeof(cmd)); + memset(cmd_redacted, 0, sizeof(cmd_redacted)); + + struct StrList *arg_map = strlist_init(); + if (!arg_map) { + return -1; + } + + char *auth_args = NULL; + if (auth_required(task)) { + // String options + jfrt_register_opt_str(auth->url, "url", &arg_map); + jfrt_register_opt_str(auth->user, "user", &arg_map); + jfrt_register_opt_str(auth->access_token, "access-token", &arg_map); + jfrt_register_opt_str(auth->password, "password", &arg_map); + jfrt_register_opt_str(auth->ssh_key_path, "ssh-key-path", &arg_map); + jfrt_register_opt_str(auth->ssh_passphrase, "ssh-passphrase", &arg_map); + jfrt_register_opt_str(auth->client_cert_key_path, "client-cert-key-path", &arg_map); + jfrt_register_opt_str(auth->client_cert_path, "client-cert-path", &arg_map); + jfrt_register_opt_bool(auth->insecure_tls, "insecure-tls", &arg_map); + jfrt_register_opt_str(auth->server_id, "server-id", &arg_map); + } + + auth_args = join(arg_map->data, " "); + if (!auth_args) { + return -1; + } + + const char *redactable[] = { + auth->access_token, + auth->ssh_key_path, + auth->ssh_passphrase, + auth->client_cert_key_path, + auth->client_cert_path, + auth->password, + }; + snprintf(cmd, sizeof(cmd) - 1, "jf %s %s %s %s", subsystem, task, auth_args, args ? args : ""); + redact_sensitive(redactable, sizeof(redactable) / sizeof (*redactable), cmd, cmd_redacted, sizeof(cmd_redacted) - 1); + + guard_free(auth_args); + guard_strlist_free(&arg_map); + + // Pings are noisy. Squelch them. + if (task && !strstr(task, "ping")) { + msg(STASIS_MSG_L2, "Executing: %s\n", cmd_redacted); + } + + if (!globals.verbose) { + strcpy(proc.f_stdout, "/dev/null"); + strcpy(proc.f_stderr, "/dev/null"); + } + status = shell(&proc, cmd); + return status; +} + +static int jfrog_cli_rt(struct JFRT_Auth *auth, char *task, char *args) { + return jfrog_cli(auth, "rt", task, args); +} + +int jfrog_cli_rt_build_collect_env(struct JFRT_Auth *auth, char *build_name, char *build_number) { + char cmd[STASIS_BUFSIZ]; + memset(cmd, 0, sizeof(cmd)); + snprintf(cmd, sizeof(cmd) - 1, "\"%s\" \"%s\"", build_name, build_number); + return jfrog_cli(auth, "rt", "build-collect-env", cmd); +} + +int jfrog_cli_rt_build_publish(struct JFRT_Auth *auth, char *build_name, char *build_number) { + char cmd[STASIS_BUFSIZ]; + memset(cmd, 0, sizeof(cmd)); + snprintf(cmd, sizeof(cmd) - 1, "\"%s\" \"%s\"", build_name, build_number); + return jfrog_cli(auth, "rt", "build-publish", cmd); +} + +int jfrog_cli_rt_ping(struct JFRT_Auth *auth) { + return jfrog_cli_rt(auth, "ping", NULL); +} + +int jfrog_cli_rt_download(struct JFRT_Auth *auth, struct JFRT_Download *ctx, char *repo_path, char *dest) { + char cmd[STASIS_BUFSIZ]; + memset(cmd, 0, sizeof(cmd)); + + if (isempty(repo_path)) { + fprintf(stderr, "repo_path argument must be a valid artifactory repository path\n"); + return -1; + } + + // dest is an optional argument, therefore may be NULL or an empty string + + struct StrList *arg_map = strlist_init(); + if (!arg_map) { + return -1; + } + + jfrt_register_opt_str(ctx->archive_entries, "archive-entries", &arg_map); + jfrt_register_opt_str(ctx->build, "build", &arg_map); + jfrt_register_opt_str(ctx->build_name, "build-name", &arg_map); + jfrt_register_opt_str(ctx->build_number, "build-number", &arg_map); + jfrt_register_opt_str(ctx->bundle, "bundle", &arg_map); + jfrt_register_opt_str(ctx->exclude_artifacts, "exclude-artifacts", &arg_map); + jfrt_register_opt_str(ctx->exclude_props, "exclude-props", &arg_map); + jfrt_register_opt_str(ctx->exclusions, "exclusions", &arg_map); + jfrt_register_opt_str(ctx->gpg_key, "gpg-key", &arg_map); + jfrt_register_opt_str(ctx->include_deps, "include-deps", &arg_map); + jfrt_register_opt_str(ctx->include_dirs, "include-dirs", &arg_map); + jfrt_register_opt_str(ctx->module, "module", &arg_map); + jfrt_register_opt_str(ctx->project, "project", &arg_map); + jfrt_register_opt_str(ctx->props, "props", &arg_map); + jfrt_register_opt_str(ctx->sort_by, "sort-by", &arg_map); + jfrt_register_opt_str(ctx->sort_order, "sort-order", &arg_map); + jfrt_register_opt_str(ctx->spec, "spec", &arg_map); + jfrt_register_opt_str(ctx->spec_vars, "spec-vars", &arg_map); + + jfrt_register_opt_bool(ctx->detailed_summary, "detailed-summary", &arg_map); + jfrt_register_opt_bool(ctx->dry_run, "dry-run", &arg_map); + jfrt_register_opt_bool(ctx->explode, "explode", &arg_map); + jfrt_register_opt_bool(ctx->fail_no_op, "fail-no-op", &arg_map); + jfrt_register_opt_bool(ctx->flat, "flat", &arg_map); + jfrt_register_opt_bool(ctx->quiet, "quiet", &arg_map); + jfrt_register_opt_bool(ctx->recursive, "recursive", &arg_map); + jfrt_register_opt_bool(ctx->retries, "retries", &arg_map); + jfrt_register_opt_bool(ctx->retry_wait_time, "retry-wait-time", &arg_map); + jfrt_register_opt_bool(ctx->skip_checksum, "skip-checksum", &arg_map); + + jfrt_register_opt_int(ctx->limit, "limit", &arg_map); + jfrt_register_opt_int(ctx->min_split, "min-split", &arg_map); + jfrt_register_opt_int(ctx->offset, "offset", &arg_map); + jfrt_register_opt_int(ctx->split_count, "split-count", &arg_map); + jfrt_register_opt_int(ctx->sync_deletes, "sync-deletes", &arg_map); + jfrt_register_opt_int(ctx->threads, "threads", &arg_map); + jfrt_register_opt_int(ctx->validate_symlinks, "validate-symlinks", &arg_map); + + char *args = join(arg_map->data, " "); + if (!args) { + return -1; + } + + snprintf(cmd, sizeof(cmd) - 1, "%s '%s' %s", args, repo_path, dest ? dest : ""); + guard_free(args); + guard_strlist_free(&arg_map); + + int status = jfrog_cli_rt(auth, "download", cmd); + return status; +} + +int jfrog_cli_rt_upload(struct JFRT_Auth *auth, struct JFRT_Upload *ctx, char *src, char *repo_path) { + char cmd[STASIS_BUFSIZ]; + memset(cmd, 0, sizeof(cmd)); + + if (isempty(src)) { + fprintf(stderr, "src argument must be a valid file system path\n"); + return -1; + } + + if (isempty(repo_path)) { + fprintf(stderr, "repo_path argument must be a valid artifactory repository path\n"); + return -1; + } + + struct StrList *arg_map = strlist_init(); + if (!arg_map) { + return -1; + } + + // String options + jfrt_register_opt_str(ctx->build_name, "build-name", &arg_map); + jfrt_register_opt_str(ctx->build_number, "build-number", &arg_map); + jfrt_register_opt_str(ctx->exclusions, "exclusions", &arg_map); + jfrt_register_opt_str(ctx->module, "module", &arg_map); + jfrt_register_opt_str(ctx->spec, "spec", &arg_map); + jfrt_register_opt_str(ctx->spec_vars, "spec-vars", &arg_map); + jfrt_register_opt_str(ctx->project, "project", &arg_map); + jfrt_register_opt_str(ctx->target_props, "target-props", &arg_map); + + // Boolean options + jfrt_register_opt_bool(ctx->quiet, "quiet", &arg_map); + jfrt_register_opt_bool(ctx->ant, "ant", &arg_map); + jfrt_register_opt_bool(ctx->archive, "archive", &arg_map); + jfrt_register_opt_bool(ctx->deb, "deb", &arg_map); + jfrt_register_opt_bool(ctx->detailed_summary, "detailed-summary", &arg_map); + jfrt_register_opt_bool(ctx->dry_run, "dry-run", &arg_map); + jfrt_register_opt_bool(ctx->explode, "explode", &arg_map); + jfrt_register_opt_bool(ctx->fail_no_op, "fail-no-op", &arg_map); + jfrt_register_opt_bool(ctx->flat, "flat", &arg_map); + jfrt_register_opt_bool(ctx->include_dirs, "include-dirs", &arg_map); + jfrt_register_opt_bool(ctx->recursive, "recursive", &arg_map); + jfrt_register_opt_bool(ctx->symlinks, "symlinks", &arg_map); + jfrt_register_opt_bool(ctx->sync_deletes, "sync-deletes", &arg_map); + jfrt_register_opt_bool(ctx->regexp, "regexp", &arg_map); + + // Integer options + jfrt_register_opt_int(ctx->retries, "retries", &arg_map); + jfrt_register_opt_int(ctx->retry_wait_time, "retry-wait-time", &arg_map); + jfrt_register_opt_int(ctx->threads, "threads", &arg_map); + + char *args = join(arg_map->data, " "); + if (!args) { + return -1; + } + + char *new_src = NULL; + char *base = NULL; + if (ctx->workaround_parent_only) { + struct StrList *components = strlist_init(); + + strlist_append_tokenize(components, src, "/"); + int max_components = (int) strlist_count(components); + for (int i = 0; i < max_components; i++) { + if (strstr(components->data[i], "*")) { + max_components = i; + break; + } + } + base = join(&components->data[max_components], "/"); + guard_free(components->data[max_components]); + new_src = join(components->data, "/"); + guard_strlist_free(&components); + } + + if (new_src) { + if (base) { + src = base; + } else { + strcat(src, "/"); + } + pushd(new_src); + } + + snprintf(cmd, sizeof(cmd) - 1, "%s '%s' \"%s\"", args, src, repo_path); + guard_free(args); + guard_strlist_free(&arg_map); + + int status = jfrog_cli_rt(auth, "upload", cmd); + if (new_src) { + popd(); + guard_free(new_src); + } + if (base) { + guard_free(base); + } + + return status; +} + +int jfrog_cli_rt_search(struct JFRT_Auth *auth, struct JFRT_Search *ctx, char *repo_path, char *pattern) { + char cmd[STASIS_BUFSIZ]; + memset(cmd, 0, sizeof(cmd)); + + if (isempty(repo_path)) { + fprintf(stderr, "repo_path argument must be a valid artifactory repository path\n"); + return -1; + } + + struct StrList *arg_map = strlist_init(); + if (!arg_map) { + return -1; + } + + jfrt_register_opt_str(ctx->archive_entries, "archive-entries", &arg_map); + jfrt_register_opt_str(ctx->build, "build", &arg_map); + jfrt_register_opt_str(ctx->bundle, "bundle", &arg_map); + jfrt_register_opt_str(ctx->exclusions, "exclusions", &arg_map); + jfrt_register_opt_str(ctx->exclude_patterns, "exclude-patterns", &arg_map); + jfrt_register_opt_str(ctx->exclude_artifacts, "exclude-artifacts", &arg_map); + jfrt_register_opt_str(ctx->exclude_props, "exclude-props", &arg_map); + jfrt_register_opt_str(ctx->include, "include", &arg_map); + jfrt_register_opt_str(ctx->include_deps, "include_deps", &arg_map); + jfrt_register_opt_str(ctx->include_dirs, "include_dirs", &arg_map); + jfrt_register_opt_str(ctx->project, "project", &arg_map); + jfrt_register_opt_str(ctx->props, "props", &arg_map); + jfrt_register_opt_str(ctx->sort_by, "sort-by", &arg_map); + jfrt_register_opt_str(ctx->sort_order, "sort-order", &arg_map); + jfrt_register_opt_str(ctx->spec, "spec", &arg_map); + jfrt_register_opt_str(ctx->spec_vars, "spec-vars", &arg_map); + + jfrt_register_opt_bool(ctx->count, "count", &arg_map); + jfrt_register_opt_bool(ctx->fail_no_op, "fail-no-op", &arg_map); + jfrt_register_opt_bool(ctx->recursive, "recursive", &arg_map); + jfrt_register_opt_bool(ctx->transitive, "transitive", &arg_map); + + jfrt_register_opt_int(ctx->limit, "limit", &arg_map); + jfrt_register_opt_int(ctx->offset, "offset", &arg_map); + + char *args = join(arg_map->data, " "); + if (!args) { + return -1; + } + + snprintf(cmd, sizeof(cmd) - 1, "%s '%s/%s'", args, repo_path, pattern ? pattern: ""); + guard_free(args); + guard_strlist_free(&arg_map); + + int status = jfrog_cli_rt(auth, "search", cmd); + return status; +} diff --git a/src/lib/core/conda.c b/src/lib/core/conda.c new file mode 100644 index 0000000..35caf02 --- /dev/null +++ b/src/lib/core/conda.c @@ -0,0 +1,465 @@ +// +// Created by jhunk on 5/14/23. +// + +#include "conda.h" + +int micromamba(struct MicromambaInfo *info, char *command, ...) { + struct utsname sys; + uname(&sys); + + tolower_s(sys.sysname); + if (!strcmp(sys.sysname, "darwin")) { + strcpy(sys.sysname, "osx"); + } + + if (!strcmp(sys.machine, "x86_64")) { + strcpy(sys.machine, "64"); + } + + char url[PATH_MAX]; + sprintf(url, "https://micro.mamba.pm/api/micromamba/%s-%s/latest", sys.sysname, sys.machine); + + char installer_path[PATH_MAX]; + sprintf(installer_path, "%s/latest", getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"); + + if (access(installer_path, F_OK)) { + download(url, installer_path, NULL); + } + + char mmbin[PATH_MAX]; + sprintf(mmbin, "%s/micromamba", info->micromamba_prefix); + + if (access(mmbin, F_OK)) { + char untarcmd[PATH_MAX * 2]; + mkdirs(info->micromamba_prefix, 0755); + sprintf(untarcmd, "tar -xvf %s -C %s --strip-components=1 bin/micromamba 1>/dev/null", installer_path, info->micromamba_prefix); + int untarcmd_status = system(untarcmd); + if (untarcmd_status) { + return -1; + } + } + + char cmd[STASIS_BUFSIZ]; + memset(cmd, 0, sizeof(cmd)); + sprintf(cmd, "%s -r %s -p %s ", mmbin, info->conda_prefix, info->conda_prefix); + va_list args; + va_start(args, command); + vsprintf(cmd + strlen(cmd), command, args); + va_end(args); + + mkdirs(info->conda_prefix, 0755); + + char rcpath[PATH_MAX]; + sprintf(rcpath, "%s/.condarc", info->conda_prefix); + touch(rcpath); + + setenv("CONDARC", rcpath, 1); + setenv("MAMBA_ROOT_PREFIX", info->conda_prefix, 1); + int status = system(cmd); + unsetenv("MAMBA_ROOT_PREFIX"); + + return status; +} + +int python_exec(const char *args) { + char command[PATH_MAX]; + memset(command, 0, sizeof(command)); + snprintf(command, sizeof(command) - 1, "python %s", args); + msg(STASIS_MSG_L3, "Executing: %s\n", command); + return system(command); +} + +int pip_exec(const char *args) { + char command[PATH_MAX]; + memset(command, 0, sizeof(command)); + snprintf(command, sizeof(command) - 1, "python -m pip %s", args); + msg(STASIS_MSG_L3, "Executing: %s\n", command); + return system(command); +} + +int pip_index_provides(const char *index_url, const char *spec) { + char cmd[PATH_MAX] = {0}; + char spec_local[255] = {0}; + + if (isempty((char *) spec)) { + // NULL or zero-length; no package spec means there's nothing to do. + return -1; + } + + // Normalize the local spec string + strncpy(spec_local, spec, sizeof(spec_local) - 1); + tolower_s(spec_local); + lstrip(spec_local); + strip(spec_local); + + char logfile[] = "/tmp/STASIS-package_exists.XXXXXX"; + int logfd = mkstemp(logfile); + if (logfd < 0) { + perror(logfile); + remove(logfile); // fail harmlessly if not present + return -1; + } + + + int status = 0; + struct Process proc; + memset(&proc, 0, sizeof(proc)); + proc.redirect_stderr = 1; + strcpy(proc.f_stdout, logfile); + + // Do an installation in dry-run mode to see if the package exists in the given index. + snprintf(cmd, sizeof(cmd) - 1, "python -m pip install --dry-run --no-deps --index-url=%s %s", index_url, spec_local); + status = shell(&proc, cmd); + + // Print errors only when shell() itself throws one + // If some day we want to see the errors thrown by pip too, use this condition instead: (status != 0) + if (status < 0) { + FILE *fp = fdopen(logfd, "r"); + if (!fp) { + remove(logfile); + return -1; + } else { + char line[BUFSIZ] = {0}; + fflush(stdout); + fflush(stderr); + while (fgets(line, sizeof(line) - 1, fp) != NULL) { + fprintf(stderr, "%s", line); + } + fflush(stderr); + fclose(fp); + } + } + remove(logfile); + return proc.returncode == 0; +} + +int conda_exec(const char *args) { + char command[PATH_MAX]; + const char *mamba_commands[] = { + "build", + "install", + "update", + "create", + "list", + "search", + "run", + "info", + "clean", + "activate", + "deactivate", + NULL + }; + char conda_as[6]; + memset(conda_as, 0, sizeof(conda_as)); + + strcpy(conda_as, "conda"); + for (size_t i = 0; mamba_commands[i] != NULL; i++) { + if (startswith(args, mamba_commands[i])) { + strcpy(conda_as, "mamba"); + break; + } + } + + snprintf(command, sizeof(command) - 1, "%s %s", conda_as, args); + msg(STASIS_MSG_L3, "Executing: %s\n", command); + return system(command); +} + +int conda_activate(const char *root, const char *env_name) { + int fd = -1; + FILE *fp = NULL; + const char *init_script_conda = "/etc/profile.d/conda.sh"; + const char *init_script_mamba = "/etc/profile.d/mamba.sh"; + char path_conda[PATH_MAX] = {0}; + char path_mamba[PATH_MAX] = {0}; + char logfile[PATH_MAX] = {0}; + struct Process proc; + memset(&proc, 0, sizeof(proc)); + + // Where to find conda's init scripts + sprintf(path_conda, "%s%s", root, init_script_conda); + sprintf(path_mamba, "%s%s", root, init_script_mamba); + + // Set the path to our stdout log + // Emulate mktemp()'s behavior. Give us a unique file name, but don't use + // the file handle at all. We'll open it as a FILE stream soon enough. + sprintf(logfile, "%s/%s", globals.tmpdir, "shell_XXXXXX"); + fd = mkstemp(logfile); + if (fd < 0) { + perror(logfile); + return -1; + } + close(fd); + + // Configure our process for output to a log file + strcpy(proc.f_stdout, logfile); + + // Verify conda's init scripts are available + if (access(path_conda, F_OK) < 0) { + perror(path_conda); + remove(logfile); + return -1; + } + + if (access(path_mamba, F_OK) < 0) { + perror(path_mamba); + remove(logfile); + return -1; + } + + // Fully activate conda and record its effect on the runtime environment + char command[PATH_MAX * 3]; + snprintf(command, sizeof(command) - 1, "set -a; source %s; source %s; conda activate %s &>/dev/null; env -0", path_conda, path_mamba, env_name); + int retval = shell(&proc, command); + if (retval) { + // it didn't work; drop out for cleanup + remove(logfile); + return retval; + } + + // Parse the log file: + // 1. Extract the environment keys and values from the sub-shell + // 2. Apply it to STASIS's runtime environment + // 3. Now we're ready to execute conda commands anywhere + fp = fopen(proc.f_stdout, "r"); + if (!fp) { + perror(logfile); + return -1; + } + + while (!feof(fp)) { + char buf[STASIS_BUFSIZ] = {0}; + int ch = 0; + size_t z = 0; + // We are ingesting output from "env -0" and can't use fgets() + // Copy each character into the buffer until we encounter '\0' or EOF + while (z < sizeof(buf) && (ch = (int) fgetc(fp)) != 0) { + if (ch == EOF) { + break; + } + buf[z] = (char) ch; + z++; + } + buf[strlen(buf)] = 0; + + if (!strlen(buf)) { + continue; + } + + char **part = split(buf, "=", 1); + if (!part) { + perror("unable to split environment variable buffer"); + return -1; + } + if (!part[0]) { + msg(STASIS_MSG_WARN | STASIS_MSG_L1, "Invalid environment variable key ignored: '%s'\n", buf); + } else if (!part[1]) { + msg(STASIS_MSG_WARN | STASIS_MSG_L1, "Invalid environment variable value ignored: '%s'\n", buf); + } else { + setenv(part[0], part[1], 1); + } + GENERIC_ARRAY_FREE(part); + } + fclose(fp); + remove(logfile); + return 0; +} + +int conda_check_required() { + int status = 0; + struct StrList *result = NULL; + char cmd[PATH_MAX] = {0}; + const char *conda_minimum_viable_tools[] = { + "boa", + "conda-build", + "conda-verify", + NULL + }; + + // Construct a "conda list" command that searches for all required packages + // using conda's (python's) regex matching + strcat(cmd, "conda list '"); + for (size_t i = 0; conda_minimum_viable_tools[i] != NULL; i++) { + strcat(cmd, "^"); + strcat(cmd, conda_minimum_viable_tools[i]); + if (conda_minimum_viable_tools[i + 1] != NULL) { + strcat(cmd, "|"); + } + } + strcat(cmd, "' | cut -d ' ' -f 1"); + + // Verify all required packages are installed + char *cmd_out = shell_output(cmd, &status); + if (cmd_out) { + size_t found = 0; + result = strlist_init(); + strlist_append_tokenize(result, cmd_out, "\n"); + for (size_t i = 0; i < strlist_count(result); i++) { + char *item = strlist_item(result, i); + if (isempty(item) || startswith(item, "#")) { + continue; + } + + for (size_t x = 0; conda_minimum_viable_tools[x] != NULL; x++) { + if (!strcmp(item, conda_minimum_viable_tools[x])) { + found++; + } + } + } + if (found < (sizeof(conda_minimum_viable_tools) / sizeof(*conda_minimum_viable_tools)) - 1) { + guard_free(cmd_out); + guard_strlist_free(&result); + return 1; + } + guard_free(cmd_out); + guard_strlist_free(&result); + } else { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "The base package requirement check could not be performed\n"); + return 2; + } + return 0; +} + +int conda_setup_headless() { + if (globals.verbose) { + conda_exec("config --system --set quiet false"); + } else { + // Not verbose, so squelch conda's noise + conda_exec("config --system --set quiet true"); + } + + // Configure conda for headless CI + conda_exec("config --system --set auto_update_conda false"); // never update conda automatically + conda_exec("config --system --set notify_outdated_conda false"); // never notify about outdated conda version + conda_exec("config --system --set always_yes true"); // never prompt for input + conda_exec("config --system --set safety_checks disabled"); // speedup + conda_exec("config --system --set rollback_enabled false"); // speedup + conda_exec("config --system --set report_errors false"); // disable data sharing + conda_exec("config --system --set solver libmamba"); // use a real solver + + char cmd[PATH_MAX]; + size_t total = 0; + if (globals.conda_packages && strlist_count(globals.conda_packages)) { + memset(cmd, 0, sizeof(cmd)); + strcpy(cmd, "install "); + + total = strlist_count(globals.conda_packages); + for (size_t i = 0; i < total; i++) { + char *item = strlist_item(globals.conda_packages, i); + if (isempty(item)) { + continue; + } + sprintf(cmd + strlen(cmd), "'%s'", item); + if (i < total - 1) { + strcat(cmd, " "); + } + } + + if (conda_exec(cmd)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Unable to install user-defined base packages (conda)\n"); + return 1; + } + } + + if (globals.pip_packages && strlist_count(globals.pip_packages)) { + memset(cmd, 0, sizeof(cmd)); + strcpy(cmd, "install "); + + total = strlist_count(globals.pip_packages); + for (size_t i = 0; i < total; i++) { + char *item = strlist_item(globals.pip_packages, i); + if (isempty(item)) { + continue; + } + sprintf(cmd + strlen(cmd), "'%s'", item); + if (i < total - 1) { + strcat(cmd, " "); + } + } + + if (pip_exec(cmd)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Unable to install user-defined base packages (pip)\n"); + return 1; + } + } + + if (conda_check_required()) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Your STASIS configuration lacks the bare" + " minimum software required to build conda packages." + " Please fix it.\n"); + return 1; + } + + if (globals.always_update_base_environment) { + if (conda_exec("update --all")) { + fprintf(stderr, "conda update was unsuccessful\n"); + return 1; + } + } + + return 0; +} + +int conda_env_create_from_uri(char *name, char *uri) { + char env_command[PATH_MAX]; + sprintf(env_command, "env create -n %s -f %s", name, uri); + return conda_exec(env_command); +} + +int conda_env_create(char *name, char *python_version, char *packages) { + char env_command[PATH_MAX]; + sprintf(env_command, "create -n %s python=%s %s", name, python_version, packages ? packages : ""); + return conda_exec(env_command); +} + +int conda_env_remove(char *name) { + char env_command[PATH_MAX]; + sprintf(env_command, "env remove -n %s", name); + return conda_exec(env_command); +} + +int conda_env_export(char *name, char *output_dir, char *output_filename) { + char env_command[PATH_MAX]; + sprintf(env_command, "env export -n %s -f %s/%s.yml", name, output_dir, output_filename); + return conda_exec(env_command); +} + +char *conda_get_active_environment() { + const char *name = getenv("CONDA_DEFAULT_ENV"); + if (!name) { + return NULL; + } + + char *result = NULL; + result = strdup(name); + if (!result) { + return NULL; + } + + return result; +} + +int conda_provides(const char *spec) { + struct Process proc; + memset(&proc, 0, sizeof(proc)); + strcpy(proc.f_stdout, "/dev/null"); + strcpy(proc.f_stderr, "/dev/null"); + + // It's worth noting the departure from using conda_exec() here: + // conda_exec() expects the program output to be visible to the user. + // For this operation we only need the exit value. + char cmd[PATH_MAX] = {0}; + snprintf(cmd, sizeof(cmd) - 1, "mamba search --use-index-cache %s", spec); + if (shell(&proc, cmd) < 0) { + fprintf(stderr, "shell: %s", strerror(errno)); + return -1; + } + return proc.returncode == 0; +} + +int conda_index(const char *path) { + char command[PATH_MAX]; + sprintf(command, "index %s", path); + return conda_exec(command); +} diff --git a/src/lib/core/copy.c b/src/lib/core/copy.c new file mode 100644 index 0000000..f69a756 --- /dev/null +++ b/src/lib/core/copy.c @@ -0,0 +1,86 @@ +#include "copy.h" + +int copy2(const char *src, const char *dest, unsigned int op) { + size_t bytes_read; + size_t bytes_written; + char buf[STASIS_BUFSIZ]; + struct stat src_stat, dnamest; + FILE *fp1, *fp2; + + if (lstat(src, &src_stat) < 0) { + perror(src); + return -1; + } + + if (access(dest, F_OK) == 0) { + unlink(dest); + } + + char dname[1024] = {0}; + strcpy(dname, dest); + char *dname_endptr; + + dname_endptr = strrchr(dname, '/'); + if (dname_endptr != NULL) { + *dname_endptr = '\0'; + } + + stat(dname, &dnamest); + if (S_ISLNK(src_stat.st_mode)) { + char lpath[1024] = {0}; + if (readlink(src, lpath, sizeof(lpath)) < 0) { + perror(src); + return -1; + } + if (symlink(lpath, dest) < 0) { + // silent + return -1; + } + } else if (S_ISREG(src_stat.st_mode) && src_stat.st_nlink > 2 && src_stat.st_dev == dnamest.st_dev) { + if (link(src, dest) < 0) { + perror(src); + return -1; + } + } else if (S_ISFIFO(src_stat.st_mode) || S_ISBLK(src_stat.st_mode) || S_ISCHR(src_stat.st_mode) || S_ISSOCK(src_stat.st_mode)) { + if (mknod(dest, src_stat.st_mode, src_stat.st_rdev) < 0) { + perror(src); + return -1; + } + } else if (S_ISREG(src_stat.st_mode)) { + fp1 = fopen(src, "rb"); + if (!fp1) { + perror(src); + return -1; + } + + fp2 = fopen(dest, "w+b"); + if (!fp2) { + perror(dest); + return -1; + } + + bytes_written = 0; + while ((bytes_read = fread(buf, sizeof(char), sizeof(buf), fp1)) != 0) { + bytes_written += fwrite(buf, sizeof(char), bytes_read, fp2); + } + fclose(fp1); + fclose(fp2); + + if (bytes_written != (size_t) src_stat.st_size) { + fprintf(stderr, "%s: SHORT WRITE (expected %zu bytes, but wrote %zu bytes)\n", dest, src_stat.st_size, bytes_written); + return -1; + } + + if (op & CT_OWNER && chown(dest, src_stat.st_uid, src_stat.st_gid) < 0) { + perror(dest); + } + + if (op & CT_PERM && chmod(dest, src_stat.st_mode) < 0) { + perror(dest); + } + } else { + errno = EOPNOTSUPP; + return -1; + } + return 0; +} diff --git a/src/lib/core/delivery.c b/src/lib/core/delivery.c new file mode 100644 index 0000000..e32ed4c --- /dev/null +++ b/src/lib/core/delivery.c @@ -0,0 +1,317 @@ +#include "delivery.h" + +void delivery_free(struct Delivery *ctx) { + guard_free(ctx->system.arch); + GENERIC_ARRAY_FREE(ctx->system.platform); + guard_free(ctx->meta.name); + guard_free(ctx->meta.version); + guard_free(ctx->meta.codename); + guard_free(ctx->meta.mission); + guard_free(ctx->meta.python); + guard_free(ctx->meta.mission); + guard_free(ctx->meta.python_compact); + guard_free(ctx->meta.based_on); + guard_runtime_free(ctx->runtime.environ); + guard_free(ctx->storage.root); + guard_free(ctx->storage.tmpdir); + guard_free(ctx->storage.delivery_dir); + guard_free(ctx->storage.tools_dir); + guard_free(ctx->storage.package_dir); + guard_free(ctx->storage.results_dir); + guard_free(ctx->storage.output_dir); + guard_free(ctx->storage.conda_install_prefix); + guard_free(ctx->storage.conda_artifact_dir); + guard_free(ctx->storage.conda_staging_dir); + guard_free(ctx->storage.conda_staging_url); + guard_free(ctx->storage.wheel_artifact_dir); + guard_free(ctx->storage.wheel_staging_dir); + guard_free(ctx->storage.wheel_staging_url); + guard_free(ctx->storage.build_dir); + guard_free(ctx->storage.build_recipes_dir); + guard_free(ctx->storage.build_sources_dir); + guard_free(ctx->storage.build_testing_dir); + guard_free(ctx->storage.build_docker_dir); + guard_free(ctx->storage.mission_dir); + guard_free(ctx->storage.docker_artifact_dir); + guard_free(ctx->storage.meta_dir); + guard_free(ctx->storage.package_dir); + guard_free(ctx->storage.cfgdump_dir); + guard_free(ctx->info.time_str_epoch); + guard_free(ctx->info.build_name); + guard_free(ctx->info.build_number); + guard_free(ctx->info.release_name); + guard_free(ctx->conda.installer_baseurl); + guard_free(ctx->conda.installer_name); + guard_free(ctx->conda.installer_version); + guard_free(ctx->conda.installer_platform); + guard_free(ctx->conda.installer_arch); + guard_free(ctx->conda.installer_path); + guard_free(ctx->conda.tool_version); + guard_free(ctx->conda.tool_build_version); + guard_strlist_free(&ctx->conda.conda_packages); + guard_strlist_free(&ctx->conda.conda_packages_defer); + guard_strlist_free(&ctx->conda.pip_packages); + guard_strlist_free(&ctx->conda.pip_packages_defer); + guard_strlist_free(&ctx->conda.wheels_packages); + + for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { + guard_free(ctx->tests[i].name); + guard_free(ctx->tests[i].version); + guard_free(ctx->tests[i].repository); + guard_free(ctx->tests[i].repository_info_ref); + guard_free(ctx->tests[i].repository_info_tag); + guard_strlist_free(&ctx->tests[i].repository_remove_tags); + guard_free(ctx->tests[i].script); + guard_free(ctx->tests[i].build_recipe); + // test-specific runtime variables + guard_runtime_free(ctx->tests[i].runtime.environ); + } + + guard_free(ctx->rules.release_fmt); + guard_free(ctx->rules.build_name_fmt); + guard_free(ctx->rules.build_number_fmt); + + guard_free(ctx->deploy.docker.test_script); + guard_free(ctx->deploy.docker.registry); + guard_free(ctx->deploy.docker.image_compression); + guard_strlist_free(&ctx->deploy.docker.tags); + guard_strlist_free(&ctx->deploy.docker.build_args); + + for (size_t i = 0; i < sizeof(ctx->deploy.jfrog) / sizeof(ctx->deploy.jfrog[0]); i++) { + guard_free(ctx->deploy.jfrog[i].repo); + guard_free(ctx->deploy.jfrog[i].dest); + guard_strlist_free(&ctx->deploy.jfrog[i].files); + } + + if (ctx->_stasis_ini_fp.delivery) { + ini_free(&ctx->_stasis_ini_fp.delivery); + } + guard_free(ctx->_stasis_ini_fp.delivery_path); + + if (ctx->_stasis_ini_fp.cfg) { + // optional extras + ini_free(&ctx->_stasis_ini_fp.cfg); + } + guard_free(ctx->_stasis_ini_fp.cfg_path); + + if (ctx->_stasis_ini_fp.mission) { + ini_free(&ctx->_stasis_ini_fp.mission); + } + guard_free(ctx->_stasis_ini_fp.mission_path); +} + +int delivery_format_str(struct Delivery *ctx, char **dest, const char *fmt) { + size_t fmt_len = strlen(fmt); + + if (!*dest) { + *dest = calloc(STASIS_NAME_MAX, sizeof(**dest)); + if (!*dest) { + return -1; + } + } + + for (size_t i = 0; i < fmt_len; i++) { + if (fmt[i] == '%' && strlen(&fmt[i])) { + i++; + switch (fmt[i]) { + case 'n': // name + strcat(*dest, ctx->meta.name); + break; + case 'c': // codename + strcat(*dest, ctx->meta.codename); + break; + case 'm': // mission + strcat(*dest, ctx->meta.mission); + break; + case 'r': // revision + sprintf(*dest + strlen(*dest), "%d", ctx->meta.rc); + break; + case 'R': // "final"-aware revision + if (ctx->meta.final) + strcat(*dest, "final"); + else + sprintf(*dest + strlen(*dest), "%d", ctx->meta.rc); + break; + case 'v': // version + strcat(*dest, ctx->meta.version); + break; + case 'P': // python version + strcat(*dest, ctx->meta.python); + break; + case 'p': // python version major/minor + strcat(*dest, ctx->meta.python_compact); + break; + case 'a': // system architecture name + strcat(*dest, ctx->system.arch); + break; + case 'o': // system platform (OS) name + strcat(*dest, ctx->system.platform[DELIVERY_PLATFORM_RELEASE]); + break; + case 't': // unix epoch + sprintf(*dest + strlen(*dest), "%ld", ctx->info.time_now); + break; + default: // unknown formatter, write as-is + sprintf(*dest + strlen(*dest), "%c%c", fmt[i - 1], fmt[i]); + break; + } + } else { // write non-format text + sprintf(*dest + strlen(*dest), "%c", fmt[i]); + } + } + return 0; +} + +void delivery_defer_packages(struct Delivery *ctx, int type) { + struct StrList *dataptr = NULL; + struct StrList *deferred = NULL; + char *name = NULL; + char cmd[PATH_MAX]; + + memset(cmd, 0, sizeof(cmd)); + + char mode[10]; + if (DEFER_CONDA == type) { + dataptr = ctx->conda.conda_packages; + deferred = ctx->conda.conda_packages_defer; + strcpy(mode, "conda"); + } else if (DEFER_PIP == type) { + dataptr = ctx->conda.pip_packages; + deferred = ctx->conda.pip_packages_defer; + strcpy(mode, "pip"); + } else { + SYSERROR("BUG: type %d does not map to a supported package manager!\n", type); + exit(1); + } + msg(STASIS_MSG_L2, "Filtering %s packages by test definition...\n", mode); + + struct StrList *filtered = NULL; + filtered = strlist_init(); + for (size_t i = 0; i < strlist_count(dataptr); i++) { + int build_for_host = 0; + + name = strlist_item(dataptr, i); + if (!strlen(name) || isblank(*name) || isspace(*name)) { + // no data + continue; + } + + // Compile a list of packages that are *also* to be tested. + char *spec_begin = strpbrk(name, "@~=<>!"); + char *spec_end = spec_begin; + char package_name[255] = {0}; + + if (spec_end) { + // A version is present in the package name. Jump past operator(s). + while (*spec_end != '\0' && !isalnum(*spec_end)) { + spec_end++; + } + strncpy(package_name, name, spec_begin - name); + } else { + strncpy(package_name, name, sizeof(package_name) - 1); + } + + msg(STASIS_MSG_L3, "package '%s': ", package_name); + + // When spec is present in name, set tests->version to the version detected in the name + for (size_t x = 0; x < sizeof(ctx->tests) / sizeof(ctx->tests[0]) && ctx->tests[x].name != NULL; x++) { + struct Test *test = &ctx->tests[x]; + char nametmp[1024] = {0}; + + if (spec_end != NULL && spec_begin != NULL) { + strncpy(nametmp, name, spec_begin - name); + } else { + strcpy(nametmp, name); + } + // Is the [test:NAME] in the package name? + if (!strcmp(nametmp, test->name)) { + // Override test->version when a version is provided by the (pip|conda)_package list item + guard_free(test->version); + if (spec_begin && spec_end) { + test->version = strdup(spec_end); + } else { + // There are too many possible default branches nowadays: master, main, develop, xyz, etc. + // HEAD is a safe bet. + test->version = strdup("HEAD"); + } + + // Is the list item a git+schema:// URL? + if (strstr(nametmp, "git+") && strstr(nametmp, "://")) { + char *xrepo = strstr(nametmp, "+"); + if (xrepo) { + xrepo++; + guard_free(test->repository); + test->repository = strdup(xrepo); + xrepo = NULL; + } + // Extract the name of the package + char *xbasename = path_basename(nametmp); + if (xbasename) { + // Replace the git+schema:// URL with the package name + strlist_set(&dataptr, i, xbasename); + name = strlist_item(dataptr, i); + } + } + + int upstream_exists = 0; + if (DEFER_PIP == type) { + upstream_exists = pip_index_provides(PYPI_INDEX_DEFAULT, name); + } else if (DEFER_CONDA == type) { + upstream_exists = conda_provides(name); + } else { + fprintf(stderr, "\nUnknown package type: %d\n", type); + exit(1); + } + + if (upstream_exists < 0) { + fprintf(stderr, "%s's existence command failed for '%s'\n" + "(This may be due to a network/firewall issue!)\n", mode, name); + exit(1); + } + if (!upstream_exists) { + build_for_host = 1; + } else { + build_for_host = 0; + } + + break; + } + } + + if (build_for_host) { + printf("BUILD FOR HOST\n"); + strlist_append(&deferred, name); + } else { + printf("USE EXTERNAL\n"); + strlist_append(&filtered, name); + } + } + + if (!strlist_count(deferred)) { + msg(STASIS_MSG_WARN | STASIS_MSG_L2, "No %s packages were filtered by test definitions\n", mode); + } else { + if (DEFER_CONDA == type) { + strlist_free(&ctx->conda.conda_packages); + ctx->conda.conda_packages = strlist_copy(filtered); + } else if (DEFER_PIP == type) { + strlist_free(&ctx->conda.pip_packages); + ctx->conda.pip_packages = strlist_copy(filtered); + } + } + if (filtered) { + strlist_free(&filtered); + } +} + +void delivery_gather_tool_versions(struct Delivery *ctx) { + int status = 0; + + // Extract version from tool output + ctx->conda.tool_version = shell_output("conda --version", &status); + if (ctx->conda.tool_version) + strip(ctx->conda.tool_version); + + ctx->conda.tool_build_version = shell_output("conda build --version", &status); + if (ctx->conda.tool_build_version) + strip(ctx->conda.tool_version); +} + diff --git a/src/lib/core/delivery_artifactory.c b/src/lib/core/delivery_artifactory.c new file mode 100644 index 0000000..27f4823 --- /dev/null +++ b/src/lib/core/delivery_artifactory.c @@ -0,0 +1,192 @@ +#include "delivery.h" + +int delivery_init_artifactory(struct Delivery *ctx) { + int status = 0; + char dest[PATH_MAX] = {0}; + char filepath[PATH_MAX] = {0}; + snprintf(dest, sizeof(dest) - 1, "%s/bin", ctx->storage.tools_dir); + snprintf(filepath, sizeof(dest) - 1, "%s/bin/jf", ctx->storage.tools_dir); + + if (!access(filepath, F_OK)) { + // already have it + msg(STASIS_MSG_L3, "Skipped download, %s already exists\n", filepath); + goto delivery_init_artifactory_envsetup; + } + + char *platform = ctx->system.platform[DELIVERY_PLATFORM]; + msg(STASIS_MSG_L3, "Downloading %s for %s %s\n", globals.jfrog.remote_filename, platform, ctx->system.arch); + if ((status = artifactory_download_cli(dest, + globals.jfrog.jfrog_artifactory_base_url, + globals.jfrog.jfrog_artifactory_product, + globals.jfrog.cli_major_ver, + globals.jfrog.version, + platform, + ctx->system.arch, + globals.jfrog.remote_filename))) { + remove(filepath); + } + + delivery_init_artifactory_envsetup: + // CI (ridiculously generic, why?) disables interactive prompts and progress bar output + setenv("CI", "1", 1); + + // JFROG_CLI_HOME_DIR is where .jfrog is stored + char path[PATH_MAX] = {0}; + snprintf(path, sizeof(path) - 1, "%s/.jfrog", ctx->storage.build_dir); + setenv("JFROG_CLI_HOME_DIR", path, 1); + + // JFROG_CLI_TEMP_DIR is where the obvious is stored + setenv("JFROG_CLI_TEMP_DIR", ctx->storage.tmpdir, 1); + return status; +} + +int delivery_artifact_upload(struct Delivery *ctx) { + int status = 0; + + if (jfrt_auth_init(&ctx->deploy.jfrog_auth)) { + fprintf(stderr, "Failed to initialize Artifactory authentication context\n"); + return -1; + } + + for (size_t i = 0; i < sizeof(ctx->deploy.jfrog) / sizeof(*ctx->deploy.jfrog); i++) { + if (!ctx->deploy.jfrog[i].files || !ctx->deploy.jfrog[i].dest) { + break; + } + jfrt_upload_init(&ctx->deploy.jfrog[i].upload_ctx); + + if (!globals.jfrog.repo) { + msg(STASIS_MSG_WARN, "Artifactory repository path is not configured!\n"); + fprintf(stderr, "set STASIS_JF_REPO environment variable...\nOr append to configuration file:\n\n"); + fprintf(stderr, "[deploy:artifactory]\nrepo = example/generic/repo/path\n\n"); + status++; + break; + } else if (!ctx->deploy.jfrog[i].repo) { + ctx->deploy.jfrog[i].repo = strdup(globals.jfrog.repo); + } + + if (!ctx->deploy.jfrog[i].repo || isempty(ctx->deploy.jfrog[i].repo) || !strlen(ctx->deploy.jfrog[i].repo)) { + // Unlikely to trigger if the config parser is working correctly + msg(STASIS_MSG_ERROR, "Artifactory repository path is empty. Cannot continue.\n"); + status++; + break; + } + + ctx->deploy.jfrog[i].upload_ctx.workaround_parent_only = true; + ctx->deploy.jfrog[i].upload_ctx.build_name = ctx->info.build_name; + ctx->deploy.jfrog[i].upload_ctx.build_number = ctx->info.build_number; + + char files[PATH_MAX]; + char dest[PATH_MAX]; // repo + remote dir + + if (jfrog_cli_rt_ping(&ctx->deploy.jfrog_auth)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Unable to contact artifactory server: %s\n", ctx->deploy.jfrog_auth.url); + return -1; + } + + if (strlist_count(ctx->deploy.jfrog[i].files)) { + for (size_t f = 0; f < strlist_count(ctx->deploy.jfrog[i].files); f++) { + memset(dest, 0, sizeof(dest)); + memset(files, 0, sizeof(files)); + snprintf(dest, sizeof(dest) - 1, "%s/%s", ctx->deploy.jfrog[i].repo, ctx->deploy.jfrog[i].dest); + snprintf(files, sizeof(files) - 1, "%s", strlist_item(ctx->deploy.jfrog[i].files, f)); + status += jfrog_cli_rt_upload(&ctx->deploy.jfrog_auth, &ctx->deploy.jfrog[i].upload_ctx, files, dest); + } + } + } + + if (globals.enable_artifactory_build_info) { + if (!status && ctx->deploy.jfrog[0].files && ctx->deploy.jfrog[0].dest) { + jfrog_cli_rt_build_collect_env(&ctx->deploy.jfrog_auth, ctx->deploy.jfrog[0].upload_ctx.build_name, + ctx->deploy.jfrog[0].upload_ctx.build_number); + jfrog_cli_rt_build_publish(&ctx->deploy.jfrog_auth, ctx->deploy.jfrog[0].upload_ctx.build_name, + ctx->deploy.jfrog[0].upload_ctx.build_number); + } + } else { + msg(STASIS_MSG_WARN | STASIS_MSG_L2, "Artifactory build info upload is disabled by CLI argument\n"); + } + + return status; +} + +int delivery_mission_render_files(struct Delivery *ctx) { + if (!ctx->storage.mission_dir) { + fprintf(stderr, "Mission directory is not configured. Context not initialized?\n"); + return -1; + } + struct Data { + char *src; + char *dest; + } data; + struct INIFILE *cfg = ctx->_stasis_ini_fp.mission; + union INIVal val; + + memset(&data, 0, sizeof(data)); + data.src = calloc(PATH_MAX, sizeof(*data.src)); + if (!data.src) { + perror("data.src"); + return -1; + } + + for (size_t i = 0; i < cfg->section_count; i++) { + char *section_name = cfg->section[i]->key; + if (!startswith(section_name, "template:")) { + continue; + } + val.as_char_p = strchr(section_name, ':') + 1; + if (val.as_char_p && isempty(val.as_char_p)) { + guard_free(data.src); + return 1; + } + sprintf(data.src, "%s/%s/%s", ctx->storage.mission_dir, ctx->meta.mission, val.as_char_p); + msg(STASIS_MSG_L2, "%s\n", data.src); + + int err = 0; + data.dest = ini_getval_str(cfg, section_name, "destination", INI_READ_RENDER, &err); + + char *contents; + struct stat st; + if (lstat(data.src, &st)) { + perror(data.src); + guard_free(data.dest); + continue; + } + + contents = calloc(st.st_size + 1, sizeof(*contents)); + if (!contents) { + perror("template file contents"); + guard_free(data.dest); + continue; + } + + FILE *fp; + fp = fopen(data.src, "rb"); + if (!fp) { + perror(data.src); + guard_free(contents); + guard_free(data.dest); + continue; + } + + if (fread(contents, st.st_size, sizeof(*contents), fp) < 1) { + perror("while reading template file"); + guard_free(contents); + guard_free(data.dest); + fclose(fp); + continue; + } + fclose(fp); + + msg(STASIS_MSG_L3, "Writing %s\n", data.dest); + if (tpl_render_to_file(contents, data.dest)) { + guard_free(contents); + guard_free(data.dest); + continue; + } + guard_free(contents); + guard_free(data.dest); + } + + guard_free(data.src); + return 0; +} + diff --git a/src/lib/core/delivery_build.c b/src/lib/core/delivery_build.c new file mode 100644 index 0000000..b4d610a --- /dev/null +++ b/src/lib/core/delivery_build.c @@ -0,0 +1,190 @@ +#include "delivery.h" + +int delivery_build_recipes(struct Delivery *ctx) { + for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { + char *recipe_dir = NULL; + if (ctx->tests[i].build_recipe) { // build a conda recipe + int recipe_type; + int status; + if (recipe_clone(ctx->storage.build_recipes_dir, ctx->tests[i].build_recipe, NULL, &recipe_dir)) { + fprintf(stderr, "Encountered an issue while cloning recipe for: %s\n", ctx->tests[i].name); + return -1; + } + if (!recipe_dir) { + fprintf(stderr, "BUG: recipe_clone() succeeded but recipe_dir is NULL: %s\n", strerror(errno)); + return -1; + } + recipe_type = recipe_get_type(recipe_dir); + if(!pushd(recipe_dir)) { + if (RECIPE_TYPE_ASTROCONDA == recipe_type) { + pushd(path_basename(ctx->tests[i].repository)); + } else if (RECIPE_TYPE_CONDA_FORGE == recipe_type) { + pushd("recipe"); + } + + char recipe_version[100]; + char recipe_buildno[100]; + char recipe_git_url[PATH_MAX]; + char recipe_git_rev[PATH_MAX]; + + //sprintf(recipe_version, "{%% set version = GIT_DESCRIBE_TAG ~ \".dev\" ~ GIT_DESCRIBE_NUMBER ~ \"+\" ~ GIT_DESCRIBE_HASH %%}"); + //sprintf(recipe_git_url, " git_url: %s", ctx->tests[i].repository); + //sprintf(recipe_git_rev, " git_rev: %s", ctx->tests[i].version); + // TODO: Conditionally download archives if github.com is the origin. Else, use raw git_* keys ^^^ + sprintf(recipe_version, "{%% set version = \"%s\" %%}", ctx->tests[i].repository_info_tag ? ctx->tests[i].repository_info_tag : ctx->tests[i].version); + sprintf(recipe_git_url, " url: %s/archive/refs/tags/{{ version }}.tar.gz", ctx->tests[i].repository); + strcpy(recipe_git_rev, ""); + sprintf(recipe_buildno, " number: 0"); + + unsigned flags = REPLACE_TRUNCATE_AFTER_MATCH; + //file_replace_text("meta.yaml", "{% set version = ", recipe_version); + if (ctx->meta.final) { // remove this. i.e. statis cannot deploy a release to conda-forge + sprintf(recipe_version, "{%% set version = \"%s\" %%}", ctx->tests[i].version); + // TODO: replace sha256 of tagged archive + // TODO: leave the recipe unchanged otherwise. in theory this should produce the same conda package hash as conda forge. + // For now, remove the sha256 requirement + file_replace_text("meta.yaml", "sha256:", "\n", flags); + } else { + file_replace_text("meta.yaml", "{% set version = ", recipe_version, flags); + file_replace_text("meta.yaml", " url:", recipe_git_url, flags); + //file_replace_text("meta.yaml", "sha256:", recipe_git_rev); + file_replace_text("meta.yaml", " sha256:", "\n", flags); + file_replace_text("meta.yaml", " number:", recipe_buildno, flags); + } + + char command[PATH_MAX]; + if (RECIPE_TYPE_CONDA_FORGE == recipe_type) { + char arch[STASIS_NAME_MAX] = {0}; + char platform[STASIS_NAME_MAX] = {0}; + + strcpy(platform, ctx->system.platform[DELIVERY_PLATFORM]); + if (strstr(platform, "Darwin")) { + memset(platform, 0, sizeof(platform)); + strcpy(platform, "osx"); + } + tolower_s(platform); + if (strstr(ctx->system.arch, "arm64")) { + strcpy(arch, "arm64"); + } else if (strstr(ctx->system.arch, "64")) { + strcpy(arch, "64"); + } else { + strcat(arch, "32"); // blind guess + } + tolower_s(arch); + + sprintf(command, "mambabuild --python=%s -m ../.ci_support/%s_%s_.yaml .", + ctx->meta.python, platform, arch); + } else { + sprintf(command, "mambabuild --python=%s .", ctx->meta.python); + } + status = conda_exec(command); + if (status) { + guard_free(recipe_dir); + return -1; + } + + if (RECIPE_TYPE_GENERIC != recipe_type) { + popd(); + } + popd(); + } else { + fprintf(stderr, "Unable to enter recipe directory %s: %s\n", recipe_dir, strerror(errno)); + guard_free(recipe_dir); + return -1; + } + } + guard_free(recipe_dir); + } + return 0; +} + +int filter_repo_tags(char *repo, struct StrList *patterns) { + int result = 0; + + if (!pushd(repo)) { + int list_status = 0; + char *tags_raw = shell_output("git tag -l", &list_status); + struct StrList *tags = strlist_init(); + strlist_append_tokenize(tags, tags_raw, LINE_SEP); + + for (size_t i = 0; tags && i < strlist_count(tags); i++) { + char *tag = strlist_item(tags, i); + for (size_t p = 0; p < strlist_count(patterns); p++) { + char *pattern = strlist_item(patterns, p); + int match = fnmatch(pattern, tag, 0); + if (!match) { + char cmd[PATH_MAX] = {0}; + sprintf(cmd, "git tag -d %s", tag); + result += system(cmd); + break; + } + } + } + guard_strlist_free(&tags); + guard_free(tags_raw); + popd(); + } else { + result = -1; + } + return result; +} + +struct StrList *delivery_build_wheels(struct Delivery *ctx) { + struct StrList *result = NULL; + struct Process proc; + memset(&proc, 0, sizeof(proc)); + + result = strlist_init(); + if (!result) { + perror("unable to allocate memory for string list"); + return NULL; + } + + for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { + if (!ctx->tests[i].build_recipe && ctx->tests[i].repository) { // build from source + char srcdir[PATH_MAX]; + char wheeldir[PATH_MAX]; + memset(srcdir, 0, sizeof(srcdir)); + memset(wheeldir, 0, sizeof(wheeldir)); + + sprintf(srcdir, "%s/%s", ctx->storage.build_sources_dir, ctx->tests[i].name); + git_clone(&proc, ctx->tests[i].repository, srcdir, ctx->tests[i].version); + + if (ctx->tests[i].repository_remove_tags && strlist_count(ctx->tests[i].repository_remove_tags)) { + filter_repo_tags(srcdir, ctx->tests[i].repository_remove_tags); + } + + if (!pushd(srcdir)) { + char dname[NAME_MAX]; + char outdir[PATH_MAX]; + char cmd[PATH_MAX * 2]; + memset(dname, 0, sizeof(dname)); + memset(outdir, 0, sizeof(outdir)); + memset(cmd, 0, sizeof(outdir)); + + strcpy(dname, ctx->tests[i].name); + tolower_s(dname); + sprintf(outdir, "%s/%s", ctx->storage.wheel_artifact_dir, dname); + if (mkdirs(outdir, 0755)) { + fprintf(stderr, "failed to create output directory: %s\n", outdir); + guard_strlist_free(&result); + return NULL; + } + + sprintf(cmd, "-m build -w -o %s", outdir); + if (python_exec(cmd)) { + fprintf(stderr, "failed to generate wheel package for %s-%s\n", ctx->tests[i].name, ctx->tests[i].version); + guard_strlist_free(&result); + return NULL; + } + popd(); + } else { + fprintf(stderr, "Unable to enter source directory %s: %s\n", srcdir, strerror(errno)); + guard_strlist_free(&result); + return NULL; + } + } + } + return result; +} + diff --git a/src/lib/core/delivery_conda.c b/src/lib/core/delivery_conda.c new file mode 100644 index 0000000..93a06fc --- /dev/null +++ b/src/lib/core/delivery_conda.c @@ -0,0 +1,110 @@ +#include "delivery.h" + +void delivery_get_conda_installer_url(struct Delivery *ctx, char *result) { + if (ctx->conda.installer_version) { + // Use version specified by configuration file + sprintf(result, "%s/%s-%s-%s-%s.sh", ctx->conda.installer_baseurl, + ctx->conda.installer_name, + ctx->conda.installer_version, + ctx->conda.installer_platform, + ctx->conda.installer_arch); + } else { + // Use latest installer + sprintf(result, "%s/%s-%s-%s.sh", ctx->conda.installer_baseurl, + ctx->conda.installer_name, + ctx->conda.installer_platform, + ctx->conda.installer_arch); + } + +} + +int delivery_get_conda_installer(struct Delivery *ctx, char *installer_url) { + char script_path[PATH_MAX]; + char *installer = path_basename(installer_url); + + memset(script_path, 0, sizeof(script_path)); + sprintf(script_path, "%s/%s", ctx->storage.tmpdir, installer); + if (access(script_path, F_OK)) { + // Script doesn't exist + long fetch_status = download(installer_url, script_path, NULL); + if (HTTP_ERROR(fetch_status) || fetch_status < 0) { + // download failed + return -1; + } + } else { + msg(STASIS_MSG_RESTRICT | STASIS_MSG_L3, "Skipped, installer already exists\n", script_path); + } + + ctx->conda.installer_path = strdup(script_path); + if (!ctx->conda.installer_path) { + SYSERROR("Unable to duplicate script_path: '%s'", script_path); + return -1; + } + + return 0; +} + +void delivery_install_conda(char *install_script, char *conda_install_dir) { + struct Process proc; + memset(&proc, 0, sizeof(proc)); + + if (globals.conda_fresh_start) { + if (!access(conda_install_dir, F_OK)) { + // directory exists so remove it + if (rmtree(conda_install_dir)) { + perror("unable to remove previous installation"); + exit(1); + } + + // Proceed with the installation + // -b = batch mode (non-interactive) + char cmd[PATH_MAX] = {0}; + snprintf(cmd, sizeof(cmd) - 1, "%s %s -b -p %s", + find_program("bash"), + install_script, + conda_install_dir); + if (shell_safe(&proc, cmd)) { + fprintf(stderr, "conda installation failed\n"); + exit(1); + } + } else { + // Proceed with the installation + // -b = batch mode (non-interactive) + char cmd[PATH_MAX] = {0}; + snprintf(cmd, sizeof(cmd) - 1, "%s %s -b -p %s", + find_program("bash"), + install_script, + conda_install_dir); + if (shell_safe(&proc, cmd)) { + fprintf(stderr, "conda installation failed\n"); + exit(1); + } + } + } else { + msg(STASIS_MSG_L3, "Conda removal disabled by configuration\n"); + } +} + +void delivery_conda_enable(struct Delivery *ctx, char *conda_install_dir) { + if (conda_activate(conda_install_dir, "base")) { + fprintf(stderr, "conda activation failed\n"); + exit(1); + } + + // Setting the CONDARC environment variable appears to be the only consistent + // way to make sure the file is used. Not setting this variable leads to strange + // behavior, especially if a conda environment is already active when STASIS is loaded. + char rcpath[PATH_MAX]; + sprintf(rcpath, "%s/%s", conda_install_dir, ".condarc"); + setenv("CONDARC", rcpath, 1); + if (runtime_replace(&ctx->runtime.environ, __environ)) { + perror("unable to replace runtime environment after activating conda"); + exit(1); + } + + if (conda_setup_headless()) { + // no COE check. this call must succeed. + exit(1); + } +} + diff --git a/src/lib/core/delivery_docker.c b/src/lib/core/delivery_docker.c new file mode 100644 index 0000000..e1d7f60 --- /dev/null +++ b/src/lib/core/delivery_docker.c @@ -0,0 +1,132 @@ +#include "delivery.h" + +int delivery_docker(struct Delivery *ctx) { + if (!docker_capable(&ctx->deploy.docker.capabilities)) { + return -1; + } + char tag[STASIS_NAME_MAX]; + char args[PATH_MAX]; + int has_registry = ctx->deploy.docker.registry != NULL; + size_t total_tags = strlist_count(ctx->deploy.docker.tags); + size_t total_build_args = strlist_count(ctx->deploy.docker.build_args); + + if (!has_registry) { + msg(STASIS_MSG_WARN | STASIS_MSG_L2, "No docker registry defined. You will need to manually retag the resulting image.\n"); + } + + if (!total_tags) { + char default_tag[PATH_MAX]; + msg(STASIS_MSG_WARN | STASIS_MSG_L2, "No docker tags defined by configuration. Generating default tag(s).\n"); + // generate local tag + memset(default_tag, 0, sizeof(default_tag)); + sprintf(default_tag, "%s:%s-py%s", ctx->meta.name, ctx->info.build_name, ctx->meta.python_compact); + tolower_s(default_tag); + + // Add tag + ctx->deploy.docker.tags = strlist_init(); + strlist_append(&ctx->deploy.docker.tags, default_tag); + + if (has_registry) { + // generate tag for target registry + memset(default_tag, 0, sizeof(default_tag)); + sprintf(default_tag, "%s/%s:%s-py%s", ctx->deploy.docker.registry, ctx->meta.name, ctx->info.build_number, ctx->meta.python_compact); + tolower_s(default_tag); + + // Add tag + strlist_append(&ctx->deploy.docker.tags, default_tag); + } + // regenerate total tag available + total_tags = strlist_count(ctx->deploy.docker.tags); + } + + memset(args, 0, sizeof(args)); + + // Append image tags to command + for (size_t i = 0; i < total_tags; i++) { + char *tag_orig = strlist_item(ctx->deploy.docker.tags, i); + strcpy(tag, tag_orig); + docker_sanitize_tag(tag); + sprintf(args + strlen(args), " -t \"%s\" ", tag); + } + + // Append build arguments to command (i.e. --build-arg "key=value" + for (size_t i = 0; i < total_build_args; i++) { + char *build_arg = strlist_item(ctx->deploy.docker.build_args, i); + if (!build_arg) { + break; + } + sprintf(args + strlen(args), " --build-arg \"%s\" ", build_arg); + } + + // Build the image + char delivery_file[PATH_MAX]; + char dest[PATH_MAX]; + char rsync_cmd[PATH_MAX * 2]; + memset(delivery_file, 0, sizeof(delivery_file)); + memset(dest, 0, sizeof(dest)); + + sprintf(delivery_file, "%s/%s.yml", ctx->storage.delivery_dir, ctx->info.release_name); + if (access(delivery_file, F_OK) < 0) { + fprintf(stderr, "docker build cannot proceed without delivery file: %s\n", delivery_file); + return -1; + } + + sprintf(dest, "%s/%s.yml", ctx->storage.build_docker_dir, ctx->info.release_name); + if (copy2(delivery_file, dest, CT_PERM)) { + fprintf(stderr, "Failed to copy delivery file to %s: %s\n", dest, strerror(errno)); + return -1; + } + + memset(dest, 0, sizeof(dest)); + sprintf(dest, "%s/packages", ctx->storage.build_docker_dir); + + msg(STASIS_MSG_L2, "Copying conda packages\n"); + memset(rsync_cmd, 0, sizeof(rsync_cmd)); + sprintf(rsync_cmd, "rsync -avi --progress '%s' '%s'", ctx->storage.conda_artifact_dir, dest); + if (system(rsync_cmd)) { + fprintf(stderr, "Failed to copy conda artifacts to docker build directory\n"); + return -1; + } + + msg(STASIS_MSG_L2, "Copying wheel packages\n"); + memset(rsync_cmd, 0, sizeof(rsync_cmd)); + sprintf(rsync_cmd, "rsync -avi --progress '%s' '%s'", ctx->storage.wheel_artifact_dir, dest); + if (system(rsync_cmd)) { + fprintf(stderr, "Failed to copy wheel artifactory to docker build directory\n"); + } + + if (docker_build(ctx->storage.build_docker_dir, args, ctx->deploy.docker.capabilities.build)) { + return -1; + } + + // Test the image + // All tags point back to the same image so test the first one we see + // regardless of how many are defined + strcpy(tag, strlist_item(ctx->deploy.docker.tags, 0)); + docker_sanitize_tag(tag); + + msg(STASIS_MSG_L2, "Executing image test script for %s\n", tag); + if (ctx->deploy.docker.test_script) { + if (isempty(ctx->deploy.docker.test_script)) { + msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "Image test script has no content\n"); + } else { + int state; + if ((state = docker_script(tag, ctx->deploy.docker.test_script, 0))) { + msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "Non-zero exit (%d) from test script. %s image archive will not be generated.\n", state >> 8, tag); + // test failed -- don't save the image + return -1; + } + } + } else { + msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "No image test script defined\n"); + } + + // Test successful, save image + if (docker_save(path_basename(tag), ctx->storage.docker_artifact_dir, ctx->deploy.docker.image_compression)) { + // save failed + return -1; + } + + return 0; +} + diff --git a/src/lib/core/delivery_init.c b/src/lib/core/delivery_init.c new file mode 100644 index 0000000..e914f99 --- /dev/null +++ b/src/lib/core/delivery_init.c @@ -0,0 +1,345 @@ +#include "delivery.h" + +int has_mount_flags(const char *mount_point, const unsigned long flags) { + struct statvfs st; + if (statvfs(mount_point, &st)) { + SYSERROR("Unable to determine mount-point flags: %s", strerror(errno)); + return -1; + } + return (st.f_flag & flags) != 0; +} + +int delivery_init_tmpdir(struct Delivery *ctx) { + char *tmpdir = NULL; + char *x = NULL; + int unusable = 0; + errno = 0; + + x = getenv("TMPDIR"); + if (x) { + guard_free(ctx->storage.tmpdir); + tmpdir = strdup(x); + } else { + tmpdir = ctx->storage.tmpdir; + } + + if (!tmpdir) { + // memory error + return -1; + } + + // If the directory doesn't exist, create it + if (access(tmpdir, F_OK) < 0) { + if (mkdirs(tmpdir, 0755) < 0) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Unable to create temporary storage directory: %s (%s)\n", tmpdir, strerror(errno)); + goto l_delivery_init_tmpdir_fatal; + } + } + + // If we can't read, write, or execute, then die + if (access(tmpdir, R_OK | W_OK | X_OK) < 0) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "%s requires at least 0755 permissions.\n"); + goto l_delivery_init_tmpdir_fatal; + } + + struct statvfs st; + if (statvfs(tmpdir, &st) < 0) { + goto l_delivery_init_tmpdir_fatal; + } + +#if defined(STASIS_OS_LINUX) + // If we can't execute programs, or write data to the file system at all, then die + if ((st.f_flag & ST_NOEXEC) != 0) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "%s is mounted with noexec\n", tmpdir); + goto l_delivery_init_tmpdir_fatal; + } +#endif + if ((st.f_flag & ST_RDONLY) != 0) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "%s is mounted read-only\n", tmpdir); + goto l_delivery_init_tmpdir_fatal; + } + + if (!globals.tmpdir) { + globals.tmpdir = strdup(tmpdir); + } + + if (!ctx->storage.tmpdir) { + ctx->storage.tmpdir = strdup(globals.tmpdir); + } + return unusable; + + l_delivery_init_tmpdir_fatal: + unusable = 1; + return unusable; +} + +void delivery_init_dirs_stage2(struct Delivery *ctx) { + path_store(&ctx->storage.build_recipes_dir, PATH_MAX, ctx->storage.build_dir, "recipes"); + path_store(&ctx->storage.build_sources_dir, PATH_MAX, ctx->storage.build_dir, "sources"); + path_store(&ctx->storage.build_testing_dir, PATH_MAX, ctx->storage.build_dir, "testing"); + path_store(&ctx->storage.build_docker_dir, PATH_MAX, ctx->storage.build_dir, "docker"); + + path_store(&ctx->storage.delivery_dir, PATH_MAX, ctx->storage.output_dir, "delivery"); + path_store(&ctx->storage.results_dir, PATH_MAX, ctx->storage.output_dir, "results"); + path_store(&ctx->storage.package_dir, PATH_MAX, ctx->storage.output_dir, "packages"); + path_store(&ctx->storage.cfgdump_dir, PATH_MAX, ctx->storage.output_dir, "config"); + path_store(&ctx->storage.meta_dir, PATH_MAX, ctx->storage.output_dir, "meta"); + + path_store(&ctx->storage.conda_artifact_dir, PATH_MAX, ctx->storage.package_dir, "conda"); + path_store(&ctx->storage.wheel_artifact_dir, PATH_MAX, ctx->storage.package_dir, "wheels"); + path_store(&ctx->storage.docker_artifact_dir, PATH_MAX, ctx->storage.package_dir, "docker"); +} + +void delivery_init_dirs_stage1(struct Delivery *ctx) { + char *rootdir = getenv("STASIS_ROOT"); + if (rootdir) { + if (isempty(rootdir)) { + fprintf(stderr, "STASIS_ROOT is set, but empty. Please assign a file system path to this environment variable.\n"); + exit(1); + } + path_store(&ctx->storage.root, PATH_MAX, rootdir, ctx->info.build_name); + } else { + // use "stasis" in current working directory + path_store(&ctx->storage.root, PATH_MAX, "stasis", ctx->info.build_name); + } + path_store(&ctx->storage.tools_dir, PATH_MAX, ctx->storage.root, "tools"); + path_store(&ctx->storage.tmpdir, PATH_MAX, ctx->storage.root, "tmp"); + if (delivery_init_tmpdir(ctx)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Set $TMPDIR to a location other than %s\n", globals.tmpdir); + if (globals.tmpdir) + guard_free(globals.tmpdir); + exit(1); + } + + path_store(&ctx->storage.build_dir, PATH_MAX, ctx->storage.root, "build"); + path_store(&ctx->storage.output_dir, PATH_MAX, ctx->storage.root, "output"); + + if (!ctx->storage.mission_dir) { + path_store(&ctx->storage.mission_dir, PATH_MAX, globals.sysconfdir, "mission"); + } + + if (access(ctx->storage.mission_dir, F_OK)) { + msg(STASIS_MSG_L1, "%s: %s\n", ctx->storage.mission_dir, strerror(errno)); + exit(1); + } + + // Override installation prefix using global configuration key + if (globals.conda_install_prefix && strlen(globals.conda_install_prefix)) { + // user wants a specific path + globals.conda_fresh_start = false; + /* + if (mkdirs(globals.conda_install_prefix, 0755)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Unable to create directory: %s: %s\n", + strerror(errno), globals.conda_install_prefix); + exit(1); + } + */ + /* + ctx->storage.conda_install_prefix = realpath(globals.conda_install_prefix, NULL); + if (!ctx->storage.conda_install_prefix) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "realpath(): Conda installation prefix reassignment failed\n"); + exit(1); + } + ctx->storage.conda_install_prefix = strdup(globals.conda_install_prefix); + */ + path_store(&ctx->storage.conda_install_prefix, PATH_MAX, globals.conda_install_prefix, "conda"); + } else { + // install conda under the STASIS tree + path_store(&ctx->storage.conda_install_prefix, PATH_MAX, ctx->storage.tools_dir, "conda"); + } +} + +int delivery_init_platform(struct Delivery *ctx) { + msg(STASIS_MSG_L2, "Setting architecture\n"); + char archsuffix[20]; + struct utsname uts; + if (uname(&uts)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "uname() failed: %s\n", strerror(errno)); + return -1; + } + + ctx->system.platform = calloc(DELIVERY_PLATFORM_MAX + 1, sizeof(*ctx->system.platform)); + if (!ctx->system.platform) { + SYSERROR("Unable to allocate %d records for platform array\n", DELIVERY_PLATFORM_MAX); + return -1; + } + for (size_t i = 0; i < DELIVERY_PLATFORM_MAX; i++) { + ctx->system.platform[i] = calloc(DELIVERY_PLATFORM_MAXLEN, sizeof(*ctx->system.platform[0])); + } + + ctx->system.arch = strdup(uts.machine); + if (!ctx->system.arch) { + // memory error + return -1; + } + + if (!strcmp(ctx->system.arch, "x86_64")) { + strcpy(archsuffix, "64"); + } else { + strcpy(archsuffix, ctx->system.arch); + } + + msg(STASIS_MSG_L2, "Setting platform\n"); + strcpy(ctx->system.platform[DELIVERY_PLATFORM], uts.sysname); + if (!strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Darwin")) { + sprintf(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], "osx-%s", archsuffix); + strcpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], "MacOSX"); + strcpy(ctx->system.platform[DELIVERY_PLATFORM_RELEASE], "macos"); + } else if (!strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Linux")) { + sprintf(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], "linux-%s", archsuffix); + strcpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], "Linux"); + strcpy(ctx->system.platform[DELIVERY_PLATFORM_RELEASE], "linux"); + } else { + // Not explicitly supported systems + strcpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], ctx->system.platform[DELIVERY_PLATFORM]); + strcpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], ctx->system.platform[DELIVERY_PLATFORM]); + strcpy(ctx->system.platform[DELIVERY_PLATFORM_RELEASE], ctx->system.platform[DELIVERY_PLATFORM]); + tolower_s(ctx->system.platform[DELIVERY_PLATFORM_RELEASE]); + } + + long cpu_count = get_cpu_count(); + if (!cpu_count) { + fprintf(stderr, "Unable to determine CPU count. Falling back to 1.\n"); + cpu_count = 1; + } + char ncpus[100] = {0}; + sprintf(ncpus, "%ld", cpu_count); + + // Declare some important bits as environment variables + setenv("CPU_COUNT", ncpus, 1); + setenv("STASIS_CPU_COUNT", ncpus, 1); + setenv("STASIS_ARCH", ctx->system.arch, 1); + setenv("STASIS_PLATFORM", ctx->system.platform[DELIVERY_PLATFORM], 1); + setenv("STASIS_CONDA_ARCH", ctx->system.arch, 1); + setenv("STASIS_CONDA_PLATFORM", ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], 1); + setenv("STASIS_CONDA_PLATFORM_SUBDIR", ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], 1); + + // Register template variables + // These were moved out of main() because we can't take the address of system.platform[x] + // _before_ the array has been initialized. + tpl_register("system.arch", &ctx->system.arch); + tpl_register("system.platform", &ctx->system.platform[DELIVERY_PLATFORM_RELEASE]); + + return 0; +} + +int delivery_init(struct Delivery *ctx, int render_mode) { + populate_info(ctx); + populate_delivery_cfg(ctx, INI_READ_RENDER); + + // Set artifactory URL via environment variable if possible + char *jfurl = getenv("STASIS_JF_ARTIFACTORY_URL"); + if (jfurl) { + if (globals.jfrog.url) { + guard_free(globals.jfrog.url); + } + globals.jfrog.url = strdup(jfurl); + } + + // Set artifactory repository via environment if possible + char *jfrepo = getenv("STASIS_JF_REPO"); + if (jfrepo) { + if (globals.jfrog.repo) { + guard_free(globals.jfrog.repo); + } + globals.jfrog.repo = strdup(jfrepo); + } + + // Configure architecture and platform information + delivery_init_platform(ctx); + + // Create STASIS directory structure + delivery_init_dirs_stage1(ctx); + + char config_local[PATH_MAX]; + sprintf(config_local, "%s/%s", ctx->storage.tmpdir, "config"); + setenv("XDG_CONFIG_HOME", config_local, 1); + + char cache_local[PATH_MAX]; + sprintf(cache_local, "%s/%s", ctx->storage.tmpdir, "cache"); + setenv("XDG_CACHE_HOME", ctx->storage.tmpdir, 1); + + // add tools to PATH + char pathvar_tmp[STASIS_BUFSIZ]; + sprintf(pathvar_tmp, "%s/bin:%s", ctx->storage.tools_dir, getenv("PATH")); + setenv("PATH", pathvar_tmp, 1); + + // Prevent git from paginating output + setenv("GIT_PAGER", "", 1); + + populate_delivery_ini(ctx, render_mode); + + if (ctx->deploy.docker.tags) { + for (size_t i = 0; i < strlist_count(ctx->deploy.docker.tags); i++) { + char *item = strlist_item(ctx->deploy.docker.tags, i); + tolower_s(item); + } + } + + if (ctx->deploy.docker.image_compression) { + if (docker_validate_compression_program(ctx->deploy.docker.image_compression)) { + SYSERROR("[deploy:docker].image_compression - invalid command / program is not installed: %s", ctx->deploy.docker.image_compression); + return -1; + } + } + return 0; +} + +int bootstrap_build_info(struct Delivery *ctx) { + struct Delivery local; + memset(&local, 0, sizeof(local)); + local._stasis_ini_fp.cfg = ini_open(ctx->_stasis_ini_fp.cfg_path); + local._stasis_ini_fp.delivery = ini_open(ctx->_stasis_ini_fp.delivery_path); + delivery_init_platform(&local); + populate_delivery_cfg(&local, INI_READ_RENDER); + populate_delivery_ini(&local, INI_READ_RENDER); + populate_info(&local); + ctx->info.build_name = strdup(local.info.build_name); + ctx->info.build_number = strdup(local.info.build_number); + ctx->info.release_name = strdup(local.info.release_name); + ctx->info.time_info = malloc(sizeof(*ctx->info.time_info)); + if (!ctx->info.time_info) { + SYSERROR("Unable to allocate %zu bytes for tm struct: %s", sizeof(*local.info.time_info), strerror(errno)); + return -1; + } + memcpy(ctx->info.time_info, local.info.time_info, sizeof(*local.info.time_info)); + ctx->info.time_now = local.info.time_now; + ctx->info.time_str_epoch = strdup(local.info.time_str_epoch); + delivery_free(&local); + return 0; +} + +int delivery_exists(struct Delivery *ctx) { + int release_exists = 0; + char release_pattern[PATH_MAX] = {0}; + sprintf(release_pattern, "*%s*", ctx->info.release_name); + + if (globals.enable_artifactory) { + if (jfrt_auth_init(&ctx->deploy.jfrog_auth)) { + fprintf(stderr, "Failed to initialize Artifactory authentication context\n"); + return -1; // error + } + + struct JFRT_Search search = {.fail_no_op = true}; + release_exists = jfrog_cli_rt_search(&ctx->deploy.jfrog_auth, &search, globals.jfrog.repo, release_pattern); + if (release_exists != 2) { + if (!globals.enable_overwrite && !release_exists) { + // --fail_no_op returns 2 on failure + // without: it returns an empty list "[]" and exit code 0 + return 1; // found + } + } + } else { + struct StrList *files = listdir(ctx->storage.delivery_dir); + for (size_t i = 0; i < strlist_count(files); i++) { + char *filename = strlist_item(files, i); + release_exists = fnmatch(release_pattern, filename, FNM_PATHNAME); + if (!globals.enable_overwrite && !release_exists) { + guard_strlist_free(&files); + return 1; // found + } + } + guard_strlist_free(&files); + } + return 0; // not found +} diff --git a/src/lib/core/delivery_install.c b/src/lib/core/delivery_install.c new file mode 100644 index 0000000..76c3f4a --- /dev/null +++ b/src/lib/core/delivery_install.c @@ -0,0 +1,224 @@ +#include "delivery.h" + +static struct Test *requirement_from_test(struct Delivery *ctx, const char *name) { + struct Test *result = NULL; + for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { + if (ctx->tests[i].name && !strcmp(name, ctx->tests[i].name)) { + result = &ctx->tests[i]; + break; + } + } + return result; +} + +static char *have_spec_in_config(struct Delivery *ctx, const char *name) { + for (size_t x = 0; x < strlist_count(ctx->conda.pip_packages); x++) { + char *config_spec = strlist_item(ctx->conda.pip_packages, x); + char *op = find_version_spec(config_spec); + char package[255] = {0}; + if (op) { + strncpy(package, config_spec, op - config_spec); + } else { + strncpy(package, config_spec, sizeof(package) - 1); + } + if (strncmp(package, name, strlen(package)) == 0) { + return config_spec; + } + } + return NULL; +} + +int delivery_overlay_packages_from_env(struct Delivery *ctx, const char *env_name) { + char *current_env = conda_get_active_environment(); + int need_restore = current_env && strcmp(env_name, current_env) != 0; + + conda_activate(ctx->storage.conda_install_prefix, env_name); + // Retrieve a listing of python packages installed under "env_name" + int freeze_status = 0; + char *freeze_output = shell_output("python -m pip freeze", &freeze_status); + if (freeze_status) { + guard_free(freeze_output); + guard_free(current_env); + return -1; + } + + if (need_restore) { + // Restore the original conda environment + conda_activate(ctx->storage.conda_install_prefix, current_env); + } + guard_free(current_env); + + struct StrList *frozen_list = strlist_init(); + strlist_append_tokenize(frozen_list, freeze_output, LINE_SEP); + guard_free(freeze_output); + + struct StrList *new_list = strlist_init(); + + // - consume package specs that have no test blocks. + // - these will be third-party packages like numpy, scipy, etc. + // - and they need to be present at the head of the list so they + // get installed first. + for (size_t i = 0; i < strlist_count(ctx->conda.pip_packages); i++) { + char *spec = strlist_item(ctx->conda.pip_packages, i); + char spec_name[255] = {0}; + char *op = find_version_spec(spec); + if (op) { + strncpy(spec_name, spec, op - spec); + } else { + strncpy(spec_name, spec, sizeof(spec_name) - 1); + } + struct Test *test_block = requirement_from_test(ctx, spec_name); + if (!test_block) { + msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "from config without test: %s\n", spec); + strlist_append(&new_list, spec); + } + } + + // now consume packages that have a test block + // if the ini provides a spec, override the environment's version. + // otherwise, use the spec derived from the environment + for (size_t i = 0; i < strlist_count(frozen_list); i++) { + char *frozen_spec = strlist_item(frozen_list, i); + char frozen_name[255] = {0}; + char *op = find_version_spec(frozen_spec); + // we only care about packages with specs here. if something else arrives, ignore it + if (op) { + strncpy(frozen_name, frozen_spec, op - frozen_spec); + } else { + strncpy(frozen_name, frozen_spec, sizeof(frozen_name) - 1); + } + struct Test *test = requirement_from_test(ctx, frozen_name); + if (test && strcmp(test->name, frozen_name) == 0) { + char *config_spec = have_spec_in_config(ctx, frozen_name); + if (config_spec) { + msg(STASIS_MSG_L2, "from config: %s\n", config_spec); + strlist_append(&new_list, config_spec); + } else { + msg(STASIS_MSG_L2, "from environment: %s\n", frozen_spec); + strlist_append(&new_list, frozen_spec); + } + } + } + + // Replace the package manifest as needed + if (strlist_count(new_list)) { + guard_strlist_free(&ctx->conda.pip_packages); + ctx->conda.pip_packages = strlist_copy(new_list); + } + guard_strlist_free(&new_list); + guard_strlist_free(&frozen_list); + return 0; +} + +int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, char *env_name, int type, struct StrList **manifest) { + char cmd[PATH_MAX]; + char pkgs[STASIS_BUFSIZ]; + char *env_current = getenv("CONDA_DEFAULT_ENV"); + + if (env_current) { + // The requested environment is not the current environment + if (strcmp(env_current, env_name) != 0) { + // Activate the requested environment + printf("Activating: %s\n", env_name); + conda_activate(conda_install_dir, env_name); + runtime_replace(&ctx->runtime.environ, __environ); + } + } + + memset(cmd, 0, sizeof(cmd)); + memset(pkgs, 0, sizeof(pkgs)); + strcat(cmd, "install"); + + typedef int (*Runner)(const char *); + Runner runner = NULL; + if (INSTALL_PKG_CONDA & type) { + runner = conda_exec; + } else if (INSTALL_PKG_PIP & type) { + runner = pip_exec; + } + + if (INSTALL_PKG_CONDA_DEFERRED & type) { + strcat(cmd, " --use-local"); + } else if (INSTALL_PKG_PIP_DEFERRED & type) { + // Don't change the baseline package set unless we're working with a + // new build. Release candidates will need to keep packages as stable + // as possible between releases. + if (!ctx->meta.based_on) { + strcat(cmd, " --upgrade"); + } + sprintf(cmd + strlen(cmd), " --extra-index-url 'file://%s'", ctx->storage.wheel_artifact_dir); + } + + for (size_t x = 0; manifest[x] != NULL; x++) { + char *name = NULL; + for (size_t p = 0; p < strlist_count(manifest[x]); p++) { + name = strlist_item(manifest[x], p); + strip(name); + if (!strlen(name)) { + continue; + } + if (INSTALL_PKG_PIP_DEFERRED & type) { + struct Test *info = requirement_from_test(ctx, name); + if (info) { + if (!strcmp(info->version, "HEAD")) { + struct StrList *tag_data = strlist_init(); + if (!tag_data) { + SYSERROR("%s", "Unable to allocate memory for tag data\n"); + return -1; + } + strlist_append_tokenize(tag_data, info->repository_info_tag, "-"); + + struct Wheel *whl = NULL; + char *post_commit = NULL; + char *hash = NULL; + if (strlist_count(tag_data) > 1) { + post_commit = strlist_item(tag_data, 1); + hash = strlist_item(tag_data, 2); + } + + // We can't match on version here (index 0). The wheel's version is not guaranteed to be + // equal to the tag; setuptools_scm auto-increments the value, the user can change it manually, + // etc. + errno = 0; + whl = get_wheel_info(ctx->storage.wheel_artifact_dir, info->name, + (char *[]) {ctx->meta.python_compact, ctx->system.arch, + "none", "any", + post_commit, hash, + NULL}, WHEEL_MATCH_ANY); + if (!whl && errno) { + // error + SYSERROR("Unable to read Python wheel info: %s\n", strerror(errno)); + exit(1); + } else if (!whl) { + // not found + fprintf(stderr, "No wheel packages found that match the description of '%s'", info->name); + } else { + // found + guard_strlist_free(&tag_data); + info->version = strdup(whl->version); + } + wheel_free(&whl); + } + snprintf(cmd + strlen(cmd), + sizeof(cmd) - strlen(cmd) - strlen(info->name) - strlen(info->version) + 5, + " '%s==%s'", info->name, info->version); + } else { + fprintf(stderr, "Deferred package '%s' is not present in the tested package list!\n", name); + return -1; + } + } else { + if (startswith(name, "--") || startswith(name, "-")) { + sprintf(cmd + strlen(cmd), " %s", name); + } else { + sprintf(cmd + strlen(cmd), " '%s'", name); + } + } + } + int status = runner(cmd); + if (status) { + return status; + } + } + return 0; +} + diff --git a/src/lib/core/delivery_populate.c b/src/lib/core/delivery_populate.c new file mode 100644 index 0000000..b37f677 --- /dev/null +++ b/src/lib/core/delivery_populate.c @@ -0,0 +1,348 @@ +#include "delivery.h" + +static void ini_has_key_required(struct INIFILE *ini, const char *section_name, char *key) { + int status = ini_has_key(ini, section_name, key); + if (!status) { + SYSERROR("%s:%s key is required but not defined", section_name, key); + exit(1); + } +} + +static void conv_str(char **x, union INIVal val) { + if (*x) { + guard_free(*x); + } + if (val.as_char_p) { + char *tplop = tpl_render(val.as_char_p); + if (tplop) { + *x = tplop; + } else { + *x = NULL; + } + } else { + *x = NULL; + } +} + + + +int populate_info(struct Delivery *ctx) { + if (!ctx->info.time_str_epoch) { + // Record timestamp used for release + time(&ctx->info.time_now); + ctx->info.time_info = localtime(&ctx->info.time_now); + + ctx->info.time_str_epoch = calloc(STASIS_TIME_STR_MAX, sizeof(*ctx->info.time_str_epoch)); + if (!ctx->info.time_str_epoch) { + msg(STASIS_MSG_ERROR, "Unable to allocate memory for Unix epoch string\n"); + return -1; + } + snprintf(ctx->info.time_str_epoch, STASIS_TIME_STR_MAX - 1, "%li", ctx->info.time_now); + } + return 0; +} + +int populate_delivery_cfg(struct Delivery *ctx, int render_mode) { + struct INIFILE *cfg = ctx->_stasis_ini_fp.cfg; + if (!cfg) { + return -1; + } + int err = 0; + ctx->storage.conda_staging_dir = ini_getval_str(cfg, "default", "conda_staging_dir", render_mode, &err); + ctx->storage.conda_staging_url = ini_getval_str(cfg, "default", "conda_staging_url", render_mode, &err); + ctx->storage.wheel_staging_dir = ini_getval_str(cfg, "default", "wheel_staging_dir", render_mode, &err); + ctx->storage.wheel_staging_url = ini_getval_str(cfg, "default", "wheel_staging_url", render_mode, &err); + globals.conda_fresh_start = ini_getval_bool(cfg, "default", "conda_fresh_start", render_mode, &err); + if (!globals.continue_on_error) { + globals.continue_on_error = ini_getval_bool(cfg, "default", "continue_on_error", render_mode, &err); + } + if (!globals.always_update_base_environment) { + globals.always_update_base_environment = ini_getval_bool(cfg, "default", "always_update_base_environment", render_mode, &err); + } + globals.conda_install_prefix = ini_getval_str(cfg, "default", "conda_install_prefix", render_mode, &err); + globals.conda_packages = ini_getval_strlist(cfg, "default", "conda_packages", LINE_SEP, render_mode, &err); + globals.pip_packages = ini_getval_strlist(cfg, "default", "pip_packages", LINE_SEP, render_mode, &err); + + globals.jfrog.jfrog_artifactory_base_url = ini_getval_str(cfg, "jfrog_cli_download", "url", render_mode, &err); + globals.jfrog.jfrog_artifactory_product = ini_getval_str(cfg, "jfrog_cli_download", "product", render_mode, &err); + globals.jfrog.cli_major_ver = ini_getval_str(cfg, "jfrog_cli_download", "version_series", render_mode, &err); + globals.jfrog.version = ini_getval_str(cfg, "jfrog_cli_download", "version", render_mode, &err); + globals.jfrog.remote_filename = ini_getval_str(cfg, "jfrog_cli_download", "filename", render_mode, &err); + globals.jfrog.url = ini_getval_str(cfg, "deploy:artifactory", "url", render_mode, &err); + globals.jfrog.repo = ini_getval_str(cfg, "deploy:artifactory", "repo", render_mode, &err); + + return 0; +} + +int populate_delivery_ini(struct Delivery *ctx, int render_mode) { + union INIVal val; + struct INIFILE *ini = ctx->_stasis_ini_fp.delivery; + struct INIData *rtdata; + RuntimeEnv *rt; + + validate_delivery_ini(ini); + // Populate runtime variables first they may be interpreted by other + // keys in the configuration + rt = runtime_copy(__environ); + while ((rtdata = ini_getall(ini, "runtime")) != NULL) { + char rec[STASIS_BUFSIZ]; + sprintf(rec, "%s=%s", lstrip(strip(rtdata->key)), lstrip(strip(rtdata->value))); + runtime_set(rt, rtdata->key, rtdata->value); + } + runtime_apply(rt); + ctx->runtime.environ = rt; + + int err = 0; + ctx->meta.mission = ini_getval_str(ini, "meta", "mission", render_mode, &err); + + if (!strcasecmp(ctx->meta.mission, "hst")) { + ctx->meta.codename = ini_getval_str(ini, "meta", "codename", render_mode, &err); + } else { + ctx->meta.codename = NULL; + } + + ctx->meta.version = ini_getval_str(ini, "meta", "version", render_mode, &err); + ctx->meta.name = ini_getval_str(ini, "meta", "name", render_mode, &err); + ctx->meta.rc = ini_getval_int(ini, "meta", "rc", render_mode, &err); + ctx->meta.final = ini_getval_bool(ini, "meta", "final", render_mode, &err); + ctx->meta.based_on = ini_getval_str(ini, "meta", "based_on", render_mode, &err); + + if (!ctx->meta.python) { + ctx->meta.python = ini_getval_str(ini, "meta", "python", render_mode, &err); + guard_free(ctx->meta.python_compact); + ctx->meta.python_compact = to_short_version(ctx->meta.python); + } else { + ini_setval(&ini, INI_SETVAL_REPLACE, "meta", "python", ctx->meta.python); + } + + ctx->conda.installer_name = ini_getval_str(ini, "conda", "installer_name", render_mode, &err); + ctx->conda.installer_version = ini_getval_str(ini, "conda", "installer_version", render_mode, &err); + ctx->conda.installer_platform = ini_getval_str(ini, "conda", "installer_platform", render_mode, &err); + ctx->conda.installer_arch = ini_getval_str(ini, "conda", "installer_arch", render_mode, &err); + ctx->conda.installer_baseurl = ini_getval_str(ini, "conda", "installer_baseurl", render_mode, &err); + ctx->conda.conda_packages = ini_getval_strlist(ini, "conda", "conda_packages", " "LINE_SEP, render_mode, &err); + + if (ctx->conda.conda_packages->data && ctx->conda.conda_packages->data[0] && strpbrk(ctx->conda.conda_packages->data[0], " \t")) { + normalize_space(ctx->conda.conda_packages->data[0]); + replace_text(ctx->conda.conda_packages->data[0], " ", LINE_SEP, 0); + char *pip_packages_replacement = join(ctx->conda.conda_packages->data, LINE_SEP); + ini_setval(&ini, INI_SETVAL_REPLACE, "conda", "conda_packages", pip_packages_replacement); + guard_free(pip_packages_replacement); + guard_strlist_free(&ctx->conda.conda_packages); + ctx->conda.conda_packages = ini_getval_strlist(ini, "conda", "conda_packages", LINE_SEP, render_mode, &err); + } + + for (size_t i = 0; i < strlist_count(ctx->conda.conda_packages); i++) { + char *pkg = strlist_item(ctx->conda.conda_packages, i); + if (strpbrk(pkg, ";#") || isempty(pkg)) { + strlist_remove(ctx->conda.conda_packages, i); + } + } + + ctx->conda.pip_packages = ini_getval_strlist(ini, "conda", "pip_packages", LINE_SEP, render_mode, &err); + if (ctx->conda.pip_packages->data && ctx->conda.pip_packages->data[0] && strpbrk(ctx->conda.pip_packages->data[0], " \t")) { + normalize_space(ctx->conda.pip_packages->data[0]); + replace_text(ctx->conda.pip_packages->data[0], " ", LINE_SEP, 0); + char *pip_packages_replacement = join(ctx->conda.pip_packages->data, LINE_SEP); + ini_setval(&ini, INI_SETVAL_REPLACE, "conda", "pip_packages", pip_packages_replacement); + guard_free(pip_packages_replacement); + guard_strlist_free(&ctx->conda.pip_packages); + ctx->conda.pip_packages = ini_getval_strlist(ini, "conda", "pip_packages", LINE_SEP, render_mode, &err); + } + + for (size_t i = 0; i < strlist_count(ctx->conda.pip_packages); i++) { + char *pkg = strlist_item(ctx->conda.pip_packages, i); + if (strpbrk(pkg, ";#") || isempty(pkg)) { + strlist_remove(ctx->conda.pip_packages, i); + } + } + + // Delivery metadata consumed + populate_mission_ini(&ctx, render_mode); + + if (ctx->info.release_name) { + guard_free(ctx->info.release_name); + guard_free(ctx->info.build_name); + guard_free(ctx->info.build_number); + } + + if (delivery_format_str(ctx, &ctx->info.release_name, ctx->rules.release_fmt)) { + fprintf(stderr, "Failed to generate release name. Format used: %s\n", ctx->rules.release_fmt); + return -1; + } + + if (!ctx->info.build_name) { + delivery_format_str(ctx, &ctx->info.build_name, ctx->rules.build_name_fmt); + } + if (!ctx->info.build_number) { + delivery_format_str(ctx, &ctx->info.build_number, ctx->rules.build_number_fmt); + } + + // Best I can do to make output directories unique. Annoying. + delivery_init_dirs_stage2(ctx); + + if (!ctx->conda.conda_packages_defer) { + ctx->conda.conda_packages_defer = strlist_init(); + } + if (!ctx->conda.pip_packages_defer) { + ctx->conda.pip_packages_defer = strlist_init(); + } + + for (size_t z = 0, i = 0; i < ini->section_count; i++) { + char *section_name = ini->section[i]->key; + if (startswith(section_name, "test:")) { + struct Test *test = &ctx->tests[z]; + val.as_char_p = strchr(ini->section[i]->key, ':') + 1; + if (val.as_char_p && isempty(val.as_char_p)) { + return 1; + } + conv_str(&test->name, val); + + test->version = ini_getval_str(ini, section_name, "version", render_mode, &err); + test->repository = ini_getval_str(ini, section_name, "repository", render_mode, &err); + test->script_setup = ini_getval_str(ini, section_name, "script_setup", INI_READ_RAW, &err); + test->script = ini_getval_str(ini, section_name, "script", INI_READ_RAW, &err); + test->disable = ini_getval_bool(ini, section_name, "disable", render_mode, &err); + test->parallel = ini_getval_bool(ini, section_name, "parallel", render_mode, &err); + if (err) { + test->parallel = true; + } + test->repository_remove_tags = ini_getval_strlist(ini, section_name, "repository_remove_tags", LINE_SEP, render_mode, &err); + test->build_recipe = ini_getval_str(ini, section_name, "build_recipe", render_mode, &err); + test->runtime.environ = ini_getval_strlist(ini, section_name, "runtime", LINE_SEP, render_mode, &err); + z++; + } + } + + for (size_t z = 0, i = 0; i < ini->section_count; i++) { + char *section_name = ini->section[i]->key; + struct Deploy *deploy = &ctx->deploy; + if (startswith(section_name, "deploy:artifactory")) { + struct JFrog *jfrog = &deploy->jfrog[z]; + // Artifactory base configuration + + jfrog->upload_ctx.workaround_parent_only = ini_getval_bool(ini, section_name, "workaround_parent_only", render_mode, &err); + jfrog->upload_ctx.exclusions = ini_getval_str(ini, section_name, "exclusions", render_mode, &err); + jfrog->upload_ctx.explode = ini_getval_bool(ini, section_name, "explode", render_mode, &err); + jfrog->upload_ctx.recursive = ini_getval_bool(ini, section_name, "recursive", render_mode, &err); + jfrog->upload_ctx.retries = ini_getval_int(ini, section_name, "retries", render_mode, &err); + jfrog->upload_ctx.retry_wait_time = ini_getval_int(ini, section_name, "retry_wait_time", render_mode, &err); + jfrog->upload_ctx.detailed_summary = ini_getval_bool(ini, section_name, "detailed_summary", render_mode, &err); + jfrog->upload_ctx.quiet = ini_getval_bool(ini, section_name, "quiet", render_mode, &err); + jfrog->upload_ctx.regexp = ini_getval_bool(ini, section_name, "regexp", render_mode, &err); + jfrog->upload_ctx.spec = ini_getval_str(ini, section_name, "spec", render_mode, &err); + jfrog->upload_ctx.flat = ini_getval_bool(ini, section_name, "flat", render_mode, &err); + jfrog->repo = ini_getval_str(ini, section_name, "repo", render_mode, &err); + jfrog->dest = ini_getval_str(ini, section_name, "dest", render_mode, &err); + jfrog->files = ini_getval_strlist(ini, section_name, "files", LINE_SEP, render_mode, &err); + z++; + } + } + + for (size_t i = 0; i < ini->section_count; i++) { + char *section_name = ini->section[i]->key; + struct Deploy *deploy = &ctx->deploy; + if (startswith(ini->section[i]->key, "deploy:docker")) { + struct Docker *docker = &deploy->docker; + + docker->registry = ini_getval_str(ini, section_name, "registry", render_mode, &err); + docker->image_compression = ini_getval_str(ini, section_name, "image_compression", render_mode, &err); + docker->test_script = ini_getval_str(ini, section_name, "test_script", render_mode, &err); + docker->build_args = ini_getval_strlist(ini, section_name, "build_args", LINE_SEP, render_mode, &err); + docker->tags = ini_getval_strlist(ini, section_name, "tags", LINE_SEP, render_mode, &err); + } + } + return 0; +} + +int populate_mission_ini(struct Delivery **ctx, int render_mode) { + int err = 0; + struct INIFILE *ini; + + if ((*ctx)->_stasis_ini_fp.mission) { + return 0; + } + + // Now populate the rules + char missionfile[PATH_MAX] = {0}; + if (getenv("STASIS_SYSCONFDIR")) { + sprintf(missionfile, "%s/%s/%s/%s.ini", + getenv("STASIS_SYSCONFDIR"), "mission", (*ctx)->meta.mission, (*ctx)->meta.mission); + } else { + sprintf(missionfile, "%s/%s/%s/%s.ini", + globals.sysconfdir, "mission", (*ctx)->meta.mission, (*ctx)->meta.mission); + } + + msg(STASIS_MSG_L2, "Reading mission configuration: %s\n", missionfile); + (*ctx)->_stasis_ini_fp.mission = ini_open(missionfile); + ini = (*ctx)->_stasis_ini_fp.mission; + if (!ini) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read mission configuration: %s, %s\n", missionfile, strerror(errno)); + exit(1); + } + (*ctx)->_stasis_ini_fp.mission_path = strdup(missionfile); + + (*ctx)->rules.release_fmt = ini_getval_str(ini, "meta", "release_fmt", render_mode, &err); + + // Used for setting artifactory build info + (*ctx)->rules.build_name_fmt = ini_getval_str(ini, "meta", "build_name_fmt", render_mode, &err); + + // Used for setting artifactory build info + (*ctx)->rules.build_number_fmt = ini_getval_str(ini, "meta", "build_number_fmt", render_mode, &err); + return 0; +} + +void validate_delivery_ini(struct INIFILE *ini) { + if (!ini) { + SYSERROR("%s", "INIFILE is NULL!"); + exit(1); + } + if (ini_section_search(&ini, INI_SEARCH_EXACT, "meta")) { + ini_has_key_required(ini, "meta", "name"); + ini_has_key_required(ini, "meta", "version"); + ini_has_key_required(ini, "meta", "rc"); + ini_has_key_required(ini, "meta", "mission"); + ini_has_key_required(ini, "meta", "python"); + } else { + SYSERROR("%s", "[meta] configuration section is required"); + exit(1); + } + + if (ini_section_search(&ini, INI_SEARCH_EXACT, "conda")) { + ini_has_key_required(ini, "conda", "installer_name"); + ini_has_key_required(ini, "conda", "installer_version"); + ini_has_key_required(ini, "conda", "installer_platform"); + ini_has_key_required(ini, "conda", "installer_arch"); + } else { + SYSERROR("%s", "[conda] configuration section is required"); + exit(1); + } + + for (size_t i = 0; i < ini->section_count; i++) { + struct INISection *section = ini->section[i]; + if (section && startswith(section->key, "test:")) { + char *name = strstr(section->key, ":"); + if (name && strlen(name) > 1) { + name = &name[1]; + } + //ini_has_key_required(ini, section->key, "version"); + //ini_has_key_required(ini, section->key, "repository"); + if (globals.enable_testing) { + ini_has_key_required(ini, section->key, "script"); + } + } + } + + if (ini_section_search(&ini, INI_SEARCH_EXACT, "deploy:docker")) { + // yeah? + } + + for (size_t i = 0; i < ini->section_count; i++) { + struct INISection *section = ini->section[i]; + if (section && startswith(section->key, "deploy:artifactory")) { + ini_has_key_required(ini, section->key, "files"); + ini_has_key_required(ini, section->key, "dest"); + } + } +} + diff --git a/src/lib/core/delivery_postprocess.c b/src/lib/core/delivery_postprocess.c new file mode 100644 index 0000000..1a902e3 --- /dev/null +++ b/src/lib/core/delivery_postprocess.c @@ -0,0 +1,266 @@ +#include "delivery.h" + + +const char *release_header = "# delivery_name: %s\n" + "# delivery_fmt: %s\n" + "# creation_time: %s\n" + "# conda_ident: %s\n" + "# conda_build_ident: %s\n"; + +char *delivery_get_release_header(struct Delivery *ctx) { + char output[STASIS_BUFSIZ]; + char stamp[100]; + strftime(stamp, sizeof(stamp) - 1, "%c", ctx->info.time_info); + sprintf(output, release_header, + ctx->info.release_name, + ctx->rules.release_fmt, + stamp, + ctx->conda.tool_version, + ctx->conda.tool_build_version); + return strdup(output); +} + +int delivery_dump_metadata(struct Delivery *ctx) { + FILE *fp; + char filename[PATH_MAX]; + sprintf(filename, "%s/meta-%s.stasis", ctx->storage.meta_dir, ctx->info.release_name); + fp = fopen(filename, "w+"); + if (!fp) { + return -1; + } + if (globals.verbose) { + printf("%s\n", filename); + } + fprintf(fp, "name %s\n", ctx->meta.name); + fprintf(fp, "version %s\n", ctx->meta.version); + fprintf(fp, "rc %d\n", ctx->meta.rc); + fprintf(fp, "python %s\n", ctx->meta.python); + fprintf(fp, "python_compact %s\n", ctx->meta.python_compact); + fprintf(fp, "mission %s\n", ctx->meta.mission); + fprintf(fp, "codename %s\n", ctx->meta.codename ? ctx->meta.codename : ""); + fprintf(fp, "platform %s %s %s %s\n", + ctx->system.platform[DELIVERY_PLATFORM], + ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], + ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], + ctx->system.platform[DELIVERY_PLATFORM_RELEASE]); + fprintf(fp, "arch %s\n", ctx->system.arch); + fprintf(fp, "time %s\n", ctx->info.time_str_epoch); + fprintf(fp, "release_fmt %s\n", ctx->rules.release_fmt); + fprintf(fp, "release_name %s\n", ctx->info.release_name); + fprintf(fp, "build_name_fmt %s\n", ctx->rules.build_name_fmt); + fprintf(fp, "build_name %s\n", ctx->info.build_name); + fprintf(fp, "build_number_fmt %s\n", ctx->rules.build_number_fmt); + fprintf(fp, "build_number %s\n", ctx->info.build_number); + fprintf(fp, "conda_installer_baseurl %s\n", ctx->conda.installer_baseurl); + fprintf(fp, "conda_installer_name %s\n", ctx->conda.installer_name); + fprintf(fp, "conda_installer_version %s\n", ctx->conda.installer_version); + fprintf(fp, "conda_installer_platform %s\n", ctx->conda.installer_platform); + fprintf(fp, "conda_installer_arch %s\n", ctx->conda.installer_arch); + + fclose(fp); + return 0; +} + +void delivery_rewrite_spec(struct Delivery *ctx, char *filename, unsigned stage) { + char output[PATH_MAX]; + char *header = NULL; + char *tempfile = NULL; + FILE *tp = NULL; + + if (stage == DELIVERY_REWRITE_SPEC_STAGE_1) { + header = delivery_get_release_header(ctx); + if (!header) { + msg(STASIS_MSG_ERROR, "failed to generate release header string\n", filename); + exit(1); + } + tempfile = xmkstemp(&tp, "w+"); + if (!tempfile || !tp) { + msg(STASIS_MSG_ERROR, "%s: unable to create temporary file\n", strerror(errno)); + exit(1); + } + fprintf(tp, "%s", header); + + // Read the original file + char **contents = file_readlines(filename, 0, 0, NULL); + if (!contents) { + msg(STASIS_MSG_ERROR, "%s: unable to read %s", filename); + exit(1); + } + + // Write temporary data + for (size_t i = 0; contents[i] != NULL; i++) { + if (startswith(contents[i], "channels:")) { + // Allow for additional conda channel injection + if (ctx->conda.conda_packages_defer && strlist_count(ctx->conda.conda_packages_defer)) { + fprintf(tp, "%s - @CONDA_CHANNEL@\n", contents[i]); + continue; + } + } else if (strstr(contents[i], "- pip:")) { + if (ctx->conda.pip_packages_defer && strlist_count(ctx->conda.pip_packages_defer)) { + // Allow for additional pip argument injection + fprintf(tp, "%s - @PIP_ARGUMENTS@\n", contents[i]); + continue; + } + } else if (startswith(contents[i], "prefix:")) { + // Remove the prefix key + if (strstr(contents[i], "/") || strstr(contents[i], "\\")) { + // path is on the same line as the key + continue; + } else { + // path is on the next line? + if (contents[i + 1] && (strstr(contents[i + 1], "/") || strstr(contents[i + 1], "\\"))) { + i++; + } + continue; + } + } + fprintf(tp, "%s", contents[i]); + } + GENERIC_ARRAY_FREE(contents); + guard_free(header); + fflush(tp); + fclose(tp); + + // Replace the original file with our temporary data + if (copy2(tempfile, filename, CT_PERM) < 0) { + fprintf(stderr, "%s: could not rename '%s' to '%s'\n", strerror(errno), tempfile, filename); + exit(1); + } + remove(tempfile); + guard_free(tempfile); + } else if (globals.enable_rewrite_spec_stage_2 && stage == DELIVERY_REWRITE_SPEC_STAGE_2) { + // Replace "local" channel with the staging URL + if (ctx->storage.conda_staging_url) { + file_replace_text(filename, "@CONDA_CHANNEL@", ctx->storage.conda_staging_url, 0); + } else if (globals.jfrog.repo) { + sprintf(output, "%s/%s/%s/%s/packages/conda", globals.jfrog.url, globals.jfrog.repo, ctx->meta.mission, ctx->info.build_name); + file_replace_text(filename, "@CONDA_CHANNEL@", output, 0); + } else { + msg(STASIS_MSG_WARN, "conda_staging_dir is not configured. Using fallback: '%s'\n", ctx->storage.conda_artifact_dir); + file_replace_text(filename, "@CONDA_CHANNEL@", ctx->storage.conda_artifact_dir, 0); + } + + if (ctx->storage.wheel_staging_url) { + file_replace_text(filename, "@PIP_ARGUMENTS@", ctx->storage.wheel_staging_url, 0); + } else if (globals.enable_artifactory && globals.jfrog.url && globals.jfrog.repo) { + sprintf(output, "--extra-index-url %s/%s/%s/%s/packages/wheels", globals.jfrog.url, globals.jfrog.repo, ctx->meta.mission, ctx->info.build_name); + file_replace_text(filename, "@PIP_ARGUMENTS@", output, 0); + } else { + msg(STASIS_MSG_WARN, "wheel_staging_dir is not configured. Using fallback: '%s'\n", ctx->storage.wheel_artifact_dir); + sprintf(output, "--extra-index-url file://%s", ctx->storage.wheel_artifact_dir); + file_replace_text(filename, "@PIP_ARGUMENTS@", output, 0); + } + } +} + +int delivery_copy_conda_artifacts(struct Delivery *ctx) { + char cmd[STASIS_BUFSIZ]; + char conda_build_dir[PATH_MAX]; + char subdir[PATH_MAX]; + memset(cmd, 0, sizeof(cmd)); + memset(conda_build_dir, 0, sizeof(conda_build_dir)); + memset(subdir, 0, sizeof(subdir)); + + sprintf(conda_build_dir, "%s/%s", ctx->storage.conda_install_prefix, "conda-bld"); + // One must run conda build at least once to create the "conda-bld" directory. + // When this directory is missing there can be no build artifacts. + if (access(conda_build_dir, F_OK) < 0) { + msg(STASIS_MSG_RESTRICT | STASIS_MSG_WARN | STASIS_MSG_L3, + "Skipped: 'conda build' has never been executed.\n"); + return 0; + } + + snprintf(cmd, sizeof(cmd) - 1, "rsync -avi --progress %s/%s %s", + conda_build_dir, + ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], + ctx->storage.conda_artifact_dir); + + return system(cmd); +} + +int delivery_index_conda_artifacts(struct Delivery *ctx) { + return conda_index(ctx->storage.conda_artifact_dir); +} + +int delivery_copy_wheel_artifacts(struct Delivery *ctx) { + char cmd[PATH_MAX]; + memset(cmd, 0, sizeof(cmd)); + snprintf(cmd, sizeof(cmd) - 1, "rsync -avi --progress %s/*/dist/*.whl %s", + ctx->storage.build_sources_dir, + ctx->storage.wheel_artifact_dir); + return system(cmd); +} + +int delivery_index_wheel_artifacts(struct Delivery *ctx) { + struct dirent *rec; + DIR *dp; + FILE *top_fp; + + dp = opendir(ctx->storage.wheel_artifact_dir); + if (!dp) { + return -1; + } + + // Generate a "dumb" local pypi index that is compatible with: + // pip install --extra-index-url + char top_index[PATH_MAX]; + memset(top_index, 0, sizeof(top_index)); + sprintf(top_index, "%s/index.html", ctx->storage.wheel_artifact_dir); + top_fp = fopen(top_index, "w+"); + if (!top_fp) { + closedir(dp); + return -2; + } + + while ((rec = readdir(dp)) != NULL) { + // skip directories + if (DT_REG == rec->d_type || !strcmp(rec->d_name, "..") || !strcmp(rec->d_name, ".")) { + continue; + } + + FILE *bottom_fp; + char bottom_index[PATH_MAX * 2]; + memset(bottom_index, 0, sizeof(bottom_index)); + sprintf(bottom_index, "%s/%s/index.html", ctx->storage.wheel_artifact_dir, rec->d_name); + bottom_fp = fopen(bottom_index, "w+"); + if (!bottom_fp) { + closedir(dp); + return -3; + } + + if (globals.verbose) { + printf("+ %s\n", rec->d_name); + } + // Add record to top level index + fprintf(top_fp, "%s
\n", rec->d_name, rec->d_name); + + char dpath[PATH_MAX * 2]; + memset(dpath, 0, sizeof(dpath)); + sprintf(dpath, "%s/%s", ctx->storage.wheel_artifact_dir, rec->d_name); + struct StrList *packages = listdir(dpath); + if (!packages) { + closedir(dp); + fclose(top_fp); + fclose(bottom_fp); + return -4; + } + + for (size_t i = 0; i < strlist_count(packages); i++) { + char *package = strlist_item(packages, i); + if (!endswith(package, ".whl")) { + continue; + } + if (globals.verbose) { + printf("`- %s\n", package); + } + // Write record to bottom level index + fprintf(bottom_fp, "%s
\n", package, package); + } + fclose(bottom_fp); + + guard_strlist_free(&packages); + } + closedir(dp); + fclose(top_fp); + return 0; +} diff --git a/src/lib/core/delivery_show.c b/src/lib/core/delivery_show.c new file mode 100644 index 0000000..adfa1be --- /dev/null +++ b/src/lib/core/delivery_show.c @@ -0,0 +1,117 @@ +#include "delivery.h" + +void delivery_debug_show(struct Delivery *ctx) { + printf("\n====DEBUG====\n"); + printf("%-20s %-10s\n", "System configuration directory:", globals.sysconfdir); + printf("%-20s %-10s\n", "Mission directory:", ctx->storage.mission_dir); + printf("%-20s %-10s\n", "Testing enabled:", globals.enable_testing ? "Yes" : "No"); + printf("%-20s %-10s\n", "Docker image builds enabled:", globals.enable_docker ? "Yes" : "No"); + printf("%-20s %-10s\n", "Artifact uploading enabled:", globals.enable_artifactory ? "Yes" : "No"); +} + +void delivery_meta_show(struct Delivery *ctx) { + if (globals.verbose) { + delivery_debug_show(ctx); + } + + printf("\n====DELIVERY====\n"); + printf("%-20s %-10s\n", "Target Python:", ctx->meta.python); + printf("%-20s %-10s\n", "Name:", ctx->meta.name); + printf("%-20s %-10s\n", "Mission:", ctx->meta.mission); + if (ctx->meta.codename) { + printf("%-20s %-10s\n", "Codename:", ctx->meta.codename); + } + if (ctx->meta.version) { + printf("%-20s %-10s\n", "Version", ctx->meta.version); + } + if (!ctx->meta.final) { + printf("%-20s %-10d\n", "RC Level:", ctx->meta.rc); + } + printf("%-20s %-10s\n", "Final Release:", ctx->meta.final ? "Yes" : "No"); + printf("%-20s %-10s\n", "Based On:", ctx->meta.based_on ? ctx->meta.based_on : "New"); +} + +void delivery_conda_show(struct Delivery *ctx) { + printf("\n====CONDA====\n"); + printf("%-20s %-10s\n", "Prefix:", ctx->storage.conda_install_prefix); + + puts("Native Packages:"); + if (strlist_count(ctx->conda.conda_packages) || strlist_count(ctx->conda.conda_packages_defer)) { + struct StrList *list_conda = strlist_init(); + if (strlist_count(ctx->conda.conda_packages)) { + strlist_append_strlist(list_conda, ctx->conda.conda_packages); + } + if (strlist_count(ctx->conda.conda_packages_defer)) { + strlist_append_strlist(list_conda, ctx->conda.conda_packages_defer); + } + strlist_sort(list_conda, STASIS_SORT_ALPHA); + + for (size_t i = 0; i < strlist_count(list_conda); i++) { + char *token = strlist_item(list_conda, i); + if (isempty(token) || isblank(*token) || startswith(token, "-")) { + continue; + } + printf("%21s%s\n", "", token); + } + guard_strlist_free(&list_conda); + } else { + printf("%21s%s\n", "", "N/A"); + } + + puts("Python Packages:"); + if (strlist_count(ctx->conda.pip_packages) || strlist_count(ctx->conda.pip_packages_defer)) { + struct StrList *list_python = strlist_init(); + if (strlist_count(ctx->conda.pip_packages)) { + strlist_append_strlist(list_python, ctx->conda.pip_packages); + } + if (strlist_count(ctx->conda.pip_packages_defer)) { + strlist_append_strlist(list_python, ctx->conda.pip_packages_defer); + } + strlist_sort(list_python, STASIS_SORT_ALPHA); + + for (size_t i = 0; i < strlist_count(list_python); i++) { + char *token = strlist_item(list_python, i); + if (isempty(token) || isblank(*token) || startswith(token, "-")) { + continue; + } + printf("%21s%s\n", "", token); + } + guard_strlist_free(&list_python); + } else { + printf("%21s%s\n", "", "N/A"); + } +} + +void delivery_tests_show(struct Delivery *ctx) { + printf("\n====TESTS====\n"); + for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { + if (!ctx->tests[i].name) { + continue; + } + printf("%-20s %-20s %s\n", ctx->tests[i].name, + ctx->tests[i].version, + ctx->tests[i].repository); + } +} + +void delivery_runtime_show(struct Delivery *ctx) { + printf("\n====RUNTIME====\n"); + struct StrList *rt = NULL; + rt = strlist_copy(ctx->runtime.environ); + if (!rt) { + // no data + return; + } + strlist_sort(rt, STASIS_SORT_ALPHA); + size_t total = strlist_count(rt); + for (size_t i = 0; i < total; i++) { + char *item = strlist_item(rt, i); + if (!item) { + // not supposed to occur + msg(STASIS_MSG_WARN | STASIS_MSG_L1, "Encountered unexpected NULL at record %zu of %zu of runtime array.\n", i); + return; + } + printf("%s\n", item); + } +} + diff --git a/src/lib/core/delivery_test.c b/src/lib/core/delivery_test.c new file mode 100644 index 0000000..cb78f64 --- /dev/null +++ b/src/lib/core/delivery_test.c @@ -0,0 +1,295 @@ +#include "delivery.h" + +void delivery_tests_run(struct Delivery *ctx) { + static const int SETUP = 0; + static const int PARALLEL = 1; + static const int SERIAL = 2; + struct MultiProcessingPool *pool[3]; + struct Process proc; + memset(&proc, 0, sizeof(proc)); + + if (!globals.workaround.conda_reactivate) { + globals.workaround.conda_reactivate = calloc(PATH_MAX, sizeof(*globals.workaround.conda_reactivate)); + } else { + memset(globals.workaround.conda_reactivate, 0, PATH_MAX); + } + // Test blocks always run with xtrace enabled. Disable, and reenable it. Conda's wrappers produce an incredible + // amount of debug information. + snprintf(globals.workaround.conda_reactivate, PATH_MAX - 1, "\nset +x; mamba activate ${CONDA_DEFAULT_ENV}; set -x\n"); + + if (!ctx->tests[0].name) { + msg(STASIS_MSG_WARN | STASIS_MSG_L2, "no tests are defined!\n"); + } else { + pool[PARALLEL] = mp_pool_init("parallel", ctx->storage.tmpdir); + if (!pool[PARALLEL]) { + perror("mp_pool_init/parallel"); + exit(1); + } + pool[PARALLEL]->status_interval = globals.pool_status_interval; + + pool[SERIAL] = mp_pool_init("serial", ctx->storage.tmpdir); + if (!pool[SERIAL]) { + perror("mp_pool_init/serial"); + exit(1); + } + pool[SERIAL]->status_interval = globals.pool_status_interval; + + pool[SETUP] = mp_pool_init("setup", ctx->storage.tmpdir); + if (!pool[SETUP]) { + perror("mp_pool_init/setup"); + exit(1); + } + pool[SETUP]->status_interval = globals.pool_status_interval; + + // Test block scripts shall exit non-zero on error. + // This will fail a test block immediately if "string" is not found in file.txt: + // grep string file.txt + // + // And this is how to avoid that scenario: + // #1: + // if ! grep string file.txt; then + // # handle error + // fi + // + // #2: + // grep string file.txt || handle error + // + // #3: + // # Use ':' as a NO-OP if/when the result doesn't matter + // grep string file.txt || : + const char *runner_cmd_fmt = "set -e -x\n%s\n"; + + // Iterate over our test records, retrieving the source code for each package, and assigning its scripted tasks + // to the appropriate processing pool + for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { + struct Test *test = &ctx->tests[i]; + if (!test->name && !test->repository && !test->script) { + // skip unused test records + continue; + } + msg(STASIS_MSG_L2, "Loading tests for %s %s\n", test->name, test->version); + if (!test->script || !strlen(test->script)) { + msg(STASIS_MSG_WARN | STASIS_MSG_L3, "Nothing to do. To fix, declare a 'script' in section: [test:%s]\n", + test->name); + continue; + } + + char destdir[PATH_MAX]; + sprintf(destdir, "%s/%s", ctx->storage.build_sources_dir, path_basename(test->repository)); + + if (!access(destdir, F_OK)) { + msg(STASIS_MSG_L3, "Purging repository %s\n", destdir); + if (rmtree(destdir)) { + COE_CHECK_ABORT(1, "Unable to remove repository\n"); + } + } + msg(STASIS_MSG_L3, "Cloning repository %s\n", test->repository); + if (!git_clone(&proc, test->repository, destdir, test->version)) { + test->repository_info_tag = strdup(git_describe(destdir)); + test->repository_info_ref = strdup(git_rev_parse(destdir, "HEAD")); + } else { + COE_CHECK_ABORT(1, "Unable to clone repository\n"); + } + + if (test->repository_remove_tags && strlist_count(test->repository_remove_tags)) { + filter_repo_tags(destdir, test->repository_remove_tags); + } + + if (pushd(destdir)) { + COE_CHECK_ABORT(1, "Unable to enter repository directory\n"); + } else { + char *cmd = calloc(strlen(test->script) + STASIS_BUFSIZ, sizeof(*cmd)); + if (!cmd) { + SYSERROR("Unable to allocate test script buffer: %s", strerror(errno)); + exit(1); + } + + msg(STASIS_MSG_L3, "Queuing task for %s\n", test->name); + memset(&proc, 0, sizeof(proc)); + + strcpy(cmd, test->script); + char *cmd_rendered = tpl_render(cmd); + if (cmd_rendered) { + if (strcmp(cmd_rendered, cmd) != 0) { + strcpy(cmd, cmd_rendered); + cmd[strlen(cmd_rendered) ? strlen(cmd_rendered) - 1 : 0] = 0; + } + guard_free(cmd_rendered); + } else { + SYSERROR("An error occurred while rendering the following:\n%s", cmd); + exit(1); + } + + if (test->disable) { + msg(STASIS_MSG_L2, "Script execution disabled by configuration\n", test->name); + guard_free(cmd); + continue; + } + + char *runner_cmd = NULL; + char pool_name[100] = "parallel"; + struct MultiProcessingTask *task = NULL; + int selected = PARALLEL; + if (!globals.enable_parallel || !test->parallel) { + selected = SERIAL; + memset(pool_name, 0, sizeof(pool_name)); + strcpy(pool_name, "serial"); + } + + if (asprintf(&runner_cmd, runner_cmd_fmt, cmd) < 0) { + SYSERROR("Unable to allocate memory for runner command: %s", strerror(errno)); + exit(1); + } + task = mp_pool_task(pool[selected], test->name, destdir, runner_cmd); + if (!task) { + SYSERROR("Failed to add task to %s pool: %s", pool_name, runner_cmd); + popd(); + if (!globals.continue_on_error) { + guard_free(runner_cmd); + tpl_free(); + delivery_free(ctx); + globals_free(); + } + exit(1); + } + guard_free(runner_cmd); + guard_free(cmd); + popd(); + + } + } + + // Configure "script_setup" tasks + // Directories should exist now, so no need to go through initializing everything all over again. + for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { + struct Test *test = &ctx->tests[i]; + if (test->script_setup) { + char destdir[PATH_MAX]; + sprintf(destdir, "%s/%s", ctx->storage.build_sources_dir, path_basename(test->repository)); + if (access(destdir, F_OK)) { + SYSERROR("%s: %s", destdir, strerror(errno)); + exit(1); + } + if (!pushd(destdir)) { + const size_t cmd_len = strlen(test->script_setup) + STASIS_BUFSIZ; + char *cmd = calloc(cmd_len, sizeof(*cmd)); + if (!cmd) { + SYSERROR("Unable to allocate test script_setup buffer: %s", strerror(errno)); + exit(1); + } + + strncpy(cmd, test->script_setup, cmd_len - 1); + char *cmd_rendered = tpl_render(cmd); + if (cmd_rendered) { + if (strcmp(cmd_rendered, cmd) != 0) { + strncpy(cmd, cmd_rendered, cmd_len - 1); + cmd[strlen(cmd_rendered) ? strlen(cmd_rendered) - 1 : 0] = 0; + } + guard_free(cmd_rendered); + } else { + SYSERROR("An error occurred while rendering the following:\n%s", cmd); + exit(1); + } + + struct MultiProcessingTask *task = NULL; + char *runner_cmd = NULL; + if (asprintf(&runner_cmd, runner_cmd_fmt, cmd) < 0) { + SYSERROR("Unable to allocate memory for runner command: %s", strerror(errno)); + exit(1); + } + + task = mp_pool_task(pool[SETUP], test->name, destdir, runner_cmd); + if (!task) { + SYSERROR("Failed to add task %s to setup pool: %s", test->name, runner_cmd); + popd(); + if (!globals.continue_on_error) { + guard_free(runner_cmd); + tpl_free(); + delivery_free(ctx); + globals_free(); + } + exit(1); + } + guard_free(runner_cmd); + guard_free(cmd); + popd(); + } else { + SYSERROR("Failed to change directory: %s\n", destdir); + exit(1); + } + } + } + + size_t opt_flags = 0; + if (globals.parallel_fail_fast) { + opt_flags |= MP_POOL_FAIL_FAST; + } + + // Execute all queued tasks + for (size_t p = 0; p < sizeof(pool) / sizeof(*pool); p++) { + int pool_status; + long jobs = globals.cpu_limit; + + if (!pool[p]->num_used) { + // Skip empty pool + continue; + } + + // Setup tasks run sequentially + if (p == (size_t) SETUP || p == (size_t) SERIAL) { + jobs = 1; + } + + // Run tasks in the pool + // 1. Setup (builds) + // 2. Parallel (fast jobs) + // 3. Serial (long jobs) + pool_status = mp_pool_join(pool[p], jobs, opt_flags); + + // On error show a summary of the current pool, and die + if (pool_status != 0) { + mp_pool_show_summary(pool[p]); + COE_CHECK_ABORT(true, "Task failure"); + } + } + + // All tasks were successful + for (size_t p = 0; p < sizeof(pool) / sizeof(*pool); p++) { + if (pool[p]->num_used) { + // Only show pools that actually had jobs to run + mp_pool_show_summary(pool[p]); + } + mp_pool_free(&pool[p]); + } + } +} + +int delivery_fixup_test_results(struct Delivery *ctx) { + struct dirent *rec; + DIR *dp; + + dp = opendir(ctx->storage.results_dir); + if (!dp) { + perror(ctx->storage.results_dir); + return -1; + } + + while ((rec = readdir(dp)) != NULL) { + char path[PATH_MAX]; + memset(path, 0, sizeof(path)); + + if (!strcmp(rec->d_name, ".") || !strcmp(rec->d_name, "..") || !endswith(rec->d_name, ".xml")) { + continue; + } + + sprintf(path, "%s/%s", ctx->storage.results_dir, rec->d_name); + msg(STASIS_MSG_L3, "%s\n", rec->d_name); + if (xml_pretty_print_in_place(path, STASIS_XML_PRETTY_PRINT_PROG, STASIS_XML_PRETTY_PRINT_ARGS)) { + msg(STASIS_MSG_L3 | STASIS_MSG_WARN, "Failed to rewrite file '%s'\n", rec->d_name); + } + } + + closedir(dp); + return 0; +} + diff --git a/src/lib/core/docker.c b/src/lib/core/docker.c new file mode 100644 index 0000000..5834ef9 --- /dev/null +++ b/src/lib/core/docker.c @@ -0,0 +1,204 @@ +#include "docker.h" + + +int docker_exec(const char *args, unsigned flags) { + struct Process proc; + char cmd[PATH_MAX]; + + memset(&proc, 0, sizeof(proc)); + memset(cmd, 0, sizeof(cmd)); + snprintf(cmd, sizeof(cmd) - 1, "docker %s", args); + if (flags & STASIS_DOCKER_QUIET) { + strcpy(proc.f_stdout, "/dev/null"); + strcpy(proc.f_stderr, "/dev/null"); + } else { + msg(STASIS_MSG_L2, "Executing: %s\n", cmd); + } + + shell(&proc, cmd); + return proc.returncode; +} + +int docker_script(const char *image, char *data, unsigned flags) { + (void)flags; // TODO: placeholder + FILE *infile; + FILE *outfile; + char cmd[PATH_MAX]; + char buffer[STASIS_BUFSIZ]; + + memset(cmd, 0, sizeof(cmd)); + snprintf(cmd, sizeof(cmd) - 1, "docker run --rm -i %s /bin/sh -", image); + + outfile = popen(cmd, "w"); + if (!outfile) { + // opening command pipe for writing failed + return -1; + } + + infile = fmemopen(data, strlen(data), "r"); + if (!infile) { + // opening memory file for reading failed + return -1; + } + + do { + memset(buffer, 0, sizeof(buffer)); + if (fgets(buffer, sizeof(buffer) - 1, infile) != NULL) { + fputs(buffer, outfile); + } + } while (!feof(infile)); + + fclose(infile); + return pclose(outfile); +} + +int docker_build(const char *dirpath, const char *args, int engine) { + char cmd[PATH_MAX]; + char build[15]; + + memset(build, 0, sizeof(build)); + memset(cmd, 0, sizeof(cmd)); + + if (engine & STASIS_DOCKER_BUILD) { + strcpy(build, "build"); + } + if (engine & STASIS_DOCKER_BUILD_X) { + strcpy(build, "buildx build"); + } + snprintf(cmd, sizeof(cmd) - 1, "%s %s %s", build, args, dirpath); + return docker_exec(cmd, 0); +} + +int docker_save(const char *image, const char *destdir, const char *compression_program) { + char cmd[PATH_MAX]; + + memset(cmd, 0, sizeof(cmd)); + + if (compression_program && strlen(compression_program)) { + char ext[255]; + memset(ext, 0, sizeof(ext)); + if (startswith(compression_program, "zstd")) { + strcpy(ext, "zst"); + } else if (startswith(compression_program, "xz")) { + strcpy(ext, "xz"); + } else if (startswith(compression_program, "gzip")) { + strcpy(ext, "gz"); + } else if (startswith(compression_program, "bzip2")) { + strcpy(ext, "bz2"); + } else { + strncpy(ext, compression_program, sizeof(ext) - 1); + } + sprintf(cmd, "save \"%s\" | %s > \"%s/%s.tar.%s\"", image, compression_program, destdir, image, ext); + } else { + sprintf(cmd, "save \"%s\" -o \"%s/%s.tar\"", image, destdir, image); + + } + return docker_exec(cmd, 0); +} + +static int docker_exists() { + if (find_program("docker")) { + return true; + } + return false; +} + +static char *docker_ident() { + FILE *fp = NULL; + char *tempfile = NULL; + char line[PATH_MAX]; + struct Process proc; + + tempfile = xmkstemp(&fp, "w+"); + if (!fp || !tempfile) { + return NULL; + } + + memset(&proc, 0, sizeof(proc)); + strcpy(proc.f_stdout, tempfile); + strcpy(proc.f_stderr, "/dev/null"); + shell(&proc, "docker --version"); + + if (!freopen(tempfile, "r", fp)) { + remove(tempfile); + guard_free(tempfile); + return NULL; + } + + if (!fgets(line, sizeof(line) - 1, fp)) { + fclose(fp); + remove(tempfile); + guard_free(tempfile); + return NULL; + } + + fclose(fp); + remove(tempfile); + guard_free(tempfile); + + return strdup(line); +} + +int docker_capable(struct DockerCapabilities *result) { + char *version = NULL; + memset(result, 0, sizeof(*result)); + + if (!docker_exists()) { + // docker isn't available + return false; + } + result->available = true; + + if (docker_exec("ps", STASIS_DOCKER_QUIET)) { + // user cannot connect to the socket + return false; + } + + version = docker_ident(); + if (version && startswith(version, "podman")) { + result->podman = true; + } + guard_free(version); + + if (!docker_exec("buildx build --help", STASIS_DOCKER_QUIET)) { + result->build |= STASIS_DOCKER_BUILD_X; + } + if (!docker_exec("build --help", STASIS_DOCKER_QUIET)) { + result->build |= STASIS_DOCKER_BUILD; + } + if (!result->build) { + // can't use docker without a build plugin + return false; + } + result->usable = true; + return true; +} + +void docker_sanitize_tag(char *str) { + char *pos = str; + while (*pos != 0) { + if (!isalnum(*pos)) { + if (*pos != '.' && *pos != ':' && *pos != '/') { + *pos = '-'; + } + } + pos++; + } +} + +int docker_validate_compression_program(char *prog) { + int result = -1; + char **parts = NULL; + if (!prog) { + goto invalid; + } + parts = split(prog, " ", 1); + if (!parts) { + goto invalid; + } + result = find_program(parts[0]) ? 0 : -1; + + invalid: + GENERIC_ARRAY_FREE(parts); + return result; +} diff --git a/src/lib/core/download.c b/src/lib/core/download.c new file mode 100644 index 0000000..bfb323e --- /dev/null +++ b/src/lib/core/download.c @@ -0,0 +1,59 @@ +// +// Created by jhunk on 10/5/23. +// + +#include "download.h" + +size_t download_writer(void *fp, size_t size, size_t nmemb, void *stream) { + size_t bytes = fwrite(fp, size, nmemb, (FILE *) stream); + return bytes; +} + +long download(char *url, const char *filename, char **errmsg) { + extern char *VERSION; + CURL *c; + CURLcode curl_code; + long http_code = -1; + FILE *fp; + char user_agent[20]; + sprintf(user_agent, "stasis/%s", VERSION); + long timeout = 30L; + char *timeout_str = getenv("STASIS_DOWNLOAD_TIMEOUT"); + + curl_global_init(CURL_GLOBAL_ALL); + c = curl_easy_init(); + curl_easy_setopt(c, CURLOPT_URL, url); + curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, download_writer); + fp = fopen(filename, "wb"); + if (!fp) { + return -1; + } + + curl_easy_setopt(c, CURLOPT_VERBOSE, 0L); + curl_easy_setopt(c, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(c, CURLOPT_USERAGENT, user_agent); + curl_easy_setopt(c, CURLOPT_NOPROGRESS, 0L); + curl_easy_setopt(c, CURLOPT_WRITEDATA, fp); + + if (timeout_str) { + timeout = strtol(timeout_str, NULL, 10); + } + curl_easy_setopt(c, CURLOPT_CONNECTTIMEOUT, timeout); + + curl_code = curl_easy_perform(c); + if (curl_code != CURLE_OK) { + if (errmsg) { + strcpy(*errmsg, curl_easy_strerror(curl_code)); + } else { + fprintf(stderr, "\nCURL ERROR: %s\n", curl_easy_strerror(curl_code)); + } + goto failed; + } + curl_easy_getinfo(c, CURLINFO_RESPONSE_CODE, &http_code); + + failed: + fclose(fp); + curl_easy_cleanup(c); + curl_global_cleanup(); + return http_code; +} \ No newline at end of file diff --git a/src/lib/core/envctl.c b/src/lib/core/envctl.c new file mode 100644 index 0000000..9037d9d --- /dev/null +++ b/src/lib/core/envctl.c @@ -0,0 +1,124 @@ +#include "envctl.h" + +struct EnvCtl *envctl_init() { + struct EnvCtl *result; + + result = calloc(1, sizeof(*result)); + if (!result) { + return NULL; + } + + result->num_alloc = STASIS_ENVCTL_DEFAULT_ALLOC; + result->item = calloc(result->num_alloc + 1, sizeof(result->item)); + if (!result->item) { + guard_free(result); + return NULL; + } + + return result; +} + +static int callback_builtin_nop(const void *a, const void *b) { + return STASIS_ENVCTL_RET_SUCCESS; +} + +int envctl_register(struct EnvCtl **envctl, unsigned flags, envctl_except_fn *callback, const char *name) { + if ((*envctl)->num_used == (*envctl)->num_alloc) { + (*envctl)->num_alloc += STASIS_ENVCTL_DEFAULT_ALLOC; + struct EnvCtl_Item **tmp = realloc((*envctl)->item, (*envctl)->num_alloc + 1 * sizeof((*envctl)->item)); + if (!tmp) { + return 1; + } else { + (*envctl)->item = tmp; + } + } + + struct EnvCtl_Item **item = (*envctl)->item; + item[(*envctl)->num_used] = calloc(1, sizeof(*item[0])); + if (!item[(*envctl)->num_used]) { + return 1; + } + if (!callback) { + callback = &callback_builtin_nop; + } + item[(*envctl)->num_used]->callback = callback; + item[(*envctl)->num_used]->name = name; + item[(*envctl)->num_used]->flags = flags; + + (*envctl)->num_used++; + return 0; +} + +size_t envctl_get_index(const struct EnvCtl *envctl, const char *name) { + for (size_t i = 0; i < envctl->num_used; i++) { + if (!strcmp(envctl->item[i]->name, name)) { + // pack state flag, outer (struct) index and inner (name) index + return 1L << 63L | i; + } + } + return 0; +} + +void envctl_decode_index(size_t in_i, size_t *state, size_t *out_i, size_t *name_i) { + *state = ((in_i >> 63L) & 1); + *out_i = in_i & 0xffffffffL; +} + +unsigned envctl_check_required(unsigned flags) { + return flags & STASIS_ENVCTL_REQUIRED; +} + +unsigned envctl_check_redact(unsigned flags) { + return flags & STASIS_ENVCTL_REDACT; +} + +int envctl_check_present(const struct EnvCtl_Item *item, const char *name) { + return ((!strcmp(item->name, name)) && getenv(name)) ? 1 : 0; +} + +unsigned envctl_get_flags(const struct EnvCtl *envctl, const char *name) { + size_t poll_index = envctl_get_index(envctl, name); + size_t id = 0; + size_t name_id = 0; + size_t state = 0; + envctl_decode_index(poll_index, &state, &id, &name_id); + if (!state) { + return 0; + } else { + fprintf(stderr, "managed environment variable: %s\n", name); + } + return envctl->item[id]->flags; +} + +void envctl_do_required(const struct EnvCtl *envctl, int verbose) { + for (size_t i = 0; i < envctl->num_used; i++) { + struct EnvCtl_Item *item = envctl->item[i]; + const char *name = item->name; + envctl_except_fn *callback = item->callback; + + if (verbose) { + msg(STASIS_MSG_L2, "Verifying %s\n", name); + } + int code = callback((const void *) item, (const void *) name); + if (code == STASIS_ENVCTL_RET_IGNORE || code == STASIS_ENVCTL_RET_SUCCESS) { + continue; + } else if (code == STASIS_ENVCTL_RET_FAIL) { + fprintf(stderr, "\n%s must be set. Exiting.\n", name); + exit(1); + } else { + fprintf(stderr, "\nan unknown envctl callback code occurred: %d\n", code); + exit(1); + } + } +} + +void envctl_free(struct EnvCtl **envctl) { + if (!envctl) { + return; + } + for (size_t i = 0; i < (*envctl)->num_used; i++) { + guard_free((*envctl)->item[i]); + } + guard_free((*envctl)->item); + guard_free(*envctl); +} \ No newline at end of file diff --git a/src/lib/core/environment.c b/src/lib/core/environment.c new file mode 100644 index 0000000..580062c --- /dev/null +++ b/src/lib/core/environment.c @@ -0,0 +1,443 @@ +/** + * @file environment.c + */ +#include "environment.h" +#include "utils.h" +#include "strlist.h" + +//extern char **__environ; + +/** + * Print a shell-specific listing of environment variables to `stdout` + * + * Example: + * ~~~{.c} + * int main(int argc, char *argv[], char *arge[]) { + * RuntimeEnv *rt = runtime_copy(arge); + * runtime_export(rt, NULL); + * runtime_free(rt); + * return 0; + * } + * ~~~ + * + * Usage: + * ~~~{.sh} + * $ gcc program.c + * $ ./a.out + * PATH="/thing/stuff/bin:/example/please/bin" + * SHELL="/your/shell" + * CC="/your/compiler" + * ...=... + * + * # You can also use this to modify the shell environment + * # (use `runtime_set` to manipulate the output) + * $ source $(./a.out) + * ~~~ + * + * Example of exporting specific keys from the environment: + * + * ~~~{.c} + * int main(int argc, char *argv[], char *arge[]) { + * RuntimeEnv *rt = runtime_copy(arge); + * + * // inline declaration + * runtime_export(rt, (char *[]) {"PATH", "LS_COLORS", NULL}); + * + * // standard declaration + * char *keys_to_export[] = { + * "PATH", "LS_COLORS", NULL + * } + * runtime_export(rt, keys_to_export); + * + * runtime_free(rt); + * return 0; + * } + * ~~~ + * + * @param env `RuntimeEnv` structure + * @param keys Array of keys to export. A value of `NULL` exports all environment keys + */ +void runtime_export(RuntimeEnv *env, char **keys) { + char *borne[] = { + "bash", + "dash", + "zsh", + NULL, + }; + char *unborne[] = { + "csh" + "tcsh", + NULL, + }; + + char output[STASIS_BUFSIZ]; + char export_command[7]; // export=6 and setenv=6... convenient + char *_sh = getenv("SHELL"); + char *sh = path_basename(_sh); + if (sh == NULL) { + fprintf(stderr, "echo SHELL environment variable is not defined"); + exit(1); + } + + for (size_t i = 0; borne[i] != NULL; i++) { + if (strcmp(sh, borne[i]) == 0) { + strcpy(export_command, "export"); + break; + } + } + for (size_t i = 0; unborne[i] != NULL; i++) { + if (strcmp(sh, unborne[i]) == 0) { + strcpy(export_command, "setenv"); + break; + } + } + + for (size_t i = 0; i < strlist_count(env); i++) { + char **pair = split(strlist_item(env, i), "=", 0); + char *key = pair[0]; + char *value = NULL; + + // We split a potentially large string by "=" so: + // Recombine elements pair[1..N] into a single string by "=" + if (pair[1] != NULL) { + value = join(&pair[1], "="); + } + + if (keys != NULL) { + for (size_t j = 0; keys[j] != NULL; j++) { + if (strcmp(keys[j], key) == 0) { + //sprintf(output, "%s=\"%s\"\n%s %s", key, value ? value : "", export_command, key); + sprintf(output, "%s %s=\"%s\"", export_command, key, value ? value : ""); + puts(output); + } + } + } + else { + sprintf(output, "%s %s=\"%s\"", export_command, key, value ? value : ""); + puts(output); + } + guard_free(value); + GENERIC_ARRAY_FREE(pair); + } +} + +/** + * Populate a `RuntimeEnv` structure + * + * Example: + * + * ~~~{.c} + * int main(int argc, char *argv[], char *arge[]) { + * RuntimeEnv *rt = NULL; + * // Example 1: Copy the shell environment + * rt = runtime_copy(arge); + * // Example 2: Create your own environment + * rt = runtime_copy((char *[]) {"SHELL=/bin/bash", "PATH=/opt/secure:/bin:/usr/bin"}) + * + * runtime_free(rt); + * return 0; + * } + * ~~~ + * + * @param env Array of strings in `var=value` format + * @return `RuntimeEnv` structure + */ +RuntimeEnv *runtime_copy(char **env) { + RuntimeEnv *rt = NULL; + size_t env_count; + for (env_count = 0; env[env_count] != NULL; env_count++); + + rt = strlist_init(); + for (size_t i = 0; i < env_count; i++) { + strlist_append(&rt, env[i]); + } + return rt; +} + +/** + * Replace the contents of `dest` with `src` + * @param dest pointer of type `RuntimeEnv` + * @param src pointer to environment array + * @return 0 on success, <0 on error + */ +int runtime_replace(RuntimeEnv **dest, char **src) { + RuntimeEnv *rt_tmp = runtime_copy(src); + if (!rt_tmp) { + return -1; + } + runtime_free((*dest)); + + (*dest) = runtime_copy(rt_tmp->data); + if (!(*dest)) { + return -1; + } + runtime_free(rt_tmp); + + runtime_apply((*dest)); + return 0; +} + +/** + * Determine whether or not a key exists in the runtime environment + * + * Example: + * + * ~~~{.c} + * int main(int argc, char *argv[], char *arge[]) { + * RuntimeEnv *rt = runtime_copy(arge); + * if (runtime_contains(rt, "PATH") { + * // $PATH is present + * } + * else { + * // $PATH is NOT present + * } + * + * runtime_free(rt); + * return 0; + * } + * ~~~ + * + * @param env `RuntimeEnv` structure + * @param key Environment variable string + * @return -1=no, positive_value=yes + */ +ssize_t runtime_contains(RuntimeEnv *env, const char *key) { + ssize_t result = -1; + for (ssize_t i = 0; i < (ssize_t) strlist_count(env); i++) { + char **pair = split(strlist_item(env, i), "=", 0); + if (pair == NULL) { + break; + } + if (strcmp(pair[0], key) == 0) { + result = i; + GENERIC_ARRAY_FREE(pair); + break; + } + GENERIC_ARRAY_FREE(pair); + } + return result; +} + +/** + * Retrieve the value of a runtime environment variable + * + * Example: + * + * ~~~{.c} + * int main(int argc, char *argv[], char *arge[]) { + * RuntimeEnv *rt = runtime_copy(arge); + * char *path = runtime_get("PATH"); + * if (path == NULL) { + * // handle error + * } + * + * runtime_free(rt); + * return 0; + * } + * ~~~ + * + * @param env `RuntimeEnv` structure + * @param key Environment variable string + * @return success=string, failure=`NULL` + */ +char *runtime_get(RuntimeEnv *env, const char *key) { + char *result = NULL; + ssize_t key_offset = runtime_contains(env, key); + if (key_offset != -1) { + char **pair = split(strlist_item(env, key_offset), "=", 0); + result = join(&pair[1], "="); + GENERIC_ARRAY_FREE(pair); + } + return result; +} + +/** + * Parse an input string and expand any environment variable(s) found + * + * Example: + * + * ~~~{.c} + * int main(int argc, char *argv[], char *arge[]) { + * RuntimeEnv *rt = runtime_copy(arge); + * char *secure_path = runtime_expand_var(rt, "/opt/secure:$PATH:/aux/bin"); + * if (secure_path == NULL) { + * // handle error + * } + * // secure_path = "/opt/secure:/your/original/path/here:/aux/bin"; + * + * runtime_free(rt); + * return 0; + * } + * ~~~ + * + * @param env `RuntimeEnv` structure + * @param input String to parse + * @return success=expanded string, failure=`NULL` + */ +char *runtime_expand_var(RuntimeEnv *env, char *input) { + const char delim = '$'; + const char *delim_literal = "$$"; + char *expanded = NULL; + + // Input is invalid + if (!input) { + return NULL; + } + + // If there's no environment variables to process return the input string + if (strchr(input, delim) == NULL) { + //return strdup(input); + return input; + } + + expanded = calloc(STASIS_BUFSIZ, sizeof(char)); + if (expanded == NULL) { + SYSERROR("could not allocate %d bytes for runtime_expand_var buffer", STASIS_BUFSIZ); + return NULL; + } + + // Parse the input string + size_t i; + for (i = 0; i < strlen(input); i++) { + char var[MAXNAMLEN]; // environment variable name + memset(var, '\0', MAXNAMLEN); // zero out name + + // Handle literal statement "$$var" + // Value becomes "$var" (unexpanded) + if (strncmp(&input[i], delim_literal, strlen(delim_literal)) == 0) { + strncat(expanded, &delim, 2); + i += strlen(delim_literal); + // Ignore opening brace + if (input[i] == '{') { + i++; + } + } + + // Handle variable when encountering a single $ + // Value expands from "$var" to "environment value of var" + if (input[i] == delim) { + // Ignore opening brace + if (input[i+1] == '{') { + i++; + } + char *tmp = NULL; + i++; + + // Construct environment variable name from input + // "$ var" == no + // "$-*)!@ == no + // "$var" == yes + for (size_t c = 0; isalnum(input[i]) || input[i] == '_'; c++, i++) { + // Ignore closing brace + if (input[i] == '}') { + i++; + } + var[c] = input[i]; + } + + if (env) { + tmp = runtime_get(env, var); + } else { + tmp = getenv(var); + } + if (tmp == NULL) { + // This mimics shell behavior in general. + // Prevent appending whitespace when an environment variable does not exist + if (i > 0) { + i--; + } + continue; + } + // Append expanded environment variable to output + strncat(expanded, tmp, STASIS_BUFSIZ - 1); + if (env) { + guard_free(tmp); + } + } + + // Nothing to do so append input to output + if (input[i] == '}') { + // Unless we ended on a closing brace + continue; + } + strncat(expanded, &input[i], 1); + } + + return expanded; +} + +/** + * Set a runtime environment variable. + * + * + * Note: `_value` is passed through `runtime_expand_var` to provide shell expansion + * + * + * Example: + * + * ~~~{.c} + * int main(int argc, char *argv[], char *arge[]) { + * RuntimeEnv *rt = runtime_copy(arge); + * + * runtime_set(rt, "new_var", "1"); + * char *new_var = runtime_get("new_var"); + * // new_var = 1; + * + * char *path = runtime_get("PATH"); + * // path = /your/path:/here + * + * runtime_set(rt, "PATH", "/opt/secure:$PATH"); + * char *secure_path = runtime_get("PATH"); + * // secure_path = /opt/secure:/your/path:/here + * // NOTE: path and secure_path are COPIES, unlike `getenv()` and `setenv()` that reuse their pointers in `environ` + * + * runtime_free(rt); + * return 0; + * } + * ~~~ + * + * + * @param env `RuntimeEnv` structure + * @param _key Environment variable to set + * @param _value New environment variable value + */ +void runtime_set(RuntimeEnv *env, const char *_key, char *_value) { + if (_key == NULL) { + return; + } + char *key = strdup(_key); + ssize_t key_offset = runtime_contains(env, key); + char *value = runtime_expand_var(env, _value); + char *now = join((char *[]) {key, value, NULL}, "="); + + if (key_offset < 0) { + strlist_append(&env, now); + } else { + strlist_set(&env, key_offset, now); + } + guard_free(now); + guard_free(key); +} + +/** + * Update the global `environ` array with data from `RuntimeEnv` + * @param env `RuntimeEnv` structure + */ +void runtime_apply(RuntimeEnv *env) { + for (size_t i = 0; i < strlist_count(env); i++) { + char **pair = split(strlist_item(env, i), "=", 1); + setenv(pair[0], pair[1], 1); + GENERIC_ARRAY_FREE(pair); + } +} + +/** + * Free `RuntimeEnv` allocated by `runtime_copy` + * @param env `RuntimeEnv` structure + */ +void runtime_free(RuntimeEnv *env) { + if (env == NULL) { + return; + } + strlist_free(&env); +} diff --git a/src/lib/core/github.c b/src/lib/core/github.c new file mode 100644 index 0000000..c5e4534 --- /dev/null +++ b/src/lib/core/github.c @@ -0,0 +1,134 @@ +#include +#include +#include +#include "core.h" +#include "github.h" + +struct GHContent { + char *data; + size_t len; +}; + +static size_t writer(void *contents, size_t size, size_t nmemb, void *result) { + const size_t newlen = size * nmemb; + struct GHContent *content = (struct GHContent *) result; + + char *ptr = realloc(content->data, content->len + newlen + 1); + if (!ptr) { + perror("realloc failed"); + return 0; + } + + content->data = ptr; + memcpy(&(content->data[content->len]), contents, newlen); + content->len += newlen; + content->data[content->len] = 0; + + return newlen; +} + +static char *unescape_lf(char *value) { + char *seq = strstr(value, "\\n"); + while (seq != NULL) { + size_t cur_len = strlen(seq); + memmove(seq, seq + 1, strlen(seq) - 1); + *seq = '\n'; + if (strlen(seq) && cur_len) { + seq[cur_len - 1] = 0; + } + seq = strstr(value, "\\n"); + } + return value; +} + +int get_github_release_notes(const char *api_token, const char *repo, const char *tag, const char *target_commitish, char **output) { + const char *field_body = "\"body\":\""; + const char *field_message = "\"message\":\""; + const char *endpoint_header_auth_fmt = "Authorization: Bearer %s"; + const char *endpoint_header_api_version = "X-GitHub-Api-Version: " STASIS_GITHUB_API_VERSION; + const char *endpoint_post_fields_fmt = "{\"tag_name\":\"%s\", \"target_commitish\":\"%s\"}"; + const char *endpoint_url_fmt = "https://api.github.com/repos/%s/releases/generate-notes"; + char endpoint_header_auth[PATH_MAX] = {0}; + char endpoint_post_fields[PATH_MAX] = {0}; + char endpoint_url[PATH_MAX] = {0}; + struct curl_slist *list = NULL; + struct GHContent content; + + CURL *curl = curl_easy_init(); + if (!curl) { + return -1; + } + + // Render the header data + sprintf(endpoint_header_auth, endpoint_header_auth_fmt, api_token); + sprintf(endpoint_post_fields, endpoint_post_fields_fmt, tag, target_commitish); + sprintf(endpoint_url, endpoint_url_fmt, repo); + + // Begin curl configuration + curl_easy_setopt(curl, CURLOPT_URL, endpoint_url); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, endpoint_post_fields); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writer); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) &content); + + // Append headers to the request + list = curl_slist_append(list, "Accept: application/vnd.github+json"); + list = curl_slist_append(list, endpoint_header_auth); + list = curl_slist_append(list, endpoint_header_api_version); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list); + + // Set the user-agent (github requires one) + char user_agent[20] = {0}; + sprintf(user_agent, "stasis/%s", VERSION); + curl_easy_setopt(curl, CURLOPT_USERAGENT, user_agent); + + // Execute curl request + memset(&content, 0, sizeof(content)); + CURLcode res; + res = curl_easy_perform(curl); + + // Clean up + curl_slist_free_all(list); + curl_easy_cleanup(curl); + + if(res != CURLE_OK) { + fprintf(stderr, "curl_easy_perform() failed: %s\n", + curl_easy_strerror(res)); + return -1; + } + + // Replace all "\\n" literals with new line characters + char *line = unescape_lf(content.data); + if (line) { + char *data_offset = NULL; + if ((data_offset = strstr(line, field_body))) { + // Skip past the body field + data_offset += strlen(field_body); + // Remove quotation mark (and trailing comma if it exists) + int trim = 2; + char last_char = data_offset[strlen(data_offset) - trim]; + if (last_char == ',') { + trim++; + } + data_offset[strlen(data_offset) - trim] = 0; + // Extract release notes + *output = strdup(data_offset); + } else if ((data_offset = strstr(line, field_message))) { + // Skip past the message field + data_offset += strlen(field_message); + *(strchr(data_offset, '"')) = 0; + fprintf(stderr, "GitHub API Error: '%s'\n", data_offset); + fprintf(stderr, "URL: %s\n", endpoint_url); + fprintf(stderr, "POST: %s\n", endpoint_post_fields); + guard_free(content.data); + return -1; + } + } else { + fprintf(stderr, "Unknown error\n"); + guard_free(content.data); + return -1; + } + + guard_free(content.data); + return 0; +} \ No newline at end of file diff --git a/src/lib/core/globals.c b/src/lib/core/globals.c new file mode 100644 index 0000000..83465f1 --- /dev/null +++ b/src/lib/core/globals.c @@ -0,0 +1,66 @@ +#include +#include +#include "core.h" +#include "envctl.h" + +const char *VERSION = "1.0.0"; +const char *AUTHOR = "Joseph Hunkeler"; +const char *BANNER = + "------------------------------------------------------------------------\n" +#if defined(STASIS_DUMB_TERMINAL) + " STASIS \n" +#else + " _____ _______ _____ _____ _____ \n" + " / ____|__ __|/\\ / ____|_ _|/ ____| \n" + " | (___ | | / \\ | (___ | | | (___ \n" + " \\___ \\ | | / /\\ \\ \\___ \\ | | \\___ \\ \n" + " ____) | | |/ ____ \\ ____) |_| |_ ____) | \n" + " |_____/ |_/_/ \\_\\_____/|_____|_____/ \n" + "\n" +#endif + "------------------------------------------------------------------------\n" + " Delivery Generator \n" + " v%s \n" + "------------------------------------------------------------------------\n" + "Copyright (C) 2023-2024 %s,\n" + "Association of Universities for Research in Astronomy (AURA)\n"; + +struct STASIS_GLOBAL globals = { + .verbose = false, ///< Toggle verbose mode + .continue_on_error = false, ///< Do not stop program on error + .always_update_base_environment = false, ///< Run "conda update --all" after installing Conda + .conda_fresh_start = true, ///< Remove/reinstall Conda at startup + .conda_install_prefix = NULL, ///< Path to install Conda + .conda_packages = NULL, ///< Conda packages to install + .pip_packages = NULL, ///< Python packages to install + .tmpdir = NULL, ///< Path to store temporary data + .enable_docker = true, ///< Toggle docker usage + .enable_artifactory = true, ///< Toggle artifactory server usage + .enable_artifactory_build_info = true, ///< Toggle build-info uploads + .enable_testing = true, ///< Toggle [test] block "script" execution. "script_setup" always executes. + .enable_rewrite_spec_stage_2 = true, ///< Leave template stings in output files + .enable_parallel = true, ///< Toggle testing in parallel + .parallel_fail_fast = false, ///< Kill ALL multiprocessing tasks immediately on error + .pool_status_interval = 30, ///< Report "Task is running" +}; + +void globals_free() { + guard_free(globals.tmpdir); + guard_free(globals.sysconfdir); + guard_free(globals.conda_install_prefix); + guard_strlist_free(&globals.conda_packages); + guard_strlist_free(&globals.pip_packages); + guard_free(globals.jfrog.arch); + guard_free(globals.jfrog.os); + guard_free(globals.jfrog.url); + guard_free(globals.jfrog.repo); + guard_free(globals.jfrog.version); + guard_free(globals.jfrog.cli_major_ver); + guard_free(globals.jfrog.jfrog_artifactory_base_url); + guard_free(globals.jfrog.jfrog_artifactory_product); + guard_free(globals.jfrog.remote_filename); + guard_free(globals.workaround.conda_reactivate); + if (globals.envctl) { + envctl_free(&globals.envctl); + } +} diff --git a/src/lib/core/ini.c b/src/lib/core/ini.c new file mode 100644 index 0000000..d44e1cc --- /dev/null +++ b/src/lib/core/ini.c @@ -0,0 +1,678 @@ +#include +#include +#include +#include +#include "core.h" +#include "ini.h" + +struct INIFILE *ini_init() { + struct INIFILE *ini; + ini = calloc(1, sizeof(*ini)); + ini->section_count = 0; + return ini; +} + +void ini_section_init(struct INIFILE **ini) { + (*ini)->section = calloc((*ini)->section_count + 1, sizeof(**(*ini)->section)); +} + +struct INISection *ini_section_search(struct INIFILE **ini, unsigned mode, const char *value) { + struct INISection *result = NULL; + for (size_t i = 0; i < (*ini)->section_count; i++) { + if ((*ini)->section[i]->key != NULL) { + if (mode == INI_SEARCH_EXACT) { + if (!strcmp((*ini)->section[i]->key, value)) { + result = (*ini)->section[i]; + break; + } + } else if (mode == INI_SEARCH_BEGINS) { + if (startswith((*ini)->section[i]->key, value)) { + result = (*ini)->section[i]; + break; + } + } else if (mode == INI_SEARCH_SUBSTR) { + if (strstr((*ini)->section[i]->key, value)) { + result = (*ini)->section[i]; + break; + } + } + } + } + return result; +} + +int ini_data_init(struct INIFILE **ini, char *section_name) { + struct INISection *section = ini_section_search(ini, INI_SEARCH_EXACT, section_name); + if (section == NULL) { + return 1; + } + section->data = calloc(section->data_count + 1, sizeof(**section->data)); + return 0; +} + +int ini_has_key(struct INIFILE *ini, const char *section_name, const char *key) { + if (!ini || !section_name || !key) { + return 0; + } + struct INISection *section = ini_section_search(&ini, INI_SEARCH_EXACT, section_name); + if (!section) { + return 0; + } + for (size_t i = 0; i < section->data_count; i++) { + const struct INIData *data = section->data[i]; + if (data && data->key) { + if (!strcmp(data->key, key)) { + return 1; + } + } + } + return 0; +} + +struct INIData *ini_data_get(struct INIFILE *ini, char *section_name, char *key) { + struct INISection *section = NULL; + + section = ini_section_search(&ini, INI_SEARCH_EXACT, section_name); + if (!section) { + return NULL; + } + + for (size_t i = 0; i < section->data_count; i++) { + if (section->data[i]->key != NULL) { + if (!strcmp(section->data[i]->key, key)) { + return section->data[i]; + } + } + } + + return NULL; +} + +struct INIData *ini_getall(struct INIFILE *ini, char *section_name) { + struct INISection *section = NULL; + struct INIData *result = NULL; + static size_t i = 0; + + section = ini_section_search(&ini, INI_SEARCH_EXACT, section_name); + if (!section) { + return NULL; + } + if (i == section->data_count) { + i = 0; + return NULL; + } + if (section->data_count) { + result = section->data[i]; + i++; + } else { + result = NULL; + i = 0; + } + + return result; +} + +int ini_getval(struct INIFILE *ini, char *section_name, char *key, int type, int flags, union INIVal *result) { + char *token = NULL; + char tbuf[STASIS_BUFSIZ]; + char *tbufp = tbuf; + struct INIData *data; + data = ini_data_get(ini, section_name, key); + if (!data) { + result->as_char_p = NULL; + return -1; + } + + char *data_copy = strdup(data->value); + if (flags == INI_READ_RENDER) { + char *render = tpl_render(data_copy); + if (render && strcmp(render, data_copy) != 0) { + guard_free(data_copy); + data_copy = render; + } else { + guard_free(render); + } + } + lstrip(data_copy); + + switch (type) { + case INIVAL_TYPE_CHAR: + result->as_char = (char) strtol(data_copy, NULL, 10); + break; + case INIVAL_TYPE_UCHAR: + result->as_uchar = (unsigned char) strtoul(data_copy, NULL, 10); + break; + case INIVAL_TYPE_SHORT: + result->as_short = (short) strtol(data_copy, NULL, 10); + break; + case INIVAL_TYPE_USHORT: + result->as_ushort = (unsigned short) strtoul(data_copy, NULL, 10); + break; + case INIVAL_TYPE_INT: + result->as_int = (int) strtol(data_copy, NULL, 10); + break; + case INIVAL_TYPE_UINT: + result->as_uint = (unsigned int) strtoul(data_copy, NULL, 10); + break; + case INIVAL_TYPE_LONG: + result->as_long = (long) strtol(data_copy, NULL, 10); + break; + case INIVAL_TYPE_ULONG: + result->as_ulong = (unsigned long) strtoul(data_copy, NULL, 10); + break; + case INIVAL_TYPE_LLONG: + result->as_llong = (long long) strtoll(data_copy, NULL, 10); + break; + case INIVAL_TYPE_ULLONG: + result->as_ullong = (unsigned long long) strtoull(data_copy, NULL, 10); + break; + case INIVAL_TYPE_DOUBLE: + result->as_double = (double) strtod(data_copy, NULL); + break; + case INIVAL_TYPE_FLOAT: + result->as_float = strtof(data_copy, NULL); + break; + case INIVAL_TYPE_STR: + result->as_char_p = strdup(data_copy); + if (!result->as_char_p) { + return -1; + } + break; + case INIVAL_TYPE_STR_ARRAY: + strcpy(tbufp, data_copy); + guard_free(data_copy); + data_copy = calloc(STASIS_BUFSIZ, sizeof(*data_copy)); + if (!data_copy) { + return -1; + } + while ((token = strsep(&tbufp, "\n")) != NULL) { + //lstrip(token); + if (!isempty(token)) { + strcat(data_copy, token); + strcat(data_copy, "\n"); + } + } + strip(data_copy); + result->as_char_p = strdup(data_copy); + break; + case INIVAL_TYPE_BOOL: + result->as_bool = false; + if ((!strcmp(data_copy, "true") || !strcmp(data_copy, "True")) || + (!strcmp(data_copy, "yes") || !strcmp(data_copy, "Yes")) || + strtol(data_copy, NULL, 10)) { + result->as_bool = true; + } + break; + default: + memset(result, 0, sizeof(*result)); + break; + } + guard_free(data_copy); + return 0; +} + +#define getval_returns(t) return result.t +#define getval_setup(t, f) \ + union INIVal result; \ + int state_local = 0; \ + state_local = ini_getval(ini, section_name, key, t, f, &result); \ + if (state != NULL) { \ + *state = state_local; \ + } + +int ini_getval_int(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + getval_setup(INIVAL_TYPE_INT, flags) + getval_returns(as_int); +} + +unsigned int ini_getval_uint(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + getval_setup(INIVAL_TYPE_UINT, flags) + getval_returns(as_uint); +} + +long ini_getval_long(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + getval_setup(INIVAL_TYPE_LONG, flags) + getval_returns(as_long); +} + +unsigned long ini_getval_ulong(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + getval_setup(INIVAL_TYPE_ULONG, flags) + getval_returns(as_ulong); +} + +long long ini_getval_llong(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + getval_setup(INIVAL_TYPE_LLONG, flags) + getval_returns(as_llong); +} + +unsigned long long ini_getval_ullong(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + getval_setup(INIVAL_TYPE_ULLONG, flags) + getval_returns(as_ullong); +} + +float ini_getval_float(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + getval_setup(INIVAL_TYPE_FLOAT, flags) + getval_returns(as_float); +} + +double ini_getval_double(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + getval_setup(INIVAL_TYPE_DOUBLE, flags) + getval_returns(as_double); +} + +bool ini_getval_bool(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + getval_setup(INIVAL_TYPE_BOOL, flags) + getval_returns(as_bool); +} + +short ini_getval_short(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + getval_setup(INIVAL_TYPE_SHORT, flags) + getval_returns(as_short); +} + +unsigned short ini_getval_ushort(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + getval_setup(INIVAL_TYPE_USHORT, flags) + getval_returns(as_ushort); +} + +char ini_getval_char(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + getval_setup(INIVAL_TYPE_CHAR, flags) + getval_returns(as_char); +} + +unsigned char ini_getval_uchar(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + getval_setup(INIVAL_TYPE_UCHAR, flags) + getval_returns(as_uchar); +} + +char *ini_getval_char_p(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + getval_setup(INIVAL_TYPE_STR, flags) + getval_returns(as_char_p); +} + +char *ini_getval_str(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + return ini_getval_char_p(ini, section_name, key, flags, state); +} + +char *ini_getval_char_array_p(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + getval_setup(INIVAL_TYPE_STR_ARRAY, flags) + getval_returns(as_char_p); +} + +char *ini_getval_str_array(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + return ini_getval_char_array_p(ini, section_name, key, flags, state); +} + +struct StrList *ini_getval_strlist(struct INIFILE *ini, char *section_name, char *key, char *tok, int flags, int *state) { + getval_setup(INIVAL_TYPE_STR_ARRAY, flags) + struct StrList *list; + list = strlist_init(); + strlist_append_tokenize(list, result.as_char_p, tok); + guard_free(result.as_char_p); + return list; +} + +int ini_data_append(struct INIFILE **ini, char *section_name, char *key, char *value, unsigned int hint) { + struct INISection *section = ini_section_search(ini, INI_SEARCH_EXACT, section_name); + if (section == NULL) { + return 1; + } + + struct INIData **tmp = realloc(section->data, (section->data_count + 1) * sizeof(**section->data)); + if (tmp == NULL) { + return 1; + } else { + section->data = tmp; + } + if (!ini_data_get((*ini), section_name, key)) { + struct INIData **data = section->data; + data[section->data_count] = calloc(1, sizeof(*data[0])); + if (!data[section->data_count]) { + SYSERROR("Unable to allocate %zu bytes for section data", sizeof(*data[0])); + return -1; + } + data[section->data_count]->type_hint = hint; + data[section->data_count]->key = key ? strdup(key) : strdup(""); + if (!data[section->data_count]->key) { + SYSERROR("Unable to allocate data key%s", ""); + return -1; + } + data[section->data_count]->value = strdup(value); + if (!data[section->data_count]->value) { + SYSERROR("Unable to allocate data value%s", ""); + return -1; + } + section->data_count++; + } else { + struct INIData *data = ini_data_get(*ini, section_name, key); + size_t value_len_old = strlen(data->value); + size_t value_len = strlen(value); + size_t value_len_new = value_len_old + value_len; + char *value_tmp = NULL; + value_tmp = realloc(data->value, value_len_new + 2); + if (!value_tmp) { + SYSERROR("Unable to increase data->value size to %zu bytes", value_len_new + 2); + return -1; + } else { + data->value = value_tmp; + } + strcat(data->value, value); + } + return 0; +} + +int ini_setval(struct INIFILE **ini, unsigned type, char *section_name, char *key, char *value) { + struct INISection *section = ini_section_search(ini, INI_SEARCH_EXACT, section_name); + if (section == NULL) { + // no section + return -1; + } + if (ini_has_key(*ini, section_name, key)) { + if (!type) { + if (ini_data_append(ini, section_name, key, value, 0)) { + // append failed + return -1; + } + } else { + struct INIData *data = ini_data_get(*ini, section_name, key); + if (data) { + guard_free(data->value); + data->value = strdup(value); + if (!data->value) { + // allocation failed + return -1; + } + } else { + // getting data failed + return -1; + } + } + } + return 0; +} + +int ini_section_create(struct INIFILE **ini, char *key) { + struct INISection **tmp = realloc((*ini)->section, ((*ini)->section_count + 1) * sizeof(**(*ini)->section)); + if (tmp == NULL) { + return 1; + } else { + (*ini)->section = tmp; + } + + (*ini)->section[(*ini)->section_count] = calloc(1, sizeof(*(*ini)->section[0])); + if (!(*ini)->section[(*ini)->section_count]) { + return -1; + } + + (*ini)->section[(*ini)->section_count]->key = strdup(key); + if (!(*ini)->section[(*ini)->section_count]->key) { + return -1; + } + + (*ini)->section_count++; + return 0; +} + +int ini_write(struct INIFILE *ini, FILE **stream, unsigned mode) { + if (!*stream) { + return -1; + } + for (size_t x = 0; x < ini->section_count; x++) { + struct INISection *section = ini->section[x]; + char *section_name = section->key; + fprintf(*stream, "[%s]" LINE_SEP, section_name); + + for (size_t y = 0; y < ini->section[x]->data_count; y++) { + struct INIData *data = section->data[y]; + char outvalue[STASIS_BUFSIZ]; + char *key = data->key; + char *value = data->value; + unsigned *hint = &data->type_hint; + memset(outvalue, 0, sizeof(outvalue)); + + if (key && value) { + int err = 0; + char *xvalue = NULL; + if (*hint == INIVAL_TYPE_STR_ARRAY) { + xvalue = ini_getval_str_array(ini, section_name, key, (int) mode, &err); + value = xvalue; + } else { + xvalue = ini_getval_str(ini, section_name, key, (int) mode, &err); + value = xvalue; + } + char **parts = split(value, LINE_SEP, 0); + size_t parts_total = 0; + for (; parts && parts[parts_total] != NULL; parts_total++); + for (size_t p = 0; parts && parts[p] != NULL; p++) { + char *render = NULL; + if (mode == INI_WRITE_PRESERVE) { + render = tpl_render(parts[p]); + } else { + render = parts[p]; + } + + if (!render) { + SYSERROR("%s", "rendered string value can never be NULL!\n"); + return -1; + } + + if (*hint == INIVAL_TYPE_STR_ARRAY) { + int leading_space = isspace(*render); + if (leading_space) { + sprintf(outvalue + strlen(outvalue), "%s" LINE_SEP, render); + } else { + sprintf(outvalue + strlen(outvalue), " %s" LINE_SEP, render); + } + } else { + sprintf(outvalue + strlen(outvalue), "%s", render); + } + if (mode == INI_WRITE_PRESERVE) { + guard_free(render); + } + } + GENERIC_ARRAY_FREE(parts); + strip(outvalue); + strcat(outvalue, LINE_SEP); + fprintf(*stream, "%s = %s%s", ini->section[x]->data[y]->key, *hint == INIVAL_TYPE_STR_ARRAY ? LINE_SEP : "", outvalue); + guard_free(value); + } else { + fprintf(*stream, "%s = %s", ini->section[x]->data[y]->key, ini->section[x]->data[y]->value); + } + } + fprintf(*stream, LINE_SEP); + } + return 0; +} + +char *unquote(char *s) { + if ((startswith(s, "'") && endswith(s, "'")) + || (startswith(s, "\"") && endswith(s, "\""))) { + memmove(s, s + 1, strlen(s)); + s[strlen(s) - 1] = '\0'; + } + return s; +} + +void ini_free(struct INIFILE **ini) { + for (size_t section = 0; section < (*ini)->section_count; section++) { +#ifdef DEBUG + SYSERROR("freeing section: %s", (*ini)->section[section]->key); +#endif + for (size_t data = 0; data < (*ini)->section[section]->data_count; data++) { + if ((*ini)->section[section]->data[data]) { +#ifdef DEBUG + SYSERROR("freeing data key: %s", (*ini)->section[section]->data[data]->key); +#endif + guard_free((*ini)->section[section]->data[data]->key); +#ifdef DEBUG + SYSERROR("freeing data value: %s", (*ini)->section[section]->data[data]->value); +#endif + guard_free((*ini)->section[section]->data[data]->value); + guard_free((*ini)->section[section]->data[data]); + } + } + guard_free((*ini)->section[section]->data); + guard_free((*ini)->section[section]->key); + guard_free((*ini)->section[section]); + } + guard_free((*ini)->section); + guard_free((*ini)); +} + +struct INIFILE *ini_open(const char *filename) { + FILE *fp; + char line[STASIS_BUFSIZ] = {0}; + char current_section[STASIS_BUFSIZ] = {0}; + char reading_value = 0; + + struct INIFILE *ini = ini_init(); + if (ini == NULL) { + return NULL; + } + + ini_section_init(&ini); + + // Create an implicit section. [default] does not need to be present in the INI config + ini_section_create(&ini, "default"); + strcpy(current_section, "default"); + + // Open the configuration file for reading + fp = fopen(filename, "r"); + if (!fp) { + ini_free(&ini); + ini = NULL; + return NULL; + } + + unsigned hint = 0; + int multiline_data = 0; + int no_data = 0; + char inikey[2][255]; + char *key = inikey[0]; + char *key_last = inikey[1]; + char value[STASIS_BUFSIZ]; + + memset(value, 0, sizeof(value)); + memset(inikey, 0, sizeof(inikey)); + + // Read file + for (size_t i = 0; fgets(line, sizeof(line), fp) != NULL; i++) { + if (no_data && multiline_data) { + if (!isempty(line)) { + no_data = 0; + } else { + multiline_data = 0; + } + memset(value, 0, sizeof(value)); + } else { + memset(key, 0, sizeof(inikey[0])); + } + // Find pointer to first comment character + char *comment = strpbrk(line, ";#"); + if (comment) { + if (!reading_value || line - comment == 0) { + // Remove comment from line (standalone and inline comments) + if (!((comment - line > 0 && (*(comment - 1) == '\\')) || (*comment - 1) == '#')) { + *comment = '\0'; + } else { + // Handle escaped comment characters. Remove the escape character '\' + memmove(comment - 1, comment, strlen(comment)); + if (strlen(comment)) { + comment[strlen(comment) - 1] = '\0'; + } else { + comment[0] = '\0'; + } + } + } + } + + // Test for section header: [string] + if (startswith(line, "[")) { + // The previous key is irrelevant now + memset(key_last, 0, sizeof(inikey[1])); + + char *section_name = substring_between(line, "[]"); + if (!section_name) { + fprintf(stderr, "error: invalid section syntax, line %zu: '%s'\n", i + 1, line); + return NULL; + } + + // Ignore default section because we already have an implicit one + if (!strncmp(section_name, "default", strlen("default"))) { + guard_free(section_name); + continue; + } + + // Create new named section + strip(section_name); + ini_section_create(&ini, section_name); + + // Record the name of the section. This is used until another section is found. + memset(current_section, 0, sizeof(current_section)); + strcpy(current_section, section_name); + guard_free(section_name); + memset(line, 0, sizeof(line)); + continue; + } + + // no data, skip + if (!reading_value && isempty(line)) { + continue; + } + + char *operator = strchr(line, '='); + + // a value continuation line + if (multiline_data && (startswith(line, " ") || startswith(line, "\t"))) { + operator = NULL; + } + + if (operator) { + size_t key_len = operator - line; + memset(key, 0, sizeof(inikey[0])); + strncpy(key, line, key_len); + lstrip(key); + strip(key); + memset(key_last, 0, sizeof(inikey[1])); + strcpy(key_last, key); + reading_value = 1; + if (strlen(operator) > 1) { + strcpy(value, &operator[1]); + } else { + strcpy(value, ""); + } + if (isempty(value)) { + //printf("%s is probably long raw data\n", key); + hint = INIVAL_TYPE_STR_ARRAY; + multiline_data = 1; + no_data = 1; + } else { + //printf("%s is probably short data\n", key); + hint = INIVAL_TYPE_STR; + multiline_data = 0; + } + strip(value); + } else { + strcpy(key, key_last); + strcpy(value, line); + } + memset(line, 0, sizeof(line)); + + // Store key value pair in section's data array + if (strlen(key)) { + lstrip(key); + strip(key); + unquote(value); + if (!multiline_data) { + reading_value = 0; + ini_data_append(&ini, current_section, key, value, hint); + continue; + } + ini_data_append(&ini, current_section, key, value, hint); + reading_value = 1; + } + } + fclose(fp); + + return ini; +} \ No newline at end of file diff --git a/src/lib/core/junitxml.c b/src/lib/core/junitxml.c new file mode 100644 index 0000000..c7d0834 --- /dev/null +++ b/src/lib/core/junitxml.c @@ -0,0 +1,240 @@ +#include +#include +#include "strlist.h" +#include "junitxml.h" + +static void testcase_result_state_free(struct JUNIT_Testcase **testcase) { + struct JUNIT_Testcase *tc = (*testcase); + if (tc->tc_result_state_type == JUNIT_RESULT_STATE_FAILURE) { + guard_free(tc->result_state.failure->message); + guard_free(tc->result_state.failure); + } else if (tc->tc_result_state_type == JUNIT_RESULT_STATE_SKIPPED) { + guard_free(tc->result_state.skipped->message); + guard_free(tc->result_state.skipped); + } +} + +static void testcase_free(struct JUNIT_Testcase **testcase) { + struct JUNIT_Testcase *tc = (*testcase); + guard_free(tc->name); + guard_free(tc->message); + guard_free(tc->classname); + testcase_result_state_free(&tc); + guard_free(tc); +} + +void junitxml_testsuite_free(struct JUNIT_Testsuite **testsuite) { + struct JUNIT_Testsuite *suite = (*testsuite); + guard_free(suite->name); + guard_free(suite->hostname); + guard_free(suite->timestamp); + for (size_t i = 0; i < suite->_tc_alloc; i++) { + testcase_free(&suite->testcase[i]); + } + guard_free(suite); +} + +static int testsuite_append_testcase(struct JUNIT_Testsuite **testsuite, struct JUNIT_Testcase *testcase) { + struct JUNIT_Testsuite *suite = (*testsuite); + struct JUNIT_Testcase **tmp = realloc(suite->testcase, (suite->_tc_alloc + 1 ) * sizeof(*testcase)); + if (tmp == NULL) { + return -1; + } else { + suite->testcase = tmp; + } + suite->testcase[suite->_tc_inuse] = testcase; + suite->_tc_inuse++; + suite->_tc_alloc++; + return 0; +} + +static struct JUNIT_Failure *testcase_failure_from_attributes(struct StrList *attrs) { + struct JUNIT_Failure *result; + + result = calloc(1, sizeof(*result)); + if(!result) { + return NULL; + } + for (size_t x = 0; x < strlist_count(attrs); x += 2) { + char *attr_name = strlist_item(attrs, x); + char *attr_value = strlist_item(attrs, x + 1); + if (!strcmp(attr_name, "message")) { + result->message = strdup(attr_value); + } + } + return result; +} + +static struct JUNIT_Error *testcase_error_from_attributes(struct StrList *attrs) { + struct JUNIT_Error *result; + + result = calloc(1, sizeof(*result)); + if(!result) { + return NULL; + } + for (size_t x = 0; x < strlist_count(attrs); x += 2) { + char *attr_name = strlist_item(attrs, x); + char *attr_value = strlist_item(attrs, x + 1); + if (!strcmp(attr_name, "message")) { + result->message = strdup(attr_value); + } + } + return result; +} + +static struct JUNIT_Skipped *testcase_skipped_from_attributes(struct StrList *attrs) { + struct JUNIT_Skipped *result; + + result = calloc(1, sizeof(*result)); + if(!result) { + return NULL; + } + for (size_t x = 0; x < strlist_count(attrs); x += 2) { + char *attr_name = strlist_item(attrs, x); + char *attr_value = strlist_item(attrs, x + 1); + if (!strcmp(attr_name, "message")) { + result->message = strdup(attr_value); + } + } + return result; +} + +static struct JUNIT_Testcase *testcase_from_attributes(struct StrList *attrs) { + struct JUNIT_Testcase *result; + + result = calloc(1, sizeof(*result)); + if(!result) { + return NULL; + } + for (size_t x = 0; x < strlist_count(attrs); x += 2) { + char *attr_name = strlist_item(attrs, x); + char *attr_value = strlist_item(attrs, x + 1); + if (!strcmp(attr_name, "name")) { + result->name = strdup(attr_value); + } else if (!strcmp(attr_name, "classname")) { + result->classname = strdup(attr_value); + } else if (!strcmp(attr_name, "time")) { + result->time = strtof(attr_value, NULL); + } else if (!strcmp(attr_name, "message")) { + result->message = strdup(attr_value); + } + } + return result; +} + +static struct StrList *attributes_to_strlist(xmlTextReaderPtr reader) { + struct StrList *list; + xmlNodePtr node = xmlTextReaderCurrentNode(reader); + if (!node) { + return NULL; + } + + list = strlist_init(); + if (xmlTextReaderNodeType(reader) == 1 && node->properties) { + xmlAttr *attr = node->properties; + while (attr && attr->name && attr->children) { + char *attr_name = (char *) attr->name; + char *attr_value = (char *) xmlNodeListGetString(node->doc, attr->children, 1); + strlist_append(&list, attr_name ? attr_name : ""); + strlist_append(&list, attr_value ? attr_value : ""); + xmlFree((xmlChar *) attr_value); + attr = attr->next; + } + } + return list; +} + +static int read_xml_data(xmlTextReaderPtr reader, struct JUNIT_Testsuite **testsuite) { + const xmlChar *name; + //const xmlChar *value; + + name = xmlTextReaderConstName(reader); + if (!name) { + // name could not be converted to string + name = BAD_CAST "--"; + } + //value = xmlTextReaderConstValue(reader); + const char *node_name = (char *) name; + //const char *node_value = (char *) value; + + struct StrList *attrs = attributes_to_strlist(reader); + if (attrs && strlist_count(attrs)) { + if (!strcmp(node_name, "testsuite")) { + for (size_t x = 0; x < strlist_count(attrs); x += 2) { + char *attr_name = strlist_item(attrs, x); + char *attr_value = strlist_item(attrs, x + 1); + if (!strcmp(attr_name, "name")) { + (*testsuite)->name = strdup(attr_value); + } else if (!strcmp(attr_name, "errors")) { + (*testsuite)->errors = (int) strtol(attr_value, NULL, 10); + } else if (!strcmp(attr_name, "failures")) { + (*testsuite)->failures = (int) strtol(attr_value, NULL, 0); + } else if (!strcmp(attr_name, "skipped")) { + (*testsuite)->skipped = (int) strtol(attr_value, NULL, 0); + } else if (!strcmp(attr_name, "tests")) { + (*testsuite)->tests = (int) strtol(attr_value, NULL, 0); + } else if (!strcmp(attr_name, "time")) { + (*testsuite)->time = strtof(attr_value, NULL); + } else if (!strcmp(attr_name, "timestamp")) { + (*testsuite)->timestamp = strdup(attr_value); + } else if (!strcmp(attr_name, "hostname")) { + (*testsuite)->hostname = strdup(attr_value); + } + } + } else if (!strcmp(node_name, "testcase")) { + struct JUNIT_Testcase *testcase = testcase_from_attributes(attrs); + testsuite_append_testcase(testsuite, testcase); + } else if (!strcmp(node_name, "failure")) { + size_t cur_tc = (*testsuite)->_tc_inuse > 0 ? (*testsuite)->_tc_inuse - 1 : (*testsuite)->_tc_inuse; + struct JUNIT_Failure *failure = testcase_failure_from_attributes(attrs); + (*testsuite)->testcase[cur_tc]->tc_result_state_type = JUNIT_RESULT_STATE_FAILURE; + (*testsuite)->testcase[cur_tc]->result_state.failure = failure; + } else if (!strcmp(node_name, "error")) { + size_t cur_tc = (*testsuite)->_tc_inuse > 0 ? (*testsuite)->_tc_inuse - 1 : (*testsuite)->_tc_inuse; + struct JUNIT_Error *error = testcase_error_from_attributes(attrs); + (*testsuite)->testcase[cur_tc]->tc_result_state_type = JUNIT_RESULT_STATE_ERROR; + (*testsuite)->testcase[cur_tc]->result_state.error = error; + } else if (!strcmp(node_name, "skipped")) { + size_t cur_tc = (*testsuite)->_tc_inuse > 0 ? (*testsuite)->_tc_inuse - 1 : (*testsuite)->_tc_inuse; + struct JUNIT_Skipped *skipped = testcase_skipped_from_attributes(attrs); + (*testsuite)->testcase[cur_tc]->tc_result_state_type = JUNIT_RESULT_STATE_SKIPPED; + (*testsuite)->testcase[cur_tc]->result_state.skipped = skipped; + } + } + guard_strlist_free(&attrs); + return 0; +} + +static int read_xml_file(const char *filename, struct JUNIT_Testsuite **testsuite) { + xmlTextReaderPtr reader; + int result; + + reader = xmlReaderForFile(filename, NULL, 0); + if (!reader) { + return -1; + } + + result = xmlTextReaderRead(reader); + while (result == 1) { + read_xml_data(reader, testsuite); + result = xmlTextReaderRead(reader); + } + + xmlFreeTextReader(reader); + return 0; +} + +struct JUNIT_Testsuite *junitxml_testsuite_read(const char *filename) { + struct JUNIT_Testsuite *result; + + if (access(filename, F_OK)) { + return NULL; + } + + result = calloc(1, sizeof(*result)); + if (!result) { + return NULL; + } + read_xml_file(filename, &result); + return result; +} \ No newline at end of file diff --git a/src/lib/core/multiprocessing.c b/src/lib/core/multiprocessing.c new file mode 100644 index 0000000..484c566 --- /dev/null +++ b/src/lib/core/multiprocessing.c @@ -0,0 +1,449 @@ +#include "core.h" +#include "multiprocessing.h" + +/// The sum of all tasks started by mp_task() +size_t mp_global_task_count = 0; + +static struct MultiProcessingTask *mp_pool_next_available(struct MultiProcessingPool *pool) { + return &pool->task[pool->num_used]; +} + +int child(struct MultiProcessingPool *pool, struct MultiProcessingTask *task) { + FILE *fp_log = NULL; + + // The task starts inside the requested working directory + if (chdir(task->working_dir)) { + perror(task->working_dir); + exit(1); + } + + // Record the task start time + if (clock_gettime(CLOCK_REALTIME, &task->time_data.t_start) < 0) { + perror("clock_gettime"); + exit(1); + } + + // Redirect stdout and stderr to the log file + fflush(stdout); + fflush(stderr); + // Set log file name + sprintf(task->log_file + strlen(task->log_file), "task-%zu-%d.log", mp_global_task_count, task->parent_pid); + fp_log = freopen(task->log_file, "w+", stdout); + if (!fp_log) { + fprintf(stderr, "unable to open '%s' for writing: %s\n", task->log_file, strerror(errno)); + return -1; + } + dup2(fileno(stdout), fileno(stderr)); + + // Generate timestamp for log header + time_t t = time(NULL); + char *timebuf = ctime(&t); + if (timebuf) { + // strip line feed from timestamp + timebuf[strlen(timebuf) ? strlen(timebuf) - 1 : 0] = 0; + } + + // Generate log header + fprintf(fp_log, "# STARTED: %s\n", timebuf ? timebuf : "unknown"); + fprintf(fp_log, "# PID: %d\n", task->parent_pid); + fprintf(fp_log, "# WORKDIR: %s\n", task->working_dir); + fprintf(fp_log, "# COMMAND:\n%s\n", task->cmd); + fprintf(fp_log, "# OUTPUT:\n"); + // Commit header to log file / clean up + fflush(fp_log); + + // Execute task + fflush(stdout); + fflush(stderr); + char *args[] = {"bash", "--norc", task->parent_script, (char *) NULL}; + return execvp("/bin/bash", args); +} + +int parent(struct MultiProcessingPool *pool, struct MultiProcessingTask *task, pid_t pid, int *child_status) { + printf("[%s:%s] Task started (pid: %d)\n", pool->ident, task->ident, pid); + + // Give the child process access to our PID value + task->pid = pid; + task->parent_pid = pid; + + mp_global_task_count++; + + // Check child's status + pid_t code = waitpid(pid, child_status, WUNTRACED | WCONTINUED | WNOHANG); + if (code < 0) { + perror("waitpid failed"); + return -1; + } + return 0; +} + +static int mp_task_fork(struct MultiProcessingPool *pool, struct MultiProcessingTask *task) { + pid_t pid = fork(); + int child_status = 0; + if (pid == -1) { + return -1; + } else if (pid == 0) { + child(pool, task); + } + return parent(pool, task, pid, &child_status); +} + +struct MultiProcessingTask *mp_pool_task(struct MultiProcessingPool *pool, const char *ident, char *working_dir, char *cmd) { + struct MultiProcessingTask *slot = mp_pool_next_available(pool); + if (pool->num_used != pool->num_alloc) { + pool->num_used++; + } else { + fprintf(stderr, "Maximum number of tasks reached\n"); + return NULL; + } + + // Set default status to "error" + slot->status = -1; + + // Set task identifier string + memset(slot->ident, 0, sizeof(slot->ident)); + strncpy(slot->ident, ident, sizeof(slot->ident) - 1); + + // Set log file path + memset(slot->log_file, 0, sizeof(*slot->log_file)); + strcat(slot->log_file, pool->log_root); + strcat(slot->log_file, "/"); + + // Set working directory + if (isempty(working_dir)) { + strcpy(slot->working_dir, "."); + } else { + strncpy(slot->working_dir, working_dir, PATH_MAX - 1); + } + + // Create a temporary file to act as our intermediate command script + FILE *tp = NULL; + char *t_name = NULL; + t_name = xmkstemp(&tp, "w"); + if (!t_name || !tp) { + return NULL; + } + + // Set the script's permissions so that only the calling user can use it + // This should help prevent eavesdropping if keys are applied in plain-text + // somewhere. + chmod(t_name, 0700); + + // Record the script path + memset(slot->parent_script, 0, sizeof(slot->parent_script)); + strncpy(slot->parent_script, t_name, PATH_MAX - 1); + guard_free(t_name); + + // Populate the script + fprintf(tp, "#!/bin/bash\n%s\n", cmd); + fflush(tp); + fclose(tp); + + // Record the command(s) + slot->cmd_len = (strlen(cmd) * sizeof(*cmd)) + 1; + slot->cmd = mmap(NULL, slot->cmd_len, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); + memset(slot->cmd, 0, slot->cmd_len); + strncpy(slot->cmd, cmd, slot->cmd_len); + + return slot; +} + +static void get_task_duration(struct MultiProcessingTask *task, struct timespec *result) { + // based on the timersub() macro in time.h + // This implementation uses timespec and increases the resolution from microseconds to nanoseconds. + struct timespec *start = &task->time_data.t_start; + struct timespec *stop = &task->time_data.t_stop; + result->tv_sec = (stop->tv_sec - start->tv_sec); + result->tv_nsec = (stop->tv_nsec - start->tv_nsec); + if (result->tv_nsec < 0) { + --result->tv_sec; + result->tv_nsec += 1000000000L; + } +} + +void mp_pool_show_summary(struct MultiProcessingPool *pool) { + print_banner("=", 79); + printf("Pool execution summary for \"%s\"\n", pool->ident); + print_banner("=", 79); + printf("STATUS PID DURATION IDENT\n"); + for (size_t i = 0; i < pool->num_used; i++) { + struct MultiProcessingTask *task = &pool->task[i]; + char status_str[10] = {0}; + if (!task->status && !task->signaled_by) { + strcpy(status_str, "DONE"); + } else if (task->signaled_by) { + strcpy(status_str, "TERM"); + } else { + strcpy(status_str, "FAIL"); + } + + struct timespec duration; + get_task_duration(task, &duration); + long diff = duration.tv_sec + duration.tv_nsec / 1000000000L; + printf("%-4s %10d %7lds %-10s\n", status_str, task->parent_pid, diff, task->ident) ; + } + puts(""); +} + +static int show_log_contents(FILE *stream, struct MultiProcessingTask *task) { + FILE *fp = fopen(task->log_file, "r"); + if (!fp) { + return -1; + } + char buf[BUFSIZ] = {0}; + while ((fgets(buf, sizeof(buf) - 1, fp)) != NULL) { + fprintf(stream, "%s", buf); + memset(buf, 0, sizeof(buf)); + } + fprintf(stream, "\n"); + fclose(fp); + return 0; +} + +int mp_pool_kill(struct MultiProcessingPool *pool, int signum) { + printf("Sending signal %d to pool '%s'\n", signum, pool->ident); + for (size_t i = 0; i < pool->num_used; i++) { + struct MultiProcessingTask *slot = &pool->task[i]; + if (!slot) { + return -1; + } + // Kill tasks in progress + if (slot->pid > 0) { + int status; + printf("Sending signal %d to task '%s' (pid: %d)\n", signum, slot->ident, slot->pid); + status = kill(slot->pid, signum); + if (status && errno != ESRCH) { + fprintf(stderr, "Task '%s' (pid: %d) did not respond: %s\n", slot->ident, slot->pid, strerror(errno)); + } else { + // Wait for process to handle the signal, then set the status accordingly + if (waitpid(slot->pid, &status, 0) >= 0) { + slot->signaled_by = WTERMSIG(status); + // Record the task stop time + if (clock_gettime(CLOCK_REALTIME, &slot->time_data.t_stop) < 0) { + perror("clock_gettime"); + exit(1); + } + // We are short-circuiting the normal flow, and the process is now dead, so mark it as such + slot->pid = MP_POOL_PID_UNUSED; + } + } + } + if (!access(slot->log_file, F_OK)) { + remove(slot->log_file); + } + if (!access(slot->parent_script, F_OK)) { + remove(slot->parent_script); + } + } + return 0; +} + +int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) { + int status = 0; + int failures = 0; + size_t tasks_complete = 0; + size_t lower_i = 0; + size_t upper_i = jobs; + + do { + size_t hang_check = 0; + if (upper_i >= pool->num_used) { + size_t x = upper_i - pool->num_used; + upper_i -= (size_t) x; + } + + for (size_t i = lower_i; i < upper_i; i++) { + struct MultiProcessingTask *slot = &pool->task[i]; + if (slot->status == -1) { + if (mp_task_fork(pool, slot)) { + fprintf(stderr, "%s: mp_task_fork failed\n", slot->ident); + kill(0, SIGTERM); + } + } + + // Has the child been processed already? + if (slot->pid == MP_POOL_PID_UNUSED) { + // Child is already used up, skip it + hang_check++; + if (hang_check >= pool->num_used) { + // If you join a pool that's already finished it will spin + // forever. This protects the program from entering an + // infinite loop. + fprintf(stderr, "%s is deadlocked\n", pool->ident); + failures++; + goto pool_deadlocked; + } + continue; + } + + // Is the process finished? + pid_t pid = waitpid(slot->pid, &status, WNOHANG | WUNTRACED | WCONTINUED); + int task_ended = WIFEXITED(status); + int task_ended_by_signal = WIFSIGNALED(status); + int task_stopped = WIFSTOPPED(status); + int task_continued = WIFCONTINUED(status); + int status_exit = WEXITSTATUS(status); + int status_signal = WTERMSIG(status); + int status_stopped = WSTOPSIG(status); + + // Update status + slot->status = status_exit; + slot->signaled_by = status_signal; + + char progress[1024] = {0}; + if (pid > 0) { + double percent = ((double) (tasks_complete + 1) / (double) pool->num_used) * 100; + snprintf(progress, sizeof(progress) - 1, "[%s:%s] [%3.1f%%]", pool->ident, slot->ident, percent); + + // The process ended in one the following ways + // Note: SIGSTOP nor SIGCONT will not increment the tasks_complete counter + if (task_stopped) { + printf("%s Task was suspended (%d)\n", progress, status_stopped); + continue; + } else if (task_continued) { + printf("%s Task was resumed\n", progress); + continue; + } else if (task_ended_by_signal) { + printf("%s Task ended by signal %d (%s)\n", progress, status_signal, strsignal(status_signal)); + tasks_complete++; + } else if (task_ended) { + printf("%s Task ended (status: %d)\n", progress, status_exit); + tasks_complete++; + } else { + fprintf(stderr, "%s Task state is unknown (0x%04X)\n", progress, status); + } + + // Show the log (always) + if (show_log_contents(stdout, slot)) { + perror(slot->log_file); + } + + // Record the task stop time + if (clock_gettime(CLOCK_REALTIME, &slot->time_data.t_stop) < 0) { + perror("clock_gettime"); + exit(1); + } + + if (status >> 8 != 0 || (status & 0xff) != 0) { + fprintf(stderr, "%s Task failed\n", progress); + failures++; + + if (flags & MP_POOL_FAIL_FAST && pool->num_used > 1) { + mp_pool_kill(pool, SIGTERM); + return -2; + } + } else { + printf("%s Task finished\n", progress); + } + + // Clean up logs and scripts left behind by the task + if (remove(slot->log_file)) { + fprintf(stderr, "%s Unable to remove log file: '%s': %s\n", progress, slot->parent_script, strerror(errno)); + } + if (remove(slot->parent_script)) { + fprintf(stderr, "%s Unable to remove temporary script '%s': %s\n", progress, slot->parent_script, strerror(errno)); + } + + // Update progress and tell the poller to ignore the PID. The process is gone. + slot->pid = MP_POOL_PID_UNUSED; + } else if (pid < 0) { + fprintf(stderr, "waitpid failed: %s\n", strerror(errno)); + return -1; + } else { + // Track the number of seconds elapsed for each task. + // When a task has executed for longer than status_intervals, print a status update + // _seconds represents the time between intervals, not the total runtime of the task + slot->_seconds = time(NULL) - slot->_now; + if (slot->_seconds > pool->status_interval) { + slot->_now = time(NULL); + slot->_seconds = 0; + } + if (slot->_seconds == 0) { + printf("[%s:%s] Task is running (pid: %d)\n", pool->ident, slot->ident, slot->parent_pid); + } + } + } + + if (tasks_complete == pool->num_used) { + break; + } + + if (tasks_complete == upper_i) { + lower_i += jobs; + upper_i += jobs; + } + + // Poll again after a short delay + sleep(1); + } while (1); + + pool_deadlocked: + puts(""); + return failures; +} + + +struct MultiProcessingPool *mp_pool_init(const char *ident, const char *log_root) { + struct MultiProcessingPool *pool; + + if (!ident || !log_root) { + // Pool must have an ident string + // log_root must be set + return NULL; + } + + // The pool is shared with children + pool = mmap(NULL, sizeof(*pool), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); + + // Set pool identity string + memset(pool->ident, 0, sizeof(pool->ident)); + strncpy(pool->ident, ident, sizeof(pool->ident) - 1); + + // Set logging base directory + memset(pool->log_root, 0, sizeof(pool->log_root)); + strncpy(pool->log_root, log_root, sizeof(pool->log_root) - 1); + pool->num_used = 0; + pool->num_alloc = MP_POOL_TASK_MAX; + + // Create the log directory + if (mkdirs(log_root, 0700) < 0) { + if (errno != EEXIST) { + perror(log_root); + mp_pool_free(&pool); + return NULL; + } + } + + // Task array is shared with children + pool->task = mmap(NULL, (pool->num_alloc + 1) * sizeof(*pool->task), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); + if (pool->task == MAP_FAILED) { + perror("mmap"); + mp_pool_free(&pool); + return NULL; + } + + return pool; +} + +void mp_pool_free(struct MultiProcessingPool **pool) { + for (size_t i = 0; i < (*pool)->num_alloc; i++) { + } + // Unmap all pool tasks + if ((*pool)->task) { + if ((*pool)->task->cmd) { + if (munmap((*pool)->task->cmd, (*pool)->task->cmd_len) < 0) { + perror("munmap"); + } + } + if (munmap((*pool)->task, sizeof(*(*pool)->task) * (*pool)->num_alloc) < 0) { + perror("munmap"); + } + } + // Unmap the pool + if ((*pool)) { + if (munmap((*pool), sizeof(*(*pool))) < 0) { + perror("munmap"); + } + (*pool) = NULL; + } +} \ No newline at end of file diff --git a/src/lib/core/recipe.c b/src/lib/core/recipe.c new file mode 100644 index 0000000..833908c --- /dev/null +++ b/src/lib/core/recipe.c @@ -0,0 +1,64 @@ +#include "recipe.h" + +int recipe_clone(char *recipe_dir, char *url, char *gitref, char **result) { + struct Process proc; + char destdir[PATH_MAX]; + char *reponame = NULL; + + memset(&proc, 0, sizeof(proc)); + memset(destdir, 0, sizeof(destdir)); + reponame = path_basename(url); + + sprintf(destdir, "%s/%s", recipe_dir, reponame); + if (!*result) { + *result = calloc(PATH_MAX, sizeof(*result)); + if (!*result) { + return -1; + } + } + strncpy(*result, destdir, PATH_MAX); + + if (!access(destdir, F_OK)) { + if (!strcmp(destdir, "/")) { + fprintf(stderr, "STASIS is misconfigured. Please check your output path(s) immediately.\n"); + fprintf(stderr, "recipe_dir = '%s'\nreponame = '%s'\ndestdir = '%s'\n", + recipe_dir, reponame, destdir); + exit(1); + } + if (rmtree(destdir)) { + guard_free(*result); + *result = NULL; + return -1; + } + } + return git_clone(&proc, url, destdir, gitref); +} + + +int recipe_get_type(char *repopath) { + int result; + char path[PATH_MAX]; + // conda-forge is a collection of repositories + // "conda-forge.yml" is guaranteed to exist + const char *marker[] = { + "conda-forge.yml", + "stsci", + "meta.yaml", + NULL + }; + const int type[] = { + RECIPE_TYPE_CONDA_FORGE, + RECIPE_TYPE_ASTROCONDA, + RECIPE_TYPE_GENERIC + }; + + for (size_t i = 0; marker[i] != NULL; i++) { + sprintf(path, "%s/%s", repopath, marker[i]); + result = access(path, F_OK); + if (!result) { + return type[i]; + } + } + + return RECIPE_TYPE_UNKNOWN; +} \ No newline at end of file diff --git a/src/lib/core/relocation.c b/src/lib/core/relocation.c new file mode 100644 index 0000000..852aca4 --- /dev/null +++ b/src/lib/core/relocation.c @@ -0,0 +1,155 @@ +/** + * @file relocation.c + */ +#include "relocation.h" +#include "str.h" + +/** + * Replace all occurrences of `target` with `replacement` in `original` + * + * ~~~{.c} + * char *str = calloc(100, sizeof(char)); + * strcpy(str, "This are a test."); + * if (replace_text(str, "are", "is")) { + * fprintf(stderr, "string replacement failed\n"); + * exit(1); + * } + * // str is: "This is a test." + * free(str); + * ~~~ + * + * @param original string to modify + * @param target string value to replace + * @param replacement string value + * @return 0 on success, -1 on error + */ +int replace_text(char *original, const char *target, const char *replacement, unsigned flags) { + char buffer[STASIS_BUFSIZ]; + char *pos = original; + char *match = NULL; + size_t original_len = strlen(original); + size_t target_len = strlen(target); + size_t rep_len = strlen(replacement); + size_t buffer_len = 0; + + if (original_len > sizeof(buffer)) { + errno = EINVAL; + SYSERROR("The original string is larger than buffer: %zu > %zu\n", original_len, sizeof(buffer)); + return -1; + } + + memset(buffer, 0, sizeof(buffer)); + if ((match = strstr(pos, target))) { + while (*pos != '\0') { + // append to buffer the bytes leading up to the match + strncat(buffer, pos, match - pos); + // position in the string jump ahead to the beginning of the match + pos = match; + + // replacement is shorter than the target + if (rep_len < target_len) { + // shrink the string + strcat(buffer, replacement); + memmove(pos, pos + target_len, strlen(pos) - target_len); + memset(pos + (strlen(pos) - target_len), 0, target_len); + } else { // replacement is longer than the target + // write the replacement value to the buffer + strcat(buffer, replacement); + // target consumed. jump to the end of the substring. + pos += target_len; + } + if (flags & REPLACE_TRUNCATE_AFTER_MATCH) { + if (strstr(pos, LINE_SEP)) { + strcat(buffer, LINE_SEP); + } + break; + } + // find more matches + if (!(match = strstr(pos, target))) { + // no more matches + // append whatever remains to the buffer + strcat(buffer, pos); + // stop + break; + } + } + } else { + return 0; + } + + buffer_len = strlen(buffer); + if (buffer_len < original_len) { + // truncate whatever remains of the original buffer + memset(original + buffer_len, 0, original_len - buffer_len); + } + // replace original with contents of buffer + strcpy(original, buffer); + return 0; +} + +/** + * Replace `target` with `replacement` in `filename` + * + * ~~~{.c} + * if (file_replace_text("/path/to/file.txt", "are", "is")) { + * fprintf(stderr, "failed to replace strings in file\n"); + * exit(1); + * } + * ~~~ + * + * @param filename path to file + * @param target string value to replace + * @param replacement string + * @return 0 on success, -1 on error + */ +int file_replace_text(const char* filename, const char* target, const char* replacement, unsigned flags) { + int result; + char buffer[STASIS_BUFSIZ]; + char tempfilename[] = "tempfileXXXXXX"; + FILE *fp; + FILE *tfp; + + fp = fopen(filename, "r"); + if (!fp) { + fprintf(stderr, "unable to open for reading: %s\n", filename); + return -1; + } + + tfp = fopen(tempfilename, "w+"); + if (!tfp) { + SYSERROR("unable to open temporary fp for writing: %s", tempfilename); + fclose(fp); + return -1; + } + + // Write modified strings to temporary file + result = 0; + while (fgets(buffer, sizeof(buffer), fp) != NULL) { + if (strstr(buffer, target)) { + if (replace_text(buffer, target, replacement, flags)) { + result = -1; + } + } + fputs(buffer, tfp); + } + fflush(tfp); + + // Replace original with modified copy + fclose(fp); + fp = fopen(filename, "w+"); + if (!fp) { + SYSERROR("unable to reopen %s for writing", filename); + return -1; + } + + // Update original file + rewind(tfp); + while (fgets(buffer, sizeof(buffer), tfp) != NULL) { + fputs(buffer, fp); + } + fclose(fp); + fclose(tfp); + + remove(tempfilename); + return result; +} \ No newline at end of file diff --git a/src/lib/core/rules.c b/src/lib/core/rules.c new file mode 100644 index 0000000..e42ee07 --- /dev/null +++ b/src/lib/core/rules.c @@ -0,0 +1,5 @@ +// +// Created by jhunk on 12/18/23. +// + +#include "rules.h" diff --git a/src/lib/core/str.c b/src/lib/core/str.c new file mode 100644 index 0000000..868a6c7 --- /dev/null +++ b/src/lib/core/str.c @@ -0,0 +1,654 @@ +/** + * @file strings.c + */ +#include +#include "str.h" + +int num_chars(const char *sptr, int ch) { + int result = 0; + for (int i = 0; sptr[i] != '\0'; i++) { + if (sptr[i] == ch) { + result++; + } + } + return result; +} + +int startswith(const char *sptr, const char *pattern) { + if (!sptr || !pattern) { + return 0; + } + for (size_t i = 0; i < strlen(pattern); i++) { + if (sptr[i] != pattern[i]) { + return 0; + } + } + return 1; +} + +int endswith(const char *sptr, const char *pattern) { + if (!sptr || !pattern) { + return 0; + } + ssize_t sptr_size = (ssize_t) strlen(sptr); + ssize_t pattern_size = (ssize_t) strlen(pattern); + + if (sptr_size == pattern_size) { + if (strcmp(sptr, pattern) == 0) { + return 1; // yes + } + return 0; // no + } + + ssize_t s = sptr_size - pattern_size; + if (s < 0) { + return 0; + } + + for (size_t p = 0 ; s < sptr_size; s++, p++) { + if (sptr[s] != pattern[p]) { + // sptr does not end with pattern + return 0; + } + } + // sptr ends with pattern + return 1; +} + +void strchrdel(char *sptr, const char *chars) { + if (sptr == NULL || chars == NULL) { + return; + } + + for (size_t i = 0; i < strlen(chars); i++) { + char ch[2] = {0}; + strncpy(ch, &chars[i], 1); + replace_text(sptr, ch, "", 0); + } +} + +char** split(char *_sptr, const char* delim, size_t max) +{ + if (_sptr == NULL || delim == NULL) { + return NULL; + } + size_t split_alloc = 0; + // Duplicate the input string and save a copy of the pointer to be freed later + char *orig = _sptr; + char *sptr = strdup(orig); + + if (!sptr) { + return NULL; + } + + // Determine how many delimiters are present + for (size_t i = 0; i < strlen(delim); i++) { + if (max && i > max) { + break; + } + split_alloc += num_chars(sptr, delim[i]); + } + + // Preallocate enough records based on the number of delimiters + char **result = calloc(split_alloc + 2, sizeof(result[0])); + if (!result) { + guard_free(sptr); + return NULL; + } + + // No delimiter, but the string was not NULL, so return the original string + if (split_alloc == 0) { + result[0] = sptr; + return result; + } + + // Separate the string into individual parts and store them in the result array + char *token = NULL; + char *sptr_tmp = sptr; + size_t pos = 0; + size_t i; + for (i = 0; (token = strsep(&sptr_tmp, delim)) != NULL; i++) { + // When max is zero, record all tokens + if (max > 0 && i == max) { + // Maximum number of splits occurred. + // Record position in string + pos = token - sptr; + break; + } + result[i] = calloc(STASIS_BUFSIZ, sizeof(char)); + if (!result[i]) { + return NULL; + } + strcpy(result[i], token); + } + + // pos is non-zero when maximum split is reached + if (pos) { + // append the remaining string contents to array + result[i] = calloc(STASIS_BUFSIZ, sizeof(char)); + if (!result[i]) { + return NULL; + } + strcpy(result[i], &orig[pos]); + } + + guard_free(sptr); + return result; +} + +char *join(char **arr, const char *separator) { + char *result = NULL; + int records = 0; + size_t total_bytes = 0; + + if (!arr || !separator) { + return NULL; + } + + for (int i = 0; arr[i] != NULL; i++) { + total_bytes += strlen(arr[i]); + records++; + } + total_bytes += (records * strlen(separator)) + 1; + + result = (char *)calloc(total_bytes, sizeof(char)); + for (int i = 0; i < records; i++) { + strcat(result, arr[i]); + if (i < (records - 1)) { + strcat(result, separator); + } + } + return result; +} + +char *join_ex(char *separator, ...) { + va_list ap; // Variadic argument list + size_t separator_len = 0; // Length of separator string + size_t size = 0; // Length of output string + size_t argc = 0; // Number of arguments ^ "..." + char **argv = NULL; // Arguments + char *current = NULL; // Current argument + char *result = NULL; // Output string + + if (separator == NULL) { + return NULL; + } + + // Initialize array + argv = calloc(argc + 1, sizeof(char **)); + if (argv == NULL) { + perror("join_ex calloc failed"); + return NULL; + } + + // Get length of the separator + separator_len = strlen(separator); + + // Process variadic arguments: + // 1. Iterate over argument list `ap` + // 2. Assign `current` with the value of argument in `ap` + // 3. Extend the `argv` array by the latest argument count `argc` + // 4. Sum the length of the argument and the `separator` passed to the function + // 5. Append `current` string to `argv` array + // 6. Update argument counter `argc` + va_start(ap, separator); + for(argc = 0; (current = va_arg(ap, char *)) != NULL; argc++) { + char **tmp = realloc(argv, (argc + 1) * sizeof(char *)); + if (tmp == NULL) { + perror("join_ex realloc failed"); + guard_free(argv); + return NULL; + } else { + argv = tmp; + } + size += strlen(current) + separator_len; + argv[argc] = strdup(current); + } + va_end(ap); + + // Generate output string + result = calloc(size + 1, sizeof(char)); + for (size_t i = 0; i < argc; i++) { + // Append argument to string + strcat(result, argv[i]); + + // Do not append a trailing separator when we reach the last argument + if (i < (argc - 1)) { + strcat(result, separator); + } + guard_free(argv[i]); + } + guard_free(argv); + + return result; +} + +char *substring_between(char *sptr, const char *delims) { + char delim_open[255] = {0}; + char delim_close[255] = {0}; + if (sptr == NULL || delims == NULL) { + return NULL; + } + + // Ensure we have enough delimiters to continue + size_t delim_count = strlen(delims); + if (delim_count < 2 || delim_count % 2 || (delim_count > (sizeof(delim_open) - 1)) != 0) { + return NULL; + } + size_t delim_take = delim_count / 2; + + // How else am I supposed to consume the first and last n chars of the string? Give me a break. + // warning: ‘__builtin___strncpy_chk’ specified bound depends on the length of the source argument + // --- + //strncpy(delim_open, delims, delim_take); + size_t i = 0; + while (i < delim_take && i < sizeof(delim_open)) { + delim_open[i] = delims[i]; + i++; + } + + //strncpy(delim_close, &delims[delim_take], delim_take); + i = 0; + while (i < delim_take && i < sizeof(delim_close)) { + delim_close[i] = delims[i + delim_take]; + i++; + } + + // Create pointers to the delimiters + char *start = strstr(sptr, delim_open); + if (start == NULL || strlen(start) == 0) { + return NULL; + } + + char *end = strstr(start + 1, delim_close); + if (end == NULL) { + return NULL; + } + + start += delim_count / 2; // ignore leading delimiter + + // Get length of the substring + size_t length = strlen(start) - strlen(end); + if (!length) { + return NULL; + } + + // Return the contents of the substring + return strndup(start, length); +} + +/* + * Comparison functions for `strsort` + */ +static int _strsort_alpha_compare(const void *a, const void *b) { + const char *aa = *(const char **)a; + const char *bb = *(const char **)b; + int result = strcmp(aa, bb); + return result; +} + +static int _strsort_numeric_compare(const void *a, const void *b) { + const char *aa = *(const char **)a; + const char *bb = *(const char **)b; + + if (isdigit(*aa) && isdigit(*bb)) { + long ia = strtol(aa, NULL, 10); + long ib = strtol(bb, NULL, 10); + + if (ia == ib) { + return 0; + } else if (ia < ib) { + return -1; + } else if (ia > ib) { + return 1; + } + } + return 0; +} + +static int _strsort_asc_compare(const void *a, const void *b) { + const char *aa = *(const char**)a; + const char *bb = *(const char**)b; + size_t len_a = strlen(aa); + size_t len_b = strlen(bb); + return len_a > len_b; +} + +/* + * Helper function for `strsortlen` + */ +static int _strsort_dsc_compare(const void *a, const void *b) { + const char *aa = *(const char**)a; + const char *bb = *(const char**)b; + size_t len_a = strlen(aa); + size_t len_b = strlen(bb); + return len_a < len_b; +} + +void strsort(char **arr, unsigned int sort_mode) { + if (arr == NULL) { + return; + } + + typedef int (*compar)(const void *, const void *); + // Default mode is alphabetic sort + compar fn = _strsort_alpha_compare; + + if (sort_mode == STASIS_SORT_LEN_DESCENDING) { + fn = _strsort_dsc_compare; + } else if (sort_mode == STASIS_SORT_LEN_ASCENDING) { + fn = _strsort_asc_compare; + } else if (sort_mode == STASIS_SORT_ALPHA) { + fn = _strsort_alpha_compare; // ^ still selectable though ^ + } else if (sort_mode == STASIS_SORT_NUMERIC) { + fn = _strsort_numeric_compare; + } + + size_t arr_size = 0; + + // Determine size of array (+ terminator) + for (size_t i = 0; arr[i] != NULL; i++) { + arr_size = i; + } + arr_size++; + + qsort(arr, arr_size, sizeof(char *), fn); +} + + +char *strstr_array(char **arr, const char *str) { + if (arr == NULL || str == NULL) { + return NULL; + } + + for (int i = 0; arr[i] != NULL; i++) { + if (strstr(arr[i], str) != NULL) { + return arr[i]; + } + } + return NULL; +} + + +char **strdeldup(char **arr) { + if (!arr) { + return NULL; + } + + size_t records; + // Determine the length of the array + for (records = 0; arr[records] != NULL; records++); + + // Allocate enough memory to store the original array contents + // (It might not have duplicate values, for example) + char **result = (char **)calloc(records + 1, sizeof(char *)); + if (!result) { + return NULL; + } + + int rec = 0; + size_t i = 0; + while(i < records) { + // Search for value in results + if (strstr_array(result, arr[i]) != NULL) { + // value already exists in results so ignore it + i++; + continue; + } + + // Store unique value + result[rec] = strdup(arr[i]); + if (!result[rec]) { + for (size_t die = 0; result[die] != NULL; die++) { + guard_free(result[die]); + } + guard_free(result); + return NULL; + } + + i++; + rec++; + } + return result; +} + +char *lstrip(char *sptr) { + char *tmp = sptr; + size_t bytes = 0; + + if (sptr == NULL) { + return NULL; + } + + while (strlen(tmp) > 1 && (isblank(*tmp) || isspace(*tmp))) { + bytes++; + tmp++; + } + if (tmp != sptr) { + memmove(sptr, sptr + bytes, strlen(sptr) - bytes); + memset((sptr + strlen(sptr)) - bytes, '\0', bytes); + } + return sptr; +} + +char *strip(char *sptr) { + if (sptr == NULL) { + return NULL; + } + + size_t len = strlen(sptr); + if (len == 0) { + return sptr; + } else if (len == 1 && (isblank(*sptr) || isspace(*sptr))) { + *sptr = '\0'; + return sptr; + } for (size_t i = len; i != 0; --i) { + if (sptr[i] == '\0') { + continue; + } + if (isspace(sptr[i]) || isblank(sptr[i])) { + sptr[i] = '\0'; + } else { + break; + } + } + return sptr; +} + +int isempty(char *sptr) { + if (sptr == NULL) { + return -1; + } + + char *tmp = sptr; + while (*tmp) { + if (!isblank(*tmp) && !isspace(*tmp) && !iscntrl(*tmp)) { + return 0; + } + tmp++; + } + return 1; +} + + +int isquoted(char *sptr) { + const char *quotes = "'\""; + + if (sptr == NULL) { + return -1; + } + + char *quote_open = strpbrk(sptr, quotes); + if (!quote_open) { + return 0; + } + char *quote_close = strpbrk(quote_open + 1, quotes); + if (!quote_close) { + return 0; + } + return 1; +} + +int isrelational(char ch) { + char symbols[] = "~!=<>"; + char *symbol = symbols; + while (*symbol != '\0') { + if (ch == *symbol) { + return 1; + } + symbol++; + } + return 0; +} + +void print_banner(const char *s, int len) { + size_t s_len = strlen(s); + if (!s_len) { + return; + } + for (size_t i = 0; i < (len / s_len); i++) { + for (size_t c = 0; c < s_len; c++) { + putchar(s[c]); + } + } + putchar('\n'); +} + +/** + * Collapse whitespace in `s`. The string is modified in place. + * @param s + * @return pointer to `s` + */ +char *normalize_space(char *s) { + size_t len; + size_t trim_pos; + int add_whitespace = 0; + char *result = s; + char *tmp; + + if (s == NULL) { + return NULL; + } + + if ((tmp = calloc(strlen(s) + 1, sizeof(char))) == NULL) { + perror("could not allocate memory for temporary string"); + return NULL; + } + char *tmp_orig = tmp; + + // count whitespace, if any + for (trim_pos = 0; isblank(s[trim_pos]); trim_pos++); + // trim whitespace from the left, if any + memmove(s, &s[trim_pos], strlen(&s[trim_pos])); + // cull bytes not part of the string after moving + len = strlen(s); + s[len - trim_pos] = '\0'; + + // Generate a new string with extra whitespace stripped out + while (*s != '\0') { + // Skip over any whitespace, but record that we encountered it + if (isblank(*s)) { + s++; + add_whitespace = 1; + continue; + } + // This gate avoids filling tmp with whitespace; we want to make our own + if (add_whitespace) { + *tmp = ' '; + tmp++; + add_whitespace = 0; + } + // Write character in s to tmp + *tmp = *s; + // Increment string pointers + s++; + tmp++; + } + + // Rewrite the input string + strcpy(result, tmp_orig); + guard_free(tmp_orig); + return result; +} + +char **strdup_array(char **array) { + char **result = NULL; + size_t elems = 0; + + // Guard + if (array == NULL) { + return NULL; + } + + // Count elements in `array` + for (elems = 0; array[elems] != NULL; elems++); + + // Create new array + result = calloc(elems + 1, sizeof(*result)); + for (size_t i = 0; i < elems; i++) { + result[i] = strdup(array[i]); + } + + return result; +} + +int strcmp_array(const char **a, const char **b) { + size_t a_len = 0; + size_t b_len = 0; + + // This could lead to false-positives depending on what the caller plans to achieve + if (a == NULL && b == NULL) { + return 0; + } else if (a == NULL) { + return -1; + } else if (b == NULL) { + return 1; + } + + // Get length of arrays + for (a_len = 0; a[a_len] != NULL; a_len++); + for (b_len = 0; b[b_len] != NULL; b_len++); + + // Check lengths are equal + if (a_len < b_len) return (int)(b_len - a_len); + else if (a_len > b_len) return (int)(a_len - b_len); + + // Compare strings in the arrays returning the total difference in bytes + int result = 0; + for (size_t ai = 0, bi = 0 ;a[ai] != NULL || b[bi] != NULL; ai++, bi++) { + int status = 0; + if ((status = strcmp(a[ai], b[bi]) != 0)) { + result += status; + } + } + return result; +} + +int isdigit_s(const char *s) { + if (!s || !strlen(s)) { + return 0; // nothing to do, fail + } + for (size_t i = 0; s[i] != '\0'; i++) { + if (isdigit(s[i]) == 0) { + return 0; // non-digit found, fail + } + } + return 1; // all digits, succeed +} + +char *tolower_s(char *s) { + for (size_t i = 0; s[i] != '\0'; i++) { + s[i] = (char)tolower(s[i]); + } + return s; +} + +char *to_short_version(const char *s) { + char *result; + result = strdup(s); + if (!result) { + return NULL; + } + strchrdel(result, "."); + return result; +} diff --git a/src/lib/core/strlist.c b/src/lib/core/strlist.c new file mode 100644 index 0000000..f0bffa8 --- /dev/null +++ b/src/lib/core/strlist.c @@ -0,0 +1,659 @@ +/** + * String array convenience functions + * @file strlist.c + */ +#include "download.h" +#include "strlist.h" +#include "utils.h" + +/** + * + * @param pStrList `StrList` + */ +void strlist_free(struct StrList **pStrList) { + if (!(*pStrList)) { + return; + } + + for (size_t i = 0; i < (*pStrList)->num_inuse; i++) { + if ((*pStrList)->data[i]) { + guard_free((*pStrList)->data[i]); + } + } + if ((*pStrList)->data) { + guard_free((*pStrList)->data); + } + guard_free((*pStrList)); +} + +/** + * Append a value to the list + * @param pStrList `StrList` + * @param str + */ +void strlist_append(struct StrList **pStrList, char *str) { + char **tmp = NULL; + + if (pStrList == NULL) { + return; + } + + tmp = realloc((*pStrList)->data, ((*pStrList)->num_alloc + 1) * sizeof(char *)); + if (tmp == NULL) { + guard_strlist_free(pStrList); + perror("failed to append to array"); + exit(1); + } else if (tmp != (*pStrList)->data) { + (*pStrList)->data = tmp; + } + (*pStrList)->data[(*pStrList)->num_inuse] = strdup(str); + (*pStrList)->data[(*pStrList)->num_alloc] = NULL; + strcpy((*pStrList)->data[(*pStrList)->num_inuse], str); + (*pStrList)->num_inuse++; + (*pStrList)->num_alloc++; +} + +static int reader_strlist_append_file(size_t lineno, char **line) { + (void)(lineno); // unused parameter + (void)(line); // unused parameter + return 0; +} + +/** + * Append lines from a local file or remote URL (HTTP/s only) + * @param pStrList + * @param path file path or HTTP/s address + * @param readerFn pointer to a reader function (use NULL to retrieve all data) + * @return 0=success 1=no data, -1=error (spmerrno set) + */ +int strlist_append_file(struct StrList *pStrList, char *_path, ReaderFn *readerFn) { + int retval = 0; + char *path = NULL; + char *filename = NULL; + char **data = NULL; + int is_url = strstr(_path, "://") != NULL; + + if (readerFn == NULL) { + readerFn = reader_strlist_append_file; + } + + path = strdup(_path); + if (path == NULL) { + retval = -1; + goto fatal; + } + + if (is_url) { + int fd; + char tempfile[PATH_MAX] = {0}; + strcpy(tempfile, "/tmp/.remote_file.XXXXXX"); + if ((fd = mkstemp(tempfile)) < 0) { + retval = -1; + goto fatal; + } + close(fd); + filename = strdup(tempfile); + long http_code = download(path, filename, NULL); + if (HTTP_ERROR(http_code)) { + retval = -1; + goto fatal; + } + } else { + filename = expandpath(path); + if (filename == NULL) { + retval = -1; + goto fatal; + } + } + + data = file_readlines(filename, 0, 0, readerFn); + if (data == NULL) { + retval = 1; + goto fatal; + } + for (size_t record = 0; data[record] != NULL; record++) { + strlist_append(&pStrList, data[record]); + guard_free(data[record]); + } + if (is_url) { + // remove temporary data + remove(filename); + } + guard_free(data); + +fatal: + guard_free(filename); + if (path != NULL) { + guard_free(path); + } + + return retval; +} + +/** + * Append the contents of `pStrList2` to `pStrList1` + * @param pStrList1 `StrList` + * @param pStrList2 `StrList` + */ +void strlist_append_strlist(struct StrList *pStrList1, struct StrList *pStrList2) { + size_t count = 0; + + if (pStrList1 == NULL || pStrList2 == NULL) { + return; + } + + count = strlist_count(pStrList2); + for (size_t i = 0; i < count; i++) { + char *item = strlist_item(pStrList2, i); + strlist_append(&pStrList1, item); + } +} + +/** + * Append the contents of an array of pointers to char + * @param pStrList `StrList` + * @param arr NULL terminated array of strings + */ + void strlist_append_array(struct StrList *pStrList, char **arr) { + if (!pStrList || !arr) { + return; + } + for (size_t i = 0; arr[i] != NULL; i++) { + strlist_append(&pStrList, arr[i]); + } + } + +/** + * Append the contents of a newline delimited string + * @param pStrList `StrList` + * @param str + * @param delim + */ + void strlist_append_tokenize(struct StrList *pStrList, char *str, char *delim) { + char **token; + if (!str || !delim) { + return; + } + + char *tmp = strdup(str); + token = split(tmp, delim, 0); + if (token) { + for (size_t i = 0; token[i] != NULL; i++) { + lstrip(token[i]); + strlist_append(&pStrList, token[i]); + } + GENERIC_ARRAY_FREE(token); + } + guard_free(tmp); + } + +/** + * Produce a new copy of a `StrList` + * @param pStrList `StrList` + * @return `StrList` copy + */ +struct StrList *strlist_copy(struct StrList *pStrList) { + struct StrList *result; + if (pStrList == NULL) { + return NULL; + } + + result = strlist_init(); + if (!result) { + return NULL; + } + + for (size_t i = 0; i < strlist_count(pStrList); i++) { + strlist_append(&result, strlist_item(pStrList, i)); + } + return result; +} + +/** + * Remove a record by index from a `StrList` + * @param pStrList + * @param index + */ +void strlist_remove(struct StrList *pStrList, size_t index) { + size_t count = strlist_count(pStrList); + if (count == 0) { + return; + } + if (pStrList->data[index] != NULL) { + for (size_t i = index; i < count; i++) { + pStrList->data[i] = pStrList->data[i + 1]; + } + if (pStrList->num_inuse) { + pStrList->num_inuse--; + } + } +} + +/** + * Compare two `StrList`s + * @param a `StrList` structure + * @param b `StrList` structure + * @return same=0, different=1, error=-1 (a is NULL), -2 (b is NULL) + */ +int strlist_cmp(struct StrList *a, struct StrList *b) { + if (a == NULL) { + return -1; + } + + if (b == NULL) { + return -2; + } + + if (a->num_alloc != b->num_alloc || a->num_inuse != b->num_inuse) { + return 1; + } + + + for (size_t i = 0; i < strlist_count(a); i++) { + if (strcmp(strlist_item(a, i), strlist_item(b, i)) != 0) { + return 1; + } + } + + return 0; +} + +/** + * Sort a `StrList` by `mode` + * @param pStrList + * @param mode Available modes: `STRLIST_DEFAULT` (alphabetic), `STRLIST_ASC` (ascending), `STRLIST_DSC` (descending) + */ +void strlist_sort(struct StrList *pStrList, unsigned int mode) { + if (pStrList == NULL) { + return; + } + strsort(pStrList->data, mode); +} + +/** + * Reverse the order of a `StrList` + * @param pStrList + */ +void strlist_reverse(struct StrList *pStrList) { + char *tmp = NULL; + size_t i = 0; + size_t j = 0; + + if (pStrList == NULL) { + return; + } + + j = pStrList->num_inuse - 1; + for (i = 0; i < j; i++) { + tmp = pStrList->data[i]; + pStrList->data[i] = pStrList->data[j]; + pStrList->data[j] = tmp; + j--; + } +} + +/** + * Get the count of values stored in a `StrList` + * @param pStrList + * @return + */ +size_t strlist_count(struct StrList *pStrList) { + size_t result; + if (pStrList != NULL) { + result = pStrList->num_inuse; + } else { + result = 0; + } + return result; +} + +/** + * Set value at index + * @param pStrList + * @param value string + * @return + */ +void strlist_set(struct StrList **pStrList, size_t index, char *value) { + char *tmp = NULL; + if (*pStrList == NULL || index > strlist_count(*pStrList)) { + strlist_errno = STRLIST_E_OUT_OF_RANGE; + return; + } + + if (value == NULL) { + guard_free((*pStrList)->data[index]); + } else { + tmp = realloc((*pStrList)->data[index], (strlen(value) + 1) * sizeof(char *)); + if (!tmp) { + perror("realloc strlist_set replacement value"); + return; + } else if (tmp != (*pStrList)->data[index]) { + (*pStrList)->data[index] = tmp; + } + + memset((*pStrList)->data[index], '\0', strlen(value) + 1); + strcpy((*pStrList)->data[index], value); + } +} + +const char *strlist_error_msgs[] = { + "success", + "index out of range", + "invalid value for type", + "unknown error", +}; +int strlist_errno = 0; + +void strlist_set_error(int flag) { + strlist_errno = flag; +} + +const char *strlist_get_error(int flag) { + if (flag < STRLIST_E_SUCCESS || flag > STRLIST_E_UNKNOWN) { + return strlist_error_msgs[STRLIST_E_UNKNOWN]; + } + return strlist_error_msgs[flag]; +} + +void strlist_clear_error() { + strlist_errno = STRLIST_E_SUCCESS; +} + +/** + * Retrieve data from a `StrList` + * @param pStrList + * @param index + * @return string + */ +char *strlist_item(struct StrList *pStrList, size_t index) { + if (pStrList && pStrList->data && pStrList->data[index]) { + return pStrList->data[index]; + } + return NULL; +} + +/** + * Alias of `strlist_item` + * @param pStrList + * @param index + * @return string + */ +char *strlist_item_as_str(struct StrList *pStrList, size_t index) { + return strlist_item(pStrList, index); +} + +/** + * Convert value at index to `char` + * @param pStrList + * @param index + * @return `char` + */ +char strlist_item_as_char(struct StrList *pStrList, size_t index) { + char *error_p; + char result; + + strlist_clear_error(); + result = (char) strtol(strlist_item(pStrList, index), &error_p, 10); + if (!result && error_p && *error_p != 0) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return 0; + } + error_p = NULL; + return result; +} + +/** + * Convert value at index to `unsigned char` + * @param pStrList + * @param index + * @return `unsigned char` + */ +unsigned char strlist_item_as_uchar(struct StrList *pStrList, size_t index) { + char *error_p; + unsigned char result; + + strlist_clear_error(); + result = (unsigned char) strtoul(strlist_item(pStrList, index), &error_p, 10); + if (!result && error_p && *error_p != 0) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return 0; + } + error_p = NULL; + return result; +} + +/** + * Convert value at index to `short` + * @param pStrList + * @param index + * @return `short` + */ +short strlist_item_as_short(struct StrList *pStrList, size_t index) { + char *error_p; + short result; + + strlist_clear_error(); + result = (short) strtol(strlist_item(pStrList, index), &error_p, 10); + if (!result && error_p && *error_p != 0) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return 0; + } + error_p = NULL; + return result; +} + +/** + * Convert value at index to `unsigned short` + * @param pStrList + * @param index + * @return `unsigned short` + */ +unsigned short strlist_item_as_ushort(struct StrList *pStrList, size_t index) { + char *error_p; + unsigned short result; + + strlist_clear_error(); + result = (unsigned short) strtoul(strlist_item(pStrList, index), &error_p, 10); + if (!result && error_p && *error_p != 0) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return 0; + } + error_p = NULL; + return result; +} + +/** + * Convert value at index to `int` + * @param pStrList + * @param index + * @return `int` + */ +int strlist_item_as_int(struct StrList *pStrList, size_t index) { + char *error_p; + int result; + + strlist_clear_error(); + result = (int) strtol(strlist_item(pStrList, index), &error_p, 10); + if (!result && error_p && *error_p != 0) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return 0; + } + error_p = NULL; + return result; +} + +/** + * Convert value at index to `unsigned int` + * @param pStrList + * @param index + * @return `unsigned int` + */ +unsigned int strlist_item_as_uint(struct StrList *pStrList, size_t index) { + char *error_p; + unsigned int result; + + strlist_clear_error(); + result = (unsigned int) strtoul(strlist_item(pStrList, index), &error_p, 10); + if (!result && error_p && *error_p != 0) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return 0; + } + error_p = NULL; + return result; +} + +/** + * Convert value at index to `long` + * @param pStrList + * @param index + * @return `long` + */ +long strlist_item_as_long(struct StrList *pStrList, size_t index) { + char *error_p; + long result; + + strlist_clear_error(); + result = (long) strtol(strlist_item(pStrList, index), &error_p, 10); + if (!result && error_p && *error_p != 0) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return 0; + } + error_p = NULL; + return result; +} + +/** + * Convert value at index to `unsigned long` + * @param pStrList + * @param index + * @return `unsigned long` + */ +unsigned long strlist_item_as_ulong(struct StrList *pStrList, size_t index) { + char *error_p; + unsigned long result; + + strlist_clear_error(); + result = (unsigned long) strtoul(strlist_item(pStrList, index), &error_p, 10); + if (!result && error_p && *error_p != 0) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return 0; + } + error_p = NULL; + return result; +} + +/** + * Convert value at index to `long long` + * @param pStrList + * @param index + * @return `long long` + */ +long long strlist_item_as_long_long(struct StrList *pStrList, size_t index) { + char *error_p; + long long result; + + strlist_clear_error(); + result = (long long) strtoll(strlist_item(pStrList, index), &error_p, 10); + if (!result && error_p && *error_p != 0) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return 0; + } + error_p = NULL; + return result; +} + +/** + * Convert value at index to `unsigned long long` + * @param pStrList + * @param index + * @return `unsigned long long` + */ +unsigned long long strlist_item_as_ulong_long(struct StrList *pStrList, size_t index) { + char *error_p; + unsigned long long result; + + strlist_clear_error(); + result = (unsigned long long) strtol(strlist_item(pStrList, index), &error_p, 10); + if (!result && error_p && *error_p != 0) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return 0; + } + error_p = NULL; + return result; +} + +/** + * Convert value at index to `float` + * @param pStrList + * @param index + * @return `float` + */ +float strlist_item_as_float(struct StrList *pStrList, size_t index) { + char *error_p; + float result; + + strlist_clear_error(); + result = (float) strtof(strlist_item(pStrList, index), &error_p); + if (!result && error_p && *error_p != 0) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return 0; + } + error_p = NULL; + return result; +} + +/** + * Convert value at index to `double` + * @param pStrList + * @param index + * @return `double` + */ +double strlist_item_as_double(struct StrList *pStrList, size_t index) { + char *error_p; + double result; + + strlist_clear_error(); + result = (double) strtod(strlist_item(pStrList, index), &error_p); + if (!result && error_p && *error_p != 0) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return 0; + } + error_p = NULL; + return result; +} + +/** + * Convert value at index to `long double` + * @param pStrList + * @param index + * @return `long double` + */ +long double strlist_item_as_long_double(struct StrList *pStrList, size_t index) { + char *error_p; + long double result; + + strlist_clear_error(); + result = (long double) strtold(strlist_item(pStrList, index), &error_p); + if (!result && error_p && *error_p != 0) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return 0; + } + error_p = NULL; + return result; +} + +/** + * Initialize an empty `StrList` + * @return `StrList` + */ +struct StrList *strlist_init() { + struct StrList *pStrList = calloc(1, sizeof(struct StrList)); + if (pStrList == NULL) { + perror("failed to allocate array"); + return NULL; + } + pStrList->num_inuse = 0; + pStrList->num_alloc = 1; + pStrList->data = calloc(pStrList->num_alloc, sizeof(char *)); + return pStrList; +} diff --git a/src/lib/core/system.c b/src/lib/core/system.c new file mode 100644 index 0000000..4e605ec --- /dev/null +++ b/src/lib/core/system.c @@ -0,0 +1,173 @@ +#include "system.h" +#include "core.h" + +int shell(struct Process *proc, char *args) { + struct Process selfproc; + pid_t pid; + pid_t status; + status = 0; + errno = 0; + + if (!proc) { + // provide our own proc structure + // albeit not accessible to the user + memset(&selfproc, 0, sizeof(selfproc)); + proc = &selfproc; + } + + if (!args) { + proc->returncode = -1; + return -1; + } + + FILE *tp = NULL; + char *t_name; + t_name = xmkstemp(&tp, "w"); + if (!t_name || !tp) { + return -1; + } + + fprintf(tp, "#!/bin/bash\n%s\n", args); + fflush(tp); + fclose(tp); + + // Set the script's permissions so that only the calling user can use it + // This should help prevent eavesdropping if keys are applied in plain-text + // somewhere. + chmod(t_name, 0700); + + pid = fork(); + if (pid == -1) { + fprintf(stderr, "fork failed\n"); + exit(1); + } else if (pid == 0) { + FILE *fp_out = NULL; + FILE *fp_err = NULL; + + if (strlen(proc->f_stdout)) { + fp_out = freopen(proc->f_stdout, "w+", stdout); + if (!fp_out) { + fprintf(stderr, "Unable to redirect stdout to %s: %s\n", proc->f_stdout, strerror(errno)); + exit(1); + } + } + + if (strlen(proc->f_stderr)) { + if (!proc->redirect_stderr) { + fp_err = freopen(proc->f_stderr, "w+", stderr); + if (!fp_err) { + fprintf(stderr, "Unable to redirect stderr to %s: %s\n", proc->f_stdout, strerror(errno)); + exit(1); + } + } + } + + if (proc->redirect_stderr) { + if (fp_err) { + fclose(fp_err); + fclose(stderr); + } + if (dup2(fileno(stdout), fileno(stderr)) < 0) { + fprintf(stderr, "Unable to redirect stderr to stdout: %s\n", strerror(errno)); + exit(1); + } + } + + return execl("/bin/bash", "bash", "--norc", t_name, (char *) NULL); + } else { + if (waitpid(pid, &status, WUNTRACED) > 0) { + if (WIFEXITED(status) && WEXITSTATUS(status)) { + if (WEXITSTATUS(status) == 127) { + fprintf(stderr, "execv failed\n"); + } + } else if (WIFSIGNALED(status)) { + fprintf(stderr, "signal received: %d\n", WIFSIGNALED(status)); + } + } else { + fprintf(stderr, "waitpid() failed\n"); + } + } + + if (!access(t_name, F_OK)) { + remove(t_name); + } + + proc->returncode = status; + guard_free(t_name); + return WEXITSTATUS(status); +} + +int shell_safe(struct Process *proc, char *args) { + FILE *fp; + char buf[1024] = {0}; + int result; + + char *invalid_ch = strpbrk(args, STASIS_SHELL_SAFE_RESTRICT); + if (invalid_ch) { + args = NULL; + } + + result = shell(proc, args); + if (strlen(proc->f_stdout)) { + fp = fopen(proc->f_stdout, "r"); + if (fp) { + while (fgets(buf, sizeof(buf) - 1, fp)) { + fprintf(stdout, "%s", buf); + buf[0] = '\0'; + } + fclose(fp); + fp = NULL; + } + } + if (strlen(proc->f_stderr)) { + fp = fopen(proc->f_stderr, "r"); + if (fp) { + while (fgets(buf, sizeof(buf) - 1, fp)) { + fprintf(stderr, "%s", buf); + buf[0] = '\0'; + } + fclose(fp); + fp = NULL; + } + } + return result; +} + +char *shell_output(const char *command, int *status) { + const size_t initial_size = STASIS_BUFSIZ; + size_t current_size = initial_size; + char *result = NULL; + char line[STASIS_BUFSIZ]; + FILE *pp; + + errno = 0; + *status = 0; + pp = popen(command, "r"); + if (!pp) { + *status = -1; + return NULL; + } + + if (errno) { + *status = 1; + } + result = calloc(initial_size, sizeof(result)); + memset(line, 0, sizeof(line)); + while (fread(line, sizeof(char), sizeof(line) - 1, pp) != 0) { + size_t result_len = strlen(result); + size_t need_realloc = (result_len + strlen(line)) > current_size; + if (need_realloc) { + current_size += initial_size; + char *tmp = realloc(result, sizeof(*result) * current_size); + if (!tmp) { + return NULL; + } else if (tmp != result) { + result = tmp; + } + } + strcat(result, line); + memset(line, 0, sizeof(line)); + } + *status = pclose(pp); + return result; +} diff --git a/src/lib/core/template.c b/src/lib/core/template.c new file mode 100644 index 0000000..a412fa8 --- /dev/null +++ b/src/lib/core/template.c @@ -0,0 +1,318 @@ +// +// Created by jhunk on 12/17/23. +// + +#include "template.h" + +#include +#include +#include +#include + + +struct tpl_item { + char *key; + char **ptr; +}; +struct tpl_item *tpl_pool[1024] = {0}; +unsigned tpl_pool_used = 0; +struct tplfunc_frame *tpl_pool_func[1024] = {0}; +unsigned tpl_pool_func_used = 0; + +extern void tpl_reset() { + tpl_free(); + tpl_pool_used = 0; + tpl_pool_func_used = 0; +} + +void tpl_register_func(char *key, void *tplfunc_ptr, int argc, void *data_in) { + struct tplfunc_frame *frame = calloc(1, sizeof(*frame)); + frame->key = strdup(key); + frame->argc = argc; + frame->func = tplfunc_ptr; + frame->data_in = data_in; + + tpl_pool_func[tpl_pool_func_used] = frame; + tpl_pool_func_used++; +} + +int tpl_key_exists(char *key) { + for (size_t i = 0; i < tpl_pool_used; i++) { + if (tpl_pool[i]->key) { + if (!strcmp(tpl_pool[i]->key, key)) { + return true; + } + } + } + return false; +} + +void tpl_register(char *key, char **ptr) { + struct tpl_item *item = NULL; + int replacing = 0; + + if (tpl_key_exists(key)) { + for (size_t i = 0; i < tpl_pool_used; i++) { + if (tpl_pool[i]->key) { + if (!strcmp(tpl_pool[i]->key, key)) { + item = tpl_pool[i]; + break; + } + } + } + replacing = 1; + } else { + item = calloc(1, sizeof(*item)); + item->key = strdup(key); + } + + if (!item) { + SYSERROR("unable to register tpl_item for %s", key); + exit(1); + } + + item->ptr = ptr; + if (!replacing) { + tpl_pool[tpl_pool_used] = item; + tpl_pool_used++; + } +} + +void tpl_free() { + for (unsigned i = 0; i < tpl_pool_used; i++) { + struct tpl_item *item = tpl_pool[i]; + if (item) { + if (item->key) { +#ifdef DEBUG + SYSERROR("freeing template item key: %s", item->key); +#endif + guard_free(item->key); + } +#ifdef DEBUG + SYSERROR("freeing template item: %p", item); +#endif + item->ptr = NULL; + } + guard_free(item); + } + for (unsigned i = 0; i < tpl_pool_func_used; i++) { + struct tplfunc_frame *item = tpl_pool_func[i]; + guard_free(item->key); + guard_free(item); + } +} + +char *tpl_getval(char *key) { + char *result = NULL; + for (size_t i = 0; i < tpl_pool_used; i++) { + if (tpl_pool[i]->key) { + if (!strcmp(tpl_pool[i]->key, key)) { + result = *tpl_pool[i]->ptr; + break; + } + } + } + return result; +} + +struct tplfunc_frame *tpl_getfunc(char *key) { + struct tplfunc_frame *result = NULL; + for (size_t i = 0; i < tpl_pool_func_used; i++) { + if (tpl_pool_func[i]->key) { + if (!strcmp(tpl_pool_func[i]->key, key)) { + result = tpl_pool_func[i]; + break; + } + } + } + return result; +} + +static int grow(size_t z, size_t *output_bytes, char **output) { + if (z >= *output_bytes) { + size_t new_size = *output_bytes + z + 1; +#ifdef DEBUG + fprintf(stderr, "template output buffer new size: %zu\n", new_size); +#endif + char *tmp = realloc(*output, new_size); + if (!tmp) { + perror("realloc failed"); + return -1; + } else if (tmp != *output) { + *output = tmp; + } + *output_bytes = new_size; + } + return 0; +} + +char *tpl_render(char *str) { + if (!str) { + return NULL; + } else if (!strlen(str)) { + return strdup(""); + } + size_t output_bytes = 1024 + strlen(str); // TODO: Is grow working correctly? + char *output = NULL; + char *b_close = NULL; + char *pos = NULL; + pos = str; + + output = calloc(output_bytes, sizeof(*output)); + if (!output) { + perror("unable to allocate output buffer"); + return NULL; + } + + for (size_t off = 0, z = 0; off < strlen(str); off++) { + char key[255] = {0}; + char *value = NULL; + + memset(key, 0, sizeof(key)); + grow(z, &output_bytes, &output); + // At opening brace + if (!strncmp(&pos[off], "{{", 2)) { + // Scan until key is reached + while (!isalnum(pos[off])) { + off++; + } + + // Read key name + size_t key_len = 0; + while (isalnum(pos[off]) || pos[off] != '}') { + if (isspace(pos[off]) || isblank(pos[off])) { + // skip whitespace in key + off++; + continue; + } + key[key_len] = pos[off]; + key_len++; + off++; + } + + char *type_stop = NULL; + type_stop = strchr(key, ':'); + + int do_env = 0; + int do_func = 0; + if (type_stop) { + if (!strncmp(key, "env", type_stop - key)) { + do_env = 1; + } else if (!strncmp(key, "func", type_stop - key)) { + do_func = 1; + } + } + + // Find closing brace + b_close = strstr(&pos[off], "}}"); + if (!b_close) { + fprintf(stderr, "error while templating '%s'\n\nunbalanced brace at position %zu\n", str, z); + return NULL; + } else { + // Jump past closing brace + off = ((b_close + 2) - pos); + } + + if (do_env) { // {{ env:VAR }} + char *k = type_stop + 1; + size_t klen = strlen(k); + memmove(key, k, klen); + key[klen] = 0; + char *env_val = getenv(key); + value = strdup(env_val ? env_val : ""); + } else if (do_func) { // {{ func:NAME(a, ...) }} + char func_name_temp[STASIS_NAME_MAX] = {0}; + strcpy(func_name_temp, type_stop + 1); + char *param_begin = strchr(func_name_temp, '('); + if (!param_begin) { + fprintf(stderr, "At position %zu in %s\nfunction name must be followed by a '('\n", off, key); + guard_free(output); + return NULL; + } + *param_begin = 0; + param_begin++; + char *param_end = strrchr(param_begin, ')'); + if (!param_end) { + fprintf(stderr, "At position %zu in %s\nfunction arguments must be closed with a ')'\n", off, key); + guard_free(output); + return NULL; + } + *param_end = 0; + char *k = func_name_temp; + char **params = split(param_begin, ",", 0); + int params_count; + for (params_count = 0; params[params_count] != NULL; params_count++); + + struct tplfunc_frame *frame = tpl_getfunc(k); + if (params_count > frame->argc || params_count < frame->argc) { + fprintf(stderr, "At position %zu in %s\nIncorrect number of arguments for function: %s (expected %d, got %d)\n", off, key, frame->key, frame->argc, params_count); + value = strdup(""); + } else { + for (size_t p = 0; p < sizeof(frame->argv) / sizeof(*frame->argv) && params[p] != NULL; p++) { + lstrip(params[p]); + strip(params[p]); + frame->argv[p].t_char_ptr = params[p]; + } + char *func_result = NULL; + int func_status = 0; + if ((func_status = frame->func(frame, &func_result))) { + fprintf(stderr, "%s returned non-zero status: %d\n", frame->key, func_status); + } + value = strdup(func_result ? func_result : ""); + guard_free(func_result); + } + GENERIC_ARRAY_FREE(params); + } else { + // Read replacement value + value = strdup(tpl_getval(key) ? tpl_getval(key) : ""); + } + } + + if (value) { + // Set output iterator to end of replacement value + z += strlen(value); + + // Append replacement value + grow(z, &output_bytes, &output); + strcat(output, value); + guard_free(value); + output[z] = 0; + } + +#ifdef DEBUG + fprintf(stderr, "z=%zu, output_bytes=%zu\n", z, output_bytes); +#endif + output[z] = pos[off]; + z++; + } +#ifdef DEBUG + fprintf(stderr, "template output length: %zu\n", strlen(output)); + fprintf(stderr, "template output bytes: %zu\n", output_bytes); +#endif + return output; +} + +int tpl_render_to_file(char *str, const char *filename) { + char *result; + FILE *fp; + + // Render the input string + result = tpl_render(str); + if (!result) { + return -1; + } + + // Open the destination file for writing + fp = fopen(filename, "w+"); + if (!fp) { + guard_free(result); + return -1; + } + + // Write rendered string to file + fprintf(fp, "%s", result); + fclose(fp); + + guard_free(result); + return 0; +} \ No newline at end of file diff --git a/src/lib/core/template_func_proto.c b/src/lib/core/template_func_proto.c new file mode 100644 index 0000000..3305b4d --- /dev/null +++ b/src/lib/core/template_func_proto.c @@ -0,0 +1,160 @@ +#include "template_func_proto.h" +#include "delivery.h" +#include "github.h" + +int get_github_release_notes_tplfunc_entrypoint(void *frame, void *data_out) { + int result; + char **output = (char **) data_out; + struct tplfunc_frame *f = (struct tplfunc_frame *) frame; + char *api_token = getenv("STASIS_GH_TOKEN"); + if (!api_token) { + api_token = getenv("GITHUB_TOKEN"); + } + result = get_github_release_notes(api_token ? api_token : "anonymous", + (const char *) f->argv[0].t_char_ptr, + (const char *) f->argv[1].t_char_ptr, + (const char *) f->argv[2].t_char_ptr, + output); + return result; +} + +int get_github_release_notes_auto_tplfunc_entrypoint(void *frame, void *data_out) { + int result = 0; + char **output = (char **) data_out; + struct tplfunc_frame *f = (struct tplfunc_frame *) frame; + char *api_token = getenv("STASIS_GH_TOKEN"); + if (!api_token) { + api_token = getenv("GITHUB_TOKEN"); + } + + const struct Delivery *ctx = (struct Delivery *) f->data_in; + struct StrList *notes_list = strlist_init(); + for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(*ctx->tests); i++) { + // Get test context + const struct Test *test = &ctx->tests[i]; + if (test->name && test->version && test->repository) { + char *repository = strdup(test->repository); + char *match = strstr(repository, "spacetelescope/"); + // Cull repository URL + if (match) { + replace_text(repository, "https://github.com/", "", 0); + if (endswith(repository, ".git")) { + replace_text(repository, ".git", "", 0); + } + // Record release notes for version relative to HEAD + // Using HEAD, GitHub returns the previous tag + char *note = NULL; + char h1_title[NAME_MAX] = {0}; + sprintf(h1_title, "# %s", test->name); + strlist_append(¬es_list, h1_title); + result += get_github_release_notes(api_token ? api_token : "anonymous", + repository, + test->version, + "HEAD", + ¬e); + if (note) { + strlist_append(¬es_list, note); + guard_free(note); + } + guard_free(repository); + } + } + } + // Return all notes as a single string + if (strlist_count(notes_list)) { + *output = join(notes_list->data, "\n\n"); + } + guard_strlist_free(¬es_list); + + return result; +} + +int get_junitxml_file_entrypoint(void *frame, void *data_out) { + int result = 0; + char **output = (char **) data_out; + struct tplfunc_frame *f = (struct tplfunc_frame *) frame; + const struct Delivery *ctx = (const struct Delivery *) f->data_in; + + char cwd[PATH_MAX] = {0}; + if (!getcwd(cwd, PATH_MAX - 1)) { + SYSERROR("unable to determine current working directory: %s", strerror(errno)); + return -1; + } + char nametmp[PATH_MAX] = {0}; + strcpy(nametmp, cwd); + char *name = path_basename(nametmp); + + *output = calloc(PATH_MAX, sizeof(**output)); + if (!*output) { + SYSERROR("failed to allocate output string: %s", strerror(errno)); + return -1; + } + sprintf(*output, "%s/results-%s-%s.xml", ctx->storage.results_dir, name, ctx->info.release_name); + + return result; +} + +int get_basetemp_dir_entrypoint(void *frame, void *data_out) { + int result = 0; + char **output = (char **) data_out; + struct tplfunc_frame *f = (struct tplfunc_frame *) frame; + const struct Delivery *ctx = (const struct Delivery *) f->data_in; + + char cwd[PATH_MAX] = {0}; + if (!getcwd(cwd, PATH_MAX - 1)) { + SYSERROR("unable to determine current working directory: %s", strerror(errno)); + return -1; + } + char nametmp[PATH_MAX] = {0}; + strcpy(nametmp, cwd); + char *name = path_basename(nametmp); + + *output = calloc(PATH_MAX, sizeof(**output)); + if (!*output) { + SYSERROR("failed to allocate output string: %s", strerror(errno)); + return -1; + } + sprintf(*output, "%s/truth-%s-%s", ctx->storage.tmpdir, name, ctx->info.release_name); + + return result; +} + +int tox_run_entrypoint(void *frame, void *data_out) { + char **output = (char **) data_out; + struct tplfunc_frame *f = (struct tplfunc_frame *) frame; + const struct Delivery *ctx = (const struct Delivery *) f->data_in; + + // Apply workaround for tox positional arguments + char *toxconf = NULL; + if (!access("tox.ini", F_OK)) { + if (!fix_tox_conf("tox.ini", &toxconf)) { + msg(STASIS_MSG_L3, "Fixing tox positional arguments\n"); + *output = calloc(STASIS_BUFSIZ, sizeof(**output)); + if (!*output) { + return -1; + } + char *basetemp_path = NULL; + if (get_basetemp_dir_entrypoint(f, &basetemp_path)) { + return -2; + } + char *jxml_path = NULL; + if (get_junitxml_file_entrypoint(f, &jxml_path)) { + guard_free(basetemp_path); + return -3; + } + const char *tox_target = f->argv[0].t_char_ptr; + const char *pytest_args = f->argv[1].t_char_ptr; + if (isempty(toxconf) || !strcmp(toxconf, "/")) { + SYSERROR("Unsafe toxconf path: '%s'", toxconf); + guard_free(basetemp_path); + guard_free(jxml_path); + return -4; + } + snprintf(*output, STASIS_BUFSIZ - 1, "\npip install tox && (tox -e py%s%s -c %s --root . -- --basetemp=\"%s\" --junitxml=\"%s\" %s ; rm -f '%s')\n", ctx->meta.python_compact, tox_target, toxconf, basetemp_path, jxml_path, pytest_args ? pytest_args : "", toxconf); + + guard_free(jxml_path); + guard_free(basetemp_path); + } + } + return 0; +} \ No newline at end of file diff --git a/src/lib/core/utils.c b/src/lib/core/utils.c new file mode 100644 index 0000000..89950df --- /dev/null +++ b/src/lib/core/utils.c @@ -0,0 +1,820 @@ +#include +#include "core.h" +#include "utils.h" + +char *dirstack[STASIS_DIRSTACK_MAX]; +const ssize_t dirstack_max = sizeof(dirstack) / sizeof(dirstack[0]); +ssize_t dirstack_len = 0; + +int pushd(const char *path) { + if (access(path, F_OK)) { + // the requested path doesn't exist. + return -1; + } + if (dirstack_len + 1 > dirstack_max) { + // the stack is full + return -1; + } + + dirstack[dirstack_len] = realpath(".", NULL); + dirstack_len++; + return chdir(path); +} + +int popd() { + int result = -1; + if (dirstack_len - 1 < 0) { + return result; + } + dirstack_len--; + result = chdir(dirstack[dirstack_len]); + guard_free(dirstack[dirstack_len]); + return result; +} + +int rmtree(char *_path) { + int status = 0; + char path[PATH_MAX] = {0}; + strncpy(path, _path, sizeof(path) - 1); + DIR *dir; + struct dirent *d_entity; + + dir = opendir(path); + if (!dir) { + return 1; + } + + while ((d_entity = readdir(dir)) != NULL) { + char abspath[PATH_MAX] = {0}; + strcat(abspath, path); + strcat(abspath, DIR_SEP); + strcat(abspath, d_entity->d_name); + + if (!strcmp(d_entity->d_name, ".") || !strcmp(d_entity->d_name, "..") || !strcmp(abspath, path)) { + continue; + } + + // Test for sufficient privilege + if (access(abspath, F_OK) < 0 && errno == EACCES) { + continue; + } + + // Push directories on to the stack first + if (d_entity->d_type == DT_DIR) { + rmtree(abspath); + } else { + remove(abspath); + } + } + closedir(dir); + + if (access(path, F_OK) == 0) { + remove(path); + } + return status; +} + +char *expandpath(const char *_path) { + if (_path == NULL) { + return NULL; + } + const char *homes[] = { + "HOME", + "USERPROFILE", + }; + char home[PATH_MAX]; + char tmp[PATH_MAX]; + char *ptmp = tmp; + char result[PATH_MAX]; + char *sep = NULL; + + memset(home, '\0', sizeof(home)); + memset(ptmp, '\0', sizeof(tmp)); + memset(result, '\0', sizeof(result)); + + strncpy(ptmp, _path, PATH_MAX - 1); + + // Check whether there's a reason to continue processing the string + if (*ptmp != '~') { + return strdup(ptmp); + } + + // Remove tilde from the string and shift its contents to the left + strchrdel(ptmp, "~"); + + // Figure out where the user's home directory resides + for (size_t i = 0; i < sizeof(homes) / sizeof(*homes); i++) { + char *tmphome; + if ((tmphome = getenv(homes[i])) != NULL) { + strncpy(home, tmphome, PATH_MAX - 1); + break; + } + } + + // A broken runtime environment means we can't do anything else here + if (isempty(home)) { + return NULL; + } + + // Scan the path for a directory separator + if ((sep = strpbrk(ptmp, "/\\")) != NULL) { + // Jump past it + ptmp = sep + 1; + } + + // Construct the new path + strncat(result, home, sizeof(result) - strlen(home) + 1); + if (sep) { + strncat(result, DIR_SEP, sizeof(result) - strlen(home) + 1); + strncat(result, ptmp, sizeof(result) - strlen(home) + 1); + } + + return strdup(result); +} + +char *path_basename(char *path) { + char *result = NULL; + char *last = NULL; + + if ((last = strrchr(path, '/')) == NULL) { + return path; + } + // Perform a lookahead ensuring the string is valid beyond the last separator + if (last++ != NULL) { + result = last; + } + + return result; +} + +char *path_dirname(char *path) { + if (!path) { + return ""; + } + if (strlen(path) == 1 && *path == '/') { + return "/"; + } + char *pos = strrchr(path, '/'); + if (!pos) { + return "."; + } + *pos = '\0'; + + return path; +} + +char **file_readlines(const char *filename, size_t start, size_t limit, ReaderFn *readerFn) { + FILE *fp = NULL; + char **result = NULL; + char *buffer = NULL; + size_t lines = 0; + int use_stdin = 0; + + if (strcmp(filename, "-") == 0) { + use_stdin = 1; + } + + if (use_stdin) { + fp = stdin; + } else { + fp = fopen(filename, "r"); + } + + if (fp == NULL) { + perror(filename); + SYSERROR("failed to open %s for reading", filename); + return NULL; + } + + // Allocate buffer + if ((buffer = calloc(STASIS_BUFSIZ, sizeof(char))) == NULL) { + SYSERROR("unable to allocate %d bytes for buffer", STASIS_BUFSIZ); + if (!use_stdin) { + fclose(fp); + } + return NULL; + } + + // count number the of lines in the file + while ((fgets(buffer, STASIS_BUFSIZ - 1, fp)) != NULL) { + lines++; + } + + if (!lines) { + guard_free(buffer); + if (!use_stdin) { + fclose(fp); + } + return NULL; + } + + rewind(fp); + + // Handle invalid start offset + if (start > lines) { + start = 0; + } + + // Adjust line count when start offset is non-zero + if (start != 0 && start < lines) { + lines -= start; + } + + + // Handle minimum and maximum limits + if (limit == 0 || limit > lines) { + limit = lines; + } + + // Populate results array + result = calloc(limit + 1, sizeof(char *)); + for (size_t i = start; i < limit; i++) { + if (i < start) { + continue; + } + + if (fgets(buffer, STASIS_BUFSIZ - 1, fp) == NULL) { + break; + } + + if (readerFn != NULL) { + int status = readerFn(i - start, &buffer); + // A status greater than zero indicates we should ignore this line entirely and "continue" + // A status less than zero indicates we should "break" + // A zero status proceeds normally + if (status > 0) { + i--; + continue; + } else if (status < 0) { + break; + } + } + result[i] = strdup(buffer); + memset(buffer, '\0', STASIS_BUFSIZ); + } + + guard_free(buffer); + if (!use_stdin) { + fclose(fp); + } + return result; +} + +char *find_program(const char *name) { + static char result[PATH_MAX] = {0}; + char *_env_path = getenv(PATH_ENV_VAR); + if (!_env_path) { + errno = EINVAL; + return NULL; + } + char *path = strdup(_env_path); + char *path_orig = path; + char *path_elem = NULL; + + if (!path) { + errno = ENOMEM; + return NULL; + } + + result[0] = '\0'; + while ((path_elem = strsep(&path, PATH_SEP))) { + char abspath[PATH_MAX] = {0}; + strcat(abspath, path_elem); + strcat(abspath, DIR_SEP); + strcat(abspath, name); + if (access(abspath, F_OK) < 0) { + continue; + } + strncpy(result, abspath, sizeof(result)); + break; + } + path = path_orig; + guard_free(path); + return strlen(result) ? result : NULL; +} + +int touch(const char *filename) { + if (access(filename, F_OK) == 0) { + return 0; + } + + FILE *fp = fopen(filename, "w"); + if (!fp) { + perror(filename); + return 1; + } + fclose(fp); + return 0; +} + +int git_clone(struct Process *proc, char *url, char *destdir, char *gitref) { + int result = -1; + char *chdir_to = NULL; + char *program = find_program("git"); + if (!program) { + return result; + } + + static char command[PATH_MAX]; + sprintf(command, "%s clone -c advice.detachedHead=false --recursive %s", program, url); + if (destdir && access(destdir, F_OK) < 0) { + sprintf(command + strlen(command), " %s", destdir); + result = shell(proc, command); + } + + if (destdir) { + chdir_to = destdir; + } else { + chdir_to = path_basename(url); + } + + pushd(chdir_to); + { + memset(command, 0, sizeof(command)); + sprintf(command, "%s fetch --all", program); + result += shell(proc, command); + + if (gitref != NULL) { + memset(command, 0, sizeof(command)); + sprintf(command, "%s checkout %s", program, gitref); + result += shell(proc, command); + } + popd(); + } + return result; +} + + +char *git_describe(const char *path) { + static char version[NAME_MAX]; + FILE *pp; + + memset(version, 0, sizeof(version)); + if (pushd(path)) { + return NULL; + } + + pp = popen("git describe --first-parent --always --tags", "r"); + if (!pp) { + return NULL; + } + fgets(version, sizeof(version) - 1, pp); + strip(version); + pclose(pp); + popd(); + return version; +} + +char *git_rev_parse(const char *path, char *args) { + static char version[NAME_MAX]; + char cmd[PATH_MAX]; + FILE *pp; + + memset(version, 0, sizeof(version)); + if (isempty(args)) { + fprintf(stderr, "git_rev_parse args cannot be empty\n"); + return NULL; + } + + if (pushd(path)) { + return NULL; + } + + sprintf(cmd, "git rev-parse %s", args); + pp = popen(cmd, "r"); + if (!pp) { + return NULL; + } + fgets(version, sizeof(version) - 1, pp); + strip(version); + pclose(pp); + popd(); + return version; +} + +void msg(unsigned type, char *fmt, ...) { + FILE *stream = NULL; + char header[255]; + char status[20]; + + if (type & STASIS_MSG_NOP) { + // quiet mode + return; + } + + if (!globals.verbose && type & STASIS_MSG_RESTRICT) { + // Verbose mode is not active + return; + } + + memset(header, 0, sizeof(header)); + memset(status, 0, sizeof(status)); + + va_list args; + va_start(args, fmt); + + stream = stdout; + fprintf(stream, "%s", STASIS_COLOR_RESET); + if (type & STASIS_MSG_ERROR) { + // for error output + stream = stderr; + fprintf(stream, "%s", STASIS_COLOR_RED); + strcpy(status, " ERROR: "); + } else if (type & STASIS_MSG_WARN) { + stream = stderr; + fprintf(stream, "%s", STASIS_COLOR_YELLOW); + strcpy(status, " WARNING: "); + } else { + fprintf(stream, "%s", STASIS_COLOR_GREEN); + strcpy(status, " "); + } + + if (type & STASIS_MSG_L1) { + sprintf(header, "==>%s" STASIS_COLOR_RESET STASIS_COLOR_WHITE, status); + } else if (type & STASIS_MSG_L2) { + sprintf(header, " ->%s" STASIS_COLOR_RESET, status); + } else if (type & STASIS_MSG_L3) { + sprintf(header, STASIS_COLOR_BLUE " ->%s" STASIS_COLOR_RESET, status); + } + + fprintf(stream, "%s", header); + vfprintf(stream, fmt, args); + fprintf(stream, "%s", STASIS_COLOR_RESET); + va_end(args); +} + +void debug_shell() { + msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "ENTERING STASIS DEBUG SHELL\n" STASIS_COLOR_RESET); + if (system("/bin/bash -c 'PS1=\"(STASIS DEBUG) \\W $ \" bash --norc --noprofile'") < 0) { + SYSERROR("unable to spawn debug shell: %s", strerror(errno)); + exit(errno); + } + msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "EXITING STASIS DEBUG SHELL\n" STASIS_COLOR_RESET); + exit(255); +} + +char *xmkstemp(FILE **fp, const char *mode) { + int fd = -1; + char tmpdir[PATH_MAX]; + char t_name[PATH_MAX * 2]; + + if (globals.tmpdir) { + strcpy(tmpdir, globals.tmpdir); + } else { + strcpy(tmpdir, "/tmp"); + } + memset(t_name, 0, sizeof(t_name)); + sprintf(t_name, "%s/%s", tmpdir, "STASIS.XXXXXX"); + + fd = mkstemp(t_name); + *fp = fdopen(fd, mode); + if (!*fp) { + // unable to open, die + if (fd > 0) + close(fd); + *fp = NULL; + return NULL; + } + + char *path = strdup(t_name); + if (!path) { + // strdup failed, die + if (*fp) { + // close the file handle + fclose(*fp); + *fp = NULL; + } + // fall through. path is NULL. + } + return path; +} + +int isempty_dir(const char *path) { + DIR *dp; + struct dirent *rec; + size_t count = 0; + + dp = opendir(path); + if (!dp) { + return -1; + } + while ((rec = readdir(dp)) != NULL) { + if (!strcmp(rec->d_name, ".") || !strcmp(rec->d_name, "..")) { + continue; + } + count++; + } + closedir(dp); + return count == 0; +} + +int path_store(char **destptr, size_t maxlen, const char *base, const char *path) { + char *path_tmp; + size_t base_len = 0; + size_t path_len = 0; + + // Both path elements need to be defined to continue + if (!base || !path) { + return -1; + } + + // Initialize destination pointer to length of maxlen + path_tmp = calloc(maxlen, sizeof(*path_tmp)); + if (!path_tmp) { + return -1; + } + + // Ensure generated path will fit in destination + base_len = strlen(base); + path_len = strlen(path); + // 2 = directory separator and NUL terminator + if (2 + (base_len + path_len) > maxlen) { + goto l_path_setup_error; + } + + snprintf(path_tmp, maxlen - 1, "%s/%s", base, path); + if (mkdirs(path_tmp, 0755)) { + goto l_path_setup_error; + } + + if (*destptr) { + guard_free(*destptr); + } + + if (!(*destptr = realpath(path_tmp, NULL))) { + goto l_path_setup_error; + } + + guard_free(path_tmp); + return 0; + + l_path_setup_error: + guard_free(path_tmp); + return -1; +} + +int xml_pretty_print_in_place(const char *filename, const char *pretty_print_prog, const char *pretty_print_args) { + int status = 0; + char *tempfile = NULL; + char *result = NULL; + FILE *fp = NULL; + FILE *tmpfp = NULL; + char cmd[PATH_MAX]; + if (!find_program(pretty_print_prog)) { + // Pretty printing is optional. 99% chance the XML data will + // be passed to a report generator; not inspected by a human. + return 0; + } + memset(cmd, 0, sizeof(cmd)); + snprintf(cmd, sizeof(cmd) - 1, "%s %s %s", pretty_print_prog, pretty_print_args, filename); + result = shell_output(cmd, &status); + if (status || !result) { + goto pretty_print_failed; + } + + tempfile = xmkstemp(&tmpfp, "w+"); + if (!tmpfp || !tempfile) { + goto pretty_print_failed; + } + + fprintf(tmpfp, "%s", result); + fflush(tmpfp); + fclose(tmpfp); + + fp = fopen(filename, "w+"); + if (!fp) { + goto pretty_print_failed; + } + + if (copy2(tempfile, filename, CT_PERM)) { + goto pretty_print_failed; + } + + if (remove(tempfile)) { + goto pretty_print_failed; + } + + fclose(fp); + guard_free(tempfile); + guard_free(result); + return 0; + + pretty_print_failed: + if (fp) { + fclose(fp); + } + if (tmpfp) { + fclose(tmpfp); + } + guard_free(tempfile); + guard_free(result); + return -1; +} + +/** + * + * @param filename /path/to/tox.ini + * @param result path of replacement tox.ini configuration + * @return 0 on success, -1 on error + */ +int fix_tox_conf(const char *filename, char **result) { + struct INIFILE *toxini; + FILE *fptemp; + char *tempfile; + const char *with_posargs = " \\\n {posargs}\n"; + + // Create new temporary tox configuration file + tempfile = xmkstemp(&fptemp, "w+"); + if (!tempfile) { + return -1; + } + + // If the result pointer is NULL, allocate enough to store a filesystem path + if (!*result) { + *result = calloc(PATH_MAX, sizeof(**result)); + if (!*result) { + guard_free(tempfile); + return -1; + } + } + + // Consume the original tox.ini configuration + toxini = ini_open(filename); + if (!toxini) { + if (fptemp) { + guard_free(result); + guard_free(tempfile); + fclose(fptemp); + } + return -1; + } + + // Modify tox configuration + // - Allow passing positional arguments pytest + for (size_t i = 0; i < toxini->section_count; i++) { + struct INISection *section = toxini->section[i]; + if (section) { + char *section_name = section->key; + for (size_t k = 0; k < section->data_count; k++) { + struct INIData *data = section->data[k]; + if (data) { + int err = 0; + char *key = data->key; + char *value = ini_getval_str(toxini, section->key, data->key, INI_READ_RENDER, &err); + if (key && value) { + if (startswith(value, "pytest") && !strstr(value, "{posargs}")) { + strip(value); + char *tmp; + tmp = realloc(value, strlen(value) + strlen(with_posargs) + 1); + if (!tmp) { + SYSERROR("failed to increase size to +%zu bytes", + strlen(value) + strlen(with_posargs) + 1); + guard_free(*result); + return -1; + } else if (tmp != value) { + value = tmp; + } + strcat(value, with_posargs); + ini_setval(&toxini, INI_SETVAL_REPLACE, section_name, key, value); + } + } + guard_free(value); + } + } + } + } + + // Save modified configuration + ini_write(toxini, &fptemp, INI_WRITE_RAW); + fclose(fptemp); + + // Store path to modified config + strcpy(*result, tempfile); + guard_free(tempfile); + + ini_free(&toxini); + return 0; +} + +static size_t count_blanks(char *s) { + // return the number of leading blanks (tab/space) in a string + size_t blank = 0; + for (size_t i = 0; i < strlen(s); i++) { + if (isblank(s[i])) { + blank++; + } else { + break; + } + } + return blank; +} + +char *collapse_whitespace(char **s) { + char *x = (*s); + size_t len = strlen(x); + for (size_t i = 0; i < len; i++) { + size_t blank = count_blanks(&x[i]); + if (blank > 1) { + memmove(&x[i], &x[i] + blank, strlen(&x[i])); + } + } + + return *s; +} + +/** + * Replace sensitive text in strings with ***REDACTED*** + * @param to_redact a list of tokens to redact + * @param src to read + * @param dest to write modified string + * @param maxlen maximum length of dest string + * @return 0 on success, -1 on error + */ +int redact_sensitive(const char **to_redact, size_t to_redact_size, char *src, char *dest, size_t maxlen) { + const char *redacted = "***REDACTED***"; + + char *tmp = calloc(strlen(redacted) + strlen(src) + 1, sizeof(*tmp)); + if (!tmp) { + return -1; + } + strcpy(tmp, src); + + for (size_t i = 0; i < to_redact_size; i++) { + if (to_redact[i] && strstr(tmp, to_redact[i])) { + replace_text(tmp, to_redact[i], redacted, 0); + break; + } + } + + memset(dest, 0, maxlen); + strncpy(dest, tmp, maxlen - 1); + guard_free(tmp); + + return 0; +} + +/** + * Retrieve file names in a directory + * (no metadata, non-recursive) + * + * @param path directory path + * @return StrList structure + */ +struct StrList *listdir(const char *path) { + struct StrList *node; + DIR *dp; + struct dirent *rec; + + dp = opendir(path); + if (!dp) { + return NULL; + } + node = strlist_init(); + + while ((rec = readdir(dp)) != NULL) { + if (!strcmp(rec->d_name, ".") || !strcmp(rec->d_name, "..")) { + continue; + } + strlist_append(&node, rec->d_name); + } + closedir(dp); + return node; +} + +long get_cpu_count() { +#if defined(STASIS_OS_LINUX) || defined(STASIS_OS_DARWIN) + return sysconf(_SC_NPROCESSORS_ONLN); +#else + return 0; +#endif +} + +int mkdirs(const char *_path, mode_t mode) { + int status; + char *token; + char pathbuf[PATH_MAX] = {0}; + char *path; + path = pathbuf; + strcpy(path, _path); + errno = 0; + + char result[PATH_MAX] = {0}; + status = 0; + while ((token = strsep(&path, "/")) != NULL && !status) { + if (token[0] == '.') + continue; + strcat(result, token); + strcat(result, "/"); + status = mkdir(result, mode); + if (status && errno == EEXIST) { + status = 0; + errno = 0; + continue; + } + } + return status; +} + +char *find_version_spec(char *str) { + return strpbrk(str, "@~=<>!"); +} diff --git a/src/lib/core/wheel.c b/src/lib/core/wheel.c new file mode 100644 index 0000000..4692d0a --- /dev/null +++ b/src/lib/core/wheel.c @@ -0,0 +1,126 @@ +#include "wheel.h" + +struct Wheel *get_wheel_info(const char *basepath, const char *name, char *to_match[], unsigned match_mode) { + DIR *dp; + struct dirent *rec; + struct Wheel *result = NULL; + char package_path[PATH_MAX]; + char package_name[NAME_MAX]; + + strcpy(package_name, name); + tolower_s(package_name); + sprintf(package_path, "%s/%s", basepath, package_name); + + dp = opendir(package_path); + if (!dp) { + return NULL; + } + + while ((rec = readdir(dp)) != NULL) { + if (!strcmp(rec->d_name, ".") || !strcmp(rec->d_name, "..")) { + continue; + } + char filename[NAME_MAX]; + strcpy(filename, rec->d_name); + char *ext = strstr(filename, ".whl"); + if (ext) { + *ext = '\0'; + } else { + // not a wheel file. nothing to do + continue; + } + + size_t match = 0; + size_t pattern_count = 0; + for (; to_match[pattern_count] != NULL; pattern_count++) { + if (strstr(filename, to_match[pattern_count])) { + match++; + } + } + + if (!startswith(rec->d_name, name)) { + continue; + } + + if (match_mode == WHEEL_MATCH_EXACT && match != pattern_count) { + continue; + } + + result = calloc(1, sizeof(*result)); + if (!result) { + SYSERROR("Unable to allocate %zu bytes for wheel struct", sizeof(*result)); + closedir(dp); + return NULL; + } + + result->path_name = realpath(package_path, NULL); + if (!result->path_name) { + SYSERROR("Unable to resolve absolute path to %s: %s", filename, strerror(errno)); + wheel_free(&result); + closedir(dp); + return NULL; + } + result->file_name = strdup(rec->d_name); + if (!result->file_name) { + SYSERROR("Unable to allocate bytes for %s: %s", rec->d_name, strerror(errno)); + wheel_free(&result); + closedir(dp); + return NULL; + } + + size_t parts_total; + char **parts = split(filename, "-", 0); + if (!parts) { + // This shouldn't happen unless a wheel file is present in the + // directory with a malformed file name, or we've managed to + // exhaust the system's memory + SYSERROR("%s has no '-' separators! (Delete this file and try again)", filename); + wheel_free(&result); + closedir(dp); + return NULL; + } + + for (parts_total = 0; parts[parts_total] != NULL; parts_total++); + if (parts_total == 5) { + // no build tag + result->distribution = strdup(parts[0]); + result->version = strdup(parts[1]); + result->build_tag = NULL; + result->python_tag = strdup(parts[2]); + result->abi_tag = strdup(parts[3]); + result->platform_tag = strdup(parts[4]); + } else if (parts_total == 6) { + // has build tag + result->distribution = strdup(parts[0]); + result->version = strdup(parts[1]); + result->build_tag = strdup(parts[2]); + result->python_tag = strdup(parts[3]); + result->abi_tag = strdup(parts[4]); + result->platform_tag = strdup(parts[5]); + } else { + SYSERROR("Unknown wheel name format: %s. Expected 5 or 6 strings " + "separated by '-', but got %zu instead", filename, parts_total); + GENERIC_ARRAY_FREE(parts); + wheel_free(&result); + closedir(dp); + return NULL; + } + GENERIC_ARRAY_FREE(parts); + break; + } + closedir(dp); + return result; +} + +void wheel_free(struct Wheel **wheel) { + struct Wheel *w = (*wheel); + guard_free(w->path_name); + guard_free(w->file_name); + guard_free(w->distribution); + guard_free(w->version); + guard_free(w->build_tag); + guard_free(w->python_tag); + guard_free(w->abi_tag); + guard_free(w->python_tag); + guard_free(w); +} diff --git a/src/multiprocessing.c b/src/multiprocessing.c deleted file mode 100644 index 2a1b350..0000000 --- a/src/multiprocessing.c +++ /dev/null @@ -1,448 +0,0 @@ -#include "core.h" - -/// The sum of all tasks started by mp_task() -size_t mp_global_task_count = 0; - -static struct MultiProcessingTask *mp_pool_next_available(struct MultiProcessingPool *pool) { - return &pool->task[pool->num_used]; -} - -int child(struct MultiProcessingPool *pool, struct MultiProcessingTask *task) { - FILE *fp_log = NULL; - - // The task starts inside the requested working directory - if (chdir(task->working_dir)) { - perror(task->working_dir); - exit(1); - } - - // Record the task start time - if (clock_gettime(CLOCK_REALTIME, &task->time_data.t_start) < 0) { - perror("clock_gettime"); - exit(1); - } - - // Redirect stdout and stderr to the log file - fflush(stdout); - fflush(stderr); - // Set log file name - sprintf(task->log_file + strlen(task->log_file), "task-%zu-%d.log", mp_global_task_count, task->parent_pid); - fp_log = freopen(task->log_file, "w+", stdout); - if (!fp_log) { - fprintf(stderr, "unable to open '%s' for writing: %s\n", task->log_file, strerror(errno)); - return -1; - } - dup2(fileno(stdout), fileno(stderr)); - - // Generate timestamp for log header - time_t t = time(NULL); - char *timebuf = ctime(&t); - if (timebuf) { - // strip line feed from timestamp - timebuf[strlen(timebuf) ? strlen(timebuf) - 1 : 0] = 0; - } - - // Generate log header - fprintf(fp_log, "# STARTED: %s\n", timebuf ? timebuf : "unknown"); - fprintf(fp_log, "# PID: %d\n", task->parent_pid); - fprintf(fp_log, "# WORKDIR: %s\n", task->working_dir); - fprintf(fp_log, "# COMMAND:\n%s\n", task->cmd); - fprintf(fp_log, "# OUTPUT:\n"); - // Commit header to log file / clean up - fflush(fp_log); - - // Execute task - fflush(stdout); - fflush(stderr); - char *args[] = {"bash", "--norc", task->parent_script, (char *) NULL}; - return execvp("/bin/bash", args); -} - -int parent(struct MultiProcessingPool *pool, struct MultiProcessingTask *task, pid_t pid, int *child_status) { - printf("[%s:%s] Task started (pid: %d)\n", pool->ident, task->ident, pid); - - // Give the child process access to our PID value - task->pid = pid; - task->parent_pid = pid; - - mp_global_task_count++; - - // Check child's status - pid_t code = waitpid(pid, child_status, WUNTRACED | WCONTINUED | WNOHANG); - if (code < 0) { - perror("waitpid failed"); - return -1; - } - return 0; -} - -static int mp_task_fork(struct MultiProcessingPool *pool, struct MultiProcessingTask *task) { - pid_t pid = fork(); - int child_status = 0; - if (pid == -1) { - return -1; - } else if (pid == 0) { - child(pool, task); - } - return parent(pool, task, pid, &child_status); -} - -struct MultiProcessingTask *mp_pool_task(struct MultiProcessingPool *pool, const char *ident, char *working_dir, char *cmd) { - struct MultiProcessingTask *slot = mp_pool_next_available(pool); - if (pool->num_used != pool->num_alloc) { - pool->num_used++; - } else { - fprintf(stderr, "Maximum number of tasks reached\n"); - return NULL; - } - - // Set default status to "error" - slot->status = -1; - - // Set task identifier string - memset(slot->ident, 0, sizeof(slot->ident)); - strncpy(slot->ident, ident, sizeof(slot->ident) - 1); - - // Set log file path - memset(slot->log_file, 0, sizeof(*slot->log_file)); - strcat(slot->log_file, pool->log_root); - strcat(slot->log_file, "/"); - - // Set working directory - if (isempty(working_dir)) { - strcpy(slot->working_dir, "."); - } else { - strncpy(slot->working_dir, working_dir, PATH_MAX - 1); - } - - // Create a temporary file to act as our intermediate command script - FILE *tp = NULL; - char *t_name = NULL; - t_name = xmkstemp(&tp, "w"); - if (!t_name || !tp) { - return NULL; - } - - // Set the script's permissions so that only the calling user can use it - // This should help prevent eavesdropping if keys are applied in plain-text - // somewhere. - chmod(t_name, 0700); - - // Record the script path - memset(slot->parent_script, 0, sizeof(slot->parent_script)); - strncpy(slot->parent_script, t_name, PATH_MAX - 1); - guard_free(t_name); - - // Populate the script - fprintf(tp, "#!/bin/bash\n%s\n", cmd); - fflush(tp); - fclose(tp); - - // Record the command(s) - slot->cmd_len = (strlen(cmd) * sizeof(*cmd)) + 1; - slot->cmd = mmap(NULL, slot->cmd_len, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); - memset(slot->cmd, 0, slot->cmd_len); - strncpy(slot->cmd, cmd, slot->cmd_len); - - return slot; -} - -static void get_task_duration(struct MultiProcessingTask *task, struct timespec *result) { - // based on the timersub() macro in time.h - // This implementation uses timespec and increases the resolution from microseconds to nanoseconds. - struct timespec *start = &task->time_data.t_start; - struct timespec *stop = &task->time_data.t_stop; - result->tv_sec = (stop->tv_sec - start->tv_sec); - result->tv_nsec = (stop->tv_nsec - start->tv_nsec); - if (result->tv_nsec < 0) { - --result->tv_sec; - result->tv_nsec += 1000000000L; - } -} - -void mp_pool_show_summary(struct MultiProcessingPool *pool) { - print_banner("=", 79); - printf("Pool execution summary for \"%s\"\n", pool->ident); - print_banner("=", 79); - printf("STATUS PID DURATION IDENT\n"); - for (size_t i = 0; i < pool->num_used; i++) { - struct MultiProcessingTask *task = &pool->task[i]; - char status_str[10] = {0}; - if (!task->status && !task->signaled_by) { - strcpy(status_str, "DONE"); - } else if (task->signaled_by) { - strcpy(status_str, "TERM"); - } else { - strcpy(status_str, "FAIL"); - } - - struct timespec duration; - get_task_duration(task, &duration); - long diff = duration.tv_sec + duration.tv_nsec / 1000000000L; - printf("%-4s %10d %7lds %-10s\n", status_str, task->parent_pid, diff, task->ident) ; - } - puts(""); -} - -static int show_log_contents(FILE *stream, struct MultiProcessingTask *task) { - FILE *fp = fopen(task->log_file, "r"); - if (!fp) { - return -1; - } - char buf[BUFSIZ] = {0}; - while ((fgets(buf, sizeof(buf) - 1, fp)) != NULL) { - fprintf(stream, "%s", buf); - memset(buf, 0, sizeof(buf)); - } - fprintf(stream, "\n"); - fclose(fp); - return 0; -} - -int mp_pool_kill(struct MultiProcessingPool *pool, int signum) { - printf("Sending signal %d to pool '%s'\n", signum, pool->ident); - for (size_t i = 0; i < pool->num_used; i++) { - struct MultiProcessingTask *slot = &pool->task[i]; - if (!slot) { - return -1; - } - // Kill tasks in progress - if (slot->pid > 0) { - int status; - printf("Sending signal %d to task '%s' (pid: %d)\n", signum, slot->ident, slot->pid); - status = kill(slot->pid, signum); - if (status && errno != ESRCH) { - fprintf(stderr, "Task '%s' (pid: %d) did not respond: %s\n", slot->ident, slot->pid, strerror(errno)); - } else { - // Wait for process to handle the signal, then set the status accordingly - if (waitpid(slot->pid, &status, 0) >= 0) { - slot->signaled_by = WTERMSIG(status); - // Record the task stop time - if (clock_gettime(CLOCK_REALTIME, &slot->time_data.t_stop) < 0) { - perror("clock_gettime"); - exit(1); - } - // We are short-circuiting the normal flow, and the process is now dead, so mark it as such - slot->pid = MP_POOL_PID_UNUSED; - } - } - } - if (!access(slot->log_file, F_OK)) { - remove(slot->log_file); - } - if (!access(slot->parent_script, F_OK)) { - remove(slot->parent_script); - } - } - return 0; -} - -int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) { - int status = 0; - int failures = 0; - size_t tasks_complete = 0; - size_t lower_i = 0; - size_t upper_i = jobs; - - do { - size_t hang_check = 0; - if (upper_i >= pool->num_used) { - size_t x = upper_i - pool->num_used; - upper_i -= (size_t) x; - } - - for (size_t i = lower_i; i < upper_i; i++) { - struct MultiProcessingTask *slot = &pool->task[i]; - if (slot->status == -1) { - if (mp_task_fork(pool, slot)) { - fprintf(stderr, "%s: mp_task_fork failed\n", slot->ident); - kill(0, SIGTERM); - } - } - - // Has the child been processed already? - if (slot->pid == MP_POOL_PID_UNUSED) { - // Child is already used up, skip it - hang_check++; - if (hang_check >= pool->num_used) { - // If you join a pool that's already finished it will spin - // forever. This protects the program from entering an - // infinite loop. - fprintf(stderr, "%s is deadlocked\n", pool->ident); - failures++; - goto pool_deadlocked; - } - continue; - } - - // Is the process finished? - pid_t pid = waitpid(slot->pid, &status, WNOHANG | WUNTRACED | WCONTINUED); - int task_ended = WIFEXITED(status); - int task_ended_by_signal = WIFSIGNALED(status); - int task_stopped = WIFSTOPPED(status); - int task_continued = WIFCONTINUED(status); - int status_exit = WEXITSTATUS(status); - int status_signal = WTERMSIG(status); - int status_stopped = WSTOPSIG(status); - - // Update status - slot->status = status_exit; - slot->signaled_by = status_signal; - - char progress[1024] = {0}; - if (pid > 0) { - double percent = ((double) (tasks_complete + 1) / (double) pool->num_used) * 100; - snprintf(progress, sizeof(progress) - 1, "[%s:%s] [%3.1f%%]", pool->ident, slot->ident, percent); - - // The process ended in one the following ways - // Note: SIGSTOP nor SIGCONT will not increment the tasks_complete counter - if (task_stopped) { - printf("%s Task was suspended (%d)\n", progress, status_stopped); - continue; - } else if (task_continued) { - printf("%s Task was resumed\n", progress); - continue; - } else if (task_ended_by_signal) { - printf("%s Task ended by signal %d (%s)\n", progress, status_signal, strsignal(status_signal)); - tasks_complete++; - } else if (task_ended) { - printf("%s Task ended (status: %d)\n", progress, status_exit); - tasks_complete++; - } else { - fprintf(stderr, "%s Task state is unknown (0x%04X)\n", progress, status); - } - - // Show the log (always) - if (show_log_contents(stdout, slot)) { - perror(slot->log_file); - } - - // Record the task stop time - if (clock_gettime(CLOCK_REALTIME, &slot->time_data.t_stop) < 0) { - perror("clock_gettime"); - exit(1); - } - - if (status >> 8 != 0 || (status & 0xff) != 0) { - fprintf(stderr, "%s Task failed\n", progress); - failures++; - - if (flags & MP_POOL_FAIL_FAST && pool->num_used > 1) { - mp_pool_kill(pool, SIGTERM); - return -2; - } - } else { - printf("%s Task finished\n", progress); - } - - // Clean up logs and scripts left behind by the task - if (remove(slot->log_file)) { - fprintf(stderr, "%s Unable to remove log file: '%s': %s\n", progress, slot->parent_script, strerror(errno)); - } - if (remove(slot->parent_script)) { - fprintf(stderr, "%s Unable to remove temporary script '%s': %s\n", progress, slot->parent_script, strerror(errno)); - } - - // Update progress and tell the poller to ignore the PID. The process is gone. - slot->pid = MP_POOL_PID_UNUSED; - } else if (pid < 0) { - fprintf(stderr, "waitpid failed: %s\n", strerror(errno)); - return -1; - } else { - // Track the number of seconds elapsed for each task. - // When a task has executed for longer than status_intervals, print a status update - // _seconds represents the time between intervals, not the total runtime of the task - slot->_seconds = time(NULL) - slot->_now; - if (slot->_seconds > pool->status_interval) { - slot->_now = time(NULL); - slot->_seconds = 0; - } - if (slot->_seconds == 0) { - printf("[%s:%s] Task is running (pid: %d)\n", pool->ident, slot->ident, slot->parent_pid); - } - } - } - - if (tasks_complete == pool->num_used) { - break; - } - - if (tasks_complete == upper_i) { - lower_i += jobs; - upper_i += jobs; - } - - // Poll again after a short delay - sleep(1); - } while (1); - - pool_deadlocked: - puts(""); - return failures; -} - - -struct MultiProcessingPool *mp_pool_init(const char *ident, const char *log_root) { - struct MultiProcessingPool *pool; - - if (!ident || !log_root) { - // Pool must have an ident string - // log_root must be set - return NULL; - } - - // The pool is shared with children - pool = mmap(NULL, sizeof(*pool), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); - - // Set pool identity string - memset(pool->ident, 0, sizeof(pool->ident)); - strncpy(pool->ident, ident, sizeof(pool->ident) - 1); - - // Set logging base directory - memset(pool->log_root, 0, sizeof(pool->log_root)); - strncpy(pool->log_root, log_root, sizeof(pool->log_root) - 1); - pool->num_used = 0; - pool->num_alloc = MP_POOL_TASK_MAX; - - // Create the log directory - if (mkdirs(log_root, 0700) < 0) { - if (errno != EEXIST) { - perror(log_root); - mp_pool_free(&pool); - return NULL; - } - } - - // Task array is shared with children - pool->task = mmap(NULL, (pool->num_alloc + 1) * sizeof(*pool->task), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); - if (pool->task == MAP_FAILED) { - perror("mmap"); - mp_pool_free(&pool); - return NULL; - } - - return pool; -} - -void mp_pool_free(struct MultiProcessingPool **pool) { - for (size_t i = 0; i < (*pool)->num_alloc; i++) { - } - // Unmap all pool tasks - if ((*pool)->task) { - if ((*pool)->task->cmd) { - if (munmap((*pool)->task->cmd, (*pool)->task->cmd_len) < 0) { - perror("munmap"); - } - } - if (munmap((*pool)->task, sizeof(*(*pool)->task) * (*pool)->num_alloc) < 0) { - perror("munmap"); - } - } - // Unmap the pool - if ((*pool)) { - if (munmap((*pool), sizeof(*(*pool))) < 0) { - perror("munmap"); - } - (*pool) = NULL; - } -} \ No newline at end of file diff --git a/src/recipe.c b/src/recipe.c deleted file mode 100644 index 833908c..0000000 --- a/src/recipe.c +++ /dev/null @@ -1,64 +0,0 @@ -#include "recipe.h" - -int recipe_clone(char *recipe_dir, char *url, char *gitref, char **result) { - struct Process proc; - char destdir[PATH_MAX]; - char *reponame = NULL; - - memset(&proc, 0, sizeof(proc)); - memset(destdir, 0, sizeof(destdir)); - reponame = path_basename(url); - - sprintf(destdir, "%s/%s", recipe_dir, reponame); - if (!*result) { - *result = calloc(PATH_MAX, sizeof(*result)); - if (!*result) { - return -1; - } - } - strncpy(*result, destdir, PATH_MAX); - - if (!access(destdir, F_OK)) { - if (!strcmp(destdir, "/")) { - fprintf(stderr, "STASIS is misconfigured. Please check your output path(s) immediately.\n"); - fprintf(stderr, "recipe_dir = '%s'\nreponame = '%s'\ndestdir = '%s'\n", - recipe_dir, reponame, destdir); - exit(1); - } - if (rmtree(destdir)) { - guard_free(*result); - *result = NULL; - return -1; - } - } - return git_clone(&proc, url, destdir, gitref); -} - - -int recipe_get_type(char *repopath) { - int result; - char path[PATH_MAX]; - // conda-forge is a collection of repositories - // "conda-forge.yml" is guaranteed to exist - const char *marker[] = { - "conda-forge.yml", - "stsci", - "meta.yaml", - NULL - }; - const int type[] = { - RECIPE_TYPE_CONDA_FORGE, - RECIPE_TYPE_ASTROCONDA, - RECIPE_TYPE_GENERIC - }; - - for (size_t i = 0; marker[i] != NULL; i++) { - sprintf(path, "%s/%s", repopath, marker[i]); - result = access(path, F_OK); - if (!result) { - return type[i]; - } - } - - return RECIPE_TYPE_UNKNOWN; -} \ No newline at end of file diff --git a/src/relocation.c b/src/relocation.c deleted file mode 100644 index 852aca4..0000000 --- a/src/relocation.c +++ /dev/null @@ -1,155 +0,0 @@ -/** - * @file relocation.c - */ -#include "relocation.h" -#include "str.h" - -/** - * Replace all occurrences of `target` with `replacement` in `original` - * - * ~~~{.c} - * char *str = calloc(100, sizeof(char)); - * strcpy(str, "This are a test."); - * if (replace_text(str, "are", "is")) { - * fprintf(stderr, "string replacement failed\n"); - * exit(1); - * } - * // str is: "This is a test." - * free(str); - * ~~~ - * - * @param original string to modify - * @param target string value to replace - * @param replacement string value - * @return 0 on success, -1 on error - */ -int replace_text(char *original, const char *target, const char *replacement, unsigned flags) { - char buffer[STASIS_BUFSIZ]; - char *pos = original; - char *match = NULL; - size_t original_len = strlen(original); - size_t target_len = strlen(target); - size_t rep_len = strlen(replacement); - size_t buffer_len = 0; - - if (original_len > sizeof(buffer)) { - errno = EINVAL; - SYSERROR("The original string is larger than buffer: %zu > %zu\n", original_len, sizeof(buffer)); - return -1; - } - - memset(buffer, 0, sizeof(buffer)); - if ((match = strstr(pos, target))) { - while (*pos != '\0') { - // append to buffer the bytes leading up to the match - strncat(buffer, pos, match - pos); - // position in the string jump ahead to the beginning of the match - pos = match; - - // replacement is shorter than the target - if (rep_len < target_len) { - // shrink the string - strcat(buffer, replacement); - memmove(pos, pos + target_len, strlen(pos) - target_len); - memset(pos + (strlen(pos) - target_len), 0, target_len); - } else { // replacement is longer than the target - // write the replacement value to the buffer - strcat(buffer, replacement); - // target consumed. jump to the end of the substring. - pos += target_len; - } - if (flags & REPLACE_TRUNCATE_AFTER_MATCH) { - if (strstr(pos, LINE_SEP)) { - strcat(buffer, LINE_SEP); - } - break; - } - // find more matches - if (!(match = strstr(pos, target))) { - // no more matches - // append whatever remains to the buffer - strcat(buffer, pos); - // stop - break; - } - } - } else { - return 0; - } - - buffer_len = strlen(buffer); - if (buffer_len < original_len) { - // truncate whatever remains of the original buffer - memset(original + buffer_len, 0, original_len - buffer_len); - } - // replace original with contents of buffer - strcpy(original, buffer); - return 0; -} - -/** - * Replace `target` with `replacement` in `filename` - * - * ~~~{.c} - * if (file_replace_text("/path/to/file.txt", "are", "is")) { - * fprintf(stderr, "failed to replace strings in file\n"); - * exit(1); - * } - * ~~~ - * - * @param filename path to file - * @param target string value to replace - * @param replacement string - * @return 0 on success, -1 on error - */ -int file_replace_text(const char* filename, const char* target, const char* replacement, unsigned flags) { - int result; - char buffer[STASIS_BUFSIZ]; - char tempfilename[] = "tempfileXXXXXX"; - FILE *fp; - FILE *tfp; - - fp = fopen(filename, "r"); - if (!fp) { - fprintf(stderr, "unable to open for reading: %s\n", filename); - return -1; - } - - tfp = fopen(tempfilename, "w+"); - if (!tfp) { - SYSERROR("unable to open temporary fp for writing: %s", tempfilename); - fclose(fp); - return -1; - } - - // Write modified strings to temporary file - result = 0; - while (fgets(buffer, sizeof(buffer), fp) != NULL) { - if (strstr(buffer, target)) { - if (replace_text(buffer, target, replacement, flags)) { - result = -1; - } - } - fputs(buffer, tfp); - } - fflush(tfp); - - // Replace original with modified copy - fclose(fp); - fp = fopen(filename, "w+"); - if (!fp) { - SYSERROR("unable to reopen %s for writing", filename); - return -1; - } - - // Update original file - rewind(tfp); - while (fgets(buffer, sizeof(buffer), tfp) != NULL) { - fputs(buffer, fp); - } - fclose(fp); - fclose(tfp); - - remove(tempfilename); - return result; -} \ No newline at end of file diff --git a/src/rules.c b/src/rules.c deleted file mode 100644 index e42ee07..0000000 --- a/src/rules.c +++ /dev/null @@ -1,5 +0,0 @@ -// -// Created by jhunk on 12/18/23. -// - -#include "rules.h" diff --git a/src/stasis_indexer.c b/src/stasis_indexer.c deleted file mode 100644 index 05b9e39..0000000 --- a/src/stasis_indexer.c +++ /dev/null @@ -1,948 +0,0 @@ -#include -#include -#include "core.h" - -static struct option long_options[] = { - {"help", no_argument, 0, 'h'}, - {"destdir", required_argument, 0, 'd'}, - {"verbose", no_argument, 0, 'v'}, - {"unbuffered", no_argument, 0, 'U'}, - {"web", no_argument, 0, 'w'}, - {0, 0, 0, 0}, -}; - -const char *long_options_help[] = { - "Display this usage statement", - "Destination directory", - "Increase output verbosity", - "Disable line buffering", - "Generate HTML indexes (requires pandoc)", - NULL, -}; - -static void usage(char *name) { - int maxopts = sizeof(long_options) / sizeof(long_options[0]); - unsigned char *opts = calloc(maxopts + 1, sizeof(char)); - for (int i = 0; i < maxopts; i++) { - opts[i] = long_options[i].val; - } - printf("usage: %s [-%s] {{STASIS_ROOT}...}\n", name, opts); - guard_free(opts); - - for (int i = 0; i < maxopts - 1; i++) { - char line[255]; - sprintf(line, " --%s -%c %-20s", long_options[i].name, long_options[i].val, long_options_help[i]); - puts(line); - } - -} - -int indexer_combine_rootdirs(const char *dest, char **rootdirs, const size_t rootdirs_total) { - char cmd[PATH_MAX]; - char destdir_bare[PATH_MAX]; - char destdir_with_output[PATH_MAX]; - char *destdir = destdir_bare; - - memset(cmd, 0, sizeof(cmd)); - memset(destdir_bare, 0, sizeof(destdir_bare)); - memset(destdir_with_output, 0, sizeof(destdir_bare)); - - strcpy(destdir_bare, dest); - strcpy(destdir_with_output, dest); - strcat(destdir_with_output, "/output"); - - if (!access(destdir_with_output, F_OK)) { - destdir = destdir_with_output; - } - - sprintf(cmd, "rsync -ah%s --delete --exclude 'tools/' --exclude 'tmp/' --exclude 'build/' ", globals.verbose ? "v" : "q"); - for (size_t i = 0; i < rootdirs_total; i++) { - char srcdir_bare[PATH_MAX] = {0}; - char srcdir_with_output[PATH_MAX] = {0}; - char *srcdir = srcdir_bare; - strcpy(srcdir_bare, rootdirs[i]); - strcpy(srcdir_with_output, rootdirs[i]); - strcat(srcdir_with_output, "/output"); - - if (access(srcdir_bare, F_OK)) { - fprintf(stderr, "%s does not exist\n", srcdir_bare); - continue; - } - - if (!access(srcdir_with_output, F_OK)) { - srcdir = srcdir_with_output; - } - snprintf(cmd + strlen(cmd), sizeof(srcdir) - strlen(srcdir) + 4, "'%s'/ ", srcdir); - } - snprintf(cmd + strlen(cmd), sizeof(cmd) - strlen(destdir) + 1, " %s/", destdir); - - if (globals.verbose) { - puts(cmd); - } - - if (system(cmd)) { - return -1; - } - return 0; -} - -int indexer_wheels(struct Delivery *ctx) { - return delivery_index_wheel_artifacts(ctx); -} - -int indexer_load_metadata(struct Delivery *ctx, const char *filename) { - char line[STASIS_NAME_MAX] = {0}; - FILE *fp; - - fp = fopen(filename, "r"); - if (!fp) { - return -1; - } - - while (fgets(line, sizeof(line) - 1, fp) != NULL) { - char **parts = split(line, " ", 1); - char *name = parts[0]; - char *value = parts[1]; - strip(value); - if (!strcmp(name, "name")) { - ctx->meta.name = strdup(value); - } else if (!strcmp(name, "version")) { - ctx->meta.version = strdup(value); - } else if (!strcmp(name, "rc")) { - ctx->meta.rc = (int) strtol(value, NULL, 10); - } else if (!strcmp(name, "python")) { - ctx->meta.python = strdup(value); - } else if (!strcmp(name, "python_compact")) { - ctx->meta.python_compact = strdup(value); - } else if (!strcmp(name, "mission")) { - ctx->meta.mission = strdup(value); - } else if (!strcmp(name, "codename")) { - ctx->meta.codename = strdup(value); - } else if (!strcmp(name, "platform")) { - ctx->system.platform = split(value, " ", 0); - } else if (!strcmp(name, "arch")) { - ctx->system.arch = strdup(value); - } else if (!strcmp(name, "time")) { - ctx->info.time_str_epoch = strdup(value); - } else if (!strcmp(name, "release_fmt")) { - ctx->rules.release_fmt = strdup(value); - } else if (!strcmp(name, "release_name")) { - ctx->info.release_name = strdup(value); - } else if (!strcmp(name, "build_name_fmt")) { - ctx->rules.build_name_fmt = strdup(value); - } else if (!strcmp(name, "build_name")) { - ctx->info.build_name = strdup(value); - } else if (!strcmp(name, "build_number_fmt")) { - ctx->rules.build_number_fmt = strdup(value); - } else if (!strcmp(name, "build_number")) { - ctx->info.build_number = strdup(value); - } else if (!strcmp(name, "conda_installer_baseurl")) { - ctx->conda.installer_baseurl = strdup(value); - } else if (!strcmp(name, "conda_installer_name")) { - ctx->conda.installer_name = strdup(value); - } else if (!strcmp(name, "conda_installer_version")) { - ctx->conda.installer_version = strdup(value); - } else if (!strcmp(name, "conda_installer_platform")) { - ctx->conda.installer_platform = strdup(value); - } else if (!strcmp(name, "conda_installer_arch")) { - ctx->conda.installer_arch = strdup(value); - } - GENERIC_ARRAY_FREE(parts); - } - fclose(fp); - - return 0; -} - -int indexer_get_files(struct StrList **out, const char *path, const char *pattern, ...) { - va_list args; - va_start(args, pattern); - char userpattern[PATH_MAX] = {0}; - vsprintf(userpattern, pattern, args); - va_end(args); - struct StrList *list = listdir(path); - if (!list) { - return -1; - } - - if (!(*out)) { - (*out) = strlist_init(); - if (!(*out)) { - guard_strlist_free(&list); - return -1; - } - } - - size_t no_match = 0; - for (size_t i = 0; i < strlist_count(list); i++) { - char *item = strlist_item(list, i); - if (fnmatch(userpattern, item, 0)) { - no_match++; - continue; - } else { - strlist_append(&(*out), item); - } - } - if (no_match >= strlist_count(list)) { - fprintf(stderr, "no files matching the pattern: %s\n", userpattern); - guard_strlist_free(&list); - return -1; - } - guard_strlist_free(&list); - return 0; -} - -int get_latest_rc(struct Delivery ctx[], size_t nelem) { - int result = 0; - for (size_t i = 0; i < nelem; i++) { - if (ctx[i].meta.rc > result) { - result = ctx[i].meta.rc; - } - } - return result; -} - -struct Delivery **get_latest_deliveries(struct Delivery ctx[], size_t nelem) { - struct Delivery **result = NULL; - int latest = 0; - size_t n = 0; - - result = calloc(nelem + 1, sizeof(result)); - if (!result) { - fprintf(stderr, "Unable to allocate %zu bytes for result delivery array: %s\n", nelem * sizeof(result), strerror(errno)); - return NULL; - } - - latest = get_latest_rc(ctx, nelem); - for (size_t i = 0; i < nelem; i++) { - if (ctx[i].meta.rc == latest) { - result[n] = &ctx[i]; - n++; - } - } - - return result; -} - -int get_pandoc_version(size_t *result) { - *result = 0; - int state = 0; - char *version_str = shell_output("pandoc --version", &state); - if (state || !version_str) { - // an error occurred - return -1; - } - - // Verify that we're looking at pandoc - if (strlen(version_str) > 7 && !strncmp(version_str, "pandoc ", 7)) { - // we have pandoc - char *v_begin = &version_str[7]; - if (!v_begin) { - SYSERROR("unexpected pandoc output: %s", version_str); - return -1; - } - char *v_end = strchr(version_str, '\n'); - if (v_end) { - *v_end = 0; - } - - char **parts = split(v_begin, ".", 0); - if (!parts) { - SYSERROR("unable to split pandoc version string, '%s': %s", version_str, strerror(errno)); - return -1; - } - - size_t parts_total; - for (parts_total = 0; parts[parts_total] != NULL; parts_total++); - - // generate the version as an integer - // note: pandoc version scheme never exceeds four elements (or bytes in this case) - for (size_t i = 0; i < 4; i++) { - unsigned char tmp = 0; - if (i < parts_total) { - // only process version elements we have. the rest will be zeros. - tmp = strtoul(parts[i], NULL, 10); - } - // pack version element into result - *result = (*result << 8) | tmp; - } - } else { - // invalid version string - return 1; - } - - return 0; -} - -int indexer_make_website(struct Delivery *ctx) { - char cmd[PATH_MAX]; - const char *pattern = "*.md"; - - if (!find_program("pandoc")) { - fprintf(stderr, "pandoc is not installed: unable to generate HTML indexes\n"); - return 0; - } - - char *css_filename = calloc(PATH_MAX, sizeof(*css_filename)); - if (!css_filename) { - SYSERROR("unable to allocate string for CSS file path: %s", strerror(errno)); - return -1; - } - - sprintf(css_filename, "%s/%s", globals.sysconfdir, "stasis_pandoc.css"); - int have_css = access(css_filename, F_OK | R_OK) == 0; - - char pandoc_versioned_args[255] = {0}; - size_t pandoc_version = 0; - - if (!get_pandoc_version(&pandoc_version)) { - // < 2.19 - if (pandoc_version < 0x02130000) { - strcat(pandoc_versioned_args, "--self-contained "); - } else { - // >= 2.19 - strcat(pandoc_versioned_args, "--embed-resources "); - } - - // >= 1.15.0.4 - if (pandoc_version >= 0x010f0004) { - strcat(pandoc_versioned_args, "--standalone "); - } - - // >= 1.10.0.1 - if (pandoc_version >= 0x010a0001) { - strcat(pandoc_versioned_args, "-f gfm+autolink_bare_uris "); - } - - // > 3.1.9 - if (pandoc_version > 0x03010900) { - strcat(pandoc_versioned_args, "-f gfm+alerts "); - } - } - - struct StrList *dirs = strlist_init(); - strlist_append(&dirs, ctx->storage.delivery_dir); - strlist_append(&dirs, ctx->storage.results_dir); - - struct StrList *inputs = NULL; - for (size_t i = 0; i < strlist_count(dirs); i++) { - if (indexer_get_files(&inputs, ctx->storage.delivery_dir, pattern)) { - SYSERROR("%s does not contain files with pattern: %s", ctx->storage.delivery_dir, pattern); - guard_strlist_free(&inputs); - continue; - } - char *root = strlist_item(dirs, i); - for (size_t x = 0; x < strlist_count(inputs); x++) { - char *filename = strlist_item(inputs, x); - char fullpath_src[PATH_MAX] = {0}; - char fullpath_dest[PATH_MAX] = {0}; - sprintf(fullpath_src, "%s/%s", root, filename); - if (access(fullpath_src, F_OK)) { - continue; - } - - // Replace *.md extension with *.html. - strcpy(fullpath_dest, fullpath_src); - char *ext = strrchr(fullpath_dest, '.'); - if (ext) { - *ext = '\0'; - } - strcat(fullpath_dest, ".html"); - - // Converts a markdown file to html - strcpy(cmd, "pandoc "); - strcat(cmd, pandoc_versioned_args); - if (have_css) { - strcat(cmd, "--css "); - strcat(cmd, css_filename); - } - strcat(cmd, " "); - strcat(cmd, "--metadata title=\"STASIS\" "); - strcat(cmd, "-o "); - strcat(cmd, fullpath_dest); - strcat(cmd, " "); - strcat(cmd, fullpath_src); - if (globals.verbose) { - puts(cmd); - } - // This might be negative when killed by a signal. - // Otherwise, the return code is not critical to us. - if (system(cmd) < 0) { - guard_free(css_filename); - guard_strlist_free(&dirs); - return 1; - } - if (file_replace_text(fullpath_dest, ".md", ".html", 0)) { - // inform-only - SYSERROR("%s: failed to rewrite *.md urls with *.html extension", fullpath_dest); - } - - // Link the nearest README.html to index.html - if (!strcmp(filename, "README.md")) { - char link_from[PATH_MAX] = {0}; - char link_dest[PATH_MAX] = {0}; - strcpy(link_from, "README.html"); - sprintf(link_dest, "%s/%s", root, "index.html"); - if (symlink(link_from, link_dest)) { - SYSERROR("Warning: symlink(%s, %s) failed: %s", link_from, link_dest, strerror(errno)); - } - } - } - guard_strlist_free(&inputs); - } - guard_free(css_filename); - guard_strlist_free(&dirs); - - return 0; -} - -static int micromamba_configure(const struct Delivery *ctx, struct MicromambaInfo *m) { - int status = 0; - char *micromamba_prefix = NULL; - if (asprintf(µmamba_prefix, "%s/bin", ctx->storage.tools_dir) < 0) { - return -1; - } - m->conda_prefix = globals.conda_install_prefix; - m->micromamba_prefix = micromamba_prefix; - - size_t pathvar_len = (strlen(getenv("PATH")) + strlen(m->micromamba_prefix) + strlen(m->conda_prefix)) + 3 + 4 + 1; - // ^^^^^^^^^^^^^^^^^^ - // 3 = separators - // 4 = chars (/bin) - // 1 = nul terminator - char *pathvar = calloc(pathvar_len, sizeof(*pathvar)); - if (!pathvar) { - SYSERROR("%s", "Unable to allocate bytes for temporary path string"); - exit(1); - } - snprintf(pathvar, pathvar_len, "%s/bin:%s:%s", m->conda_prefix, m->micromamba_prefix, getenv("PATH")); - setenv("PATH", pathvar, 1); - guard_free(pathvar); - - status += micromamba(m, "config prepend --env channels conda-forge"); - if (!globals.verbose) { - status += micromamba(m, "config set --env quiet true"); - } - status += micromamba(m, "config set --env always_yes true"); - status += micromamba(m, "install conda-build pandoc"); - - return status; -} - -int indexer_conda(struct Delivery *ctx, struct MicromambaInfo m) { - int status = 0; - - status += micromamba(&m, "run conda index %s", ctx->storage.conda_artifact_dir); - return status; -} - -static struct StrList *get_architectures(struct Delivery ctx[], size_t nelem) { - struct StrList *architectures = strlist_init(); - for (size_t i = 0; i < nelem; i++) { - if (!strstr_array(architectures->data, ctx[i].system.arch)) { - strlist_append(&architectures, ctx[i].system.arch); - } - } - return architectures; -} - -static struct StrList *get_platforms(struct Delivery ctx[], size_t nelem) { - struct StrList *platforms = strlist_init(); - for (size_t i = 0; i < nelem; i++) { - if (!strstr_array(platforms->data, ctx[i].system.platform[DELIVERY_PLATFORM_RELEASE])) { - strlist_append(&platforms, ctx[i].system.platform[DELIVERY_PLATFORM_RELEASE]); - } - } - return platforms; -} - -int indexer_symlinks(struct Delivery ctx[], size_t nelem) { - struct Delivery **data = NULL; - data = get_latest_deliveries(ctx, nelem); - //int latest = get_latest_rc(ctx, nelem); - - if (!pushd(ctx->storage.delivery_dir)) { - for (size_t i = 0; i < nelem; i++) { - char link_name_spec[PATH_MAX]; - char link_name_readme[PATH_MAX]; - - char file_name_spec[PATH_MAX]; - char file_name_readme[PATH_MAX]; - - if (!data[i]) { - continue; - } - sprintf(link_name_spec, "latest-py%s-%s-%s.yml", data[i]->meta.python_compact, data[i]->system.platform[DELIVERY_PLATFORM_RELEASE], data[i]->system.arch); - sprintf(file_name_spec, "%s.yml", data[i]->info.release_name); - - sprintf(link_name_readme, "README-py%s-%s-%s.md", data[i]->meta.python_compact, data[i]->system.platform[DELIVERY_PLATFORM_RELEASE], data[i]->system.arch); - sprintf(file_name_readme, "README-%s.md", data[i]->info.release_name); - - if (!access(link_name_spec, F_OK)) { - if (unlink(link_name_spec)) { - fprintf(stderr, "Unable to remove spec link: %s\n", link_name_spec); - } - } - if (!access(link_name_readme, F_OK)) { - if (unlink(link_name_readme)) { - fprintf(stderr, "Unable to remove readme link: %s\n", link_name_readme); - } - } - - if (globals.verbose) { - printf("%s -> %s\n", file_name_spec, link_name_spec); - } - if (symlink(file_name_spec, link_name_spec)) { - fprintf(stderr, "Unable to link %s as %s\n", file_name_spec, link_name_spec); - } - - if (globals.verbose) { - printf("%s -> %s\n", file_name_readme, link_name_readme); - } - if (symlink(file_name_readme, link_name_readme)) { - fprintf(stderr, "Unable to link %s as %s\n", file_name_readme, link_name_readme); - } - } - popd(); - } else { - fprintf(stderr, "Unable to enter delivery directory: %s\n", ctx->storage.delivery_dir); - guard_free(data); - return -1; - } - - // "latest" is an array of pointers to ctx[]. Do not free the contents of the array. - guard_free(data); - return 0; -} - -int indexer_readmes(struct Delivery ctx[], size_t nelem) { - struct Delivery **latest = NULL; - latest = get_latest_deliveries(ctx, nelem); - - char indexfile[PATH_MAX] = {0}; - sprintf(indexfile, "%s/README.md", ctx->storage.delivery_dir); - - if (!pushd(ctx->storage.delivery_dir)) { - FILE *indexfp; - indexfp = fopen(indexfile, "w+"); - if (!indexfp) { - fprintf(stderr, "Unable to open %s for writing\n", indexfile); - return -1; - } - struct StrList *archs = get_architectures(*latest, nelem); - struct StrList *platforms = get_platforms(*latest, nelem); - - fprintf(indexfp, "# %s-%s\n\n", ctx->meta.name, ctx->meta.version); - fprintf(indexfp, "## Current Release\n\n"); - for (size_t p = 0; p < strlist_count(platforms); p++) { - char *platform = strlist_item(platforms, p); - for (size_t a = 0; a < strlist_count(archs); a++) { - char *arch = strlist_item(archs, a); - int have_combo = 0; - for (size_t i = 0; i < nelem; i++) { - if (latest[i] && latest[i]->system.platform) { - if (strstr(latest[i]->system.platform[DELIVERY_PLATFORM_RELEASE], platform) && - strstr(latest[i]->system.arch, arch)) { - have_combo = 1; - } - } - } - if (!have_combo) { - continue; - } - fprintf(indexfp, "### %s-%s\n\n", platform, arch); - - fprintf(indexfp, "|Release|Info|Receipt|\n"); - fprintf(indexfp, "|:----:|:----:|:----:|\n"); - for (size_t i = 0; i < nelem; i++) { - char link_name[PATH_MAX]; - char readme_name[PATH_MAX]; - char conf_name[PATH_MAX]; - char conf_name_relative[PATH_MAX]; - if (!latest[i]) { - continue; - } - sprintf(link_name, "latest-py%s-%s-%s.yml", latest[i]->meta.python_compact, latest[i]->system.platform[DELIVERY_PLATFORM_RELEASE], latest[i]->system.arch); - sprintf(readme_name, "README-py%s-%s-%s.md", latest[i]->meta.python_compact, latest[i]->system.platform[DELIVERY_PLATFORM_RELEASE], latest[i]->system.arch); - sprintf(conf_name, "%s.ini", latest[i]->info.release_name); - sprintf(conf_name_relative, "../config/%s-rendered.ini", latest[i]->info.release_name); - if (strstr(link_name, platform) && strstr(link_name, arch)) { - fprintf(indexfp, "|[%s](%s)|[%s](%s)|[%s](%s)|\n", link_name, link_name, readme_name, readme_name, conf_name, conf_name_relative); - } - } - fprintf(indexfp, "\n"); - } - fprintf(indexfp, "\n"); - } - guard_strlist_free(&archs); - guard_strlist_free(&platforms); - fclose(indexfp); - popd(); - } else { - fprintf(stderr, "Unable to enter delivery directory: %s\n", ctx->storage.delivery_dir); - guard_free(latest); - return -1; - } - - // "latest" is an array of pointers to ctxs[]. Do not free the contents of the array. - guard_free(latest); - return 0; -} - -int indexer_junitxml_report(struct Delivery ctx[], size_t nelem) { - struct Delivery **latest = NULL; - latest = get_latest_deliveries(ctx, nelem); - - char indexfile[PATH_MAX] = {0}; - sprintf(indexfile, "%s/README.md", ctx->storage.results_dir); - - struct StrList *file_listing = listdir(ctx->storage.results_dir); - if (!file_listing) { - // no test results to process - return 0; - } - - if (!pushd(ctx->storage.results_dir)) { - FILE *indexfp; - indexfp = fopen(indexfile, "w+"); - if (!indexfp) { - fprintf(stderr, "Unable to open %s for writing\n", indexfile); - return -1; - } - struct StrList *archs = get_architectures(*latest, nelem); - struct StrList *platforms = get_platforms(*latest, nelem); - - fprintf(indexfp, "# %s-%s Test Report\n\n", ctx->meta.name, ctx->meta.version); - fprintf(indexfp, "## Current Release\n\n"); - for (size_t p = 0; p < strlist_count(platforms); p++) { - char *platform = strlist_item(platforms, p); - for (size_t a = 0; a < strlist_count(archs); a++) { - char *arch = strlist_item(archs, a); - int have_combo = 0; - for (size_t i = 0; i < nelem; i++) { - if (latest[i] && latest[i]->system.platform) { - if (strstr(latest[i]->system.platform[DELIVERY_PLATFORM_RELEASE], platform) && - strstr(latest[i]->system.arch, arch)) { - have_combo = 1; - break; - } - } - } - if (!have_combo) { - continue; - } - fprintf(indexfp, "### %s-%s\n\n", platform, arch); - - fprintf(indexfp, "|Suite|Duration|Fail |Skip |Error |\n"); - fprintf(indexfp, "|:----|:------:|:------:|:---:|:----:|\n"); - for (size_t f = 0; f < strlist_count(file_listing); f++) { - char *filename = strlist_item(file_listing, f); - if (!endswith(filename, ".xml")) { - continue; - } - - if (strstr(filename, platform) && strstr(filename, arch)) { - struct JUNIT_Testsuite *testsuite = junitxml_testsuite_read(filename); - if (testsuite) { - if (globals.verbose) { - printf("%s: duration: %0.4f, failed: %d, skipped: %d, errors: %d\n", filename, testsuite->time, testsuite->failures, testsuite->skipped, testsuite->errors); - } - fprintf(indexfp, "|[%s](%s)|%0.4f|%d|%d|%d|\n", filename, filename, testsuite->time, testsuite->failures, testsuite->skipped, testsuite->errors); - /* - * TODO: Display failure/skip/error output. - * - for (size_t i = 0; i < testsuite->_tc_inuse; i++) { - if (testsuite->testcase[i]->tc_result_state_type) { - printf("testcase: %s :: %s\n", testsuite->testcase[i]->classname, testsuite->testcase[i]->name); - if (testsuite->testcase[i]->tc_result_state_type == JUNIT_RESULT_STATE_FAILURE) { - printf("failure: %s\n", testsuite->testcase[i]->result_state.failure->message); - } else if (testsuite->testcase[i]->tc_result_state_type == JUNIT_RESULT_STATE_SKIPPED) { - printf("skipped: %s\n", testsuite->testcase[i]->result_state.skipped->message); - } - } - } - */ - junitxml_testsuite_free(&testsuite); - } else { - fprintf(stderr, "bad test suite: %s: %s\n", strerror(errno), filename); - continue; - } - } - } - fprintf(indexfp, "\n"); - } - fprintf(indexfp, "\n"); - } - guard_strlist_free(&archs); - guard_strlist_free(&platforms); - fclose(indexfp); - popd(); - } else { - fprintf(stderr, "Unable to enter delivery directory: %s\n", ctx->storage.delivery_dir); - guard_free(latest); - return -1; - } - - // "latest" is an array of pointers to ctxs[]. Do not free the contents of the array. - guard_free(latest); - return 0; -} - -void indexer_init_dirs(struct Delivery *ctx, const char *workdir) { - path_store(&ctx->storage.root, PATH_MAX, workdir, ""); - path_store(&ctx->storage.tmpdir, PATH_MAX, ctx->storage.root, "tmp"); - if (delivery_init_tmpdir(ctx)) { - fprintf(stderr, "Failed to configure temporary storage directory\n"); - exit(1); - } - path_store(&ctx->storage.output_dir, PATH_MAX, ctx->storage.root, ""); - path_store(&ctx->storage.tools_dir, PATH_MAX, ctx->storage.output_dir, "tools"); - path_store(&globals.conda_install_prefix, PATH_MAX, ctx->storage.tools_dir, "conda"); - path_store(&ctx->storage.cfgdump_dir, PATH_MAX, ctx->storage.output_dir, "config"); - path_store(&ctx->storage.meta_dir, PATH_MAX, ctx->storage.output_dir, "meta"); - path_store(&ctx->storage.delivery_dir, PATH_MAX, ctx->storage.output_dir, "delivery"); - path_store(&ctx->storage.package_dir, PATH_MAX, ctx->storage.output_dir, "packages"); - path_store(&ctx->storage.results_dir, PATH_MAX, ctx->storage.output_dir, "results"); - path_store(&ctx->storage.wheel_artifact_dir, PATH_MAX, ctx->storage.package_dir, "wheels"); - path_store(&ctx->storage.conda_artifact_dir, PATH_MAX, ctx->storage.package_dir, "conda"); - - char newpath[PATH_MAX] = {0}; - if (getenv("PATH")) { - sprintf(newpath, "%s/bin:%s", ctx->storage.tools_dir, getenv("PATH")); - setenv("PATH", newpath, 1); - } else { - SYSERROR("%s", "environment variable PATH is undefined. Unable to continue."); - exit(1); - } -} - -int main(int argc, char *argv[]) { - size_t rootdirs_total = 0; - char *destdir = NULL; - char **rootdirs = NULL; - int do_html = 0; - int c = 0; - int option_index = 0; - while ((c = getopt_long(argc, argv, "hd:vUw", long_options, &option_index)) != -1) { - switch (c) { - case 'h': - usage(path_basename(argv[0])); - exit(0); - case 'd': - if (mkdir(optarg, 0755)) { - if (errno != 0 && errno != EEXIST) { - SYSERROR("Unable to create destination directory, '%s': %s", optarg, strerror(errno)); - exit(1); - } - } - destdir = realpath(optarg, NULL); - break; - case 'U': - fflush(stdout); - fflush(stderr); - setvbuf(stdout, NULL, _IONBF, 0); - setvbuf(stderr, NULL, _IONBF, 0); - break; - case 'v': - globals.verbose = 1; - break; - case 'w': - do_html = 1; - break; - case '?': - default: - exit(1); - } - } - - int current_index = optind; - if (optind < argc) { - rootdirs_total = argc - current_index; - rootdirs = calloc(rootdirs_total + 1, sizeof(**rootdirs)); - - int i = 0; - while (optind < argc) { - if (argv[optind]) { - if (access(argv[optind], F_OK) < 0) { - fprintf(stderr, "%s: %s\n", argv[optind], strerror(errno)); - exit(1); - } - } - // use first positional argument - rootdirs[i] = realpath(argv[optind], NULL); - optind++; - break; - } - } - - if (isempty(destdir)) { - if (mkdir("output", 0755)) { - if (errno != 0 && errno != EEXIST) { - SYSERROR("Unable to create destination directory, '%s': %s", "output", strerror(errno)); - exit(1); - } - } - destdir = realpath("output", NULL); - } - - if (!rootdirs || !rootdirs_total) { - fprintf(stderr, "You must specify at least one STASIS root directory to index\n"); - exit(1); - } else { - for (size_t i = 0; i < rootdirs_total; i++) { - if (isempty(rootdirs[i]) || !strcmp(rootdirs[i], "/") || !strcmp(rootdirs[i], "\\")) { - SYSERROR("Unsafe directory: %s", rootdirs[i]); - exit(1); - } else if (access(rootdirs[i], F_OK)) { - SYSERROR("%s: %s", rootdirs[i], strerror(errno)); - exit(1); - } - } - } - - char stasis_sysconfdir_tmp[PATH_MAX]; - if (getenv("STASIS_SYSCONFDIR")) { - strncpy(stasis_sysconfdir_tmp, getenv("STASIS_SYSCONFDIR"), sizeof(stasis_sysconfdir_tmp) - 1); - } else { - strncpy(stasis_sysconfdir_tmp, STASIS_SYSCONFDIR, sizeof(stasis_sysconfdir_tmp) - 1); - } - - globals.sysconfdir = realpath(stasis_sysconfdir_tmp, NULL); - if (!globals.sysconfdir) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Unable to resolve path to configuration directory: %s\n", stasis_sysconfdir_tmp); - exit(1); - } - - char *workdir; - char workdir_template[PATH_MAX] = {0}; - char *system_tmp = getenv("TMPDIR"); - if (system_tmp) { - strcat(workdir_template, system_tmp); - } else { - strcat(workdir_template, "/tmp"); - } - strcat(workdir_template, "/stasis-combine.XXXXXX"); - workdir = mkdtemp(workdir_template); - if (!workdir) { - SYSERROR("Unable to create temporary directory: %s", workdir_template); - exit(1); - } else if (isempty(workdir) || !strcmp(workdir, "/") || !strcmp(workdir, "\\")) { - SYSERROR("Unsafe directory: %s", workdir); - exit(1); - } - - struct Delivery ctx; - memset(&ctx, 0, sizeof(ctx)); - - printf(BANNER, VERSION, AUTHOR); - - indexer_init_dirs(&ctx, workdir); - - msg(STASIS_MSG_L1, "%s delivery root %s\n", - rootdirs_total > 1 ? "Merging" : "Indexing", - rootdirs_total > 1 ? "directories" : "directory"); - if (indexer_combine_rootdirs(workdir, rootdirs, rootdirs_total)) { - SYSERROR("%s", "Copy operation failed"); - rmtree(workdir); - exit(1); - } - - if (access(ctx.storage.conda_artifact_dir, F_OK)) { - mkdirs(ctx.storage.conda_artifact_dir, 0755); - } - - if (access(ctx.storage.wheel_artifact_dir, F_OK)) { - mkdirs(ctx.storage.wheel_artifact_dir, 0755); - } - - struct MicromambaInfo m; - if (micromamba_configure(&ctx, &m)) { - SYSERROR("%s", "Unable to configure micromamba"); - exit(1); - } - - msg(STASIS_MSG_L1, "Indexing conda packages\n"); - if (indexer_conda(&ctx, m)) { - SYSERROR("%s", "Conda package indexing operation failed"); - exit(1); - } - - msg(STASIS_MSG_L1, "Indexing wheel packages\n"); - if (indexer_wheels(&ctx)) { - SYSERROR("%s", "Python package indexing operation failed"); - exit(1); - } - - msg(STASIS_MSG_L1, "Loading metadata\n"); - struct StrList *metafiles = NULL; - indexer_get_files(&metafiles, ctx.storage.meta_dir, "*.stasis"); - strlist_sort(metafiles, STASIS_SORT_LEN_ASCENDING); - struct Delivery local[strlist_count(metafiles)]; - - for (size_t i = 0; i < strlist_count(metafiles); i++) { - char *item = strlist_item(metafiles, i); - memset(&local[i], 0, sizeof(ctx)); - memcpy(&local[i], &ctx, sizeof(ctx)); - char path[PATH_MAX]; - sprintf(path, "%s/%s", ctx.storage.meta_dir, item); - if (globals.verbose) { - puts(path); - } - indexer_load_metadata(&local[i], path); - } - - msg(STASIS_MSG_L1, "Generating links to latest release iteration\n"); - if (indexer_symlinks(local, strlist_count(metafiles))) { - SYSERROR("%s", "Link generation failed"); - exit(1); - } - - msg(STASIS_MSG_L1, "Generating README.md\n"); - if (indexer_readmes(local, strlist_count(metafiles))) { - SYSERROR("%s", "README indexing operation failed"); - exit(1); - } - - msg(STASIS_MSG_L1, "Indexing test results\n"); - if (indexer_junitxml_report(local, strlist_count(metafiles))) { - SYSERROR("%s", "Test result indexing operation failed"); - exit(1); - } - - if (do_html) { - msg(STASIS_MSG_L1, "Generating HTML indexes\n"); - if (indexer_make_website(local)) { - SYSERROR("%s", "Site creation failed"); - exit(1); - } - } - - msg(STASIS_MSG_L1, "Copying indexed delivery to '%s'\n", destdir); - char cmd[PATH_MAX]; - memset(cmd, 0, sizeof(cmd)); - sprintf(cmd, "rsync -ah%s --delete --exclude 'tmp/' --exclude 'tools/' '%s/' '%s/'", globals.verbose ? "v" : "q", workdir, destdir); - guard_free(destdir); - - if (globals.verbose) { - puts(cmd); - } - - if (system(cmd)) { - SYSERROR("%s", "Copy operation failed"); - rmtree(workdir); - exit(1); - } - - msg(STASIS_MSG_L1, "Removing work directory: %s\n", workdir); - if (rmtree(workdir)) { - SYSERROR("Failed to remove work directory: %s", strerror(errno)); - } - - guard_free(destdir); - GENERIC_ARRAY_FREE(rootdirs); - guard_strlist_free(&metafiles); - delivery_free(&ctx); - globals_free(); - msg(STASIS_MSG_L1, "Done!\n"); - return 0; -} diff --git a/src/stasis_main.c b/src/stasis_main.c deleted file mode 100644 index eecc419..0000000 --- a/src/stasis_main.c +++ /dev/null @@ -1,794 +0,0 @@ -#include -#include -#include -#include -#include -#include "core.h" - -#define OPT_ALWAYS_UPDATE_BASE 1000 -#define OPT_NO_DOCKER 1001 -#define OPT_NO_ARTIFACTORY 1002 -#define OPT_NO_ARTIFACTORY_BUILD_INFO 1003 -#define OPT_NO_TESTING 1004 -#define OPT_OVERWRITE 1005 -#define OPT_NO_REWRITE_SPEC_STAGE_2 1006 -#define OPT_FAIL_FAST 1007 -#define OPT_NO_PARALLEL 1008 -#define OPT_POOL_STATUS_INTERVAL 1009 - -static struct option long_options[] = { - {"help", no_argument, 0, 'h'}, - {"version", no_argument, 0, 'V'}, - {"continue-on-error", no_argument, 0, 'C'}, - {"config", required_argument, 0, 'c'}, - {"cpu-limit", required_argument, 0, 'l'}, - {"pool-status-interval", required_argument, 0, OPT_POOL_STATUS_INTERVAL}, - {"python", required_argument, 0, 'p'}, - {"verbose", no_argument, 0, 'v'}, - {"unbuffered", no_argument, 0, 'U'}, - {"update-base", no_argument, 0, OPT_ALWAYS_UPDATE_BASE}, - {"fail-fast", no_argument, 0, OPT_FAIL_FAST}, - {"overwrite", no_argument, 0, OPT_OVERWRITE}, - {"no-docker", no_argument, 0, OPT_NO_DOCKER}, - {"no-artifactory", no_argument, 0, OPT_NO_ARTIFACTORY}, - {"no-artifactory-build-info", no_argument, 0, OPT_NO_ARTIFACTORY_BUILD_INFO}, - {"no-testing", no_argument, 0, OPT_NO_TESTING}, - {"no-parallel", no_argument, 0, OPT_NO_PARALLEL}, - {"no-rewrite", no_argument, 0, OPT_NO_REWRITE_SPEC_STAGE_2}, - {0, 0, 0, 0}, -}; - -const char *long_options_help[] = { - "Display this usage statement", - "Display program version", - "Allow tests to fail", - "Read configuration file", - "Number of processes to spawn concurrently (default: cpus - 1)", - "Report task status every n seconds (default: 30)", - "Override version of Python in configuration", - "Increase output verbosity", - "Disable line buffering", - "Update conda installation prior to STASIS environment creation", - "On error, immediately terminate all tasks", - "Overwrite an existing release", - "Do not build docker images", - "Do not upload artifacts to Artifactory", - "Do not upload build info objects to Artifactory", - "Do not execute test scripts", - "Do not execute tests in parallel", - "Do not rewrite paths and URLs in output files", - NULL, -}; - -static int get_option_max_width(struct option option[]) { - int i = 0; - int max = 0; - const int indent = 4; - while (option[i].name != 0) { - int len = (int) strlen(option[i].name); - if (option[i].has_arg) { - len += indent; - } - if (len > max) { - max = len; - } - i++; - } - return max; -} - -static void usage(char *progname) { - printf("usage: %s ", progname); - printf("[-"); - for (int x = 0; long_options[x].val != 0; x++) { - if (long_options[x].has_arg == no_argument && long_options[x].val <= 'z') { - putchar(long_options[x].val); - } - } - printf("] {DELIVERY_FILE}\n"); - - int width = get_option_max_width(long_options); - for (int x = 0; long_options[x].name != 0; x++) { - char tmp[STASIS_NAME_MAX] = {0}; - char output[sizeof(tmp)] = {0}; - char opt_long[50] = {0}; // --? [ARG]? - char opt_short[50] = {0}; // -? [ARG]? - - strcat(opt_long, "--"); - strcat(opt_long, long_options[x].name); - if (long_options[x].has_arg) { - strcat(opt_long, " ARG"); - } - - if (long_options[x].val <= 'z') { - strcat(opt_short, "-"); - opt_short[1] = (char) long_options[x].val; - if (long_options[x].has_arg) { - strcat(opt_short, " ARG"); - } - } else { - strcat(opt_short, " "); - } - - sprintf(tmp, " %%-%ds\t%%s\t\t%%s", width + 4); - sprintf(output, tmp, opt_long, opt_short, long_options_help[x]); - puts(output); - } -} - -static int callback_except_jf(const void *a, const void *b) { - const struct EnvCtl_Item *item = a; - const char *name = b; - - if (!globals.enable_artifactory) { - return STASIS_ENVCTL_RET_IGNORE; - } - - if (envctl_check_required(item->flags)) { - const char *content = getenv(name); - if (!content || isempty((char *) content)) { - return STASIS_ENVCTL_RET_FAIL; - } - } - - return STASIS_ENVCTL_RET_SUCCESS; -} - -static int callback_except_gh(const void *a, const void *b) { - const struct EnvCtl_Item *item = a; - const char *name = b; - //printf("GH exception check: %s\n", name); - if (envctl_check_required(item->flags) && envctl_check_present(item, name)) { - return STASIS_ENVCTL_RET_SUCCESS; - } - - return STASIS_ENVCTL_RET_FAIL; -} - -static void check_system_env_requirements() { - msg(STASIS_MSG_L1, "Checking environment\n"); - globals.envctl = envctl_init(); - envctl_register(&globals.envctl, STASIS_ENVCTL_PASSTHRU, NULL, "TMPDIR"); - envctl_register(&globals.envctl, STASIS_ENVCTL_PASSTHRU, NULL, "STASIS_ROOT"); - envctl_register(&globals.envctl, STASIS_ENVCTL_PASSTHRU, NULL, "STASIS_SYSCONFDIR"); - envctl_register(&globals.envctl, STASIS_ENVCTL_PASSTHRU, NULL, "STASIS_CPU_COUNT"); - envctl_register(&globals.envctl, STASIS_ENVCTL_REQUIRED | STASIS_ENVCTL_REDACT, callback_except_gh, "STASIS_GH_TOKEN"); - envctl_register(&globals.envctl, STASIS_ENVCTL_REQUIRED, callback_except_jf, "STASIS_JF_ARTIFACTORY_URL"); - envctl_register(&globals.envctl, STASIS_ENVCTL_REDACT, NULL, "STASIS_JF_ACCESS_TOKEN"); - envctl_register(&globals.envctl, STASIS_ENVCTL_PASSTHRU, NULL, "STASIS_JF_USER"); - envctl_register(&globals.envctl, STASIS_ENVCTL_REDACT, NULL, "STASIS_JF_PASSWORD"); - envctl_register(&globals.envctl, STASIS_ENVCTL_REDACT, NULL, "STASIS_JF_SSH_KEY_PATH"); - envctl_register(&globals.envctl, STASIS_ENVCTL_REDACT, NULL, "STASIS_JF_SSH_PASSPHRASE"); - envctl_register(&globals.envctl, STASIS_ENVCTL_REDACT, NULL, "STASIS_JF_CLIENT_CERT_CERT_PATH"); - envctl_register(&globals.envctl, STASIS_ENVCTL_REDACT, NULL, "STASIS_JF_CLIENT_CERT_KEY_PATH"); - envctl_register(&globals.envctl, STASIS_ENVCTL_REQUIRED, callback_except_jf, "STASIS_JF_REPO"); - envctl_do_required(globals.envctl, globals.verbose); -} - -static void check_system_requirements(struct Delivery *ctx) { - const char *tools_required[] = { - "rsync", - NULL, - }; - - msg(STASIS_MSG_L1, "Checking system requirements\n"); - for (size_t i = 0; tools_required[i] != NULL; i++) { - if (!find_program(tools_required[i])) { - msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "'%s' must be installed.\n", tools_required[i]); - exit(1); - } - } - - if (!globals.tmpdir && !ctx->storage.tmpdir) { - delivery_init_tmpdir(ctx); - } - - struct DockerCapabilities dcap; - if (!docker_capable(&dcap)) { - msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "Docker is broken\n"); - msg(STASIS_MSG_L3, "Available: %s\n", dcap.available ? "Yes" : "No"); - msg(STASIS_MSG_L3, "Usable: %s\n", dcap.usable ? "Yes" : "No"); - msg(STASIS_MSG_L3, "Podman [Docker Emulation]: %s\n", dcap.podman ? "Yes" : "No"); - msg(STASIS_MSG_L3, "Build plugin(s): "); - if (dcap.usable) { - if (dcap.build & STASIS_DOCKER_BUILD) { - printf("build "); - } - if (dcap.build & STASIS_DOCKER_BUILD_X) { - printf("buildx "); - } - puts(""); - } else { - printf("N/A\n"); - } - - // disable docker builds - globals.enable_docker = false; - } -} - -static void check_requirements(struct Delivery *ctx) { - check_system_requirements(ctx); - check_system_env_requirements(); -} - -int main(int argc, char *argv[]) { - struct Delivery ctx; - struct Process proc = { - .f_stdout = "", - .f_stderr = "", - .redirect_stderr = 0, - }; - char env_name[STASIS_NAME_MAX] = {0}; - char env_name_testing[STASIS_NAME_MAX] = {0}; - char *delivery_input = NULL; - char *config_input = NULL; - char installer_url[PATH_MAX]; - char python_override_version[STASIS_NAME_MAX]; - int user_disabled_docker = false; - globals.cpu_limit = get_cpu_count(); - if (globals.cpu_limit > 1) { - globals.cpu_limit--; // max - 1 - } - - memset(env_name, 0, sizeof(env_name)); - memset(env_name_testing, 0, sizeof(env_name_testing)); - memset(installer_url, 0, sizeof(installer_url)); - memset(python_override_version, 0, sizeof(python_override_version)); - memset(&proc, 0, sizeof(proc)); - memset(&ctx, 0, sizeof(ctx)); - - int c; - int option_index = 0; - while ((c = getopt_long(argc, argv, "hVCc:p:vU", long_options, &option_index)) != -1) { - switch (c) { - case 'h': - usage(path_basename(argv[0])); - exit(0); - case 'V': - puts(VERSION); - exit(0); - case 'c': - config_input = strdup(optarg); - break; - case 'C': - globals.continue_on_error = true; - break; - case 'p': - strcpy(python_override_version, optarg); - break; - case 'l': - globals.cpu_limit = strtol(optarg, NULL, 10); - if (globals.cpu_limit <= 1) { - globals.cpu_limit = 1; - globals.enable_parallel = false; // No point - } - break; - case OPT_ALWAYS_UPDATE_BASE: - globals.always_update_base_environment = true; - break; - case OPT_FAIL_FAST: - globals.parallel_fail_fast = true; - break; - case OPT_POOL_STATUS_INTERVAL: - globals.pool_status_interval = (int) strtol(optarg, NULL, 10); - if (globals.pool_status_interval < 1) { - globals.pool_status_interval = 1; - } else if (globals.pool_status_interval > 60 * 10) { - // Possible poor choice alert - fprintf(stderr, "Caution: Excessive pausing between status updates may cause third-party CI/CD" - " jobs to fail if the stdout/stderr streams are idle for too long!\n"); - } - break; - case 'U': - setenv("PYTHONUNBUFFERED", "1", 1); - fflush(stdout); - fflush(stderr); - setvbuf(stdout, NULL, _IONBF, 0); - setvbuf(stderr, NULL, _IONBF, 0); - break; - case 'v': - globals.verbose = true; - break; - case OPT_OVERWRITE: - globals.enable_overwrite = true; - break; - case OPT_NO_DOCKER: - globals.enable_docker = false; - user_disabled_docker = true; - break; - case OPT_NO_ARTIFACTORY: - globals.enable_artifactory = false; - break; - case OPT_NO_ARTIFACTORY_BUILD_INFO: - globals.enable_artifactory_build_info = false; - break; - case OPT_NO_TESTING: - globals.enable_testing = false; - break; - case OPT_NO_REWRITE_SPEC_STAGE_2: - globals.enable_rewrite_spec_stage_2 = false; - break; - case OPT_NO_PARALLEL: - globals.enable_parallel = false; - break; - case '?': - default: - exit(1); - } - } - - if (optind < argc) { - while (optind < argc) { - // use first positional argument - delivery_input = argv[optind++]; - break; - } - } - - if (!delivery_input) { - fprintf(stderr, "error: a DELIVERY_FILE is required\n"); - usage(path_basename(argv[0])); - exit(1); - } - - printf(BANNER, VERSION, AUTHOR); - - msg(STASIS_MSG_L1, "Setup\n"); - - // Expose variables for use with the template engine - // NOTE: These pointers are populated by delivery_init() so please avoid using - // tpl_render() until then. - tpl_register("meta.name", &ctx.meta.name); - tpl_register("meta.version", &ctx.meta.version); - tpl_register("meta.codename", &ctx.meta.codename); - tpl_register("meta.mission", &ctx.meta.mission); - tpl_register("meta.python", &ctx.meta.python); - tpl_register("meta.python_compact", &ctx.meta.python_compact); - tpl_register("info.time_str_epoch", &ctx.info.time_str_epoch); - tpl_register("info.release_name", &ctx.info.release_name); - tpl_register("info.build_name", &ctx.info.build_name); - tpl_register("info.build_number", &ctx.info.build_number); - tpl_register("storage.tmpdir", &ctx.storage.tmpdir); - tpl_register("storage.output_dir", &ctx.storage.output_dir); - tpl_register("storage.delivery_dir", &ctx.storage.delivery_dir); - tpl_register("storage.conda_artifact_dir", &ctx.storage.conda_artifact_dir); - tpl_register("storage.wheel_artifact_dir", &ctx.storage.wheel_artifact_dir); - tpl_register("storage.build_sources_dir", &ctx.storage.build_sources_dir); - tpl_register("storage.build_docker_dir", &ctx.storage.build_docker_dir); - tpl_register("storage.results_dir", &ctx.storage.results_dir); - tpl_register("storage.tools_dir", &ctx.storage.tools_dir); - tpl_register("conda.installer_baseurl", &ctx.conda.installer_baseurl); - tpl_register("conda.installer_name", &ctx.conda.installer_name); - tpl_register("conda.installer_version", &ctx.conda.installer_version); - tpl_register("conda.installer_arch", &ctx.conda.installer_arch); - tpl_register("conda.installer_platform", &ctx.conda.installer_platform); - tpl_register("deploy.jfrog.repo", &globals.jfrog.repo); - tpl_register("deploy.jfrog.url", &globals.jfrog.url); - tpl_register("deploy.docker.registry", &ctx.deploy.docker.registry); - tpl_register("workaround.conda_reactivate", &globals.workaround.conda_reactivate); - - // Expose function(s) to the template engine - // Prototypes can be found in template_func_proto.h - tpl_register_func("get_github_release_notes", &get_github_release_notes_tplfunc_entrypoint, 3, NULL); - tpl_register_func("get_github_release_notes_auto", &get_github_release_notes_auto_tplfunc_entrypoint, 1, &ctx); - tpl_register_func("junitxml_file", &get_junitxml_file_entrypoint, 1, &ctx); - tpl_register_func("basetemp_dir", &get_basetemp_dir_entrypoint, 1, &ctx); - tpl_register_func("tox_run", &tox_run_entrypoint, 2, &ctx); - - // Set up PREFIX/etc directory information - // The user may manipulate the base directory path with STASIS_SYSCONFDIR - // environment variable - char stasis_sysconfdir_tmp[PATH_MAX]; - if (getenv("STASIS_SYSCONFDIR")) { - strncpy(stasis_sysconfdir_tmp, getenv("STASIS_SYSCONFDIR"), sizeof(stasis_sysconfdir_tmp) - 1); - } else { - strncpy(stasis_sysconfdir_tmp, STASIS_SYSCONFDIR, sizeof(stasis_sysconfdir_tmp) - 1); - } - - globals.sysconfdir = realpath(stasis_sysconfdir_tmp, NULL); - if (!globals.sysconfdir) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Unable to resolve path to configuration directory: %s\n", stasis_sysconfdir_tmp); - exit(1); - } - - // Override Python version from command-line, if any - if (strlen(python_override_version)) { - guard_free(ctx.meta.python); - ctx.meta.python = strdup(python_override_version); - guard_free(ctx.meta.python_compact); - ctx.meta.python_compact = to_short_version(ctx.meta.python); - } - - if (!config_input) { - // no configuration passed by argument. use basic config. - char cfgfile[PATH_MAX * 2]; - sprintf(cfgfile, "%s/%s", globals.sysconfdir, "stasis.ini"); - if (!access(cfgfile, F_OK | R_OK)) { - config_input = strdup(cfgfile); - } else { - msg(STASIS_MSG_WARN, "STASIS global configuration is not readable, or does not exist: %s", cfgfile); - } - } - - if (config_input) { - msg(STASIS_MSG_L2, "Reading STASIS global configuration: %s\n", config_input); - ctx._stasis_ini_fp.cfg = ini_open(config_input); - if (!ctx._stasis_ini_fp.cfg) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read config file: %s, %s\n", delivery_input, strerror(errno)); - exit(1); - } - ctx._stasis_ini_fp.cfg_path = strdup(config_input); - guard_free(config_input); - } - - msg(STASIS_MSG_L2, "Reading STASIS delivery configuration: %s\n", delivery_input); - ctx._stasis_ini_fp.delivery = ini_open(delivery_input); - if (!ctx._stasis_ini_fp.delivery) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read delivery file: %s, %s\n", delivery_input, strerror(errno)); - exit(1); - } - ctx._stasis_ini_fp.delivery_path = strdup(delivery_input); - - msg(STASIS_MSG_L2, "Bootstrapping delivery context\n"); - if (bootstrap_build_info(&ctx)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to bootstrap delivery context\n"); - exit(1); - } - - msg(STASIS_MSG_L2, "Initializing delivery context\n"); - if (delivery_init(&ctx, INI_READ_RENDER)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to initialize delivery context\n"); - exit(1); - } - check_requirements(&ctx); - - msg(STASIS_MSG_L2, "Configuring JFrog CLI\n"); - if (delivery_init_artifactory(&ctx)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "JFrog CLI configuration failed\n"); - exit(1); - } - - runtime_apply(ctx.runtime.environ); - strcpy(env_name, ctx.info.release_name); - strcpy(env_name_testing, env_name); - strcat(env_name_testing, "-test"); - - // Safety gate: Avoid clobbering a delivered release unless the user wants that behavior - msg(STASIS_MSG_L1, "Checking release history\n"); - if (delivery_exists(&ctx)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Refusing to overwrite release: %s\nUse --overwrite to enable release clobbering.\n", ctx.info.release_name); - exit(1); - } - - msg(STASIS_MSG_L1, "Conda setup\n"); - delivery_get_conda_installer_url(&ctx, installer_url); - msg(STASIS_MSG_L2, "Downloading: %s\n", installer_url); - if (delivery_get_conda_installer(&ctx, installer_url)) { - msg(STASIS_MSG_ERROR, "download failed: %s\n", installer_url); - exit(1); - } - - // Unlikely to occur: this should help prevent rmtree() from destroying your entire filesystem - // if path is "/" then, die - // or if empty string, die - if (!strcmp(ctx.storage.conda_install_prefix, DIR_SEP) || !strlen(ctx.storage.conda_install_prefix)) { - fprintf(stderr, "error: ctx.storage.conda_install_prefix is malformed!\n"); - exit(1); - } - - msg(STASIS_MSG_L2, "Installing: %s\n", ctx.conda.installer_name); - delivery_install_conda(ctx.conda.installer_path, ctx.storage.conda_install_prefix); - - msg(STASIS_MSG_L2, "Configuring: %s\n", ctx.storage.conda_install_prefix); - delivery_conda_enable(&ctx, ctx.storage.conda_install_prefix); - - char *pathvar = NULL; - pathvar = getenv("PATH"); - if (!pathvar) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "PATH variable is not set. Cannot continue.\n"); - exit(1); - } else { - char pathvar_tmp[STASIS_BUFSIZ]; - sprintf(pathvar_tmp, "%s/bin:%s", ctx.storage.conda_install_prefix, pathvar); - setenv("PATH", pathvar_tmp, 1); - pathvar = NULL; - } - - - // - // Implied environment creation modes/actions - // - // 1. No base environment config - // 1a. Caller is warned - // 1b. Caller has full control over all packages - // 2. Default base environment (etc/stasis/mission/[name]/base.yml) - // 2a. Depends on packages defined by base.yml - // 2b. Caller may issue a reduced package set in the INI config - // 2c. Caller must be vigilant to avoid incompatible packages (base.yml - // *should* have no version constraints) - // 3. External base environment (based_on=schema://[release_name].yml) - // 3a. Depends on a previous release or arbitrary yaml configuration - // 3b. Bugs, conflicts, and dependency resolution issues are inherited and - // must be handled in the INI config - msg(STASIS_MSG_L1, "Creating release environment(s)\n"); - - char *mission_base = NULL; - if (isempty(ctx.meta.based_on)) { - guard_free(ctx.meta.based_on); - char *mission_base_orig = NULL; - - if (asprintf(&mission_base_orig, "%s/%s/base.yml", ctx.storage.mission_dir, ctx.meta.mission) < 0) { - SYSERROR("Unable to allocate bytes for %s/%s/base.yml path\n", ctx.storage.mission_dir, ctx.meta.mission); - exit(1); - } - - if (access(mission_base_orig, F_OK) < 0) { - msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "Mission does not provide a base.yml configuration: %s (%s)\n", - ctx.meta.mission, ctx.storage.mission_dir); - } else { - msg(STASIS_MSG_L2, "Using base environment configuration: %s\n", mission_base_orig); - if (asprintf(&mission_base, "%s/%s-base.yml", ctx.storage.tmpdir, ctx.info.release_name) < 0) { - SYSERROR("%s", "Unable to allocate bytes for temporary base.yml configuration"); - remove(mission_base); - exit(1); - } - copy2(mission_base_orig, mission_base, CT_OWNER | CT_PERM); - char spec[255] = {0}; - snprintf(spec, sizeof(spec) - 1, "- python=%s\n", ctx.meta.python); - file_replace_text(mission_base, "- python\n", spec, 0); - ctx.meta.based_on = mission_base; - } - guard_free(mission_base_orig); - } - - if (!isempty(ctx.meta.based_on)) { - if (conda_env_remove(env_name)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to remove release environment: %s\n", env_name); - exit(1); - } - - msg(STASIS_MSG_L2, "Based on: %s\n", ctx.meta.based_on); - if (conda_env_create_from_uri(env_name, ctx.meta.based_on)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "unable to install release environment using configuration file\n"); - exit(1); - } - - if (conda_env_remove(env_name_testing)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to remove testing environment %s\n", env_name_testing); - exit(1); - } - if (conda_env_create_from_uri(env_name_testing, ctx.meta.based_on)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "unable to install testing environment using configuration file\n"); - exit(1); - } - } else { - if (conda_env_create(env_name, ctx.meta.python, NULL)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to create release environment\n"); - exit(1); - } - if (conda_env_create(env_name_testing, ctx.meta.python, NULL)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed to create testing environment\n"); - exit(1); - } - } - // The base environment configuration not used past this point - remove(mission_base); - - // Activate test environment - msg(STASIS_MSG_L1, "Activating test environment\n"); - if (conda_activate(ctx.storage.conda_install_prefix, env_name_testing)) { - fprintf(stderr, "failed to activate test environment\n"); - exit(1); - } - - delivery_gather_tool_versions(&ctx); - if (!ctx.conda.tool_version) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Could not determine conda version\n"); - exit(1); - } - if (!ctx.conda.tool_build_version) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Could not determine conda-build version\n"); - exit(1); - } - - if (pip_exec("install build")) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "'build' tool installation failed\n"); - exit(1); - } - - if (!isempty(ctx.meta.based_on)) { - msg(STASIS_MSG_L1, "Generating package overlay from environment: %s\n", env_name); - if (delivery_overlay_packages_from_env(&ctx, env_name)) { - msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "%s", "Failed to generate package overlay. Resulting environment integrity cannot be guaranteed.\n"); - exit(1); - } - } - - msg(STASIS_MSG_L1, "Filter deliverable packages\n"); - delivery_defer_packages(&ctx, DEFER_CONDA); - delivery_defer_packages(&ctx, DEFER_PIP); - - msg(STASIS_MSG_L1, "Overview\n"); - delivery_meta_show(&ctx); - delivery_conda_show(&ctx); - if (globals.verbose) { - //delivery_runtime_show(&ctx); - } - - // Execute configuration-defined tests - if (globals.enable_testing) { - delivery_tests_show(&ctx); - - msg(STASIS_MSG_L1, "Begin test execution\n"); - delivery_tests_run(&ctx); - msg(STASIS_MSG_L2, "Rewriting test results\n"); - delivery_fixup_test_results(&ctx); - } else { - msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Test execution is disabled\n"); - } - - if (ctx.conda.conda_packages_defer && strlist_count(ctx.conda.conda_packages_defer)) { - msg(STASIS_MSG_L2, "Building Conda recipe(s)\n"); - if (delivery_build_recipes(&ctx)) { - exit(1); - } - msg(STASIS_MSG_L3, "Copying artifacts\n"); - if (delivery_copy_conda_artifacts(&ctx)) { - exit(1); - } - msg(STASIS_MSG_L3, "Indexing artifacts\n"); - if (delivery_index_conda_artifacts(&ctx)) { - exit(1); - } - } - - if (strlist_count(ctx.conda.pip_packages_defer)) { - if (!(ctx.conda.wheels_packages = delivery_build_wheels(&ctx))) { - exit(1); - } - if (delivery_index_wheel_artifacts(&ctx)) { - exit(1); - } - - } - - // Populate the release environment - msg(STASIS_MSG_L1, "Populating release environment\n"); - msg(STASIS_MSG_L2, "Installing conda packages\n"); - if (strlist_count(ctx.conda.conda_packages)) { - if (delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_CONDA, (struct StrList *[]) {ctx.conda.conda_packages, NULL})) { - exit(1); - } - } - if (strlist_count(ctx.conda.conda_packages_defer)) { - msg(STASIS_MSG_L3, "Installing deferred conda packages\n"); - if (delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_CONDA | INSTALL_PKG_CONDA_DEFERRED, (struct StrList *[]) {ctx.conda.conda_packages_defer, NULL})) { - exit(1); - } - } else { - msg(STASIS_MSG_L3, "No deferred conda packages\n"); - } - - msg(STASIS_MSG_L2, "Installing pip packages\n"); - if (strlist_count(ctx.conda.pip_packages)) { - if (delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_PIP, (struct StrList *[]) {ctx.conda.pip_packages, NULL})) { - exit(1); - } - } - - if (strlist_count(ctx.conda.pip_packages_defer)) { - msg(STASIS_MSG_L3, "Installing deferred pip packages\n"); - if (delivery_install_packages(&ctx, ctx.storage.conda_install_prefix, env_name, INSTALL_PKG_PIP | INSTALL_PKG_PIP_DEFERRED, (struct StrList *[]) {ctx.conda.pip_packages_defer, NULL})) { - exit(1); - } - } else { - msg(STASIS_MSG_L3, "No deferred pip packages\n"); - } - - conda_exec("list"); - - msg(STASIS_MSG_L1, "Creating release\n"); - msg(STASIS_MSG_L2, "Exporting delivery configuration\n"); - if (!pushd(ctx.storage.cfgdump_dir)) { - char filename[PATH_MAX] = {0}; - sprintf(filename, "%s.ini", ctx.info.release_name); - FILE *spec = fopen(filename, "w+"); - if (!spec) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", filename); - exit(1); - } - ini_write(ctx._stasis_ini_fp.delivery, &spec, INI_WRITE_RAW); - fclose(spec); - - memset(filename, 0, sizeof(filename)); - sprintf(filename, "%s-rendered.ini", ctx.info.release_name); - spec = fopen(filename, "w+"); - if (!spec) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", filename); - exit(1); - } - ini_write(ctx._stasis_ini_fp.delivery, &spec, INI_WRITE_PRESERVE); - fclose(spec); - popd(); - } else { - SYSERROR("Failed to enter directory: %s", ctx.storage.delivery_dir); - exit(1); - } - - msg(STASIS_MSG_L2, "Exporting %s\n", env_name_testing); - if (conda_env_export(env_name_testing, ctx.storage.delivery_dir, env_name_testing)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", env_name_testing); - exit(1); - } - - msg(STASIS_MSG_L2, "Exporting %s\n", env_name); - if (conda_env_export(env_name, ctx.storage.delivery_dir, env_name)) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "failed %s\n", env_name); - exit(1); - } - - // Rewrite release environment output (i.e. set package origin(s) to point to the deployment server, etc.) - char specfile[PATH_MAX]; - sprintf(specfile, "%s/%s.yml", ctx.storage.delivery_dir, env_name); - msg(STASIS_MSG_L3, "Rewriting release spec file (stage 1): %s\n", path_basename(specfile)); - delivery_rewrite_spec(&ctx, specfile, DELIVERY_REWRITE_SPEC_STAGE_1); - - msg(STASIS_MSG_L1, "Rendering mission templates\n"); - delivery_mission_render_files(&ctx); - - int want_docker = ini_section_search(&ctx._stasis_ini_fp.delivery, INI_SEARCH_BEGINS, "deploy:docker") ? true : false; - int want_artifactory = ini_section_search(&ctx._stasis_ini_fp.delivery, INI_SEARCH_BEGINS, "deploy:artifactory") ? true : false; - - if (want_docker) { - if (user_disabled_docker) { - msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Docker image building is disabled by CLI argument\n"); - } else { - char dockerfile[PATH_MAX] = {0}; - sprintf(dockerfile, "%s/%s", ctx.storage.build_docker_dir, "Dockerfile"); - if (globals.enable_docker) { - if (!access(dockerfile, F_OK)) { - msg(STASIS_MSG_L1, "Building Docker image\n"); - if (delivery_docker(&ctx)) { - msg(STASIS_MSG_L1 | STASIS_MSG_ERROR, "Failed to build docker image!\n"); - COE_CHECK_ABORT(1, "Failed to build docker image"); - } - } else { - msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Docker image building is disabled. No Dockerfile found in %s\n", ctx.storage.build_docker_dir); - } - } else { - msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Docker image building is disabled. System configuration error\n"); - } - } - } else { - msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Docker image building is disabled. deploy:docker is not configured\n"); - } - - msg(STASIS_MSG_L3, "Rewriting release spec file (stage 2): %s\n", path_basename(specfile)); - delivery_rewrite_spec(&ctx, specfile, DELIVERY_REWRITE_SPEC_STAGE_2); - - msg(STASIS_MSG_L1, "Dumping metadata\n"); - if (delivery_dump_metadata(&ctx)) { - msg(STASIS_MSG_L1 | STASIS_MSG_ERROR, "Metadata dump failed\n"); - } - - if (want_artifactory) { - if (globals.enable_artifactory) { - msg(STASIS_MSG_L1, "Uploading artifacts\n"); - delivery_artifact_upload(&ctx); - } else { - msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Artifactory upload is disabled by CLI argument\n"); - } - } else { - msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "Artifactory upload is disabled. deploy:artifactory is not configured\n"); - } - - msg(STASIS_MSG_L1, "Cleaning up\n"); - delivery_free(&ctx); - globals_free(); - tpl_free(); - - msg(STASIS_MSG_L1, "Done!\n"); - return 0; -} - diff --git a/src/str.c b/src/str.c deleted file mode 100644 index 868a6c7..0000000 --- a/src/str.c +++ /dev/null @@ -1,654 +0,0 @@ -/** - * @file strings.c - */ -#include -#include "str.h" - -int num_chars(const char *sptr, int ch) { - int result = 0; - for (int i = 0; sptr[i] != '\0'; i++) { - if (sptr[i] == ch) { - result++; - } - } - return result; -} - -int startswith(const char *sptr, const char *pattern) { - if (!sptr || !pattern) { - return 0; - } - for (size_t i = 0; i < strlen(pattern); i++) { - if (sptr[i] != pattern[i]) { - return 0; - } - } - return 1; -} - -int endswith(const char *sptr, const char *pattern) { - if (!sptr || !pattern) { - return 0; - } - ssize_t sptr_size = (ssize_t) strlen(sptr); - ssize_t pattern_size = (ssize_t) strlen(pattern); - - if (sptr_size == pattern_size) { - if (strcmp(sptr, pattern) == 0) { - return 1; // yes - } - return 0; // no - } - - ssize_t s = sptr_size - pattern_size; - if (s < 0) { - return 0; - } - - for (size_t p = 0 ; s < sptr_size; s++, p++) { - if (sptr[s] != pattern[p]) { - // sptr does not end with pattern - return 0; - } - } - // sptr ends with pattern - return 1; -} - -void strchrdel(char *sptr, const char *chars) { - if (sptr == NULL || chars == NULL) { - return; - } - - for (size_t i = 0; i < strlen(chars); i++) { - char ch[2] = {0}; - strncpy(ch, &chars[i], 1); - replace_text(sptr, ch, "", 0); - } -} - -char** split(char *_sptr, const char* delim, size_t max) -{ - if (_sptr == NULL || delim == NULL) { - return NULL; - } - size_t split_alloc = 0; - // Duplicate the input string and save a copy of the pointer to be freed later - char *orig = _sptr; - char *sptr = strdup(orig); - - if (!sptr) { - return NULL; - } - - // Determine how many delimiters are present - for (size_t i = 0; i < strlen(delim); i++) { - if (max && i > max) { - break; - } - split_alloc += num_chars(sptr, delim[i]); - } - - // Preallocate enough records based on the number of delimiters - char **result = calloc(split_alloc + 2, sizeof(result[0])); - if (!result) { - guard_free(sptr); - return NULL; - } - - // No delimiter, but the string was not NULL, so return the original string - if (split_alloc == 0) { - result[0] = sptr; - return result; - } - - // Separate the string into individual parts and store them in the result array - char *token = NULL; - char *sptr_tmp = sptr; - size_t pos = 0; - size_t i; - for (i = 0; (token = strsep(&sptr_tmp, delim)) != NULL; i++) { - // When max is zero, record all tokens - if (max > 0 && i == max) { - // Maximum number of splits occurred. - // Record position in string - pos = token - sptr; - break; - } - result[i] = calloc(STASIS_BUFSIZ, sizeof(char)); - if (!result[i]) { - return NULL; - } - strcpy(result[i], token); - } - - // pos is non-zero when maximum split is reached - if (pos) { - // append the remaining string contents to array - result[i] = calloc(STASIS_BUFSIZ, sizeof(char)); - if (!result[i]) { - return NULL; - } - strcpy(result[i], &orig[pos]); - } - - guard_free(sptr); - return result; -} - -char *join(char **arr, const char *separator) { - char *result = NULL; - int records = 0; - size_t total_bytes = 0; - - if (!arr || !separator) { - return NULL; - } - - for (int i = 0; arr[i] != NULL; i++) { - total_bytes += strlen(arr[i]); - records++; - } - total_bytes += (records * strlen(separator)) + 1; - - result = (char *)calloc(total_bytes, sizeof(char)); - for (int i = 0; i < records; i++) { - strcat(result, arr[i]); - if (i < (records - 1)) { - strcat(result, separator); - } - } - return result; -} - -char *join_ex(char *separator, ...) { - va_list ap; // Variadic argument list - size_t separator_len = 0; // Length of separator string - size_t size = 0; // Length of output string - size_t argc = 0; // Number of arguments ^ "..." - char **argv = NULL; // Arguments - char *current = NULL; // Current argument - char *result = NULL; // Output string - - if (separator == NULL) { - return NULL; - } - - // Initialize array - argv = calloc(argc + 1, sizeof(char **)); - if (argv == NULL) { - perror("join_ex calloc failed"); - return NULL; - } - - // Get length of the separator - separator_len = strlen(separator); - - // Process variadic arguments: - // 1. Iterate over argument list `ap` - // 2. Assign `current` with the value of argument in `ap` - // 3. Extend the `argv` array by the latest argument count `argc` - // 4. Sum the length of the argument and the `separator` passed to the function - // 5. Append `current` string to `argv` array - // 6. Update argument counter `argc` - va_start(ap, separator); - for(argc = 0; (current = va_arg(ap, char *)) != NULL; argc++) { - char **tmp = realloc(argv, (argc + 1) * sizeof(char *)); - if (tmp == NULL) { - perror("join_ex realloc failed"); - guard_free(argv); - return NULL; - } else { - argv = tmp; - } - size += strlen(current) + separator_len; - argv[argc] = strdup(current); - } - va_end(ap); - - // Generate output string - result = calloc(size + 1, sizeof(char)); - for (size_t i = 0; i < argc; i++) { - // Append argument to string - strcat(result, argv[i]); - - // Do not append a trailing separator when we reach the last argument - if (i < (argc - 1)) { - strcat(result, separator); - } - guard_free(argv[i]); - } - guard_free(argv); - - return result; -} - -char *substring_between(char *sptr, const char *delims) { - char delim_open[255] = {0}; - char delim_close[255] = {0}; - if (sptr == NULL || delims == NULL) { - return NULL; - } - - // Ensure we have enough delimiters to continue - size_t delim_count = strlen(delims); - if (delim_count < 2 || delim_count % 2 || (delim_count > (sizeof(delim_open) - 1)) != 0) { - return NULL; - } - size_t delim_take = delim_count / 2; - - // How else am I supposed to consume the first and last n chars of the string? Give me a break. - // warning: ‘__builtin___strncpy_chk’ specified bound depends on the length of the source argument - // --- - //strncpy(delim_open, delims, delim_take); - size_t i = 0; - while (i < delim_take && i < sizeof(delim_open)) { - delim_open[i] = delims[i]; - i++; - } - - //strncpy(delim_close, &delims[delim_take], delim_take); - i = 0; - while (i < delim_take && i < sizeof(delim_close)) { - delim_close[i] = delims[i + delim_take]; - i++; - } - - // Create pointers to the delimiters - char *start = strstr(sptr, delim_open); - if (start == NULL || strlen(start) == 0) { - return NULL; - } - - char *end = strstr(start + 1, delim_close); - if (end == NULL) { - return NULL; - } - - start += delim_count / 2; // ignore leading delimiter - - // Get length of the substring - size_t length = strlen(start) - strlen(end); - if (!length) { - return NULL; - } - - // Return the contents of the substring - return strndup(start, length); -} - -/* - * Comparison functions for `strsort` - */ -static int _strsort_alpha_compare(const void *a, const void *b) { - const char *aa = *(const char **)a; - const char *bb = *(const char **)b; - int result = strcmp(aa, bb); - return result; -} - -static int _strsort_numeric_compare(const void *a, const void *b) { - const char *aa = *(const char **)a; - const char *bb = *(const char **)b; - - if (isdigit(*aa) && isdigit(*bb)) { - long ia = strtol(aa, NULL, 10); - long ib = strtol(bb, NULL, 10); - - if (ia == ib) { - return 0; - } else if (ia < ib) { - return -1; - } else if (ia > ib) { - return 1; - } - } - return 0; -} - -static int _strsort_asc_compare(const void *a, const void *b) { - const char *aa = *(const char**)a; - const char *bb = *(const char**)b; - size_t len_a = strlen(aa); - size_t len_b = strlen(bb); - return len_a > len_b; -} - -/* - * Helper function for `strsortlen` - */ -static int _strsort_dsc_compare(const void *a, const void *b) { - const char *aa = *(const char**)a; - const char *bb = *(const char**)b; - size_t len_a = strlen(aa); - size_t len_b = strlen(bb); - return len_a < len_b; -} - -void strsort(char **arr, unsigned int sort_mode) { - if (arr == NULL) { - return; - } - - typedef int (*compar)(const void *, const void *); - // Default mode is alphabetic sort - compar fn = _strsort_alpha_compare; - - if (sort_mode == STASIS_SORT_LEN_DESCENDING) { - fn = _strsort_dsc_compare; - } else if (sort_mode == STASIS_SORT_LEN_ASCENDING) { - fn = _strsort_asc_compare; - } else if (sort_mode == STASIS_SORT_ALPHA) { - fn = _strsort_alpha_compare; // ^ still selectable though ^ - } else if (sort_mode == STASIS_SORT_NUMERIC) { - fn = _strsort_numeric_compare; - } - - size_t arr_size = 0; - - // Determine size of array (+ terminator) - for (size_t i = 0; arr[i] != NULL; i++) { - arr_size = i; - } - arr_size++; - - qsort(arr, arr_size, sizeof(char *), fn); -} - - -char *strstr_array(char **arr, const char *str) { - if (arr == NULL || str == NULL) { - return NULL; - } - - for (int i = 0; arr[i] != NULL; i++) { - if (strstr(arr[i], str) != NULL) { - return arr[i]; - } - } - return NULL; -} - - -char **strdeldup(char **arr) { - if (!arr) { - return NULL; - } - - size_t records; - // Determine the length of the array - for (records = 0; arr[records] != NULL; records++); - - // Allocate enough memory to store the original array contents - // (It might not have duplicate values, for example) - char **result = (char **)calloc(records + 1, sizeof(char *)); - if (!result) { - return NULL; - } - - int rec = 0; - size_t i = 0; - while(i < records) { - // Search for value in results - if (strstr_array(result, arr[i]) != NULL) { - // value already exists in results so ignore it - i++; - continue; - } - - // Store unique value - result[rec] = strdup(arr[i]); - if (!result[rec]) { - for (size_t die = 0; result[die] != NULL; die++) { - guard_free(result[die]); - } - guard_free(result); - return NULL; - } - - i++; - rec++; - } - return result; -} - -char *lstrip(char *sptr) { - char *tmp = sptr; - size_t bytes = 0; - - if (sptr == NULL) { - return NULL; - } - - while (strlen(tmp) > 1 && (isblank(*tmp) || isspace(*tmp))) { - bytes++; - tmp++; - } - if (tmp != sptr) { - memmove(sptr, sptr + bytes, strlen(sptr) - bytes); - memset((sptr + strlen(sptr)) - bytes, '\0', bytes); - } - return sptr; -} - -char *strip(char *sptr) { - if (sptr == NULL) { - return NULL; - } - - size_t len = strlen(sptr); - if (len == 0) { - return sptr; - } else if (len == 1 && (isblank(*sptr) || isspace(*sptr))) { - *sptr = '\0'; - return sptr; - } for (size_t i = len; i != 0; --i) { - if (sptr[i] == '\0') { - continue; - } - if (isspace(sptr[i]) || isblank(sptr[i])) { - sptr[i] = '\0'; - } else { - break; - } - } - return sptr; -} - -int isempty(char *sptr) { - if (sptr == NULL) { - return -1; - } - - char *tmp = sptr; - while (*tmp) { - if (!isblank(*tmp) && !isspace(*tmp) && !iscntrl(*tmp)) { - return 0; - } - tmp++; - } - return 1; -} - - -int isquoted(char *sptr) { - const char *quotes = "'\""; - - if (sptr == NULL) { - return -1; - } - - char *quote_open = strpbrk(sptr, quotes); - if (!quote_open) { - return 0; - } - char *quote_close = strpbrk(quote_open + 1, quotes); - if (!quote_close) { - return 0; - } - return 1; -} - -int isrelational(char ch) { - char symbols[] = "~!=<>"; - char *symbol = symbols; - while (*symbol != '\0') { - if (ch == *symbol) { - return 1; - } - symbol++; - } - return 0; -} - -void print_banner(const char *s, int len) { - size_t s_len = strlen(s); - if (!s_len) { - return; - } - for (size_t i = 0; i < (len / s_len); i++) { - for (size_t c = 0; c < s_len; c++) { - putchar(s[c]); - } - } - putchar('\n'); -} - -/** - * Collapse whitespace in `s`. The string is modified in place. - * @param s - * @return pointer to `s` - */ -char *normalize_space(char *s) { - size_t len; - size_t trim_pos; - int add_whitespace = 0; - char *result = s; - char *tmp; - - if (s == NULL) { - return NULL; - } - - if ((tmp = calloc(strlen(s) + 1, sizeof(char))) == NULL) { - perror("could not allocate memory for temporary string"); - return NULL; - } - char *tmp_orig = tmp; - - // count whitespace, if any - for (trim_pos = 0; isblank(s[trim_pos]); trim_pos++); - // trim whitespace from the left, if any - memmove(s, &s[trim_pos], strlen(&s[trim_pos])); - // cull bytes not part of the string after moving - len = strlen(s); - s[len - trim_pos] = '\0'; - - // Generate a new string with extra whitespace stripped out - while (*s != '\0') { - // Skip over any whitespace, but record that we encountered it - if (isblank(*s)) { - s++; - add_whitespace = 1; - continue; - } - // This gate avoids filling tmp with whitespace; we want to make our own - if (add_whitespace) { - *tmp = ' '; - tmp++; - add_whitespace = 0; - } - // Write character in s to tmp - *tmp = *s; - // Increment string pointers - s++; - tmp++; - } - - // Rewrite the input string - strcpy(result, tmp_orig); - guard_free(tmp_orig); - return result; -} - -char **strdup_array(char **array) { - char **result = NULL; - size_t elems = 0; - - // Guard - if (array == NULL) { - return NULL; - } - - // Count elements in `array` - for (elems = 0; array[elems] != NULL; elems++); - - // Create new array - result = calloc(elems + 1, sizeof(*result)); - for (size_t i = 0; i < elems; i++) { - result[i] = strdup(array[i]); - } - - return result; -} - -int strcmp_array(const char **a, const char **b) { - size_t a_len = 0; - size_t b_len = 0; - - // This could lead to false-positives depending on what the caller plans to achieve - if (a == NULL && b == NULL) { - return 0; - } else if (a == NULL) { - return -1; - } else if (b == NULL) { - return 1; - } - - // Get length of arrays - for (a_len = 0; a[a_len] != NULL; a_len++); - for (b_len = 0; b[b_len] != NULL; b_len++); - - // Check lengths are equal - if (a_len < b_len) return (int)(b_len - a_len); - else if (a_len > b_len) return (int)(a_len - b_len); - - // Compare strings in the arrays returning the total difference in bytes - int result = 0; - for (size_t ai = 0, bi = 0 ;a[ai] != NULL || b[bi] != NULL; ai++, bi++) { - int status = 0; - if ((status = strcmp(a[ai], b[bi]) != 0)) { - result += status; - } - } - return result; -} - -int isdigit_s(const char *s) { - if (!s || !strlen(s)) { - return 0; // nothing to do, fail - } - for (size_t i = 0; s[i] != '\0'; i++) { - if (isdigit(s[i]) == 0) { - return 0; // non-digit found, fail - } - } - return 1; // all digits, succeed -} - -char *tolower_s(char *s) { - for (size_t i = 0; s[i] != '\0'; i++) { - s[i] = (char)tolower(s[i]); - } - return s; -} - -char *to_short_version(const char *s) { - char *result; - result = strdup(s); - if (!result) { - return NULL; - } - strchrdel(result, "."); - return result; -} diff --git a/src/strlist.c b/src/strlist.c deleted file mode 100644 index 7a045f1..0000000 --- a/src/strlist.c +++ /dev/null @@ -1,658 +0,0 @@ -/** - * String array convenience functions - * @file strlist.c - */ -#include "strlist.h" -#include "utils.h" - -/** - * - * @param pStrList `StrList` - */ -void strlist_free(struct StrList **pStrList) { - if (!(*pStrList)) { - return; - } - - for (size_t i = 0; i < (*pStrList)->num_inuse; i++) { - if ((*pStrList)->data[i]) { - guard_free((*pStrList)->data[i]); - } - } - if ((*pStrList)->data) { - guard_free((*pStrList)->data); - } - guard_free((*pStrList)); -} - -/** - * Append a value to the list - * @param pStrList `StrList` - * @param str - */ -void strlist_append(struct StrList **pStrList, char *str) { - char **tmp = NULL; - - if (pStrList == NULL) { - return; - } - - tmp = realloc((*pStrList)->data, ((*pStrList)->num_alloc + 1) * sizeof(char *)); - if (tmp == NULL) { - guard_strlist_free(pStrList); - perror("failed to append to array"); - exit(1); - } else if (tmp != (*pStrList)->data) { - (*pStrList)->data = tmp; - } - (*pStrList)->data[(*pStrList)->num_inuse] = strdup(str); - (*pStrList)->data[(*pStrList)->num_alloc] = NULL; - strcpy((*pStrList)->data[(*pStrList)->num_inuse], str); - (*pStrList)->num_inuse++; - (*pStrList)->num_alloc++; -} - -static int reader_strlist_append_file(size_t lineno, char **line) { - (void)(lineno); // unused parameter - (void)(line); // unused parameter - return 0; -} - -/** - * Append lines from a local file or remote URL (HTTP/s only) - * @param pStrList - * @param path file path or HTTP/s address - * @param readerFn pointer to a reader function (use NULL to retrieve all data) - * @return 0=success 1=no data, -1=error (spmerrno set) - */ -int strlist_append_file(struct StrList *pStrList, char *_path, ReaderFn *readerFn) { - int retval = 0; - char *path = NULL; - char *filename = NULL; - char **data = NULL; - int is_url = strstr(_path, "://") != NULL; - - if (readerFn == NULL) { - readerFn = reader_strlist_append_file; - } - - path = strdup(_path); - if (path == NULL) { - retval = -1; - goto fatal; - } - - if (is_url) { - int fd; - char tempfile[PATH_MAX] = {0}; - strcpy(tempfile, "/tmp/.remote_file.XXXXXX"); - if ((fd = mkstemp(tempfile)) < 0) { - retval = -1; - goto fatal; - } - close(fd); - filename = strdup(tempfile); - long http_code = download(path, filename, NULL); - if (HTTP_ERROR(http_code)) { - retval = -1; - goto fatal; - } - } else { - filename = expandpath(path); - if (filename == NULL) { - retval = -1; - goto fatal; - } - } - - data = file_readlines(filename, 0, 0, readerFn); - if (data == NULL) { - retval = 1; - goto fatal; - } - for (size_t record = 0; data[record] != NULL; record++) { - strlist_append(&pStrList, data[record]); - guard_free(data[record]); - } - if (is_url) { - // remove temporary data - remove(filename); - } - guard_free(data); - -fatal: - guard_free(filename); - if (path != NULL) { - guard_free(path); - } - - return retval; -} - -/** - * Append the contents of `pStrList2` to `pStrList1` - * @param pStrList1 `StrList` - * @param pStrList2 `StrList` - */ -void strlist_append_strlist(struct StrList *pStrList1, struct StrList *pStrList2) { - size_t count = 0; - - if (pStrList1 == NULL || pStrList2 == NULL) { - return; - } - - count = strlist_count(pStrList2); - for (size_t i = 0; i < count; i++) { - char *item = strlist_item(pStrList2, i); - strlist_append(&pStrList1, item); - } -} - -/** - * Append the contents of an array of pointers to char - * @param pStrList `StrList` - * @param arr NULL terminated array of strings - */ - void strlist_append_array(struct StrList *pStrList, char **arr) { - if (!pStrList || !arr) { - return; - } - for (size_t i = 0; arr[i] != NULL; i++) { - strlist_append(&pStrList, arr[i]); - } - } - -/** - * Append the contents of a newline delimited string - * @param pStrList `StrList` - * @param str - * @param delim - */ - void strlist_append_tokenize(struct StrList *pStrList, char *str, char *delim) { - char **token; - if (!str || !delim) { - return; - } - - char *tmp = strdup(str); - token = split(tmp, delim, 0); - if (token) { - for (size_t i = 0; token[i] != NULL; i++) { - lstrip(token[i]); - strlist_append(&pStrList, token[i]); - } - GENERIC_ARRAY_FREE(token); - } - guard_free(tmp); - } - -/** - * Produce a new copy of a `StrList` - * @param pStrList `StrList` - * @return `StrList` copy - */ -struct StrList *strlist_copy(struct StrList *pStrList) { - struct StrList *result; - if (pStrList == NULL) { - return NULL; - } - - result = strlist_init(); - if (!result) { - return NULL; - } - - for (size_t i = 0; i < strlist_count(pStrList); i++) { - strlist_append(&result, strlist_item(pStrList, i)); - } - return result; -} - -/** - * Remove a record by index from a `StrList` - * @param pStrList - * @param index - */ -void strlist_remove(struct StrList *pStrList, size_t index) { - size_t count = strlist_count(pStrList); - if (count == 0) { - return; - } - if (pStrList->data[index] != NULL) { - for (size_t i = index; i < count; i++) { - pStrList->data[i] = pStrList->data[i + 1]; - } - if (pStrList->num_inuse) { - pStrList->num_inuse--; - } - } -} - -/** - * Compare two `StrList`s - * @param a `StrList` structure - * @param b `StrList` structure - * @return same=0, different=1, error=-1 (a is NULL), -2 (b is NULL) - */ -int strlist_cmp(struct StrList *a, struct StrList *b) { - if (a == NULL) { - return -1; - } - - if (b == NULL) { - return -2; - } - - if (a->num_alloc != b->num_alloc || a->num_inuse != b->num_inuse) { - return 1; - } - - - for (size_t i = 0; i < strlist_count(a); i++) { - if (strcmp(strlist_item(a, i), strlist_item(b, i)) != 0) { - return 1; - } - } - - return 0; -} - -/** - * Sort a `StrList` by `mode` - * @param pStrList - * @param mode Available modes: `STRLIST_DEFAULT` (alphabetic), `STRLIST_ASC` (ascending), `STRLIST_DSC` (descending) - */ -void strlist_sort(struct StrList *pStrList, unsigned int mode) { - if (pStrList == NULL) { - return; - } - strsort(pStrList->data, mode); -} - -/** - * Reverse the order of a `StrList` - * @param pStrList - */ -void strlist_reverse(struct StrList *pStrList) { - char *tmp = NULL; - size_t i = 0; - size_t j = 0; - - if (pStrList == NULL) { - return; - } - - j = pStrList->num_inuse - 1; - for (i = 0; i < j; i++) { - tmp = pStrList->data[i]; - pStrList->data[i] = pStrList->data[j]; - pStrList->data[j] = tmp; - j--; - } -} - -/** - * Get the count of values stored in a `StrList` - * @param pStrList - * @return - */ -size_t strlist_count(struct StrList *pStrList) { - size_t result; - if (pStrList != NULL) { - result = pStrList->num_inuse; - } else { - result = 0; - } - return result; -} - -/** - * Set value at index - * @param pStrList - * @param value string - * @return - */ -void strlist_set(struct StrList **pStrList, size_t index, char *value) { - char *tmp = NULL; - if (*pStrList == NULL || index > strlist_count(*pStrList)) { - strlist_errno = STRLIST_E_OUT_OF_RANGE; - return; - } - - if (value == NULL) { - guard_free((*pStrList)->data[index]); - } else { - tmp = realloc((*pStrList)->data[index], (strlen(value) + 1) * sizeof(char *)); - if (!tmp) { - perror("realloc strlist_set replacement value"); - return; - } else if (tmp != (*pStrList)->data[index]) { - (*pStrList)->data[index] = tmp; - } - - memset((*pStrList)->data[index], '\0', strlen(value) + 1); - strcpy((*pStrList)->data[index], value); - } -} - -const char *strlist_error_msgs[] = { - "success", - "index out of range", - "invalid value for type", - "unknown error", -}; -int strlist_errno = 0; - -void strlist_set_error(int flag) { - strlist_errno = flag; -} - -const char *strlist_get_error(int flag) { - if (flag < STRLIST_E_SUCCESS || flag > STRLIST_E_UNKNOWN) { - return strlist_error_msgs[STRLIST_E_UNKNOWN]; - } - return strlist_error_msgs[flag]; -} - -void strlist_clear_error() { - strlist_errno = STRLIST_E_SUCCESS; -} - -/** - * Retrieve data from a `StrList` - * @param pStrList - * @param index - * @return string - */ -char *strlist_item(struct StrList *pStrList, size_t index) { - if (pStrList && pStrList->data && pStrList->data[index]) { - return pStrList->data[index]; - } - return NULL; -} - -/** - * Alias of `strlist_item` - * @param pStrList - * @param index - * @return string - */ -char *strlist_item_as_str(struct StrList *pStrList, size_t index) { - return strlist_item(pStrList, index); -} - -/** - * Convert value at index to `char` - * @param pStrList - * @param index - * @return `char` - */ -char strlist_item_as_char(struct StrList *pStrList, size_t index) { - char *error_p; - char result; - - strlist_clear_error(); - result = (char) strtol(strlist_item(pStrList, index), &error_p, 10); - if (!result && error_p && *error_p != 0) { - strlist_set_error(STRLIST_E_INVALID_VALUE); - return 0; - } - error_p = NULL; - return result; -} - -/** - * Convert value at index to `unsigned char` - * @param pStrList - * @param index - * @return `unsigned char` - */ -unsigned char strlist_item_as_uchar(struct StrList *pStrList, size_t index) { - char *error_p; - unsigned char result; - - strlist_clear_error(); - result = (unsigned char) strtoul(strlist_item(pStrList, index), &error_p, 10); - if (!result && error_p && *error_p != 0) { - strlist_set_error(STRLIST_E_INVALID_VALUE); - return 0; - } - error_p = NULL; - return result; -} - -/** - * Convert value at index to `short` - * @param pStrList - * @param index - * @return `short` - */ -short strlist_item_as_short(struct StrList *pStrList, size_t index) { - char *error_p; - short result; - - strlist_clear_error(); - result = (short) strtol(strlist_item(pStrList, index), &error_p, 10); - if (!result && error_p && *error_p != 0) { - strlist_set_error(STRLIST_E_INVALID_VALUE); - return 0; - } - error_p = NULL; - return result; -} - -/** - * Convert value at index to `unsigned short` - * @param pStrList - * @param index - * @return `unsigned short` - */ -unsigned short strlist_item_as_ushort(struct StrList *pStrList, size_t index) { - char *error_p; - unsigned short result; - - strlist_clear_error(); - result = (unsigned short) strtoul(strlist_item(pStrList, index), &error_p, 10); - if (!result && error_p && *error_p != 0) { - strlist_set_error(STRLIST_E_INVALID_VALUE); - return 0; - } - error_p = NULL; - return result; -} - -/** - * Convert value at index to `int` - * @param pStrList - * @param index - * @return `int` - */ -int strlist_item_as_int(struct StrList *pStrList, size_t index) { - char *error_p; - int result; - - strlist_clear_error(); - result = (int) strtol(strlist_item(pStrList, index), &error_p, 10); - if (!result && error_p && *error_p != 0) { - strlist_set_error(STRLIST_E_INVALID_VALUE); - return 0; - } - error_p = NULL; - return result; -} - -/** - * Convert value at index to `unsigned int` - * @param pStrList - * @param index - * @return `unsigned int` - */ -unsigned int strlist_item_as_uint(struct StrList *pStrList, size_t index) { - char *error_p; - unsigned int result; - - strlist_clear_error(); - result = (unsigned int) strtoul(strlist_item(pStrList, index), &error_p, 10); - if (!result && error_p && *error_p != 0) { - strlist_set_error(STRLIST_E_INVALID_VALUE); - return 0; - } - error_p = NULL; - return result; -} - -/** - * Convert value at index to `long` - * @param pStrList - * @param index - * @return `long` - */ -long strlist_item_as_long(struct StrList *pStrList, size_t index) { - char *error_p; - long result; - - strlist_clear_error(); - result = (long) strtol(strlist_item(pStrList, index), &error_p, 10); - if (!result && error_p && *error_p != 0) { - strlist_set_error(STRLIST_E_INVALID_VALUE); - return 0; - } - error_p = NULL; - return result; -} - -/** - * Convert value at index to `unsigned long` - * @param pStrList - * @param index - * @return `unsigned long` - */ -unsigned long strlist_item_as_ulong(struct StrList *pStrList, size_t index) { - char *error_p; - unsigned long result; - - strlist_clear_error(); - result = (unsigned long) strtoul(strlist_item(pStrList, index), &error_p, 10); - if (!result && error_p && *error_p != 0) { - strlist_set_error(STRLIST_E_INVALID_VALUE); - return 0; - } - error_p = NULL; - return result; -} - -/** - * Convert value at index to `long long` - * @param pStrList - * @param index - * @return `long long` - */ -long long strlist_item_as_long_long(struct StrList *pStrList, size_t index) { - char *error_p; - long long result; - - strlist_clear_error(); - result = (long long) strtoll(strlist_item(pStrList, index), &error_p, 10); - if (!result && error_p && *error_p != 0) { - strlist_set_error(STRLIST_E_INVALID_VALUE); - return 0; - } - error_p = NULL; - return result; -} - -/** - * Convert value at index to `unsigned long long` - * @param pStrList - * @param index - * @return `unsigned long long` - */ -unsigned long long strlist_item_as_ulong_long(struct StrList *pStrList, size_t index) { - char *error_p; - unsigned long long result; - - strlist_clear_error(); - result = (unsigned long long) strtol(strlist_item(pStrList, index), &error_p, 10); - if (!result && error_p && *error_p != 0) { - strlist_set_error(STRLIST_E_INVALID_VALUE); - return 0; - } - error_p = NULL; - return result; -} - -/** - * Convert value at index to `float` - * @param pStrList - * @param index - * @return `float` - */ -float strlist_item_as_float(struct StrList *pStrList, size_t index) { - char *error_p; - float result; - - strlist_clear_error(); - result = (float) strtof(strlist_item(pStrList, index), &error_p); - if (!result && error_p && *error_p != 0) { - strlist_set_error(STRLIST_E_INVALID_VALUE); - return 0; - } - error_p = NULL; - return result; -} - -/** - * Convert value at index to `double` - * @param pStrList - * @param index - * @return `double` - */ -double strlist_item_as_double(struct StrList *pStrList, size_t index) { - char *error_p; - double result; - - strlist_clear_error(); - result = (double) strtod(strlist_item(pStrList, index), &error_p); - if (!result && error_p && *error_p != 0) { - strlist_set_error(STRLIST_E_INVALID_VALUE); - return 0; - } - error_p = NULL; - return result; -} - -/** - * Convert value at index to `long double` - * @param pStrList - * @param index - * @return `long double` - */ -long double strlist_item_as_long_double(struct StrList *pStrList, size_t index) { - char *error_p; - long double result; - - strlist_clear_error(); - result = (long double) strtold(strlist_item(pStrList, index), &error_p); - if (!result && error_p && *error_p != 0) { - strlist_set_error(STRLIST_E_INVALID_VALUE); - return 0; - } - error_p = NULL; - return result; -} - -/** - * Initialize an empty `StrList` - * @return `StrList` - */ -struct StrList *strlist_init() { - struct StrList *pStrList = calloc(1, sizeof(struct StrList)); - if (pStrList == NULL) { - perror("failed to allocate array"); - return NULL; - } - pStrList->num_inuse = 0; - pStrList->num_alloc = 1; - pStrList->data = calloc(pStrList->num_alloc, sizeof(char *)); - return pStrList; -} diff --git a/src/system.c b/src/system.c deleted file mode 100644 index 4e605ec..0000000 --- a/src/system.c +++ /dev/null @@ -1,173 +0,0 @@ -#include "system.h" -#include "core.h" - -int shell(struct Process *proc, char *args) { - struct Process selfproc; - pid_t pid; - pid_t status; - status = 0; - errno = 0; - - if (!proc) { - // provide our own proc structure - // albeit not accessible to the user - memset(&selfproc, 0, sizeof(selfproc)); - proc = &selfproc; - } - - if (!args) { - proc->returncode = -1; - return -1; - } - - FILE *tp = NULL; - char *t_name; - t_name = xmkstemp(&tp, "w"); - if (!t_name || !tp) { - return -1; - } - - fprintf(tp, "#!/bin/bash\n%s\n", args); - fflush(tp); - fclose(tp); - - // Set the script's permissions so that only the calling user can use it - // This should help prevent eavesdropping if keys are applied in plain-text - // somewhere. - chmod(t_name, 0700); - - pid = fork(); - if (pid == -1) { - fprintf(stderr, "fork failed\n"); - exit(1); - } else if (pid == 0) { - FILE *fp_out = NULL; - FILE *fp_err = NULL; - - if (strlen(proc->f_stdout)) { - fp_out = freopen(proc->f_stdout, "w+", stdout); - if (!fp_out) { - fprintf(stderr, "Unable to redirect stdout to %s: %s\n", proc->f_stdout, strerror(errno)); - exit(1); - } - } - - if (strlen(proc->f_stderr)) { - if (!proc->redirect_stderr) { - fp_err = freopen(proc->f_stderr, "w+", stderr); - if (!fp_err) { - fprintf(stderr, "Unable to redirect stderr to %s: %s\n", proc->f_stdout, strerror(errno)); - exit(1); - } - } - } - - if (proc->redirect_stderr) { - if (fp_err) { - fclose(fp_err); - fclose(stderr); - } - if (dup2(fileno(stdout), fileno(stderr)) < 0) { - fprintf(stderr, "Unable to redirect stderr to stdout: %s\n", strerror(errno)); - exit(1); - } - } - - return execl("/bin/bash", "bash", "--norc", t_name, (char *) NULL); - } else { - if (waitpid(pid, &status, WUNTRACED) > 0) { - if (WIFEXITED(status) && WEXITSTATUS(status)) { - if (WEXITSTATUS(status) == 127) { - fprintf(stderr, "execv failed\n"); - } - } else if (WIFSIGNALED(status)) { - fprintf(stderr, "signal received: %d\n", WIFSIGNALED(status)); - } - } else { - fprintf(stderr, "waitpid() failed\n"); - } - } - - if (!access(t_name, F_OK)) { - remove(t_name); - } - - proc->returncode = status; - guard_free(t_name); - return WEXITSTATUS(status); -} - -int shell_safe(struct Process *proc, char *args) { - FILE *fp; - char buf[1024] = {0}; - int result; - - char *invalid_ch = strpbrk(args, STASIS_SHELL_SAFE_RESTRICT); - if (invalid_ch) { - args = NULL; - } - - result = shell(proc, args); - if (strlen(proc->f_stdout)) { - fp = fopen(proc->f_stdout, "r"); - if (fp) { - while (fgets(buf, sizeof(buf) - 1, fp)) { - fprintf(stdout, "%s", buf); - buf[0] = '\0'; - } - fclose(fp); - fp = NULL; - } - } - if (strlen(proc->f_stderr)) { - fp = fopen(proc->f_stderr, "r"); - if (fp) { - while (fgets(buf, sizeof(buf) - 1, fp)) { - fprintf(stderr, "%s", buf); - buf[0] = '\0'; - } - fclose(fp); - fp = NULL; - } - } - return result; -} - -char *shell_output(const char *command, int *status) { - const size_t initial_size = STASIS_BUFSIZ; - size_t current_size = initial_size; - char *result = NULL; - char line[STASIS_BUFSIZ]; - FILE *pp; - - errno = 0; - *status = 0; - pp = popen(command, "r"); - if (!pp) { - *status = -1; - return NULL; - } - - if (errno) { - *status = 1; - } - result = calloc(initial_size, sizeof(result)); - memset(line, 0, sizeof(line)); - while (fread(line, sizeof(char), sizeof(line) - 1, pp) != 0) { - size_t result_len = strlen(result); - size_t need_realloc = (result_len + strlen(line)) > current_size; - if (need_realloc) { - current_size += initial_size; - char *tmp = realloc(result, sizeof(*result) * current_size); - if (!tmp) { - return NULL; - } else if (tmp != result) { - result = tmp; - } - } - strcat(result, line); - memset(line, 0, sizeof(line)); - } - *status = pclose(pp); - return result; -} diff --git a/src/template.c b/src/template.c deleted file mode 100644 index a412fa8..0000000 --- a/src/template.c +++ /dev/null @@ -1,318 +0,0 @@ -// -// Created by jhunk on 12/17/23. -// - -#include "template.h" - -#include -#include -#include -#include - - -struct tpl_item { - char *key; - char **ptr; -}; -struct tpl_item *tpl_pool[1024] = {0}; -unsigned tpl_pool_used = 0; -struct tplfunc_frame *tpl_pool_func[1024] = {0}; -unsigned tpl_pool_func_used = 0; - -extern void tpl_reset() { - tpl_free(); - tpl_pool_used = 0; - tpl_pool_func_used = 0; -} - -void tpl_register_func(char *key, void *tplfunc_ptr, int argc, void *data_in) { - struct tplfunc_frame *frame = calloc(1, sizeof(*frame)); - frame->key = strdup(key); - frame->argc = argc; - frame->func = tplfunc_ptr; - frame->data_in = data_in; - - tpl_pool_func[tpl_pool_func_used] = frame; - tpl_pool_func_used++; -} - -int tpl_key_exists(char *key) { - for (size_t i = 0; i < tpl_pool_used; i++) { - if (tpl_pool[i]->key) { - if (!strcmp(tpl_pool[i]->key, key)) { - return true; - } - } - } - return false; -} - -void tpl_register(char *key, char **ptr) { - struct tpl_item *item = NULL; - int replacing = 0; - - if (tpl_key_exists(key)) { - for (size_t i = 0; i < tpl_pool_used; i++) { - if (tpl_pool[i]->key) { - if (!strcmp(tpl_pool[i]->key, key)) { - item = tpl_pool[i]; - break; - } - } - } - replacing = 1; - } else { - item = calloc(1, sizeof(*item)); - item->key = strdup(key); - } - - if (!item) { - SYSERROR("unable to register tpl_item for %s", key); - exit(1); - } - - item->ptr = ptr; - if (!replacing) { - tpl_pool[tpl_pool_used] = item; - tpl_pool_used++; - } -} - -void tpl_free() { - for (unsigned i = 0; i < tpl_pool_used; i++) { - struct tpl_item *item = tpl_pool[i]; - if (item) { - if (item->key) { -#ifdef DEBUG - SYSERROR("freeing template item key: %s", item->key); -#endif - guard_free(item->key); - } -#ifdef DEBUG - SYSERROR("freeing template item: %p", item); -#endif - item->ptr = NULL; - } - guard_free(item); - } - for (unsigned i = 0; i < tpl_pool_func_used; i++) { - struct tplfunc_frame *item = tpl_pool_func[i]; - guard_free(item->key); - guard_free(item); - } -} - -char *tpl_getval(char *key) { - char *result = NULL; - for (size_t i = 0; i < tpl_pool_used; i++) { - if (tpl_pool[i]->key) { - if (!strcmp(tpl_pool[i]->key, key)) { - result = *tpl_pool[i]->ptr; - break; - } - } - } - return result; -} - -struct tplfunc_frame *tpl_getfunc(char *key) { - struct tplfunc_frame *result = NULL; - for (size_t i = 0; i < tpl_pool_func_used; i++) { - if (tpl_pool_func[i]->key) { - if (!strcmp(tpl_pool_func[i]->key, key)) { - result = tpl_pool_func[i]; - break; - } - } - } - return result; -} - -static int grow(size_t z, size_t *output_bytes, char **output) { - if (z >= *output_bytes) { - size_t new_size = *output_bytes + z + 1; -#ifdef DEBUG - fprintf(stderr, "template output buffer new size: %zu\n", new_size); -#endif - char *tmp = realloc(*output, new_size); - if (!tmp) { - perror("realloc failed"); - return -1; - } else if (tmp != *output) { - *output = tmp; - } - *output_bytes = new_size; - } - return 0; -} - -char *tpl_render(char *str) { - if (!str) { - return NULL; - } else if (!strlen(str)) { - return strdup(""); - } - size_t output_bytes = 1024 + strlen(str); // TODO: Is grow working correctly? - char *output = NULL; - char *b_close = NULL; - char *pos = NULL; - pos = str; - - output = calloc(output_bytes, sizeof(*output)); - if (!output) { - perror("unable to allocate output buffer"); - return NULL; - } - - for (size_t off = 0, z = 0; off < strlen(str); off++) { - char key[255] = {0}; - char *value = NULL; - - memset(key, 0, sizeof(key)); - grow(z, &output_bytes, &output); - // At opening brace - if (!strncmp(&pos[off], "{{", 2)) { - // Scan until key is reached - while (!isalnum(pos[off])) { - off++; - } - - // Read key name - size_t key_len = 0; - while (isalnum(pos[off]) || pos[off] != '}') { - if (isspace(pos[off]) || isblank(pos[off])) { - // skip whitespace in key - off++; - continue; - } - key[key_len] = pos[off]; - key_len++; - off++; - } - - char *type_stop = NULL; - type_stop = strchr(key, ':'); - - int do_env = 0; - int do_func = 0; - if (type_stop) { - if (!strncmp(key, "env", type_stop - key)) { - do_env = 1; - } else if (!strncmp(key, "func", type_stop - key)) { - do_func = 1; - } - } - - // Find closing brace - b_close = strstr(&pos[off], "}}"); - if (!b_close) { - fprintf(stderr, "error while templating '%s'\n\nunbalanced brace at position %zu\n", str, z); - return NULL; - } else { - // Jump past closing brace - off = ((b_close + 2) - pos); - } - - if (do_env) { // {{ env:VAR }} - char *k = type_stop + 1; - size_t klen = strlen(k); - memmove(key, k, klen); - key[klen] = 0; - char *env_val = getenv(key); - value = strdup(env_val ? env_val : ""); - } else if (do_func) { // {{ func:NAME(a, ...) }} - char func_name_temp[STASIS_NAME_MAX] = {0}; - strcpy(func_name_temp, type_stop + 1); - char *param_begin = strchr(func_name_temp, '('); - if (!param_begin) { - fprintf(stderr, "At position %zu in %s\nfunction name must be followed by a '('\n", off, key); - guard_free(output); - return NULL; - } - *param_begin = 0; - param_begin++; - char *param_end = strrchr(param_begin, ')'); - if (!param_end) { - fprintf(stderr, "At position %zu in %s\nfunction arguments must be closed with a ')'\n", off, key); - guard_free(output); - return NULL; - } - *param_end = 0; - char *k = func_name_temp; - char **params = split(param_begin, ",", 0); - int params_count; - for (params_count = 0; params[params_count] != NULL; params_count++); - - struct tplfunc_frame *frame = tpl_getfunc(k); - if (params_count > frame->argc || params_count < frame->argc) { - fprintf(stderr, "At position %zu in %s\nIncorrect number of arguments for function: %s (expected %d, got %d)\n", off, key, frame->key, frame->argc, params_count); - value = strdup(""); - } else { - for (size_t p = 0; p < sizeof(frame->argv) / sizeof(*frame->argv) && params[p] != NULL; p++) { - lstrip(params[p]); - strip(params[p]); - frame->argv[p].t_char_ptr = params[p]; - } - char *func_result = NULL; - int func_status = 0; - if ((func_status = frame->func(frame, &func_result))) { - fprintf(stderr, "%s returned non-zero status: %d\n", frame->key, func_status); - } - value = strdup(func_result ? func_result : ""); - guard_free(func_result); - } - GENERIC_ARRAY_FREE(params); - } else { - // Read replacement value - value = strdup(tpl_getval(key) ? tpl_getval(key) : ""); - } - } - - if (value) { - // Set output iterator to end of replacement value - z += strlen(value); - - // Append replacement value - grow(z, &output_bytes, &output); - strcat(output, value); - guard_free(value); - output[z] = 0; - } - -#ifdef DEBUG - fprintf(stderr, "z=%zu, output_bytes=%zu\n", z, output_bytes); -#endif - output[z] = pos[off]; - z++; - } -#ifdef DEBUG - fprintf(stderr, "template output length: %zu\n", strlen(output)); - fprintf(stderr, "template output bytes: %zu\n", output_bytes); -#endif - return output; -} - -int tpl_render_to_file(char *str, const char *filename) { - char *result; - FILE *fp; - - // Render the input string - result = tpl_render(str); - if (!result) { - return -1; - } - - // Open the destination file for writing - fp = fopen(filename, "w+"); - if (!fp) { - guard_free(result); - return -1; - } - - // Write rendered string to file - fprintf(fp, "%s", result); - fclose(fp); - - guard_free(result); - return 0; -} \ No newline at end of file diff --git a/src/template_func_proto.c b/src/template_func_proto.c deleted file mode 100644 index 9c325bb..0000000 --- a/src/template_func_proto.c +++ /dev/null @@ -1,158 +0,0 @@ -#include "template_func_proto.h" - -int get_github_release_notes_tplfunc_entrypoint(void *frame, void *data_out) { - int result; - char **output = (char **) data_out; - struct tplfunc_frame *f = (struct tplfunc_frame *) frame; - char *api_token = getenv("STASIS_GH_TOKEN"); - if (!api_token) { - api_token = getenv("GITHUB_TOKEN"); - } - result = get_github_release_notes(api_token ? api_token : "anonymous", - (const char *) f->argv[0].t_char_ptr, - (const char *) f->argv[1].t_char_ptr, - (const char *) f->argv[2].t_char_ptr, - output); - return result; -} - -int get_github_release_notes_auto_tplfunc_entrypoint(void *frame, void *data_out) { - int result = 0; - char **output = (char **) data_out; - struct tplfunc_frame *f = (struct tplfunc_frame *) frame; - char *api_token = getenv("STASIS_GH_TOKEN"); - if (!api_token) { - api_token = getenv("GITHUB_TOKEN"); - } - - const struct Delivery *ctx = (struct Delivery *) f->data_in; - struct StrList *notes_list = strlist_init(); - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(*ctx->tests); i++) { - // Get test context - const struct Test *test = &ctx->tests[i]; - if (test->name && test->version && test->repository) { - char *repository = strdup(test->repository); - char *match = strstr(repository, "spacetelescope/"); - // Cull repository URL - if (match) { - replace_text(repository, "https://github.com/", "", 0); - if (endswith(repository, ".git")) { - replace_text(repository, ".git", "", 0); - } - // Record release notes for version relative to HEAD - // Using HEAD, GitHub returns the previous tag - char *note = NULL; - char h1_title[NAME_MAX] = {0}; - sprintf(h1_title, "# %s", test->name); - strlist_append(¬es_list, h1_title); - result += get_github_release_notes(api_token ? api_token : "anonymous", - repository, - test->version, - "HEAD", - ¬e); - if (note) { - strlist_append(¬es_list, note); - guard_free(note); - } - guard_free(repository); - } - } - } - // Return all notes as a single string - if (strlist_count(notes_list)) { - *output = join(notes_list->data, "\n\n"); - } - guard_strlist_free(¬es_list); - - return result; -} - -int get_junitxml_file_entrypoint(void *frame, void *data_out) { - int result = 0; - char **output = (char **) data_out; - struct tplfunc_frame *f = (struct tplfunc_frame *) frame; - const struct Delivery *ctx = (const struct Delivery *) f->data_in; - - char cwd[PATH_MAX] = {0}; - if (!getcwd(cwd, PATH_MAX - 1)) { - SYSERROR("unable to determine current working directory: %s", strerror(errno)); - return -1; - } - char nametmp[PATH_MAX] = {0}; - strcpy(nametmp, cwd); - char *name = path_basename(nametmp); - - *output = calloc(PATH_MAX, sizeof(**output)); - if (!*output) { - SYSERROR("failed to allocate output string: %s", strerror(errno)); - return -1; - } - sprintf(*output, "%s/results-%s-%s.xml", ctx->storage.results_dir, name, ctx->info.release_name); - - return result; -} - -int get_basetemp_dir_entrypoint(void *frame, void *data_out) { - int result = 0; - char **output = (char **) data_out; - struct tplfunc_frame *f = (struct tplfunc_frame *) frame; - const struct Delivery *ctx = (const struct Delivery *) f->data_in; - - char cwd[PATH_MAX] = {0}; - if (!getcwd(cwd, PATH_MAX - 1)) { - SYSERROR("unable to determine current working directory: %s", strerror(errno)); - return -1; - } - char nametmp[PATH_MAX] = {0}; - strcpy(nametmp, cwd); - char *name = path_basename(nametmp); - - *output = calloc(PATH_MAX, sizeof(**output)); - if (!*output) { - SYSERROR("failed to allocate output string: %s", strerror(errno)); - return -1; - } - sprintf(*output, "%s/truth-%s-%s", ctx->storage.tmpdir, name, ctx->info.release_name); - - return result; -} - -int tox_run_entrypoint(void *frame, void *data_out) { - char **output = (char **) data_out; - struct tplfunc_frame *f = (struct tplfunc_frame *) frame; - const struct Delivery *ctx = (const struct Delivery *) f->data_in; - - // Apply workaround for tox positional arguments - char *toxconf = NULL; - if (!access("tox.ini", F_OK)) { - if (!fix_tox_conf("tox.ini", &toxconf)) { - msg(STASIS_MSG_L3, "Fixing tox positional arguments\n"); - *output = calloc(STASIS_BUFSIZ, sizeof(**output)); - if (!*output) { - return -1; - } - char *basetemp_path = NULL; - if (get_basetemp_dir_entrypoint(f, &basetemp_path)) { - return -2; - } - char *jxml_path = NULL; - if (get_junitxml_file_entrypoint(f, &jxml_path)) { - guard_free(basetemp_path); - return -3; - } - const char *tox_target = f->argv[0].t_char_ptr; - const char *pytest_args = f->argv[1].t_char_ptr; - if (isempty(toxconf) || !strcmp(toxconf, "/")) { - SYSERROR("Unsafe toxconf path: '%s'", toxconf); - guard_free(basetemp_path); - guard_free(jxml_path); - return -4; - } - snprintf(*output, STASIS_BUFSIZ - 1, "\npip install tox && (tox -e py%s%s -c %s --root . -- --basetemp=\"%s\" --junitxml=\"%s\" %s ; rm -f '%s')\n", ctx->meta.python_compact, tox_target, toxconf, basetemp_path, jxml_path, pytest_args ? pytest_args : "", toxconf); - - guard_free(jxml_path); - guard_free(basetemp_path); - } - } - return 0; -} \ No newline at end of file diff --git a/src/utils.c b/src/utils.c deleted file mode 100644 index 6381cea..0000000 --- a/src/utils.c +++ /dev/null @@ -1,819 +0,0 @@ -#include -#include "core.h" - -char *dirstack[STASIS_DIRSTACK_MAX]; -const ssize_t dirstack_max = sizeof(dirstack) / sizeof(dirstack[0]); -ssize_t dirstack_len = 0; - -int pushd(const char *path) { - if (access(path, F_OK)) { - // the requested path doesn't exist. - return -1; - } - if (dirstack_len + 1 > dirstack_max) { - // the stack is full - return -1; - } - - dirstack[dirstack_len] = realpath(".", NULL); - dirstack_len++; - return chdir(path); -} - -int popd() { - int result = -1; - if (dirstack_len - 1 < 0) { - return result; - } - dirstack_len--; - result = chdir(dirstack[dirstack_len]); - guard_free(dirstack[dirstack_len]); - return result; -} - -int rmtree(char *_path) { - int status = 0; - char path[PATH_MAX] = {0}; - strncpy(path, _path, sizeof(path) - 1); - DIR *dir; - struct dirent *d_entity; - - dir = opendir(path); - if (!dir) { - return 1; - } - - while ((d_entity = readdir(dir)) != NULL) { - char abspath[PATH_MAX] = {0}; - strcat(abspath, path); - strcat(abspath, DIR_SEP); - strcat(abspath, d_entity->d_name); - - if (!strcmp(d_entity->d_name, ".") || !strcmp(d_entity->d_name, "..") || !strcmp(abspath, path)) { - continue; - } - - // Test for sufficient privilege - if (access(abspath, F_OK) < 0 && errno == EACCES) { - continue; - } - - // Push directories on to the stack first - if (d_entity->d_type == DT_DIR) { - rmtree(abspath); - } else { - remove(abspath); - } - } - closedir(dir); - - if (access(path, F_OK) == 0) { - remove(path); - } - return status; -} - -char *expandpath(const char *_path) { - if (_path == NULL) { - return NULL; - } - const char *homes[] = { - "HOME", - "USERPROFILE", - }; - char home[PATH_MAX]; - char tmp[PATH_MAX]; - char *ptmp = tmp; - char result[PATH_MAX]; - char *sep = NULL; - - memset(home, '\0', sizeof(home)); - memset(ptmp, '\0', sizeof(tmp)); - memset(result, '\0', sizeof(result)); - - strncpy(ptmp, _path, PATH_MAX - 1); - - // Check whether there's a reason to continue processing the string - if (*ptmp != '~') { - return strdup(ptmp); - } - - // Remove tilde from the string and shift its contents to the left - strchrdel(ptmp, "~"); - - // Figure out where the user's home directory resides - for (size_t i = 0; i < sizeof(homes) / sizeof(*homes); i++) { - char *tmphome; - if ((tmphome = getenv(homes[i])) != NULL) { - strncpy(home, tmphome, PATH_MAX - 1); - break; - } - } - - // A broken runtime environment means we can't do anything else here - if (isempty(home)) { - return NULL; - } - - // Scan the path for a directory separator - if ((sep = strpbrk(ptmp, "/\\")) != NULL) { - // Jump past it - ptmp = sep + 1; - } - - // Construct the new path - strncat(result, home, sizeof(result) - strlen(home) + 1); - if (sep) { - strncat(result, DIR_SEP, sizeof(result) - strlen(home) + 1); - strncat(result, ptmp, sizeof(result) - strlen(home) + 1); - } - - return strdup(result); -} - -char *path_basename(char *path) { - char *result = NULL; - char *last = NULL; - - if ((last = strrchr(path, '/')) == NULL) { - return path; - } - // Perform a lookahead ensuring the string is valid beyond the last separator - if (last++ != NULL) { - result = last; - } - - return result; -} - -char *path_dirname(char *path) { - if (!path) { - return ""; - } - if (strlen(path) == 1 && *path == '/') { - return "/"; - } - char *pos = strrchr(path, '/'); - if (!pos) { - return "."; - } - *pos = '\0'; - - return path; -} - -char **file_readlines(const char *filename, size_t start, size_t limit, ReaderFn *readerFn) { - FILE *fp = NULL; - char **result = NULL; - char *buffer = NULL; - size_t lines = 0; - int use_stdin = 0; - - if (strcmp(filename, "-") == 0) { - use_stdin = 1; - } - - if (use_stdin) { - fp = stdin; - } else { - fp = fopen(filename, "r"); - } - - if (fp == NULL) { - perror(filename); - SYSERROR("failed to open %s for reading", filename); - return NULL; - } - - // Allocate buffer - if ((buffer = calloc(STASIS_BUFSIZ, sizeof(char))) == NULL) { - SYSERROR("unable to allocate %d bytes for buffer", STASIS_BUFSIZ); - if (!use_stdin) { - fclose(fp); - } - return NULL; - } - - // count number the of lines in the file - while ((fgets(buffer, STASIS_BUFSIZ - 1, fp)) != NULL) { - lines++; - } - - if (!lines) { - guard_free(buffer); - if (!use_stdin) { - fclose(fp); - } - return NULL; - } - - rewind(fp); - - // Handle invalid start offset - if (start > lines) { - start = 0; - } - - // Adjust line count when start offset is non-zero - if (start != 0 && start < lines) { - lines -= start; - } - - - // Handle minimum and maximum limits - if (limit == 0 || limit > lines) { - limit = lines; - } - - // Populate results array - result = calloc(limit + 1, sizeof(char *)); - for (size_t i = start; i < limit; i++) { - if (i < start) { - continue; - } - - if (fgets(buffer, STASIS_BUFSIZ - 1, fp) == NULL) { - break; - } - - if (readerFn != NULL) { - int status = readerFn(i - start, &buffer); - // A status greater than zero indicates we should ignore this line entirely and "continue" - // A status less than zero indicates we should "break" - // A zero status proceeds normally - if (status > 0) { - i--; - continue; - } else if (status < 0) { - break; - } - } - result[i] = strdup(buffer); - memset(buffer, '\0', STASIS_BUFSIZ); - } - - guard_free(buffer); - if (!use_stdin) { - fclose(fp); - } - return result; -} - -char *find_program(const char *name) { - static char result[PATH_MAX] = {0}; - char *_env_path = getenv(PATH_ENV_VAR); - if (!_env_path) { - errno = EINVAL; - return NULL; - } - char *path = strdup(_env_path); - char *path_orig = path; - char *path_elem = NULL; - - if (!path) { - errno = ENOMEM; - return NULL; - } - - result[0] = '\0'; - while ((path_elem = strsep(&path, PATH_SEP))) { - char abspath[PATH_MAX] = {0}; - strcat(abspath, path_elem); - strcat(abspath, DIR_SEP); - strcat(abspath, name); - if (access(abspath, F_OK) < 0) { - continue; - } - strncpy(result, abspath, sizeof(result)); - break; - } - path = path_orig; - guard_free(path); - return strlen(result) ? result : NULL; -} - -int touch(const char *filename) { - if (access(filename, F_OK) == 0) { - return 0; - } - - FILE *fp = fopen(filename, "w"); - if (!fp) { - perror(filename); - return 1; - } - fclose(fp); - return 0; -} - -int git_clone(struct Process *proc, char *url, char *destdir, char *gitref) { - int result = -1; - char *chdir_to = NULL; - char *program = find_program("git"); - if (!program) { - return result; - } - - static char command[PATH_MAX]; - sprintf(command, "%s clone -c advice.detachedHead=false --recursive %s", program, url); - if (destdir && access(destdir, F_OK) < 0) { - sprintf(command + strlen(command), " %s", destdir); - result = shell(proc, command); - } - - if (destdir) { - chdir_to = destdir; - } else { - chdir_to = path_basename(url); - } - - pushd(chdir_to); - { - memset(command, 0, sizeof(command)); - sprintf(command, "%s fetch --all", program); - result += shell(proc, command); - - if (gitref != NULL) { - memset(command, 0, sizeof(command)); - sprintf(command, "%s checkout %s", program, gitref); - result += shell(proc, command); - } - popd(); - } - return result; -} - - -char *git_describe(const char *path) { - static char version[NAME_MAX]; - FILE *pp; - - memset(version, 0, sizeof(version)); - if (pushd(path)) { - return NULL; - } - - pp = popen("git describe --first-parent --always --tags", "r"); - if (!pp) { - return NULL; - } - fgets(version, sizeof(version) - 1, pp); - strip(version); - pclose(pp); - popd(); - return version; -} - -char *git_rev_parse(const char *path, char *args) { - static char version[NAME_MAX]; - char cmd[PATH_MAX]; - FILE *pp; - - memset(version, 0, sizeof(version)); - if (isempty(args)) { - fprintf(stderr, "git_rev_parse args cannot be empty\n"); - return NULL; - } - - if (pushd(path)) { - return NULL; - } - - sprintf(cmd, "git rev-parse %s", args); - pp = popen(cmd, "r"); - if (!pp) { - return NULL; - } - fgets(version, sizeof(version) - 1, pp); - strip(version); - pclose(pp); - popd(); - return version; -} - -void msg(unsigned type, char *fmt, ...) { - FILE *stream = NULL; - char header[255]; - char status[20]; - - if (type & STASIS_MSG_NOP) { - // quiet mode - return; - } - - if (!globals.verbose && type & STASIS_MSG_RESTRICT) { - // Verbose mode is not active - return; - } - - memset(header, 0, sizeof(header)); - memset(status, 0, sizeof(status)); - - va_list args; - va_start(args, fmt); - - stream = stdout; - fprintf(stream, "%s", STASIS_COLOR_RESET); - if (type & STASIS_MSG_ERROR) { - // for error output - stream = stderr; - fprintf(stream, "%s", STASIS_COLOR_RED); - strcpy(status, " ERROR: "); - } else if (type & STASIS_MSG_WARN) { - stream = stderr; - fprintf(stream, "%s", STASIS_COLOR_YELLOW); - strcpy(status, " WARNING: "); - } else { - fprintf(stream, "%s", STASIS_COLOR_GREEN); - strcpy(status, " "); - } - - if (type & STASIS_MSG_L1) { - sprintf(header, "==>%s" STASIS_COLOR_RESET STASIS_COLOR_WHITE, status); - } else if (type & STASIS_MSG_L2) { - sprintf(header, " ->%s" STASIS_COLOR_RESET, status); - } else if (type & STASIS_MSG_L3) { - sprintf(header, STASIS_COLOR_BLUE " ->%s" STASIS_COLOR_RESET, status); - } - - fprintf(stream, "%s", header); - vfprintf(stream, fmt, args); - fprintf(stream, "%s", STASIS_COLOR_RESET); - va_end(args); -} - -void debug_shell() { - msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "ENTERING STASIS DEBUG SHELL\n" STASIS_COLOR_RESET); - if (system("/bin/bash -c 'PS1=\"(STASIS DEBUG) \\W $ \" bash --norc --noprofile'") < 0) { - SYSERROR("unable to spawn debug shell: %s", strerror(errno)); - exit(errno); - } - msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "EXITING STASIS DEBUG SHELL\n" STASIS_COLOR_RESET); - exit(255); -} - -char *xmkstemp(FILE **fp, const char *mode) { - int fd = -1; - char tmpdir[PATH_MAX]; - char t_name[PATH_MAX * 2]; - - if (globals.tmpdir) { - strcpy(tmpdir, globals.tmpdir); - } else { - strcpy(tmpdir, "/tmp"); - } - memset(t_name, 0, sizeof(t_name)); - sprintf(t_name, "%s/%s", tmpdir, "STASIS.XXXXXX"); - - fd = mkstemp(t_name); - *fp = fdopen(fd, mode); - if (!*fp) { - // unable to open, die - if (fd > 0) - close(fd); - *fp = NULL; - return NULL; - } - - char *path = strdup(t_name); - if (!path) { - // strdup failed, die - if (*fp) { - // close the file handle - fclose(*fp); - *fp = NULL; - } - // fall through. path is NULL. - } - return path; -} - -int isempty_dir(const char *path) { - DIR *dp; - struct dirent *rec; - size_t count = 0; - - dp = opendir(path); - if (!dp) { - return -1; - } - while ((rec = readdir(dp)) != NULL) { - if (!strcmp(rec->d_name, ".") || !strcmp(rec->d_name, "..")) { - continue; - } - count++; - } - closedir(dp); - return count == 0; -} - -int path_store(char **destptr, size_t maxlen, const char *base, const char *path) { - char *path_tmp; - size_t base_len = 0; - size_t path_len = 0; - - // Both path elements need to be defined to continue - if (!base || !path) { - return -1; - } - - // Initialize destination pointer to length of maxlen - path_tmp = calloc(maxlen, sizeof(*path_tmp)); - if (!path_tmp) { - return -1; - } - - // Ensure generated path will fit in destination - base_len = strlen(base); - path_len = strlen(path); - // 2 = directory separator and NUL terminator - if (2 + (base_len + path_len) > maxlen) { - goto l_path_setup_error; - } - - snprintf(path_tmp, maxlen - 1, "%s/%s", base, path); - if (mkdirs(path_tmp, 0755)) { - goto l_path_setup_error; - } - - if (*destptr) { - guard_free(*destptr); - } - - if (!(*destptr = realpath(path_tmp, NULL))) { - goto l_path_setup_error; - } - - guard_free(path_tmp); - return 0; - - l_path_setup_error: - guard_free(path_tmp); - return -1; -} - -int xml_pretty_print_in_place(const char *filename, const char *pretty_print_prog, const char *pretty_print_args) { - int status = 0; - char *tempfile = NULL; - char *result = NULL; - FILE *fp = NULL; - FILE *tmpfp = NULL; - char cmd[PATH_MAX]; - if (!find_program(pretty_print_prog)) { - // Pretty printing is optional. 99% chance the XML data will - // be passed to a report generator; not inspected by a human. - return 0; - } - memset(cmd, 0, sizeof(cmd)); - snprintf(cmd, sizeof(cmd) - 1, "%s %s %s", pretty_print_prog, pretty_print_args, filename); - result = shell_output(cmd, &status); - if (status || !result) { - goto pretty_print_failed; - } - - tempfile = xmkstemp(&tmpfp, "w+"); - if (!tmpfp || !tempfile) { - goto pretty_print_failed; - } - - fprintf(tmpfp, "%s", result); - fflush(tmpfp); - fclose(tmpfp); - - fp = fopen(filename, "w+"); - if (!fp) { - goto pretty_print_failed; - } - - if (copy2(tempfile, filename, CT_PERM)) { - goto pretty_print_failed; - } - - if (remove(tempfile)) { - goto pretty_print_failed; - } - - fclose(fp); - guard_free(tempfile); - guard_free(result); - return 0; - - pretty_print_failed: - if (fp) { - fclose(fp); - } - if (tmpfp) { - fclose(tmpfp); - } - guard_free(tempfile); - guard_free(result); - return -1; -} - -/** - * - * @param filename /path/to/tox.ini - * @param result path of replacement tox.ini configuration - * @return 0 on success, -1 on error - */ -int fix_tox_conf(const char *filename, char **result) { - struct INIFILE *toxini; - FILE *fptemp; - char *tempfile; - const char *with_posargs = " \\\n {posargs}\n"; - - // Create new temporary tox configuration file - tempfile = xmkstemp(&fptemp, "w+"); - if (!tempfile) { - return -1; - } - - // If the result pointer is NULL, allocate enough to store a filesystem path - if (!*result) { - *result = calloc(PATH_MAX, sizeof(**result)); - if (!*result) { - guard_free(tempfile); - return -1; - } - } - - // Consume the original tox.ini configuration - toxini = ini_open(filename); - if (!toxini) { - if (fptemp) { - guard_free(result); - guard_free(tempfile); - fclose(fptemp); - } - return -1; - } - - // Modify tox configuration - // - Allow passing positional arguments pytest - for (size_t i = 0; i < toxini->section_count; i++) { - struct INISection *section = toxini->section[i]; - if (section) { - char *section_name = section->key; - for (size_t k = 0; k < section->data_count; k++) { - struct INIData *data = section->data[k]; - if (data) { - int err = 0; - char *key = data->key; - char *value = ini_getval_str(toxini, section->key, data->key, INI_READ_RENDER, &err); - if (key && value) { - if (startswith(value, "pytest") && !strstr(value, "{posargs}")) { - strip(value); - char *tmp; - tmp = realloc(value, strlen(value) + strlen(with_posargs) + 1); - if (!tmp) { - SYSERROR("failed to increase size to +%zu bytes", - strlen(value) + strlen(with_posargs) + 1); - guard_free(*result); - return -1; - } else if (tmp != value) { - value = tmp; - } - strcat(value, with_posargs); - ini_setval(&toxini, INI_SETVAL_REPLACE, section_name, key, value); - } - } - guard_free(value); - } - } - } - } - - // Save modified configuration - ini_write(toxini, &fptemp, INI_WRITE_RAW); - fclose(fptemp); - - // Store path to modified config - strcpy(*result, tempfile); - guard_free(tempfile); - - ini_free(&toxini); - return 0; -} - -static size_t count_blanks(char *s) { - // return the number of leading blanks (tab/space) in a string - size_t blank = 0; - for (size_t i = 0; i < strlen(s); i++) { - if (isblank(s[i])) { - blank++; - } else { - break; - } - } - return blank; -} - -char *collapse_whitespace(char **s) { - char *x = (*s); - size_t len = strlen(x); - for (size_t i = 0; i < len; i++) { - size_t blank = count_blanks(&x[i]); - if (blank > 1) { - memmove(&x[i], &x[i] + blank, strlen(&x[i])); - } - } - - return *s; -} - -/** - * Replace sensitive text in strings with ***REDACTED*** - * @param to_redact a list of tokens to redact - * @param src to read - * @param dest to write modified string - * @param maxlen maximum length of dest string - * @return 0 on success, -1 on error - */ -int redact_sensitive(const char **to_redact, size_t to_redact_size, char *src, char *dest, size_t maxlen) { - const char *redacted = "***REDACTED***"; - - char *tmp = calloc(strlen(redacted) + strlen(src) + 1, sizeof(*tmp)); - if (!tmp) { - return -1; - } - strcpy(tmp, src); - - for (size_t i = 0; i < to_redact_size; i++) { - if (to_redact[i] && strstr(tmp, to_redact[i])) { - replace_text(tmp, to_redact[i], redacted, 0); - break; - } - } - - memset(dest, 0, maxlen); - strncpy(dest, tmp, maxlen - 1); - guard_free(tmp); - - return 0; -} - -/** - * Retrieve file names in a directory - * (no metadata, non-recursive) - * - * @param path directory path - * @return StrList structure - */ -struct StrList *listdir(const char *path) { - struct StrList *node; - DIR *dp; - struct dirent *rec; - - dp = opendir(path); - if (!dp) { - return NULL; - } - node = strlist_init(); - - while ((rec = readdir(dp)) != NULL) { - if (!strcmp(rec->d_name, ".") || !strcmp(rec->d_name, "..")) { - continue; - } - strlist_append(&node, rec->d_name); - } - closedir(dp); - return node; -} - -long get_cpu_count() { -#if defined(STASIS_OS_LINUX) || defined(STASIS_OS_DARWIN) - return sysconf(_SC_NPROCESSORS_ONLN); -#else - return 0; -#endif -} - -int mkdirs(const char *_path, mode_t mode) { - int status; - char *token; - char pathbuf[PATH_MAX] = {0}; - char *path; - path = pathbuf; - strcpy(path, _path); - errno = 0; - - char result[PATH_MAX] = {0}; - status = 0; - while ((token = strsep(&path, "/")) != NULL && !status) { - if (token[0] == '.') - continue; - strcat(result, token); - strcat(result, "/"); - status = mkdir(result, mode); - if (status && errno == EEXIST) { - status = 0; - errno = 0; - continue; - } - } - return status; -} - -char *find_version_spec(char *str) { - return strpbrk(str, "@~=<>!"); -} diff --git a/src/wheel.c b/src/wheel.c deleted file mode 100644 index 4692d0a..0000000 --- a/src/wheel.c +++ /dev/null @@ -1,126 +0,0 @@ -#include "wheel.h" - -struct Wheel *get_wheel_info(const char *basepath, const char *name, char *to_match[], unsigned match_mode) { - DIR *dp; - struct dirent *rec; - struct Wheel *result = NULL; - char package_path[PATH_MAX]; - char package_name[NAME_MAX]; - - strcpy(package_name, name); - tolower_s(package_name); - sprintf(package_path, "%s/%s", basepath, package_name); - - dp = opendir(package_path); - if (!dp) { - return NULL; - } - - while ((rec = readdir(dp)) != NULL) { - if (!strcmp(rec->d_name, ".") || !strcmp(rec->d_name, "..")) { - continue; - } - char filename[NAME_MAX]; - strcpy(filename, rec->d_name); - char *ext = strstr(filename, ".whl"); - if (ext) { - *ext = '\0'; - } else { - // not a wheel file. nothing to do - continue; - } - - size_t match = 0; - size_t pattern_count = 0; - for (; to_match[pattern_count] != NULL; pattern_count++) { - if (strstr(filename, to_match[pattern_count])) { - match++; - } - } - - if (!startswith(rec->d_name, name)) { - continue; - } - - if (match_mode == WHEEL_MATCH_EXACT && match != pattern_count) { - continue; - } - - result = calloc(1, sizeof(*result)); - if (!result) { - SYSERROR("Unable to allocate %zu bytes for wheel struct", sizeof(*result)); - closedir(dp); - return NULL; - } - - result->path_name = realpath(package_path, NULL); - if (!result->path_name) { - SYSERROR("Unable to resolve absolute path to %s: %s", filename, strerror(errno)); - wheel_free(&result); - closedir(dp); - return NULL; - } - result->file_name = strdup(rec->d_name); - if (!result->file_name) { - SYSERROR("Unable to allocate bytes for %s: %s", rec->d_name, strerror(errno)); - wheel_free(&result); - closedir(dp); - return NULL; - } - - size_t parts_total; - char **parts = split(filename, "-", 0); - if (!parts) { - // This shouldn't happen unless a wheel file is present in the - // directory with a malformed file name, or we've managed to - // exhaust the system's memory - SYSERROR("%s has no '-' separators! (Delete this file and try again)", filename); - wheel_free(&result); - closedir(dp); - return NULL; - } - - for (parts_total = 0; parts[parts_total] != NULL; parts_total++); - if (parts_total == 5) { - // no build tag - result->distribution = strdup(parts[0]); - result->version = strdup(parts[1]); - result->build_tag = NULL; - result->python_tag = strdup(parts[2]); - result->abi_tag = strdup(parts[3]); - result->platform_tag = strdup(parts[4]); - } else if (parts_total == 6) { - // has build tag - result->distribution = strdup(parts[0]); - result->version = strdup(parts[1]); - result->build_tag = strdup(parts[2]); - result->python_tag = strdup(parts[3]); - result->abi_tag = strdup(parts[4]); - result->platform_tag = strdup(parts[5]); - } else { - SYSERROR("Unknown wheel name format: %s. Expected 5 or 6 strings " - "separated by '-', but got %zu instead", filename, parts_total); - GENERIC_ARRAY_FREE(parts); - wheel_free(&result); - closedir(dp); - return NULL; - } - GENERIC_ARRAY_FREE(parts); - break; - } - closedir(dp); - return result; -} - -void wheel_free(struct Wheel **wheel) { - struct Wheel *w = (*wheel); - guard_free(w->path_name); - guard_free(w->file_name); - guard_free(w->distribution); - guard_free(w->version); - guard_free(w->build_tag); - guard_free(w->python_tag); - guard_free(w->abi_tag); - guard_free(w->python_tag); - guard_free(w); -} -- cgit