diff options
author | Joseph Hunkeler <jhunkeler@gmail.com> | 2024-10-14 09:32:03 -0400 |
---|---|---|
committer | Joseph Hunkeler <jhunkeler@gmail.com> | 2024-10-14 09:43:31 -0400 |
commit | 5a9688e9e78a25a42bddfc4388fb4ce3311ded74 (patch) | |
tree | bcc1b54c3f8a7f1eab0d6b3e129f098721a41537 /src/lib/core | |
parent | b98088f7b7cfe4b08eb39fa1b6b86210cb6c08b8 (diff) | |
download | stasis-5a9688e9e78a25a42bddfc4388fb4ce3311ded74.tar.gz |
Refactor directory structure
* Move core library sources into src/lib/core
* Move command-line programs into src/cli
Diffstat (limited to 'src/lib/core')
34 files changed, 9152 insertions, 0 deletions
diff --git a/src/lib/core/CMakeLists.txt b/src/lib/core/CMakeLists.txt new file mode 100644 index 0000000..c569187 --- /dev/null +++ b/src/lib/core/CMakeLists.txt @@ -0,0 +1,38 @@ +include_directories(${PROJECT_BINARY_DIR}) + +add_library(stasis_core STATIC + globals.c + str.c + strlist.c + ini.c + conda.c + environment.c + utils.c + system.c + download.c + delivery_postprocess.c + delivery_conda.c + delivery_docker.c + delivery_install.c + delivery_artifactory.c + delivery_test.c + delivery_build.c + delivery_show.c + delivery_populate.c + delivery_init.c + delivery.c + recipe.c + relocation.c + wheel.c + copy.c + artifactory.c + template.c + rules.c + docker.c + junitxml.c + github.c + template_func_proto.c + envctl.c + multiprocessing.c +) + diff --git a/src/lib/core/artifactory.c b/src/lib/core/artifactory.c new file mode 100644 index 0000000..6b9635d --- /dev/null +++ b/src/lib/core/artifactory.c @@ -0,0 +1,496 @@ +#include "artifactory.h" + +extern struct STASIS_GLOBAL globals; + +int artifactory_download_cli(char *dest, + char *jfrog_artifactory_base_url, + char *jfrog_artifactory_product, + char *cli_major_ver, + char *version, + char *os, + char *arch, + char *remote_filename) { + char url[PATH_MAX] = {0}; + char path[PATH_MAX] = {0}; + char os_ident[STASIS_NAME_MAX] = {0}; + char arch_ident[STASIS_NAME_MAX] = {0}; + + // convert platform string to lower-case + strcpy(os_ident, os); + tolower_s(os_ident); + + // translate OS identifier + if (!strcmp(os_ident, "darwin") || startswith(os_ident, "macos")) { + strcpy(os_ident, "mac"); + } else if (!strcmp(os_ident, "linux")) { + strcpy(os_ident, "linux"); + } else { + fprintf(stderr, "%s: unknown operating system: %s\n", __FUNCTION__, os_ident); + return -1; + } + + // translate ARCH identifier + strcpy(arch_ident, arch); + if (startswith(arch_ident, "i") && endswith(arch_ident, "86")) { + strcpy(arch_ident, "386"); + } else if (!strcmp(arch_ident, "amd64") || !strcmp(arch_ident, "x86_64") || !strcmp(arch_ident, "x64")) { + if (!strcmp(os_ident, "mac")) { + strcpy(arch_ident, "386"); + } else { + strcpy(arch_ident, "amd64"); + } + } else if (!strcmp(arch_ident, "arm64") || !strcmp(arch_ident, "aarch64")) { + strcpy(arch_ident, "arm64"); + } else { + fprintf(stderr, "%s: unknown architecture: %s\n", __FUNCTION__, arch_ident); + return -1; + } + + snprintf(url, sizeof(url) - 1, "%s/%s/%s/%s/%s-%s-%s/%s", + jfrog_artifactory_base_url, // https://releases.jfrog.io/artifactory + jfrog_artifactory_product, // jfrog-cli + cli_major_ver, // v\d+(-jf)? + version, // 1.2.3 + jfrog_artifactory_product, // ... + os_ident, // ... + arch_ident, // jfrog-cli-linux-x86_64 + remote_filename); // jf + strcpy(path, dest); + + if (mkdirs(path, 0755)) { + fprintf(stderr, "%s: %s: %s", __FUNCTION__, path, strerror(errno)); + return -1; + } + + sprintf(path + strlen(path), "/%s", remote_filename); + long fetch_status = download(url, path, NULL); + if (HTTP_ERROR(fetch_status) || fetch_status < 0) { + fprintf(stderr, "%s: download failed: %s\n", __FUNCTION__, url); + return -1; + } + chmod(path, 0755); + return 0; +} + +void jfrt_register_opt_str(char *jfrt_val, const char *opt_name, struct StrList **opt_map) { + char data[STASIS_BUFSIZ]; + memset(data, 0, sizeof(data)); + + if (jfrt_val == NULL) { + // no data + return; + } + snprintf(data, sizeof(data) - 1, "--%s=\"%s\"", opt_name, jfrt_val); + strlist_append(&*opt_map, data); +} + +void jfrt_register_opt_bool(bool jfrt_val, const char *opt_name, struct StrList **opt_map) { + char data[STASIS_BUFSIZ]; + memset(data, 0, sizeof(data)); + + if (jfrt_val == false) { + // option will not be used + return; + } + snprintf(data, sizeof(data) - 1, "--%s", opt_name); + strlist_append(&*opt_map, data); +} + +void jfrt_register_opt_int(int jfrt_val, const char *opt_name, struct StrList **opt_map) { + char data[STASIS_BUFSIZ]; + memset(data, 0, sizeof(data)); + + if (jfrt_val == 0) { + // option will not be used + return; + } + snprintf(data, sizeof(data) - 1, "--%s=%d", opt_name, jfrt_val); + strlist_append(&*opt_map, data); +} + +void jfrt_register_opt_long(long jfrt_val, const char *opt_name, struct StrList **opt_map) { + char data[STASIS_BUFSIZ]; + memset(data, 0, sizeof(data)); + + if (jfrt_val == 0) { + // option will not be used + return; + } + snprintf(data, sizeof(data) - 1, "--%s=%ld", opt_name, jfrt_val); + strlist_append(&*opt_map, data); +} + +void jfrt_upload_init(struct JFRT_Upload *ctx) { + memset(ctx, 0, sizeof(*ctx)); + ctx->recursive = true; + ctx->threads = 3; + ctx->retries = 3; +} + +static int auth_required(const char *cmd) { + const char *modes[] = { + "build-collect-env", + NULL, + }; + for (size_t i = 0; modes[i] != NULL; i++) { + if (!startswith(cmd, modes[i])) { + return 1; + } + } + return 0; +} + +int jfrt_auth_init(struct JFRT_Auth *auth_ctx) { + char *url = getenv("STASIS_JF_ARTIFACTORY_URL"); + char *user = getenv("STASIS_JF_USER"); + char *access_token = getenv("STASIS_JF_ACCESS_TOKEN"); + char *password = getenv("STASIS_JF_PASSWORD"); + char *ssh_key_path = getenv("STASIS_JF_SSH_KEY_PATH"); + char *ssh_passphrase = getenv("STASIS_JF_SSH_PASSPHRASE"); + char *client_cert_key_path = getenv("STASIS_JF_CLIENT_CERT_KEY_PATH"); + char *client_cert_path = getenv("STASIS_JF_CLIENT_CERT_PATH"); + + if (!url) { + fprintf(stderr, "Artifactory URL is not configured:\n"); + fprintf(stderr, "please set STASIS_JF_ARTIFACTORY_URL\n"); + return -1; + } + auth_ctx->url = url; + + if (access_token) { + auth_ctx->user = NULL; + auth_ctx->access_token = access_token; + auth_ctx->password = NULL; + auth_ctx->ssh_key_path = NULL; + } else if (user && password) { + auth_ctx->user = user; + auth_ctx->password = password; + auth_ctx->access_token = NULL; + auth_ctx->ssh_key_path = NULL; + } else if (ssh_key_path) { + auth_ctx->user = NULL; + auth_ctx->ssh_key_path = ssh_key_path; + if (ssh_passphrase) { + auth_ctx->ssh_passphrase = ssh_passphrase; + } + auth_ctx->password = NULL; + auth_ctx->access_token = NULL; + } else if (client_cert_key_path && client_cert_path) { + auth_ctx->user = NULL; + auth_ctx->password = NULL; + auth_ctx->access_token = NULL; + auth_ctx->ssh_key_path = NULL; + auth_ctx->client_cert_key_path = client_cert_key_path; + auth_ctx->client_cert_path = client_cert_path; + } else { + fprintf(stderr, "Artifactory authentication is not configured:\n"); + fprintf(stderr, "set STASIS_JF_USER and STASIS_JF_PASSWORD\n"); + fprintf(stderr, "or, set STASIS_JF_ACCESS_TOKEN\n"); + fprintf(stderr, "or, set STASIS_JF_SSH_KEY_PATH and STASIS_JF_SSH_KEY_PASSPHRASE\n"); + fprintf(stderr, "or, set STASIS_JF_CLIENT_CERT_KEY_PATH and STASIS_JF_CLIENT_CERT_PATH\n"); + return -1; + } + return 0; +} + +int jfrog_cli(struct JFRT_Auth *auth, const char *subsystem, const char *task, char *args) { + struct Process proc; + char cmd[STASIS_BUFSIZ]; + char cmd_redacted[STASIS_BUFSIZ]; + int status; + + memset(&proc, 0, sizeof(proc)); + memset(cmd, 0, sizeof(cmd)); + memset(cmd_redacted, 0, sizeof(cmd_redacted)); + + struct StrList *arg_map = strlist_init(); + if (!arg_map) { + return -1; + } + + char *auth_args = NULL; + if (auth_required(task)) { + // String options + jfrt_register_opt_str(auth->url, "url", &arg_map); + jfrt_register_opt_str(auth->user, "user", &arg_map); + jfrt_register_opt_str(auth->access_token, "access-token", &arg_map); + jfrt_register_opt_str(auth->password, "password", &arg_map); + jfrt_register_opt_str(auth->ssh_key_path, "ssh-key-path", &arg_map); + jfrt_register_opt_str(auth->ssh_passphrase, "ssh-passphrase", &arg_map); + jfrt_register_opt_str(auth->client_cert_key_path, "client-cert-key-path", &arg_map); + jfrt_register_opt_str(auth->client_cert_path, "client-cert-path", &arg_map); + jfrt_register_opt_bool(auth->insecure_tls, "insecure-tls", &arg_map); + jfrt_register_opt_str(auth->server_id, "server-id", &arg_map); + } + + auth_args = join(arg_map->data, " "); + if (!auth_args) { + return -1; + } + + const char *redactable[] = { + auth->access_token, + auth->ssh_key_path, + auth->ssh_passphrase, + auth->client_cert_key_path, + auth->client_cert_path, + auth->password, + }; + snprintf(cmd, sizeof(cmd) - 1, "jf %s %s %s %s", subsystem, task, auth_args, args ? args : ""); + redact_sensitive(redactable, sizeof(redactable) / sizeof (*redactable), cmd, cmd_redacted, sizeof(cmd_redacted) - 1); + + guard_free(auth_args); + guard_strlist_free(&arg_map); + + // Pings are noisy. Squelch them. + if (task && !strstr(task, "ping")) { + msg(STASIS_MSG_L2, "Executing: %s\n", cmd_redacted); + } + + if (!globals.verbose) { + strcpy(proc.f_stdout, "/dev/null"); + strcpy(proc.f_stderr, "/dev/null"); + } + status = shell(&proc, cmd); + return status; +} + +static int jfrog_cli_rt(struct JFRT_Auth *auth, char *task, char *args) { + return jfrog_cli(auth, "rt", task, args); +} + +int jfrog_cli_rt_build_collect_env(struct JFRT_Auth *auth, char *build_name, char *build_number) { + char cmd[STASIS_BUFSIZ]; + memset(cmd, 0, sizeof(cmd)); + snprintf(cmd, sizeof(cmd) - 1, "\"%s\" \"%s\"", build_name, build_number); + return jfrog_cli(auth, "rt", "build-collect-env", cmd); +} + +int jfrog_cli_rt_build_publish(struct JFRT_Auth *auth, char *build_name, char *build_number) { + char cmd[STASIS_BUFSIZ]; + memset(cmd, 0, sizeof(cmd)); + snprintf(cmd, sizeof(cmd) - 1, "\"%s\" \"%s\"", build_name, build_number); + return jfrog_cli(auth, "rt", "build-publish", cmd); +} + +int jfrog_cli_rt_ping(struct JFRT_Auth *auth) { + return jfrog_cli_rt(auth, "ping", NULL); +} + +int jfrog_cli_rt_download(struct JFRT_Auth *auth, struct JFRT_Download *ctx, char *repo_path, char *dest) { + char cmd[STASIS_BUFSIZ]; + memset(cmd, 0, sizeof(cmd)); + + if (isempty(repo_path)) { + fprintf(stderr, "repo_path argument must be a valid artifactory repository path\n"); + return -1; + } + + // dest is an optional argument, therefore may be NULL or an empty string + + struct StrList *arg_map = strlist_init(); + if (!arg_map) { + return -1; + } + + jfrt_register_opt_str(ctx->archive_entries, "archive-entries", &arg_map); + jfrt_register_opt_str(ctx->build, "build", &arg_map); + jfrt_register_opt_str(ctx->build_name, "build-name", &arg_map); + jfrt_register_opt_str(ctx->build_number, "build-number", &arg_map); + jfrt_register_opt_str(ctx->bundle, "bundle", &arg_map); + jfrt_register_opt_str(ctx->exclude_artifacts, "exclude-artifacts", &arg_map); + jfrt_register_opt_str(ctx->exclude_props, "exclude-props", &arg_map); + jfrt_register_opt_str(ctx->exclusions, "exclusions", &arg_map); + jfrt_register_opt_str(ctx->gpg_key, "gpg-key", &arg_map); + jfrt_register_opt_str(ctx->include_deps, "include-deps", &arg_map); + jfrt_register_opt_str(ctx->include_dirs, "include-dirs", &arg_map); + jfrt_register_opt_str(ctx->module, "module", &arg_map); + jfrt_register_opt_str(ctx->project, "project", &arg_map); + jfrt_register_opt_str(ctx->props, "props", &arg_map); + jfrt_register_opt_str(ctx->sort_by, "sort-by", &arg_map); + jfrt_register_opt_str(ctx->sort_order, "sort-order", &arg_map); + jfrt_register_opt_str(ctx->spec, "spec", &arg_map); + jfrt_register_opt_str(ctx->spec_vars, "spec-vars", &arg_map); + + jfrt_register_opt_bool(ctx->detailed_summary, "detailed-summary", &arg_map); + jfrt_register_opt_bool(ctx->dry_run, "dry-run", &arg_map); + jfrt_register_opt_bool(ctx->explode, "explode", &arg_map); + jfrt_register_opt_bool(ctx->fail_no_op, "fail-no-op", &arg_map); + jfrt_register_opt_bool(ctx->flat, "flat", &arg_map); + jfrt_register_opt_bool(ctx->quiet, "quiet", &arg_map); + jfrt_register_opt_bool(ctx->recursive, "recursive", &arg_map); + jfrt_register_opt_bool(ctx->retries, "retries", &arg_map); + jfrt_register_opt_bool(ctx->retry_wait_time, "retry-wait-time", &arg_map); + jfrt_register_opt_bool(ctx->skip_checksum, "skip-checksum", &arg_map); + + jfrt_register_opt_int(ctx->limit, "limit", &arg_map); + jfrt_register_opt_int(ctx->min_split, "min-split", &arg_map); + jfrt_register_opt_int(ctx->offset, "offset", &arg_map); + jfrt_register_opt_int(ctx->split_count, "split-count", &arg_map); + jfrt_register_opt_int(ctx->sync_deletes, "sync-deletes", &arg_map); + jfrt_register_opt_int(ctx->threads, "threads", &arg_map); + jfrt_register_opt_int(ctx->validate_symlinks, "validate-symlinks", &arg_map); + + char *args = join(arg_map->data, " "); + if (!args) { + return -1; + } + + snprintf(cmd, sizeof(cmd) - 1, "%s '%s' %s", args, repo_path, dest ? dest : ""); + guard_free(args); + guard_strlist_free(&arg_map); + + int status = jfrog_cli_rt(auth, "download", cmd); + return status; +} + +int jfrog_cli_rt_upload(struct JFRT_Auth *auth, struct JFRT_Upload *ctx, char *src, char *repo_path) { + char cmd[STASIS_BUFSIZ]; + memset(cmd, 0, sizeof(cmd)); + + if (isempty(src)) { + fprintf(stderr, "src argument must be a valid file system path\n"); + return -1; + } + + if (isempty(repo_path)) { + fprintf(stderr, "repo_path argument must be a valid artifactory repository path\n"); + return -1; + } + + struct StrList *arg_map = strlist_init(); + if (!arg_map) { + return -1; + } + + // String options + jfrt_register_opt_str(ctx->build_name, "build-name", &arg_map); + jfrt_register_opt_str(ctx->build_number, "build-number", &arg_map); + jfrt_register_opt_str(ctx->exclusions, "exclusions", &arg_map); + jfrt_register_opt_str(ctx->module, "module", &arg_map); + jfrt_register_opt_str(ctx->spec, "spec", &arg_map); + jfrt_register_opt_str(ctx->spec_vars, "spec-vars", &arg_map); + jfrt_register_opt_str(ctx->project, "project", &arg_map); + jfrt_register_opt_str(ctx->target_props, "target-props", &arg_map); + + // Boolean options + jfrt_register_opt_bool(ctx->quiet, "quiet", &arg_map); + jfrt_register_opt_bool(ctx->ant, "ant", &arg_map); + jfrt_register_opt_bool(ctx->archive, "archive", &arg_map); + jfrt_register_opt_bool(ctx->deb, "deb", &arg_map); + jfrt_register_opt_bool(ctx->detailed_summary, "detailed-summary", &arg_map); + jfrt_register_opt_bool(ctx->dry_run, "dry-run", &arg_map); + jfrt_register_opt_bool(ctx->explode, "explode", &arg_map); + jfrt_register_opt_bool(ctx->fail_no_op, "fail-no-op", &arg_map); + jfrt_register_opt_bool(ctx->flat, "flat", &arg_map); + jfrt_register_opt_bool(ctx->include_dirs, "include-dirs", &arg_map); + jfrt_register_opt_bool(ctx->recursive, "recursive", &arg_map); + jfrt_register_opt_bool(ctx->symlinks, "symlinks", &arg_map); + jfrt_register_opt_bool(ctx->sync_deletes, "sync-deletes", &arg_map); + jfrt_register_opt_bool(ctx->regexp, "regexp", &arg_map); + + // Integer options + jfrt_register_opt_int(ctx->retries, "retries", &arg_map); + jfrt_register_opt_int(ctx->retry_wait_time, "retry-wait-time", &arg_map); + jfrt_register_opt_int(ctx->threads, "threads", &arg_map); + + char *args = join(arg_map->data, " "); + if (!args) { + return -1; + } + + char *new_src = NULL; + char *base = NULL; + if (ctx->workaround_parent_only) { + struct StrList *components = strlist_init(); + + strlist_append_tokenize(components, src, "/"); + int max_components = (int) strlist_count(components); + for (int i = 0; i < max_components; i++) { + if (strstr(components->data[i], "*")) { + max_components = i; + break; + } + } + base = join(&components->data[max_components], "/"); + guard_free(components->data[max_components]); + new_src = join(components->data, "/"); + guard_strlist_free(&components); + } + + if (new_src) { + if (base) { + src = base; + } else { + strcat(src, "/"); + } + pushd(new_src); + } + + snprintf(cmd, sizeof(cmd) - 1, "%s '%s' \"%s\"", args, src, repo_path); + guard_free(args); + guard_strlist_free(&arg_map); + + int status = jfrog_cli_rt(auth, "upload", cmd); + if (new_src) { + popd(); + guard_free(new_src); + } + if (base) { + guard_free(base); + } + + return status; +} + +int jfrog_cli_rt_search(struct JFRT_Auth *auth, struct JFRT_Search *ctx, char *repo_path, char *pattern) { + char cmd[STASIS_BUFSIZ]; + memset(cmd, 0, sizeof(cmd)); + + if (isempty(repo_path)) { + fprintf(stderr, "repo_path argument must be a valid artifactory repository path\n"); + return -1; + } + + struct StrList *arg_map = strlist_init(); + if (!arg_map) { + return -1; + } + + jfrt_register_opt_str(ctx->archive_entries, "archive-entries", &arg_map); + jfrt_register_opt_str(ctx->build, "build", &arg_map); + jfrt_register_opt_str(ctx->bundle, "bundle", &arg_map); + jfrt_register_opt_str(ctx->exclusions, "exclusions", &arg_map); + jfrt_register_opt_str(ctx->exclude_patterns, "exclude-patterns", &arg_map); + jfrt_register_opt_str(ctx->exclude_artifacts, "exclude-artifacts", &arg_map); + jfrt_register_opt_str(ctx->exclude_props, "exclude-props", &arg_map); + jfrt_register_opt_str(ctx->include, "include", &arg_map); + jfrt_register_opt_str(ctx->include_deps, "include_deps", &arg_map); + jfrt_register_opt_str(ctx->include_dirs, "include_dirs", &arg_map); + jfrt_register_opt_str(ctx->project, "project", &arg_map); + jfrt_register_opt_str(ctx->props, "props", &arg_map); + jfrt_register_opt_str(ctx->sort_by, "sort-by", &arg_map); + jfrt_register_opt_str(ctx->sort_order, "sort-order", &arg_map); + jfrt_register_opt_str(ctx->spec, "spec", &arg_map); + jfrt_register_opt_str(ctx->spec_vars, "spec-vars", &arg_map); + + jfrt_register_opt_bool(ctx->count, "count", &arg_map); + jfrt_register_opt_bool(ctx->fail_no_op, "fail-no-op", &arg_map); + jfrt_register_opt_bool(ctx->recursive, "recursive", &arg_map); + jfrt_register_opt_bool(ctx->transitive, "transitive", &arg_map); + + jfrt_register_opt_int(ctx->limit, "limit", &arg_map); + jfrt_register_opt_int(ctx->offset, "offset", &arg_map); + + char *args = join(arg_map->data, " "); + if (!args) { + return -1; + } + + snprintf(cmd, sizeof(cmd) - 1, "%s '%s/%s'", args, repo_path, pattern ? pattern: ""); + guard_free(args); + guard_strlist_free(&arg_map); + + int status = jfrog_cli_rt(auth, "search", cmd); + return status; +} diff --git a/src/lib/core/conda.c b/src/lib/core/conda.c new file mode 100644 index 0000000..35caf02 --- /dev/null +++ b/src/lib/core/conda.c @@ -0,0 +1,465 @@ +// +// Created by jhunk on 5/14/23. +// + +#include "conda.h" + +int micromamba(struct MicromambaInfo *info, char *command, ...) { + struct utsname sys; + uname(&sys); + + tolower_s(sys.sysname); + if (!strcmp(sys.sysname, "darwin")) { + strcpy(sys.sysname, "osx"); + } + + if (!strcmp(sys.machine, "x86_64")) { + strcpy(sys.machine, "64"); + } + + char url[PATH_MAX]; + sprintf(url, "https://micro.mamba.pm/api/micromamba/%s-%s/latest", sys.sysname, sys.machine); + + char installer_path[PATH_MAX]; + sprintf(installer_path, "%s/latest", getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"); + + if (access(installer_path, F_OK)) { + download(url, installer_path, NULL); + } + + char mmbin[PATH_MAX]; + sprintf(mmbin, "%s/micromamba", info->micromamba_prefix); + + if (access(mmbin, F_OK)) { + char untarcmd[PATH_MAX * 2]; + mkdirs(info->micromamba_prefix, 0755); + sprintf(untarcmd, "tar -xvf %s -C %s --strip-components=1 bin/micromamba 1>/dev/null", installer_path, info->micromamba_prefix); + int untarcmd_status = system(untarcmd); + if (untarcmd_status) { + return -1; + } + } + + char cmd[STASIS_BUFSIZ]; + memset(cmd, 0, sizeof(cmd)); + sprintf(cmd, "%s -r %s -p %s ", mmbin, info->conda_prefix, info->conda_prefix); + va_list args; + va_start(args, command); + vsprintf(cmd + strlen(cmd), command, args); + va_end(args); + + mkdirs(info->conda_prefix, 0755); + + char rcpath[PATH_MAX]; + sprintf(rcpath, "%s/.condarc", info->conda_prefix); + touch(rcpath); + + setenv("CONDARC", rcpath, 1); + setenv("MAMBA_ROOT_PREFIX", info->conda_prefix, 1); + int status = system(cmd); + unsetenv("MAMBA_ROOT_PREFIX"); + + return status; +} + +int python_exec(const char *args) { + char command[PATH_MAX]; + memset(command, 0, sizeof(command)); + snprintf(command, sizeof(command) - 1, "python %s", args); + msg(STASIS_MSG_L3, "Executing: %s\n", command); + return system(command); +} + +int pip_exec(const char *args) { + char command[PATH_MAX]; + memset(command, 0, sizeof(command)); + snprintf(command, sizeof(command) - 1, "python -m pip %s", args); + msg(STASIS_MSG_L3, "Executing: %s\n", command); + return system(command); +} + +int pip_index_provides(const char *index_url, const char *spec) { + char cmd[PATH_MAX] = {0}; + char spec_local[255] = {0}; + + if (isempty((char *) spec)) { + // NULL or zero-length; no package spec means there's nothing to do. + return -1; + } + + // Normalize the local spec string + strncpy(spec_local, spec, sizeof(spec_local) - 1); + tolower_s(spec_local); + lstrip(spec_local); + strip(spec_local); + + char logfile[] = "/tmp/STASIS-package_exists.XXXXXX"; + int logfd = mkstemp(logfile); + if (logfd < 0) { + perror(logfile); + remove(logfile); // fail harmlessly if not present + return -1; + } + + + int status = 0; + struct Process proc; + memset(&proc, 0, sizeof(proc)); + proc.redirect_stderr = 1; + strcpy(proc.f_stdout, logfile); + + // Do an installation in dry-run mode to see if the package exists in the given index. + snprintf(cmd, sizeof(cmd) - 1, "python -m pip install --dry-run --no-deps --index-url=%s %s", index_url, spec_local); + status = shell(&proc, cmd); + + // Print errors only when shell() itself throws one + // If some day we want to see the errors thrown by pip too, use this condition instead: (status != 0) + if (status < 0) { + FILE *fp = fdopen(logfd, "r"); + if (!fp) { + remove(logfile); + return -1; + } else { + char line[BUFSIZ] = {0}; + fflush(stdout); + fflush(stderr); + while (fgets(line, sizeof(line) - 1, fp) != NULL) { + fprintf(stderr, "%s", line); + } + fflush(stderr); + fclose(fp); + } + } + remove(logfile); + return proc.returncode == 0; +} + +int conda_exec(const char *args) { + char command[PATH_MAX]; + const char *mamba_commands[] = { + "build", + "install", + "update", + "create", + "list", + "search", + "run", + "info", + "clean", + "activate", + "deactivate", + NULL + }; + char conda_as[6]; + memset(conda_as, 0, sizeof(conda_as)); + + strcpy(conda_as, "conda"); + for (size_t i = 0; mamba_commands[i] != NULL; i++) { + if (startswith(args, mamba_commands[i])) { + strcpy(conda_as, "mamba"); + break; + } + } + + snprintf(command, sizeof(command) - 1, "%s %s", conda_as, args); + msg(STASIS_MSG_L3, "Executing: %s\n", command); + return system(command); +} + +int conda_activate(const char *root, const char *env_name) { + int fd = -1; + FILE *fp = NULL; + const char *init_script_conda = "/etc/profile.d/conda.sh"; + const char *init_script_mamba = "/etc/profile.d/mamba.sh"; + char path_conda[PATH_MAX] = {0}; + char path_mamba[PATH_MAX] = {0}; + char logfile[PATH_MAX] = {0}; + struct Process proc; + memset(&proc, 0, sizeof(proc)); + + // Where to find conda's init scripts + sprintf(path_conda, "%s%s", root, init_script_conda); + sprintf(path_mamba, "%s%s", root, init_script_mamba); + + // Set the path to our stdout log + // Emulate mktemp()'s behavior. Give us a unique file name, but don't use + // the file handle at all. We'll open it as a FILE stream soon enough. + sprintf(logfile, "%s/%s", globals.tmpdir, "shell_XXXXXX"); + fd = mkstemp(logfile); + if (fd < 0) { + perror(logfile); + return -1; + } + close(fd); + + // Configure our process for output to a log file + strcpy(proc.f_stdout, logfile); + + // Verify conda's init scripts are available + if (access(path_conda, F_OK) < 0) { + perror(path_conda); + remove(logfile); + return -1; + } + + if (access(path_mamba, F_OK) < 0) { + perror(path_mamba); + remove(logfile); + return -1; + } + + // Fully activate conda and record its effect on the runtime environment + char command[PATH_MAX * 3]; + snprintf(command, sizeof(command) - 1, "set -a; source %s; source %s; conda activate %s &>/dev/null; env -0", path_conda, path_mamba, env_name); + int retval = shell(&proc, command); + if (retval) { + // it didn't work; drop out for cleanup + remove(logfile); + return retval; + } + + // Parse the log file: + // 1. Extract the environment keys and values from the sub-shell + // 2. Apply it to STASIS's runtime environment + // 3. Now we're ready to execute conda commands anywhere + fp = fopen(proc.f_stdout, "r"); + if (!fp) { + perror(logfile); + return -1; + } + + while (!feof(fp)) { + char buf[STASIS_BUFSIZ] = {0}; + int ch = 0; + size_t z = 0; + // We are ingesting output from "env -0" and can't use fgets() + // Copy each character into the buffer until we encounter '\0' or EOF + while (z < sizeof(buf) && (ch = (int) fgetc(fp)) != 0) { + if (ch == EOF) { + break; + } + buf[z] = (char) ch; + z++; + } + buf[strlen(buf)] = 0; + + if (!strlen(buf)) { + continue; + } + + char **part = split(buf, "=", 1); + if (!part) { + perror("unable to split environment variable buffer"); + return -1; + } + if (!part[0]) { + msg(STASIS_MSG_WARN | STASIS_MSG_L1, "Invalid environment variable key ignored: '%s'\n", buf); + } else if (!part[1]) { + msg(STASIS_MSG_WARN | STASIS_MSG_L1, "Invalid environment variable value ignored: '%s'\n", buf); + } else { + setenv(part[0], part[1], 1); + } + GENERIC_ARRAY_FREE(part); + } + fclose(fp); + remove(logfile); + return 0; +} + +int conda_check_required() { + int status = 0; + struct StrList *result = NULL; + char cmd[PATH_MAX] = {0}; + const char *conda_minimum_viable_tools[] = { + "boa", + "conda-build", + "conda-verify", + NULL + }; + + // Construct a "conda list" command that searches for all required packages + // using conda's (python's) regex matching + strcat(cmd, "conda list '"); + for (size_t i = 0; conda_minimum_viable_tools[i] != NULL; i++) { + strcat(cmd, "^"); + strcat(cmd, conda_minimum_viable_tools[i]); + if (conda_minimum_viable_tools[i + 1] != NULL) { + strcat(cmd, "|"); + } + } + strcat(cmd, "' | cut -d ' ' -f 1"); + + // Verify all required packages are installed + char *cmd_out = shell_output(cmd, &status); + if (cmd_out) { + size_t found = 0; + result = strlist_init(); + strlist_append_tokenize(result, cmd_out, "\n"); + for (size_t i = 0; i < strlist_count(result); i++) { + char *item = strlist_item(result, i); + if (isempty(item) || startswith(item, "#")) { + continue; + } + + for (size_t x = 0; conda_minimum_viable_tools[x] != NULL; x++) { + if (!strcmp(item, conda_minimum_viable_tools[x])) { + found++; + } + } + } + if (found < (sizeof(conda_minimum_viable_tools) / sizeof(*conda_minimum_viable_tools)) - 1) { + guard_free(cmd_out); + guard_strlist_free(&result); + return 1; + } + guard_free(cmd_out); + guard_strlist_free(&result); + } else { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "The base package requirement check could not be performed\n"); + return 2; + } + return 0; +} + +int conda_setup_headless() { + if (globals.verbose) { + conda_exec("config --system --set quiet false"); + } else { + // Not verbose, so squelch conda's noise + conda_exec("config --system --set quiet true"); + } + + // Configure conda for headless CI + conda_exec("config --system --set auto_update_conda false"); // never update conda automatically + conda_exec("config --system --set notify_outdated_conda false"); // never notify about outdated conda version + conda_exec("config --system --set always_yes true"); // never prompt for input + conda_exec("config --system --set safety_checks disabled"); // speedup + conda_exec("config --system --set rollback_enabled false"); // speedup + conda_exec("config --system --set report_errors false"); // disable data sharing + conda_exec("config --system --set solver libmamba"); // use a real solver + + char cmd[PATH_MAX]; + size_t total = 0; + if (globals.conda_packages && strlist_count(globals.conda_packages)) { + memset(cmd, 0, sizeof(cmd)); + strcpy(cmd, "install "); + + total = strlist_count(globals.conda_packages); + for (size_t i = 0; i < total; i++) { + char *item = strlist_item(globals.conda_packages, i); + if (isempty(item)) { + continue; + } + sprintf(cmd + strlen(cmd), "'%s'", item); + if (i < total - 1) { + strcat(cmd, " "); + } + } + + if (conda_exec(cmd)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Unable to install user-defined base packages (conda)\n"); + return 1; + } + } + + if (globals.pip_packages && strlist_count(globals.pip_packages)) { + memset(cmd, 0, sizeof(cmd)); + strcpy(cmd, "install "); + + total = strlist_count(globals.pip_packages); + for (size_t i = 0; i < total; i++) { + char *item = strlist_item(globals.pip_packages, i); + if (isempty(item)) { + continue; + } + sprintf(cmd + strlen(cmd), "'%s'", item); + if (i < total - 1) { + strcat(cmd, " "); + } + } + + if (pip_exec(cmd)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Unable to install user-defined base packages (pip)\n"); + return 1; + } + } + + if (conda_check_required()) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Your STASIS configuration lacks the bare" + " minimum software required to build conda packages." + " Please fix it.\n"); + return 1; + } + + if (globals.always_update_base_environment) { + if (conda_exec("update --all")) { + fprintf(stderr, "conda update was unsuccessful\n"); + return 1; + } + } + + return 0; +} + +int conda_env_create_from_uri(char *name, char *uri) { + char env_command[PATH_MAX]; + sprintf(env_command, "env create -n %s -f %s", name, uri); + return conda_exec(env_command); +} + +int conda_env_create(char *name, char *python_version, char *packages) { + char env_command[PATH_MAX]; + sprintf(env_command, "create -n %s python=%s %s", name, python_version, packages ? packages : ""); + return conda_exec(env_command); +} + +int conda_env_remove(char *name) { + char env_command[PATH_MAX]; + sprintf(env_command, "env remove -n %s", name); + return conda_exec(env_command); +} + +int conda_env_export(char *name, char *output_dir, char *output_filename) { + char env_command[PATH_MAX]; + sprintf(env_command, "env export -n %s -f %s/%s.yml", name, output_dir, output_filename); + return conda_exec(env_command); +} + +char *conda_get_active_environment() { + const char *name = getenv("CONDA_DEFAULT_ENV"); + if (!name) { + return NULL; + } + + char *result = NULL; + result = strdup(name); + if (!result) { + return NULL; + } + + return result; +} + +int conda_provides(const char *spec) { + struct Process proc; + memset(&proc, 0, sizeof(proc)); + strcpy(proc.f_stdout, "/dev/null"); + strcpy(proc.f_stderr, "/dev/null"); + + // It's worth noting the departure from using conda_exec() here: + // conda_exec() expects the program output to be visible to the user. + // For this operation we only need the exit value. + char cmd[PATH_MAX] = {0}; + snprintf(cmd, sizeof(cmd) - 1, "mamba search --use-index-cache %s", spec); + if (shell(&proc, cmd) < 0) { + fprintf(stderr, "shell: %s", strerror(errno)); + return -1; + } + return proc.returncode == 0; +} + +int conda_index(const char *path) { + char command[PATH_MAX]; + sprintf(command, "index %s", path); + return conda_exec(command); +} diff --git a/src/lib/core/copy.c b/src/lib/core/copy.c new file mode 100644 index 0000000..f69a756 --- /dev/null +++ b/src/lib/core/copy.c @@ -0,0 +1,86 @@ +#include "copy.h" + +int copy2(const char *src, const char *dest, unsigned int op) { + size_t bytes_read; + size_t bytes_written; + char buf[STASIS_BUFSIZ]; + struct stat src_stat, dnamest; + FILE *fp1, *fp2; + + if (lstat(src, &src_stat) < 0) { + perror(src); + return -1; + } + + if (access(dest, F_OK) == 0) { + unlink(dest); + } + + char dname[1024] = {0}; + strcpy(dname, dest); + char *dname_endptr; + + dname_endptr = strrchr(dname, '/'); + if (dname_endptr != NULL) { + *dname_endptr = '\0'; + } + + stat(dname, &dnamest); + if (S_ISLNK(src_stat.st_mode)) { + char lpath[1024] = {0}; + if (readlink(src, lpath, sizeof(lpath)) < 0) { + perror(src); + return -1; + } + if (symlink(lpath, dest) < 0) { + // silent + return -1; + } + } else if (S_ISREG(src_stat.st_mode) && src_stat.st_nlink > 2 && src_stat.st_dev == dnamest.st_dev) { + if (link(src, dest) < 0) { + perror(src); + return -1; + } + } else if (S_ISFIFO(src_stat.st_mode) || S_ISBLK(src_stat.st_mode) || S_ISCHR(src_stat.st_mode) || S_ISSOCK(src_stat.st_mode)) { + if (mknod(dest, src_stat.st_mode, src_stat.st_rdev) < 0) { + perror(src); + return -1; + } + } else if (S_ISREG(src_stat.st_mode)) { + fp1 = fopen(src, "rb"); + if (!fp1) { + perror(src); + return -1; + } + + fp2 = fopen(dest, "w+b"); + if (!fp2) { + perror(dest); + return -1; + } + + bytes_written = 0; + while ((bytes_read = fread(buf, sizeof(char), sizeof(buf), fp1)) != 0) { + bytes_written += fwrite(buf, sizeof(char), bytes_read, fp2); + } + fclose(fp1); + fclose(fp2); + + if (bytes_written != (size_t) src_stat.st_size) { + fprintf(stderr, "%s: SHORT WRITE (expected %zu bytes, but wrote %zu bytes)\n", dest, src_stat.st_size, bytes_written); + return -1; + } + + if (op & CT_OWNER && chown(dest, src_stat.st_uid, src_stat.st_gid) < 0) { + perror(dest); + } + + if (op & CT_PERM && chmod(dest, src_stat.st_mode) < 0) { + perror(dest); + } + } else { + errno = EOPNOTSUPP; + return -1; + } + return 0; +} diff --git a/src/lib/core/delivery.c b/src/lib/core/delivery.c new file mode 100644 index 0000000..e32ed4c --- /dev/null +++ b/src/lib/core/delivery.c @@ -0,0 +1,317 @@ +#include "delivery.h" + +void delivery_free(struct Delivery *ctx) { + guard_free(ctx->system.arch); + GENERIC_ARRAY_FREE(ctx->system.platform); + guard_free(ctx->meta.name); + guard_free(ctx->meta.version); + guard_free(ctx->meta.codename); + guard_free(ctx->meta.mission); + guard_free(ctx->meta.python); + guard_free(ctx->meta.mission); + guard_free(ctx->meta.python_compact); + guard_free(ctx->meta.based_on); + guard_runtime_free(ctx->runtime.environ); + guard_free(ctx->storage.root); + guard_free(ctx->storage.tmpdir); + guard_free(ctx->storage.delivery_dir); + guard_free(ctx->storage.tools_dir); + guard_free(ctx->storage.package_dir); + guard_free(ctx->storage.results_dir); + guard_free(ctx->storage.output_dir); + guard_free(ctx->storage.conda_install_prefix); + guard_free(ctx->storage.conda_artifact_dir); + guard_free(ctx->storage.conda_staging_dir); + guard_free(ctx->storage.conda_staging_url); + guard_free(ctx->storage.wheel_artifact_dir); + guard_free(ctx->storage.wheel_staging_dir); + guard_free(ctx->storage.wheel_staging_url); + guard_free(ctx->storage.build_dir); + guard_free(ctx->storage.build_recipes_dir); + guard_free(ctx->storage.build_sources_dir); + guard_free(ctx->storage.build_testing_dir); + guard_free(ctx->storage.build_docker_dir); + guard_free(ctx->storage.mission_dir); + guard_free(ctx->storage.docker_artifact_dir); + guard_free(ctx->storage.meta_dir); + guard_free(ctx->storage.package_dir); + guard_free(ctx->storage.cfgdump_dir); + guard_free(ctx->info.time_str_epoch); + guard_free(ctx->info.build_name); + guard_free(ctx->info.build_number); + guard_free(ctx->info.release_name); + guard_free(ctx->conda.installer_baseurl); + guard_free(ctx->conda.installer_name); + guard_free(ctx->conda.installer_version); + guard_free(ctx->conda.installer_platform); + guard_free(ctx->conda.installer_arch); + guard_free(ctx->conda.installer_path); + guard_free(ctx->conda.tool_version); + guard_free(ctx->conda.tool_build_version); + guard_strlist_free(&ctx->conda.conda_packages); + guard_strlist_free(&ctx->conda.conda_packages_defer); + guard_strlist_free(&ctx->conda.pip_packages); + guard_strlist_free(&ctx->conda.pip_packages_defer); + guard_strlist_free(&ctx->conda.wheels_packages); + + for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { + guard_free(ctx->tests[i].name); + guard_free(ctx->tests[i].version); + guard_free(ctx->tests[i].repository); + guard_free(ctx->tests[i].repository_info_ref); + guard_free(ctx->tests[i].repository_info_tag); + guard_strlist_free(&ctx->tests[i].repository_remove_tags); + guard_free(ctx->tests[i].script); + guard_free(ctx->tests[i].build_recipe); + // test-specific runtime variables + guard_runtime_free(ctx->tests[i].runtime.environ); + } + + guard_free(ctx->rules.release_fmt); + guard_free(ctx->rules.build_name_fmt); + guard_free(ctx->rules.build_number_fmt); + + guard_free(ctx->deploy.docker.test_script); + guard_free(ctx->deploy.docker.registry); + guard_free(ctx->deploy.docker.image_compression); + guard_strlist_free(&ctx->deploy.docker.tags); + guard_strlist_free(&ctx->deploy.docker.build_args); + + for (size_t i = 0; i < sizeof(ctx->deploy.jfrog) / sizeof(ctx->deploy.jfrog[0]); i++) { + guard_free(ctx->deploy.jfrog[i].repo); + guard_free(ctx->deploy.jfrog[i].dest); + guard_strlist_free(&ctx->deploy.jfrog[i].files); + } + + if (ctx->_stasis_ini_fp.delivery) { + ini_free(&ctx->_stasis_ini_fp.delivery); + } + guard_free(ctx->_stasis_ini_fp.delivery_path); + + if (ctx->_stasis_ini_fp.cfg) { + // optional extras + ini_free(&ctx->_stasis_ini_fp.cfg); + } + guard_free(ctx->_stasis_ini_fp.cfg_path); + + if (ctx->_stasis_ini_fp.mission) { + ini_free(&ctx->_stasis_ini_fp.mission); + } + guard_free(ctx->_stasis_ini_fp.mission_path); +} + +int delivery_format_str(struct Delivery *ctx, char **dest, const char *fmt) { + size_t fmt_len = strlen(fmt); + + if (!*dest) { + *dest = calloc(STASIS_NAME_MAX, sizeof(**dest)); + if (!*dest) { + return -1; + } + } + + for (size_t i = 0; i < fmt_len; i++) { + if (fmt[i] == '%' && strlen(&fmt[i])) { + i++; + switch (fmt[i]) { + case 'n': // name + strcat(*dest, ctx->meta.name); + break; + case 'c': // codename + strcat(*dest, ctx->meta.codename); + break; + case 'm': // mission + strcat(*dest, ctx->meta.mission); + break; + case 'r': // revision + sprintf(*dest + strlen(*dest), "%d", ctx->meta.rc); + break; + case 'R': // "final"-aware revision + if (ctx->meta.final) + strcat(*dest, "final"); + else + sprintf(*dest + strlen(*dest), "%d", ctx->meta.rc); + break; + case 'v': // version + strcat(*dest, ctx->meta.version); + break; + case 'P': // python version + strcat(*dest, ctx->meta.python); + break; + case 'p': // python version major/minor + strcat(*dest, ctx->meta.python_compact); + break; + case 'a': // system architecture name + strcat(*dest, ctx->system.arch); + break; + case 'o': // system platform (OS) name + strcat(*dest, ctx->system.platform[DELIVERY_PLATFORM_RELEASE]); + break; + case 't': // unix epoch + sprintf(*dest + strlen(*dest), "%ld", ctx->info.time_now); + break; + default: // unknown formatter, write as-is + sprintf(*dest + strlen(*dest), "%c%c", fmt[i - 1], fmt[i]); + break; + } + } else { // write non-format text + sprintf(*dest + strlen(*dest), "%c", fmt[i]); + } + } + return 0; +} + +void delivery_defer_packages(struct Delivery *ctx, int type) { + struct StrList *dataptr = NULL; + struct StrList *deferred = NULL; + char *name = NULL; + char cmd[PATH_MAX]; + + memset(cmd, 0, sizeof(cmd)); + + char mode[10]; + if (DEFER_CONDA == type) { + dataptr = ctx->conda.conda_packages; + deferred = ctx->conda.conda_packages_defer; + strcpy(mode, "conda"); + } else if (DEFER_PIP == type) { + dataptr = ctx->conda.pip_packages; + deferred = ctx->conda.pip_packages_defer; + strcpy(mode, "pip"); + } else { + SYSERROR("BUG: type %d does not map to a supported package manager!\n", type); + exit(1); + } + msg(STASIS_MSG_L2, "Filtering %s packages by test definition...\n", mode); + + struct StrList *filtered = NULL; + filtered = strlist_init(); + for (size_t i = 0; i < strlist_count(dataptr); i++) { + int build_for_host = 0; + + name = strlist_item(dataptr, i); + if (!strlen(name) || isblank(*name) || isspace(*name)) { + // no data + continue; + } + + // Compile a list of packages that are *also* to be tested. + char *spec_begin = strpbrk(name, "@~=<>!"); + char *spec_end = spec_begin; + char package_name[255] = {0}; + + if (spec_end) { + // A version is present in the package name. Jump past operator(s). + while (*spec_end != '\0' && !isalnum(*spec_end)) { + spec_end++; + } + strncpy(package_name, name, spec_begin - name); + } else { + strncpy(package_name, name, sizeof(package_name) - 1); + } + + msg(STASIS_MSG_L3, "package '%s': ", package_name); + + // When spec is present in name, set tests->version to the version detected in the name + for (size_t x = 0; x < sizeof(ctx->tests) / sizeof(ctx->tests[0]) && ctx->tests[x].name != NULL; x++) { + struct Test *test = &ctx->tests[x]; + char nametmp[1024] = {0}; + + if (spec_end != NULL && spec_begin != NULL) { + strncpy(nametmp, name, spec_begin - name); + } else { + strcpy(nametmp, name); + } + // Is the [test:NAME] in the package name? + if (!strcmp(nametmp, test->name)) { + // Override test->version when a version is provided by the (pip|conda)_package list item + guard_free(test->version); + if (spec_begin && spec_end) { + test->version = strdup(spec_end); + } else { + // There are too many possible default branches nowadays: master, main, develop, xyz, etc. + // HEAD is a safe bet. + test->version = strdup("HEAD"); + } + + // Is the list item a git+schema:// URL? + if (strstr(nametmp, "git+") && strstr(nametmp, "://")) { + char *xrepo = strstr(nametmp, "+"); + if (xrepo) { + xrepo++; + guard_free(test->repository); + test->repository = strdup(xrepo); + xrepo = NULL; + } + // Extract the name of the package + char *xbasename = path_basename(nametmp); + if (xbasename) { + // Replace the git+schema:// URL with the package name + strlist_set(&dataptr, i, xbasename); + name = strlist_item(dataptr, i); + } + } + + int upstream_exists = 0; + if (DEFER_PIP == type) { + upstream_exists = pip_index_provides(PYPI_INDEX_DEFAULT, name); + } else if (DEFER_CONDA == type) { + upstream_exists = conda_provides(name); + } else { + fprintf(stderr, "\nUnknown package type: %d\n", type); + exit(1); + } + + if (upstream_exists < 0) { + fprintf(stderr, "%s's existence command failed for '%s'\n" + "(This may be due to a network/firewall issue!)\n", mode, name); + exit(1); + } + if (!upstream_exists) { + build_for_host = 1; + } else { + build_for_host = 0; + } + + break; + } + } + + if (build_for_host) { + printf("BUILD FOR HOST\n"); + strlist_append(&deferred, name); + } else { + printf("USE EXTERNAL\n"); + strlist_append(&filtered, name); + } + } + + if (!strlist_count(deferred)) { + msg(STASIS_MSG_WARN | STASIS_MSG_L2, "No %s packages were filtered by test definitions\n", mode); + } else { + if (DEFER_CONDA == type) { + strlist_free(&ctx->conda.conda_packages); + ctx->conda.conda_packages = strlist_copy(filtered); + } else if (DEFER_PIP == type) { + strlist_free(&ctx->conda.pip_packages); + ctx->conda.pip_packages = strlist_copy(filtered); + } + } + if (filtered) { + strlist_free(&filtered); + } +} + +void delivery_gather_tool_versions(struct Delivery *ctx) { + int status = 0; + + // Extract version from tool output + ctx->conda.tool_version = shell_output("conda --version", &status); + if (ctx->conda.tool_version) + strip(ctx->conda.tool_version); + + ctx->conda.tool_build_version = shell_output("conda build --version", &status); + if (ctx->conda.tool_build_version) + strip(ctx->conda.tool_version); +} + diff --git a/src/lib/core/delivery_artifactory.c b/src/lib/core/delivery_artifactory.c new file mode 100644 index 0000000..27f4823 --- /dev/null +++ b/src/lib/core/delivery_artifactory.c @@ -0,0 +1,192 @@ +#include "delivery.h" + +int delivery_init_artifactory(struct Delivery *ctx) { + int status = 0; + char dest[PATH_MAX] = {0}; + char filepath[PATH_MAX] = {0}; + snprintf(dest, sizeof(dest) - 1, "%s/bin", ctx->storage.tools_dir); + snprintf(filepath, sizeof(dest) - 1, "%s/bin/jf", ctx->storage.tools_dir); + + if (!access(filepath, F_OK)) { + // already have it + msg(STASIS_MSG_L3, "Skipped download, %s already exists\n", filepath); + goto delivery_init_artifactory_envsetup; + } + + char *platform = ctx->system.platform[DELIVERY_PLATFORM]; + msg(STASIS_MSG_L3, "Downloading %s for %s %s\n", globals.jfrog.remote_filename, platform, ctx->system.arch); + if ((status = artifactory_download_cli(dest, + globals.jfrog.jfrog_artifactory_base_url, + globals.jfrog.jfrog_artifactory_product, + globals.jfrog.cli_major_ver, + globals.jfrog.version, + platform, + ctx->system.arch, + globals.jfrog.remote_filename))) { + remove(filepath); + } + + delivery_init_artifactory_envsetup: + // CI (ridiculously generic, why?) disables interactive prompts and progress bar output + setenv("CI", "1", 1); + + // JFROG_CLI_HOME_DIR is where .jfrog is stored + char path[PATH_MAX] = {0}; + snprintf(path, sizeof(path) - 1, "%s/.jfrog", ctx->storage.build_dir); + setenv("JFROG_CLI_HOME_DIR", path, 1); + + // JFROG_CLI_TEMP_DIR is where the obvious is stored + setenv("JFROG_CLI_TEMP_DIR", ctx->storage.tmpdir, 1); + return status; +} + +int delivery_artifact_upload(struct Delivery *ctx) { + int status = 0; + + if (jfrt_auth_init(&ctx->deploy.jfrog_auth)) { + fprintf(stderr, "Failed to initialize Artifactory authentication context\n"); + return -1; + } + + for (size_t i = 0; i < sizeof(ctx->deploy.jfrog) / sizeof(*ctx->deploy.jfrog); i++) { + if (!ctx->deploy.jfrog[i].files || !ctx->deploy.jfrog[i].dest) { + break; + } + jfrt_upload_init(&ctx->deploy.jfrog[i].upload_ctx); + + if (!globals.jfrog.repo) { + msg(STASIS_MSG_WARN, "Artifactory repository path is not configured!\n"); + fprintf(stderr, "set STASIS_JF_REPO environment variable...\nOr append to configuration file:\n\n"); + fprintf(stderr, "[deploy:artifactory]\nrepo = example/generic/repo/path\n\n"); + status++; + break; + } else if (!ctx->deploy.jfrog[i].repo) { + ctx->deploy.jfrog[i].repo = strdup(globals.jfrog.repo); + } + + if (!ctx->deploy.jfrog[i].repo || isempty(ctx->deploy.jfrog[i].repo) || !strlen(ctx->deploy.jfrog[i].repo)) { + // Unlikely to trigger if the config parser is working correctly + msg(STASIS_MSG_ERROR, "Artifactory repository path is empty. Cannot continue.\n"); + status++; + break; + } + + ctx->deploy.jfrog[i].upload_ctx.workaround_parent_only = true; + ctx->deploy.jfrog[i].upload_ctx.build_name = ctx->info.build_name; + ctx->deploy.jfrog[i].upload_ctx.build_number = ctx->info.build_number; + + char files[PATH_MAX]; + char dest[PATH_MAX]; // repo + remote dir + + if (jfrog_cli_rt_ping(&ctx->deploy.jfrog_auth)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Unable to contact artifactory server: %s\n", ctx->deploy.jfrog_auth.url); + return -1; + } + + if (strlist_count(ctx->deploy.jfrog[i].files)) { + for (size_t f = 0; f < strlist_count(ctx->deploy.jfrog[i].files); f++) { + memset(dest, 0, sizeof(dest)); + memset(files, 0, sizeof(files)); + snprintf(dest, sizeof(dest) - 1, "%s/%s", ctx->deploy.jfrog[i].repo, ctx->deploy.jfrog[i].dest); + snprintf(files, sizeof(files) - 1, "%s", strlist_item(ctx->deploy.jfrog[i].files, f)); + status += jfrog_cli_rt_upload(&ctx->deploy.jfrog_auth, &ctx->deploy.jfrog[i].upload_ctx, files, dest); + } + } + } + + if (globals.enable_artifactory_build_info) { + if (!status && ctx->deploy.jfrog[0].files && ctx->deploy.jfrog[0].dest) { + jfrog_cli_rt_build_collect_env(&ctx->deploy.jfrog_auth, ctx->deploy.jfrog[0].upload_ctx.build_name, + ctx->deploy.jfrog[0].upload_ctx.build_number); + jfrog_cli_rt_build_publish(&ctx->deploy.jfrog_auth, ctx->deploy.jfrog[0].upload_ctx.build_name, + ctx->deploy.jfrog[0].upload_ctx.build_number); + } + } else { + msg(STASIS_MSG_WARN | STASIS_MSG_L2, "Artifactory build info upload is disabled by CLI argument\n"); + } + + return status; +} + +int delivery_mission_render_files(struct Delivery *ctx) { + if (!ctx->storage.mission_dir) { + fprintf(stderr, "Mission directory is not configured. Context not initialized?\n"); + return -1; + } + struct Data { + char *src; + char *dest; + } data; + struct INIFILE *cfg = ctx->_stasis_ini_fp.mission; + union INIVal val; + + memset(&data, 0, sizeof(data)); + data.src = calloc(PATH_MAX, sizeof(*data.src)); + if (!data.src) { + perror("data.src"); + return -1; + } + + for (size_t i = 0; i < cfg->section_count; i++) { + char *section_name = cfg->section[i]->key; + if (!startswith(section_name, "template:")) { + continue; + } + val.as_char_p = strchr(section_name, ':') + 1; + if (val.as_char_p && isempty(val.as_char_p)) { + guard_free(data.src); + return 1; + } + sprintf(data.src, "%s/%s/%s", ctx->storage.mission_dir, ctx->meta.mission, val.as_char_p); + msg(STASIS_MSG_L2, "%s\n", data.src); + + int err = 0; + data.dest = ini_getval_str(cfg, section_name, "destination", INI_READ_RENDER, &err); + + char *contents; + struct stat st; + if (lstat(data.src, &st)) { + perror(data.src); + guard_free(data.dest); + continue; + } + + contents = calloc(st.st_size + 1, sizeof(*contents)); + if (!contents) { + perror("template file contents"); + guard_free(data.dest); + continue; + } + + FILE *fp; + fp = fopen(data.src, "rb"); + if (!fp) { + perror(data.src); + guard_free(contents); + guard_free(data.dest); + continue; + } + + if (fread(contents, st.st_size, sizeof(*contents), fp) < 1) { + perror("while reading template file"); + guard_free(contents); + guard_free(data.dest); + fclose(fp); + continue; + } + fclose(fp); + + msg(STASIS_MSG_L3, "Writing %s\n", data.dest); + if (tpl_render_to_file(contents, data.dest)) { + guard_free(contents); + guard_free(data.dest); + continue; + } + guard_free(contents); + guard_free(data.dest); + } + + guard_free(data.src); + return 0; +} + diff --git a/src/lib/core/delivery_build.c b/src/lib/core/delivery_build.c new file mode 100644 index 0000000..b4d610a --- /dev/null +++ b/src/lib/core/delivery_build.c @@ -0,0 +1,190 @@ +#include "delivery.h" + +int delivery_build_recipes(struct Delivery *ctx) { + for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { + char *recipe_dir = NULL; + if (ctx->tests[i].build_recipe) { // build a conda recipe + int recipe_type; + int status; + if (recipe_clone(ctx->storage.build_recipes_dir, ctx->tests[i].build_recipe, NULL, &recipe_dir)) { + fprintf(stderr, "Encountered an issue while cloning recipe for: %s\n", ctx->tests[i].name); + return -1; + } + if (!recipe_dir) { + fprintf(stderr, "BUG: recipe_clone() succeeded but recipe_dir is NULL: %s\n", strerror(errno)); + return -1; + } + recipe_type = recipe_get_type(recipe_dir); + if(!pushd(recipe_dir)) { + if (RECIPE_TYPE_ASTROCONDA == recipe_type) { + pushd(path_basename(ctx->tests[i].repository)); + } else if (RECIPE_TYPE_CONDA_FORGE == recipe_type) { + pushd("recipe"); + } + + char recipe_version[100]; + char recipe_buildno[100]; + char recipe_git_url[PATH_MAX]; + char recipe_git_rev[PATH_MAX]; + + //sprintf(recipe_version, "{%% set version = GIT_DESCRIBE_TAG ~ \".dev\" ~ GIT_DESCRIBE_NUMBER ~ \"+\" ~ GIT_DESCRIBE_HASH %%}"); + //sprintf(recipe_git_url, " git_url: %s", ctx->tests[i].repository); + //sprintf(recipe_git_rev, " git_rev: %s", ctx->tests[i].version); + // TODO: Conditionally download archives if github.com is the origin. Else, use raw git_* keys ^^^ + sprintf(recipe_version, "{%% set version = \"%s\" %%}", ctx->tests[i].repository_info_tag ? ctx->tests[i].repository_info_tag : ctx->tests[i].version); + sprintf(recipe_git_url, " url: %s/archive/refs/tags/{{ version }}.tar.gz", ctx->tests[i].repository); + strcpy(recipe_git_rev, ""); + sprintf(recipe_buildno, " number: 0"); + + unsigned flags = REPLACE_TRUNCATE_AFTER_MATCH; + //file_replace_text("meta.yaml", "{% set version = ", recipe_version); + if (ctx->meta.final) { // remove this. i.e. statis cannot deploy a release to conda-forge + sprintf(recipe_version, "{%% set version = \"%s\" %%}", ctx->tests[i].version); + // TODO: replace sha256 of tagged archive + // TODO: leave the recipe unchanged otherwise. in theory this should produce the same conda package hash as conda forge. + // For now, remove the sha256 requirement + file_replace_text("meta.yaml", "sha256:", "\n", flags); + } else { + file_replace_text("meta.yaml", "{% set version = ", recipe_version, flags); + file_replace_text("meta.yaml", " url:", recipe_git_url, flags); + //file_replace_text("meta.yaml", "sha256:", recipe_git_rev); + file_replace_text("meta.yaml", " sha256:", "\n", flags); + file_replace_text("meta.yaml", " number:", recipe_buildno, flags); + } + + char command[PATH_MAX]; + if (RECIPE_TYPE_CONDA_FORGE == recipe_type) { + char arch[STASIS_NAME_MAX] = {0}; + char platform[STASIS_NAME_MAX] = {0}; + + strcpy(platform, ctx->system.platform[DELIVERY_PLATFORM]); + if (strstr(platform, "Darwin")) { + memset(platform, 0, sizeof(platform)); + strcpy(platform, "osx"); + } + tolower_s(platform); + if (strstr(ctx->system.arch, "arm64")) { + strcpy(arch, "arm64"); + } else if (strstr(ctx->system.arch, "64")) { + strcpy(arch, "64"); + } else { + strcat(arch, "32"); // blind guess + } + tolower_s(arch); + + sprintf(command, "mambabuild --python=%s -m ../.ci_support/%s_%s_.yaml .", + ctx->meta.python, platform, arch); + } else { + sprintf(command, "mambabuild --python=%s .", ctx->meta.python); + } + status = conda_exec(command); + if (status) { + guard_free(recipe_dir); + return -1; + } + + if (RECIPE_TYPE_GENERIC != recipe_type) { + popd(); + } + popd(); + } else { + fprintf(stderr, "Unable to enter recipe directory %s: %s\n", recipe_dir, strerror(errno)); + guard_free(recipe_dir); + return -1; + } + } + guard_free(recipe_dir); + } + return 0; +} + +int filter_repo_tags(char *repo, struct StrList *patterns) { + int result = 0; + + if (!pushd(repo)) { + int list_status = 0; + char *tags_raw = shell_output("git tag -l", &list_status); + struct StrList *tags = strlist_init(); + strlist_append_tokenize(tags, tags_raw, LINE_SEP); + + for (size_t i = 0; tags && i < strlist_count(tags); i++) { + char *tag = strlist_item(tags, i); + for (size_t p = 0; p < strlist_count(patterns); p++) { + char *pattern = strlist_item(patterns, p); + int match = fnmatch(pattern, tag, 0); + if (!match) { + char cmd[PATH_MAX] = {0}; + sprintf(cmd, "git tag -d %s", tag); + result += system(cmd); + break; + } + } + } + guard_strlist_free(&tags); + guard_free(tags_raw); + popd(); + } else { + result = -1; + } + return result; +} + +struct StrList *delivery_build_wheels(struct Delivery *ctx) { + struct StrList *result = NULL; + struct Process proc; + memset(&proc, 0, sizeof(proc)); + + result = strlist_init(); + if (!result) { + perror("unable to allocate memory for string list"); + return NULL; + } + + for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { + if (!ctx->tests[i].build_recipe && ctx->tests[i].repository) { // build from source + char srcdir[PATH_MAX]; + char wheeldir[PATH_MAX]; + memset(srcdir, 0, sizeof(srcdir)); + memset(wheeldir, 0, sizeof(wheeldir)); + + sprintf(srcdir, "%s/%s", ctx->storage.build_sources_dir, ctx->tests[i].name); + git_clone(&proc, ctx->tests[i].repository, srcdir, ctx->tests[i].version); + + if (ctx->tests[i].repository_remove_tags && strlist_count(ctx->tests[i].repository_remove_tags)) { + filter_repo_tags(srcdir, ctx->tests[i].repository_remove_tags); + } + + if (!pushd(srcdir)) { + char dname[NAME_MAX]; + char outdir[PATH_MAX]; + char cmd[PATH_MAX * 2]; + memset(dname, 0, sizeof(dname)); + memset(outdir, 0, sizeof(outdir)); + memset(cmd, 0, sizeof(outdir)); + + strcpy(dname, ctx->tests[i].name); + tolower_s(dname); + sprintf(outdir, "%s/%s", ctx->storage.wheel_artifact_dir, dname); + if (mkdirs(outdir, 0755)) { + fprintf(stderr, "failed to create output directory: %s\n", outdir); + guard_strlist_free(&result); + return NULL; + } + + sprintf(cmd, "-m build -w -o %s", outdir); + if (python_exec(cmd)) { + fprintf(stderr, "failed to generate wheel package for %s-%s\n", ctx->tests[i].name, ctx->tests[i].version); + guard_strlist_free(&result); + return NULL; + } + popd(); + } else { + fprintf(stderr, "Unable to enter source directory %s: %s\n", srcdir, strerror(errno)); + guard_strlist_free(&result); + return NULL; + } + } + } + return result; +} + diff --git a/src/lib/core/delivery_conda.c b/src/lib/core/delivery_conda.c new file mode 100644 index 0000000..93a06fc --- /dev/null +++ b/src/lib/core/delivery_conda.c @@ -0,0 +1,110 @@ +#include "delivery.h" + +void delivery_get_conda_installer_url(struct Delivery *ctx, char *result) { + if (ctx->conda.installer_version) { + // Use version specified by configuration file + sprintf(result, "%s/%s-%s-%s-%s.sh", ctx->conda.installer_baseurl, + ctx->conda.installer_name, + ctx->conda.installer_version, + ctx->conda.installer_platform, + ctx->conda.installer_arch); + } else { + // Use latest installer + sprintf(result, "%s/%s-%s-%s.sh", ctx->conda.installer_baseurl, + ctx->conda.installer_name, + ctx->conda.installer_platform, + ctx->conda.installer_arch); + } + +} + +int delivery_get_conda_installer(struct Delivery *ctx, char *installer_url) { + char script_path[PATH_MAX]; + char *installer = path_basename(installer_url); + + memset(script_path, 0, sizeof(script_path)); + sprintf(script_path, "%s/%s", ctx->storage.tmpdir, installer); + if (access(script_path, F_OK)) { + // Script doesn't exist + long fetch_status = download(installer_url, script_path, NULL); + if (HTTP_ERROR(fetch_status) || fetch_status < 0) { + // download failed + return -1; + } + } else { + msg(STASIS_MSG_RESTRICT | STASIS_MSG_L3, "Skipped, installer already exists\n", script_path); + } + + ctx->conda.installer_path = strdup(script_path); + if (!ctx->conda.installer_path) { + SYSERROR("Unable to duplicate script_path: '%s'", script_path); + return -1; + } + + return 0; +} + +void delivery_install_conda(char *install_script, char *conda_install_dir) { + struct Process proc; + memset(&proc, 0, sizeof(proc)); + + if (globals.conda_fresh_start) { + if (!access(conda_install_dir, F_OK)) { + // directory exists so remove it + if (rmtree(conda_install_dir)) { + perror("unable to remove previous installation"); + exit(1); + } + + // Proceed with the installation + // -b = batch mode (non-interactive) + char cmd[PATH_MAX] = {0}; + snprintf(cmd, sizeof(cmd) - 1, "%s %s -b -p %s", + find_program("bash"), + install_script, + conda_install_dir); + if (shell_safe(&proc, cmd)) { + fprintf(stderr, "conda installation failed\n"); + exit(1); + } + } else { + // Proceed with the installation + // -b = batch mode (non-interactive) + char cmd[PATH_MAX] = {0}; + snprintf(cmd, sizeof(cmd) - 1, "%s %s -b -p %s", + find_program("bash"), + install_script, + conda_install_dir); + if (shell_safe(&proc, cmd)) { + fprintf(stderr, "conda installation failed\n"); + exit(1); + } + } + } else { + msg(STASIS_MSG_L3, "Conda removal disabled by configuration\n"); + } +} + +void delivery_conda_enable(struct Delivery *ctx, char *conda_install_dir) { + if (conda_activate(conda_install_dir, "base")) { + fprintf(stderr, "conda activation failed\n"); + exit(1); + } + + // Setting the CONDARC environment variable appears to be the only consistent + // way to make sure the file is used. Not setting this variable leads to strange + // behavior, especially if a conda environment is already active when STASIS is loaded. + char rcpath[PATH_MAX]; + sprintf(rcpath, "%s/%s", conda_install_dir, ".condarc"); + setenv("CONDARC", rcpath, 1); + if (runtime_replace(&ctx->runtime.environ, __environ)) { + perror("unable to replace runtime environment after activating conda"); + exit(1); + } + + if (conda_setup_headless()) { + // no COE check. this call must succeed. + exit(1); + } +} + diff --git a/src/lib/core/delivery_docker.c b/src/lib/core/delivery_docker.c new file mode 100644 index 0000000..e1d7f60 --- /dev/null +++ b/src/lib/core/delivery_docker.c @@ -0,0 +1,132 @@ +#include "delivery.h" + +int delivery_docker(struct Delivery *ctx) { + if (!docker_capable(&ctx->deploy.docker.capabilities)) { + return -1; + } + char tag[STASIS_NAME_MAX]; + char args[PATH_MAX]; + int has_registry = ctx->deploy.docker.registry != NULL; + size_t total_tags = strlist_count(ctx->deploy.docker.tags); + size_t total_build_args = strlist_count(ctx->deploy.docker.build_args); + + if (!has_registry) { + msg(STASIS_MSG_WARN | STASIS_MSG_L2, "No docker registry defined. You will need to manually retag the resulting image.\n"); + } + + if (!total_tags) { + char default_tag[PATH_MAX]; + msg(STASIS_MSG_WARN | STASIS_MSG_L2, "No docker tags defined by configuration. Generating default tag(s).\n"); + // generate local tag + memset(default_tag, 0, sizeof(default_tag)); + sprintf(default_tag, "%s:%s-py%s", ctx->meta.name, ctx->info.build_name, ctx->meta.python_compact); + tolower_s(default_tag); + + // Add tag + ctx->deploy.docker.tags = strlist_init(); + strlist_append(&ctx->deploy.docker.tags, default_tag); + + if (has_registry) { + // generate tag for target registry + memset(default_tag, 0, sizeof(default_tag)); + sprintf(default_tag, "%s/%s:%s-py%s", ctx->deploy.docker.registry, ctx->meta.name, ctx->info.build_number, ctx->meta.python_compact); + tolower_s(default_tag); + + // Add tag + strlist_append(&ctx->deploy.docker.tags, default_tag); + } + // regenerate total tag available + total_tags = strlist_count(ctx->deploy.docker.tags); + } + + memset(args, 0, sizeof(args)); + + // Append image tags to command + for (size_t i = 0; i < total_tags; i++) { + char *tag_orig = strlist_item(ctx->deploy.docker.tags, i); + strcpy(tag, tag_orig); + docker_sanitize_tag(tag); + sprintf(args + strlen(args), " -t \"%s\" ", tag); + } + + // Append build arguments to command (i.e. --build-arg "key=value" + for (size_t i = 0; i < total_build_args; i++) { + char *build_arg = strlist_item(ctx->deploy.docker.build_args, i); + if (!build_arg) { + break; + } + sprintf(args + strlen(args), " --build-arg \"%s\" ", build_arg); + } + + // Build the image + char delivery_file[PATH_MAX]; + char dest[PATH_MAX]; + char rsync_cmd[PATH_MAX * 2]; + memset(delivery_file, 0, sizeof(delivery_file)); + memset(dest, 0, sizeof(dest)); + + sprintf(delivery_file, "%s/%s.yml", ctx->storage.delivery_dir, ctx->info.release_name); + if (access(delivery_file, F_OK) < 0) { + fprintf(stderr, "docker build cannot proceed without delivery file: %s\n", delivery_file); + return -1; + } + + sprintf(dest, "%s/%s.yml", ctx->storage.build_docker_dir, ctx->info.release_name); + if (copy2(delivery_file, dest, CT_PERM)) { + fprintf(stderr, "Failed to copy delivery file to %s: %s\n", dest, strerror(errno)); + return -1; + } + + memset(dest, 0, sizeof(dest)); + sprintf(dest, "%s/packages", ctx->storage.build_docker_dir); + + msg(STASIS_MSG_L2, "Copying conda packages\n"); + memset(rsync_cmd, 0, sizeof(rsync_cmd)); + sprintf(rsync_cmd, "rsync -avi --progress '%s' '%s'", ctx->storage.conda_artifact_dir, dest); + if (system(rsync_cmd)) { + fprintf(stderr, "Failed to copy conda artifacts to docker build directory\n"); + return -1; + } + + msg(STASIS_MSG_L2, "Copying wheel packages\n"); + memset(rsync_cmd, 0, sizeof(rsync_cmd)); + sprintf(rsync_cmd, "rsync -avi --progress '%s' '%s'", ctx->storage.wheel_artifact_dir, dest); + if (system(rsync_cmd)) { + fprintf(stderr, "Failed to copy wheel artifactory to docker build directory\n"); + } + + if (docker_build(ctx->storage.build_docker_dir, args, ctx->deploy.docker.capabilities.build)) { + return -1; + } + + // Test the image + // All tags point back to the same image so test the first one we see + // regardless of how many are defined + strcpy(tag, strlist_item(ctx->deploy.docker.tags, 0)); + docker_sanitize_tag(tag); + + msg(STASIS_MSG_L2, "Executing image test script for %s\n", tag); + if (ctx->deploy.docker.test_script) { + if (isempty(ctx->deploy.docker.test_script)) { + msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "Image test script has no content\n"); + } else { + int state; + if ((state = docker_script(tag, ctx->deploy.docker.test_script, 0))) { + msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "Non-zero exit (%d) from test script. %s image archive will not be generated.\n", state >> 8, tag); + // test failed -- don't save the image + return -1; + } + } + } else { + msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "No image test script defined\n"); + } + + // Test successful, save image + if (docker_save(path_basename(tag), ctx->storage.docker_artifact_dir, ctx->deploy.docker.image_compression)) { + // save failed + return -1; + } + + return 0; +} + diff --git a/src/lib/core/delivery_init.c b/src/lib/core/delivery_init.c new file mode 100644 index 0000000..e914f99 --- /dev/null +++ b/src/lib/core/delivery_init.c @@ -0,0 +1,345 @@ +#include "delivery.h" + +int has_mount_flags(const char *mount_point, const unsigned long flags) { + struct statvfs st; + if (statvfs(mount_point, &st)) { + SYSERROR("Unable to determine mount-point flags: %s", strerror(errno)); + return -1; + } + return (st.f_flag & flags) != 0; +} + +int delivery_init_tmpdir(struct Delivery *ctx) { + char *tmpdir = NULL; + char *x = NULL; + int unusable = 0; + errno = 0; + + x = getenv("TMPDIR"); + if (x) { + guard_free(ctx->storage.tmpdir); + tmpdir = strdup(x); + } else { + tmpdir = ctx->storage.tmpdir; + } + + if (!tmpdir) { + // memory error + return -1; + } + + // If the directory doesn't exist, create it + if (access(tmpdir, F_OK) < 0) { + if (mkdirs(tmpdir, 0755) < 0) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Unable to create temporary storage directory: %s (%s)\n", tmpdir, strerror(errno)); + goto l_delivery_init_tmpdir_fatal; + } + } + + // If we can't read, write, or execute, then die + if (access(tmpdir, R_OK | W_OK | X_OK) < 0) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "%s requires at least 0755 permissions.\n"); + goto l_delivery_init_tmpdir_fatal; + } + + struct statvfs st; + if (statvfs(tmpdir, &st) < 0) { + goto l_delivery_init_tmpdir_fatal; + } + +#if defined(STASIS_OS_LINUX) + // If we can't execute programs, or write data to the file system at all, then die + if ((st.f_flag & ST_NOEXEC) != 0) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "%s is mounted with noexec\n", tmpdir); + goto l_delivery_init_tmpdir_fatal; + } +#endif + if ((st.f_flag & ST_RDONLY) != 0) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "%s is mounted read-only\n", tmpdir); + goto l_delivery_init_tmpdir_fatal; + } + + if (!globals.tmpdir) { + globals.tmpdir = strdup(tmpdir); + } + + if (!ctx->storage.tmpdir) { + ctx->storage.tmpdir = strdup(globals.tmpdir); + } + return unusable; + + l_delivery_init_tmpdir_fatal: + unusable = 1; + return unusable; +} + +void delivery_init_dirs_stage2(struct Delivery *ctx) { + path_store(&ctx->storage.build_recipes_dir, PATH_MAX, ctx->storage.build_dir, "recipes"); + path_store(&ctx->storage.build_sources_dir, PATH_MAX, ctx->storage.build_dir, "sources"); + path_store(&ctx->storage.build_testing_dir, PATH_MAX, ctx->storage.build_dir, "testing"); + path_store(&ctx->storage.build_docker_dir, PATH_MAX, ctx->storage.build_dir, "docker"); + + path_store(&ctx->storage.delivery_dir, PATH_MAX, ctx->storage.output_dir, "delivery"); + path_store(&ctx->storage.results_dir, PATH_MAX, ctx->storage.output_dir, "results"); + path_store(&ctx->storage.package_dir, PATH_MAX, ctx->storage.output_dir, "packages"); + path_store(&ctx->storage.cfgdump_dir, PATH_MAX, ctx->storage.output_dir, "config"); + path_store(&ctx->storage.meta_dir, PATH_MAX, ctx->storage.output_dir, "meta"); + + path_store(&ctx->storage.conda_artifact_dir, PATH_MAX, ctx->storage.package_dir, "conda"); + path_store(&ctx->storage.wheel_artifact_dir, PATH_MAX, ctx->storage.package_dir, "wheels"); + path_store(&ctx->storage.docker_artifact_dir, PATH_MAX, ctx->storage.package_dir, "docker"); +} + +void delivery_init_dirs_stage1(struct Delivery *ctx) { + char *rootdir = getenv("STASIS_ROOT"); + if (rootdir) { + if (isempty(rootdir)) { + fprintf(stderr, "STASIS_ROOT is set, but empty. Please assign a file system path to this environment variable.\n"); + exit(1); + } + path_store(&ctx->storage.root, PATH_MAX, rootdir, ctx->info.build_name); + } else { + // use "stasis" in current working directory + path_store(&ctx->storage.root, PATH_MAX, "stasis", ctx->info.build_name); + } + path_store(&ctx->storage.tools_dir, PATH_MAX, ctx->storage.root, "tools"); + path_store(&ctx->storage.tmpdir, PATH_MAX, ctx->storage.root, "tmp"); + if (delivery_init_tmpdir(ctx)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Set $TMPDIR to a location other than %s\n", globals.tmpdir); + if (globals.tmpdir) + guard_free(globals.tmpdir); + exit(1); + } + + path_store(&ctx->storage.build_dir, PATH_MAX, ctx->storage.root, "build"); + path_store(&ctx->storage.output_dir, PATH_MAX, ctx->storage.root, "output"); + + if (!ctx->storage.mission_dir) { + path_store(&ctx->storage.mission_dir, PATH_MAX, globals.sysconfdir, "mission"); + } + + if (access(ctx->storage.mission_dir, F_OK)) { + msg(STASIS_MSG_L1, "%s: %s\n", ctx->storage.mission_dir, strerror(errno)); + exit(1); + } + + // Override installation prefix using global configuration key + if (globals.conda_install_prefix && strlen(globals.conda_install_prefix)) { + // user wants a specific path + globals.conda_fresh_start = false; + /* + if (mkdirs(globals.conda_install_prefix, 0755)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Unable to create directory: %s: %s\n", + strerror(errno), globals.conda_install_prefix); + exit(1); + } + */ + /* + ctx->storage.conda_install_prefix = realpath(globals.conda_install_prefix, NULL); + if (!ctx->storage.conda_install_prefix) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "realpath(): Conda installation prefix reassignment failed\n"); + exit(1); + } + ctx->storage.conda_install_prefix = strdup(globals.conda_install_prefix); + */ + path_store(&ctx->storage.conda_install_prefix, PATH_MAX, globals.conda_install_prefix, "conda"); + } else { + // install conda under the STASIS tree + path_store(&ctx->storage.conda_install_prefix, PATH_MAX, ctx->storage.tools_dir, "conda"); + } +} + +int delivery_init_platform(struct Delivery *ctx) { + msg(STASIS_MSG_L2, "Setting architecture\n"); + char archsuffix[20]; + struct utsname uts; + if (uname(&uts)) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "uname() failed: %s\n", strerror(errno)); + return -1; + } + + ctx->system.platform = calloc(DELIVERY_PLATFORM_MAX + 1, sizeof(*ctx->system.platform)); + if (!ctx->system.platform) { + SYSERROR("Unable to allocate %d records for platform array\n", DELIVERY_PLATFORM_MAX); + return -1; + } + for (size_t i = 0; i < DELIVERY_PLATFORM_MAX; i++) { + ctx->system.platform[i] = calloc(DELIVERY_PLATFORM_MAXLEN, sizeof(*ctx->system.platform[0])); + } + + ctx->system.arch = strdup(uts.machine); + if (!ctx->system.arch) { + // memory error + return -1; + } + + if (!strcmp(ctx->system.arch, "x86_64")) { + strcpy(archsuffix, "64"); + } else { + strcpy(archsuffix, ctx->system.arch); + } + + msg(STASIS_MSG_L2, "Setting platform\n"); + strcpy(ctx->system.platform[DELIVERY_PLATFORM], uts.sysname); + if (!strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Darwin")) { + sprintf(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], "osx-%s", archsuffix); + strcpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], "MacOSX"); + strcpy(ctx->system.platform[DELIVERY_PLATFORM_RELEASE], "macos"); + } else if (!strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Linux")) { + sprintf(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], "linux-%s", archsuffix); + strcpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], "Linux"); + strcpy(ctx->system.platform[DELIVERY_PLATFORM_RELEASE], "linux"); + } else { + // Not explicitly supported systems + strcpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], ctx->system.platform[DELIVERY_PLATFORM]); + strcpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], ctx->system.platform[DELIVERY_PLATFORM]); + strcpy(ctx->system.platform[DELIVERY_PLATFORM_RELEASE], ctx->system.platform[DELIVERY_PLATFORM]); + tolower_s(ctx->system.platform[DELIVERY_PLATFORM_RELEASE]); + } + + long cpu_count = get_cpu_count(); + if (!cpu_count) { + fprintf(stderr, "Unable to determine CPU count. Falling back to 1.\n"); + cpu_count = 1; + } + char ncpus[100] = {0}; + sprintf(ncpus, "%ld", cpu_count); + + // Declare some important bits as environment variables + setenv("CPU_COUNT", ncpus, 1); + setenv("STASIS_CPU_COUNT", ncpus, 1); + setenv("STASIS_ARCH", ctx->system.arch, 1); + setenv("STASIS_PLATFORM", ctx->system.platform[DELIVERY_PLATFORM], 1); + setenv("STASIS_CONDA_ARCH", ctx->system.arch, 1); + setenv("STASIS_CONDA_PLATFORM", ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], 1); + setenv("STASIS_CONDA_PLATFORM_SUBDIR", ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], 1); + + // Register template variables + // These were moved out of main() because we can't take the address of system.platform[x] + // _before_ the array has been initialized. + tpl_register("system.arch", &ctx->system.arch); + tpl_register("system.platform", &ctx->system.platform[DELIVERY_PLATFORM_RELEASE]); + + return 0; +} + +int delivery_init(struct Delivery *ctx, int render_mode) { + populate_info(ctx); + populate_delivery_cfg(ctx, INI_READ_RENDER); + + // Set artifactory URL via environment variable if possible + char *jfurl = getenv("STASIS_JF_ARTIFACTORY_URL"); + if (jfurl) { + if (globals.jfrog.url) { + guard_free(globals.jfrog.url); + } + globals.jfrog.url = strdup(jfurl); + } + + // Set artifactory repository via environment if possible + char *jfrepo = getenv("STASIS_JF_REPO"); + if (jfrepo) { + if (globals.jfrog.repo) { + guard_free(globals.jfrog.repo); + } + globals.jfrog.repo = strdup(jfrepo); + } + + // Configure architecture and platform information + delivery_init_platform(ctx); + + // Create STASIS directory structure + delivery_init_dirs_stage1(ctx); + + char config_local[PATH_MAX]; + sprintf(config_local, "%s/%s", ctx->storage.tmpdir, "config"); + setenv("XDG_CONFIG_HOME", config_local, 1); + + char cache_local[PATH_MAX]; + sprintf(cache_local, "%s/%s", ctx->storage.tmpdir, "cache"); + setenv("XDG_CACHE_HOME", ctx->storage.tmpdir, 1); + + // add tools to PATH + char pathvar_tmp[STASIS_BUFSIZ]; + sprintf(pathvar_tmp, "%s/bin:%s", ctx->storage.tools_dir, getenv("PATH")); + setenv("PATH", pathvar_tmp, 1); + + // Prevent git from paginating output + setenv("GIT_PAGER", "", 1); + + populate_delivery_ini(ctx, render_mode); + + if (ctx->deploy.docker.tags) { + for (size_t i = 0; i < strlist_count(ctx->deploy.docker.tags); i++) { + char *item = strlist_item(ctx->deploy.docker.tags, i); + tolower_s(item); + } + } + + if (ctx->deploy.docker.image_compression) { + if (docker_validate_compression_program(ctx->deploy.docker.image_compression)) { + SYSERROR("[deploy:docker].image_compression - invalid command / program is not installed: %s", ctx->deploy.docker.image_compression); + return -1; + } + } + return 0; +} + +int bootstrap_build_info(struct Delivery *ctx) { + struct Delivery local; + memset(&local, 0, sizeof(local)); + local._stasis_ini_fp.cfg = ini_open(ctx->_stasis_ini_fp.cfg_path); + local._stasis_ini_fp.delivery = ini_open(ctx->_stasis_ini_fp.delivery_path); + delivery_init_platform(&local); + populate_delivery_cfg(&local, INI_READ_RENDER); + populate_delivery_ini(&local, INI_READ_RENDER); + populate_info(&local); + ctx->info.build_name = strdup(local.info.build_name); + ctx->info.build_number = strdup(local.info.build_number); + ctx->info.release_name = strdup(local.info.release_name); + ctx->info.time_info = malloc(sizeof(*ctx->info.time_info)); + if (!ctx->info.time_info) { + SYSERROR("Unable to allocate %zu bytes for tm struct: %s", sizeof(*local.info.time_info), strerror(errno)); + return -1; + } + memcpy(ctx->info.time_info, local.info.time_info, sizeof(*local.info.time_info)); + ctx->info.time_now = local.info.time_now; + ctx->info.time_str_epoch = strdup(local.info.time_str_epoch); + delivery_free(&local); + return 0; +} + +int delivery_exists(struct Delivery *ctx) { + int release_exists = 0; + char release_pattern[PATH_MAX] = {0}; + sprintf(release_pattern, "*%s*", ctx->info.release_name); + + if (globals.enable_artifactory) { + if (jfrt_auth_init(&ctx->deploy.jfrog_auth)) { + fprintf(stderr, "Failed to initialize Artifactory authentication context\n"); + return -1; // error + } + + struct JFRT_Search search = {.fail_no_op = true}; + release_exists = jfrog_cli_rt_search(&ctx->deploy.jfrog_auth, &search, globals.jfrog.repo, release_pattern); + if (release_exists != 2) { + if (!globals.enable_overwrite && !release_exists) { + // --fail_no_op returns 2 on failure + // without: it returns an empty list "[]" and exit code 0 + return 1; // found + } + } + } else { + struct StrList *files = listdir(ctx->storage.delivery_dir); + for (size_t i = 0; i < strlist_count(files); i++) { + char *filename = strlist_item(files, i); + release_exists = fnmatch(release_pattern, filename, FNM_PATHNAME); + if (!globals.enable_overwrite && !release_exists) { + guard_strlist_free(&files); + return 1; // found + } + } + guard_strlist_free(&files); + } + return 0; // not found +} diff --git a/src/lib/core/delivery_install.c b/src/lib/core/delivery_install.c new file mode 100644 index 0000000..76c3f4a --- /dev/null +++ b/src/lib/core/delivery_install.c @@ -0,0 +1,224 @@ +#include "delivery.h" + +static struct Test *requirement_from_test(struct Delivery *ctx, const char *name) { + struct Test *result = NULL; + for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { + if (ctx->tests[i].name && !strcmp(name, ctx->tests[i].name)) { + result = &ctx->tests[i]; + break; + } + } + return result; +} + +static char *have_spec_in_config(struct Delivery *ctx, const char *name) { + for (size_t x = 0; x < strlist_count(ctx->conda.pip_packages); x++) { + char *config_spec = strlist_item(ctx->conda.pip_packages, x); + char *op = find_version_spec(config_spec); + char package[255] = {0}; + if (op) { + strncpy(package, config_spec, op - config_spec); + } else { + strncpy(package, config_spec, sizeof(package) - 1); + } + if (strncmp(package, name, strlen(package)) == 0) { + return config_spec; + } + } + return NULL; +} + +int delivery_overlay_packages_from_env(struct Delivery *ctx, const char *env_name) { + char *current_env = conda_get_active_environment(); + int need_restore = current_env && strcmp(env_name, current_env) != 0; + + conda_activate(ctx->storage.conda_install_prefix, env_name); + // Retrieve a listing of python packages installed under "env_name" + int freeze_status = 0; + char *freeze_output = shell_output("python -m pip freeze", &freeze_status); + if (freeze_status) { + guard_free(freeze_output); + guard_free(current_env); + return -1; + } + + if (need_restore) { + // Restore the original conda environment + conda_activate(ctx->storage.conda_install_prefix, current_env); + } + guard_free(current_env); + + struct StrList *frozen_list = strlist_init(); + strlist_append_tokenize(frozen_list, freeze_output, LINE_SEP); + guard_free(freeze_output); + + struct StrList *new_list = strlist_init(); + + // - consume package specs that have no test blocks. + // - these will be third-party packages like numpy, scipy, etc. + // - and they need to be present at the head of the list so they + // get installed first. + for (size_t i = 0; i < strlist_count(ctx->conda.pip_packages); i++) { + char *spec = strlist_item(ctx->conda.pip_packages, i); + char spec_name[255] = {0}; + char *op = find_version_spec(spec); + if (op) { + strncpy(spec_name, spec, op - spec); + } else { + strncpy(spec_name, spec, sizeof(spec_name) - 1); + } + struct Test *test_block = requirement_from_test(ctx, spec_name); + if (!test_block) { + msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "from config without test: %s\n", spec); + strlist_append(&new_list, spec); + } + } + + // now consume packages that have a test block + // if the ini provides a spec, override the environment's version. + // otherwise, use the spec derived from the environment + for (size_t i = 0; i < strlist_count(frozen_list); i++) { + char *frozen_spec = strlist_item(frozen_list, i); + char frozen_name[255] = {0}; + char *op = find_version_spec(frozen_spec); + // we only care about packages with specs here. if something else arrives, ignore it + if (op) { + strncpy(frozen_name, frozen_spec, op - frozen_spec); + } else { + strncpy(frozen_name, frozen_spec, sizeof(frozen_name) - 1); + } + struct Test *test = requirement_from_test(ctx, frozen_name); + if (test && strcmp(test->name, frozen_name) == 0) { + char *config_spec = have_spec_in_config(ctx, frozen_name); + if (config_spec) { + msg(STASIS_MSG_L2, "from config: %s\n", config_spec); + strlist_append(&new_list, config_spec); + } else { + msg(STASIS_MSG_L2, "from environment: %s\n", frozen_spec); + strlist_append(&new_list, frozen_spec); + } + } + } + + // Replace the package manifest as needed + if (strlist_count(new_list)) { + guard_strlist_free(&ctx->conda.pip_packages); + ctx->conda.pip_packages = strlist_copy(new_list); + } + guard_strlist_free(&new_list); + guard_strlist_free(&frozen_list); + return 0; +} + +int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, char *env_name, int type, struct StrList **manifest) { + char cmd[PATH_MAX]; + char pkgs[STASIS_BUFSIZ]; + char *env_current = getenv("CONDA_DEFAULT_ENV"); + + if (env_current) { + // The requested environment is not the current environment + if (strcmp(env_current, env_name) != 0) { + // Activate the requested environment + printf("Activating: %s\n", env_name); + conda_activate(conda_install_dir, env_name); + runtime_replace(&ctx->runtime.environ, __environ); + } + } + + memset(cmd, 0, sizeof(cmd)); + memset(pkgs, 0, sizeof(pkgs)); + strcat(cmd, "install"); + + typedef int (*Runner)(const char *); + Runner runner = NULL; + if (INSTALL_PKG_CONDA & type) { + runner = conda_exec; + } else if (INSTALL_PKG_PIP & type) { + runner = pip_exec; + } + + if (INSTALL_PKG_CONDA_DEFERRED & type) { + strcat(cmd, " --use-local"); + } else if (INSTALL_PKG_PIP_DEFERRED & type) { + // Don't change the baseline package set unless we're working with a + // new build. Release candidates will need to keep packages as stable + // as possible between releases. + if (!ctx->meta.based_on) { + strcat(cmd, " --upgrade"); + } + sprintf(cmd + strlen(cmd), " --extra-index-url 'file://%s'", ctx->storage.wheel_artifact_dir); + } + + for (size_t x = 0; manifest[x] != NULL; x++) { + char *name = NULL; + for (size_t p = 0; p < strlist_count(manifest[x]); p++) { + name = strlist_item(manifest[x], p); + strip(name); + if (!strlen(name)) { + continue; + } + if (INSTALL_PKG_PIP_DEFERRED & type) { + struct Test *info = requirement_from_test(ctx, name); + if (info) { + if (!strcmp(info->version, "HEAD")) { + struct StrList *tag_data = strlist_init(); + if (!tag_data) { + SYSERROR("%s", "Unable to allocate memory for tag data\n"); + return -1; + } + strlist_append_tokenize(tag_data, info->repository_info_tag, "-"); + + struct Wheel *whl = NULL; + char *post_commit = NULL; + char *hash = NULL; + if (strlist_count(tag_data) > 1) { + post_commit = strlist_item(tag_data, 1); + hash = strlist_item(tag_data, 2); + } + + // We can't match on version here (index 0). The wheel's version is not guaranteed to be + // equal to the tag; setuptools_scm auto-increments the value, the user can change it manually, + // etc. + errno = 0; + whl = get_wheel_info(ctx->storage.wheel_artifact_dir, info->name, + (char *[]) {ctx->meta.python_compact, ctx->system.arch, + "none", "any", + post_commit, hash, + NULL}, WHEEL_MATCH_ANY); + if (!whl && errno) { + // error + SYSERROR("Unable to read Python wheel info: %s\n", strerror(errno)); + exit(1); + } else if (!whl) { + // not found + fprintf(stderr, "No wheel packages found that match the description of '%s'", info->name); + } else { + // found + guard_strlist_free(&tag_data); + info->version = strdup(whl->version); + } + wheel_free(&whl); + } + snprintf(cmd + strlen(cmd), + sizeof(cmd) - strlen(cmd) - strlen(info->name) - strlen(info->version) + 5, + " '%s==%s'", info->name, info->version); + } else { + fprintf(stderr, "Deferred package '%s' is not present in the tested package list!\n", name); + return -1; + } + } else { + if (startswith(name, "--") || startswith(name, "-")) { + sprintf(cmd + strlen(cmd), " %s", name); + } else { + sprintf(cmd + strlen(cmd), " '%s'", name); + } + } + } + int status = runner(cmd); + if (status) { + return status; + } + } + return 0; +} + diff --git a/src/lib/core/delivery_populate.c b/src/lib/core/delivery_populate.c new file mode 100644 index 0000000..b37f677 --- /dev/null +++ b/src/lib/core/delivery_populate.c @@ -0,0 +1,348 @@ +#include "delivery.h" + +static void ini_has_key_required(struct INIFILE *ini, const char *section_name, char *key) { + int status = ini_has_key(ini, section_name, key); + if (!status) { + SYSERROR("%s:%s key is required but not defined", section_name, key); + exit(1); + } +} + +static void conv_str(char **x, union INIVal val) { + if (*x) { + guard_free(*x); + } + if (val.as_char_p) { + char *tplop = tpl_render(val.as_char_p); + if (tplop) { + *x = tplop; + } else { + *x = NULL; + } + } else { + *x = NULL; + } +} + + + +int populate_info(struct Delivery *ctx) { + if (!ctx->info.time_str_epoch) { + // Record timestamp used for release + time(&ctx->info.time_now); + ctx->info.time_info = localtime(&ctx->info.time_now); + + ctx->info.time_str_epoch = calloc(STASIS_TIME_STR_MAX, sizeof(*ctx->info.time_str_epoch)); + if (!ctx->info.time_str_epoch) { + msg(STASIS_MSG_ERROR, "Unable to allocate memory for Unix epoch string\n"); + return -1; + } + snprintf(ctx->info.time_str_epoch, STASIS_TIME_STR_MAX - 1, "%li", ctx->info.time_now); + } + return 0; +} + +int populate_delivery_cfg(struct Delivery *ctx, int render_mode) { + struct INIFILE *cfg = ctx->_stasis_ini_fp.cfg; + if (!cfg) { + return -1; + } + int err = 0; + ctx->storage.conda_staging_dir = ini_getval_str(cfg, "default", "conda_staging_dir", render_mode, &err); + ctx->storage.conda_staging_url = ini_getval_str(cfg, "default", "conda_staging_url", render_mode, &err); + ctx->storage.wheel_staging_dir = ini_getval_str(cfg, "default", "wheel_staging_dir", render_mode, &err); + ctx->storage.wheel_staging_url = ini_getval_str(cfg, "default", "wheel_staging_url", render_mode, &err); + globals.conda_fresh_start = ini_getval_bool(cfg, "default", "conda_fresh_start", render_mode, &err); + if (!globals.continue_on_error) { + globals.continue_on_error = ini_getval_bool(cfg, "default", "continue_on_error", render_mode, &err); + } + if (!globals.always_update_base_environment) { + globals.always_update_base_environment = ini_getval_bool(cfg, "default", "always_update_base_environment", render_mode, &err); + } + globals.conda_install_prefix = ini_getval_str(cfg, "default", "conda_install_prefix", render_mode, &err); + globals.conda_packages = ini_getval_strlist(cfg, "default", "conda_packages", LINE_SEP, render_mode, &err); + globals.pip_packages = ini_getval_strlist(cfg, "default", "pip_packages", LINE_SEP, render_mode, &err); + + globals.jfrog.jfrog_artifactory_base_url = ini_getval_str(cfg, "jfrog_cli_download", "url", render_mode, &err); + globals.jfrog.jfrog_artifactory_product = ini_getval_str(cfg, "jfrog_cli_download", "product", render_mode, &err); + globals.jfrog.cli_major_ver = ini_getval_str(cfg, "jfrog_cli_download", "version_series", render_mode, &err); + globals.jfrog.version = ini_getval_str(cfg, "jfrog_cli_download", "version", render_mode, &err); + globals.jfrog.remote_filename = ini_getval_str(cfg, "jfrog_cli_download", "filename", render_mode, &err); + globals.jfrog.url = ini_getval_str(cfg, "deploy:artifactory", "url", render_mode, &err); + globals.jfrog.repo = ini_getval_str(cfg, "deploy:artifactory", "repo", render_mode, &err); + + return 0; +} + +int populate_delivery_ini(struct Delivery *ctx, int render_mode) { + union INIVal val; + struct INIFILE *ini = ctx->_stasis_ini_fp.delivery; + struct INIData *rtdata; + RuntimeEnv *rt; + + validate_delivery_ini(ini); + // Populate runtime variables first they may be interpreted by other + // keys in the configuration + rt = runtime_copy(__environ); + while ((rtdata = ini_getall(ini, "runtime")) != NULL) { + char rec[STASIS_BUFSIZ]; + sprintf(rec, "%s=%s", lstrip(strip(rtdata->key)), lstrip(strip(rtdata->value))); + runtime_set(rt, rtdata->key, rtdata->value); + } + runtime_apply(rt); + ctx->runtime.environ = rt; + + int err = 0; + ctx->meta.mission = ini_getval_str(ini, "meta", "mission", render_mode, &err); + + if (!strcasecmp(ctx->meta.mission, "hst")) { + ctx->meta.codename = ini_getval_str(ini, "meta", "codename", render_mode, &err); + } else { + ctx->meta.codename = NULL; + } + + ctx->meta.version = ini_getval_str(ini, "meta", "version", render_mode, &err); + ctx->meta.name = ini_getval_str(ini, "meta", "name", render_mode, &err); + ctx->meta.rc = ini_getval_int(ini, "meta", "rc", render_mode, &err); + ctx->meta.final = ini_getval_bool(ini, "meta", "final", render_mode, &err); + ctx->meta.based_on = ini_getval_str(ini, "meta", "based_on", render_mode, &err); + + if (!ctx->meta.python) { + ctx->meta.python = ini_getval_str(ini, "meta", "python", render_mode, &err); + guard_free(ctx->meta.python_compact); + ctx->meta.python_compact = to_short_version(ctx->meta.python); + } else { + ini_setval(&ini, INI_SETVAL_REPLACE, "meta", "python", ctx->meta.python); + } + + ctx->conda.installer_name = ini_getval_str(ini, "conda", "installer_name", render_mode, &err); + ctx->conda.installer_version = ini_getval_str(ini, "conda", "installer_version", render_mode, &err); + ctx->conda.installer_platform = ini_getval_str(ini, "conda", "installer_platform", render_mode, &err); + ctx->conda.installer_arch = ini_getval_str(ini, "conda", "installer_arch", render_mode, &err); + ctx->conda.installer_baseurl = ini_getval_str(ini, "conda", "installer_baseurl", render_mode, &err); + ctx->conda.conda_packages = ini_getval_strlist(ini, "conda", "conda_packages", " "LINE_SEP, render_mode, &err); + + if (ctx->conda.conda_packages->data && ctx->conda.conda_packages->data[0] && strpbrk(ctx->conda.conda_packages->data[0], " \t")) { + normalize_space(ctx->conda.conda_packages->data[0]); + replace_text(ctx->conda.conda_packages->data[0], " ", LINE_SEP, 0); + char *pip_packages_replacement = join(ctx->conda.conda_packages->data, LINE_SEP); + ini_setval(&ini, INI_SETVAL_REPLACE, "conda", "conda_packages", pip_packages_replacement); + guard_free(pip_packages_replacement); + guard_strlist_free(&ctx->conda.conda_packages); + ctx->conda.conda_packages = ini_getval_strlist(ini, "conda", "conda_packages", LINE_SEP, render_mode, &err); + } + + for (size_t i = 0; i < strlist_count(ctx->conda.conda_packages); i++) { + char *pkg = strlist_item(ctx->conda.conda_packages, i); + if (strpbrk(pkg, ";#") || isempty(pkg)) { + strlist_remove(ctx->conda.conda_packages, i); + } + } + + ctx->conda.pip_packages = ini_getval_strlist(ini, "conda", "pip_packages", LINE_SEP, render_mode, &err); + if (ctx->conda.pip_packages->data && ctx->conda.pip_packages->data[0] && strpbrk(ctx->conda.pip_packages->data[0], " \t")) { + normalize_space(ctx->conda.pip_packages->data[0]); + replace_text(ctx->conda.pip_packages->data[0], " ", LINE_SEP, 0); + char *pip_packages_replacement = join(ctx->conda.pip_packages->data, LINE_SEP); + ini_setval(&ini, INI_SETVAL_REPLACE, "conda", "pip_packages", pip_packages_replacement); + guard_free(pip_packages_replacement); + guard_strlist_free(&ctx->conda.pip_packages); + ctx->conda.pip_packages = ini_getval_strlist(ini, "conda", "pip_packages", LINE_SEP, render_mode, &err); + } + + for (size_t i = 0; i < strlist_count(ctx->conda.pip_packages); i++) { + char *pkg = strlist_item(ctx->conda.pip_packages, i); + if (strpbrk(pkg, ";#") || isempty(pkg)) { + strlist_remove(ctx->conda.pip_packages, i); + } + } + + // Delivery metadata consumed + populate_mission_ini(&ctx, render_mode); + + if (ctx->info.release_name) { + guard_free(ctx->info.release_name); + guard_free(ctx->info.build_name); + guard_free(ctx->info.build_number); + } + + if (delivery_format_str(ctx, &ctx->info.release_name, ctx->rules.release_fmt)) { + fprintf(stderr, "Failed to generate release name. Format used: %s\n", ctx->rules.release_fmt); + return -1; + } + + if (!ctx->info.build_name) { + delivery_format_str(ctx, &ctx->info.build_name, ctx->rules.build_name_fmt); + } + if (!ctx->info.build_number) { + delivery_format_str(ctx, &ctx->info.build_number, ctx->rules.build_number_fmt); + } + + // Best I can do to make output directories unique. Annoying. + delivery_init_dirs_stage2(ctx); + + if (!ctx->conda.conda_packages_defer) { + ctx->conda.conda_packages_defer = strlist_init(); + } + if (!ctx->conda.pip_packages_defer) { + ctx->conda.pip_packages_defer = strlist_init(); + } + + for (size_t z = 0, i = 0; i < ini->section_count; i++) { + char *section_name = ini->section[i]->key; + if (startswith(section_name, "test:")) { + struct Test *test = &ctx->tests[z]; + val.as_char_p = strchr(ini->section[i]->key, ':') + 1; + if (val.as_char_p && isempty(val.as_char_p)) { + return 1; + } + conv_str(&test->name, val); + + test->version = ini_getval_str(ini, section_name, "version", render_mode, &err); + test->repository = ini_getval_str(ini, section_name, "repository", render_mode, &err); + test->script_setup = ini_getval_str(ini, section_name, "script_setup", INI_READ_RAW, &err); + test->script = ini_getval_str(ini, section_name, "script", INI_READ_RAW, &err); + test->disable = ini_getval_bool(ini, section_name, "disable", render_mode, &err); + test->parallel = ini_getval_bool(ini, section_name, "parallel", render_mode, &err); + if (err) { + test->parallel = true; + } + test->repository_remove_tags = ini_getval_strlist(ini, section_name, "repository_remove_tags", LINE_SEP, render_mode, &err); + test->build_recipe = ini_getval_str(ini, section_name, "build_recipe", render_mode, &err); + test->runtime.environ = ini_getval_strlist(ini, section_name, "runtime", LINE_SEP, render_mode, &err); + z++; + } + } + + for (size_t z = 0, i = 0; i < ini->section_count; i++) { + char *section_name = ini->section[i]->key; + struct Deploy *deploy = &ctx->deploy; + if (startswith(section_name, "deploy:artifactory")) { + struct JFrog *jfrog = &deploy->jfrog[z]; + // Artifactory base configuration + + jfrog->upload_ctx.workaround_parent_only = ini_getval_bool(ini, section_name, "workaround_parent_only", render_mode, &err); + jfrog->upload_ctx.exclusions = ini_getval_str(ini, section_name, "exclusions", render_mode, &err); + jfrog->upload_ctx.explode = ini_getval_bool(ini, section_name, "explode", render_mode, &err); + jfrog->upload_ctx.recursive = ini_getval_bool(ini, section_name, "recursive", render_mode, &err); + jfrog->upload_ctx.retries = ini_getval_int(ini, section_name, "retries", render_mode, &err); + jfrog->upload_ctx.retry_wait_time = ini_getval_int(ini, section_name, "retry_wait_time", render_mode, &err); + jfrog->upload_ctx.detailed_summary = ini_getval_bool(ini, section_name, "detailed_summary", render_mode, &err); + jfrog->upload_ctx.quiet = ini_getval_bool(ini, section_name, "quiet", render_mode, &err); + jfrog->upload_ctx.regexp = ini_getval_bool(ini, section_name, "regexp", render_mode, &err); + jfrog->upload_ctx.spec = ini_getval_str(ini, section_name, "spec", render_mode, &err); + jfrog->upload_ctx.flat = ini_getval_bool(ini, section_name, "flat", render_mode, &err); + jfrog->repo = ini_getval_str(ini, section_name, "repo", render_mode, &err); + jfrog->dest = ini_getval_str(ini, section_name, "dest", render_mode, &err); + jfrog->files = ini_getval_strlist(ini, section_name, "files", LINE_SEP, render_mode, &err); + z++; + } + } + + for (size_t i = 0; i < ini->section_count; i++) { + char *section_name = ini->section[i]->key; + struct Deploy *deploy = &ctx->deploy; + if (startswith(ini->section[i]->key, "deploy:docker")) { + struct Docker *docker = &deploy->docker; + + docker->registry = ini_getval_str(ini, section_name, "registry", render_mode, &err); + docker->image_compression = ini_getval_str(ini, section_name, "image_compression", render_mode, &err); + docker->test_script = ini_getval_str(ini, section_name, "test_script", render_mode, &err); + docker->build_args = ini_getval_strlist(ini, section_name, "build_args", LINE_SEP, render_mode, &err); + docker->tags = ini_getval_strlist(ini, section_name, "tags", LINE_SEP, render_mode, &err); + } + } + return 0; +} + +int populate_mission_ini(struct Delivery **ctx, int render_mode) { + int err = 0; + struct INIFILE *ini; + + if ((*ctx)->_stasis_ini_fp.mission) { + return 0; + } + + // Now populate the rules + char missionfile[PATH_MAX] = {0}; + if (getenv("STASIS_SYSCONFDIR")) { + sprintf(missionfile, "%s/%s/%s/%s.ini", + getenv("STASIS_SYSCONFDIR"), "mission", (*ctx)->meta.mission, (*ctx)->meta.mission); + } else { + sprintf(missionfile, "%s/%s/%s/%s.ini", + globals.sysconfdir, "mission", (*ctx)->meta.mission, (*ctx)->meta.mission); + } + + msg(STASIS_MSG_L2, "Reading mission configuration: %s\n", missionfile); + (*ctx)->_stasis_ini_fp.mission = ini_open(missionfile); + ini = (*ctx)->_stasis_ini_fp.mission; + if (!ini) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read mission configuration: %s, %s\n", missionfile, strerror(errno)); + exit(1); + } + (*ctx)->_stasis_ini_fp.mission_path = strdup(missionfile); + + (*ctx)->rules.release_fmt = ini_getval_str(ini, "meta", "release_fmt", render_mode, &err); + + // Used for setting artifactory build info + (*ctx)->rules.build_name_fmt = ini_getval_str(ini, "meta", "build_name_fmt", render_mode, &err); + + // Used for setting artifactory build info + (*ctx)->rules.build_number_fmt = ini_getval_str(ini, "meta", "build_number_fmt", render_mode, &err); + return 0; +} + +void validate_delivery_ini(struct INIFILE *ini) { + if (!ini) { + SYSERROR("%s", "INIFILE is NULL!"); + exit(1); + } + if (ini_section_search(&ini, INI_SEARCH_EXACT, "meta")) { + ini_has_key_required(ini, "meta", "name"); + ini_has_key_required(ini, "meta", "version"); + ini_has_key_required(ini, "meta", "rc"); + ini_has_key_required(ini, "meta", "mission"); + ini_has_key_required(ini, "meta", "python"); + } else { + SYSERROR("%s", "[meta] configuration section is required"); + exit(1); + } + + if (ini_section_search(&ini, INI_SEARCH_EXACT, "conda")) { + ini_has_key_required(ini, "conda", "installer_name"); + ini_has_key_required(ini, "conda", "installer_version"); + ini_has_key_required(ini, "conda", "installer_platform"); + ini_has_key_required(ini, "conda", "installer_arch"); + } else { + SYSERROR("%s", "[conda] configuration section is required"); + exit(1); + } + + for (size_t i = 0; i < ini->section_count; i++) { + struct INISection *section = ini->section[i]; + if (section && startswith(section->key, "test:")) { + char *name = strstr(section->key, ":"); + if (name && strlen(name) > 1) { + name = &name[1]; + } + //ini_has_key_required(ini, section->key, "version"); + //ini_has_key_required(ini, section->key, "repository"); + if (globals.enable_testing) { + ini_has_key_required(ini, section->key, "script"); + } + } + } + + if (ini_section_search(&ini, INI_SEARCH_EXACT, "deploy:docker")) { + // yeah? + } + + for (size_t i = 0; i < ini->section_count; i++) { + struct INISection *section = ini->section[i]; + if (section && startswith(section->key, "deploy:artifactory")) { + ini_has_key_required(ini, section->key, "files"); + ini_has_key_required(ini, section->key, "dest"); + } + } +} + diff --git a/src/lib/core/delivery_postprocess.c b/src/lib/core/delivery_postprocess.c new file mode 100644 index 0000000..1a902e3 --- /dev/null +++ b/src/lib/core/delivery_postprocess.c @@ -0,0 +1,266 @@ +#include "delivery.h" + + +const char *release_header = "# delivery_name: %s\n" + "# delivery_fmt: %s\n" + "# creation_time: %s\n" + "# conda_ident: %s\n" + "# conda_build_ident: %s\n"; + +char *delivery_get_release_header(struct Delivery *ctx) { + char output[STASIS_BUFSIZ]; + char stamp[100]; + strftime(stamp, sizeof(stamp) - 1, "%c", ctx->info.time_info); + sprintf(output, release_header, + ctx->info.release_name, + ctx->rules.release_fmt, + stamp, + ctx->conda.tool_version, + ctx->conda.tool_build_version); + return strdup(output); +} + +int delivery_dump_metadata(struct Delivery *ctx) { + FILE *fp; + char filename[PATH_MAX]; + sprintf(filename, "%s/meta-%s.stasis", ctx->storage.meta_dir, ctx->info.release_name); + fp = fopen(filename, "w+"); + if (!fp) { + return -1; + } + if (globals.verbose) { + printf("%s\n", filename); + } + fprintf(fp, "name %s\n", ctx->meta.name); + fprintf(fp, "version %s\n", ctx->meta.version); + fprintf(fp, "rc %d\n", ctx->meta.rc); + fprintf(fp, "python %s\n", ctx->meta.python); + fprintf(fp, "python_compact %s\n", ctx->meta.python_compact); + fprintf(fp, "mission %s\n", ctx->meta.mission); + fprintf(fp, "codename %s\n", ctx->meta.codename ? ctx->meta.codename : ""); + fprintf(fp, "platform %s %s %s %s\n", + ctx->system.platform[DELIVERY_PLATFORM], + ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], + ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], + ctx->system.platform[DELIVERY_PLATFORM_RELEASE]); + fprintf(fp, "arch %s\n", ctx->system.arch); + fprintf(fp, "time %s\n", ctx->info.time_str_epoch); + fprintf(fp, "release_fmt %s\n", ctx->rules.release_fmt); + fprintf(fp, "release_name %s\n", ctx->info.release_name); + fprintf(fp, "build_name_fmt %s\n", ctx->rules.build_name_fmt); + fprintf(fp, "build_name %s\n", ctx->info.build_name); + fprintf(fp, "build_number_fmt %s\n", ctx->rules.build_number_fmt); + fprintf(fp, "build_number %s\n", ctx->info.build_number); + fprintf(fp, "conda_installer_baseurl %s\n", ctx->conda.installer_baseurl); + fprintf(fp, "conda_installer_name %s\n", ctx->conda.installer_name); + fprintf(fp, "conda_installer_version %s\n", ctx->conda.installer_version); + fprintf(fp, "conda_installer_platform %s\n", ctx->conda.installer_platform); + fprintf(fp, "conda_installer_arch %s\n", ctx->conda.installer_arch); + + fclose(fp); + return 0; +} + +void delivery_rewrite_spec(struct Delivery *ctx, char *filename, unsigned stage) { + char output[PATH_MAX]; + char *header = NULL; + char *tempfile = NULL; + FILE *tp = NULL; + + if (stage == DELIVERY_REWRITE_SPEC_STAGE_1) { + header = delivery_get_release_header(ctx); + if (!header) { + msg(STASIS_MSG_ERROR, "failed to generate release header string\n", filename); + exit(1); + } + tempfile = xmkstemp(&tp, "w+"); + if (!tempfile || !tp) { + msg(STASIS_MSG_ERROR, "%s: unable to create temporary file\n", strerror(errno)); + exit(1); + } + fprintf(tp, "%s", header); + + // Read the original file + char **contents = file_readlines(filename, 0, 0, NULL); + if (!contents) { + msg(STASIS_MSG_ERROR, "%s: unable to read %s", filename); + exit(1); + } + + // Write temporary data + for (size_t i = 0; contents[i] != NULL; i++) { + if (startswith(contents[i], "channels:")) { + // Allow for additional conda channel injection + if (ctx->conda.conda_packages_defer && strlist_count(ctx->conda.conda_packages_defer)) { + fprintf(tp, "%s - @CONDA_CHANNEL@\n", contents[i]); + continue; + } + } else if (strstr(contents[i], "- pip:")) { + if (ctx->conda.pip_packages_defer && strlist_count(ctx->conda.pip_packages_defer)) { + // Allow for additional pip argument injection + fprintf(tp, "%s - @PIP_ARGUMENTS@\n", contents[i]); + continue; + } + } else if (startswith(contents[i], "prefix:")) { + // Remove the prefix key + if (strstr(contents[i], "/") || strstr(contents[i], "\\")) { + // path is on the same line as the key + continue; + } else { + // path is on the next line? + if (contents[i + 1] && (strstr(contents[i + 1], "/") || strstr(contents[i + 1], "\\"))) { + i++; + } + continue; + } + } + fprintf(tp, "%s", contents[i]); + } + GENERIC_ARRAY_FREE(contents); + guard_free(header); + fflush(tp); + fclose(tp); + + // Replace the original file with our temporary data + if (copy2(tempfile, filename, CT_PERM) < 0) { + fprintf(stderr, "%s: could not rename '%s' to '%s'\n", strerror(errno), tempfile, filename); + exit(1); + } + remove(tempfile); + guard_free(tempfile); + } else if (globals.enable_rewrite_spec_stage_2 && stage == DELIVERY_REWRITE_SPEC_STAGE_2) { + // Replace "local" channel with the staging URL + if (ctx->storage.conda_staging_url) { + file_replace_text(filename, "@CONDA_CHANNEL@", ctx->storage.conda_staging_url, 0); + } else if (globals.jfrog.repo) { + sprintf(output, "%s/%s/%s/%s/packages/conda", globals.jfrog.url, globals.jfrog.repo, ctx->meta.mission, ctx->info.build_name); + file_replace_text(filename, "@CONDA_CHANNEL@", output, 0); + } else { + msg(STASIS_MSG_WARN, "conda_staging_dir is not configured. Using fallback: '%s'\n", ctx->storage.conda_artifact_dir); + file_replace_text(filename, "@CONDA_CHANNEL@", ctx->storage.conda_artifact_dir, 0); + } + + if (ctx->storage.wheel_staging_url) { + file_replace_text(filename, "@PIP_ARGUMENTS@", ctx->storage.wheel_staging_url, 0); + } else if (globals.enable_artifactory && globals.jfrog.url && globals.jfrog.repo) { + sprintf(output, "--extra-index-url %s/%s/%s/%s/packages/wheels", globals.jfrog.url, globals.jfrog.repo, ctx->meta.mission, ctx->info.build_name); + file_replace_text(filename, "@PIP_ARGUMENTS@", output, 0); + } else { + msg(STASIS_MSG_WARN, "wheel_staging_dir is not configured. Using fallback: '%s'\n", ctx->storage.wheel_artifact_dir); + sprintf(output, "--extra-index-url file://%s", ctx->storage.wheel_artifact_dir); + file_replace_text(filename, "@PIP_ARGUMENTS@", output, 0); + } + } +} + +int delivery_copy_conda_artifacts(struct Delivery *ctx) { + char cmd[STASIS_BUFSIZ]; + char conda_build_dir[PATH_MAX]; + char subdir[PATH_MAX]; + memset(cmd, 0, sizeof(cmd)); + memset(conda_build_dir, 0, sizeof(conda_build_dir)); + memset(subdir, 0, sizeof(subdir)); + + sprintf(conda_build_dir, "%s/%s", ctx->storage.conda_install_prefix, "conda-bld"); + // One must run conda build at least once to create the "conda-bld" directory. + // When this directory is missing there can be no build artifacts. + if (access(conda_build_dir, F_OK) < 0) { + msg(STASIS_MSG_RESTRICT | STASIS_MSG_WARN | STASIS_MSG_L3, + "Skipped: 'conda build' has never been executed.\n"); + return 0; + } + + snprintf(cmd, sizeof(cmd) - 1, "rsync -avi --progress %s/%s %s", + conda_build_dir, + ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], + ctx->storage.conda_artifact_dir); + + return system(cmd); +} + +int delivery_index_conda_artifacts(struct Delivery *ctx) { + return conda_index(ctx->storage.conda_artifact_dir); +} + +int delivery_copy_wheel_artifacts(struct Delivery *ctx) { + char cmd[PATH_MAX]; + memset(cmd, 0, sizeof(cmd)); + snprintf(cmd, sizeof(cmd) - 1, "rsync -avi --progress %s/*/dist/*.whl %s", + ctx->storage.build_sources_dir, + ctx->storage.wheel_artifact_dir); + return system(cmd); +} + +int delivery_index_wheel_artifacts(struct Delivery *ctx) { + struct dirent *rec; + DIR *dp; + FILE *top_fp; + + dp = opendir(ctx->storage.wheel_artifact_dir); + if (!dp) { + return -1; + } + + // Generate a "dumb" local pypi index that is compatible with: + // pip install --extra-index-url + char top_index[PATH_MAX]; + memset(top_index, 0, sizeof(top_index)); + sprintf(top_index, "%s/index.html", ctx->storage.wheel_artifact_dir); + top_fp = fopen(top_index, "w+"); + if (!top_fp) { + closedir(dp); + return -2; + } + + while ((rec = readdir(dp)) != NULL) { + // skip directories + if (DT_REG == rec->d_type || !strcmp(rec->d_name, "..") || !strcmp(rec->d_name, ".")) { + continue; + } + + FILE *bottom_fp; + char bottom_index[PATH_MAX * 2]; + memset(bottom_index, 0, sizeof(bottom_index)); + sprintf(bottom_index, "%s/%s/index.html", ctx->storage.wheel_artifact_dir, rec->d_name); + bottom_fp = fopen(bottom_index, "w+"); + if (!bottom_fp) { + closedir(dp); + return -3; + } + + if (globals.verbose) { + printf("+ %s\n", rec->d_name); + } + // Add record to top level index + fprintf(top_fp, "<a href=\"%s/\">%s</a><br/>\n", rec->d_name, rec->d_name); + + char dpath[PATH_MAX * 2]; + memset(dpath, 0, sizeof(dpath)); + sprintf(dpath, "%s/%s", ctx->storage.wheel_artifact_dir, rec->d_name); + struct StrList *packages = listdir(dpath); + if (!packages) { + closedir(dp); + fclose(top_fp); + fclose(bottom_fp); + return -4; + } + + for (size_t i = 0; i < strlist_count(packages); i++) { + char *package = strlist_item(packages, i); + if (!endswith(package, ".whl")) { + continue; + } + if (globals.verbose) { + printf("`- %s\n", package); + } + // Write record to bottom level index + fprintf(bottom_fp, "<a href=\"%s\">%s</a><br/>\n", package, package); + } + fclose(bottom_fp); + + guard_strlist_free(&packages); + } + closedir(dp); + fclose(top_fp); + return 0; +} diff --git a/src/lib/core/delivery_show.c b/src/lib/core/delivery_show.c new file mode 100644 index 0000000..adfa1be --- /dev/null +++ b/src/lib/core/delivery_show.c @@ -0,0 +1,117 @@ +#include "delivery.h" + +void delivery_debug_show(struct Delivery *ctx) { + printf("\n====DEBUG====\n"); + printf("%-20s %-10s\n", "System configuration directory:", globals.sysconfdir); + printf("%-20s %-10s\n", "Mission directory:", ctx->storage.mission_dir); + printf("%-20s %-10s\n", "Testing enabled:", globals.enable_testing ? "Yes" : "No"); + printf("%-20s %-10s\n", "Docker image builds enabled:", globals.enable_docker ? "Yes" : "No"); + printf("%-20s %-10s\n", "Artifact uploading enabled:", globals.enable_artifactory ? "Yes" : "No"); +} + +void delivery_meta_show(struct Delivery *ctx) { + if (globals.verbose) { + delivery_debug_show(ctx); + } + + printf("\n====DELIVERY====\n"); + printf("%-20s %-10s\n", "Target Python:", ctx->meta.python); + printf("%-20s %-10s\n", "Name:", ctx->meta.name); + printf("%-20s %-10s\n", "Mission:", ctx->meta.mission); + if (ctx->meta.codename) { + printf("%-20s %-10s\n", "Codename:", ctx->meta.codename); + } + if (ctx->meta.version) { + printf("%-20s %-10s\n", "Version", ctx->meta.version); + } + if (!ctx->meta.final) { + printf("%-20s %-10d\n", "RC Level:", ctx->meta.rc); + } + printf("%-20s %-10s\n", "Final Release:", ctx->meta.final ? "Yes" : "No"); + printf("%-20s %-10s\n", "Based On:", ctx->meta.based_on ? ctx->meta.based_on : "New"); +} + +void delivery_conda_show(struct Delivery *ctx) { + printf("\n====CONDA====\n"); + printf("%-20s %-10s\n", "Prefix:", ctx->storage.conda_install_prefix); + + puts("Native Packages:"); + if (strlist_count(ctx->conda.conda_packages) || strlist_count(ctx->conda.conda_packages_defer)) { + struct StrList *list_conda = strlist_init(); + if (strlist_count(ctx->conda.conda_packages)) { + strlist_append_strlist(list_conda, ctx->conda.conda_packages); + } + if (strlist_count(ctx->conda.conda_packages_defer)) { + strlist_append_strlist(list_conda, ctx->conda.conda_packages_defer); + } + strlist_sort(list_conda, STASIS_SORT_ALPHA); + + for (size_t i = 0; i < strlist_count(list_conda); i++) { + char *token = strlist_item(list_conda, i); + if (isempty(token) || isblank(*token) || startswith(token, "-")) { + continue; + } + printf("%21s%s\n", "", token); + } + guard_strlist_free(&list_conda); + } else { + printf("%21s%s\n", "", "N/A"); + } + + puts("Python Packages:"); + if (strlist_count(ctx->conda.pip_packages) || strlist_count(ctx->conda.pip_packages_defer)) { + struct StrList *list_python = strlist_init(); + if (strlist_count(ctx->conda.pip_packages)) { + strlist_append_strlist(list_python, ctx->conda.pip_packages); + } + if (strlist_count(ctx->conda.pip_packages_defer)) { + strlist_append_strlist(list_python, ctx->conda.pip_packages_defer); + } + strlist_sort(list_python, STASIS_SORT_ALPHA); + + for (size_t i = 0; i < strlist_count(list_python); i++) { + char *token = strlist_item(list_python, i); + if (isempty(token) || isblank(*token) || startswith(token, "-")) { + continue; + } + printf("%21s%s\n", "", token); + } + guard_strlist_free(&list_python); + } else { + printf("%21s%s\n", "", "N/A"); + } +} + +void delivery_tests_show(struct Delivery *ctx) { + printf("\n====TESTS====\n"); + for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { + if (!ctx->tests[i].name) { + continue; + } + printf("%-20s %-20s %s\n", ctx->tests[i].name, + ctx->tests[i].version, + ctx->tests[i].repository); + } +} + +void delivery_runtime_show(struct Delivery *ctx) { + printf("\n====RUNTIME====\n"); + struct StrList *rt = NULL; + rt = strlist_copy(ctx->runtime.environ); + if (!rt) { + // no data + return; + } + strlist_sort(rt, STASIS_SORT_ALPHA); + size_t total = strlist_count(rt); + for (size_t i = 0; i < total; i++) { + char *item = strlist_item(rt, i); + if (!item) { + // not supposed to occur + msg(STASIS_MSG_WARN | STASIS_MSG_L1, "Encountered unexpected NULL at record %zu of %zu of runtime array.\n", i); + return; + } + printf("%s\n", item); + } +} + diff --git a/src/lib/core/delivery_test.c b/src/lib/core/delivery_test.c new file mode 100644 index 0000000..cb78f64 --- /dev/null +++ b/src/lib/core/delivery_test.c @@ -0,0 +1,295 @@ +#include "delivery.h" + +void delivery_tests_run(struct Delivery *ctx) { + static const int SETUP = 0; + static const int PARALLEL = 1; + static const int SERIAL = 2; + struct MultiProcessingPool *pool[3]; + struct Process proc; + memset(&proc, 0, sizeof(proc)); + + if (!globals.workaround.conda_reactivate) { + globals.workaround.conda_reactivate = calloc(PATH_MAX, sizeof(*globals.workaround.conda_reactivate)); + } else { + memset(globals.workaround.conda_reactivate, 0, PATH_MAX); + } + // Test blocks always run with xtrace enabled. Disable, and reenable it. Conda's wrappers produce an incredible + // amount of debug information. + snprintf(globals.workaround.conda_reactivate, PATH_MAX - 1, "\nset +x; mamba activate ${CONDA_DEFAULT_ENV}; set -x\n"); + + if (!ctx->tests[0].name) { + msg(STASIS_MSG_WARN | STASIS_MSG_L2, "no tests are defined!\n"); + } else { + pool[PARALLEL] = mp_pool_init("parallel", ctx->storage.tmpdir); + if (!pool[PARALLEL]) { + perror("mp_pool_init/parallel"); + exit(1); + } + pool[PARALLEL]->status_interval = globals.pool_status_interval; + + pool[SERIAL] = mp_pool_init("serial", ctx->storage.tmpdir); + if (!pool[SERIAL]) { + perror("mp_pool_init/serial"); + exit(1); + } + pool[SERIAL]->status_interval = globals.pool_status_interval; + + pool[SETUP] = mp_pool_init("setup", ctx->storage.tmpdir); + if (!pool[SETUP]) { + perror("mp_pool_init/setup"); + exit(1); + } + pool[SETUP]->status_interval = globals.pool_status_interval; + + // Test block scripts shall exit non-zero on error. + // This will fail a test block immediately if "string" is not found in file.txt: + // grep string file.txt + // + // And this is how to avoid that scenario: + // #1: + // if ! grep string file.txt; then + // # handle error + // fi + // + // #2: + // grep string file.txt || handle error + // + // #3: + // # Use ':' as a NO-OP if/when the result doesn't matter + // grep string file.txt || : + const char *runner_cmd_fmt = "set -e -x\n%s\n"; + + // Iterate over our test records, retrieving the source code for each package, and assigning its scripted tasks + // to the appropriate processing pool + for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { + struct Test *test = &ctx->tests[i]; + if (!test->name && !test->repository && !test->script) { + // skip unused test records + continue; + } + msg(STASIS_MSG_L2, "Loading tests for %s %s\n", test->name, test->version); + if (!test->script || !strlen(test->script)) { + msg(STASIS_MSG_WARN | STASIS_MSG_L3, "Nothing to do. To fix, declare a 'script' in section: [test:%s]\n", + test->name); + continue; + } + + char destdir[PATH_MAX]; + sprintf(destdir, "%s/%s", ctx->storage.build_sources_dir, path_basename(test->repository)); + + if (!access(destdir, F_OK)) { + msg(STASIS_MSG_L3, "Purging repository %s\n", destdir); + if (rmtree(destdir)) { + COE_CHECK_ABORT(1, "Unable to remove repository\n"); + } + } + msg(STASIS_MSG_L3, "Cloning repository %s\n", test->repository); + if (!git_clone(&proc, test->repository, destdir, test->version)) { + test->repository_info_tag = strdup(git_describe(destdir)); + test->repository_info_ref = strdup(git_rev_parse(destdir, "HEAD")); + } else { + COE_CHECK_ABORT(1, "Unable to clone repository\n"); + } + + if (test->repository_remove_tags && strlist_count(test->repository_remove_tags)) { + filter_repo_tags(destdir, test->repository_remove_tags); + } + + if (pushd(destdir)) { + COE_CHECK_ABORT(1, "Unable to enter repository directory\n"); + } else { + char *cmd = calloc(strlen(test->script) + STASIS_BUFSIZ, sizeof(*cmd)); + if (!cmd) { + SYSERROR("Unable to allocate test script buffer: %s", strerror(errno)); + exit(1); + } + + msg(STASIS_MSG_L3, "Queuing task for %s\n", test->name); + memset(&proc, 0, sizeof(proc)); + + strcpy(cmd, test->script); + char *cmd_rendered = tpl_render(cmd); + if (cmd_rendered) { + if (strcmp(cmd_rendered, cmd) != 0) { + strcpy(cmd, cmd_rendered); + cmd[strlen(cmd_rendered) ? strlen(cmd_rendered) - 1 : 0] = 0; + } + guard_free(cmd_rendered); + } else { + SYSERROR("An error occurred while rendering the following:\n%s", cmd); + exit(1); + } + + if (test->disable) { + msg(STASIS_MSG_L2, "Script execution disabled by configuration\n", test->name); + guard_free(cmd); + continue; + } + + char *runner_cmd = NULL; + char pool_name[100] = "parallel"; + struct MultiProcessingTask *task = NULL; + int selected = PARALLEL; + if (!globals.enable_parallel || !test->parallel) { + selected = SERIAL; + memset(pool_name, 0, sizeof(pool_name)); + strcpy(pool_name, "serial"); + } + + if (asprintf(&runner_cmd, runner_cmd_fmt, cmd) < 0) { + SYSERROR("Unable to allocate memory for runner command: %s", strerror(errno)); + exit(1); + } + task = mp_pool_task(pool[selected], test->name, destdir, runner_cmd); + if (!task) { + SYSERROR("Failed to add task to %s pool: %s", pool_name, runner_cmd); + popd(); + if (!globals.continue_on_error) { + guard_free(runner_cmd); + tpl_free(); + delivery_free(ctx); + globals_free(); + } + exit(1); + } + guard_free(runner_cmd); + guard_free(cmd); + popd(); + + } + } + + // Configure "script_setup" tasks + // Directories should exist now, so no need to go through initializing everything all over again. + for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { + struct Test *test = &ctx->tests[i]; + if (test->script_setup) { + char destdir[PATH_MAX]; + sprintf(destdir, "%s/%s", ctx->storage.build_sources_dir, path_basename(test->repository)); + if (access(destdir, F_OK)) { + SYSERROR("%s: %s", destdir, strerror(errno)); + exit(1); + } + if (!pushd(destdir)) { + const size_t cmd_len = strlen(test->script_setup) + STASIS_BUFSIZ; + char *cmd = calloc(cmd_len, sizeof(*cmd)); + if (!cmd) { + SYSERROR("Unable to allocate test script_setup buffer: %s", strerror(errno)); + exit(1); + } + + strncpy(cmd, test->script_setup, cmd_len - 1); + char *cmd_rendered = tpl_render(cmd); + if (cmd_rendered) { + if (strcmp(cmd_rendered, cmd) != 0) { + strncpy(cmd, cmd_rendered, cmd_len - 1); + cmd[strlen(cmd_rendered) ? strlen(cmd_rendered) - 1 : 0] = 0; + } + guard_free(cmd_rendered); + } else { + SYSERROR("An error occurred while rendering the following:\n%s", cmd); + exit(1); + } + + struct MultiProcessingTask *task = NULL; + char *runner_cmd = NULL; + if (asprintf(&runner_cmd, runner_cmd_fmt, cmd) < 0) { + SYSERROR("Unable to allocate memory for runner command: %s", strerror(errno)); + exit(1); + } + + task = mp_pool_task(pool[SETUP], test->name, destdir, runner_cmd); + if (!task) { + SYSERROR("Failed to add task %s to setup pool: %s", test->name, runner_cmd); + popd(); + if (!globals.continue_on_error) { + guard_free(runner_cmd); + tpl_free(); + delivery_free(ctx); + globals_free(); + } + exit(1); + } + guard_free(runner_cmd); + guard_free(cmd); + popd(); + } else { + SYSERROR("Failed to change directory: %s\n", destdir); + exit(1); + } + } + } + + size_t opt_flags = 0; + if (globals.parallel_fail_fast) { + opt_flags |= MP_POOL_FAIL_FAST; + } + + // Execute all queued tasks + for (size_t p = 0; p < sizeof(pool) / sizeof(*pool); p++) { + int pool_status; + long jobs = globals.cpu_limit; + + if (!pool[p]->num_used) { + // Skip empty pool + continue; + } + + // Setup tasks run sequentially + if (p == (size_t) SETUP || p == (size_t) SERIAL) { + jobs = 1; + } + + // Run tasks in the pool + // 1. Setup (builds) + // 2. Parallel (fast jobs) + // 3. Serial (long jobs) + pool_status = mp_pool_join(pool[p], jobs, opt_flags); + + // On error show a summary of the current pool, and die + if (pool_status != 0) { + mp_pool_show_summary(pool[p]); + COE_CHECK_ABORT(true, "Task failure"); + } + } + + // All tasks were successful + for (size_t p = 0; p < sizeof(pool) / sizeof(*pool); p++) { + if (pool[p]->num_used) { + // Only show pools that actually had jobs to run + mp_pool_show_summary(pool[p]); + } + mp_pool_free(&pool[p]); + } + } +} + +int delivery_fixup_test_results(struct Delivery *ctx) { + struct dirent *rec; + DIR *dp; + + dp = opendir(ctx->storage.results_dir); + if (!dp) { + perror(ctx->storage.results_dir); + return -1; + } + + while ((rec = readdir(dp)) != NULL) { + char path[PATH_MAX]; + memset(path, 0, sizeof(path)); + + if (!strcmp(rec->d_name, ".") || !strcmp(rec->d_name, "..") || !endswith(rec->d_name, ".xml")) { + continue; + } + + sprintf(path, "%s/%s", ctx->storage.results_dir, rec->d_name); + msg(STASIS_MSG_L3, "%s\n", rec->d_name); + if (xml_pretty_print_in_place(path, STASIS_XML_PRETTY_PRINT_PROG, STASIS_XML_PRETTY_PRINT_ARGS)) { + msg(STASIS_MSG_L3 | STASIS_MSG_WARN, "Failed to rewrite file '%s'\n", rec->d_name); + } + } + + closedir(dp); + return 0; +} + diff --git a/src/lib/core/docker.c b/src/lib/core/docker.c new file mode 100644 index 0000000..5834ef9 --- /dev/null +++ b/src/lib/core/docker.c @@ -0,0 +1,204 @@ +#include "docker.h" + + +int docker_exec(const char *args, unsigned flags) { + struct Process proc; + char cmd[PATH_MAX]; + + memset(&proc, 0, sizeof(proc)); + memset(cmd, 0, sizeof(cmd)); + snprintf(cmd, sizeof(cmd) - 1, "docker %s", args); + if (flags & STASIS_DOCKER_QUIET) { + strcpy(proc.f_stdout, "/dev/null"); + strcpy(proc.f_stderr, "/dev/null"); + } else { + msg(STASIS_MSG_L2, "Executing: %s\n", cmd); + } + + shell(&proc, cmd); + return proc.returncode; +} + +int docker_script(const char *image, char *data, unsigned flags) { + (void)flags; // TODO: placeholder + FILE *infile; + FILE *outfile; + char cmd[PATH_MAX]; + char buffer[STASIS_BUFSIZ]; + + memset(cmd, 0, sizeof(cmd)); + snprintf(cmd, sizeof(cmd) - 1, "docker run --rm -i %s /bin/sh -", image); + + outfile = popen(cmd, "w"); + if (!outfile) { + // opening command pipe for writing failed + return -1; + } + + infile = fmemopen(data, strlen(data), "r"); + if (!infile) { + // opening memory file for reading failed + return -1; + } + + do { + memset(buffer, 0, sizeof(buffer)); + if (fgets(buffer, sizeof(buffer) - 1, infile) != NULL) { + fputs(buffer, outfile); + } + } while (!feof(infile)); + + fclose(infile); + return pclose(outfile); +} + +int docker_build(const char *dirpath, const char *args, int engine) { + char cmd[PATH_MAX]; + char build[15]; + + memset(build, 0, sizeof(build)); + memset(cmd, 0, sizeof(cmd)); + + if (engine & STASIS_DOCKER_BUILD) { + strcpy(build, "build"); + } + if (engine & STASIS_DOCKER_BUILD_X) { + strcpy(build, "buildx build"); + } + snprintf(cmd, sizeof(cmd) - 1, "%s %s %s", build, args, dirpath); + return docker_exec(cmd, 0); +} + +int docker_save(const char *image, const char *destdir, const char *compression_program) { + char cmd[PATH_MAX]; + + memset(cmd, 0, sizeof(cmd)); + + if (compression_program && strlen(compression_program)) { + char ext[255]; + memset(ext, 0, sizeof(ext)); + if (startswith(compression_program, "zstd")) { + strcpy(ext, "zst"); + } else if (startswith(compression_program, "xz")) { + strcpy(ext, "xz"); + } else if (startswith(compression_program, "gzip")) { + strcpy(ext, "gz"); + } else if (startswith(compression_program, "bzip2")) { + strcpy(ext, "bz2"); + } else { + strncpy(ext, compression_program, sizeof(ext) - 1); + } + sprintf(cmd, "save \"%s\" | %s > \"%s/%s.tar.%s\"", image, compression_program, destdir, image, ext); + } else { + sprintf(cmd, "save \"%s\" -o \"%s/%s.tar\"", image, destdir, image); + + } + return docker_exec(cmd, 0); +} + +static int docker_exists() { + if (find_program("docker")) { + return true; + } + return false; +} + +static char *docker_ident() { + FILE *fp = NULL; + char *tempfile = NULL; + char line[PATH_MAX]; + struct Process proc; + + tempfile = xmkstemp(&fp, "w+"); + if (!fp || !tempfile) { + return NULL; + } + + memset(&proc, 0, sizeof(proc)); + strcpy(proc.f_stdout, tempfile); + strcpy(proc.f_stderr, "/dev/null"); + shell(&proc, "docker --version"); + + if (!freopen(tempfile, "r", fp)) { + remove(tempfile); + guard_free(tempfile); + return NULL; + } + + if (!fgets(line, sizeof(line) - 1, fp)) { + fclose(fp); + remove(tempfile); + guard_free(tempfile); + return NULL; + } + + fclose(fp); + remove(tempfile); + guard_free(tempfile); + + return strdup(line); +} + +int docker_capable(struct DockerCapabilities *result) { + char *version = NULL; + memset(result, 0, sizeof(*result)); + + if (!docker_exists()) { + // docker isn't available + return false; + } + result->available = true; + + if (docker_exec("ps", STASIS_DOCKER_QUIET)) { + // user cannot connect to the socket + return false; + } + + version = docker_ident(); + if (version && startswith(version, "podman")) { + result->podman = true; + } + guard_free(version); + + if (!docker_exec("buildx build --help", STASIS_DOCKER_QUIET)) { + result->build |= STASIS_DOCKER_BUILD_X; + } + if (!docker_exec("build --help", STASIS_DOCKER_QUIET)) { + result->build |= STASIS_DOCKER_BUILD; + } + if (!result->build) { + // can't use docker without a build plugin + return false; + } + result->usable = true; + return true; +} + +void docker_sanitize_tag(char *str) { + char *pos = str; + while (*pos != 0) { + if (!isalnum(*pos)) { + if (*pos != '.' && *pos != ':' && *pos != '/') { + *pos = '-'; + } + } + pos++; + } +} + +int docker_validate_compression_program(char *prog) { + int result = -1; + char **parts = NULL; + if (!prog) { + goto invalid; + } + parts = split(prog, " ", 1); + if (!parts) { + goto invalid; + } + result = find_program(parts[0]) ? 0 : -1; + + invalid: + GENERIC_ARRAY_FREE(parts); + return result; +} diff --git a/src/lib/core/download.c b/src/lib/core/download.c new file mode 100644 index 0000000..bfb323e --- /dev/null +++ b/src/lib/core/download.c @@ -0,0 +1,59 @@ +// +// Created by jhunk on 10/5/23. +// + +#include "download.h" + +size_t download_writer(void *fp, size_t size, size_t nmemb, void *stream) { + size_t bytes = fwrite(fp, size, nmemb, (FILE *) stream); + return bytes; +} + +long download(char *url, const char *filename, char **errmsg) { + extern char *VERSION; + CURL *c; + CURLcode curl_code; + long http_code = -1; + FILE *fp; + char user_agent[20]; + sprintf(user_agent, "stasis/%s", VERSION); + long timeout = 30L; + char *timeout_str = getenv("STASIS_DOWNLOAD_TIMEOUT"); + + curl_global_init(CURL_GLOBAL_ALL); + c = curl_easy_init(); + curl_easy_setopt(c, CURLOPT_URL, url); + curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, download_writer); + fp = fopen(filename, "wb"); + if (!fp) { + return -1; + } + + curl_easy_setopt(c, CURLOPT_VERBOSE, 0L); + curl_easy_setopt(c, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(c, CURLOPT_USERAGENT, user_agent); + curl_easy_setopt(c, CURLOPT_NOPROGRESS, 0L); + curl_easy_setopt(c, CURLOPT_WRITEDATA, fp); + + if (timeout_str) { + timeout = strtol(timeout_str, NULL, 10); + } + curl_easy_setopt(c, CURLOPT_CONNECTTIMEOUT, timeout); + + curl_code = curl_easy_perform(c); + if (curl_code != CURLE_OK) { + if (errmsg) { + strcpy(*errmsg, curl_easy_strerror(curl_code)); + } else { + fprintf(stderr, "\nCURL ERROR: %s\n", curl_easy_strerror(curl_code)); + } + goto failed; + } + curl_easy_getinfo(c, CURLINFO_RESPONSE_CODE, &http_code); + + failed: + fclose(fp); + curl_easy_cleanup(c); + curl_global_cleanup(); + return http_code; +}
\ No newline at end of file diff --git a/src/lib/core/envctl.c b/src/lib/core/envctl.c new file mode 100644 index 0000000..9037d9d --- /dev/null +++ b/src/lib/core/envctl.c @@ -0,0 +1,124 @@ +#include "envctl.h" + +struct EnvCtl *envctl_init() { + struct EnvCtl *result; + + result = calloc(1, sizeof(*result)); + if (!result) { + return NULL; + } + + result->num_alloc = STASIS_ENVCTL_DEFAULT_ALLOC; + result->item = calloc(result->num_alloc + 1, sizeof(result->item)); + if (!result->item) { + guard_free(result); + return NULL; + } + + return result; +} + +static int callback_builtin_nop(const void *a, const void *b) { + return STASIS_ENVCTL_RET_SUCCESS; +} + +int envctl_register(struct EnvCtl **envctl, unsigned flags, envctl_except_fn *callback, const char *name) { + if ((*envctl)->num_used == (*envctl)->num_alloc) { + (*envctl)->num_alloc += STASIS_ENVCTL_DEFAULT_ALLOC; + struct EnvCtl_Item **tmp = realloc((*envctl)->item, (*envctl)->num_alloc + 1 * sizeof((*envctl)->item)); + if (!tmp) { + return 1; + } else { + (*envctl)->item = tmp; + } + } + + struct EnvCtl_Item **item = (*envctl)->item; + item[(*envctl)->num_used] = calloc(1, sizeof(*item[0])); + if (!item[(*envctl)->num_used]) { + return 1; + } + if (!callback) { + callback = &callback_builtin_nop; + } + item[(*envctl)->num_used]->callback = callback; + item[(*envctl)->num_used]->name = name; + item[(*envctl)->num_used]->flags = flags; + + (*envctl)->num_used++; + return 0; +} + +size_t envctl_get_index(const struct EnvCtl *envctl, const char *name) { + for (size_t i = 0; i < envctl->num_used; i++) { + if (!strcmp(envctl->item[i]->name, name)) { + // pack state flag, outer (struct) index and inner (name) index + return 1L << 63L | i; + } + } + return 0; +} + +void envctl_decode_index(size_t in_i, size_t *state, size_t *out_i, size_t *name_i) { + *state = ((in_i >> 63L) & 1); + *out_i = in_i & 0xffffffffL; +} + +unsigned envctl_check_required(unsigned flags) { + return flags & STASIS_ENVCTL_REQUIRED; +} + +unsigned envctl_check_redact(unsigned flags) { + return flags & STASIS_ENVCTL_REDACT; +} + +int envctl_check_present(const struct EnvCtl_Item *item, const char *name) { + return ((!strcmp(item->name, name)) && getenv(name)) ? 1 : 0; +} + +unsigned envctl_get_flags(const struct EnvCtl *envctl, const char *name) { + size_t poll_index = envctl_get_index(envctl, name); + size_t id = 0; + size_t name_id = 0; + size_t state = 0; + envctl_decode_index(poll_index, &state, &id, &name_id); + if (!state) { + return 0; + } else { + fprintf(stderr, "managed environment variable: %s\n", name); + } + return envctl->item[id]->flags; +} + +void envctl_do_required(const struct EnvCtl *envctl, int verbose) { + for (size_t i = 0; i < envctl->num_used; i++) { + struct EnvCtl_Item *item = envctl->item[i]; + const char *name = item->name; + envctl_except_fn *callback = item->callback; + + if (verbose) { + msg(STASIS_MSG_L2, "Verifying %s\n", name); + } + int code = callback((const void *) item, (const void *) name); + if (code == STASIS_ENVCTL_RET_IGNORE || code == STASIS_ENVCTL_RET_SUCCESS) { + continue; + } else if (code == STASIS_ENVCTL_RET_FAIL) { + fprintf(stderr, "\n%s must be set. Exiting.\n", name); + exit(1); + } else { + fprintf(stderr, "\nan unknown envctl callback code occurred: %d\n", code); + exit(1); + } + } +} + +void envctl_free(struct EnvCtl **envctl) { + if (!envctl) { + return; + } + for (size_t i = 0; i < (*envctl)->num_used; i++) { + guard_free((*envctl)->item[i]); + } + guard_free((*envctl)->item); + guard_free(*envctl); +}
\ No newline at end of file diff --git a/src/lib/core/environment.c b/src/lib/core/environment.c new file mode 100644 index 0000000..580062c --- /dev/null +++ b/src/lib/core/environment.c @@ -0,0 +1,443 @@ +/** + * @file environment.c + */ +#include "environment.h" +#include "utils.h" +#include "strlist.h" + +//extern char **__environ; + +/** + * Print a shell-specific listing of environment variables to `stdout` + * + * Example: + * ~~~{.c} + * int main(int argc, char *argv[], char *arge[]) { + * RuntimeEnv *rt = runtime_copy(arge); + * runtime_export(rt, NULL); + * runtime_free(rt); + * return 0; + * } + * ~~~ + * + * Usage: + * ~~~{.sh} + * $ gcc program.c + * $ ./a.out + * PATH="/thing/stuff/bin:/example/please/bin" + * SHELL="/your/shell" + * CC="/your/compiler" + * ...=... + * + * # You can also use this to modify the shell environment + * # (use `runtime_set` to manipulate the output) + * $ source $(./a.out) + * ~~~ + * + * Example of exporting specific keys from the environment: + * + * ~~~{.c} + * int main(int argc, char *argv[], char *arge[]) { + * RuntimeEnv *rt = runtime_copy(arge); + * + * // inline declaration + * runtime_export(rt, (char *[]) {"PATH", "LS_COLORS", NULL}); + * + * // standard declaration + * char *keys_to_export[] = { + * "PATH", "LS_COLORS", NULL + * } + * runtime_export(rt, keys_to_export); + * + * runtime_free(rt); + * return 0; + * } + * ~~~ + * + * @param env `RuntimeEnv` structure + * @param keys Array of keys to export. A value of `NULL` exports all environment keys + */ +void runtime_export(RuntimeEnv *env, char **keys) { + char *borne[] = { + "bash", + "dash", + "zsh", + NULL, + }; + char *unborne[] = { + "csh" + "tcsh", + NULL, + }; + + char output[STASIS_BUFSIZ]; + char export_command[7]; // export=6 and setenv=6... convenient + char *_sh = getenv("SHELL"); + char *sh = path_basename(_sh); + if (sh == NULL) { + fprintf(stderr, "echo SHELL environment variable is not defined"); + exit(1); + } + + for (size_t i = 0; borne[i] != NULL; i++) { + if (strcmp(sh, borne[i]) == 0) { + strcpy(export_command, "export"); + break; + } + } + for (size_t i = 0; unborne[i] != NULL; i++) { + if (strcmp(sh, unborne[i]) == 0) { + strcpy(export_command, "setenv"); + break; + } + } + + for (size_t i = 0; i < strlist_count(env); i++) { + char **pair = split(strlist_item(env, i), "=", 0); + char *key = pair[0]; + char *value = NULL; + + // We split a potentially large string by "=" so: + // Recombine elements pair[1..N] into a single string by "=" + if (pair[1] != NULL) { + value = join(&pair[1], "="); + } + + if (keys != NULL) { + for (size_t j = 0; keys[j] != NULL; j++) { + if (strcmp(keys[j], key) == 0) { + //sprintf(output, "%s=\"%s\"\n%s %s", key, value ? value : "", export_command, key); + sprintf(output, "%s %s=\"%s\"", export_command, key, value ? value : ""); + puts(output); + } + } + } + else { + sprintf(output, "%s %s=\"%s\"", export_command, key, value ? value : ""); + puts(output); + } + guard_free(value); + GENERIC_ARRAY_FREE(pair); + } +} + +/** + * Populate a `RuntimeEnv` structure + * + * Example: + * + * ~~~{.c} + * int main(int argc, char *argv[], char *arge[]) { + * RuntimeEnv *rt = NULL; + * // Example 1: Copy the shell environment + * rt = runtime_copy(arge); + * // Example 2: Create your own environment + * rt = runtime_copy((char *[]) {"SHELL=/bin/bash", "PATH=/opt/secure:/bin:/usr/bin"}) + * + * runtime_free(rt); + * return 0; + * } + * ~~~ + * + * @param env Array of strings in `var=value` format + * @return `RuntimeEnv` structure + */ +RuntimeEnv *runtime_copy(char **env) { + RuntimeEnv *rt = NULL; + size_t env_count; + for (env_count = 0; env[env_count] != NULL; env_count++); + + rt = strlist_init(); + for (size_t i = 0; i < env_count; i++) { + strlist_append(&rt, env[i]); + } + return rt; +} + +/** + * Replace the contents of `dest` with `src` + * @param dest pointer of type `RuntimeEnv` + * @param src pointer to environment array + * @return 0 on success, <0 on error + */ +int runtime_replace(RuntimeEnv **dest, char **src) { + RuntimeEnv *rt_tmp = runtime_copy(src); + if (!rt_tmp) { + return -1; + } + runtime_free((*dest)); + + (*dest) = runtime_copy(rt_tmp->data); + if (!(*dest)) { + return -1; + } + runtime_free(rt_tmp); + + runtime_apply((*dest)); + return 0; +} + +/** + * Determine whether or not a key exists in the runtime environment + * + * Example: + * + * ~~~{.c} + * int main(int argc, char *argv[], char *arge[]) { + * RuntimeEnv *rt = runtime_copy(arge); + * if (runtime_contains(rt, "PATH") { + * // $PATH is present + * } + * else { + * // $PATH is NOT present + * } + * + * runtime_free(rt); + * return 0; + * } + * ~~~ + * + * @param env `RuntimeEnv` structure + * @param key Environment variable string + * @return -1=no, positive_value=yes + */ +ssize_t runtime_contains(RuntimeEnv *env, const char *key) { + ssize_t result = -1; + for (ssize_t i = 0; i < (ssize_t) strlist_count(env); i++) { + char **pair = split(strlist_item(env, i), "=", 0); + if (pair == NULL) { + break; + } + if (strcmp(pair[0], key) == 0) { + result = i; + GENERIC_ARRAY_FREE(pair); + break; + } + GENERIC_ARRAY_FREE(pair); + } + return result; +} + +/** + * Retrieve the value of a runtime environment variable + * + * Example: + * + * ~~~{.c} + * int main(int argc, char *argv[], char *arge[]) { + * RuntimeEnv *rt = runtime_copy(arge); + * char *path = runtime_get("PATH"); + * if (path == NULL) { + * // handle error + * } + * + * runtime_free(rt); + * return 0; + * } + * ~~~ + * + * @param env `RuntimeEnv` structure + * @param key Environment variable string + * @return success=string, failure=`NULL` + */ +char *runtime_get(RuntimeEnv *env, const char *key) { + char *result = NULL; + ssize_t key_offset = runtime_contains(env, key); + if (key_offset != -1) { + char **pair = split(strlist_item(env, key_offset), "=", 0); + result = join(&pair[1], "="); + GENERIC_ARRAY_FREE(pair); + } + return result; +} + +/** + * Parse an input string and expand any environment variable(s) found + * + * Example: + * + * ~~~{.c} + * int main(int argc, char *argv[], char *arge[]) { + * RuntimeEnv *rt = runtime_copy(arge); + * char *secure_path = runtime_expand_var(rt, "/opt/secure:$PATH:/aux/bin"); + * if (secure_path == NULL) { + * // handle error + * } + * // secure_path = "/opt/secure:/your/original/path/here:/aux/bin"; + * + * runtime_free(rt); + * return 0; + * } + * ~~~ + * + * @param env `RuntimeEnv` structure + * @param input String to parse + * @return success=expanded string, failure=`NULL` + */ +char *runtime_expand_var(RuntimeEnv *env, char *input) { + const char delim = '$'; + const char *delim_literal = "$$"; + char *expanded = NULL; + + // Input is invalid + if (!input) { + return NULL; + } + + // If there's no environment variables to process return the input string + if (strchr(input, delim) == NULL) { + //return strdup(input); + return input; + } + + expanded = calloc(STASIS_BUFSIZ, sizeof(char)); + if (expanded == NULL) { + SYSERROR("could not allocate %d bytes for runtime_expand_var buffer", STASIS_BUFSIZ); + return NULL; + } + + // Parse the input string + size_t i; + for (i = 0; i < strlen(input); i++) { + char var[MAXNAMLEN]; // environment variable name + memset(var, '\0', MAXNAMLEN); // zero out name + + // Handle literal statement "$$var" + // Value becomes "$var" (unexpanded) + if (strncmp(&input[i], delim_literal, strlen(delim_literal)) == 0) { + strncat(expanded, &delim, 2); + i += strlen(delim_literal); + // Ignore opening brace + if (input[i] == '{') { + i++; + } + } + + // Handle variable when encountering a single $ + // Value expands from "$var" to "environment value of var" + if (input[i] == delim) { + // Ignore opening brace + if (input[i+1] == '{') { + i++; + } + char *tmp = NULL; + i++; + + // Construct environment variable name from input + // "$ var" == no + // "$-*)!@ == no + // "$var" == yes + for (size_t c = 0; isalnum(input[i]) || input[i] == '_'; c++, i++) { + // Ignore closing brace + if (input[i] == '}') { + i++; + } + var[c] = input[i]; + } + + if (env) { + tmp = runtime_get(env, var); + } else { + tmp = getenv(var); + } + if (tmp == NULL) { + // This mimics shell behavior in general. + // Prevent appending whitespace when an environment variable does not exist + if (i > 0) { + i--; + } + continue; + } + // Append expanded environment variable to output + strncat(expanded, tmp, STASIS_BUFSIZ - 1); + if (env) { + guard_free(tmp); + } + } + + // Nothing to do so append input to output + if (input[i] == '}') { + // Unless we ended on a closing brace + continue; + } + strncat(expanded, &input[i], 1); + } + + return expanded; +} + +/** + * Set a runtime environment variable. + * + * + * Note: `_value` is passed through `runtime_expand_var` to provide shell expansion + * + * + * Example: + * + * ~~~{.c} + * int main(int argc, char *argv[], char *arge[]) { + * RuntimeEnv *rt = runtime_copy(arge); + * + * runtime_set(rt, "new_var", "1"); + * char *new_var = runtime_get("new_var"); + * // new_var = 1; + * + * char *path = runtime_get("PATH"); + * // path = /your/path:/here + * + * runtime_set(rt, "PATH", "/opt/secure:$PATH"); + * char *secure_path = runtime_get("PATH"); + * // secure_path = /opt/secure:/your/path:/here + * // NOTE: path and secure_path are COPIES, unlike `getenv()` and `setenv()` that reuse their pointers in `environ` + * + * runtime_free(rt); + * return 0; + * } + * ~~~ + * + * + * @param env `RuntimeEnv` structure + * @param _key Environment variable to set + * @param _value New environment variable value + */ +void runtime_set(RuntimeEnv *env, const char *_key, char *_value) { + if (_key == NULL) { + return; + } + char *key = strdup(_key); + ssize_t key_offset = runtime_contains(env, key); + char *value = runtime_expand_var(env, _value); + char *now = join((char *[]) {key, value, NULL}, "="); + + if (key_offset < 0) { + strlist_append(&env, now); + } else { + strlist_set(&env, key_offset, now); + } + guard_free(now); + guard_free(key); +} + +/** + * Update the global `environ` array with data from `RuntimeEnv` + * @param env `RuntimeEnv` structure + */ +void runtime_apply(RuntimeEnv *env) { + for (size_t i = 0; i < strlist_count(env); i++) { + char **pair = split(strlist_item(env, i), "=", 1); + setenv(pair[0], pair[1], 1); + GENERIC_ARRAY_FREE(pair); + } +} + +/** + * Free `RuntimeEnv` allocated by `runtime_copy` + * @param env `RuntimeEnv` structure + */ +void runtime_free(RuntimeEnv *env) { + if (env == NULL) { + return; + } + strlist_free(&env); +} diff --git a/src/lib/core/github.c b/src/lib/core/github.c new file mode 100644 index 0000000..c5e4534 --- /dev/null +++ b/src/lib/core/github.c @@ -0,0 +1,134 @@ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include "core.h" +#include "github.h" + +struct GHContent { + char *data; + size_t len; +}; + +static size_t writer(void *contents, size_t size, size_t nmemb, void *result) { + const size_t newlen = size * nmemb; + struct GHContent *content = (struct GHContent *) result; + + char *ptr = realloc(content->data, content->len + newlen + 1); + if (!ptr) { + perror("realloc failed"); + return 0; + } + + content->data = ptr; + memcpy(&(content->data[content->len]), contents, newlen); + content->len += newlen; + content->data[content->len] = 0; + + return newlen; +} + +static char *unescape_lf(char *value) { + char *seq = strstr(value, "\\n"); + while (seq != NULL) { + size_t cur_len = strlen(seq); + memmove(seq, seq + 1, strlen(seq) - 1); + *seq = '\n'; + if (strlen(seq) && cur_len) { + seq[cur_len - 1] = 0; + } + seq = strstr(value, "\\n"); + } + return value; +} + +int get_github_release_notes(const char *api_token, const char *repo, const char *tag, const char *target_commitish, char **output) { + const char *field_body = "\"body\":\""; + const char *field_message = "\"message\":\""; + const char *endpoint_header_auth_fmt = "Authorization: Bearer %s"; + const char *endpoint_header_api_version = "X-GitHub-Api-Version: " STASIS_GITHUB_API_VERSION; + const char *endpoint_post_fields_fmt = "{\"tag_name\":\"%s\", \"target_commitish\":\"%s\"}"; + const char *endpoint_url_fmt = "https://api.github.com/repos/%s/releases/generate-notes"; + char endpoint_header_auth[PATH_MAX] = {0}; + char endpoint_post_fields[PATH_MAX] = {0}; + char endpoint_url[PATH_MAX] = {0}; + struct curl_slist *list = NULL; + struct GHContent content; + + CURL *curl = curl_easy_init(); + if (!curl) { + return -1; + } + + // Render the header data + sprintf(endpoint_header_auth, endpoint_header_auth_fmt, api_token); + sprintf(endpoint_post_fields, endpoint_post_fields_fmt, tag, target_commitish); + sprintf(endpoint_url, endpoint_url_fmt, repo); + + // Begin curl configuration + curl_easy_setopt(curl, CURLOPT_URL, endpoint_url); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, endpoint_post_fields); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writer); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) &content); + + // Append headers to the request + list = curl_slist_append(list, "Accept: application/vnd.github+json"); + list = curl_slist_append(list, endpoint_header_auth); + list = curl_slist_append(list, endpoint_header_api_version); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list); + + // Set the user-agent (github requires one) + char user_agent[20] = {0}; + sprintf(user_agent, "stasis/%s", VERSION); + curl_easy_setopt(curl, CURLOPT_USERAGENT, user_agent); + + // Execute curl request + memset(&content, 0, sizeof(content)); + CURLcode res; + res = curl_easy_perform(curl); + + // Clean up + curl_slist_free_all(list); + curl_easy_cleanup(curl); + + if(res != CURLE_OK) { + fprintf(stderr, "curl_easy_perform() failed: %s\n", + curl_easy_strerror(res)); + return -1; + } + + // Replace all "\\n" literals with new line characters + char *line = unescape_lf(content.data); + if (line) { + char *data_offset = NULL; + if ((data_offset = strstr(line, field_body))) { + // Skip past the body field + data_offset += strlen(field_body); + // Remove quotation mark (and trailing comma if it exists) + int trim = 2; + char last_char = data_offset[strlen(data_offset) - trim]; + if (last_char == ',') { + trim++; + } + data_offset[strlen(data_offset) - trim] = 0; + // Extract release notes + *output = strdup(data_offset); + } else if ((data_offset = strstr(line, field_message))) { + // Skip past the message field + data_offset += strlen(field_message); + *(strchr(data_offset, '"')) = 0; + fprintf(stderr, "GitHub API Error: '%s'\n", data_offset); + fprintf(stderr, "URL: %s\n", endpoint_url); + fprintf(stderr, "POST: %s\n", endpoint_post_fields); + guard_free(content.data); + return -1; + } + } else { + fprintf(stderr, "Unknown error\n"); + guard_free(content.data); + return -1; + } + + guard_free(content.data); + return 0; +}
\ No newline at end of file diff --git a/src/lib/core/globals.c b/src/lib/core/globals.c new file mode 100644 index 0000000..83465f1 --- /dev/null +++ b/src/lib/core/globals.c @@ -0,0 +1,66 @@ +#include <stdlib.h> +#include <stdbool.h> +#include "core.h" +#include "envctl.h" + +const char *VERSION = "1.0.0"; +const char *AUTHOR = "Joseph Hunkeler"; +const char *BANNER = + "------------------------------------------------------------------------\n" +#if defined(STASIS_DUMB_TERMINAL) + " STASIS \n" +#else + " _____ _______ _____ _____ _____ \n" + " / ____|__ __|/\\ / ____|_ _|/ ____| \n" + " | (___ | | / \\ | (___ | | | (___ \n" + " \\___ \\ | | / /\\ \\ \\___ \\ | | \\___ \\ \n" + " ____) | | |/ ____ \\ ____) |_| |_ ____) | \n" + " |_____/ |_/_/ \\_\\_____/|_____|_____/ \n" + "\n" +#endif + "------------------------------------------------------------------------\n" + " Delivery Generator \n" + " v%s \n" + "------------------------------------------------------------------------\n" + "Copyright (C) 2023-2024 %s,\n" + "Association of Universities for Research in Astronomy (AURA)\n"; + +struct STASIS_GLOBAL globals = { + .verbose = false, ///< Toggle verbose mode + .continue_on_error = false, ///< Do not stop program on error + .always_update_base_environment = false, ///< Run "conda update --all" after installing Conda + .conda_fresh_start = true, ///< Remove/reinstall Conda at startup + .conda_install_prefix = NULL, ///< Path to install Conda + .conda_packages = NULL, ///< Conda packages to install + .pip_packages = NULL, ///< Python packages to install + .tmpdir = NULL, ///< Path to store temporary data + .enable_docker = true, ///< Toggle docker usage + .enable_artifactory = true, ///< Toggle artifactory server usage + .enable_artifactory_build_info = true, ///< Toggle build-info uploads + .enable_testing = true, ///< Toggle [test] block "script" execution. "script_setup" always executes. + .enable_rewrite_spec_stage_2 = true, ///< Leave template stings in output files + .enable_parallel = true, ///< Toggle testing in parallel + .parallel_fail_fast = false, ///< Kill ALL multiprocessing tasks immediately on error + .pool_status_interval = 30, ///< Report "Task is running" +}; + +void globals_free() { + guard_free(globals.tmpdir); + guard_free(globals.sysconfdir); + guard_free(globals.conda_install_prefix); + guard_strlist_free(&globals.conda_packages); + guard_strlist_free(&globals.pip_packages); + guard_free(globals.jfrog.arch); + guard_free(globals.jfrog.os); + guard_free(globals.jfrog.url); + guard_free(globals.jfrog.repo); + guard_free(globals.jfrog.version); + guard_free(globals.jfrog.cli_major_ver); + guard_free(globals.jfrog.jfrog_artifactory_base_url); + guard_free(globals.jfrog.jfrog_artifactory_product); + guard_free(globals.jfrog.remote_filename); + guard_free(globals.workaround.conda_reactivate); + if (globals.envctl) { + envctl_free(&globals.envctl); + } +} diff --git a/src/lib/core/ini.c b/src/lib/core/ini.c new file mode 100644 index 0000000..d44e1cc --- /dev/null +++ b/src/lib/core/ini.c @@ -0,0 +1,678 @@ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <ctype.h> +#include "core.h" +#include "ini.h" + +struct INIFILE *ini_init() { + struct INIFILE *ini; + ini = calloc(1, sizeof(*ini)); + ini->section_count = 0; + return ini; +} + +void ini_section_init(struct INIFILE **ini) { + (*ini)->section = calloc((*ini)->section_count + 1, sizeof(**(*ini)->section)); +} + +struct INISection *ini_section_search(struct INIFILE **ini, unsigned mode, const char *value) { + struct INISection *result = NULL; + for (size_t i = 0; i < (*ini)->section_count; i++) { + if ((*ini)->section[i]->key != NULL) { + if (mode == INI_SEARCH_EXACT) { + if (!strcmp((*ini)->section[i]->key, value)) { + result = (*ini)->section[i]; + break; + } + } else if (mode == INI_SEARCH_BEGINS) { + if (startswith((*ini)->section[i]->key, value)) { + result = (*ini)->section[i]; + break; + } + } else if (mode == INI_SEARCH_SUBSTR) { + if (strstr((*ini)->section[i]->key, value)) { + result = (*ini)->section[i]; + break; + } + } + } + } + return result; +} + +int ini_data_init(struct INIFILE **ini, char *section_name) { + struct INISection *section = ini_section_search(ini, INI_SEARCH_EXACT, section_name); + if (section == NULL) { + return 1; + } + section->data = calloc(section->data_count + 1, sizeof(**section->data)); + return 0; +} + +int ini_has_key(struct INIFILE *ini, const char *section_name, const char *key) { + if (!ini || !section_name || !key) { + return 0; + } + struct INISection *section = ini_section_search(&ini, INI_SEARCH_EXACT, section_name); + if (!section) { + return 0; + } + for (size_t i = 0; i < section->data_count; i++) { + const struct INIData *data = section->data[i]; + if (data && data->key) { + if (!strcmp(data->key, key)) { + return 1; + } + } + } + return 0; +} + +struct INIData *ini_data_get(struct INIFILE *ini, char *section_name, char *key) { + struct INISection *section = NULL; + + section = ini_section_search(&ini, INI_SEARCH_EXACT, section_name); + if (!section) { + return NULL; + } + + for (size_t i = 0; i < section->data_count; i++) { + if (section->data[i]->key != NULL) { + if (!strcmp(section->data[i]->key, key)) { + return section->data[i]; + } + } + } + + return NULL; +} + +struct INIData *ini_getall(struct INIFILE *ini, char *section_name) { + struct INISection *section = NULL; + struct INIData *result = NULL; + static size_t i = 0; + + section = ini_section_search(&ini, INI_SEARCH_EXACT, section_name); + if (!section) { + return NULL; + } + if (i == section->data_count) { + i = 0; + return NULL; + } + if (section->data_count) { + result = section->data[i]; + i++; + } else { + result = NULL; + i = 0; + } + + return result; +} + +int ini_getval(struct INIFILE *ini, char *section_name, char *key, int type, int flags, union INIVal *result) { + char *token = NULL; + char tbuf[STASIS_BUFSIZ]; + char *tbufp = tbuf; + struct INIData *data; + data = ini_data_get(ini, section_name, key); + if (!data) { + result->as_char_p = NULL; + return -1; + } + + char *data_copy = strdup(data->value); + if (flags == INI_READ_RENDER) { + char *render = tpl_render(data_copy); + if (render && strcmp(render, data_copy) != 0) { + guard_free(data_copy); + data_copy = render; + } else { + guard_free(render); + } + } + lstrip(data_copy); + + switch (type) { + case INIVAL_TYPE_CHAR: + result->as_char = (char) strtol(data_copy, NULL, 10); + break; + case INIVAL_TYPE_UCHAR: + result->as_uchar = (unsigned char) strtoul(data_copy, NULL, 10); + break; + case INIVAL_TYPE_SHORT: + result->as_short = (short) strtol(data_copy, NULL, 10); + break; + case INIVAL_TYPE_USHORT: + result->as_ushort = (unsigned short) strtoul(data_copy, NULL, 10); + break; + case INIVAL_TYPE_INT: + result->as_int = (int) strtol(data_copy, NULL, 10); + break; + case INIVAL_TYPE_UINT: + result->as_uint = (unsigned int) strtoul(data_copy, NULL, 10); + break; + case INIVAL_TYPE_LONG: + result->as_long = (long) strtol(data_copy, NULL, 10); + break; + case INIVAL_TYPE_ULONG: + result->as_ulong = (unsigned long) strtoul(data_copy, NULL, 10); + break; + case INIVAL_TYPE_LLONG: + result->as_llong = (long long) strtoll(data_copy, NULL, 10); + break; + case INIVAL_TYPE_ULLONG: + result->as_ullong = (unsigned long long) strtoull(data_copy, NULL, 10); + break; + case INIVAL_TYPE_DOUBLE: + result->as_double = (double) strtod(data_copy, NULL); + break; + case INIVAL_TYPE_FLOAT: + result->as_float = strtof(data_copy, NULL); + break; + case INIVAL_TYPE_STR: + result->as_char_p = strdup(data_copy); + if (!result->as_char_p) { + return -1; + } + break; + case INIVAL_TYPE_STR_ARRAY: + strcpy(tbufp, data_copy); + guard_free(data_copy); + data_copy = calloc(STASIS_BUFSIZ, sizeof(*data_copy)); + if (!data_copy) { + return -1; + } + while ((token = strsep(&tbufp, "\n")) != NULL) { + //lstrip(token); + if (!isempty(token)) { + strcat(data_copy, token); + strcat(data_copy, "\n"); + } + } + strip(data_copy); + result->as_char_p = strdup(data_copy); + break; + case INIVAL_TYPE_BOOL: + result->as_bool = false; + if ((!strcmp(data_copy, "true") || !strcmp(data_copy, "True")) || + (!strcmp(data_copy, "yes") || !strcmp(data_copy, "Yes")) || + strtol(data_copy, NULL, 10)) { + result->as_bool = true; + } + break; + default: + memset(result, 0, sizeof(*result)); + break; + } + guard_free(data_copy); + return 0; +} + +#define getval_returns(t) return result.t +#define getval_setup(t, f) \ + union INIVal result; \ + int state_local = 0; \ + state_local = ini_getval(ini, section_name, key, t, f, &result); \ + if (state != NULL) { \ + *state = state_local; \ + } + +int ini_getval_int(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + getval_setup(INIVAL_TYPE_INT, flags) + getval_returns(as_int); +} + +unsigned int ini_getval_uint(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + getval_setup(INIVAL_TYPE_UINT, flags) + getval_returns(as_uint); +} + +long ini_getval_long(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + getval_setup(INIVAL_TYPE_LONG, flags) + getval_returns(as_long); +} + +unsigned long ini_getval_ulong(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + getval_setup(INIVAL_TYPE_ULONG, flags) + getval_returns(as_ulong); +} + +long long ini_getval_llong(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + getval_setup(INIVAL_TYPE_LLONG, flags) + getval_returns(as_llong); +} + +unsigned long long ini_getval_ullong(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + getval_setup(INIVAL_TYPE_ULLONG, flags) + getval_returns(as_ullong); +} + +float ini_getval_float(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + getval_setup(INIVAL_TYPE_FLOAT, flags) + getval_returns(as_float); +} + +double ini_getval_double(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + getval_setup(INIVAL_TYPE_DOUBLE, flags) + getval_returns(as_double); +} + +bool ini_getval_bool(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + getval_setup(INIVAL_TYPE_BOOL, flags) + getval_returns(as_bool); +} + +short ini_getval_short(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + getval_setup(INIVAL_TYPE_SHORT, flags) + getval_returns(as_short); +} + +unsigned short ini_getval_ushort(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + getval_setup(INIVAL_TYPE_USHORT, flags) + getval_returns(as_ushort); +} + +char ini_getval_char(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + getval_setup(INIVAL_TYPE_CHAR, flags) + getval_returns(as_char); +} + +unsigned char ini_getval_uchar(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + getval_setup(INIVAL_TYPE_UCHAR, flags) + getval_returns(as_uchar); +} + +char *ini_getval_char_p(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + getval_setup(INIVAL_TYPE_STR, flags) + getval_returns(as_char_p); +} + +char *ini_getval_str(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + return ini_getval_char_p(ini, section_name, key, flags, state); +} + +char *ini_getval_char_array_p(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + getval_setup(INIVAL_TYPE_STR_ARRAY, flags) + getval_returns(as_char_p); +} + +char *ini_getval_str_array(struct INIFILE *ini, char *section_name, char *key, int flags, int *state) { + return ini_getval_char_array_p(ini, section_name, key, flags, state); +} + +struct StrList *ini_getval_strlist(struct INIFILE *ini, char *section_name, char *key, char *tok, int flags, int *state) { + getval_setup(INIVAL_TYPE_STR_ARRAY, flags) + struct StrList *list; + list = strlist_init(); + strlist_append_tokenize(list, result.as_char_p, tok); + guard_free(result.as_char_p); + return list; +} + +int ini_data_append(struct INIFILE **ini, char *section_name, char *key, char *value, unsigned int hint) { + struct INISection *section = ini_section_search(ini, INI_SEARCH_EXACT, section_name); + if (section == NULL) { + return 1; + } + + struct INIData **tmp = realloc(section->data, (section->data_count + 1) * sizeof(**section->data)); + if (tmp == NULL) { + return 1; + } else { + section->data = tmp; + } + if (!ini_data_get((*ini), section_name, key)) { + struct INIData **data = section->data; + data[section->data_count] = calloc(1, sizeof(*data[0])); + if (!data[section->data_count]) { + SYSERROR("Unable to allocate %zu bytes for section data", sizeof(*data[0])); + return -1; + } + data[section->data_count]->type_hint = hint; + data[section->data_count]->key = key ? strdup(key) : strdup(""); + if (!data[section->data_count]->key) { + SYSERROR("Unable to allocate data key%s", ""); + return -1; + } + data[section->data_count]->value = strdup(value); + if (!data[section->data_count]->value) { + SYSERROR("Unable to allocate data value%s", ""); + return -1; + } + section->data_count++; + } else { + struct INIData *data = ini_data_get(*ini, section_name, key); + size_t value_len_old = strlen(data->value); + size_t value_len = strlen(value); + size_t value_len_new = value_len_old + value_len; + char *value_tmp = NULL; + value_tmp = realloc(data->value, value_len_new + 2); + if (!value_tmp) { + SYSERROR("Unable to increase data->value size to %zu bytes", value_len_new + 2); + return -1; + } else { + data->value = value_tmp; + } + strcat(data->value, value); + } + return 0; +} + +int ini_setval(struct INIFILE **ini, unsigned type, char *section_name, char *key, char *value) { + struct INISection *section = ini_section_search(ini, INI_SEARCH_EXACT, section_name); + if (section == NULL) { + // no section + return -1; + } + if (ini_has_key(*ini, section_name, key)) { + if (!type) { + if (ini_data_append(ini, section_name, key, value, 0)) { + // append failed + return -1; + } + } else { + struct INIData *data = ini_data_get(*ini, section_name, key); + if (data) { + guard_free(data->value); + data->value = strdup(value); + if (!data->value) { + // allocation failed + return -1; + } + } else { + // getting data failed + return -1; + } + } + } + return 0; +} + +int ini_section_create(struct INIFILE **ini, char *key) { + struct INISection **tmp = realloc((*ini)->section, ((*ini)->section_count + 1) * sizeof(**(*ini)->section)); + if (tmp == NULL) { + return 1; + } else { + (*ini)->section = tmp; + } + + (*ini)->section[(*ini)->section_count] = calloc(1, sizeof(*(*ini)->section[0])); + if (!(*ini)->section[(*ini)->section_count]) { + return -1; + } + + (*ini)->section[(*ini)->section_count]->key = strdup(key); + if (!(*ini)->section[(*ini)->section_count]->key) { + return -1; + } + + (*ini)->section_count++; + return 0; +} + +int ini_write(struct INIFILE *ini, FILE **stream, unsigned mode) { + if (!*stream) { + return -1; + } + for (size_t x = 0; x < ini->section_count; x++) { + struct INISection *section = ini->section[x]; + char *section_name = section->key; + fprintf(*stream, "[%s]" LINE_SEP, section_name); + + for (size_t y = 0; y < ini->section[x]->data_count; y++) { + struct INIData *data = section->data[y]; + char outvalue[STASIS_BUFSIZ]; + char *key = data->key; + char *value = data->value; + unsigned *hint = &data->type_hint; + memset(outvalue, 0, sizeof(outvalue)); + + if (key && value) { + int err = 0; + char *xvalue = NULL; + if (*hint == INIVAL_TYPE_STR_ARRAY) { + xvalue = ini_getval_str_array(ini, section_name, key, (int) mode, &err); + value = xvalue; + } else { + xvalue = ini_getval_str(ini, section_name, key, (int) mode, &err); + value = xvalue; + } + char **parts = split(value, LINE_SEP, 0); + size_t parts_total = 0; + for (; parts && parts[parts_total] != NULL; parts_total++); + for (size_t p = 0; parts && parts[p] != NULL; p++) { + char *render = NULL; + if (mode == INI_WRITE_PRESERVE) { + render = tpl_render(parts[p]); + } else { + render = parts[p]; + } + + if (!render) { + SYSERROR("%s", "rendered string value can never be NULL!\n"); + return -1; + } + + if (*hint == INIVAL_TYPE_STR_ARRAY) { + int leading_space = isspace(*render); + if (leading_space) { + sprintf(outvalue + strlen(outvalue), "%s" LINE_SEP, render); + } else { + sprintf(outvalue + strlen(outvalue), " %s" LINE_SEP, render); + } + } else { + sprintf(outvalue + strlen(outvalue), "%s", render); + } + if (mode == INI_WRITE_PRESERVE) { + guard_free(render); + } + } + GENERIC_ARRAY_FREE(parts); + strip(outvalue); + strcat(outvalue, LINE_SEP); + fprintf(*stream, "%s = %s%s", ini->section[x]->data[y]->key, *hint == INIVAL_TYPE_STR_ARRAY ? LINE_SEP : "", outvalue); + guard_free(value); + } else { + fprintf(*stream, "%s = %s", ini->section[x]->data[y]->key, ini->section[x]->data[y]->value); + } + } + fprintf(*stream, LINE_SEP); + } + return 0; +} + +char *unquote(char *s) { + if ((startswith(s, "'") && endswith(s, "'")) + || (startswith(s, "\"") && endswith(s, "\""))) { + memmove(s, s + 1, strlen(s)); + s[strlen(s) - 1] = '\0'; + } + return s; +} + +void ini_free(struct INIFILE **ini) { + for (size_t section = 0; section < (*ini)->section_count; section++) { +#ifdef DEBUG + SYSERROR("freeing section: %s", (*ini)->section[section]->key); +#endif + for (size_t data = 0; data < (*ini)->section[section]->data_count; data++) { + if ((*ini)->section[section]->data[data]) { +#ifdef DEBUG + SYSERROR("freeing data key: %s", (*ini)->section[section]->data[data]->key); +#endif + guard_free((*ini)->section[section]->data[data]->key); +#ifdef DEBUG + SYSERROR("freeing data value: %s", (*ini)->section[section]->data[data]->value); +#endif + guard_free((*ini)->section[section]->data[data]->value); + guard_free((*ini)->section[section]->data[data]); + } + } + guard_free((*ini)->section[section]->data); + guard_free((*ini)->section[section]->key); + guard_free((*ini)->section[section]); + } + guard_free((*ini)->section); + guard_free((*ini)); +} + +struct INIFILE *ini_open(const char *filename) { + FILE *fp; + char line[STASIS_BUFSIZ] = {0}; + char current_section[STASIS_BUFSIZ] = {0}; + char reading_value = 0; + + struct INIFILE *ini = ini_init(); + if (ini == NULL) { + return NULL; + } + + ini_section_init(&ini); + + // Create an implicit section. [default] does not need to be present in the INI config + ini_section_create(&ini, "default"); + strcpy(current_section, "default"); + + // Open the configuration file for reading + fp = fopen(filename, "r"); + if (!fp) { + ini_free(&ini); + ini = NULL; + return NULL; + } + + unsigned hint = 0; + int multiline_data = 0; + int no_data = 0; + char inikey[2][255]; + char *key = inikey[0]; + char *key_last = inikey[1]; + char value[STASIS_BUFSIZ]; + + memset(value, 0, sizeof(value)); + memset(inikey, 0, sizeof(inikey)); + + // Read file + for (size_t i = 0; fgets(line, sizeof(line), fp) != NULL; i++) { + if (no_data && multiline_data) { + if (!isempty(line)) { + no_data = 0; + } else { + multiline_data = 0; + } + memset(value, 0, sizeof(value)); + } else { + memset(key, 0, sizeof(inikey[0])); + } + // Find pointer to first comment character + char *comment = strpbrk(line, ";#"); + if (comment) { + if (!reading_value || line - comment == 0) { + // Remove comment from line (standalone and inline comments) + if (!((comment - line > 0 && (*(comment - 1) == '\\')) || (*comment - 1) == '#')) { + *comment = '\0'; + } else { + // Handle escaped comment characters. Remove the escape character '\' + memmove(comment - 1, comment, strlen(comment)); + if (strlen(comment)) { + comment[strlen(comment) - 1] = '\0'; + } else { + comment[0] = '\0'; + } + } + } + } + + // Test for section header: [string] + if (startswith(line, "[")) { + // The previous key is irrelevant now + memset(key_last, 0, sizeof(inikey[1])); + + char *section_name = substring_between(line, "[]"); + if (!section_name) { + fprintf(stderr, "error: invalid section syntax, line %zu: '%s'\n", i + 1, line); + return NULL; + } + + // Ignore default section because we already have an implicit one + if (!strncmp(section_name, "default", strlen("default"))) { + guard_free(section_name); + continue; + } + + // Create new named section + strip(section_name); + ini_section_create(&ini, section_name); + + // Record the name of the section. This is used until another section is found. + memset(current_section, 0, sizeof(current_section)); + strcpy(current_section, section_name); + guard_free(section_name); + memset(line, 0, sizeof(line)); + continue; + } + + // no data, skip + if (!reading_value && isempty(line)) { + continue; + } + + char *operator = strchr(line, '='); + + // a value continuation line + if (multiline_data && (startswith(line, " ") || startswith(line, "\t"))) { + operator = NULL; + } + + if (operator) { + size_t key_len = operator - line; + memset(key, 0, sizeof(inikey[0])); + strncpy(key, line, key_len); + lstrip(key); + strip(key); + memset(key_last, 0, sizeof(inikey[1])); + strcpy(key_last, key); + reading_value = 1; + if (strlen(operator) > 1) { + strcpy(value, &operator[1]); + } else { + strcpy(value, ""); + } + if (isempty(value)) { + //printf("%s is probably long raw data\n", key); + hint = INIVAL_TYPE_STR_ARRAY; + multiline_data = 1; + no_data = 1; + } else { + //printf("%s is probably short data\n", key); + hint = INIVAL_TYPE_STR; + multiline_data = 0; + } + strip(value); + } else { + strcpy(key, key_last); + strcpy(value, line); + } + memset(line, 0, sizeof(line)); + + // Store key value pair in section's data array + if (strlen(key)) { + lstrip(key); + strip(key); + unquote(value); + if (!multiline_data) { + reading_value = 0; + ini_data_append(&ini, current_section, key, value, hint); + continue; + } + ini_data_append(&ini, current_section, key, value, hint); + reading_value = 1; + } + } + fclose(fp); + + return ini; +}
\ No newline at end of file diff --git a/src/lib/core/junitxml.c b/src/lib/core/junitxml.c new file mode 100644 index 0000000..c7d0834 --- /dev/null +++ b/src/lib/core/junitxml.c @@ -0,0 +1,240 @@ +#include <stdlib.h> +#include <string.h> +#include "strlist.h" +#include "junitxml.h" + +static void testcase_result_state_free(struct JUNIT_Testcase **testcase) { + struct JUNIT_Testcase *tc = (*testcase); + if (tc->tc_result_state_type == JUNIT_RESULT_STATE_FAILURE) { + guard_free(tc->result_state.failure->message); + guard_free(tc->result_state.failure); + } else if (tc->tc_result_state_type == JUNIT_RESULT_STATE_SKIPPED) { + guard_free(tc->result_state.skipped->message); + guard_free(tc->result_state.skipped); + } +} + +static void testcase_free(struct JUNIT_Testcase **testcase) { + struct JUNIT_Testcase *tc = (*testcase); + guard_free(tc->name); + guard_free(tc->message); + guard_free(tc->classname); + testcase_result_state_free(&tc); + guard_free(tc); +} + +void junitxml_testsuite_free(struct JUNIT_Testsuite **testsuite) { + struct JUNIT_Testsuite *suite = (*testsuite); + guard_free(suite->name); + guard_free(suite->hostname); + guard_free(suite->timestamp); + for (size_t i = 0; i < suite->_tc_alloc; i++) { + testcase_free(&suite->testcase[i]); + } + guard_free(suite); +} + +static int testsuite_append_testcase(struct JUNIT_Testsuite **testsuite, struct JUNIT_Testcase *testcase) { + struct JUNIT_Testsuite *suite = (*testsuite); + struct JUNIT_Testcase **tmp = realloc(suite->testcase, (suite->_tc_alloc + 1 ) * sizeof(*testcase)); + if (tmp == NULL) { + return -1; + } else { + suite->testcase = tmp; + } + suite->testcase[suite->_tc_inuse] = testcase; + suite->_tc_inuse++; + suite->_tc_alloc++; + return 0; +} + +static struct JUNIT_Failure *testcase_failure_from_attributes(struct StrList *attrs) { + struct JUNIT_Failure *result; + + result = calloc(1, sizeof(*result)); + if(!result) { + return NULL; + } + for (size_t x = 0; x < strlist_count(attrs); x += 2) { + char *attr_name = strlist_item(attrs, x); + char *attr_value = strlist_item(attrs, x + 1); + if (!strcmp(attr_name, "message")) { + result->message = strdup(attr_value); + } + } + return result; +} + +static struct JUNIT_Error *testcase_error_from_attributes(struct StrList *attrs) { + struct JUNIT_Error *result; + + result = calloc(1, sizeof(*result)); + if(!result) { + return NULL; + } + for (size_t x = 0; x < strlist_count(attrs); x += 2) { + char *attr_name = strlist_item(attrs, x); + char *attr_value = strlist_item(attrs, x + 1); + if (!strcmp(attr_name, "message")) { + result->message = strdup(attr_value); + } + } + return result; +} + +static struct JUNIT_Skipped *testcase_skipped_from_attributes(struct StrList *attrs) { + struct JUNIT_Skipped *result; + + result = calloc(1, sizeof(*result)); + if(!result) { + return NULL; + } + for (size_t x = 0; x < strlist_count(attrs); x += 2) { + char *attr_name = strlist_item(attrs, x); + char *attr_value = strlist_item(attrs, x + 1); + if (!strcmp(attr_name, "message")) { + result->message = strdup(attr_value); + } + } + return result; +} + +static struct JUNIT_Testcase *testcase_from_attributes(struct StrList *attrs) { + struct JUNIT_Testcase *result; + + result = calloc(1, sizeof(*result)); + if(!result) { + return NULL; + } + for (size_t x = 0; x < strlist_count(attrs); x += 2) { + char *attr_name = strlist_item(attrs, x); + char *attr_value = strlist_item(attrs, x + 1); + if (!strcmp(attr_name, "name")) { + result->name = strdup(attr_value); + } else if (!strcmp(attr_name, "classname")) { + result->classname = strdup(attr_value); + } else if (!strcmp(attr_name, "time")) { + result->time = strtof(attr_value, NULL); + } else if (!strcmp(attr_name, "message")) { + result->message = strdup(attr_value); + } + } + return result; +} + +static struct StrList *attributes_to_strlist(xmlTextReaderPtr reader) { + struct StrList *list; + xmlNodePtr node = xmlTextReaderCurrentNode(reader); + if (!node) { + return NULL; + } + + list = strlist_init(); + if (xmlTextReaderNodeType(reader) == 1 && node->properties) { + xmlAttr *attr = node->properties; + while (attr && attr->name && attr->children) { + char *attr_name = (char *) attr->name; + char *attr_value = (char *) xmlNodeListGetString(node->doc, attr->children, 1); + strlist_append(&list, attr_name ? attr_name : ""); + strlist_append(&list, attr_value ? attr_value : ""); + xmlFree((xmlChar *) attr_value); + attr = attr->next; + } + } + return list; +} + +static int read_xml_data(xmlTextReaderPtr reader, struct JUNIT_Testsuite **testsuite) { + const xmlChar *name; + //const xmlChar *value; + + name = xmlTextReaderConstName(reader); + if (!name) { + // name could not be converted to string + name = BAD_CAST "--"; + } + //value = xmlTextReaderConstValue(reader); + const char *node_name = (char *) name; + //const char *node_value = (char *) value; + + struct StrList *attrs = attributes_to_strlist(reader); + if (attrs && strlist_count(attrs)) { + if (!strcmp(node_name, "testsuite")) { + for (size_t x = 0; x < strlist_count(attrs); x += 2) { + char *attr_name = strlist_item(attrs, x); + char *attr_value = strlist_item(attrs, x + 1); + if (!strcmp(attr_name, "name")) { + (*testsuite)->name = strdup(attr_value); + } else if (!strcmp(attr_name, "errors")) { + (*testsuite)->errors = (int) strtol(attr_value, NULL, 10); + } else if (!strcmp(attr_name, "failures")) { + (*testsuite)->failures = (int) strtol(attr_value, NULL, 0); + } else if (!strcmp(attr_name, "skipped")) { + (*testsuite)->skipped = (int) strtol(attr_value, NULL, 0); + } else if (!strcmp(attr_name, "tests")) { + (*testsuite)->tests = (int) strtol(attr_value, NULL, 0); + } else if (!strcmp(attr_name, "time")) { + (*testsuite)->time = strtof(attr_value, NULL); + } else if (!strcmp(attr_name, "timestamp")) { + (*testsuite)->timestamp = strdup(attr_value); + } else if (!strcmp(attr_name, "hostname")) { + (*testsuite)->hostname = strdup(attr_value); + } + } + } else if (!strcmp(node_name, "testcase")) { + struct JUNIT_Testcase *testcase = testcase_from_attributes(attrs); + testsuite_append_testcase(testsuite, testcase); + } else if (!strcmp(node_name, "failure")) { + size_t cur_tc = (*testsuite)->_tc_inuse > 0 ? (*testsuite)->_tc_inuse - 1 : (*testsuite)->_tc_inuse; + struct JUNIT_Failure *failure = testcase_failure_from_attributes(attrs); + (*testsuite)->testcase[cur_tc]->tc_result_state_type = JUNIT_RESULT_STATE_FAILURE; + (*testsuite)->testcase[cur_tc]->result_state.failure = failure; + } else if (!strcmp(node_name, "error")) { + size_t cur_tc = (*testsuite)->_tc_inuse > 0 ? (*testsuite)->_tc_inuse - 1 : (*testsuite)->_tc_inuse; + struct JUNIT_Error *error = testcase_error_from_attributes(attrs); + (*testsuite)->testcase[cur_tc]->tc_result_state_type = JUNIT_RESULT_STATE_ERROR; + (*testsuite)->testcase[cur_tc]->result_state.error = error; + } else if (!strcmp(node_name, "skipped")) { + size_t cur_tc = (*testsuite)->_tc_inuse > 0 ? (*testsuite)->_tc_inuse - 1 : (*testsuite)->_tc_inuse; + struct JUNIT_Skipped *skipped = testcase_skipped_from_attributes(attrs); + (*testsuite)->testcase[cur_tc]->tc_result_state_type = JUNIT_RESULT_STATE_SKIPPED; + (*testsuite)->testcase[cur_tc]->result_state.skipped = skipped; + } + } + guard_strlist_free(&attrs); + return 0; +} + +static int read_xml_file(const char *filename, struct JUNIT_Testsuite **testsuite) { + xmlTextReaderPtr reader; + int result; + + reader = xmlReaderForFile(filename, NULL, 0); + if (!reader) { + return -1; + } + + result = xmlTextReaderRead(reader); + while (result == 1) { + read_xml_data(reader, testsuite); + result = xmlTextReaderRead(reader); + } + + xmlFreeTextReader(reader); + return 0; +} + +struct JUNIT_Testsuite *junitxml_testsuite_read(const char *filename) { + struct JUNIT_Testsuite *result; + + if (access(filename, F_OK)) { + return NULL; + } + + result = calloc(1, sizeof(*result)); + if (!result) { + return NULL; + } + read_xml_file(filename, &result); + return result; +}
\ No newline at end of file diff --git a/src/lib/core/multiprocessing.c b/src/lib/core/multiprocessing.c new file mode 100644 index 0000000..484c566 --- /dev/null +++ b/src/lib/core/multiprocessing.c @@ -0,0 +1,449 @@ +#include "core.h" +#include "multiprocessing.h" + +/// The sum of all tasks started by mp_task() +size_t mp_global_task_count = 0; + +static struct MultiProcessingTask *mp_pool_next_available(struct MultiProcessingPool *pool) { + return &pool->task[pool->num_used]; +} + +int child(struct MultiProcessingPool *pool, struct MultiProcessingTask *task) { + FILE *fp_log = NULL; + + // The task starts inside the requested working directory + if (chdir(task->working_dir)) { + perror(task->working_dir); + exit(1); + } + + // Record the task start time + if (clock_gettime(CLOCK_REALTIME, &task->time_data.t_start) < 0) { + perror("clock_gettime"); + exit(1); + } + + // Redirect stdout and stderr to the log file + fflush(stdout); + fflush(stderr); + // Set log file name + sprintf(task->log_file + strlen(task->log_file), "task-%zu-%d.log", mp_global_task_count, task->parent_pid); + fp_log = freopen(task->log_file, "w+", stdout); + if (!fp_log) { + fprintf(stderr, "unable to open '%s' for writing: %s\n", task->log_file, strerror(errno)); + return -1; + } + dup2(fileno(stdout), fileno(stderr)); + + // Generate timestamp for log header + time_t t = time(NULL); + char *timebuf = ctime(&t); + if (timebuf) { + // strip line feed from timestamp + timebuf[strlen(timebuf) ? strlen(timebuf) - 1 : 0] = 0; + } + + // Generate log header + fprintf(fp_log, "# STARTED: %s\n", timebuf ? timebuf : "unknown"); + fprintf(fp_log, "# PID: %d\n", task->parent_pid); + fprintf(fp_log, "# WORKDIR: %s\n", task->working_dir); + fprintf(fp_log, "# COMMAND:\n%s\n", task->cmd); + fprintf(fp_log, "# OUTPUT:\n"); + // Commit header to log file / clean up + fflush(fp_log); + + // Execute task + fflush(stdout); + fflush(stderr); + char *args[] = {"bash", "--norc", task->parent_script, (char *) NULL}; + return execvp("/bin/bash", args); +} + +int parent(struct MultiProcessingPool *pool, struct MultiProcessingTask *task, pid_t pid, int *child_status) { + printf("[%s:%s] Task started (pid: %d)\n", pool->ident, task->ident, pid); + + // Give the child process access to our PID value + task->pid = pid; + task->parent_pid = pid; + + mp_global_task_count++; + + // Check child's status + pid_t code = waitpid(pid, child_status, WUNTRACED | WCONTINUED | WNOHANG); + if (code < 0) { + perror("waitpid failed"); + return -1; + } + return 0; +} + +static int mp_task_fork(struct MultiProcessingPool *pool, struct MultiProcessingTask *task) { + pid_t pid = fork(); + int child_status = 0; + if (pid == -1) { + return -1; + } else if (pid == 0) { + child(pool, task); + } + return parent(pool, task, pid, &child_status); +} + +struct MultiProcessingTask *mp_pool_task(struct MultiProcessingPool *pool, const char *ident, char *working_dir, char *cmd) { + struct MultiProcessingTask *slot = mp_pool_next_available(pool); + if (pool->num_used != pool->num_alloc) { + pool->num_used++; + } else { + fprintf(stderr, "Maximum number of tasks reached\n"); + return NULL; + } + + // Set default status to "error" + slot->status = -1; + + // Set task identifier string + memset(slot->ident, 0, sizeof(slot->ident)); + strncpy(slot->ident, ident, sizeof(slot->ident) - 1); + + // Set log file path + memset(slot->log_file, 0, sizeof(*slot->log_file)); + strcat(slot->log_file, pool->log_root); + strcat(slot->log_file, "/"); + + // Set working directory + if (isempty(working_dir)) { + strcpy(slot->working_dir, "."); + } else { + strncpy(slot->working_dir, working_dir, PATH_MAX - 1); + } + + // Create a temporary file to act as our intermediate command script + FILE *tp = NULL; + char *t_name = NULL; + t_name = xmkstemp(&tp, "w"); + if (!t_name || !tp) { + return NULL; + } + + // Set the script's permissions so that only the calling user can use it + // This should help prevent eavesdropping if keys are applied in plain-text + // somewhere. + chmod(t_name, 0700); + + // Record the script path + memset(slot->parent_script, 0, sizeof(slot->parent_script)); + strncpy(slot->parent_script, t_name, PATH_MAX - 1); + guard_free(t_name); + + // Populate the script + fprintf(tp, "#!/bin/bash\n%s\n", cmd); + fflush(tp); + fclose(tp); + + // Record the command(s) + slot->cmd_len = (strlen(cmd) * sizeof(*cmd)) + 1; + slot->cmd = mmap(NULL, slot->cmd_len, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); + memset(slot->cmd, 0, slot->cmd_len); + strncpy(slot->cmd, cmd, slot->cmd_len); + + return slot; +} + +static void get_task_duration(struct MultiProcessingTask *task, struct timespec *result) { + // based on the timersub() macro in time.h + // This implementation uses timespec and increases the resolution from microseconds to nanoseconds. + struct timespec *start = &task->time_data.t_start; + struct timespec *stop = &task->time_data.t_stop; + result->tv_sec = (stop->tv_sec - start->tv_sec); + result->tv_nsec = (stop->tv_nsec - start->tv_nsec); + if (result->tv_nsec < 0) { + --result->tv_sec; + result->tv_nsec += 1000000000L; + } +} + +void mp_pool_show_summary(struct MultiProcessingPool *pool) { + print_banner("=", 79); + printf("Pool execution summary for \"%s\"\n", pool->ident); + print_banner("=", 79); + printf("STATUS PID DURATION IDENT\n"); + for (size_t i = 0; i < pool->num_used; i++) { + struct MultiProcessingTask *task = &pool->task[i]; + char status_str[10] = {0}; + if (!task->status && !task->signaled_by) { + strcpy(status_str, "DONE"); + } else if (task->signaled_by) { + strcpy(status_str, "TERM"); + } else { + strcpy(status_str, "FAIL"); + } + + struct timespec duration; + get_task_duration(task, &duration); + long diff = duration.tv_sec + duration.tv_nsec / 1000000000L; + printf("%-4s %10d %7lds %-10s\n", status_str, task->parent_pid, diff, task->ident) ; + } + puts(""); +} + +static int show_log_contents(FILE *stream, struct MultiProcessingTask *task) { + FILE *fp = fopen(task->log_file, "r"); + if (!fp) { + return -1; + } + char buf[BUFSIZ] = {0}; + while ((fgets(buf, sizeof(buf) - 1, fp)) != NULL) { + fprintf(stream, "%s", buf); + memset(buf, 0, sizeof(buf)); + } + fprintf(stream, "\n"); + fclose(fp); + return 0; +} + +int mp_pool_kill(struct MultiProcessingPool *pool, int signum) { + printf("Sending signal %d to pool '%s'\n", signum, pool->ident); + for (size_t i = 0; i < pool->num_used; i++) { + struct MultiProcessingTask *slot = &pool->task[i]; + if (!slot) { + return -1; + } + // Kill tasks in progress + if (slot->pid > 0) { + int status; + printf("Sending signal %d to task '%s' (pid: %d)\n", signum, slot->ident, slot->pid); + status = kill(slot->pid, signum); + if (status && errno != ESRCH) { + fprintf(stderr, "Task '%s' (pid: %d) did not respond: %s\n", slot->ident, slot->pid, strerror(errno)); + } else { + // Wait for process to handle the signal, then set the status accordingly + if (waitpid(slot->pid, &status, 0) >= 0) { + slot->signaled_by = WTERMSIG(status); + // Record the task stop time + if (clock_gettime(CLOCK_REALTIME, &slot->time_data.t_stop) < 0) { + perror("clock_gettime"); + exit(1); + } + // We are short-circuiting the normal flow, and the process is now dead, so mark it as such + slot->pid = MP_POOL_PID_UNUSED; + } + } + } + if (!access(slot->log_file, F_OK)) { + remove(slot->log_file); + } + if (!access(slot->parent_script, F_OK)) { + remove(slot->parent_script); + } + } + return 0; +} + +int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) { + int status = 0; + int failures = 0; + size_t tasks_complete = 0; + size_t lower_i = 0; + size_t upper_i = jobs; + + do { + size_t hang_check = 0; + if (upper_i >= pool->num_used) { + size_t x = upper_i - pool->num_used; + upper_i -= (size_t) x; + } + + for (size_t i = lower_i; i < upper_i; i++) { + struct MultiProcessingTask *slot = &pool->task[i]; + if (slot->status == -1) { + if (mp_task_fork(pool, slot)) { + fprintf(stderr, "%s: mp_task_fork failed\n", slot->ident); + kill(0, SIGTERM); + } + } + + // Has the child been processed already? + if (slot->pid == MP_POOL_PID_UNUSED) { + // Child is already used up, skip it + hang_check++; + if (hang_check >= pool->num_used) { + // If you join a pool that's already finished it will spin + // forever. This protects the program from entering an + // infinite loop. + fprintf(stderr, "%s is deadlocked\n", pool->ident); + failures++; + goto pool_deadlocked; + } + continue; + } + + // Is the process finished? + pid_t pid = waitpid(slot->pid, &status, WNOHANG | WUNTRACED | WCONTINUED); + int task_ended = WIFEXITED(status); + int task_ended_by_signal = WIFSIGNALED(status); + int task_stopped = WIFSTOPPED(status); + int task_continued = WIFCONTINUED(status); + int status_exit = WEXITSTATUS(status); + int status_signal = WTERMSIG(status); + int status_stopped = WSTOPSIG(status); + + // Update status + slot->status = status_exit; + slot->signaled_by = status_signal; + + char progress[1024] = {0}; + if (pid > 0) { + double percent = ((double) (tasks_complete + 1) / (double) pool->num_used) * 100; + snprintf(progress, sizeof(progress) - 1, "[%s:%s] [%3.1f%%]", pool->ident, slot->ident, percent); + + // The process ended in one the following ways + // Note: SIGSTOP nor SIGCONT will not increment the tasks_complete counter + if (task_stopped) { + printf("%s Task was suspended (%d)\n", progress, status_stopped); + continue; + } else if (task_continued) { + printf("%s Task was resumed\n", progress); + continue; + } else if (task_ended_by_signal) { + printf("%s Task ended by signal %d (%s)\n", progress, status_signal, strsignal(status_signal)); + tasks_complete++; + } else if (task_ended) { + printf("%s Task ended (status: %d)\n", progress, status_exit); + tasks_complete++; + } else { + fprintf(stderr, "%s Task state is unknown (0x%04X)\n", progress, status); + } + + // Show the log (always) + if (show_log_contents(stdout, slot)) { + perror(slot->log_file); + } + + // Record the task stop time + if (clock_gettime(CLOCK_REALTIME, &slot->time_data.t_stop) < 0) { + perror("clock_gettime"); + exit(1); + } + + if (status >> 8 != 0 || (status & 0xff) != 0) { + fprintf(stderr, "%s Task failed\n", progress); + failures++; + + if (flags & MP_POOL_FAIL_FAST && pool->num_used > 1) { + mp_pool_kill(pool, SIGTERM); + return -2; + } + } else { + printf("%s Task finished\n", progress); + } + + // Clean up logs and scripts left behind by the task + if (remove(slot->log_file)) { + fprintf(stderr, "%s Unable to remove log file: '%s': %s\n", progress, slot->parent_script, strerror(errno)); + } + if (remove(slot->parent_script)) { + fprintf(stderr, "%s Unable to remove temporary script '%s': %s\n", progress, slot->parent_script, strerror(errno)); + } + + // Update progress and tell the poller to ignore the PID. The process is gone. + slot->pid = MP_POOL_PID_UNUSED; + } else if (pid < 0) { + fprintf(stderr, "waitpid failed: %s\n", strerror(errno)); + return -1; + } else { + // Track the number of seconds elapsed for each task. + // When a task has executed for longer than status_intervals, print a status update + // _seconds represents the time between intervals, not the total runtime of the task + slot->_seconds = time(NULL) - slot->_now; + if (slot->_seconds > pool->status_interval) { + slot->_now = time(NULL); + slot->_seconds = 0; + } + if (slot->_seconds == 0) { + printf("[%s:%s] Task is running (pid: %d)\n", pool->ident, slot->ident, slot->parent_pid); + } + } + } + + if (tasks_complete == pool->num_used) { + break; + } + + if (tasks_complete == upper_i) { + lower_i += jobs; + upper_i += jobs; + } + + // Poll again after a short delay + sleep(1); + } while (1); + + pool_deadlocked: + puts(""); + return failures; +} + + +struct MultiProcessingPool *mp_pool_init(const char *ident, const char *log_root) { + struct MultiProcessingPool *pool; + + if (!ident || !log_root) { + // Pool must have an ident string + // log_root must be set + return NULL; + } + + // The pool is shared with children + pool = mmap(NULL, sizeof(*pool), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); + + // Set pool identity string + memset(pool->ident, 0, sizeof(pool->ident)); + strncpy(pool->ident, ident, sizeof(pool->ident) - 1); + + // Set logging base directory + memset(pool->log_root, 0, sizeof(pool->log_root)); + strncpy(pool->log_root, log_root, sizeof(pool->log_root) - 1); + pool->num_used = 0; + pool->num_alloc = MP_POOL_TASK_MAX; + + // Create the log directory + if (mkdirs(log_root, 0700) < 0) { + if (errno != EEXIST) { + perror(log_root); + mp_pool_free(&pool); + return NULL; + } + } + + // Task array is shared with children + pool->task = mmap(NULL, (pool->num_alloc + 1) * sizeof(*pool->task), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); + if (pool->task == MAP_FAILED) { + perror("mmap"); + mp_pool_free(&pool); + return NULL; + } + + return pool; +} + +void mp_pool_free(struct MultiProcessingPool **pool) { + for (size_t i = 0; i < (*pool)->num_alloc; i++) { + } + // Unmap all pool tasks + if ((*pool)->task) { + if ((*pool)->task->cmd) { + if (munmap((*pool)->task->cmd, (*pool)->task->cmd_len) < 0) { + perror("munmap"); + } + } + if (munmap((*pool)->task, sizeof(*(*pool)->task) * (*pool)->num_alloc) < 0) { + perror("munmap"); + } + } + // Unmap the pool + if ((*pool)) { + if (munmap((*pool), sizeof(*(*pool))) < 0) { + perror("munmap"); + } + (*pool) = NULL; + } +}
\ No newline at end of file diff --git a/src/lib/core/recipe.c b/src/lib/core/recipe.c new file mode 100644 index 0000000..833908c --- /dev/null +++ b/src/lib/core/recipe.c @@ -0,0 +1,64 @@ +#include "recipe.h" + +int recipe_clone(char *recipe_dir, char *url, char *gitref, char **result) { + struct Process proc; + char destdir[PATH_MAX]; + char *reponame = NULL; + + memset(&proc, 0, sizeof(proc)); + memset(destdir, 0, sizeof(destdir)); + reponame = path_basename(url); + + sprintf(destdir, "%s/%s", recipe_dir, reponame); + if (!*result) { + *result = calloc(PATH_MAX, sizeof(*result)); + if (!*result) { + return -1; + } + } + strncpy(*result, destdir, PATH_MAX); + + if (!access(destdir, F_OK)) { + if (!strcmp(destdir, "/")) { + fprintf(stderr, "STASIS is misconfigured. Please check your output path(s) immediately.\n"); + fprintf(stderr, "recipe_dir = '%s'\nreponame = '%s'\ndestdir = '%s'\n", + recipe_dir, reponame, destdir); + exit(1); + } + if (rmtree(destdir)) { + guard_free(*result); + *result = NULL; + return -1; + } + } + return git_clone(&proc, url, destdir, gitref); +} + + +int recipe_get_type(char *repopath) { + int result; + char path[PATH_MAX]; + // conda-forge is a collection of repositories + // "conda-forge.yml" is guaranteed to exist + const char *marker[] = { + "conda-forge.yml", + "stsci", + "meta.yaml", + NULL + }; + const int type[] = { + RECIPE_TYPE_CONDA_FORGE, + RECIPE_TYPE_ASTROCONDA, + RECIPE_TYPE_GENERIC + }; + + for (size_t i = 0; marker[i] != NULL; i++) { + sprintf(path, "%s/%s", repopath, marker[i]); + result = access(path, F_OK); + if (!result) { + return type[i]; + } + } + + return RECIPE_TYPE_UNKNOWN; +}
\ No newline at end of file diff --git a/src/lib/core/relocation.c b/src/lib/core/relocation.c new file mode 100644 index 0000000..852aca4 --- /dev/null +++ b/src/lib/core/relocation.c @@ -0,0 +1,155 @@ +/** + * @file relocation.c + */ +#include "relocation.h" +#include "str.h" + +/** + * Replace all occurrences of `target` with `replacement` in `original` + * + * ~~~{.c} + * char *str = calloc(100, sizeof(char)); + * strcpy(str, "This are a test."); + * if (replace_text(str, "are", "is")) { + * fprintf(stderr, "string replacement failed\n"); + * exit(1); + * } + * // str is: "This is a test." + * free(str); + * ~~~ + * + * @param original string to modify + * @param target string value to replace + * @param replacement string value + * @return 0 on success, -1 on error + */ +int replace_text(char *original, const char *target, const char *replacement, unsigned flags) { + char buffer[STASIS_BUFSIZ]; + char *pos = original; + char *match = NULL; + size_t original_len = strlen(original); + size_t target_len = strlen(target); + size_t rep_len = strlen(replacement); + size_t buffer_len = 0; + + if (original_len > sizeof(buffer)) { + errno = EINVAL; + SYSERROR("The original string is larger than buffer: %zu > %zu\n", original_len, sizeof(buffer)); + return -1; + } + + memset(buffer, 0, sizeof(buffer)); + if ((match = strstr(pos, target))) { + while (*pos != '\0') { + // append to buffer the bytes leading up to the match + strncat(buffer, pos, match - pos); + // position in the string jump ahead to the beginning of the match + pos = match; + + // replacement is shorter than the target + if (rep_len < target_len) { + // shrink the string + strcat(buffer, replacement); + memmove(pos, pos + target_len, strlen(pos) - target_len); + memset(pos + (strlen(pos) - target_len), 0, target_len); + } else { // replacement is longer than the target + // write the replacement value to the buffer + strcat(buffer, replacement); + // target consumed. jump to the end of the substring. + pos += target_len; + } + if (flags & REPLACE_TRUNCATE_AFTER_MATCH) { + if (strstr(pos, LINE_SEP)) { + strcat(buffer, LINE_SEP); + } + break; + } + // find more matches + if (!(match = strstr(pos, target))) { + // no more matches + // append whatever remains to the buffer + strcat(buffer, pos); + // stop + break; + } + } + } else { + return 0; + } + + buffer_len = strlen(buffer); + if (buffer_len < original_len) { + // truncate whatever remains of the original buffer + memset(original + buffer_len, 0, original_len - buffer_len); + } + // replace original with contents of buffer + strcpy(original, buffer); + return 0; +} + +/** + * Replace `target` with `replacement` in `filename` + * + * ~~~{.c} + * if (file_replace_text("/path/to/file.txt", "are", "is")) { + * fprintf(stderr, "failed to replace strings in file\n"); + * exit(1); + * } + * ~~~ + * + * @param filename path to file + * @param target string value to replace + * @param replacement string + * @return 0 on success, -1 on error + */ +int file_replace_text(const char* filename, const char* target, const char* replacement, unsigned flags) { + int result; + char buffer[STASIS_BUFSIZ]; + char tempfilename[] = "tempfileXXXXXX"; + FILE *fp; + FILE *tfp; + + fp = fopen(filename, "r"); + if (!fp) { + fprintf(stderr, "unable to open for reading: %s\n", filename); + return -1; + } + + tfp = fopen(tempfilename, "w+"); + if (!tfp) { + SYSERROR("unable to open temporary fp for writing: %s", tempfilename); + fclose(fp); + return -1; + } + + // Write modified strings to temporary file + result = 0; + while (fgets(buffer, sizeof(buffer), fp) != NULL) { + if (strstr(buffer, target)) { + if (replace_text(buffer, target, replacement, flags)) { + result = -1; + } + } + fputs(buffer, tfp); + } + fflush(tfp); + + // Replace original with modified copy + fclose(fp); + fp = fopen(filename, "w+"); + if (!fp) { + SYSERROR("unable to reopen %s for writing", filename); + return -1; + } + + // Update original file + rewind(tfp); + while (fgets(buffer, sizeof(buffer), tfp) != NULL) { + fputs(buffer, fp); + } + fclose(fp); + fclose(tfp); + + remove(tempfilename); + return result; +}
\ No newline at end of file diff --git a/src/lib/core/rules.c b/src/lib/core/rules.c new file mode 100644 index 0000000..e42ee07 --- /dev/null +++ b/src/lib/core/rules.c @@ -0,0 +1,5 @@ +// +// Created by jhunk on 12/18/23. +// + +#include "rules.h" diff --git a/src/lib/core/str.c b/src/lib/core/str.c new file mode 100644 index 0000000..868a6c7 --- /dev/null +++ b/src/lib/core/str.c @@ -0,0 +1,654 @@ +/** + * @file strings.c + */ +#include <unistd.h> +#include "str.h" + +int num_chars(const char *sptr, int ch) { + int result = 0; + for (int i = 0; sptr[i] != '\0'; i++) { + if (sptr[i] == ch) { + result++; + } + } + return result; +} + +int startswith(const char *sptr, const char *pattern) { + if (!sptr || !pattern) { + return 0; + } + for (size_t i = 0; i < strlen(pattern); i++) { + if (sptr[i] != pattern[i]) { + return 0; + } + } + return 1; +} + +int endswith(const char *sptr, const char *pattern) { + if (!sptr || !pattern) { + return 0; + } + ssize_t sptr_size = (ssize_t) strlen(sptr); + ssize_t pattern_size = (ssize_t) strlen(pattern); + + if (sptr_size == pattern_size) { + if (strcmp(sptr, pattern) == 0) { + return 1; // yes + } + return 0; // no + } + + ssize_t s = sptr_size - pattern_size; + if (s < 0) { + return 0; + } + + for (size_t p = 0 ; s < sptr_size; s++, p++) { + if (sptr[s] != pattern[p]) { + // sptr does not end with pattern + return 0; + } + } + // sptr ends with pattern + return 1; +} + +void strchrdel(char *sptr, const char *chars) { + if (sptr == NULL || chars == NULL) { + return; + } + + for (size_t i = 0; i < strlen(chars); i++) { + char ch[2] = {0}; + strncpy(ch, &chars[i], 1); + replace_text(sptr, ch, "", 0); + } +} + +char** split(char *_sptr, const char* delim, size_t max) +{ + if (_sptr == NULL || delim == NULL) { + return NULL; + } + size_t split_alloc = 0; + // Duplicate the input string and save a copy of the pointer to be freed later + char *orig = _sptr; + char *sptr = strdup(orig); + + if (!sptr) { + return NULL; + } + + // Determine how many delimiters are present + for (size_t i = 0; i < strlen(delim); i++) { + if (max && i > max) { + break; + } + split_alloc += num_chars(sptr, delim[i]); + } + + // Preallocate enough records based on the number of delimiters + char **result = calloc(split_alloc + 2, sizeof(result[0])); + if (!result) { + guard_free(sptr); + return NULL; + } + + // No delimiter, but the string was not NULL, so return the original string + if (split_alloc == 0) { + result[0] = sptr; + return result; + } + + // Separate the string into individual parts and store them in the result array + char *token = NULL; + char *sptr_tmp = sptr; + size_t pos = 0; + size_t i; + for (i = 0; (token = strsep(&sptr_tmp, delim)) != NULL; i++) { + // When max is zero, record all tokens + if (max > 0 && i == max) { + // Maximum number of splits occurred. + // Record position in string + pos = token - sptr; + break; + } + result[i] = calloc(STASIS_BUFSIZ, sizeof(char)); + if (!result[i]) { + return NULL; + } + strcpy(result[i], token); + } + + // pos is non-zero when maximum split is reached + if (pos) { + // append the remaining string contents to array + result[i] = calloc(STASIS_BUFSIZ, sizeof(char)); + if (!result[i]) { + return NULL; + } + strcpy(result[i], &orig[pos]); + } + + guard_free(sptr); + return result; +} + +char *join(char **arr, const char *separator) { + char *result = NULL; + int records = 0; + size_t total_bytes = 0; + + if (!arr || !separator) { + return NULL; + } + + for (int i = 0; arr[i] != NULL; i++) { + total_bytes += strlen(arr[i]); + records++; + } + total_bytes += (records * strlen(separator)) + 1; + + result = (char *)calloc(total_bytes, sizeof(char)); + for (int i = 0; i < records; i++) { + strcat(result, arr[i]); + if (i < (records - 1)) { + strcat(result, separator); + } + } + return result; +} + +char *join_ex(char *separator, ...) { + va_list ap; // Variadic argument list + size_t separator_len = 0; // Length of separator string + size_t size = 0; // Length of output string + size_t argc = 0; // Number of arguments ^ "..." + char **argv = NULL; // Arguments + char *current = NULL; // Current argument + char *result = NULL; // Output string + + if (separator == NULL) { + return NULL; + } + + // Initialize array + argv = calloc(argc + 1, sizeof(char **)); + if (argv == NULL) { + perror("join_ex calloc failed"); + return NULL; + } + + // Get length of the separator + separator_len = strlen(separator); + + // Process variadic arguments: + // 1. Iterate over argument list `ap` + // 2. Assign `current` with the value of argument in `ap` + // 3. Extend the `argv` array by the latest argument count `argc` + // 4. Sum the length of the argument and the `separator` passed to the function + // 5. Append `current` string to `argv` array + // 6. Update argument counter `argc` + va_start(ap, separator); + for(argc = 0; (current = va_arg(ap, char *)) != NULL; argc++) { + char **tmp = realloc(argv, (argc + 1) * sizeof(char *)); + if (tmp == NULL) { + perror("join_ex realloc failed"); + guard_free(argv); + return NULL; + } else { + argv = tmp; + } + size += strlen(current) + separator_len; + argv[argc] = strdup(current); + } + va_end(ap); + + // Generate output string + result = calloc(size + 1, sizeof(char)); + for (size_t i = 0; i < argc; i++) { + // Append argument to string + strcat(result, argv[i]); + + // Do not append a trailing separator when we reach the last argument + if (i < (argc - 1)) { + strcat(result, separator); + } + guard_free(argv[i]); + } + guard_free(argv); + + return result; +} + +char *substring_between(char *sptr, const char *delims) { + char delim_open[255] = {0}; + char delim_close[255] = {0}; + if (sptr == NULL || delims == NULL) { + return NULL; + } + + // Ensure we have enough delimiters to continue + size_t delim_count = strlen(delims); + if (delim_count < 2 || delim_count % 2 || (delim_count > (sizeof(delim_open) - 1)) != 0) { + return NULL; + } + size_t delim_take = delim_count / 2; + + // How else am I supposed to consume the first and last n chars of the string? Give me a break. + // warning: ‘__builtin___strncpy_chk’ specified bound depends on the length of the source argument + // --- + //strncpy(delim_open, delims, delim_take); + size_t i = 0; + while (i < delim_take && i < sizeof(delim_open)) { + delim_open[i] = delims[i]; + i++; + } + + //strncpy(delim_close, &delims[delim_take], delim_take); + i = 0; + while (i < delim_take && i < sizeof(delim_close)) { + delim_close[i] = delims[i + delim_take]; + i++; + } + + // Create pointers to the delimiters + char *start = strstr(sptr, delim_open); + if (start == NULL || strlen(start) == 0) { + return NULL; + } + + char *end = strstr(start + 1, delim_close); + if (end == NULL) { + return NULL; + } + + start += delim_count / 2; // ignore leading delimiter + + // Get length of the substring + size_t length = strlen(start) - strlen(end); + if (!length) { + return NULL; + } + + // Return the contents of the substring + return strndup(start, length); +} + +/* + * Comparison functions for `strsort` + */ +static int _strsort_alpha_compare(const void *a, const void *b) { + const char *aa = *(const char **)a; + const char *bb = *(const char **)b; + int result = strcmp(aa, bb); + return result; +} + +static int _strsort_numeric_compare(const void *a, const void *b) { + const char *aa = *(const char **)a; + const char *bb = *(const char **)b; + + if (isdigit(*aa) && isdigit(*bb)) { + long ia = strtol(aa, NULL, 10); + long ib = strtol(bb, NULL, 10); + + if (ia == ib) { + return 0; + } else if (ia < ib) { + return -1; + } else if (ia > ib) { + return 1; + } + } + return 0; +} + +static int _strsort_asc_compare(const void *a, const void *b) { + const char *aa = *(const char**)a; + const char *bb = *(const char**)b; + size_t len_a = strlen(aa); + size_t len_b = strlen(bb); + return len_a > len_b; +} + +/* + * Helper function for `strsortlen` + */ +static int _strsort_dsc_compare(const void *a, const void *b) { + const char *aa = *(const char**)a; + const char *bb = *(const char**)b; + size_t len_a = strlen(aa); + size_t len_b = strlen(bb); + return len_a < len_b; +} + +void strsort(char **arr, unsigned int sort_mode) { + if (arr == NULL) { + return; + } + + typedef int (*compar)(const void *, const void *); + // Default mode is alphabetic sort + compar fn = _strsort_alpha_compare; + + if (sort_mode == STASIS_SORT_LEN_DESCENDING) { + fn = _strsort_dsc_compare; + } else if (sort_mode == STASIS_SORT_LEN_ASCENDING) { + fn = _strsort_asc_compare; + } else if (sort_mode == STASIS_SORT_ALPHA) { + fn = _strsort_alpha_compare; // ^ still selectable though ^ + } else if (sort_mode == STASIS_SORT_NUMERIC) { + fn = _strsort_numeric_compare; + } + + size_t arr_size = 0; + + // Determine size of array (+ terminator) + for (size_t i = 0; arr[i] != NULL; i++) { + arr_size = i; + } + arr_size++; + + qsort(arr, arr_size, sizeof(char *), fn); +} + + +char *strstr_array(char **arr, const char *str) { + if (arr == NULL || str == NULL) { + return NULL; + } + + for (int i = 0; arr[i] != NULL; i++) { + if (strstr(arr[i], str) != NULL) { + return arr[i]; + } + } + return NULL; +} + + +char **strdeldup(char **arr) { + if (!arr) { + return NULL; + } + + size_t records; + // Determine the length of the array + for (records = 0; arr[records] != NULL; records++); + + // Allocate enough memory to store the original array contents + // (It might not have duplicate values, for example) + char **result = (char **)calloc(records + 1, sizeof(char *)); + if (!result) { + return NULL; + } + + int rec = 0; + size_t i = 0; + while(i < records) { + // Search for value in results + if (strstr_array(result, arr[i]) != NULL) { + // value already exists in results so ignore it + i++; + continue; + } + + // Store unique value + result[rec] = strdup(arr[i]); + if (!result[rec]) { + for (size_t die = 0; result[die] != NULL; die++) { + guard_free(result[die]); + } + guard_free(result); + return NULL; + } + + i++; + rec++; + } + return result; +} + +char *lstrip(char *sptr) { + char *tmp = sptr; + size_t bytes = 0; + + if (sptr == NULL) { + return NULL; + } + + while (strlen(tmp) > 1 && (isblank(*tmp) || isspace(*tmp))) { + bytes++; + tmp++; + } + if (tmp != sptr) { + memmove(sptr, sptr + bytes, strlen(sptr) - bytes); + memset((sptr + strlen(sptr)) - bytes, '\0', bytes); + } + return sptr; +} + +char *strip(char *sptr) { + if (sptr == NULL) { + return NULL; + } + + size_t len = strlen(sptr); + if (len == 0) { + return sptr; + } else if (len == 1 && (isblank(*sptr) || isspace(*sptr))) { + *sptr = '\0'; + return sptr; + } for (size_t i = len; i != 0; --i) { + if (sptr[i] == '\0') { + continue; + } + if (isspace(sptr[i]) || isblank(sptr[i])) { + sptr[i] = '\0'; + } else { + break; + } + } + return sptr; +} + +int isempty(char *sptr) { + if (sptr == NULL) { + return -1; + } + + char *tmp = sptr; + while (*tmp) { + if (!isblank(*tmp) && !isspace(*tmp) && !iscntrl(*tmp)) { + return 0; + } + tmp++; + } + return 1; +} + + +int isquoted(char *sptr) { + const char *quotes = "'\""; + + if (sptr == NULL) { + return -1; + } + + char *quote_open = strpbrk(sptr, quotes); + if (!quote_open) { + return 0; + } + char *quote_close = strpbrk(quote_open + 1, quotes); + if (!quote_close) { + return 0; + } + return 1; +} + +int isrelational(char ch) { + char symbols[] = "~!=<>"; + char *symbol = symbols; + while (*symbol != '\0') { + if (ch == *symbol) { + return 1; + } + symbol++; + } + return 0; +} + +void print_banner(const char *s, int len) { + size_t s_len = strlen(s); + if (!s_len) { + return; + } + for (size_t i = 0; i < (len / s_len); i++) { + for (size_t c = 0; c < s_len; c++) { + putchar(s[c]); + } + } + putchar('\n'); +} + +/** + * Collapse whitespace in `s`. The string is modified in place. + * @param s + * @return pointer to `s` + */ +char *normalize_space(char *s) { + size_t len; + size_t trim_pos; + int add_whitespace = 0; + char *result = s; + char *tmp; + + if (s == NULL) { + return NULL; + } + + if ((tmp = calloc(strlen(s) + 1, sizeof(char))) == NULL) { + perror("could not allocate memory for temporary string"); + return NULL; + } + char *tmp_orig = tmp; + + // count whitespace, if any + for (trim_pos = 0; isblank(s[trim_pos]); trim_pos++); + // trim whitespace from the left, if any + memmove(s, &s[trim_pos], strlen(&s[trim_pos])); + // cull bytes not part of the string after moving + len = strlen(s); + s[len - trim_pos] = '\0'; + + // Generate a new string with extra whitespace stripped out + while (*s != '\0') { + // Skip over any whitespace, but record that we encountered it + if (isblank(*s)) { + s++; + add_whitespace = 1; + continue; + } + // This gate avoids filling tmp with whitespace; we want to make our own + if (add_whitespace) { + *tmp = ' '; + tmp++; + add_whitespace = 0; + } + // Write character in s to tmp + *tmp = *s; + // Increment string pointers + s++; + tmp++; + } + + // Rewrite the input string + strcpy(result, tmp_orig); + guard_free(tmp_orig); + return result; +} + +char **strdup_array(char **array) { + char **result = NULL; + size_t elems = 0; + + // Guard + if (array == NULL) { + return NULL; + } + + // Count elements in `array` + for (elems = 0; array[elems] != NULL; elems++); + + // Create new array + result = calloc(elems + 1, sizeof(*result)); + for (size_t i = 0; i < elems; i++) { + result[i] = strdup(array[i]); + } + + return result; +} + +int strcmp_array(const char **a, const char **b) { + size_t a_len = 0; + size_t b_len = 0; + + // This could lead to false-positives depending on what the caller plans to achieve + if (a == NULL && b == NULL) { + return 0; + } else if (a == NULL) { + return -1; + } else if (b == NULL) { + return 1; + } + + // Get length of arrays + for (a_len = 0; a[a_len] != NULL; a_len++); + for (b_len = 0; b[b_len] != NULL; b_len++); + + // Check lengths are equal + if (a_len < b_len) return (int)(b_len - a_len); + else if (a_len > b_len) return (int)(a_len - b_len); + + // Compare strings in the arrays returning the total difference in bytes + int result = 0; + for (size_t ai = 0, bi = 0 ;a[ai] != NULL || b[bi] != NULL; ai++, bi++) { + int status = 0; + if ((status = strcmp(a[ai], b[bi]) != 0)) { + result += status; + } + } + return result; +} + +int isdigit_s(const char *s) { + if (!s || !strlen(s)) { + return 0; // nothing to do, fail + } + for (size_t i = 0; s[i] != '\0'; i++) { + if (isdigit(s[i]) == 0) { + return 0; // non-digit found, fail + } + } + return 1; // all digits, succeed +} + +char *tolower_s(char *s) { + for (size_t i = 0; s[i] != '\0'; i++) { + s[i] = (char)tolower(s[i]); + } + return s; +} + +char *to_short_version(const char *s) { + char *result; + result = strdup(s); + if (!result) { + return NULL; + } + strchrdel(result, "."); + return result; +} diff --git a/src/lib/core/strlist.c b/src/lib/core/strlist.c new file mode 100644 index 0000000..f0bffa8 --- /dev/null +++ b/src/lib/core/strlist.c @@ -0,0 +1,659 @@ +/** + * String array convenience functions + * @file strlist.c + */ +#include "download.h" +#include "strlist.h" +#include "utils.h" + +/** + * + * @param pStrList `StrList` + */ +void strlist_free(struct StrList **pStrList) { + if (!(*pStrList)) { + return; + } + + for (size_t i = 0; i < (*pStrList)->num_inuse; i++) { + if ((*pStrList)->data[i]) { + guard_free((*pStrList)->data[i]); + } + } + if ((*pStrList)->data) { + guard_free((*pStrList)->data); + } + guard_free((*pStrList)); +} + +/** + * Append a value to the list + * @param pStrList `StrList` + * @param str + */ +void strlist_append(struct StrList **pStrList, char *str) { + char **tmp = NULL; + + if (pStrList == NULL) { + return; + } + + tmp = realloc((*pStrList)->data, ((*pStrList)->num_alloc + 1) * sizeof(char *)); + if (tmp == NULL) { + guard_strlist_free(pStrList); + perror("failed to append to array"); + exit(1); + } else if (tmp != (*pStrList)->data) { + (*pStrList)->data = tmp; + } + (*pStrList)->data[(*pStrList)->num_inuse] = strdup(str); + (*pStrList)->data[(*pStrList)->num_alloc] = NULL; + strcpy((*pStrList)->data[(*pStrList)->num_inuse], str); + (*pStrList)->num_inuse++; + (*pStrList)->num_alloc++; +} + +static int reader_strlist_append_file(size_t lineno, char **line) { + (void)(lineno); // unused parameter + (void)(line); // unused parameter + return 0; +} + +/** + * Append lines from a local file or remote URL (HTTP/s only) + * @param pStrList + * @param path file path or HTTP/s address + * @param readerFn pointer to a reader function (use NULL to retrieve all data) + * @return 0=success 1=no data, -1=error (spmerrno set) + */ +int strlist_append_file(struct StrList *pStrList, char *_path, ReaderFn *readerFn) { + int retval = 0; + char *path = NULL; + char *filename = NULL; + char **data = NULL; + int is_url = strstr(_path, "://") != NULL; + + if (readerFn == NULL) { + readerFn = reader_strlist_append_file; + } + + path = strdup(_path); + if (path == NULL) { + retval = -1; + goto fatal; + } + + if (is_url) { + int fd; + char tempfile[PATH_MAX] = {0}; + strcpy(tempfile, "/tmp/.remote_file.XXXXXX"); + if ((fd = mkstemp(tempfile)) < 0) { + retval = -1; + goto fatal; + } + close(fd); + filename = strdup(tempfile); + long http_code = download(path, filename, NULL); + if (HTTP_ERROR(http_code)) { + retval = -1; + goto fatal; + } + } else { + filename = expandpath(path); + if (filename == NULL) { + retval = -1; + goto fatal; + } + } + + data = file_readlines(filename, 0, 0, readerFn); + if (data == NULL) { + retval = 1; + goto fatal; + } + for (size_t record = 0; data[record] != NULL; record++) { + strlist_append(&pStrList, data[record]); + guard_free(data[record]); + } + if (is_url) { + // remove temporary data + remove(filename); + } + guard_free(data); + +fatal: + guard_free(filename); + if (path != NULL) { + guard_free(path); + } + + return retval; +} + +/** + * Append the contents of `pStrList2` to `pStrList1` + * @param pStrList1 `StrList` + * @param pStrList2 `StrList` + */ +void strlist_append_strlist(struct StrList *pStrList1, struct StrList *pStrList2) { + size_t count = 0; + + if (pStrList1 == NULL || pStrList2 == NULL) { + return; + } + + count = strlist_count(pStrList2); + for (size_t i = 0; i < count; i++) { + char *item = strlist_item(pStrList2, i); + strlist_append(&pStrList1, item); + } +} + +/** + * Append the contents of an array of pointers to char + * @param pStrList `StrList` + * @param arr NULL terminated array of strings + */ + void strlist_append_array(struct StrList *pStrList, char **arr) { + if (!pStrList || !arr) { + return; + } + for (size_t i = 0; arr[i] != NULL; i++) { + strlist_append(&pStrList, arr[i]); + } + } + +/** + * Append the contents of a newline delimited string + * @param pStrList `StrList` + * @param str + * @param delim + */ + void strlist_append_tokenize(struct StrList *pStrList, char *str, char *delim) { + char **token; + if (!str || !delim) { + return; + } + + char *tmp = strdup(str); + token = split(tmp, delim, 0); + if (token) { + for (size_t i = 0; token[i] != NULL; i++) { + lstrip(token[i]); + strlist_append(&pStrList, token[i]); + } + GENERIC_ARRAY_FREE(token); + } + guard_free(tmp); + } + +/** + * Produce a new copy of a `StrList` + * @param pStrList `StrList` + * @return `StrList` copy + */ +struct StrList *strlist_copy(struct StrList *pStrList) { + struct StrList *result; + if (pStrList == NULL) { + return NULL; + } + + result = strlist_init(); + if (!result) { + return NULL; + } + + for (size_t i = 0; i < strlist_count(pStrList); i++) { + strlist_append(&result, strlist_item(pStrList, i)); + } + return result; +} + +/** + * Remove a record by index from a `StrList` + * @param pStrList + * @param index + */ +void strlist_remove(struct StrList *pStrList, size_t index) { + size_t count = strlist_count(pStrList); + if (count == 0) { + return; + } + if (pStrList->data[index] != NULL) { + for (size_t i = index; i < count; i++) { + pStrList->data[i] = pStrList->data[i + 1]; + } + if (pStrList->num_inuse) { + pStrList->num_inuse--; + } + } +} + +/** + * Compare two `StrList`s + * @param a `StrList` structure + * @param b `StrList` structure + * @return same=0, different=1, error=-1 (a is NULL), -2 (b is NULL) + */ +int strlist_cmp(struct StrList *a, struct StrList *b) { + if (a == NULL) { + return -1; + } + + if (b == NULL) { + return -2; + } + + if (a->num_alloc != b->num_alloc || a->num_inuse != b->num_inuse) { + return 1; + } + + + for (size_t i = 0; i < strlist_count(a); i++) { + if (strcmp(strlist_item(a, i), strlist_item(b, i)) != 0) { + return 1; + } + } + + return 0; +} + +/** + * Sort a `StrList` by `mode` + * @param pStrList + * @param mode Available modes: `STRLIST_DEFAULT` (alphabetic), `STRLIST_ASC` (ascending), `STRLIST_DSC` (descending) + */ +void strlist_sort(struct StrList *pStrList, unsigned int mode) { + if (pStrList == NULL) { + return; + } + strsort(pStrList->data, mode); +} + +/** + * Reverse the order of a `StrList` + * @param pStrList + */ +void strlist_reverse(struct StrList *pStrList) { + char *tmp = NULL; + size_t i = 0; + size_t j = 0; + + if (pStrList == NULL) { + return; + } + + j = pStrList->num_inuse - 1; + for (i = 0; i < j; i++) { + tmp = pStrList->data[i]; + pStrList->data[i] = pStrList->data[j]; + pStrList->data[j] = tmp; + j--; + } +} + +/** + * Get the count of values stored in a `StrList` + * @param pStrList + * @return + */ +size_t strlist_count(struct StrList *pStrList) { + size_t result; + if (pStrList != NULL) { + result = pStrList->num_inuse; + } else { + result = 0; + } + return result; +} + +/** + * Set value at index + * @param pStrList + * @param value string + * @return + */ +void strlist_set(struct StrList **pStrList, size_t index, char *value) { + char *tmp = NULL; + if (*pStrList == NULL || index > strlist_count(*pStrList)) { + strlist_errno = STRLIST_E_OUT_OF_RANGE; + return; + } + + if (value == NULL) { + guard_free((*pStrList)->data[index]); + } else { + tmp = realloc((*pStrList)->data[index], (strlen(value) + 1) * sizeof(char *)); + if (!tmp) { + perror("realloc strlist_set replacement value"); + return; + } else if (tmp != (*pStrList)->data[index]) { + (*pStrList)->data[index] = tmp; + } + + memset((*pStrList)->data[index], '\0', strlen(value) + 1); + strcpy((*pStrList)->data[index], value); + } +} + +const char *strlist_error_msgs[] = { + "success", + "index out of range", + "invalid value for type", + "unknown error", +}; +int strlist_errno = 0; + +void strlist_set_error(int flag) { + strlist_errno = flag; +} + +const char *strlist_get_error(int flag) { + if (flag < STRLIST_E_SUCCESS || flag > STRLIST_E_UNKNOWN) { + return strlist_error_msgs[STRLIST_E_UNKNOWN]; + } + return strlist_error_msgs[flag]; +} + +void strlist_clear_error() { + strlist_errno = STRLIST_E_SUCCESS; +} + +/** + * Retrieve data from a `StrList` + * @param pStrList + * @param index + * @return string + */ +char *strlist_item(struct StrList *pStrList, size_t index) { + if (pStrList && pStrList->data && pStrList->data[index]) { + return pStrList->data[index]; + } + return NULL; +} + +/** + * Alias of `strlist_item` + * @param pStrList + * @param index + * @return string + */ +char *strlist_item_as_str(struct StrList *pStrList, size_t index) { + return strlist_item(pStrList, index); +} + +/** + * Convert value at index to `char` + * @param pStrList + * @param index + * @return `char` + */ +char strlist_item_as_char(struct StrList *pStrList, size_t index) { + char *error_p; + char result; + + strlist_clear_error(); + result = (char) strtol(strlist_item(pStrList, index), &error_p, 10); + if (!result && error_p && *error_p != 0) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return 0; + } + error_p = NULL; + return result; +} + +/** + * Convert value at index to `unsigned char` + * @param pStrList + * @param index + * @return `unsigned char` + */ +unsigned char strlist_item_as_uchar(struct StrList *pStrList, size_t index) { + char *error_p; + unsigned char result; + + strlist_clear_error(); + result = (unsigned char) strtoul(strlist_item(pStrList, index), &error_p, 10); + if (!result && error_p && *error_p != 0) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return 0; + } + error_p = NULL; + return result; +} + +/** + * Convert value at index to `short` + * @param pStrList + * @param index + * @return `short` + */ +short strlist_item_as_short(struct StrList *pStrList, size_t index) { + char *error_p; + short result; + + strlist_clear_error(); + result = (short) strtol(strlist_item(pStrList, index), &error_p, 10); + if (!result && error_p && *error_p != 0) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return 0; + } + error_p = NULL; + return result; +} + +/** + * Convert value at index to `unsigned short` + * @param pStrList + * @param index + * @return `unsigned short` + */ +unsigned short strlist_item_as_ushort(struct StrList *pStrList, size_t index) { + char *error_p; + unsigned short result; + + strlist_clear_error(); + result = (unsigned short) strtoul(strlist_item(pStrList, index), &error_p, 10); + if (!result && error_p && *error_p != 0) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return 0; + } + error_p = NULL; + return result; +} + +/** + * Convert value at index to `int` + * @param pStrList + * @param index + * @return `int` + */ +int strlist_item_as_int(struct StrList *pStrList, size_t index) { + char *error_p; + int result; + + strlist_clear_error(); + result = (int) strtol(strlist_item(pStrList, index), &error_p, 10); + if (!result && error_p && *error_p != 0) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return 0; + } + error_p = NULL; + return result; +} + +/** + * Convert value at index to `unsigned int` + * @param pStrList + * @param index + * @return `unsigned int` + */ +unsigned int strlist_item_as_uint(struct StrList *pStrList, size_t index) { + char *error_p; + unsigned int result; + + strlist_clear_error(); + result = (unsigned int) strtoul(strlist_item(pStrList, index), &error_p, 10); + if (!result && error_p && *error_p != 0) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return 0; + } + error_p = NULL; + return result; +} + +/** + * Convert value at index to `long` + * @param pStrList + * @param index + * @return `long` + */ +long strlist_item_as_long(struct StrList *pStrList, size_t index) { + char *error_p; + long result; + + strlist_clear_error(); + result = (long) strtol(strlist_item(pStrList, index), &error_p, 10); + if (!result && error_p && *error_p != 0) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return 0; + } + error_p = NULL; + return result; +} + +/** + * Convert value at index to `unsigned long` + * @param pStrList + * @param index + * @return `unsigned long` + */ +unsigned long strlist_item_as_ulong(struct StrList *pStrList, size_t index) { + char *error_p; + unsigned long result; + + strlist_clear_error(); + result = (unsigned long) strtoul(strlist_item(pStrList, index), &error_p, 10); + if (!result && error_p && *error_p != 0) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return 0; + } + error_p = NULL; + return result; +} + +/** + * Convert value at index to `long long` + * @param pStrList + * @param index + * @return `long long` + */ +long long strlist_item_as_long_long(struct StrList *pStrList, size_t index) { + char *error_p; + long long result; + + strlist_clear_error(); + result = (long long) strtoll(strlist_item(pStrList, index), &error_p, 10); + if (!result && error_p && *error_p != 0) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return 0; + } + error_p = NULL; + return result; +} + +/** + * Convert value at index to `unsigned long long` + * @param pStrList + * @param index + * @return `unsigned long long` + */ +unsigned long long strlist_item_as_ulong_long(struct StrList *pStrList, size_t index) { + char *error_p; + unsigned long long result; + + strlist_clear_error(); + result = (unsigned long long) strtol(strlist_item(pStrList, index), &error_p, 10); + if (!result && error_p && *error_p != 0) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return 0; + } + error_p = NULL; + return result; +} + +/** + * Convert value at index to `float` + * @param pStrList + * @param index + * @return `float` + */ +float strlist_item_as_float(struct StrList *pStrList, size_t index) { + char *error_p; + float result; + + strlist_clear_error(); + result = (float) strtof(strlist_item(pStrList, index), &error_p); + if (!result && error_p && *error_p != 0) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return 0; + } + error_p = NULL; + return result; +} + +/** + * Convert value at index to `double` + * @param pStrList + * @param index + * @return `double` + */ +double strlist_item_as_double(struct StrList *pStrList, size_t index) { + char *error_p; + double result; + + strlist_clear_error(); + result = (double) strtod(strlist_item(pStrList, index), &error_p); + if (!result && error_p && *error_p != 0) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return 0; + } + error_p = NULL; + return result; +} + +/** + * Convert value at index to `long double` + * @param pStrList + * @param index + * @return `long double` + */ +long double strlist_item_as_long_double(struct StrList *pStrList, size_t index) { + char *error_p; + long double result; + + strlist_clear_error(); + result = (long double) strtold(strlist_item(pStrList, index), &error_p); + if (!result && error_p && *error_p != 0) { + strlist_set_error(STRLIST_E_INVALID_VALUE); + return 0; + } + error_p = NULL; + return result; +} + +/** + * Initialize an empty `StrList` + * @return `StrList` + */ +struct StrList *strlist_init() { + struct StrList *pStrList = calloc(1, sizeof(struct StrList)); + if (pStrList == NULL) { + perror("failed to allocate array"); + return NULL; + } + pStrList->num_inuse = 0; + pStrList->num_alloc = 1; + pStrList->data = calloc(pStrList->num_alloc, sizeof(char *)); + return pStrList; +} diff --git a/src/lib/core/system.c b/src/lib/core/system.c new file mode 100644 index 0000000..4e605ec --- /dev/null +++ b/src/lib/core/system.c @@ -0,0 +1,173 @@ +#include "system.h" +#include "core.h" + +int shell(struct Process *proc, char *args) { + struct Process selfproc; + pid_t pid; + pid_t status; + status = 0; + errno = 0; + + if (!proc) { + // provide our own proc structure + // albeit not accessible to the user + memset(&selfproc, 0, sizeof(selfproc)); + proc = &selfproc; + } + + if (!args) { + proc->returncode = -1; + return -1; + } + + FILE *tp = NULL; + char *t_name; + t_name = xmkstemp(&tp, "w"); + if (!t_name || !tp) { + return -1; + } + + fprintf(tp, "#!/bin/bash\n%s\n", args); + fflush(tp); + fclose(tp); + + // Set the script's permissions so that only the calling user can use it + // This should help prevent eavesdropping if keys are applied in plain-text + // somewhere. + chmod(t_name, 0700); + + pid = fork(); + if (pid == -1) { + fprintf(stderr, "fork failed\n"); + exit(1); + } else if (pid == 0) { + FILE *fp_out = NULL; + FILE *fp_err = NULL; + + if (strlen(proc->f_stdout)) { + fp_out = freopen(proc->f_stdout, "w+", stdout); + if (!fp_out) { + fprintf(stderr, "Unable to redirect stdout to %s: %s\n", proc->f_stdout, strerror(errno)); + exit(1); + } + } + + if (strlen(proc->f_stderr)) { + if (!proc->redirect_stderr) { + fp_err = freopen(proc->f_stderr, "w+", stderr); + if (!fp_err) { + fprintf(stderr, "Unable to redirect stderr to %s: %s\n", proc->f_stdout, strerror(errno)); + exit(1); + } + } + } + + if (proc->redirect_stderr) { + if (fp_err) { + fclose(fp_err); + fclose(stderr); + } + if (dup2(fileno(stdout), fileno(stderr)) < 0) { + fprintf(stderr, "Unable to redirect stderr to stdout: %s\n", strerror(errno)); + exit(1); + } + } + + return execl("/bin/bash", "bash", "--norc", t_name, (char *) NULL); + } else { + if (waitpid(pid, &status, WUNTRACED) > 0) { + if (WIFEXITED(status) && WEXITSTATUS(status)) { + if (WEXITSTATUS(status) == 127) { + fprintf(stderr, "execv failed\n"); + } + } else if (WIFSIGNALED(status)) { + fprintf(stderr, "signal received: %d\n", WIFSIGNALED(status)); + } + } else { + fprintf(stderr, "waitpid() failed\n"); + } + } + + if (!access(t_name, F_OK)) { + remove(t_name); + } + + proc->returncode = status; + guard_free(t_name); + return WEXITSTATUS(status); +} + +int shell_safe(struct Process *proc, char *args) { + FILE *fp; + char buf[1024] = {0}; + int result; + + char *invalid_ch = strpbrk(args, STASIS_SHELL_SAFE_RESTRICT); + if (invalid_ch) { + args = NULL; + } + + result = shell(proc, args); + if (strlen(proc->f_stdout)) { + fp = fopen(proc->f_stdout, "r"); + if (fp) { + while (fgets(buf, sizeof(buf) - 1, fp)) { + fprintf(stdout, "%s", buf); + buf[0] = '\0'; + } + fclose(fp); + fp = NULL; + } + } + if (strlen(proc->f_stderr)) { + fp = fopen(proc->f_stderr, "r"); + if (fp) { + while (fgets(buf, sizeof(buf) - 1, fp)) { + fprintf(stderr, "%s", buf); + buf[0] = '\0'; + } + fclose(fp); + fp = NULL; + } + } + return result; +} + +char *shell_output(const char *command, int *status) { + const size_t initial_size = STASIS_BUFSIZ; + size_t current_size = initial_size; + char *result = NULL; + char line[STASIS_BUFSIZ]; + FILE *pp; + + errno = 0; + *status = 0; + pp = popen(command, "r"); + if (!pp) { + *status = -1; + return NULL; + } + + if (errno) { + *status = 1; + } + result = calloc(initial_size, sizeof(result)); + memset(line, 0, sizeof(line)); + while (fread(line, sizeof(char), sizeof(line) - 1, pp) != 0) { + size_t result_len = strlen(result); + size_t need_realloc = (result_len + strlen(line)) > current_size; + if (need_realloc) { + current_size += initial_size; + char *tmp = realloc(result, sizeof(*result) * current_size); + if (!tmp) { + return NULL; + } else if (tmp != result) { + result = tmp; + } + } + strcat(result, line); + memset(line, 0, sizeof(line)); + } + *status = pclose(pp); + return result; +} diff --git a/src/lib/core/template.c b/src/lib/core/template.c new file mode 100644 index 0000000..a412fa8 --- /dev/null +++ b/src/lib/core/template.c @@ -0,0 +1,318 @@ +// +// Created by jhunk on 12/17/23. +// + +#include "template.h" + +#include <stdio.h> +#include <string.h> +#include <stdlib.h> +#include <ctype.h> + + +struct tpl_item { + char *key; + char **ptr; +}; +struct tpl_item *tpl_pool[1024] = {0}; +unsigned tpl_pool_used = 0; +struct tplfunc_frame *tpl_pool_func[1024] = {0}; +unsigned tpl_pool_func_used = 0; + +extern void tpl_reset() { + tpl_free(); + tpl_pool_used = 0; + tpl_pool_func_used = 0; +} + +void tpl_register_func(char *key, void *tplfunc_ptr, int argc, void *data_in) { + struct tplfunc_frame *frame = calloc(1, sizeof(*frame)); + frame->key = strdup(key); + frame->argc = argc; + frame->func = tplfunc_ptr; + frame->data_in = data_in; + + tpl_pool_func[tpl_pool_func_used] = frame; + tpl_pool_func_used++; +} + +int tpl_key_exists(char *key) { + for (size_t i = 0; i < tpl_pool_used; i++) { + if (tpl_pool[i]->key) { + if (!strcmp(tpl_pool[i]->key, key)) { + return true; + } + } + } + return false; +} + +void tpl_register(char *key, char **ptr) { + struct tpl_item *item = NULL; + int replacing = 0; + + if (tpl_key_exists(key)) { + for (size_t i = 0; i < tpl_pool_used; i++) { + if (tpl_pool[i]->key) { + if (!strcmp(tpl_pool[i]->key, key)) { + item = tpl_pool[i]; + break; + } + } + } + replacing = 1; + } else { + item = calloc(1, sizeof(*item)); + item->key = strdup(key); + } + + if (!item) { + SYSERROR("unable to register tpl_item for %s", key); + exit(1); + } + + item->ptr = ptr; + if (!replacing) { + tpl_pool[tpl_pool_used] = item; + tpl_pool_used++; + } +} + +void tpl_free() { + for (unsigned i = 0; i < tpl_pool_used; i++) { + struct tpl_item *item = tpl_pool[i]; + if (item) { + if (item->key) { +#ifdef DEBUG + SYSERROR("freeing template item key: %s", item->key); +#endif + guard_free(item->key); + } +#ifdef DEBUG + SYSERROR("freeing template item: %p", item); +#endif + item->ptr = NULL; + } + guard_free(item); + } + for (unsigned i = 0; i < tpl_pool_func_used; i++) { + struct tplfunc_frame *item = tpl_pool_func[i]; + guard_free(item->key); + guard_free(item); + } +} + +char *tpl_getval(char *key) { + char *result = NULL; + for (size_t i = 0; i < tpl_pool_used; i++) { + if (tpl_pool[i]->key) { + if (!strcmp(tpl_pool[i]->key, key)) { + result = *tpl_pool[i]->ptr; + break; + } + } + } + return result; +} + +struct tplfunc_frame *tpl_getfunc(char *key) { + struct tplfunc_frame *result = NULL; + for (size_t i = 0; i < tpl_pool_func_used; i++) { + if (tpl_pool_func[i]->key) { + if (!strcmp(tpl_pool_func[i]->key, key)) { + result = tpl_pool_func[i]; + break; + } + } + } + return result; +} + +static int grow(size_t z, size_t *output_bytes, char **output) { + if (z >= *output_bytes) { + size_t new_size = *output_bytes + z + 1; +#ifdef DEBUG + fprintf(stderr, "template output buffer new size: %zu\n", new_size); +#endif + char *tmp = realloc(*output, new_size); + if (!tmp) { + perror("realloc failed"); + return -1; + } else if (tmp != *output) { + *output = tmp; + } + *output_bytes = new_size; + } + return 0; +} + +char *tpl_render(char *str) { + if (!str) { + return NULL; + } else if (!strlen(str)) { + return strdup(""); + } + size_t output_bytes = 1024 + strlen(str); // TODO: Is grow working correctly? + char *output = NULL; + char *b_close = NULL; + char *pos = NULL; + pos = str; + + output = calloc(output_bytes, sizeof(*output)); + if (!output) { + perror("unable to allocate output buffer"); + return NULL; + } + + for (size_t off = 0, z = 0; off < strlen(str); off++) { + char key[255] = {0}; + char *value = NULL; + + memset(key, 0, sizeof(key)); + grow(z, &output_bytes, &output); + // At opening brace + if (!strncmp(&pos[off], "{{", 2)) { + // Scan until key is reached + while (!isalnum(pos[off])) { + off++; + } + + // Read key name + size_t key_len = 0; + while (isalnum(pos[off]) || pos[off] != '}') { + if (isspace(pos[off]) || isblank(pos[off])) { + // skip whitespace in key + off++; + continue; + } + key[key_len] = pos[off]; + key_len++; + off++; + } + + char *type_stop = NULL; + type_stop = strchr(key, ':'); + + int do_env = 0; + int do_func = 0; + if (type_stop) { + if (!strncmp(key, "env", type_stop - key)) { + do_env = 1; + } else if (!strncmp(key, "func", type_stop - key)) { + do_func = 1; + } + } + + // Find closing brace + b_close = strstr(&pos[off], "}}"); + if (!b_close) { + fprintf(stderr, "error while templating '%s'\n\nunbalanced brace at position %zu\n", str, z); + return NULL; + } else { + // Jump past closing brace + off = ((b_close + 2) - pos); + } + + if (do_env) { // {{ env:VAR }} + char *k = type_stop + 1; + size_t klen = strlen(k); + memmove(key, k, klen); + key[klen] = 0; + char *env_val = getenv(key); + value = strdup(env_val ? env_val : ""); + } else if (do_func) { // {{ func:NAME(a, ...) }} + char func_name_temp[STASIS_NAME_MAX] = {0}; + strcpy(func_name_temp, type_stop + 1); + char *param_begin = strchr(func_name_temp, '('); + if (!param_begin) { + fprintf(stderr, "At position %zu in %s\nfunction name must be followed by a '('\n", off, key); + guard_free(output); + return NULL; + } + *param_begin = 0; + param_begin++; + char *param_end = strrchr(param_begin, ')'); + if (!param_end) { + fprintf(stderr, "At position %zu in %s\nfunction arguments must be closed with a ')'\n", off, key); + guard_free(output); + return NULL; + } + *param_end = 0; + char *k = func_name_temp; + char **params = split(param_begin, ",", 0); + int params_count; + for (params_count = 0; params[params_count] != NULL; params_count++); + + struct tplfunc_frame *frame = tpl_getfunc(k); + if (params_count > frame->argc || params_count < frame->argc) { + fprintf(stderr, "At position %zu in %s\nIncorrect number of arguments for function: %s (expected %d, got %d)\n", off, key, frame->key, frame->argc, params_count); + value = strdup(""); + } else { + for (size_t p = 0; p < sizeof(frame->argv) / sizeof(*frame->argv) && params[p] != NULL; p++) { + lstrip(params[p]); + strip(params[p]); + frame->argv[p].t_char_ptr = params[p]; + } + char *func_result = NULL; + int func_status = 0; + if ((func_status = frame->func(frame, &func_result))) { + fprintf(stderr, "%s returned non-zero status: %d\n", frame->key, func_status); + } + value = strdup(func_result ? func_result : ""); + guard_free(func_result); + } + GENERIC_ARRAY_FREE(params); + } else { + // Read replacement value + value = strdup(tpl_getval(key) ? tpl_getval(key) : ""); + } + } + + if (value) { + // Set output iterator to end of replacement value + z += strlen(value); + + // Append replacement value + grow(z, &output_bytes, &output); + strcat(output, value); + guard_free(value); + output[z] = 0; + } + +#ifdef DEBUG + fprintf(stderr, "z=%zu, output_bytes=%zu\n", z, output_bytes); +#endif + output[z] = pos[off]; + z++; + } +#ifdef DEBUG + fprintf(stderr, "template output length: %zu\n", strlen(output)); + fprintf(stderr, "template output bytes: %zu\n", output_bytes); +#endif + return output; +} + +int tpl_render_to_file(char *str, const char *filename) { + char *result; + FILE *fp; + + // Render the input string + result = tpl_render(str); + if (!result) { + return -1; + } + + // Open the destination file for writing + fp = fopen(filename, "w+"); + if (!fp) { + guard_free(result); + return -1; + } + + // Write rendered string to file + fprintf(fp, "%s", result); + fclose(fp); + + guard_free(result); + return 0; +}
\ No newline at end of file diff --git a/src/lib/core/template_func_proto.c b/src/lib/core/template_func_proto.c new file mode 100644 index 0000000..3305b4d --- /dev/null +++ b/src/lib/core/template_func_proto.c @@ -0,0 +1,160 @@ +#include "template_func_proto.h" +#include "delivery.h" +#include "github.h" + +int get_github_release_notes_tplfunc_entrypoint(void *frame, void *data_out) { + int result; + char **output = (char **) data_out; + struct tplfunc_frame *f = (struct tplfunc_frame *) frame; + char *api_token = getenv("STASIS_GH_TOKEN"); + if (!api_token) { + api_token = getenv("GITHUB_TOKEN"); + } + result = get_github_release_notes(api_token ? api_token : "anonymous", + (const char *) f->argv[0].t_char_ptr, + (const char *) f->argv[1].t_char_ptr, + (const char *) f->argv[2].t_char_ptr, + output); + return result; +} + +int get_github_release_notes_auto_tplfunc_entrypoint(void *frame, void *data_out) { + int result = 0; + char **output = (char **) data_out; + struct tplfunc_frame *f = (struct tplfunc_frame *) frame; + char *api_token = getenv("STASIS_GH_TOKEN"); + if (!api_token) { + api_token = getenv("GITHUB_TOKEN"); + } + + const struct Delivery *ctx = (struct Delivery *) f->data_in; + struct StrList *notes_list = strlist_init(); + for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(*ctx->tests); i++) { + // Get test context + const struct Test *test = &ctx->tests[i]; + if (test->name && test->version && test->repository) { + char *repository = strdup(test->repository); + char *match = strstr(repository, "spacetelescope/"); + // Cull repository URL + if (match) { + replace_text(repository, "https://github.com/", "", 0); + if (endswith(repository, ".git")) { + replace_text(repository, ".git", "", 0); + } + // Record release notes for version relative to HEAD + // Using HEAD, GitHub returns the previous tag + char *note = NULL; + char h1_title[NAME_MAX] = {0}; + sprintf(h1_title, "# %s", test->name); + strlist_append(¬es_list, h1_title); + result += get_github_release_notes(api_token ? api_token : "anonymous", + repository, + test->version, + "HEAD", + ¬e); + if (note) { + strlist_append(¬es_list, note); + guard_free(note); + } + guard_free(repository); + } + } + } + // Return all notes as a single string + if (strlist_count(notes_list)) { + *output = join(notes_list->data, "\n\n"); + } + guard_strlist_free(¬es_list); + + return result; +} + +int get_junitxml_file_entrypoint(void *frame, void *data_out) { + int result = 0; + char **output = (char **) data_out; + struct tplfunc_frame *f = (struct tplfunc_frame *) frame; + const struct Delivery *ctx = (const struct Delivery *) f->data_in; + + char cwd[PATH_MAX] = {0}; + if (!getcwd(cwd, PATH_MAX - 1)) { + SYSERROR("unable to determine current working directory: %s", strerror(errno)); + return -1; + } + char nametmp[PATH_MAX] = {0}; + strcpy(nametmp, cwd); + char *name = path_basename(nametmp); + + *output = calloc(PATH_MAX, sizeof(**output)); + if (!*output) { + SYSERROR("failed to allocate output string: %s", strerror(errno)); + return -1; + } + sprintf(*output, "%s/results-%s-%s.xml", ctx->storage.results_dir, name, ctx->info.release_name); + + return result; +} + +int get_basetemp_dir_entrypoint(void *frame, void *data_out) { + int result = 0; + char **output = (char **) data_out; + struct tplfunc_frame *f = (struct tplfunc_frame *) frame; + const struct Delivery *ctx = (const struct Delivery *) f->data_in; + + char cwd[PATH_MAX] = {0}; + if (!getcwd(cwd, PATH_MAX - 1)) { + SYSERROR("unable to determine current working directory: %s", strerror(errno)); + return -1; + } + char nametmp[PATH_MAX] = {0}; + strcpy(nametmp, cwd); + char *name = path_basename(nametmp); + + *output = calloc(PATH_MAX, sizeof(**output)); + if (!*output) { + SYSERROR("failed to allocate output string: %s", strerror(errno)); + return -1; + } + sprintf(*output, "%s/truth-%s-%s", ctx->storage.tmpdir, name, ctx->info.release_name); + + return result; +} + +int tox_run_entrypoint(void *frame, void *data_out) { + char **output = (char **) data_out; + struct tplfunc_frame *f = (struct tplfunc_frame *) frame; + const struct Delivery *ctx = (const struct Delivery *) f->data_in; + + // Apply workaround for tox positional arguments + char *toxconf = NULL; + if (!access("tox.ini", F_OK)) { + if (!fix_tox_conf("tox.ini", &toxconf)) { + msg(STASIS_MSG_L3, "Fixing tox positional arguments\n"); + *output = calloc(STASIS_BUFSIZ, sizeof(**output)); + if (!*output) { + return -1; + } + char *basetemp_path = NULL; + if (get_basetemp_dir_entrypoint(f, &basetemp_path)) { + return -2; + } + char *jxml_path = NULL; + if (get_junitxml_file_entrypoint(f, &jxml_path)) { + guard_free(basetemp_path); + return -3; + } + const char *tox_target = f->argv[0].t_char_ptr; + const char *pytest_args = f->argv[1].t_char_ptr; + if (isempty(toxconf) || !strcmp(toxconf, "/")) { + SYSERROR("Unsafe toxconf path: '%s'", toxconf); + guard_free(basetemp_path); + guard_free(jxml_path); + return -4; + } + snprintf(*output, STASIS_BUFSIZ - 1, "\npip install tox && (tox -e py%s%s -c %s --root . -- --basetemp=\"%s\" --junitxml=\"%s\" %s ; rm -f '%s')\n", ctx->meta.python_compact, tox_target, toxconf, basetemp_path, jxml_path, pytest_args ? pytest_args : "", toxconf); + + guard_free(jxml_path); + guard_free(basetemp_path); + } + } + return 0; +}
\ No newline at end of file diff --git a/src/lib/core/utils.c b/src/lib/core/utils.c new file mode 100644 index 0000000..89950df --- /dev/null +++ b/src/lib/core/utils.c @@ -0,0 +1,820 @@ +#include <stdarg.h> +#include "core.h" +#include "utils.h" + +char *dirstack[STASIS_DIRSTACK_MAX]; +const ssize_t dirstack_max = sizeof(dirstack) / sizeof(dirstack[0]); +ssize_t dirstack_len = 0; + +int pushd(const char *path) { + if (access(path, F_OK)) { + // the requested path doesn't exist. + return -1; + } + if (dirstack_len + 1 > dirstack_max) { + // the stack is full + return -1; + } + + dirstack[dirstack_len] = realpath(".", NULL); + dirstack_len++; + return chdir(path); +} + +int popd() { + int result = -1; + if (dirstack_len - 1 < 0) { + return result; + } + dirstack_len--; + result = chdir(dirstack[dirstack_len]); + guard_free(dirstack[dirstack_len]); + return result; +} + +int rmtree(char *_path) { + int status = 0; + char path[PATH_MAX] = {0}; + strncpy(path, _path, sizeof(path) - 1); + DIR *dir; + struct dirent *d_entity; + + dir = opendir(path); + if (!dir) { + return 1; + } + + while ((d_entity = readdir(dir)) != NULL) { + char abspath[PATH_MAX] = {0}; + strcat(abspath, path); + strcat(abspath, DIR_SEP); + strcat(abspath, d_entity->d_name); + + if (!strcmp(d_entity->d_name, ".") || !strcmp(d_entity->d_name, "..") || !strcmp(abspath, path)) { + continue; + } + + // Test for sufficient privilege + if (access(abspath, F_OK) < 0 && errno == EACCES) { + continue; + } + + // Push directories on to the stack first + if (d_entity->d_type == DT_DIR) { + rmtree(abspath); + } else { + remove(abspath); + } + } + closedir(dir); + + if (access(path, F_OK) == 0) { + remove(path); + } + return status; +} + +char *expandpath(const char *_path) { + if (_path == NULL) { + return NULL; + } + const char *homes[] = { + "HOME", + "USERPROFILE", + }; + char home[PATH_MAX]; + char tmp[PATH_MAX]; + char *ptmp = tmp; + char result[PATH_MAX]; + char *sep = NULL; + + memset(home, '\0', sizeof(home)); + memset(ptmp, '\0', sizeof(tmp)); + memset(result, '\0', sizeof(result)); + + strncpy(ptmp, _path, PATH_MAX - 1); + + // Check whether there's a reason to continue processing the string + if (*ptmp != '~') { + return strdup(ptmp); + } + + // Remove tilde from the string and shift its contents to the left + strchrdel(ptmp, "~"); + + // Figure out where the user's home directory resides + for (size_t i = 0; i < sizeof(homes) / sizeof(*homes); i++) { + char *tmphome; + if ((tmphome = getenv(homes[i])) != NULL) { + strncpy(home, tmphome, PATH_MAX - 1); + break; + } + } + + // A broken runtime environment means we can't do anything else here + if (isempty(home)) { + return NULL; + } + + // Scan the path for a directory separator + if ((sep = strpbrk(ptmp, "/\\")) != NULL) { + // Jump past it + ptmp = sep + 1; + } + + // Construct the new path + strncat(result, home, sizeof(result) - strlen(home) + 1); + if (sep) { + strncat(result, DIR_SEP, sizeof(result) - strlen(home) + 1); + strncat(result, ptmp, sizeof(result) - strlen(home) + 1); + } + + return strdup(result); +} + +char *path_basename(char *path) { + char *result = NULL; + char *last = NULL; + + if ((last = strrchr(path, '/')) == NULL) { + return path; + } + // Perform a lookahead ensuring the string is valid beyond the last separator + if (last++ != NULL) { + result = last; + } + + return result; +} + +char *path_dirname(char *path) { + if (!path) { + return ""; + } + if (strlen(path) == 1 && *path == '/') { + return "/"; + } + char *pos = strrchr(path, '/'); + if (!pos) { + return "."; + } + *pos = '\0'; + + return path; +} + +char **file_readlines(const char *filename, size_t start, size_t limit, ReaderFn *readerFn) { + FILE *fp = NULL; + char **result = NULL; + char *buffer = NULL; + size_t lines = 0; + int use_stdin = 0; + + if (strcmp(filename, "-") == 0) { + use_stdin = 1; + } + + if (use_stdin) { + fp = stdin; + } else { + fp = fopen(filename, "r"); + } + + if (fp == NULL) { + perror(filename); + SYSERROR("failed to open %s for reading", filename); + return NULL; + } + + // Allocate buffer + if ((buffer = calloc(STASIS_BUFSIZ, sizeof(char))) == NULL) { + SYSERROR("unable to allocate %d bytes for buffer", STASIS_BUFSIZ); + if (!use_stdin) { + fclose(fp); + } + return NULL; + } + + // count number the of lines in the file + while ((fgets(buffer, STASIS_BUFSIZ - 1, fp)) != NULL) { + lines++; + } + + if (!lines) { + guard_free(buffer); + if (!use_stdin) { + fclose(fp); + } + return NULL; + } + + rewind(fp); + + // Handle invalid start offset + if (start > lines) { + start = 0; + } + + // Adjust line count when start offset is non-zero + if (start != 0 && start < lines) { + lines -= start; + } + + + // Handle minimum and maximum limits + if (limit == 0 || limit > lines) { + limit = lines; + } + + // Populate results array + result = calloc(limit + 1, sizeof(char *)); + for (size_t i = start; i < limit; i++) { + if (i < start) { + continue; + } + + if (fgets(buffer, STASIS_BUFSIZ - 1, fp) == NULL) { + break; + } + + if (readerFn != NULL) { + int status = readerFn(i - start, &buffer); + // A status greater than zero indicates we should ignore this line entirely and "continue" + // A status less than zero indicates we should "break" + // A zero status proceeds normally + if (status > 0) { + i--; + continue; + } else if (status < 0) { + break; + } + } + result[i] = strdup(buffer); + memset(buffer, '\0', STASIS_BUFSIZ); + } + + guard_free(buffer); + if (!use_stdin) { + fclose(fp); + } + return result; +} + +char *find_program(const char *name) { + static char result[PATH_MAX] = {0}; + char *_env_path = getenv(PATH_ENV_VAR); + if (!_env_path) { + errno = EINVAL; + return NULL; + } + char *path = strdup(_env_path); + char *path_orig = path; + char *path_elem = NULL; + + if (!path) { + errno = ENOMEM; + return NULL; + } + + result[0] = '\0'; + while ((path_elem = strsep(&path, PATH_SEP))) { + char abspath[PATH_MAX] = {0}; + strcat(abspath, path_elem); + strcat(abspath, DIR_SEP); + strcat(abspath, name); + if (access(abspath, F_OK) < 0) { + continue; + } + strncpy(result, abspath, sizeof(result)); + break; + } + path = path_orig; + guard_free(path); + return strlen(result) ? result : NULL; +} + +int touch(const char *filename) { + if (access(filename, F_OK) == 0) { + return 0; + } + + FILE *fp = fopen(filename, "w"); + if (!fp) { + perror(filename); + return 1; + } + fclose(fp); + return 0; +} + +int git_clone(struct Process *proc, char *url, char *destdir, char *gitref) { + int result = -1; + char *chdir_to = NULL; + char *program = find_program("git"); + if (!program) { + return result; + } + + static char command[PATH_MAX]; + sprintf(command, "%s clone -c advice.detachedHead=false --recursive %s", program, url); + if (destdir && access(destdir, F_OK) < 0) { + sprintf(command + strlen(command), " %s", destdir); + result = shell(proc, command); + } + + if (destdir) { + chdir_to = destdir; + } else { + chdir_to = path_basename(url); + } + + pushd(chdir_to); + { + memset(command, 0, sizeof(command)); + sprintf(command, "%s fetch --all", program); + result += shell(proc, command); + + if (gitref != NULL) { + memset(command, 0, sizeof(command)); + sprintf(command, "%s checkout %s", program, gitref); + result += shell(proc, command); + } + popd(); + } + return result; +} + + +char *git_describe(const char *path) { + static char version[NAME_MAX]; + FILE *pp; + + memset(version, 0, sizeof(version)); + if (pushd(path)) { + return NULL; + } + + pp = popen("git describe --first-parent --always --tags", "r"); + if (!pp) { + return NULL; + } + fgets(version, sizeof(version) - 1, pp); + strip(version); + pclose(pp); + popd(); + return version; +} + +char *git_rev_parse(const char *path, char *args) { + static char version[NAME_MAX]; + char cmd[PATH_MAX]; + FILE *pp; + + memset(version, 0, sizeof(version)); + if (isempty(args)) { + fprintf(stderr, "git_rev_parse args cannot be empty\n"); + return NULL; + } + + if (pushd(path)) { + return NULL; + } + + sprintf(cmd, "git rev-parse %s", args); + pp = popen(cmd, "r"); + if (!pp) { + return NULL; + } + fgets(version, sizeof(version) - 1, pp); + strip(version); + pclose(pp); + popd(); + return version; +} + +void msg(unsigned type, char *fmt, ...) { + FILE *stream = NULL; + char header[255]; + char status[20]; + + if (type & STASIS_MSG_NOP) { + // quiet mode + return; + } + + if (!globals.verbose && type & STASIS_MSG_RESTRICT) { + // Verbose mode is not active + return; + } + + memset(header, 0, sizeof(header)); + memset(status, 0, sizeof(status)); + + va_list args; + va_start(args, fmt); + + stream = stdout; + fprintf(stream, "%s", STASIS_COLOR_RESET); + if (type & STASIS_MSG_ERROR) { + // for error output + stream = stderr; + fprintf(stream, "%s", STASIS_COLOR_RED); + strcpy(status, " ERROR: "); + } else if (type & STASIS_MSG_WARN) { + stream = stderr; + fprintf(stream, "%s", STASIS_COLOR_YELLOW); + strcpy(status, " WARNING: "); + } else { + fprintf(stream, "%s", STASIS_COLOR_GREEN); + strcpy(status, " "); + } + + if (type & STASIS_MSG_L1) { + sprintf(header, "==>%s" STASIS_COLOR_RESET STASIS_COLOR_WHITE, status); + } else if (type & STASIS_MSG_L2) { + sprintf(header, " ->%s" STASIS_COLOR_RESET, status); + } else if (type & STASIS_MSG_L3) { + sprintf(header, STASIS_COLOR_BLUE " ->%s" STASIS_COLOR_RESET, status); + } + + fprintf(stream, "%s", header); + vfprintf(stream, fmt, args); + fprintf(stream, "%s", STASIS_COLOR_RESET); + va_end(args); +} + +void debug_shell() { + msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "ENTERING STASIS DEBUG SHELL\n" STASIS_COLOR_RESET); + if (system("/bin/bash -c 'PS1=\"(STASIS DEBUG) \\W $ \" bash --norc --noprofile'") < 0) { + SYSERROR("unable to spawn debug shell: %s", strerror(errno)); + exit(errno); + } + msg(STASIS_MSG_L1 | STASIS_MSG_WARN, "EXITING STASIS DEBUG SHELL\n" STASIS_COLOR_RESET); + exit(255); +} + +char *xmkstemp(FILE **fp, const char *mode) { + int fd = -1; + char tmpdir[PATH_MAX]; + char t_name[PATH_MAX * 2]; + + if (globals.tmpdir) { + strcpy(tmpdir, globals.tmpdir); + } else { + strcpy(tmpdir, "/tmp"); + } + memset(t_name, 0, sizeof(t_name)); + sprintf(t_name, "%s/%s", tmpdir, "STASIS.XXXXXX"); + + fd = mkstemp(t_name); + *fp = fdopen(fd, mode); + if (!*fp) { + // unable to open, die + if (fd > 0) + close(fd); + *fp = NULL; + return NULL; + } + + char *path = strdup(t_name); + if (!path) { + // strdup failed, die + if (*fp) { + // close the file handle + fclose(*fp); + *fp = NULL; + } + // fall through. path is NULL. + } + return path; +} + +int isempty_dir(const char *path) { + DIR *dp; + struct dirent *rec; + size_t count = 0; + + dp = opendir(path); + if (!dp) { + return -1; + } + while ((rec = readdir(dp)) != NULL) { + if (!strcmp(rec->d_name, ".") || !strcmp(rec->d_name, "..")) { + continue; + } + count++; + } + closedir(dp); + return count == 0; +} + +int path_store(char **destptr, size_t maxlen, const char *base, const char *path) { + char *path_tmp; + size_t base_len = 0; + size_t path_len = 0; + + // Both path elements need to be defined to continue + if (!base || !path) { + return -1; + } + + // Initialize destination pointer to length of maxlen + path_tmp = calloc(maxlen, sizeof(*path_tmp)); + if (!path_tmp) { + return -1; + } + + // Ensure generated path will fit in destination + base_len = strlen(base); + path_len = strlen(path); + // 2 = directory separator and NUL terminator + if (2 + (base_len + path_len) > maxlen) { + goto l_path_setup_error; + } + + snprintf(path_tmp, maxlen - 1, "%s/%s", base, path); + if (mkdirs(path_tmp, 0755)) { + goto l_path_setup_error; + } + + if (*destptr) { + guard_free(*destptr); + } + + if (!(*destptr = realpath(path_tmp, NULL))) { + goto l_path_setup_error; + } + + guard_free(path_tmp); + return 0; + + l_path_setup_error: + guard_free(path_tmp); + return -1; +} + +int xml_pretty_print_in_place(const char *filename, const char *pretty_print_prog, const char *pretty_print_args) { + int status = 0; + char *tempfile = NULL; + char *result = NULL; + FILE *fp = NULL; + FILE *tmpfp = NULL; + char cmd[PATH_MAX]; + if (!find_program(pretty_print_prog)) { + // Pretty printing is optional. 99% chance the XML data will + // be passed to a report generator; not inspected by a human. + return 0; + } + memset(cmd, 0, sizeof(cmd)); + snprintf(cmd, sizeof(cmd) - 1, "%s %s %s", pretty_print_prog, pretty_print_args, filename); + result = shell_output(cmd, &status); + if (status || !result) { + goto pretty_print_failed; + } + + tempfile = xmkstemp(&tmpfp, "w+"); + if (!tmpfp || !tempfile) { + goto pretty_print_failed; + } + + fprintf(tmpfp, "%s", result); + fflush(tmpfp); + fclose(tmpfp); + + fp = fopen(filename, "w+"); + if (!fp) { + goto pretty_print_failed; + } + + if (copy2(tempfile, filename, CT_PERM)) { + goto pretty_print_failed; + } + + if (remove(tempfile)) { + goto pretty_print_failed; + } + + fclose(fp); + guard_free(tempfile); + guard_free(result); + return 0; + + pretty_print_failed: + if (fp) { + fclose(fp); + } + if (tmpfp) { + fclose(tmpfp); + } + guard_free(tempfile); + guard_free(result); + return -1; +} + +/** + * + * @param filename /path/to/tox.ini + * @param result path of replacement tox.ini configuration + * @return 0 on success, -1 on error + */ +int fix_tox_conf(const char *filename, char **result) { + struct INIFILE *toxini; + FILE *fptemp; + char *tempfile; + const char *with_posargs = " \\\n {posargs}\n"; + + // Create new temporary tox configuration file + tempfile = xmkstemp(&fptemp, "w+"); + if (!tempfile) { + return -1; + } + + // If the result pointer is NULL, allocate enough to store a filesystem path + if (!*result) { + *result = calloc(PATH_MAX, sizeof(**result)); + if (!*result) { + guard_free(tempfile); + return -1; + } + } + + // Consume the original tox.ini configuration + toxini = ini_open(filename); + if (!toxini) { + if (fptemp) { + guard_free(result); + guard_free(tempfile); + fclose(fptemp); + } + return -1; + } + + // Modify tox configuration + // - Allow passing positional arguments pytest + for (size_t i = 0; i < toxini->section_count; i++) { + struct INISection *section = toxini->section[i]; + if (section) { + char *section_name = section->key; + for (size_t k = 0; k < section->data_count; k++) { + struct INIData *data = section->data[k]; + if (data) { + int err = 0; + char *key = data->key; + char *value = ini_getval_str(toxini, section->key, data->key, INI_READ_RENDER, &err); + if (key && value) { + if (startswith(value, "pytest") && !strstr(value, "{posargs}")) { + strip(value); + char *tmp; + tmp = realloc(value, strlen(value) + strlen(with_posargs) + 1); + if (!tmp) { + SYSERROR("failed to increase size to +%zu bytes", + strlen(value) + strlen(with_posargs) + 1); + guard_free(*result); + return -1; + } else if (tmp != value) { + value = tmp; + } + strcat(value, with_posargs); + ini_setval(&toxini, INI_SETVAL_REPLACE, section_name, key, value); + } + } + guard_free(value); + } + } + } + } + + // Save modified configuration + ini_write(toxini, &fptemp, INI_WRITE_RAW); + fclose(fptemp); + + // Store path to modified config + strcpy(*result, tempfile); + guard_free(tempfile); + + ini_free(&toxini); + return 0; +} + +static size_t count_blanks(char *s) { + // return the number of leading blanks (tab/space) in a string + size_t blank = 0; + for (size_t i = 0; i < strlen(s); i++) { + if (isblank(s[i])) { + blank++; + } else { + break; + } + } + return blank; +} + +char *collapse_whitespace(char **s) { + char *x = (*s); + size_t len = strlen(x); + for (size_t i = 0; i < len; i++) { + size_t blank = count_blanks(&x[i]); + if (blank > 1) { + memmove(&x[i], &x[i] + blank, strlen(&x[i])); + } + } + + return *s; +} + +/** + * Replace sensitive text in strings with ***REDACTED*** + * @param to_redact a list of tokens to redact + * @param src to read + * @param dest to write modified string + * @param maxlen maximum length of dest string + * @return 0 on success, -1 on error + */ +int redact_sensitive(const char **to_redact, size_t to_redact_size, char *src, char *dest, size_t maxlen) { + const char *redacted = "***REDACTED***"; + + char *tmp = calloc(strlen(redacted) + strlen(src) + 1, sizeof(*tmp)); + if (!tmp) { + return -1; + } + strcpy(tmp, src); + + for (size_t i = 0; i < to_redact_size; i++) { + if (to_redact[i] && strstr(tmp, to_redact[i])) { + replace_text(tmp, to_redact[i], redacted, 0); + break; + } + } + + memset(dest, 0, maxlen); + strncpy(dest, tmp, maxlen - 1); + guard_free(tmp); + + return 0; +} + +/** + * Retrieve file names in a directory + * (no metadata, non-recursive) + * + * @param path directory path + * @return StrList structure + */ +struct StrList *listdir(const char *path) { + struct StrList *node; + DIR *dp; + struct dirent *rec; + + dp = opendir(path); + if (!dp) { + return NULL; + } + node = strlist_init(); + + while ((rec = readdir(dp)) != NULL) { + if (!strcmp(rec->d_name, ".") || !strcmp(rec->d_name, "..")) { + continue; + } + strlist_append(&node, rec->d_name); + } + closedir(dp); + return node; +} + +long get_cpu_count() { +#if defined(STASIS_OS_LINUX) || defined(STASIS_OS_DARWIN) + return sysconf(_SC_NPROCESSORS_ONLN); +#else + return 0; +#endif +} + +int mkdirs(const char *_path, mode_t mode) { + int status; + char *token; + char pathbuf[PATH_MAX] = {0}; + char *path; + path = pathbuf; + strcpy(path, _path); + errno = 0; + + char result[PATH_MAX] = {0}; + status = 0; + while ((token = strsep(&path, "/")) != NULL && !status) { + if (token[0] == '.') + continue; + strcat(result, token); + strcat(result, "/"); + status = mkdir(result, mode); + if (status && errno == EEXIST) { + status = 0; + errno = 0; + continue; + } + } + return status; +} + +char *find_version_spec(char *str) { + return strpbrk(str, "@~=<>!"); +} diff --git a/src/lib/core/wheel.c b/src/lib/core/wheel.c new file mode 100644 index 0000000..4692d0a --- /dev/null +++ b/src/lib/core/wheel.c @@ -0,0 +1,126 @@ +#include "wheel.h" + +struct Wheel *get_wheel_info(const char *basepath, const char *name, char *to_match[], unsigned match_mode) { + DIR *dp; + struct dirent *rec; + struct Wheel *result = NULL; + char package_path[PATH_MAX]; + char package_name[NAME_MAX]; + + strcpy(package_name, name); + tolower_s(package_name); + sprintf(package_path, "%s/%s", basepath, package_name); + + dp = opendir(package_path); + if (!dp) { + return NULL; + } + + while ((rec = readdir(dp)) != NULL) { + if (!strcmp(rec->d_name, ".") || !strcmp(rec->d_name, "..")) { + continue; + } + char filename[NAME_MAX]; + strcpy(filename, rec->d_name); + char *ext = strstr(filename, ".whl"); + if (ext) { + *ext = '\0'; + } else { + // not a wheel file. nothing to do + continue; + } + + size_t match = 0; + size_t pattern_count = 0; + for (; to_match[pattern_count] != NULL; pattern_count++) { + if (strstr(filename, to_match[pattern_count])) { + match++; + } + } + + if (!startswith(rec->d_name, name)) { + continue; + } + + if (match_mode == WHEEL_MATCH_EXACT && match != pattern_count) { + continue; + } + + result = calloc(1, sizeof(*result)); + if (!result) { + SYSERROR("Unable to allocate %zu bytes for wheel struct", sizeof(*result)); + closedir(dp); + return NULL; + } + + result->path_name = realpath(package_path, NULL); + if (!result->path_name) { + SYSERROR("Unable to resolve absolute path to %s: %s", filename, strerror(errno)); + wheel_free(&result); + closedir(dp); + return NULL; + } + result->file_name = strdup(rec->d_name); + if (!result->file_name) { + SYSERROR("Unable to allocate bytes for %s: %s", rec->d_name, strerror(errno)); + wheel_free(&result); + closedir(dp); + return NULL; + } + + size_t parts_total; + char **parts = split(filename, "-", 0); + if (!parts) { + // This shouldn't happen unless a wheel file is present in the + // directory with a malformed file name, or we've managed to + // exhaust the system's memory + SYSERROR("%s has no '-' separators! (Delete this file and try again)", filename); + wheel_free(&result); + closedir(dp); + return NULL; + } + + for (parts_total = 0; parts[parts_total] != NULL; parts_total++); + if (parts_total == 5) { + // no build tag + result->distribution = strdup(parts[0]); + result->version = strdup(parts[1]); + result->build_tag = NULL; + result->python_tag = strdup(parts[2]); + result->abi_tag = strdup(parts[3]); + result->platform_tag = strdup(parts[4]); + } else if (parts_total == 6) { + // has build tag + result->distribution = strdup(parts[0]); + result->version = strdup(parts[1]); + result->build_tag = strdup(parts[2]); + result->python_tag = strdup(parts[3]); + result->abi_tag = strdup(parts[4]); + result->platform_tag = strdup(parts[5]); + } else { + SYSERROR("Unknown wheel name format: %s. Expected 5 or 6 strings " + "separated by '-', but got %zu instead", filename, parts_total); + GENERIC_ARRAY_FREE(parts); + wheel_free(&result); + closedir(dp); + return NULL; + } + GENERIC_ARRAY_FREE(parts); + break; + } + closedir(dp); + return result; +} + +void wheel_free(struct Wheel **wheel) { + struct Wheel *w = (*wheel); + guard_free(w->path_name); + guard_free(w->file_name); + guard_free(w->distribution); + guard_free(w->version); + guard_free(w->build_tag); + guard_free(w->python_tag); + guard_free(w->abi_tag); + guard_free(w->python_tag); + guard_free(w); +} |