From f2cc2cee71c6d2fb0cdd5b7736e86efa4f999c0a Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 14 Nov 2024 13:21:00 -0500 Subject: Sort releases by python version --- src/cli/stasis_indexer/stasis_indexer.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'src') diff --git a/src/cli/stasis_indexer/stasis_indexer.c b/src/cli/stasis_indexer/stasis_indexer.c index fddf18c..f45f073 100644 --- a/src/cli/stasis_indexer/stasis_indexer.c +++ b/src/cli/stasis_indexer/stasis_indexer.c @@ -713,6 +713,21 @@ void indexer_init_dirs(struct Delivery *ctx, const char *workdir) { } } +static int sort_deliveries_cmpfn(const void *a, const void *b) { + const struct Delivery *delivery1 = (struct Delivery *) a; + const size_t delivery1_python = strtoul(delivery1->meta.python_compact, NULL, 10); + const struct Delivery *delivery2 = (struct Delivery *) b; + const size_t delivery2_python = strtoul(delivery2->meta.python_compact, NULL, 10); + + if (delivery2_python > delivery1_python) { + return 1; + } + if (delivery2_python < delivery1_python) { + return -1; + } + return 0; +} + int main(int argc, char *argv[]) { size_t rootdirs_total = 0; char *destdir = NULL; @@ -885,6 +900,7 @@ int main(int argc, char *argv[]) { } indexer_load_metadata(&local[i], path); } + qsort(local, strlist_count(metafiles), sizeof(*local), sort_deliveries_cmpfn); msg(STASIS_MSG_L1, "Generating links to latest release iteration\n"); if (indexer_symlinks(local, strlist_count(metafiles))) { -- cgit From a9d975389aaf5d79d738517b98161e375e757cba Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Mon, 18 Nov 2024 09:27:35 -0500 Subject: Break down indexer into independent source files * Generate test result output * Add helper function to manage changing file extensions --- src/cli/stasis_indexer/CMakeLists.txt | 11 +- src/cli/stasis_indexer/args.c | 38 ++ src/cli/stasis_indexer/args.h | 9 + src/cli/stasis_indexer/callbacks.c | 39 ++ src/cli/stasis_indexer/callbacks.h | 9 + src/cli/stasis_indexer/helpers.c | 242 +++++++ src/cli/stasis_indexer/helpers.h | 15 + src/cli/stasis_indexer/junitxml_report.c | 137 ++++ src/cli/stasis_indexer/junitxml_report.h | 8 + src/cli/stasis_indexer/readmes.c | 75 +++ src/cli/stasis_indexer/readmes.h | 8 + src/cli/stasis_indexer/stasis_indexer.c | 958 --------------------------- src/cli/stasis_indexer/stasis_indexer_main.c | 387 +++++++++++ src/cli/stasis_indexer/website.c | 119 ++++ src/cli/stasis_indexer/website.h | 8 + src/lib/core/utils.c | 10 + 16 files changed, 1114 insertions(+), 959 deletions(-) create mode 100644 src/cli/stasis_indexer/args.c create mode 100644 src/cli/stasis_indexer/args.h create mode 100644 src/cli/stasis_indexer/callbacks.c create mode 100644 src/cli/stasis_indexer/callbacks.h create mode 100644 src/cli/stasis_indexer/helpers.c create mode 100644 src/cli/stasis_indexer/helpers.h create mode 100644 src/cli/stasis_indexer/junitxml_report.c create mode 100644 src/cli/stasis_indexer/junitxml_report.h create mode 100644 src/cli/stasis_indexer/readmes.c create mode 100644 src/cli/stasis_indexer/readmes.h delete mode 100644 src/cli/stasis_indexer/stasis_indexer.c create mode 100644 src/cli/stasis_indexer/stasis_indexer_main.c create mode 100644 src/cli/stasis_indexer/website.c create mode 100644 src/cli/stasis_indexer/website.h (limited to 'src') diff --git a/src/cli/stasis_indexer/CMakeLists.txt b/src/cli/stasis_indexer/CMakeLists.txt index eae1394..1d4c658 100644 --- a/src/cli/stasis_indexer/CMakeLists.txt +++ b/src/cli/stasis_indexer/CMakeLists.txt @@ -1,6 +1,15 @@ add_executable(stasis_indexer - stasis_indexer.c + args.c + stasis_indexer_main.c + callbacks.c + helpers.c + junitxml_report.c + website.c + website.h + readmes.c + readmes.h ) +target_include_directories(stasis_indexer PRIVATE ${CMAKE_SOURCE_DIR}) target_link_libraries(stasis_indexer PRIVATE stasis_core) install(TARGETS stasis_indexer RUNTIME) diff --git a/src/cli/stasis_indexer/args.c b/src/cli/stasis_indexer/args.c new file mode 100644 index 0000000..2d92ab0 --- /dev/null +++ b/src/cli/stasis_indexer/args.c @@ -0,0 +1,38 @@ +#include "core.h" +#include "args.h" + +struct option long_options[] = { + {"help", no_argument, 0, 'h'}, + {"destdir", required_argument, 0, 'd'}, + {"verbose", no_argument, 0, 'v'}, + {"unbuffered", no_argument, 0, 'U'}, + {"web", no_argument, 0, 'w'}, + {0, 0, 0, 0}, +}; + +const char *long_options_help[] = { + "Display this usage statement", + "Destination directory", + "Increase output verbosity", + "Disable line buffering", + "Generate HTML indexes (requires pandoc)", + NULL, +}; + +void usage(char *name) { + const int maxopts = sizeof(long_options) / sizeof(long_options[0]); + char *opts = calloc(maxopts + 1, sizeof(char)); + for (int i = 0; i < maxopts; i++) { + opts[i] = (char) long_options[i].val; + } + printf("usage: %s [-%s] {{STASIS_ROOT}...}\n", name, opts); + guard_free(opts); + + for (int i = 0; i < maxopts - 1; i++) { + char line[255] = {0}; + sprintf(line, " --%s -%c %-20s", long_options[i].name, long_options[i].val, long_options_help[i]); + puts(line); + } + +} + diff --git a/src/cli/stasis_indexer/args.h b/src/cli/stasis_indexer/args.h new file mode 100644 index 0000000..543aa4b --- /dev/null +++ b/src/cli/stasis_indexer/args.h @@ -0,0 +1,9 @@ +#ifndef STASIS_ARGS_H +#define STASIS_ARGS_H + +#include + +extern struct option long_options[]; +void usage(char *name); + +#endif //STASIS_ARGS_H diff --git a/src/cli/stasis_indexer/callbacks.c b/src/cli/stasis_indexer/callbacks.c new file mode 100644 index 0000000..4790e09 --- /dev/null +++ b/src/cli/stasis_indexer/callbacks.c @@ -0,0 +1,39 @@ +// +// Created by jhunk on 11/15/24. +// + +#include "core.h" +#include "callbacks.h" + +// qsort callback to sort delivery contexts by compact python version +int callback_sort_deliveries_cmpfn(const void *a, const void *b) { + const struct Delivery *delivery1 = (struct Delivery *) a; + const size_t delivery1_python = strtoul(delivery1->meta.python_compact, NULL, 10); + const struct Delivery *delivery2 = (struct Delivery *) b; + const size_t delivery2_python = strtoul(delivery2->meta.python_compact, NULL, 10); + + if (delivery2_python > delivery1_python) { + return 1; + } + if (delivery2_python < delivery1_python) { + return -1; + } + return 0; +} + +// qsort callback to sort dynamically allocated delivery contexts by compact python version +int callback_sort_deliveries_dynamic_cmpfn(const void *a, const void *b) { + const struct Delivery *delivery1 = *(struct Delivery **) a; + const size_t delivery1_python = strtoul(delivery1->meta.python_compact, NULL, 10); + const struct Delivery *delivery2 = *(struct Delivery **) b; + const size_t delivery2_python = strtoul(delivery2->meta.python_compact, NULL, 10); + + if (delivery2_python > delivery1_python) { + return 1; + } + if (delivery2_python < delivery1_python) { + return -1; + } + return 0; +} + diff --git a/src/cli/stasis_indexer/callbacks.h b/src/cli/stasis_indexer/callbacks.h new file mode 100644 index 0000000..7d95cbb --- /dev/null +++ b/src/cli/stasis_indexer/callbacks.h @@ -0,0 +1,9 @@ +#ifndef CALLBACKS_H +#define CALLBACKS_H + +#include "delivery.h" + +int callback_sort_deliveries_cmpfn(const void *a, const void *b); +int callback_sort_deliveries_dynamic_cmpfn(const void *a, const void *b); + +#endif //CALLBACKS_H diff --git a/src/cli/stasis_indexer/helpers.c b/src/cli/stasis_indexer/helpers.c new file mode 100644 index 0000000..87ab922 --- /dev/null +++ b/src/cli/stasis_indexer/helpers.c @@ -0,0 +1,242 @@ +// +// Created by jhunk on 11/15/24. +// + +#include "core.h" +#include "helpers.h" + +struct StrList *get_architectures(struct Delivery ctx[], const size_t nelem) { + struct StrList *architectures = strlist_init(); + for (size_t i = 0; i < nelem; i++) { + if (!strstr_array(architectures->data, ctx[i].system.arch)) { + strlist_append(&architectures, ctx[i].system.arch); + } + } + return architectures; +} + +struct StrList *get_platforms(struct Delivery ctx[], const size_t nelem) { + struct StrList *platforms = strlist_init(); + for (size_t i = 0; i < nelem; i++) { + if (!strstr_array(platforms->data, ctx[i].system.platform[DELIVERY_PLATFORM_RELEASE])) { + strlist_append(&platforms, ctx[i].system.platform[DELIVERY_PLATFORM_RELEASE]); + } + } + return platforms; +} + +int get_pandoc_version(size_t *result) { + *result = 0; + int state = 0; + char *version_str = shell_output("pandoc --version", &state); + if (state || !version_str) { + // an error occurred + return -1; + } + + // Verify that we're looking at pandoc + if (strlen(version_str) > 7 && !strncmp(version_str, "pandoc ", 7)) { + // we have pandoc + char *v_begin = &version_str[7]; + if (!v_begin) { + SYSERROR("unexpected pandoc output: %s", version_str); + return -1; + } + char *v_end = strchr(version_str, '\n'); + if (v_end) { + *v_end = 0; + } + + char **parts = split(v_begin, ".", 0); + if (!parts) { + SYSERROR("unable to split pandoc version string, '%s': %s", version_str, strerror(errno)); + return -1; + } + + size_t parts_total; + for (parts_total = 0; parts[parts_total] != NULL; parts_total++) {} + + // generate the version as an integer + // note: pandoc version scheme never exceeds four elements (or bytes in this case) + for (size_t i = 0; i < 4; i++) { + unsigned char tmp = 0; + if (i < parts_total) { + // only process version elements we have. the rest will be zeros. + tmp = strtoul(parts[i], NULL, 10); + } + // pack version element into result + *result = *result << 8 | tmp; + } + } else { + // invalid version string + return 1; + } + + return 0; +} + +int micromamba_configure(const struct Delivery *ctx, struct MicromambaInfo *m) { + int status = 0; + char *micromamba_prefix = NULL; + if (asprintf(µmamba_prefix, "%s/bin", ctx->storage.tools_dir) < 0) { + return -1; + } + m->conda_prefix = globals.conda_install_prefix; + m->micromamba_prefix = micromamba_prefix; + + const size_t pathvar_len = strlen(getenv("PATH")) + strlen(m->micromamba_prefix) + strlen(m->conda_prefix) + 3 + 4 + 1; + // ^^^^^^^^^^^^^^^^^^ + // 3 = separators + // 4 = chars (/bin) + // 1 = nul terminator + char *pathvar = calloc(pathvar_len, sizeof(*pathvar)); + if (!pathvar) { + SYSERROR("%s", "Unable to allocate bytes for temporary path string"); + exit(1); + } + snprintf(pathvar, pathvar_len, "%s/bin:%s:%s", m->conda_prefix, m->micromamba_prefix, getenv("PATH")); + setenv("PATH", pathvar, 1); + guard_free(pathvar); + + status += micromamba(m, "config prepend --env channels conda-forge"); + if (!globals.verbose) { + status += micromamba(m, "config set --env quiet true"); + } + status += micromamba(m, "config set --env always_yes true"); + status += micromamba(m, "install conda-build pandoc"); + + return status; +} + +int get_latest_rc(struct Delivery ctx[], const size_t nelem) { + int result = 0; + for (size_t i = 0; i < nelem; i++) { + if (ctx[i].meta.rc > result) { + result = ctx[i].meta.rc; + } + } + return result; +} + +struct Delivery **get_latest_deliveries(struct Delivery ctx[], const size_t nelem) { + struct Delivery **result = NULL; + int latest = 0; + size_t n = 0; + + result = calloc(nelem + 1, sizeof(result)); + if (!result) { + fprintf(stderr, "Unable to allocate %zu bytes for result delivery array: %s\n", nelem * sizeof(result), strerror(errno)); + return NULL; + } + + latest = get_latest_rc(ctx, nelem); + for (size_t i = 0; i < nelem; i++) { + if (ctx[i].meta.rc == latest) { + result[n] = &ctx[i]; + n++; + } + } + + return result; +} + +int get_files(struct StrList **out, const char *path, const char *pattern, ...) { + va_list args; + va_start(args, pattern); + char userpattern[PATH_MAX] = {0}; + vsprintf(userpattern, pattern, args); + va_end(args); + struct StrList *list = listdir(path); + if (!list) { + return -1; + } + + if (!*out) { + *out = strlist_init(); + if (!*out) { + guard_strlist_free(&list); + return -1; + } + } + + size_t no_match = 0; + for (size_t i = 0; i < strlist_count(list); i++) { + char *item = strlist_item(list, i); + if (fnmatch(userpattern, item, 0)) { + no_match++; + } else { + strlist_append(out, item); + } + } + if (no_match >= strlist_count(list)) { + fprintf(stderr, "no files matching the pattern: %s\n", userpattern); + guard_strlist_free(&list); + return -1; + } + guard_strlist_free(&list); + return 0; +} + +int load_metadata(struct Delivery *ctx, const char *filename) { + char line[STASIS_NAME_MAX] = {0}; + + FILE *fp = fopen(filename, "r"); + if (!fp) { + return -1; + } + + while (fgets(line, sizeof(line) - 1, fp) != NULL) { + char **parts = split(line, " ", 1); + const char *name = parts[0]; + char *value = parts[1]; + + strip(value); + if (!strcmp(name, "name")) { + ctx->meta.name = strdup(value); + } else if (!strcmp(name, "version")) { + ctx->meta.version = strdup(value); + } else if (!strcmp(name, "rc")) { + ctx->meta.rc = (int) strtol(value, NULL, 10); + } else if (!strcmp(name, "python")) { + ctx->meta.python = strdup(value); + } else if (!strcmp(name, "python_compact")) { + ctx->meta.python_compact = strdup(value); + } else if (!strcmp(name, "mission")) { + ctx->meta.mission = strdup(value); + } else if (!strcmp(name, "codename")) { + ctx->meta.codename = strdup(value); + } else if (!strcmp(name, "platform")) { + ctx->system.platform = split(value, " ", 0); + } else if (!strcmp(name, "arch")) { + ctx->system.arch = strdup(value); + } else if (!strcmp(name, "time")) { + ctx->info.time_str_epoch = strdup(value); + } else if (!strcmp(name, "release_fmt")) { + ctx->rules.release_fmt = strdup(value); + } else if (!strcmp(name, "release_name")) { + ctx->info.release_name = strdup(value); + } else if (!strcmp(name, "build_name_fmt")) { + ctx->rules.build_name_fmt = strdup(value); + } else if (!strcmp(name, "build_name")) { + ctx->info.build_name = strdup(value); + } else if (!strcmp(name, "build_number_fmt")) { + ctx->rules.build_number_fmt = strdup(value); + } else if (!strcmp(name, "build_number")) { + ctx->info.build_number = strdup(value); + } else if (!strcmp(name, "conda_installer_baseurl")) { + ctx->conda.installer_baseurl = strdup(value); + } else if (!strcmp(name, "conda_installer_name")) { + ctx->conda.installer_name = strdup(value); + } else if (!strcmp(name, "conda_installer_version")) { + ctx->conda.installer_version = strdup(value); + } else if (!strcmp(name, "conda_installer_platform")) { + ctx->conda.installer_platform = strdup(value); + } else if (!strcmp(name, "conda_installer_arch")) { + ctx->conda.installer_arch = strdup(value); + } + GENERIC_ARRAY_FREE(parts); + } + fclose(fp); + + return 0; +} diff --git a/src/cli/stasis_indexer/helpers.h b/src/cli/stasis_indexer/helpers.h new file mode 100644 index 0000000..41db49e --- /dev/null +++ b/src/cli/stasis_indexer/helpers.h @@ -0,0 +1,15 @@ +#ifndef HELPERS_H +#define HELPERS_H + +#include "delivery.h" + +struct StrList *get_architectures(struct Delivery ctx[], size_t nelem); +struct StrList *get_platforms(struct Delivery ctx[], size_t nelem); +int get_pandoc_version(size_t *result); +int get_latest_rc(struct Delivery ctx[], size_t nelem); +struct Delivery **get_latest_deliveries(struct Delivery ctx[], size_t nelem); +int get_files(struct StrList **out, const char *path, const char *pattern, ...); +int load_metadata(struct Delivery *ctx, const char *filename); +int micromamba_configure(const struct Delivery *ctx, struct MicromambaInfo *m); + +#endif //HELPERS_H diff --git a/src/cli/stasis_indexer/junitxml_report.c b/src/cli/stasis_indexer/junitxml_report.c new file mode 100644 index 0000000..571f1b7 --- /dev/null +++ b/src/cli/stasis_indexer/junitxml_report.c @@ -0,0 +1,137 @@ +// +// Created by jhunk on 11/15/24. +// + +#include "core.h" +#include "callbacks.h" +#include "junitxml.h" +#include "junitxml_report.h" + +int indexer_junitxml_report(struct Delivery ctx[], const size_t nelem) { + struct Delivery **latest = NULL; + latest = get_latest_deliveries(ctx, nelem); + + char indexfile[PATH_MAX] = {0}; + sprintf(indexfile, "%s/README.md", ctx->storage.results_dir); + + struct StrList *file_listing = listdir(ctx->storage.results_dir); + if (!file_listing) { + // no test results to process + return 0; + } + + if (!pushd(ctx->storage.results_dir)) { + FILE *indexfp = fopen(indexfile, "w+"); + if (!indexfp) { + fprintf(stderr, "Unable to open %s for writing\n", indexfile); + return -1; + } + struct StrList *archs = get_architectures(*latest, nelem); + struct StrList *platforms = get_platforms(*latest, nelem); + + qsort(latest, nelem, sizeof(*latest), callback_sort_deliveries_dynamic_cmpfn); + fprintf(indexfp, "# %s-%s Test Report\n\n", ctx->meta.name, ctx->meta.version); + fprintf(indexfp, "## Current Release\n\n"); + size_t no_printable_data = 0; + for (size_t p = 0; p < strlist_count(platforms); p++) { + char *platform = strlist_item(platforms, p); + for (size_t a = 0; a < strlist_count(archs); a++) { + char *arch = strlist_item(archs, a); + int have_combo = 0; + for (size_t i = 0; i < nelem; i++) { + if (latest[i] && latest[i]->system.platform) { + if (strstr(latest[i]->system.platform[DELIVERY_PLATFORM_RELEASE], platform) && + strstr(latest[i]->system.arch, arch)) { + have_combo = 1; + break; + } + } + } + if (!have_combo) { + continue; + } + fprintf(indexfp, "### %s-%s\n\n", platform, arch); + + fprintf(indexfp, "|Suite|Duration|Fail |Skip |Error |\n"); + fprintf(indexfp, "|:----|:------:|:------:|:---:|:----:|\n"); + for (size_t f = 0; f < strlist_count(file_listing); f++) { + char *filename = strlist_item(file_listing, f); + if (!endswith(filename, ".xml")) { + continue; + } + + if (strstr(filename, platform) && strstr(filename, arch)) { + struct JUNIT_Testsuite *testsuite = junitxml_testsuite_read(filename); + if (testsuite) { + if (globals.verbose) { + printf("%s: duration: %0.4f, failed: %d, skipped: %d, errors: %d\n", filename, testsuite->time, testsuite->failures, testsuite->skipped, testsuite->errors); + } + fprintf(indexfp, "|[%s](%s)|%0.4f|%d|%d|%d|\n", filename, filename, testsuite->time, testsuite->failures, testsuite->skipped, testsuite->errors); + /* + * TODO: Display failure/skip/error output. + */ + char *bname = strdup(filename); + bname[strlen(bname) - 4] = 0; + char result_outfile[PATH_MAX] = {0}; + path_basename(bname); + mkdir(bname, 0755); + snprintf(result_outfile, sizeof(result_outfile) - 1, "%s/%s.md", bname, bname); + FILE *resultfp = fopen(result_outfile, "w+"); + if (!resultfp) { + SYSERROR("Unable to open %s for writing", result_outfile); + return -1; + } + + for (size_t i = 0; i < testsuite->_tc_inuse; i++) { + if (testsuite->testcase[i]->tc_result_state_type) { + const char *type_str = NULL; + const int state = testsuite->testcase[i]->tc_result_state_type; + const char *message = NULL; + if (state == JUNIT_RESULT_STATE_FAILURE) { + message = testsuite->testcase[i]->result_state.failure->message; + type_str = "[FAILED]"; + } else if (state == JUNIT_RESULT_STATE_ERROR) { + message = testsuite->testcase[i]->result_state.error->message; + type_str = "[ERROR]"; + } else if (state == JUNIT_RESULT_STATE_SKIPPED) { + message = testsuite->testcase[i]->result_state.skipped->message; + type_str = "[SKIPPED]"; + } + fprintf(resultfp, "### %s %s :: %s\n", type_str, testsuite->testcase[i]->classname, testsuite->testcase[i]->name); + fprintf(resultfp, "\nDuration: %0.04fs\n", testsuite->testcase[i]->time); + fprintf(resultfp, "\n```\n%s\n```\n", message); + } + } + junitxml_testsuite_free(&testsuite); + fclose(resultfp); + } else { + fprintf(stderr, "bad test suite: %s: %s\n", strerror(errno), filename); + } + } else { + if (!no_printable_data) { + fprintf(indexfp, "|No data|-|-|-|-|-|-|\n"); + no_printable_data++; + } + } + } + fprintf(indexfp, "\n"); + no_printable_data = 0; + } + fprintf(indexfp, "\n"); + } + guard_strlist_free(&archs); + guard_strlist_free(&platforms); + fclose(indexfp); + popd(); + } else { + fprintf(stderr, "Unable to enter delivery directory: %s\n", ctx->storage.delivery_dir); + guard_free(latest); + return -1; + } + + // "latest" is an array of pointers to ctxs[]. Do not free the contents of the array. + guard_free(latest); + return 0; +} + + diff --git a/src/cli/stasis_indexer/junitxml_report.h b/src/cli/stasis_indexer/junitxml_report.h new file mode 100644 index 0000000..6d2a248 --- /dev/null +++ b/src/cli/stasis_indexer/junitxml_report.h @@ -0,0 +1,8 @@ +#ifndef JUNITXML_REPORT_H +#define JUNITXML_REPORT_H + +#include "helpers.h" + +int indexer_junitxml_report(struct Delivery ctx[], size_t nelem); + +#endif //JUNITXML_REPORT_H diff --git a/src/cli/stasis_indexer/readmes.c b/src/cli/stasis_indexer/readmes.c new file mode 100644 index 0000000..75e97a9 --- /dev/null +++ b/src/cli/stasis_indexer/readmes.c @@ -0,0 +1,75 @@ +#include "core.h" +#include "readmes.h" + +int indexer_readmes(struct Delivery ctx[], const size_t nelem) { + struct Delivery **latest = NULL; + latest = get_latest_deliveries(ctx, nelem); + + char indexfile[PATH_MAX] = {0}; + sprintf(indexfile, "%s/README.md", ctx->storage.delivery_dir); + + if (!pushd(ctx->storage.delivery_dir)) { + FILE *indexfp = fopen(indexfile, "w+"); + if (!indexfp) { + fprintf(stderr, "Unable to open %s for writing\n", indexfile); + return -1; + } + struct StrList *archs = get_architectures(*latest, nelem); + struct StrList *platforms = get_platforms(*latest, nelem); + + fprintf(indexfp, "# %s-%s\n\n", ctx->meta.name, ctx->meta.version); + fprintf(indexfp, "## Current Release\n\n"); + for (size_t p = 0; p < strlist_count(platforms); p++) { + char *platform = strlist_item(platforms, p); + for (size_t a = 0; a < strlist_count(archs); a++) { + char *arch = strlist_item(archs, a); + int have_combo = 0; + for (size_t i = 0; i < nelem; i++) { + if (latest[i] && latest[i]->system.platform) { + if (strstr(latest[i]->system.platform[DELIVERY_PLATFORM_RELEASE], platform) && + strstr(latest[i]->system.arch, arch)) { + have_combo = 1; + } + } + } + if (!have_combo) { + continue; + } + fprintf(indexfp, "### %s-%s\n\n", platform, arch); + + fprintf(indexfp, "|Release|Info|Receipt|\n"); + fprintf(indexfp, "|:----:|:----:|:----:|\n"); + for (size_t i = 0; i < nelem; i++) { + char link_name[PATH_MAX]; + char readme_name[PATH_MAX]; + char conf_name[PATH_MAX]; + char conf_name_relative[PATH_MAX]; + if (!latest[i]) { + continue; + } + sprintf(link_name, "latest-py%s-%s-%s.yml", latest[i]->meta.python_compact, latest[i]->system.platform[DELIVERY_PLATFORM_RELEASE], latest[i]->system.arch); + sprintf(readme_name, "README-py%s-%s-%s.md", latest[i]->meta.python_compact, latest[i]->system.platform[DELIVERY_PLATFORM_RELEASE], latest[i]->system.arch); + sprintf(conf_name, "%s.ini", latest[i]->info.release_name); + sprintf(conf_name_relative, "../config/%s-rendered.ini", latest[i]->info.release_name); + if (strstr(link_name, platform) && strstr(link_name, arch)) { + fprintf(indexfp, "|[%s](%s)|[%s](%s)|[%s](%s)|\n", link_name, link_name, readme_name, readme_name, conf_name, conf_name_relative); + } + } + fprintf(indexfp, "\n"); + } + fprintf(indexfp, "\n"); + } + guard_strlist_free(&archs); + guard_strlist_free(&platforms); + fclose(indexfp); + popd(); + } else { + fprintf(stderr, "Unable to enter delivery directory: %s\n", ctx->storage.delivery_dir); + guard_free(latest); + return -1; + } + + // "latest" is an array of pointers to ctxs[]. Do not free the contents of the array. + guard_free(latest); + return 0; +} diff --git a/src/cli/stasis_indexer/readmes.h b/src/cli/stasis_indexer/readmes.h new file mode 100644 index 0000000..d4fa7ac --- /dev/null +++ b/src/cli/stasis_indexer/readmes.h @@ -0,0 +1,8 @@ +#ifndef READMES_H +#define READMES_H + +#include "helpers.h" + +int indexer_readmes(struct Delivery ctx[], size_t nelem); + +#endif //READMES_H diff --git a/src/cli/stasis_indexer/stasis_indexer.c b/src/cli/stasis_indexer/stasis_indexer.c deleted file mode 100644 index f45f073..0000000 --- a/src/cli/stasis_indexer/stasis_indexer.c +++ /dev/null @@ -1,958 +0,0 @@ -#include -#include -#include "delivery.h" -#include "junitxml.h" - -static struct option long_options[] = { - {"help", no_argument, 0, 'h'}, - {"destdir", required_argument, 0, 'd'}, - {"verbose", no_argument, 0, 'v'}, - {"unbuffered", no_argument, 0, 'U'}, - {"web", no_argument, 0, 'w'}, - {0, 0, 0, 0}, -}; - -const char *long_options_help[] = { - "Display this usage statement", - "Destination directory", - "Increase output verbosity", - "Disable line buffering", - "Generate HTML indexes (requires pandoc)", - NULL, -}; - -static void usage(char *name) { - int maxopts = sizeof(long_options) / sizeof(long_options[0]); - char *opts = calloc(maxopts + 1, sizeof(char)); - for (int i = 0; i < maxopts; i++) { - opts[i] = (char) long_options[i].val; - } - printf("usage: %s [-%s] {{STASIS_ROOT}...}\n", name, opts); - guard_free(opts); - - for (int i = 0; i < maxopts - 1; i++) { - char line[255] = {0}; - sprintf(line, " --%s -%c %-20s", long_options[i].name, long_options[i].val, long_options_help[i]); - puts(line); - } - -} - -int indexer_combine_rootdirs(const char *dest, char **rootdirs, const size_t rootdirs_total) { - char cmd[PATH_MAX]; - char destdir_bare[PATH_MAX]; - char destdir_with_output[PATH_MAX]; - char *destdir = destdir_bare; - - memset(cmd, 0, sizeof(cmd)); - memset(destdir_bare, 0, sizeof(destdir_bare)); - memset(destdir_with_output, 0, sizeof(destdir_bare)); - - strcpy(destdir_bare, dest); - strcpy(destdir_with_output, dest); - strcat(destdir_with_output, "/output"); - - if (!access(destdir_with_output, F_OK)) { - destdir = destdir_with_output; - } - - sprintf(cmd, "rsync -ah%s --delete --exclude 'tools/' --exclude 'tmp/' --exclude 'build/' ", globals.verbose ? "v" : "q"); - for (size_t i = 0; i < rootdirs_total; i++) { - char srcdir_bare[PATH_MAX] = {0}; - char srcdir_with_output[PATH_MAX] = {0}; - char *srcdir = srcdir_bare; - strcpy(srcdir_bare, rootdirs[i]); - strcpy(srcdir_with_output, rootdirs[i]); - strcat(srcdir_with_output, "/output"); - - if (access(srcdir_bare, F_OK)) { - fprintf(stderr, "%s does not exist\n", srcdir_bare); - continue; - } - - if (!access(srcdir_with_output, F_OK)) { - srcdir = srcdir_with_output; - } - snprintf(cmd + strlen(cmd), sizeof(srcdir) - strlen(srcdir) + 4, "'%s'/ ", srcdir); - } - snprintf(cmd + strlen(cmd), sizeof(cmd) - strlen(destdir) + 1, " %s/", destdir); - - if (globals.verbose) { - puts(cmd); - } - - if (system(cmd)) { - return -1; - } - return 0; -} - -int indexer_wheels(struct Delivery *ctx) { - return delivery_index_wheel_artifacts(ctx); -} - -int indexer_load_metadata(struct Delivery *ctx, const char *filename) { - char line[STASIS_NAME_MAX] = {0}; - - FILE *fp = fopen(filename, "r"); - if (!fp) { - return -1; - } - - while (fgets(line, sizeof(line) - 1, fp) != NULL) { - char **parts = split(line, " ", 1); - char *name = parts[0]; - char *value = parts[1]; - strip(value); - if (!strcmp(name, "name")) { - ctx->meta.name = strdup(value); - } else if (!strcmp(name, "version")) { - ctx->meta.version = strdup(value); - } else if (!strcmp(name, "rc")) { - ctx->meta.rc = (int) strtol(value, NULL, 10); - } else if (!strcmp(name, "python")) { - ctx->meta.python = strdup(value); - } else if (!strcmp(name, "python_compact")) { - ctx->meta.python_compact = strdup(value); - } else if (!strcmp(name, "mission")) { - ctx->meta.mission = strdup(value); - } else if (!strcmp(name, "codename")) { - ctx->meta.codename = strdup(value); - } else if (!strcmp(name, "platform")) { - ctx->system.platform = split(value, " ", 0); - } else if (!strcmp(name, "arch")) { - ctx->system.arch = strdup(value); - } else if (!strcmp(name, "time")) { - ctx->info.time_str_epoch = strdup(value); - } else if (!strcmp(name, "release_fmt")) { - ctx->rules.release_fmt = strdup(value); - } else if (!strcmp(name, "release_name")) { - ctx->info.release_name = strdup(value); - } else if (!strcmp(name, "build_name_fmt")) { - ctx->rules.build_name_fmt = strdup(value); - } else if (!strcmp(name, "build_name")) { - ctx->info.build_name = strdup(value); - } else if (!strcmp(name, "build_number_fmt")) { - ctx->rules.build_number_fmt = strdup(value); - } else if (!strcmp(name, "build_number")) { - ctx->info.build_number = strdup(value); - } else if (!strcmp(name, "conda_installer_baseurl")) { - ctx->conda.installer_baseurl = strdup(value); - } else if (!strcmp(name, "conda_installer_name")) { - ctx->conda.installer_name = strdup(value); - } else if (!strcmp(name, "conda_installer_version")) { - ctx->conda.installer_version = strdup(value); - } else if (!strcmp(name, "conda_installer_platform")) { - ctx->conda.installer_platform = strdup(value); - } else if (!strcmp(name, "conda_installer_arch")) { - ctx->conda.installer_arch = strdup(value); - } - GENERIC_ARRAY_FREE(parts); - } - fclose(fp); - - return 0; -} - -int indexer_get_files(struct StrList **out, const char *path, const char *pattern, ...) { - va_list args; - va_start(args, pattern); - char userpattern[PATH_MAX] = {0}; - vsprintf(userpattern, pattern, args); - va_end(args); - struct StrList *list = listdir(path); - if (!list) { - return -1; - } - - if (!(*out)) { - (*out) = strlist_init(); - if (!(*out)) { - guard_strlist_free(&list); - return -1; - } - } - - size_t no_match = 0; - for (size_t i = 0; i < strlist_count(list); i++) { - char *item = strlist_item(list, i); - if (fnmatch(userpattern, item, 0)) { - no_match++; - } else { - strlist_append((out), item); - } - } - if (no_match >= strlist_count(list)) { - fprintf(stderr, "no files matching the pattern: %s\n", userpattern); - guard_strlist_free(&list); - return -1; - } - guard_strlist_free(&list); - return 0; -} - -int get_latest_rc(struct Delivery ctx[], size_t nelem) { - int result = 0; - for (size_t i = 0; i < nelem; i++) { - if (ctx[i].meta.rc > result) { - result = ctx[i].meta.rc; - } - } - return result; -} - -struct Delivery **get_latest_deliveries(struct Delivery ctx[], size_t nelem) { - struct Delivery **result = NULL; - int latest = 0; - size_t n = 0; - - result = calloc(nelem + 1, sizeof(result)); - if (!result) { - fprintf(stderr, "Unable to allocate %zu bytes for result delivery array: %s\n", nelem * sizeof(result), strerror(errno)); - return NULL; - } - - latest = get_latest_rc(ctx, nelem); - for (size_t i = 0; i < nelem; i++) { - if (ctx[i].meta.rc == latest) { - result[n] = &ctx[i]; - n++; - } - } - - return result; -} - -int get_pandoc_version(size_t *result) { - *result = 0; - int state = 0; - char *version_str = shell_output("pandoc --version", &state); - if (state || !version_str) { - // an error occurred - return -1; - } - - // Verify that we're looking at pandoc - if (strlen(version_str) > 7 && !strncmp(version_str, "pandoc ", 7)) { - // we have pandoc - char *v_begin = &version_str[7]; - if (!v_begin) { - SYSERROR("unexpected pandoc output: %s", version_str); - return -1; - } - char *v_end = strchr(version_str, '\n'); - if (v_end) { - *v_end = 0; - } - - char **parts = split(v_begin, ".", 0); - if (!parts) { - SYSERROR("unable to split pandoc version string, '%s': %s", version_str, strerror(errno)); - return -1; - } - - size_t parts_total; - for (parts_total = 0; parts[parts_total] != NULL; parts_total++) {} - - // generate the version as an integer - // note: pandoc version scheme never exceeds four elements (or bytes in this case) - for (size_t i = 0; i < 4; i++) { - unsigned char tmp = 0; - if (i < parts_total) { - // only process version elements we have. the rest will be zeros. - tmp = strtoul(parts[i], NULL, 10); - } - // pack version element into result - *result = (*result << 8) | tmp; - } - } else { - // invalid version string - return 1; - } - - return 0; -} - -int indexer_make_website(struct Delivery *ctx) { - const char *pattern = "*.md"; - - if (!find_program("pandoc")) { - fprintf(stderr, "pandoc is not installed: unable to generate HTML indexes\n"); - return 0; - } - - char *css_filename = calloc(PATH_MAX, sizeof(*css_filename)); - if (!css_filename) { - SYSERROR("unable to allocate string for CSS file path: %s", strerror(errno)); - return -1; - } - - sprintf(css_filename, "%s/%s", globals.sysconfdir, "stasis_pandoc.css"); - int have_css = access(css_filename, F_OK | R_OK) == 0; - - char pandoc_versioned_args[255] = {0}; - size_t pandoc_version = 0; - - if (!get_pandoc_version(&pandoc_version)) { - // < 2.19 - if (pandoc_version < 0x02130000) { - strcat(pandoc_versioned_args, "--self-contained "); - } else { - // >= 2.19 - strcat(pandoc_versioned_args, "--embed-resources "); - } - - // >= 1.15.0.4 - if (pandoc_version >= 0x010f0004) { - strcat(pandoc_versioned_args, "--standalone "); - } - - // >= 1.10.0.1 - if (pandoc_version >= 0x010a0001) { - strcat(pandoc_versioned_args, "-f gfm+autolink_bare_uris "); - } - - // > 3.1.9 - if (pandoc_version > 0x03010900) { - strcat(pandoc_versioned_args, "-f gfm+alerts "); - } - } - - struct StrList *dirs = strlist_init(); - strlist_append(&dirs, ctx->storage.delivery_dir); - strlist_append(&dirs, ctx->storage.results_dir); - - struct StrList *inputs = NULL; - for (size_t i = 0; i < strlist_count(dirs); i++) { - if (indexer_get_files(&inputs, ctx->storage.delivery_dir, pattern)) { - SYSERROR("%s does not contain files with pattern: %s", ctx->storage.delivery_dir, pattern); - guard_strlist_free(&inputs); - continue; - } - char *root = strlist_item(dirs, i); - for (size_t x = 0; x < strlist_count(inputs); x++) { - char cmd[PATH_MAX] = {0}; - char *filename = strlist_item(inputs, x); - char fullpath_src[PATH_MAX] = {0}; - char fullpath_dest[PATH_MAX] = {0}; - sprintf(fullpath_src, "%s/%s", root, filename); - if (access(fullpath_src, F_OK)) { - continue; - } - - // Replace *.md extension with *.html. - strcpy(fullpath_dest, fullpath_src); - char *ext = strrchr(fullpath_dest, '.'); - if (ext) { - *ext = '\0'; - } - strcat(fullpath_dest, ".html"); - - // Converts a markdown file to html - strcpy(cmd, "pandoc "); - strcat(cmd, pandoc_versioned_args); - if (have_css) { - strcat(cmd, "--css "); - strcat(cmd, css_filename); - } - strcat(cmd, " "); - strcat(cmd, "--metadata title=\"STASIS\" "); - strcat(cmd, "-o "); - strcat(cmd, fullpath_dest); - strcat(cmd, " "); - strcat(cmd, fullpath_src); - if (globals.verbose) { - puts(cmd); - } - // This might be negative when killed by a signal. - // Otherwise, the return code is not critical to us. - if (system(cmd) < 0) { - guard_free(css_filename); - guard_strlist_free(&dirs); - return 1; - } - if (file_replace_text(fullpath_dest, ".md", ".html", 0)) { - // inform-only - SYSERROR("%s: failed to rewrite *.md urls with *.html extension", fullpath_dest); - } - - // Link the nearest README.html to index.html - if (!strcmp(filename, "README.md")) { - char link_from[PATH_MAX] = {0}; - char link_dest[PATH_MAX] = {0}; - strcpy(link_from, "README.html"); - sprintf(link_dest, "%s/%s", root, "index.html"); - if (symlink(link_from, link_dest)) { - SYSERROR("Warning: symlink(%s, %s) failed: %s", link_from, link_dest, strerror(errno)); - } - } - } - guard_strlist_free(&inputs); - } - guard_free(css_filename); - guard_strlist_free(&dirs); - - return 0; -} - -static int micromamba_configure(const struct Delivery *ctx, struct MicromambaInfo *m) { - int status = 0; - char *micromamba_prefix = NULL; - if (asprintf(µmamba_prefix, "%s/bin", ctx->storage.tools_dir) < 0) { - return -1; - } - m->conda_prefix = globals.conda_install_prefix; - m->micromamba_prefix = micromamba_prefix; - - size_t pathvar_len = (strlen(getenv("PATH")) + strlen(m->micromamba_prefix) + strlen(m->conda_prefix)) + 3 + 4 + 1; - // ^^^^^^^^^^^^^^^^^^ - // 3 = separators - // 4 = chars (/bin) - // 1 = nul terminator - char *pathvar = calloc(pathvar_len, sizeof(*pathvar)); - if (!pathvar) { - SYSERROR("%s", "Unable to allocate bytes for temporary path string"); - exit(1); - } - snprintf(pathvar, pathvar_len, "%s/bin:%s:%s", m->conda_prefix, m->micromamba_prefix, getenv("PATH")); - setenv("PATH", pathvar, 1); - guard_free(pathvar); - - status += micromamba(m, "config prepend --env channels conda-forge"); - if (!globals.verbose) { - status += micromamba(m, "config set --env quiet true"); - } - status += micromamba(m, "config set --env always_yes true"); - status += micromamba(m, "install conda-build pandoc"); - - return status; -} - -int indexer_conda(struct Delivery *ctx, struct MicromambaInfo m) { - int status = 0; - - status += micromamba(&m, "run conda index %s", ctx->storage.conda_artifact_dir); - return status; -} - -static struct StrList *get_architectures(struct Delivery ctx[], size_t nelem) { - struct StrList *architectures = strlist_init(); - for (size_t i = 0; i < nelem; i++) { - if (!strstr_array(architectures->data, ctx[i].system.arch)) { - strlist_append(&architectures, ctx[i].system.arch); - } - } - return architectures; -} - -static struct StrList *get_platforms(struct Delivery ctx[], size_t nelem) { - struct StrList *platforms = strlist_init(); - for (size_t i = 0; i < nelem; i++) { - if (!strstr_array(platforms->data, ctx[i].system.platform[DELIVERY_PLATFORM_RELEASE])) { - strlist_append(&platforms, ctx[i].system.platform[DELIVERY_PLATFORM_RELEASE]); - } - } - return platforms; -} - -int indexer_symlinks(struct Delivery ctx[], size_t nelem) { - struct Delivery **data = NULL; - data = get_latest_deliveries(ctx, nelem); - //int latest = get_latest_rc(ctx, nelem); - - if (!pushd(ctx->storage.delivery_dir)) { - for (size_t i = 0; i < nelem; i++) { - char link_name_spec[PATH_MAX]; - char link_name_readme[PATH_MAX]; - - char file_name_spec[PATH_MAX]; - char file_name_readme[PATH_MAX]; - - if (!data[i]) { - continue; - } - sprintf(link_name_spec, "latest-py%s-%s-%s.yml", data[i]->meta.python_compact, data[i]->system.platform[DELIVERY_PLATFORM_RELEASE], data[i]->system.arch); - sprintf(file_name_spec, "%s.yml", data[i]->info.release_name); - - sprintf(link_name_readme, "README-py%s-%s-%s.md", data[i]->meta.python_compact, data[i]->system.platform[DELIVERY_PLATFORM_RELEASE], data[i]->system.arch); - sprintf(file_name_readme, "README-%s.md", data[i]->info.release_name); - - if (!access(link_name_spec, F_OK)) { - if (unlink(link_name_spec)) { - fprintf(stderr, "Unable to remove spec link: %s\n", link_name_spec); - } - } - if (!access(link_name_readme, F_OK)) { - if (unlink(link_name_readme)) { - fprintf(stderr, "Unable to remove readme link: %s\n", link_name_readme); - } - } - - if (globals.verbose) { - printf("%s -> %s\n", file_name_spec, link_name_spec); - } - if (symlink(file_name_spec, link_name_spec)) { - fprintf(stderr, "Unable to link %s as %s\n", file_name_spec, link_name_spec); - } - - if (globals.verbose) { - printf("%s -> %s\n", file_name_readme, link_name_readme); - } - if (symlink(file_name_readme, link_name_readme)) { - fprintf(stderr, "Unable to link %s as %s\n", file_name_readme, link_name_readme); - } - } - popd(); - } else { - fprintf(stderr, "Unable to enter delivery directory: %s\n", ctx->storage.delivery_dir); - guard_free(data); - return -1; - } - - // "latest" is an array of pointers to ctx[]. Do not free the contents of the array. - guard_free(data); - return 0; -} - -int indexer_readmes(struct Delivery ctx[], size_t nelem) { - struct Delivery **latest = NULL; - latest = get_latest_deliveries(ctx, nelem); - - char indexfile[PATH_MAX] = {0}; - sprintf(indexfile, "%s/README.md", ctx->storage.delivery_dir); - - if (!pushd(ctx->storage.delivery_dir)) { - FILE *indexfp = fopen(indexfile, "w+"); - if (!indexfp) { - fprintf(stderr, "Unable to open %s for writing\n", indexfile); - return -1; - } - struct StrList *archs = get_architectures(*latest, nelem); - struct StrList *platforms = get_platforms(*latest, nelem); - - fprintf(indexfp, "# %s-%s\n\n", ctx->meta.name, ctx->meta.version); - fprintf(indexfp, "## Current Release\n\n"); - for (size_t p = 0; p < strlist_count(platforms); p++) { - char *platform = strlist_item(platforms, p); - for (size_t a = 0; a < strlist_count(archs); a++) { - char *arch = strlist_item(archs, a); - int have_combo = 0; - for (size_t i = 0; i < nelem; i++) { - if (latest[i] && latest[i]->system.platform) { - if (strstr(latest[i]->system.platform[DELIVERY_PLATFORM_RELEASE], platform) && - strstr(latest[i]->system.arch, arch)) { - have_combo = 1; - } - } - } - if (!have_combo) { - continue; - } - fprintf(indexfp, "### %s-%s\n\n", platform, arch); - - fprintf(indexfp, "|Release|Info|Receipt|\n"); - fprintf(indexfp, "|:----:|:----:|:----:|\n"); - for (size_t i = 0; i < nelem; i++) { - char link_name[PATH_MAX]; - char readme_name[PATH_MAX]; - char conf_name[PATH_MAX]; - char conf_name_relative[PATH_MAX]; - if (!latest[i]) { - continue; - } - sprintf(link_name, "latest-py%s-%s-%s.yml", latest[i]->meta.python_compact, latest[i]->system.platform[DELIVERY_PLATFORM_RELEASE], latest[i]->system.arch); - sprintf(readme_name, "README-py%s-%s-%s.md", latest[i]->meta.python_compact, latest[i]->system.platform[DELIVERY_PLATFORM_RELEASE], latest[i]->system.arch); - sprintf(conf_name, "%s.ini", latest[i]->info.release_name); - sprintf(conf_name_relative, "../config/%s-rendered.ini", latest[i]->info.release_name); - if (strstr(link_name, platform) && strstr(link_name, arch)) { - fprintf(indexfp, "|[%s](%s)|[%s](%s)|[%s](%s)|\n", link_name, link_name, readme_name, readme_name, conf_name, conf_name_relative); - } - } - fprintf(indexfp, "\n"); - } - fprintf(indexfp, "\n"); - } - guard_strlist_free(&archs); - guard_strlist_free(&platforms); - fclose(indexfp); - popd(); - } else { - fprintf(stderr, "Unable to enter delivery directory: %s\n", ctx->storage.delivery_dir); - guard_free(latest); - return -1; - } - - // "latest" is an array of pointers to ctxs[]. Do not free the contents of the array. - guard_free(latest); - return 0; -} - -int indexer_junitxml_report(struct Delivery ctx[], size_t nelem) { - struct Delivery **latest = NULL; - latest = get_latest_deliveries(ctx, nelem); - - char indexfile[PATH_MAX] = {0}; - sprintf(indexfile, "%s/README.md", ctx->storage.results_dir); - - struct StrList *file_listing = listdir(ctx->storage.results_dir); - if (!file_listing) { - // no test results to process - return 0; - } - - if (!pushd(ctx->storage.results_dir)) { - FILE *indexfp = fopen(indexfile, "w+"); - if (!indexfp) { - fprintf(stderr, "Unable to open %s for writing\n", indexfile); - return -1; - } - struct StrList *archs = get_architectures(*latest, nelem); - struct StrList *platforms = get_platforms(*latest, nelem); - - fprintf(indexfp, "# %s-%s Test Report\n\n", ctx->meta.name, ctx->meta.version); - fprintf(indexfp, "## Current Release\n\n"); - for (size_t p = 0; p < strlist_count(platforms); p++) { - char *platform = strlist_item(platforms, p); - for (size_t a = 0; a < strlist_count(archs); a++) { - char *arch = strlist_item(archs, a); - int have_combo = 0; - for (size_t i = 0; i < nelem; i++) { - if (latest[i] && latest[i]->system.platform) { - if (strstr(latest[i]->system.platform[DELIVERY_PLATFORM_RELEASE], platform) && - strstr(latest[i]->system.arch, arch)) { - have_combo = 1; - break; - } - } - } - if (!have_combo) { - continue; - } - fprintf(indexfp, "### %s-%s\n\n", platform, arch); - - fprintf(indexfp, "|Suite|Duration|Fail |Skip |Error |\n"); - fprintf(indexfp, "|:----|:------:|:------:|:---:|:----:|\n"); - for (size_t f = 0; f < strlist_count(file_listing); f++) { - char *filename = strlist_item(file_listing, f); - if (!endswith(filename, ".xml")) { - continue; - } - - if (strstr(filename, platform) && strstr(filename, arch)) { - struct JUNIT_Testsuite *testsuite = junitxml_testsuite_read(filename); - if (testsuite) { - if (globals.verbose) { - printf("%s: duration: %0.4f, failed: %d, skipped: %d, errors: %d\n", filename, testsuite->time, testsuite->failures, testsuite->skipped, testsuite->errors); - } - fprintf(indexfp, "|[%s](%s)|%0.4f|%d|%d|%d|\n", filename, filename, testsuite->time, testsuite->failures, testsuite->skipped, testsuite->errors); - /* - * TODO: Display failure/skip/error output. - * - for (size_t i = 0; i < testsuite->_tc_inuse; i++) { - if (testsuite->testcase[i]->tc_result_state_type) { - printf("testcase: %s :: %s\n", testsuite->testcase[i]->classname, testsuite->testcase[i]->name); - if (testsuite->testcase[i]->tc_result_state_type == JUNIT_RESULT_STATE_FAILURE) { - printf("failure: %s\n", testsuite->testcase[i]->result_state.failure->message); - } else if (testsuite->testcase[i]->tc_result_state_type == JUNIT_RESULT_STATE_SKIPPED) { - printf("skipped: %s\n", testsuite->testcase[i]->result_state.skipped->message); - } - } - } - */ - junitxml_testsuite_free(&testsuite); - } else { - fprintf(stderr, "bad test suite: %s: %s\n", strerror(errno), filename); - continue; - } - } - } - fprintf(indexfp, "\n"); - } - fprintf(indexfp, "\n"); - } - guard_strlist_free(&archs); - guard_strlist_free(&platforms); - fclose(indexfp); - popd(); - } else { - fprintf(stderr, "Unable to enter delivery directory: %s\n", ctx->storage.delivery_dir); - guard_free(latest); - return -1; - } - - // "latest" is an array of pointers to ctxs[]. Do not free the contents of the array. - guard_free(latest); - return 0; -} - -void indexer_init_dirs(struct Delivery *ctx, const char *workdir) { - path_store(&ctx->storage.root, PATH_MAX, workdir, ""); - path_store(&ctx->storage.tmpdir, PATH_MAX, ctx->storage.root, "tmp"); - if (delivery_init_tmpdir(ctx)) { - fprintf(stderr, "Failed to configure temporary storage directory\n"); - exit(1); - } - path_store(&ctx->storage.output_dir, PATH_MAX, ctx->storage.root, ""); - path_store(&ctx->storage.tools_dir, PATH_MAX, ctx->storage.output_dir, "tools"); - path_store(&globals.conda_install_prefix, PATH_MAX, ctx->storage.tools_dir, "conda"); - path_store(&ctx->storage.cfgdump_dir, PATH_MAX, ctx->storage.output_dir, "config"); - path_store(&ctx->storage.meta_dir, PATH_MAX, ctx->storage.output_dir, "meta"); - path_store(&ctx->storage.delivery_dir, PATH_MAX, ctx->storage.output_dir, "delivery"); - path_store(&ctx->storage.package_dir, PATH_MAX, ctx->storage.output_dir, "packages"); - path_store(&ctx->storage.results_dir, PATH_MAX, ctx->storage.output_dir, "results"); - path_store(&ctx->storage.wheel_artifact_dir, PATH_MAX, ctx->storage.package_dir, "wheels"); - path_store(&ctx->storage.conda_artifact_dir, PATH_MAX, ctx->storage.package_dir, "conda"); - - char newpath[PATH_MAX] = {0}; - if (getenv("PATH")) { - sprintf(newpath, "%s/bin:%s", ctx->storage.tools_dir, getenv("PATH")); - setenv("PATH", newpath, 1); - } else { - SYSERROR("%s", "environment variable PATH is undefined. Unable to continue."); - exit(1); - } -} - -static int sort_deliveries_cmpfn(const void *a, const void *b) { - const struct Delivery *delivery1 = (struct Delivery *) a; - const size_t delivery1_python = strtoul(delivery1->meta.python_compact, NULL, 10); - const struct Delivery *delivery2 = (struct Delivery *) b; - const size_t delivery2_python = strtoul(delivery2->meta.python_compact, NULL, 10); - - if (delivery2_python > delivery1_python) { - return 1; - } - if (delivery2_python < delivery1_python) { - return -1; - } - return 0; -} - -int main(int argc, char *argv[]) { - size_t rootdirs_total = 0; - char *destdir = NULL; - char **rootdirs = NULL; - int do_html = 0; - int c = 0; - int option_index = 0; - while ((c = getopt_long(argc, argv, "hd:vUw", long_options, &option_index)) != -1) { - switch (c) { - case 'h': - usage(path_basename(argv[0])); - exit(0); - case 'd': - if (mkdir(optarg, 0755)) { - if (errno != 0 && errno != EEXIST) { - SYSERROR("Unable to create destination directory, '%s': %s", optarg, strerror(errno)); - exit(1); - } - } - destdir = realpath(optarg, NULL); - break; - case 'U': - fflush(stdout); - fflush(stderr); - setvbuf(stdout, NULL, _IONBF, 0); - setvbuf(stderr, NULL, _IONBF, 0); - break; - case 'v': - globals.verbose = 1; - break; - case 'w': - do_html = 1; - break; - case '?': - default: - exit(1); - } - } - - int current_index = optind; - if (optind < argc) { - rootdirs_total = argc - current_index; - rootdirs = calloc(rootdirs_total + 1, sizeof(**rootdirs)); - - int i = 0; - while (optind < argc) { - if (argv[optind]) { - if (access(argv[optind], F_OK) < 0) { - fprintf(stderr, "%s: %s\n", argv[optind], strerror(errno)); - exit(1); - } - } - // use first positional argument - rootdirs[i] = realpath(argv[optind], NULL); - optind++; - break; - } - } - - if (isempty(destdir)) { - if (mkdir("output", 0755)) { - if (errno != 0 && errno != EEXIST) { - SYSERROR("Unable to create destination directory, '%s': %s", "output", strerror(errno)); - exit(1); - } - } - destdir = realpath("output", NULL); - } - - if (!rootdirs || !rootdirs_total) { - fprintf(stderr, "You must specify at least one STASIS root directory to index\n"); - exit(1); - } else { - for (size_t i = 0; i < rootdirs_total; i++) { - if (isempty(rootdirs[i]) || !strcmp(rootdirs[i], "/") || !strcmp(rootdirs[i], "\\")) { - SYSERROR("Unsafe directory: %s", rootdirs[i]); - exit(1); - } else if (access(rootdirs[i], F_OK)) { - SYSERROR("%s: %s", rootdirs[i], strerror(errno)); - exit(1); - } - } - } - - char stasis_sysconfdir_tmp[PATH_MAX]; - if (getenv("STASIS_SYSCONFDIR")) { - strncpy(stasis_sysconfdir_tmp, getenv("STASIS_SYSCONFDIR"), sizeof(stasis_sysconfdir_tmp) - 1); - } else { - strncpy(stasis_sysconfdir_tmp, STASIS_SYSCONFDIR, sizeof(stasis_sysconfdir_tmp) - 1); - } - - globals.sysconfdir = realpath(stasis_sysconfdir_tmp, NULL); - if (!globals.sysconfdir) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Unable to resolve path to configuration directory: %s\n", stasis_sysconfdir_tmp); - exit(1); - } - - char workdir_template[PATH_MAX] = {0}; - char *system_tmp = getenv("TMPDIR"); - if (system_tmp) { - strcat(workdir_template, system_tmp); - } else { - strcat(workdir_template, "/tmp"); - } - strcat(workdir_template, "/stasis-combine.XXXXXX"); - char *workdir = mkdtemp(workdir_template); - if (!workdir) { - SYSERROR("Unable to create temporary directory: %s", workdir_template); - exit(1); - } else if (isempty(workdir) || !strcmp(workdir, "/") || !strcmp(workdir, "\\")) { - SYSERROR("Unsafe directory: %s", workdir); - exit(1); - } - - struct Delivery ctx = {0}; - - printf(BANNER, VERSION, AUTHOR); - - indexer_init_dirs(&ctx, workdir); - - msg(STASIS_MSG_L1, "%s delivery root %s\n", - rootdirs_total > 1 ? "Merging" : "Indexing", - rootdirs_total > 1 ? "directories" : "directory"); - if (indexer_combine_rootdirs(workdir, rootdirs, rootdirs_total)) { - SYSERROR("%s", "Copy operation failed"); - rmtree(workdir); - exit(1); - } - - if (access(ctx.storage.conda_artifact_dir, F_OK)) { - mkdirs(ctx.storage.conda_artifact_dir, 0755); - } - - if (access(ctx.storage.wheel_artifact_dir, F_OK)) { - mkdirs(ctx.storage.wheel_artifact_dir, 0755); - } - - struct MicromambaInfo m; - if (micromamba_configure(&ctx, &m)) { - SYSERROR("%s", "Unable to configure micromamba"); - exit(1); - } - - msg(STASIS_MSG_L1, "Indexing conda packages\n"); - if (indexer_conda(&ctx, m)) { - SYSERROR("%s", "Conda package indexing operation failed"); - exit(1); - } - - msg(STASIS_MSG_L1, "Indexing wheel packages\n"); - if (indexer_wheels(&ctx)) { - SYSERROR("%s", "Python package indexing operation failed"); - exit(1); - } - - msg(STASIS_MSG_L1, "Loading metadata\n"); - struct StrList *metafiles = NULL; - indexer_get_files(&metafiles, ctx.storage.meta_dir, "*.stasis"); - strlist_sort(metafiles, STASIS_SORT_LEN_ASCENDING); - struct Delivery local[strlist_count(metafiles)]; - - for (size_t i = 0; i < strlist_count(metafiles); i++) { - char *item = strlist_item(metafiles, i); - memset(&local[i], 0, sizeof(ctx)); - memcpy(&local[i], &ctx, sizeof(ctx)); - char path[PATH_MAX]; - sprintf(path, "%s/%s", ctx.storage.meta_dir, item); - if (globals.verbose) { - puts(path); - } - indexer_load_metadata(&local[i], path); - } - qsort(local, strlist_count(metafiles), sizeof(*local), sort_deliveries_cmpfn); - - msg(STASIS_MSG_L1, "Generating links to latest release iteration\n"); - if (indexer_symlinks(local, strlist_count(metafiles))) { - SYSERROR("%s", "Link generation failed"); - exit(1); - } - - msg(STASIS_MSG_L1, "Generating README.md\n"); - if (indexer_readmes(local, strlist_count(metafiles))) { - SYSERROR("%s", "README indexing operation failed"); - exit(1); - } - - msg(STASIS_MSG_L1, "Indexing test results\n"); - if (indexer_junitxml_report(local, strlist_count(metafiles))) { - SYSERROR("%s", "Test result indexing operation failed"); - exit(1); - } - - if (do_html) { - msg(STASIS_MSG_L1, "Generating HTML indexes\n"); - if (indexer_make_website(local)) { - SYSERROR("%s", "Site creation failed"); - exit(1); - } - } - - msg(STASIS_MSG_L1, "Copying indexed delivery to '%s'\n", destdir); - char cmd[PATH_MAX] = {0}; - sprintf(cmd, "rsync -ah%s --delete --exclude 'tmp/' --exclude 'tools/' '%s/' '%s/'", globals.verbose ? "v" : "q", workdir, destdir); - guard_free(destdir); - - if (globals.verbose) { - puts(cmd); - } - - if (system(cmd)) { - SYSERROR("%s", "Copy operation failed"); - rmtree(workdir); - exit(1); - } - - msg(STASIS_MSG_L1, "Removing work directory: %s\n", workdir); - if (rmtree(workdir)) { - SYSERROR("Failed to remove work directory: %s", strerror(errno)); - } - - guard_free(destdir); - GENERIC_ARRAY_FREE(rootdirs); - guard_strlist_free(&metafiles); - delivery_free(&ctx); - globals_free(); - msg(STASIS_MSG_L1, "Done!\n"); - return 0; -} diff --git a/src/cli/stasis_indexer/stasis_indexer_main.c b/src/cli/stasis_indexer/stasis_indexer_main.c new file mode 100644 index 0000000..6c318c9 --- /dev/null +++ b/src/cli/stasis_indexer/stasis_indexer_main.c @@ -0,0 +1,387 @@ +#include +#include "args.h" +#include "callbacks.h" +#include "helpers.h" +#include "junitxml_report.h" +#include "website.h" +#include "readmes.h" +#include "delivery.h" + +int indexer_combine_rootdirs(const char *dest, char **rootdirs, const size_t rootdirs_total) { + char cmd[PATH_MAX]; + char destdir_bare[PATH_MAX]; + char destdir_with_output[PATH_MAX]; + char *destdir = destdir_bare; + + memset(cmd, 0, sizeof(cmd)); + memset(destdir_bare, 0, sizeof(destdir_bare)); + memset(destdir_with_output, 0, sizeof(destdir_bare)); + + strcpy(destdir_bare, dest); + strcpy(destdir_with_output, dest); + strcat(destdir_with_output, "/output"); + + if (!access(destdir_with_output, F_OK)) { + destdir = destdir_with_output; + } + + sprintf(cmd, "rsync -ah%s --delete --exclude 'tools/' --exclude 'tmp/' --exclude 'build/' ", globals.verbose ? "v" : "q"); + for (size_t i = 0; i < rootdirs_total; i++) { + char srcdir_bare[PATH_MAX] = {0}; + char srcdir_with_output[PATH_MAX] = {0}; + char *srcdir = srcdir_bare; + strcpy(srcdir_bare, rootdirs[i]); + strcpy(srcdir_with_output, rootdirs[i]); + strcat(srcdir_with_output, "/output"); + + if (access(srcdir_bare, F_OK)) { + fprintf(stderr, "%s does not exist\n", srcdir_bare); + continue; + } + + if (!access(srcdir_with_output, F_OK)) { + srcdir = srcdir_with_output; + } + snprintf(cmd + strlen(cmd), sizeof(srcdir) - strlen(srcdir) + 4, "'%s'/ ", srcdir); + } + snprintf(cmd + strlen(cmd), sizeof(cmd) - strlen(destdir) + 1, " %s/", destdir); + + if (globals.verbose) { + puts(cmd); + } + + if (system(cmd)) { + return -1; + } + return 0; +} + +int indexer_wheels(struct Delivery *ctx) { + return delivery_index_wheel_artifacts(ctx); +} + +int indexer_conda(const struct Delivery *ctx, struct MicromambaInfo m) { + int status = 0; + + status += micromamba(&m, "run conda index %s", ctx->storage.conda_artifact_dir); + return status; +} + +int indexer_symlinks(struct Delivery ctx[], const size_t nelem) { + struct Delivery **data = NULL; + data = get_latest_deliveries(ctx, nelem); + //int latest = get_latest_rc(ctx, nelem); + + if (!pushd(ctx->storage.delivery_dir)) { + for (size_t i = 0; i < nelem; i++) { + char link_name_spec[PATH_MAX]; + char link_name_readme[PATH_MAX]; + + char file_name_spec[PATH_MAX]; + char file_name_readme[PATH_MAX]; + + if (!data[i]) { + continue; + } + sprintf(link_name_spec, "latest-py%s-%s-%s.yml", data[i]->meta.python_compact, data[i]->system.platform[DELIVERY_PLATFORM_RELEASE], data[i]->system.arch); + sprintf(file_name_spec, "%s.yml", data[i]->info.release_name); + + sprintf(link_name_readme, "README-py%s-%s-%s.md", data[i]->meta.python_compact, data[i]->system.platform[DELIVERY_PLATFORM_RELEASE], data[i]->system.arch); + sprintf(file_name_readme, "README-%s.md", data[i]->info.release_name); + + if (!access(link_name_spec, F_OK)) { + if (unlink(link_name_spec)) { + fprintf(stderr, "Unable to remove spec link: %s\n", link_name_spec); + } + } + if (!access(link_name_readme, F_OK)) { + if (unlink(link_name_readme)) { + fprintf(stderr, "Unable to remove readme link: %s\n", link_name_readme); + } + } + + if (globals.verbose) { + printf("%s -> %s\n", file_name_spec, link_name_spec); + } + if (symlink(file_name_spec, link_name_spec)) { + fprintf(stderr, "Unable to link %s as %s\n", file_name_spec, link_name_spec); + } + + if (globals.verbose) { + printf("%s -> %s\n", file_name_readme, link_name_readme); + } + if (symlink(file_name_readme, link_name_readme)) { + fprintf(stderr, "Unable to link %s as %s\n", file_name_readme, link_name_readme); + } + } + popd(); + } else { + fprintf(stderr, "Unable to enter delivery directory: %s\n", ctx->storage.delivery_dir); + guard_free(data); + return -1; + } + + // "latest" is an array of pointers to ctx[]. Do not free the contents of the array. + guard_free(data); + return 0; +} + +void indexer_init_dirs(struct Delivery *ctx, const char *workdir) { + path_store(&ctx->storage.root, PATH_MAX, workdir, ""); + path_store(&ctx->storage.tmpdir, PATH_MAX, ctx->storage.root, "tmp"); + if (delivery_init_tmpdir(ctx)) { + fprintf(stderr, "Failed to configure temporary storage directory\n"); + exit(1); + } + path_store(&ctx->storage.output_dir, PATH_MAX, ctx->storage.root, ""); + path_store(&ctx->storage.tools_dir, PATH_MAX, ctx->storage.output_dir, "tools"); + path_store(&globals.conda_install_prefix, PATH_MAX, ctx->storage.tools_dir, "conda"); + path_store(&ctx->storage.cfgdump_dir, PATH_MAX, ctx->storage.output_dir, "config"); + path_store(&ctx->storage.meta_dir, PATH_MAX, ctx->storage.output_dir, "meta"); + path_store(&ctx->storage.delivery_dir, PATH_MAX, ctx->storage.output_dir, "delivery"); + path_store(&ctx->storage.package_dir, PATH_MAX, ctx->storage.output_dir, "packages"); + path_store(&ctx->storage.results_dir, PATH_MAX, ctx->storage.output_dir, "results"); + path_store(&ctx->storage.wheel_artifact_dir, PATH_MAX, ctx->storage.package_dir, "wheels"); + path_store(&ctx->storage.conda_artifact_dir, PATH_MAX, ctx->storage.package_dir, "conda"); + + char newpath[PATH_MAX] = {0}; + if (getenv("PATH")) { + sprintf(newpath, "%s/bin:%s", ctx->storage.tools_dir, getenv("PATH")); + setenv("PATH", newpath, 1); + } else { + SYSERROR("%s", "environment variable PATH is undefined. Unable to continue."); + exit(1); + } +} + +int main(const int argc, char *argv[]) { + size_t rootdirs_total = 0; + char *destdir = NULL; + char **rootdirs = NULL; + int do_html = 0; + int c = 0; + int option_index = 0; + while ((c = getopt_long(argc, argv, "hd:vUw", long_options, &option_index)) != -1) { + switch (c) { + case 'h': + usage(path_basename(argv[0])); + exit(0); + case 'd': + if (mkdir(optarg, 0755)) { + if (errno != 0 && errno != EEXIST) { + SYSERROR("Unable to create destination directory, '%s': %s", optarg, strerror(errno)); + exit(1); + } + } + destdir = realpath(optarg, NULL); + break; + case 'U': + fflush(stdout); + fflush(stderr); + setvbuf(stdout, NULL, _IONBF, 0); + setvbuf(stderr, NULL, _IONBF, 0); + break; + case 'v': + globals.verbose = 1; + break; + case 'w': + do_html = 1; + break; + case '?': + default: + exit(1); + } + } + + const int current_index = optind; + if (optind < argc) { + rootdirs_total = argc - current_index; + rootdirs = calloc(rootdirs_total + 1, sizeof(**rootdirs)); + + int i = 0; + while (optind < argc) { + if (argv[optind]) { + if (access(argv[optind], F_OK) < 0) { + fprintf(stderr, "%s: %s\n", argv[optind], strerror(errno)); + exit(1); + } + } + // use first positional argument + rootdirs[i] = realpath(argv[optind], NULL); + optind++; + break; + } + } + + if (isempty(destdir)) { + if (mkdir("output", 0755)) { + if (errno != 0 && errno != EEXIST) { + SYSERROR("Unable to create destination directory, '%s': %s", "output", strerror(errno)); + exit(1); + } + } + destdir = realpath("output", NULL); + } + + if (!rootdirs || !rootdirs_total) { + fprintf(stderr, "You must specify at least one STASIS root directory to index\n"); + exit(1); + } + + for (size_t i = 0; i < rootdirs_total; i++) { + if (isempty(rootdirs[i]) || !strcmp(rootdirs[i], "/") || !strcmp(rootdirs[i], "\\")) { + SYSERROR("Unsafe directory: %s", rootdirs[i]); + exit(1); + } + + if (access(rootdirs[i], F_OK)) { + SYSERROR("%s: %s", rootdirs[i], strerror(errno)); + exit(1); + } + } + + char stasis_sysconfdir_tmp[PATH_MAX]; + if (getenv("STASIS_SYSCONFDIR")) { + strncpy(stasis_sysconfdir_tmp, getenv("STASIS_SYSCONFDIR"), sizeof(stasis_sysconfdir_tmp) - 1); + } else { + strncpy(stasis_sysconfdir_tmp, STASIS_SYSCONFDIR, sizeof(stasis_sysconfdir_tmp) - 1); + } + + globals.sysconfdir = realpath(stasis_sysconfdir_tmp, NULL); + if (!globals.sysconfdir) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Unable to resolve path to configuration directory: %s\n", stasis_sysconfdir_tmp); + exit(1); + } + + char workdir_template[PATH_MAX] = {0}; + const char *system_tmp = getenv("TMPDIR"); + if (system_tmp) { + strcat(workdir_template, system_tmp); + } else { + strcat(workdir_template, "/tmp"); + } + strcat(workdir_template, "/stasis-combine.XXXXXX"); + char *workdir = mkdtemp(workdir_template); + if (!workdir) { + SYSERROR("Unable to create temporary directory: %s", workdir_template); + exit(1); + } + if (isempty(workdir) || !strcmp(workdir, "/") || !strcmp(workdir, "\\")) { + SYSERROR("Unsafe directory: %s", workdir); + exit(1); + } + + struct Delivery ctx = {0}; + + printf(BANNER, VERSION, AUTHOR); + + indexer_init_dirs(&ctx, workdir); + + msg(STASIS_MSG_L1, "%s delivery root %s\n", + rootdirs_total > 1 ? "Merging" : "Indexing", + rootdirs_total > 1 ? "directories" : "directory"); + if (indexer_combine_rootdirs(workdir, rootdirs, rootdirs_total)) { + SYSERROR("%s", "Copy operation failed"); + rmtree(workdir); + exit(1); + } + + if (access(ctx.storage.conda_artifact_dir, F_OK)) { + mkdirs(ctx.storage.conda_artifact_dir, 0755); + } + + if (access(ctx.storage.wheel_artifact_dir, F_OK)) { + mkdirs(ctx.storage.wheel_artifact_dir, 0755); + } + + struct MicromambaInfo m; + if (micromamba_configure(&ctx, &m)) { + SYSERROR("%s", "Unable to configure micromamba"); + exit(1); + } + + msg(STASIS_MSG_L1, "Indexing conda packages\n"); + if (indexer_conda(&ctx, m)) { + SYSERROR("%s", "Conda package indexing operation failed"); + exit(1); + } + + msg(STASIS_MSG_L1, "Indexing wheel packages\n"); + if (indexer_wheels(&ctx)) { + SYSERROR("%s", "Python package indexing operation failed"); + exit(1); + } + + msg(STASIS_MSG_L1, "Loading metadata\n"); + struct StrList *metafiles = NULL; + get_files(&metafiles, ctx.storage.meta_dir, "*.stasis"); + strlist_sort(metafiles, STASIS_SORT_LEN_ASCENDING); + struct Delivery local[strlist_count(metafiles)]; + + for (size_t i = 0; i < strlist_count(metafiles); i++) { + char *item = strlist_item(metafiles, i); + memset(&local[i], 0, sizeof(ctx)); + memcpy(&local[i], &ctx, sizeof(ctx)); + char path[PATH_MAX]; + sprintf(path, "%s/%s", ctx.storage.meta_dir, item); + if (globals.verbose) { + puts(path); + } + load_metadata(&local[i], path); + } + qsort(local, strlist_count(metafiles), sizeof(*local), callback_sort_deliveries_cmpfn); + + msg(STASIS_MSG_L1, "Generating links to latest release iteration\n"); + if (indexer_symlinks(local, strlist_count(metafiles))) { + SYSERROR("%s", "Link generation failed"); + exit(1); + } + + msg(STASIS_MSG_L1, "Generating README.md\n"); + if (indexer_readmes(local, strlist_count(metafiles))) { + SYSERROR("%s", "README indexing operation failed"); + exit(1); + } + + msg(STASIS_MSG_L1, "Indexing test results\n"); + if (indexer_junitxml_report(local, strlist_count(metafiles))) { + SYSERROR("%s", "Test result indexing operation failed"); + exit(1); + } + + if (do_html) { + msg(STASIS_MSG_L1, "Generating HTML indexes\n"); + if (indexer_make_website(local)) { + SYSERROR("%s", "Site creation failed"); + exit(1); + } + } + + msg(STASIS_MSG_L1, "Copying indexed delivery to '%s'\n", destdir); + char cmd[PATH_MAX] = {0}; + sprintf(cmd, "rsync -ah%s --delete --exclude 'tmp/' --exclude 'tools/' '%s/' '%s/'", globals.verbose ? "v" : "q", workdir, destdir); + guard_free(destdir); + + if (globals.verbose) { + puts(cmd); + } + + if (system(cmd)) { + SYSERROR("%s", "Copy operation failed"); + rmtree(workdir); + exit(1); + } + + msg(STASIS_MSG_L1, "Removing work directory: %s\n", workdir); + if (rmtree(workdir)) { + SYSERROR("Failed to remove work directory: %s", strerror(errno)); + } + + guard_free(destdir); + GENERIC_ARRAY_FREE(rootdirs); + guard_strlist_free(&metafiles); + delivery_free(&ctx); + globals_free(); + msg(STASIS_MSG_L1, "Done!\n"); + return 0; +} diff --git a/src/cli/stasis_indexer/website.c b/src/cli/stasis_indexer/website.c new file mode 100644 index 0000000..c04b7f2 --- /dev/null +++ b/src/cli/stasis_indexer/website.c @@ -0,0 +1,119 @@ +#include "core.h" +#include "website.h" + +int indexer_make_website(const struct Delivery *ctx) { + if (!find_program("pandoc")) { + fprintf(stderr, "pandoc is not installed: unable to generate HTML indexes\n"); + return 0; + } + + char *css_filename = calloc(PATH_MAX, sizeof(*css_filename)); + if (!css_filename) { + SYSERROR("unable to allocate string for CSS file path: %s", strerror(errno)); + return -1; + } + + sprintf(css_filename, "%s/%s", globals.sysconfdir, "stasis_pandoc.css"); + const int have_css = access(css_filename, F_OK | R_OK) == 0; + + char pandoc_versioned_args[255] = {0}; + size_t pandoc_version = 0; + + if (!get_pandoc_version(&pandoc_version)) { + // < 2.19 + if (pandoc_version < 0x02130000) { + strcat(pandoc_versioned_args, "--self-contained "); + } else { + // >= 2.19 + strcat(pandoc_versioned_args, "--embed-resources "); + } + + // >= 1.15.0.4 + if (pandoc_version >= 0x010f0004) { + strcat(pandoc_versioned_args, "--standalone "); + } + + // >= 1.10.0.1 + if (pandoc_version >= 0x010a0001) { + strcat(pandoc_versioned_args, "-f gfm+autolink_bare_uris "); + } + + // > 3.1.9 + if (pandoc_version > 0x03010900) { + strcat(pandoc_versioned_args, "-f gfm+alerts "); + } + } + + struct StrList *dirs = strlist_init(); + strlist_append(&dirs, ctx->storage.delivery_dir); + strlist_append(&dirs, ctx->storage.results_dir); + + struct StrList *inputs = NULL; + for (size_t i = 0; i < strlist_count(dirs); i++) { + const char *pattern = "*.md"; + if (get_files(&inputs, ctx->storage.delivery_dir, pattern)) { + SYSERROR("%s does not contain files with pattern: %s", ctx->storage.delivery_dir, pattern); + guard_strlist_free(&inputs); + continue; + } + char *root = strlist_item(dirs, i); + for (size_t x = 0; x < strlist_count(inputs); x++) { + char cmd[PATH_MAX] = {0}; + char *filename = strlist_item(inputs, x); + char fullpath_src[PATH_MAX] = {0}; + char fullpath_dest[PATH_MAX] = {0}; + sprintf(fullpath_src, "%s/%s", root, filename); + if (access(fullpath_src, F_OK)) { + continue; + } + + // Replace *.md extension with *.html. + strcpy(fullpath_dest, fullpath_src); + gen_file_extension_str(fullpath_dest, ".html"); + + // Converts a markdown file to html + strcpy(cmd, "pandoc "); + strcat(cmd, pandoc_versioned_args); + if (have_css) { + strcat(cmd, "--css "); + strcat(cmd, css_filename); + } + strcat(cmd, " "); + strcat(cmd, "--metadata title=\"STASIS\" "); + strcat(cmd, "-o "); + strcat(cmd, fullpath_dest); + strcat(cmd, " "); + strcat(cmd, fullpath_src); + if (globals.verbose) { + puts(cmd); + } + // This might be negative when killed by a signal. + // Otherwise, the return code is not critical to us. + if (system(cmd) < 0) { + guard_free(css_filename); + guard_strlist_free(&dirs); + return 1; + } + if (file_replace_text(fullpath_dest, ".md", ".html", 0)) { + // inform-only + SYSERROR("%s: failed to rewrite *.md urls with *.html extension", fullpath_dest); + } + + // Link the nearest README.html to index.html + if (!strcmp(filename, "README.md")) { + char link_from[PATH_MAX] = {0}; + char link_dest[PATH_MAX] = {0}; + strcpy(link_from, "README.html"); + sprintf(link_dest, "%s/%s", root, "index.html"); + if (symlink(link_from, link_dest)) { + SYSERROR("Warning: symlink(%s, %s) failed: %s", link_from, link_dest, strerror(errno)); + } + } + } + guard_strlist_free(&inputs); + } + guard_free(css_filename); + guard_strlist_free(&dirs); + + return 0; +} diff --git a/src/cli/stasis_indexer/website.h b/src/cli/stasis_indexer/website.h new file mode 100644 index 0000000..e67d58b --- /dev/null +++ b/src/cli/stasis_indexer/website.h @@ -0,0 +1,8 @@ +#ifndef WEBSITE_H +#define WEBSITE_H + +#include "helpers.h" + +int indexer_make_website(const struct Delivery *ctx); + +#endif //WEBSITE_H diff --git a/src/lib/core/utils.c b/src/lib/core/utils.c index 18731e6..c03f8fa 100644 --- a/src/lib/core/utils.c +++ b/src/lib/core/utils.c @@ -850,3 +850,13 @@ int env_manipulate_pathstr(const char *key, char *path, int mode) { return 0; } +int gen_file_extension_str(char *filename, const char *extension) { + char *ext_orig = strrchr(filename, '.'); + if (!ext_orig) { + strcat(filename, extension); + return 0; + } + + return replace_text(ext_orig, ext_orig, extension, 0); +} + -- cgit From e4e16a440b9ad45b81814db20da0aa912e2b28d2 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Mon, 18 Nov 2024 12:17:58 -0500 Subject: Disable "No data" row for now --- src/cli/stasis_indexer/junitxml_report.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/cli/stasis_indexer/junitxml_report.c b/src/cli/stasis_indexer/junitxml_report.c index 571f1b7..7128331 100644 --- a/src/cli/stasis_indexer/junitxml_report.c +++ b/src/cli/stasis_indexer/junitxml_report.c @@ -109,7 +109,8 @@ int indexer_junitxml_report(struct Delivery ctx[], const size_t nelem) { } } else { if (!no_printable_data) { - fprintf(indexfp, "|No data|-|-|-|-|-|-|\n"); + // Triggering for reasons unknown + //fprintf(indexfp, "|No data|-|-|-|-|-|-|\n"); no_printable_data++; } } -- cgit From 93ad037af095d490e836a16b92e23f637f1f61ab Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Wed, 20 Nov 2024 22:32:01 -0500 Subject: pip_packages now accepts name[extra1,extra2]==version strings --- src/lib/core/delivery.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/lib/core/delivery.c b/src/lib/core/delivery.c index 1ceb8b7..aa3e51a 100644 --- a/src/lib/core/delivery.c +++ b/src/lib/core/delivery.c @@ -207,6 +207,15 @@ void delivery_defer_packages(struct Delivery *ctx, int type) { strncpy(package_name, name, sizeof(package_name) - 1); } + char *extra_begin = strchr(package_name, '['); + char *extra_end = NULL; + if (extra_begin) { + extra_end = strchr(extra_begin, ']'); + if (extra_end) { + *extra_begin = '\0'; + } + } + msg(STASIS_MSG_L3, "package '%s': ", package_name); // When spec is present in name, set tests->version to the version detected in the name @@ -214,11 +223,7 @@ void delivery_defer_packages(struct Delivery *ctx, int type) { struct Test *test = &ctx->tests[x]; char nametmp[1024] = {0}; - if (spec_end != NULL && spec_begin != NULL) { - strncpy(nametmp, name, spec_begin - name); - } else { - strcpy(nametmp, name); - } + strncpy(nametmp, package_name, sizeof(nametmp) - 1); // Is the [test:NAME] in the package name? if (!strcmp(nametmp, test->name)) { // Override test->version when a version is provided by the (pip|conda)_package list item @@ -260,7 +265,9 @@ void delivery_defer_packages(struct Delivery *ctx, int type) { fprintf(stderr, "%s's existence command failed for '%s': %s\n", mode, name, pkg_index_provides_strerror(upstream_exists)); exit(1); - } else if (upstream_exists == PKG_NOT_FOUND) { + } + + if (upstream_exists == PKG_NOT_FOUND) { build_for_host = 1; } else { build_for_host = 0; -- cgit From 9173fe4a472500b3f87780d116ff7a54d4729c78 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 22 Nov 2024 15:46:56 -0500 Subject: Add basic unindent function --- src/lib/core/str.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) (limited to 'src') diff --git a/src/lib/core/str.c b/src/lib/core/str.c index a7dbab1..45fb60a 100644 --- a/src/lib/core/str.c +++ b/src/lib/core/str.c @@ -649,3 +649,28 @@ char *to_short_version(const char *s) { strchrdel(result, "."); return result; } + +void unindent(char *s) { + char *pos = NULL; + size_t leading_spaces; + + // Set position to beginning of string + pos = s; + + while (pos != NULL) { + const size_t len = strlen(s); + for (leading_spaces = 0; isspace(pos[leading_spaces]); leading_spaces++) {} + + // For each new line strip an indent + if (leading_spaces >= 4 && len >= 4) { + leading_spaces = 4; // remove first level of indentation + memmove(pos, pos + leading_spaces, len - leading_spaces); + pos[len - leading_spaces] = '\0'; + } + + pos = strchr(pos, '\n'); + if (pos && strlen(pos)) { + pos++; + } + } +} \ No newline at end of file -- cgit From ff4e8b219792684d2f09d865fab7aad1eab3a957 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 22 Nov 2024 16:23:45 -0500 Subject: Unindent script --- src/lib/core/delivery_test.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src') diff --git a/src/lib/core/delivery_test.c b/src/lib/core/delivery_test.c index 0bcf04d..8dbcd78 100644 --- a/src/lib/core/delivery_test.c +++ b/src/lib/core/delivery_test.c @@ -118,6 +118,9 @@ void delivery_tests_run(struct Delivery *ctx) { SYSERROR("An error occurred while rendering the following:\n%s", cmd); exit(1); } + // Move indents + // HEREDOCs will not work otherwise + unindent(cmd); if (test->disable) { msg(STASIS_MSG_L2, "Script execution disabled by configuration\n", test->name); -- cgit From 506269820329f019d1ff26eb681fbaa70bcb9674 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 22 Nov 2024 16:26:40 -0500 Subject: Unindent script_setup --- src/lib/core/delivery_test.c | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/lib/core/delivery_test.c b/src/lib/core/delivery_test.c index 8dbcd78..e80e0ec 100644 --- a/src/lib/core/delivery_test.c +++ b/src/lib/core/delivery_test.c @@ -192,6 +192,7 @@ void delivery_tests_run(struct Delivery *ctx) { SYSERROR("An error occurred while rendering the following:\n%s", cmd); exit(1); } + unindent(cmd); struct MultiProcessingTask *task = NULL; char *runner_cmd = NULL; -- cgit From 080866ee9400dbc1cf82a71621e55cb20861f6e8 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 3 Dec 2024 00:57:34 -0500 Subject: Add DELIVERY_[NOT_]FOUND defines * Add delivery_series_sync function to download previously delivered files from artifactory --- src/lib/core/delivery_artifactory.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'src') diff --git a/src/lib/core/delivery_artifactory.c b/src/lib/core/delivery_artifactory.c index b69615e..9ad5829 100644 --- a/src/lib/core/delivery_artifactory.c +++ b/src/lib/core/delivery_artifactory.c @@ -185,3 +185,20 @@ int delivery_mission_render_files(struct Delivery *ctx) { return 0; } +int delivery_series_sync(struct Delivery *ctx) { + struct JFRT_Download dl = {0}; + + char *remote_dir = NULL; + if (asprintf(&remote_dir, "%s/%s/%s/(*)", globals.jfrog.repo, ctx->meta.mission, ctx->info.build_name) < 0) { + SYSERROR("%s", "Unable to allocate bytes for remote directory path"); + return -1; + } + + char *dest_dir = NULL; + if (asprintf(&dest_dir, "%s/{1}", ctx->storage.output_dir) < 0) { + SYSERROR("%s", "Unable to allocate bytes for destination directory path"); + return -1; + } + + return jfrog_cli_rt_download(&ctx->deploy.jfrog_auth, &dl, remote_dir, dest_dir); +} -- cgit From cc6458bc5efffc9712bb0c731957dfe75d1ebca5 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 3 Dec 2024 00:58:40 -0500 Subject: Fix error message wording --- src/lib/core/delivery_docker.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/core/delivery_docker.c b/src/lib/core/delivery_docker.c index c170082..57015ad 100644 --- a/src/lib/core/delivery_docker.c +++ b/src/lib/core/delivery_docker.c @@ -92,7 +92,7 @@ int delivery_docker(struct Delivery *ctx) { memset(rsync_cmd, 0, sizeof(rsync_cmd)); sprintf(rsync_cmd, "rsync -avi --progress '%s' '%s'", ctx->storage.wheel_artifact_dir, dest); if (system(rsync_cmd)) { - fprintf(stderr, "Failed to copy wheel artifactory to docker build directory\n"); + fprintf(stderr, "Failed to copy wheel artifacts to docker build directory\n"); } if (docker_build(ctx->storage.build_docker_dir, args, ctx->deploy.docker.capabilities.build)) { -- cgit From 38e4789530a38ac9e7dfddfc0afcff178bd1647f Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 3 Dec 2024 01:00:22 -0500 Subject: Simplify delivery_exists() function * Returns DELIVERY_[NOT_]FOUND, or -1 on error --- src/lib/core/delivery_init.c | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) (limited to 'src') diff --git a/src/lib/core/delivery_init.c b/src/lib/core/delivery_init.c index 356a8ce..7e51f24 100644 --- a/src/lib/core/delivery_init.c +++ b/src/lib/core/delivery_init.c @@ -320,25 +320,27 @@ int delivery_exists(struct Delivery *ctx) { } struct JFRT_Search search = {.fail_no_op = true}; + // release_exists error states: + // `jf rt search --fail_no_op` returns 2 on failure + // otherwise, search returns an empty list "[]" and returns 0 release_exists = jfrog_cli_rt_search(&ctx->deploy.jfrog_auth, &search, globals.jfrog.repo, release_pattern); - if (release_exists != 2) { - if (!globals.enable_overwrite && !release_exists) { - // --fail_no_op returns 2 on failure - // without: it returns an empty list "[]" and exit code 0 - return 1; // found - } - } } else { struct StrList *files = listdir(ctx->storage.delivery_dir); for (size_t i = 0; i < strlist_count(files); i++) { char *filename = strlist_item(files, i); release_exists = fnmatch(release_pattern, filename, FNM_PATHNAME); - if (!globals.enable_overwrite && !release_exists) { - guard_strlist_free(&files); - return 1; // found + if (!release_exists) { + break; } } guard_strlist_free(&files); } - return 0; // not found + + if (release_exists < 0) { + return -1; // error + } + if (release_exists >= 1) { + return DELIVERY_NOT_FOUND; + } + return DELIVERY_FOUND; } -- cgit From d18249abd2e45b345fc8c5dc61bab11530e9dccf Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 3 Dec 2024 01:01:01 -0500 Subject: Use delivery_series_sync() function --- src/cli/stasis/stasis_main.c | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/cli/stasis/stasis_main.c b/src/cli/stasis/stasis_main.c index 093e32e..adc3cd7 100644 --- a/src/cli/stasis/stasis_main.c +++ b/src/cli/stasis/stasis_main.c @@ -220,11 +220,33 @@ int main(int argc, char *argv[]) { // Safety gate: Avoid clobbering a delivered release unless the user wants that behavior msg(STASIS_MSG_L1, "Checking release history\n"); - if (delivery_exists(&ctx)) { + const int found_delivery = delivery_exists(&ctx); + + if (!globals.enable_overwrite && found_delivery) { msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Refusing to overwrite release: %s\nUse --overwrite to enable release clobbering.\n", ctx.info.release_name); exit(1); } + if (globals.enable_artifactory) { + // We need to download previous revisions to ensure processed packages are available at build-time + // This is also a docker requirement. Python wheels must exist locally. + if (ctx.meta.rc > 1) { + msg(STASIS_MSG_L1, "Syncing delivery artifacts for %s\n", ctx.info.build_name); + if (delivery_series_sync(&ctx) != 0) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Unable to sync artifacts for %s\n", ctx.info.build_name); + msg(STASIS_MSG_L3, "Case #1:\n" + "\tIf this is a new 'version', and 'rc' is greater " + "than 1, then no previous deliveries exist remotely. " + "Reset 'rc' to 1.\n"); + msg(STASIS_MSG_L3, "Case #2:\n" + "\tThe Artifactory server %s is unreachable, or the credentials used " + "are invalid.\n", globals.jfrog.url); + // No continue-on-error check. Without the previous delivery nothing can be done. + exit(1); + } + } + } + // Unlikely to occur: this should help prevent rmtree() from destroying your entire filesystem // if path is "/" then, die // or if empty string, die -- cgit From 9098abc13882e6b665e46361721d3bcba7da55eb Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 3 Dec 2024 09:26:32 -0500 Subject: delivery_exists() returns DELIVERY_NOT_FOUND by default --- src/cli/stasis/stasis_main.c | 4 +--- src/lib/core/delivery_init.c | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 15 deletions(-) (limited to 'src') diff --git a/src/cli/stasis/stasis_main.c b/src/cli/stasis/stasis_main.c index adc3cd7..807dbbd 100644 --- a/src/cli/stasis/stasis_main.c +++ b/src/cli/stasis/stasis_main.c @@ -220,9 +220,7 @@ int main(int argc, char *argv[]) { // Safety gate: Avoid clobbering a delivered release unless the user wants that behavior msg(STASIS_MSG_L1, "Checking release history\n"); - const int found_delivery = delivery_exists(&ctx); - - if (!globals.enable_overwrite && found_delivery) { + if (!globals.enable_overwrite && delivery_exists(&ctx) == DELIVERY_FOUND) { msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Refusing to overwrite release: %s\nUse --overwrite to enable release clobbering.\n", ctx.info.release_name); exit(1); } diff --git a/src/lib/core/delivery_init.c b/src/lib/core/delivery_init.c index 7e51f24..2fced03 100644 --- a/src/lib/core/delivery_init.c +++ b/src/lib/core/delivery_init.c @@ -309,7 +309,7 @@ int bootstrap_build_info(struct Delivery *ctx) { } int delivery_exists(struct Delivery *ctx) { - int release_exists = 0; + int release_exists = DELIVERY_NOT_FOUND; char release_pattern[PATH_MAX] = {0}; sprintf(release_pattern, "*%s*", ctx->info.release_name); @@ -323,24 +323,24 @@ int delivery_exists(struct Delivery *ctx) { // release_exists error states: // `jf rt search --fail_no_op` returns 2 on failure // otherwise, search returns an empty list "[]" and returns 0 - release_exists = jfrog_cli_rt_search(&ctx->deploy.jfrog_auth, &search, globals.jfrog.repo, release_pattern); + const int match = jfrog_cli_rt_search(&ctx->deploy.jfrog_auth, &search, globals.jfrog.repo, release_pattern); + if (!match) { + release_exists = DELIVERY_FOUND; + } } else { struct StrList *files = listdir(ctx->storage.delivery_dir); - for (size_t i = 0; i < strlist_count(files); i++) { + const size_t files_count = strlist_count(files); + + for (size_t i = 0; i < files_count; i++) { char *filename = strlist_item(files, i); - release_exists = fnmatch(release_pattern, filename, FNM_PATHNAME); - if (!release_exists) { + const int match = fnmatch(release_pattern, filename, FNM_PATHNAME); + if (match == 0) { + release_exists = DELIVERY_FOUND; break; } } guard_strlist_free(&files); } - if (release_exists < 0) { - return -1; // error - } - if (release_exists >= 1) { - return DELIVERY_NOT_FOUND; - } - return DELIVERY_FOUND; + return release_exists; } -- cgit From 5796ce9338c7fe2aa8a26766ff9e01448d785c99 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 3 Dec 2024 10:47:37 -0500 Subject: Add ability to use artifactory without uploading any artifacts at the end. * New option: --no-artifactory-upload * Implies --no-artifactory-build-info * Updated README.md --- src/cli/stasis/args.c | 2 ++ src/cli/stasis/args.h | 13 +++++++------ src/cli/stasis/stasis_main.c | 6 +++++- src/lib/core/globals.c | 1 + 4 files changed, 15 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/cli/stasis/args.c b/src/cli/stasis/args.c index ed11ab9..f3ce823 100644 --- a/src/cli/stasis/args.c +++ b/src/cli/stasis/args.c @@ -17,6 +17,7 @@ struct option long_options[] = { {"no-docker", no_argument, 0, OPT_NO_DOCKER}, {"no-artifactory", no_argument, 0, OPT_NO_ARTIFACTORY}, {"no-artifactory-build-info", no_argument, 0, OPT_NO_ARTIFACTORY_BUILD_INFO}, + {"no-artifactory-upload", no_argument, 0, OPT_NO_ARTIFACTORY_UPLOAD}, {"no-testing", no_argument, 0, OPT_NO_TESTING}, {"no-parallel", no_argument, 0, OPT_NO_PARALLEL}, {"no-rewrite", no_argument, 0, OPT_NO_REWRITE_SPEC_STAGE_2}, @@ -39,6 +40,7 @@ const char *long_options_help[] = { "Do not build docker images", "Do not upload artifacts to Artifactory", "Do not upload build info objects to Artifactory", + "Do not upload artifacts to Artifactory (dry-run)", "Do not execute test scripts", "Do not execute tests in parallel", "Do not rewrite paths and URLs in output files", diff --git a/src/cli/stasis/args.h b/src/cli/stasis/args.h index 932eac7..5bad752 100644 --- a/src/cli/stasis/args.h +++ b/src/cli/stasis/args.h @@ -10,12 +10,13 @@ #define OPT_NO_DOCKER 1001 #define OPT_NO_ARTIFACTORY 1002 #define OPT_NO_ARTIFACTORY_BUILD_INFO 1003 -#define OPT_NO_TESTING 1004 -#define OPT_OVERWRITE 1005 -#define OPT_NO_REWRITE_SPEC_STAGE_2 1006 -#define OPT_FAIL_FAST 1007 -#define OPT_NO_PARALLEL 1008 -#define OPT_POOL_STATUS_INTERVAL 1009 +#define OPT_NO_ARTIFACTORY_UPLOAD 1004 +#define OPT_NO_TESTING 1005 +#define OPT_OVERWRITE 1006 +#define OPT_NO_REWRITE_SPEC_STAGE_2 1007 +#define OPT_FAIL_FAST 1009 +#define OPT_NO_PARALLEL 1010 +#define OPT_POOL_STATUS_INTERVAL 1011 extern struct option long_options[]; void usage(char *progname); diff --git a/src/cli/stasis/stasis_main.c b/src/cli/stasis/stasis_main.c index 807dbbd..dc4e2d1 100644 --- a/src/cli/stasis/stasis_main.c +++ b/src/cli/stasis/stasis_main.c @@ -102,6 +102,10 @@ int main(int argc, char *argv[]) { case OPT_NO_ARTIFACTORY_BUILD_INFO: globals.enable_artifactory_build_info = false; break; + case OPT_NO_ARTIFACTORY_UPLOAD: + globals.enable_artifactory_build_info = false; + globals.enable_artifactory_upload = false; + break; case OPT_NO_TESTING: globals.enable_testing = false; break; @@ -561,7 +565,7 @@ int main(int argc, char *argv[]) { } if (want_artifactory) { - if (globals.enable_artifactory) { + if (globals.enable_artifactory && globals.enable_artifactory_upload) { msg(STASIS_MSG_L1, "Uploading artifacts\n"); delivery_artifact_upload(&ctx); } else { diff --git a/src/lib/core/globals.c b/src/lib/core/globals.c index 83465f1..0f0941a 100644 --- a/src/lib/core/globals.c +++ b/src/lib/core/globals.c @@ -37,6 +37,7 @@ struct STASIS_GLOBAL globals = { .enable_docker = true, ///< Toggle docker usage .enable_artifactory = true, ///< Toggle artifactory server usage .enable_artifactory_build_info = true, ///< Toggle build-info uploads + .enable_artifactory_upload = true, ///< Toggle artifactory file uploads .enable_testing = true, ///< Toggle [test] block "script" execution. "script_setup" always executes. .enable_rewrite_spec_stage_2 = true, ///< Leave template stings in output files .enable_parallel = true, ///< Toggle testing in parallel -- cgit From 2a15ece791252c07ddf2fe9868709bd80eecf489 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 6 Dec 2024 08:21:58 -0500 Subject: Fix segfault in join_ex * Calculate the number of function arguments with va_copy() * Remove realloc() usage. No point. --- src/lib/core/str.c | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) (limited to 'src') diff --git a/src/lib/core/str.c b/src/lib/core/str.c index 45fb60a..d774e72 100644 --- a/src/lib/core/str.c +++ b/src/lib/core/str.c @@ -162,7 +162,7 @@ char *join(char **arr, const char *separator) { } char *join_ex(char *separator, ...) { - va_list ap; // Variadic argument list + va_list ap = {0}; // Variadic argument list size_t separator_len = 0; // Length of separator string size_t size = 0; // Length of output string size_t argc = 0; // Number of arguments ^ "..." @@ -174,13 +174,6 @@ char *join_ex(char *separator, ...) { return NULL; } - // Initialize array - argv = calloc(argc + 1, sizeof(char **)); - if (argv == NULL) { - perror("join_ex calloc failed"); - return NULL; - } - // Get length of the separator separator_len = strlen(separator); @@ -192,16 +185,21 @@ char *join_ex(char *separator, ...) { // 5. Append `current` string to `argv` array // 6. Update argument counter `argc` va_start(ap, separator); - for(argc = 0; (current = va_arg(ap, char *)) != NULL; argc++) { - char **tmp = realloc(argv, (argc + 1) * sizeof(char *)); - if (tmp == NULL) { - perror("join_ex realloc failed"); - GENERIC_ARRAY_FREE(argv); - return NULL; - } - argv = tmp; + va_list ap_tmp = {0}; + va_copy(ap_tmp, ap); + for(argc = 0; (current = va_arg(ap_tmp, char *)) != NULL; argc++) {} + va_end(ap_tmp); + + // Initialize array + argv = calloc(argc + 1, sizeof(char **)); + if (argv == NULL) { + perror("join_ex calloc failed"); + return NULL; + } + + for(size_t i = 0; i < argc && (current = va_arg(ap, char *)); i++) { size += strlen(current) + separator_len; - argv[argc] = strdup(current); + argv[i] = strdup(current); } va_end(ap); -- cgit From a2bcfc37f3634179b4e75fff38d4c36a1ab2a812 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 6 Dec 2024 08:26:35 -0500 Subject: Fix listdir() * Now returns the absolute path(s) to the file(s) * Remove restriction on reading hidden files --- src/lib/core/utils.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/lib/core/utils.c b/src/lib/core/utils.c index c03f8fa..aa4173c 100644 --- a/src/lib/core/utils.c +++ b/src/lib/core/utils.c @@ -766,7 +766,15 @@ struct StrList *listdir(const char *path) { if (!strcmp(rec->d_name, ".") || !strcmp(rec->d_name, "..")) { continue; } - strlist_append(&node, rec->d_name); + char *fullpath = join_ex("/", path, rec->d_name, NULL); + if (!fullpath) { + SYSERROR("%s", "Unable to allocate bytes to construct full path"); + guard_strlist_free(&node); + closedir(dp); + return NULL; + } + strlist_append(&node, fullpath); + guard_free(fullpath); } closedir(dp); return node; @@ -791,8 +799,6 @@ int mkdirs(const char *_path, mode_t mode) { char result[PATH_MAX] = {0}; int status = 0; while ((token = strsep(&path, "/")) != NULL && !status) { - if (token[0] == '.') - continue; strcat(result, token); strcat(result, "/"); status = mkdir(result, mode); -- cgit From bef4faf973f2caf501a456f1e78b3520a2f47410 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 6 Dec 2024 08:29:16 -0500 Subject: Check whether arch and platform are populated before accessing them --- src/cli/stasis_indexer/helpers.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/cli/stasis_indexer/helpers.c b/src/cli/stasis_indexer/helpers.c index 87ab922..cfc1ed6 100644 --- a/src/cli/stasis_indexer/helpers.c +++ b/src/cli/stasis_indexer/helpers.c @@ -8,8 +8,10 @@ struct StrList *get_architectures(struct Delivery ctx[], const size_t nelem) { struct StrList *architectures = strlist_init(); for (size_t i = 0; i < nelem; i++) { - if (!strstr_array(architectures->data, ctx[i].system.arch)) { - strlist_append(&architectures, ctx[i].system.arch); + if (ctx[i].system.arch) { + if (!strstr_array(architectures->data, ctx[i].system.arch)) { + strlist_append(&architectures, ctx[i].system.arch); + } } } return architectures; @@ -18,8 +20,10 @@ struct StrList *get_architectures(struct Delivery ctx[], const size_t nelem) { struct StrList *get_platforms(struct Delivery ctx[], const size_t nelem) { struct StrList *platforms = strlist_init(); for (size_t i = 0; i < nelem; i++) { - if (!strstr_array(platforms->data, ctx[i].system.platform[DELIVERY_PLATFORM_RELEASE])) { - strlist_append(&platforms, ctx[i].system.platform[DELIVERY_PLATFORM_RELEASE]); + if (ctx[i].system.platform) { + if (!strstr_array(platforms->data, ctx[i].system.platform[DELIVERY_PLATFORM_RELEASE])) { + strlist_append(&platforms, ctx[i].system.platform[DELIVERY_PLATFORM_RELEASE]); + } } } return platforms; -- cgit From 3336bb1399e83f10717dcad6fbbf949727a51532 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 6 Dec 2024 08:30:26 -0500 Subject: Move pandoc code into its own function pandoc_exec() --- src/cli/stasis_indexer/helpers.c | 59 ++++++++++++++++++++++++++++++++++++++++ src/cli/stasis_indexer/website.c | 59 +++------------------------------------- 2 files changed, 63 insertions(+), 55 deletions(-) (limited to 'src') diff --git a/src/cli/stasis_indexer/helpers.c b/src/cli/stasis_indexer/helpers.c index cfc1ed6..1c51e8d 100644 --- a/src/cli/stasis_indexer/helpers.c +++ b/src/cli/stasis_indexer/helpers.c @@ -79,6 +79,65 @@ int get_pandoc_version(size_t *result) { return 0; } +int pandoc_exec(const char *in_file, const char *out_file, const char *css_file, const char *title) { + if (!find_program("pandoc")) { + fprintf(stderr, "pandoc is not installed: unable to generate HTML indexes\n"); + return 0; + } + + char pandoc_versioned_args[255] = {0}; + size_t pandoc_version = 0; + if (!get_pandoc_version(&pandoc_version)) { + // < 2.19 + if (pandoc_version < 0x02130000) { + strcat(pandoc_versioned_args, "--self-contained "); + } else { + // >= 2.19 + strcat(pandoc_versioned_args, "--embed-resources "); + } + + // >= 1.15.0.4 + if (pandoc_version >= 0x010f0004) { + strcat(pandoc_versioned_args, "--standalone "); + } + + // >= 1.10.0.1 + if (pandoc_version >= 0x010a0001) { + strcat(pandoc_versioned_args, "-f gfm+autolink_bare_uris "); + } + + // > 3.1.9 + if (pandoc_version > 0x03010900) { + strcat(pandoc_versioned_args, "-f gfm+alerts "); + } + } + + // Converts a markdown file to html + char cmd[STASIS_BUFSIZ] = {0}; + strcpy(cmd, "pandoc "); + strcat(cmd, pandoc_versioned_args); + if (css_file && strlen(css_file)) { + strcat(cmd, "--css "); + strcat(cmd, css_file); + } + strcat(cmd, " "); + strcat(cmd, "--metadata title=\""); + strcat(cmd, title); + strcat(cmd, "\" "); + strcat(cmd, "-o "); + strcat(cmd, out_file); + strcat(cmd, " "); + strcat(cmd, in_file); + + if (globals.verbose) { + puts(cmd); + } + + // This might be negative when killed by a signal. + return system(cmd); +} + + int micromamba_configure(const struct Delivery *ctx, struct MicromambaInfo *m) { int status = 0; char *micromamba_prefix = NULL; diff --git a/src/cli/stasis_indexer/website.c b/src/cli/stasis_indexer/website.c index c04b7f2..cf5f3c2 100644 --- a/src/cli/stasis_indexer/website.c +++ b/src/cli/stasis_indexer/website.c @@ -2,11 +2,6 @@ #include "website.h" int indexer_make_website(const struct Delivery *ctx) { - if (!find_program("pandoc")) { - fprintf(stderr, "pandoc is not installed: unable to generate HTML indexes\n"); - return 0; - } - char *css_filename = calloc(PATH_MAX, sizeof(*css_filename)); if (!css_filename) { SYSERROR("unable to allocate string for CSS file path: %s", strerror(errno)); @@ -16,34 +11,6 @@ int indexer_make_website(const struct Delivery *ctx) { sprintf(css_filename, "%s/%s", globals.sysconfdir, "stasis_pandoc.css"); const int have_css = access(css_filename, F_OK | R_OK) == 0; - char pandoc_versioned_args[255] = {0}; - size_t pandoc_version = 0; - - if (!get_pandoc_version(&pandoc_version)) { - // < 2.19 - if (pandoc_version < 0x02130000) { - strcat(pandoc_versioned_args, "--self-contained "); - } else { - // >= 2.19 - strcat(pandoc_versioned_args, "--embed-resources "); - } - - // >= 1.15.0.4 - if (pandoc_version >= 0x010f0004) { - strcat(pandoc_versioned_args, "--standalone "); - } - - // >= 1.10.0.1 - if (pandoc_version >= 0x010a0001) { - strcat(pandoc_versioned_args, "-f gfm+autolink_bare_uris "); - } - - // > 3.1.9 - if (pandoc_version > 0x03010900) { - strcat(pandoc_versioned_args, "-f gfm+alerts "); - } - } - struct StrList *dirs = strlist_init(); strlist_append(&dirs, ctx->storage.delivery_dir); strlist_append(&dirs, ctx->storage.results_dir); @@ -71,29 +38,11 @@ int indexer_make_website(const struct Delivery *ctx) { strcpy(fullpath_dest, fullpath_src); gen_file_extension_str(fullpath_dest, ".html"); - // Converts a markdown file to html - strcpy(cmd, "pandoc "); - strcat(cmd, pandoc_versioned_args); - if (have_css) { - strcat(cmd, "--css "); - strcat(cmd, css_filename); - } - strcat(cmd, " "); - strcat(cmd, "--metadata title=\"STASIS\" "); - strcat(cmd, "-o "); - strcat(cmd, fullpath_dest); - strcat(cmd, " "); - strcat(cmd, fullpath_src); - if (globals.verbose) { - puts(cmd); - } - // This might be negative when killed by a signal. - // Otherwise, the return code is not critical to us. - if (system(cmd) < 0) { - guard_free(css_filename); - guard_strlist_free(&dirs); - return 1; + // Convert markdown to html + if (pandoc_exec(fullpath_src, fullpath_dest, have_css ? css_filename : NULL, "STASIS")) { + msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "Unable to convert %s\n", fullpath_src); } + if (file_replace_text(fullpath_dest, ".md", ".html", 0)) { // inform-only SYSERROR("%s: failed to rewrite *.md urls with *.html extension", fullpath_dest); -- cgit From 9788c4ffe8a75d1a10ff0bdc5d382b5e7f1cd70b Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 6 Dec 2024 08:31:35 -0500 Subject: Scan the directories listed in `dirs` instead of just the delivery directory --- src/cli/stasis_indexer/website.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/cli/stasis_indexer/website.c b/src/cli/stasis_indexer/website.c index cf5f3c2..9fa7b57 100644 --- a/src/cli/stasis_indexer/website.c +++ b/src/cli/stasis_indexer/website.c @@ -18,11 +18,12 @@ int indexer_make_website(const struct Delivery *ctx) { struct StrList *inputs = NULL; for (size_t i = 0; i < strlist_count(dirs); i++) { const char *pattern = "*.md"; - if (get_files(&inputs, ctx->storage.delivery_dir, pattern)) { - SYSERROR("%s does not contain files with pattern: %s", ctx->storage.delivery_dir, pattern); - guard_strlist_free(&inputs); + char *dirpath = strlist_item(dirs, i); + if (get_files(&inputs, dirpath, pattern)) { + SYSERROR("%s does not contain files with pattern: %s", dirpath, pattern); continue; } + char *root = strlist_item(dirs, i); for (size_t x = 0; x < strlist_count(inputs); x++) { char cmd[PATH_MAX] = {0}; -- cgit From 4da928cada36fcabf4f0c9138ade0beef7bf92e4 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 6 Dec 2024 08:31:56 -0500 Subject: Take the basename of the file immediately from the StrList item --- src/cli/stasis_indexer/website.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'src') diff --git a/src/cli/stasis_indexer/website.c b/src/cli/stasis_indexer/website.c index 9fa7b57..55f0c45 100644 --- a/src/cli/stasis_indexer/website.c +++ b/src/cli/stasis_indexer/website.c @@ -26,8 +26,7 @@ int indexer_make_website(const struct Delivery *ctx) { char *root = strlist_item(dirs, i); for (size_t x = 0; x < strlist_count(inputs); x++) { - char cmd[PATH_MAX] = {0}; - char *filename = strlist_item(inputs, x); + char *filename = path_basename(strlist_item(inputs, x)); char fullpath_src[PATH_MAX] = {0}; char fullpath_dest[PATH_MAX] = {0}; sprintf(fullpath_src, "%s/%s", root, filename); -- cgit From 47b0eb0ffc358393238a2e4ec8600f5a862a99b5 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 6 Dec 2024 08:32:36 -0500 Subject: Export pandoc_exec() --- src/cli/stasis_indexer/helpers.h | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'src') diff --git a/src/cli/stasis_indexer/helpers.h b/src/cli/stasis_indexer/helpers.h index 41db49e..ffd9f4d 100644 --- a/src/cli/stasis_indexer/helpers.h +++ b/src/cli/stasis_indexer/helpers.h @@ -3,9 +3,20 @@ #include "delivery.h" +#define ARRAY_COUNT_DYNAMIC(X, COUNTER) \ + do { \ + for (COUNTER = 0; X && X[COUNTER] != NULL; COUNTER++) {} \ + } while(0) + +#define ARRAY_COUNT_BY_STRUCT_MEMBER(X, MEMBER, COUNTER) \ + do { \ + for (COUNTER = 0; X[COUNTER].MEMBER != NULL; COUNTER++) {} \ + } while(0) + struct StrList *get_architectures(struct Delivery ctx[], size_t nelem); struct StrList *get_platforms(struct Delivery ctx[], size_t nelem); int get_pandoc_version(size_t *result); +int pandoc_exec(const char *in_file, const char *out_file, const char *css_file, const char *title); int get_latest_rc(struct Delivery ctx[], size_t nelem); struct Delivery **get_latest_deliveries(struct Delivery ctx[], size_t nelem); int get_files(struct StrList **out, const char *path, const char *pattern, ...); -- cgit From 0c18b5b15ed6e78cc27580518ae29f20ff23da65 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 6 Dec 2024 08:43:27 -0500 Subject: Fix pointers to Delivery struct * Dynamic allocation only. I'm completely fed up with "lost" addresses. --- src/cli/stasis_indexer/callbacks.c | 4 ++-- src/cli/stasis_indexer/helpers.c | 22 +++++++++++++++------ src/cli/stasis_indexer/helpers.h | 2 +- src/cli/stasis_indexer/junitxml_report.c | 12 +++++++++--- src/cli/stasis_indexer/readmes.c | 22 ++++++++++----------- src/cli/stasis_indexer/stasis_indexer_main.c | 29 +++++++++++++++------------- 6 files changed, 55 insertions(+), 36 deletions(-) (limited to 'src') diff --git a/src/cli/stasis_indexer/callbacks.c b/src/cli/stasis_indexer/callbacks.c index 4790e09..0186e1c 100644 --- a/src/cli/stasis_indexer/callbacks.c +++ b/src/cli/stasis_indexer/callbacks.c @@ -23,9 +23,9 @@ int callback_sort_deliveries_cmpfn(const void *a, const void *b) { // qsort callback to sort dynamically allocated delivery contexts by compact python version int callback_sort_deliveries_dynamic_cmpfn(const void *a, const void *b) { - const struct Delivery *delivery1 = *(struct Delivery **) a; + const struct Delivery *delivery1 = a; const size_t delivery1_python = strtoul(delivery1->meta.python_compact, NULL, 10); - const struct Delivery *delivery2 = *(struct Delivery **) b; + const struct Delivery *delivery2 = b; const size_t delivery2_python = strtoul(delivery2->meta.python_compact, NULL, 10); if (delivery2_python > delivery1_python) { diff --git a/src/cli/stasis_indexer/helpers.c b/src/cli/stasis_indexer/helpers.c index 1c51e8d..078be68 100644 --- a/src/cli/stasis_indexer/helpers.c +++ b/src/cli/stasis_indexer/helpers.c @@ -181,25 +181,35 @@ int get_latest_rc(struct Delivery ctx[], const size_t nelem) { return result; } -struct Delivery **get_latest_deliveries(struct Delivery ctx[], const size_t nelem) { - struct Delivery **result = NULL; +int sort_by_latest_rc(const void *a, const void *b) { + const struct Delivery *aa = a; + const struct Delivery *bb = b; + if (aa->meta.rc > bb->meta.rc) { + return -1; + } + if (aa->meta.rc < bb->meta.rc) { + return 1; + } + return 0; +} + +struct Delivery *get_latest_deliveries(struct Delivery ctx[], size_t nelem) { int latest = 0; size_t n = 0; - result = calloc(nelem + 1, sizeof(result)); + struct Delivery *result = calloc(nelem + 1, sizeof(*result)); if (!result) { - fprintf(stderr, "Unable to allocate %zu bytes for result delivery array: %s\n", nelem * sizeof(result), strerror(errno)); + fprintf(stderr, "Unable to allocate %zu bytes for result delivery array: %s\n", nelem * sizeof(*result), strerror(errno)); return NULL; } latest = get_latest_rc(ctx, nelem); for (size_t i = 0; i < nelem; i++) { if (ctx[i].meta.rc == latest) { - result[n] = &ctx[i]; + result[n] = ctx[i]; n++; } } - return result; } diff --git a/src/cli/stasis_indexer/helpers.h b/src/cli/stasis_indexer/helpers.h index ffd9f4d..733691d 100644 --- a/src/cli/stasis_indexer/helpers.h +++ b/src/cli/stasis_indexer/helpers.h @@ -18,7 +18,7 @@ struct StrList *get_platforms(struct Delivery ctx[], size_t nelem); int get_pandoc_version(size_t *result); int pandoc_exec(const char *in_file, const char *out_file, const char *css_file, const char *title); int get_latest_rc(struct Delivery ctx[], size_t nelem); -struct Delivery **get_latest_deliveries(struct Delivery ctx[], size_t nelem); +struct Delivery *get_latest_deliveries(struct Delivery ctx[], size_t nelem); int get_files(struct StrList **out, const char *path, const char *pattern, ...); int load_metadata(struct Delivery *ctx, const char *filename); int micromamba_configure(const struct Delivery *ctx, struct MicromambaInfo *m); diff --git a/src/cli/stasis_indexer/junitxml_report.c b/src/cli/stasis_indexer/junitxml_report.c index 7128331..90cae3b 100644 --- a/src/cli/stasis_indexer/junitxml_report.c +++ b/src/cli/stasis_indexer/junitxml_report.c @@ -8,8 +8,12 @@ #include "junitxml_report.h" int indexer_junitxml_report(struct Delivery ctx[], const size_t nelem) { - struct Delivery **latest = NULL; - latest = get_latest_deliveries(ctx, nelem); + struct Delivery *latest = get_latest_deliveries(ctx, nelem); + if (!latest) { + return -1; + } + size_t latest_count; + ARRAY_COUNT_BY_STRUCT_MEMBER(latest, meta.name, latest_count); char indexfile[PATH_MAX] = {0}; sprintf(indexfile, "%s/README.md", ctx->storage.results_dir); @@ -29,7 +33,9 @@ int indexer_junitxml_report(struct Delivery ctx[], const size_t nelem) { struct StrList *archs = get_architectures(*latest, nelem); struct StrList *platforms = get_platforms(*latest, nelem); - qsort(latest, nelem, sizeof(*latest), callback_sort_deliveries_dynamic_cmpfn); + struct StrList *archs = get_architectures(latest, nelem); + struct StrList *platforms = get_platforms(latest, nelem); + qsort(latest, latest_count, sizeof(*latest), callback_sort_deliveries_dynamic_cmpfn); fprintf(indexfp, "# %s-%s Test Report\n\n", ctx->meta.name, ctx->meta.version); fprintf(indexfp, "## Current Release\n\n"); size_t no_printable_data = 0; diff --git a/src/cli/stasis_indexer/readmes.c b/src/cli/stasis_indexer/readmes.c index 75e97a9..c98ecfc 100644 --- a/src/cli/stasis_indexer/readmes.c +++ b/src/cli/stasis_indexer/readmes.c @@ -2,7 +2,7 @@ #include "readmes.h" int indexer_readmes(struct Delivery ctx[], const size_t nelem) { - struct Delivery **latest = NULL; + struct Delivery *latest = NULL; latest = get_latest_deliveries(ctx, nelem); char indexfile[PATH_MAX] = {0}; @@ -14,8 +14,8 @@ int indexer_readmes(struct Delivery ctx[], const size_t nelem) { fprintf(stderr, "Unable to open %s for writing\n", indexfile); return -1; } - struct StrList *archs = get_architectures(*latest, nelem); - struct StrList *platforms = get_platforms(*latest, nelem); + struct StrList *archs = get_architectures(latest, nelem); + struct StrList *platforms = get_platforms(latest, nelem); fprintf(indexfp, "# %s-%s\n\n", ctx->meta.name, ctx->meta.version); fprintf(indexfp, "## Current Release\n\n"); @@ -25,9 +25,9 @@ int indexer_readmes(struct Delivery ctx[], const size_t nelem) { char *arch = strlist_item(archs, a); int have_combo = 0; for (size_t i = 0; i < nelem; i++) { - if (latest[i] && latest[i]->system.platform) { - if (strstr(latest[i]->system.platform[DELIVERY_PLATFORM_RELEASE], platform) && - strstr(latest[i]->system.arch, arch)) { + if (latest[i].system.platform) { + if (strstr(latest[i].system.platform[DELIVERY_PLATFORM_RELEASE], platform) && + strstr(latest[i].system.arch, arch)) { have_combo = 1; } } @@ -44,13 +44,13 @@ int indexer_readmes(struct Delivery ctx[], const size_t nelem) { char readme_name[PATH_MAX]; char conf_name[PATH_MAX]; char conf_name_relative[PATH_MAX]; - if (!latest[i]) { + if (!latest[i].meta.name) { continue; } - sprintf(link_name, "latest-py%s-%s-%s.yml", latest[i]->meta.python_compact, latest[i]->system.platform[DELIVERY_PLATFORM_RELEASE], latest[i]->system.arch); - sprintf(readme_name, "README-py%s-%s-%s.md", latest[i]->meta.python_compact, latest[i]->system.platform[DELIVERY_PLATFORM_RELEASE], latest[i]->system.arch); - sprintf(conf_name, "%s.ini", latest[i]->info.release_name); - sprintf(conf_name_relative, "../config/%s-rendered.ini", latest[i]->info.release_name); + sprintf(link_name, "latest-py%s-%s-%s.yml", latest[i].meta.python_compact, latest[i].system.platform[DELIVERY_PLATFORM_RELEASE], latest[i].system.arch); + sprintf(readme_name, "README-py%s-%s-%s.md", latest[i].meta.python_compact, latest[i].system.platform[DELIVERY_PLATFORM_RELEASE], latest[i].system.arch); + sprintf(conf_name, "%s.ini", latest[i].info.release_name); + sprintf(conf_name_relative, "../config/%s.ini", latest[i].info.release_name); if (strstr(link_name, platform) && strstr(link_name, arch)) { fprintf(indexfp, "|[%s](%s)|[%s](%s)|[%s](%s)|\n", link_name, link_name, readme_name, readme_name, conf_name, conf_name_relative); } diff --git a/src/cli/stasis_indexer/stasis_indexer_main.c b/src/cli/stasis_indexer/stasis_indexer_main.c index 6c318c9..016b26e 100644 --- a/src/cli/stasis_indexer/stasis_indexer_main.c +++ b/src/cli/stasis_indexer/stasis_indexer_main.c @@ -67,8 +67,8 @@ int indexer_conda(const struct Delivery *ctx, struct MicromambaInfo m) { return status; } -int indexer_symlinks(struct Delivery ctx[], const size_t nelem) { - struct Delivery **data = NULL; +int indexer_symlinks(struct Delivery *ctx, const size_t nelem) { + struct Delivery *data = NULL; data = get_latest_deliveries(ctx, nelem); //int latest = get_latest_rc(ctx, nelem); @@ -80,14 +80,14 @@ int indexer_symlinks(struct Delivery ctx[], const size_t nelem) { char file_name_spec[PATH_MAX]; char file_name_readme[PATH_MAX]; - if (!data[i]) { + if (!data[i].meta.name) { continue; } - sprintf(link_name_spec, "latest-py%s-%s-%s.yml", data[i]->meta.python_compact, data[i]->system.platform[DELIVERY_PLATFORM_RELEASE], data[i]->system.arch); - sprintf(file_name_spec, "%s.yml", data[i]->info.release_name); + sprintf(link_name_spec, "latest-py%s-%s-%s.yml", data[i].meta.python_compact, data[i].system.platform[DELIVERY_PLATFORM_RELEASE], data[i].system.arch); + sprintf(file_name_spec, "%s.yml", data[i].info.release_name); - sprintf(link_name_readme, "README-py%s-%s-%s.md", data[i]->meta.python_compact, data[i]->system.platform[DELIVERY_PLATFORM_RELEASE], data[i]->system.arch); - sprintf(file_name_readme, "README-%s.md", data[i]->info.release_name); + sprintf(link_name_readme, "README-py%s-%s-%s.md", data[i].meta.python_compact, data[i].system.platform[DELIVERY_PLATFORM_RELEASE], data[i].system.arch); + sprintf(file_name_readme, "README-%s.md", data[i].info.release_name); if (!access(link_name_spec, F_OK)) { if (unlink(link_name_spec)) { @@ -316,18 +316,21 @@ int main(const int argc, char *argv[]) { struct StrList *metafiles = NULL; get_files(&metafiles, ctx.storage.meta_dir, "*.stasis"); strlist_sort(metafiles, STASIS_SORT_LEN_ASCENDING); - struct Delivery local[strlist_count(metafiles)]; + + struct Delivery *local = calloc(strlist_count(metafiles) + 1, sizeof(*local)); + if (!local) { + SYSERROR("%s", "Unable to allocate bytes for local delivery context array"); + exit(1); + } for (size_t i = 0; i < strlist_count(metafiles); i++) { char *item = strlist_item(metafiles, i); - memset(&local[i], 0, sizeof(ctx)); + // Copy the pre-filled contents of the main delivery context memcpy(&local[i], &ctx, sizeof(ctx)); - char path[PATH_MAX]; - sprintf(path, "%s/%s", ctx.storage.meta_dir, item); if (globals.verbose) { - puts(path); + puts(item); } - load_metadata(&local[i], path); + load_metadata(&local[i], item); } qsort(local, strlist_count(metafiles), sizeof(*local), callback_sort_deliveries_cmpfn); -- cgit From 41c76e0ca3f34fd46235bac6c2c9eac0497b28fb Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 6 Dec 2024 08:44:39 -0500 Subject: Improved markdown output and presentation --- src/cli/stasis_indexer/helpers.c | 1 + src/cli/stasis_indexer/junitxml_report.c | 176 +++++++++++++++++-------------- src/cli/stasis_indexer/readmes.c | 18 +++- 3 files changed, 113 insertions(+), 82 deletions(-) (limited to 'src') diff --git a/src/cli/stasis_indexer/helpers.c b/src/cli/stasis_indexer/helpers.c index 078be68..8279896 100644 --- a/src/cli/stasis_indexer/helpers.c +++ b/src/cli/stasis_indexer/helpers.c @@ -204,6 +204,7 @@ struct Delivery *get_latest_deliveries(struct Delivery ctx[], size_t nelem) { } latest = get_latest_rc(ctx, nelem); + qsort(ctx, nelem, sizeof(*ctx), sort_by_latest_rc); for (size_t i = 0; i < nelem; i++) { if (ctx[i].meta.rc == latest) { result[n] = ctx[i]; diff --git a/src/cli/stasis_indexer/junitxml_report.c b/src/cli/stasis_indexer/junitxml_report.c index 90cae3b..10c336d 100644 --- a/src/cli/stasis_indexer/junitxml_report.c +++ b/src/cli/stasis_indexer/junitxml_report.c @@ -30,101 +30,121 @@ int indexer_junitxml_report(struct Delivery ctx[], const size_t nelem) { fprintf(stderr, "Unable to open %s for writing\n", indexfile); return -1; } - struct StrList *archs = get_architectures(*latest, nelem); - struct StrList *platforms = get_platforms(*latest, nelem); struct StrList *archs = get_architectures(latest, nelem); struct StrList *platforms = get_platforms(latest, nelem); qsort(latest, latest_count, sizeof(*latest), callback_sort_deliveries_dynamic_cmpfn); fprintf(indexfp, "# %s-%s Test Report\n\n", ctx->meta.name, ctx->meta.version); - fprintf(indexfp, "## Current Release\n\n"); size_t no_printable_data = 0; - for (size_t p = 0; p < strlist_count(platforms); p++) { - char *platform = strlist_item(platforms, p); - for (size_t a = 0; a < strlist_count(archs); a++) { - char *arch = strlist_item(archs, a); - int have_combo = 0; - for (size_t i = 0; i < nelem; i++) { - if (latest[i] && latest[i]->system.platform) { - if (strstr(latest[i]->system.platform[DELIVERY_PLATFORM_RELEASE], platform) && - strstr(latest[i]->system.arch, arch)) { - have_combo = 1; - break; + + size_t delivery_count = get_latest_rc(latest, latest_count); + for (size_t p = 0; p < strlist_count(platforms); p++) { + char *platform = strlist_item(platforms, p); + for (size_t a = 0; a < strlist_count(archs); a++) { + char *arch = strlist_item(archs, a); + + fprintf(indexfp, "## %s-%s\n\n", platform, arch); + for (size_t d = 0; d < delivery_count; d++) { + struct Delivery *current = &ctx[d]; + if (current->meta.rc == (int) d + 1 + && strcmp(current->system.arch, arch) != 0 + && strcmp(current->system.platform[DELIVERY_PLATFORM_RELEASE], platform) != 0) { + continue; } - } - } - if (!have_combo) { - continue; - } - fprintf(indexfp, "### %s-%s\n\n", platform, arch); - - fprintf(indexfp, "|Suite|Duration|Fail |Skip |Error |\n"); - fprintf(indexfp, "|:----|:------:|:------:|:---:|:----:|\n"); - for (size_t f = 0; f < strlist_count(file_listing); f++) { - char *filename = strlist_item(file_listing, f); - if (!endswith(filename, ".xml")) { - continue; - } - if (strstr(filename, platform) && strstr(filename, arch)) { - struct JUNIT_Testsuite *testsuite = junitxml_testsuite_read(filename); - if (testsuite) { - if (globals.verbose) { - printf("%s: duration: %0.4f, failed: %d, skipped: %d, errors: %d\n", filename, testsuite->time, testsuite->failures, testsuite->skipped, testsuite->errors); - } - fprintf(indexfp, "|[%s](%s)|%0.4f|%d|%d|%d|\n", filename, filename, testsuite->time, testsuite->failures, testsuite->skipped, testsuite->errors); - /* - * TODO: Display failure/skip/error output. - */ - char *bname = strdup(filename); - bname[strlen(bname) - 4] = 0; - char result_outfile[PATH_MAX] = {0}; - path_basename(bname); - mkdir(bname, 0755); - snprintf(result_outfile, sizeof(result_outfile) - 1, "%s/%s.md", bname, bname); - FILE *resultfp = fopen(result_outfile, "w+"); - if (!resultfp) { - SYSERROR("Unable to open %s for writing", result_outfile); - return -1; + fprintf(indexfp, "### %s\n", current->info.release_name); + fprintf(indexfp, "\n|Suite|Duration|Fail |Skip |Error |\n"); + fprintf(indexfp, "|:----|:------:|:------:|:---:|:----:|\n"); + for (size_t f = 0; f < strlist_count(file_listing); f++) { + char *filename = strlist_item(file_listing, f); + if (!endswith(filename, ".xml")) { + continue; } - for (size_t i = 0; i < testsuite->_tc_inuse; i++) { - if (testsuite->testcase[i]->tc_result_state_type) { - const char *type_str = NULL; - const int state = testsuite->testcase[i]->tc_result_state_type; - const char *message = NULL; - if (state == JUNIT_RESULT_STATE_FAILURE) { - message = testsuite->testcase[i]->result_state.failure->message; - type_str = "[FAILED]"; - } else if (state == JUNIT_RESULT_STATE_ERROR) { - message = testsuite->testcase[i]->result_state.error->message; - type_str = "[ERROR]"; - } else if (state == JUNIT_RESULT_STATE_SKIPPED) { - message = testsuite->testcase[i]->result_state.skipped->message; - type_str = "[SKIPPED]"; + char pattern[PATH_MAX] = {0}; + snprintf(pattern, sizeof(pattern) - 1, "*%s*", current->info.release_name); + if (!fnmatch(pattern, filename, 0) && strstr(filename, platform) && + strstr(filename, arch)) { + struct JUNIT_Testsuite *testsuite = junitxml_testsuite_read(filename); + if (testsuite) { + if (globals.verbose) { + printf("%s: duration: %0.4f, failed: %d, skipped: %d, errors: %d\n", filename, + testsuite->time, testsuite->failures, testsuite->skipped, + testsuite->errors); } - fprintf(resultfp, "### %s %s :: %s\n", type_str, testsuite->testcase[i]->classname, testsuite->testcase[i]->name); - fprintf(resultfp, "\nDuration: %0.04fs\n", testsuite->testcase[i]->time); - fprintf(resultfp, "\n```\n%s\n```\n", message); + + char *bname_tmp = strdup(filename); + char *bname = path_basename(bname_tmp); + bname[strlen(bname) - 4] = 0; + guard_free(bname_tmp); + + char result_outfile[PATH_MAX] = {0}; + + char result_path[PATH_MAX] = {0}; + //snprintf(result_path, sizeof(result_path) -1 , "%s/%s", platform, arch); + //mkdirs(result_path, 0755); + + char *short_name_pattern = NULL; + asprintf(&short_name_pattern, "-%s", current->info.release_name); + + char short_name[PATH_MAX] = {0}; + strncpy(short_name, bname, sizeof(short_name) - 1); + replace_text(short_name, short_name_pattern, "", 0); + replace_text(short_name, "results-", "", 0); + guard_free(short_name_pattern); + + fprintf(indexfp, "|[%s](%s.html)|%0.4f|%d|%d|%d|\n", short_name, + bname, + testsuite->time, testsuite->failures, testsuite->skipped, + testsuite->errors); + + snprintf(result_outfile, sizeof(result_outfile) - strlen(bname) - 3, "%s.md", + bname); + FILE *resultfp = fopen(result_outfile, "w+"); + if (!resultfp) { + SYSERROR("Unable to open %s for writing", result_outfile); + return -1; + } + + for (size_t i = 0; i < testsuite->_tc_inuse; i++) { + if (testsuite->testcase[i]->tc_result_state_type) { + const char *type_str = NULL; + const int state = testsuite->testcase[i]->tc_result_state_type; + const char *message = NULL; + if (state == JUNIT_RESULT_STATE_FAILURE) { + message = testsuite->testcase[i]->result_state.failure->message; + type_str = "[FAILED]"; + } else if (state == JUNIT_RESULT_STATE_ERROR) { + message = testsuite->testcase[i]->result_state.error->message; + type_str = "[ERROR]"; + } else if (state == JUNIT_RESULT_STATE_SKIPPED) { + message = testsuite->testcase[i]->result_state.skipped->message; + type_str = "[SKIPPED]"; + } + fprintf(resultfp, "### %s %s :: %s\n", type_str, + testsuite->testcase[i]->classname, testsuite->testcase[i]->name); + fprintf(resultfp, "\nDuration: %0.04fs\n", testsuite->testcase[i]->time); + fprintf(resultfp, "\n```\n%s\n```\n", message); + } + } + junitxml_testsuite_free(&testsuite); + fclose(resultfp); + } else { + fprintf(stderr, "bad test suite: %s: %s\n", strerror(errno), filename); + } + } else { + if (!no_printable_data) { + // Triggering for reasons unknown + //fprintf(indexfp, "|No data|-|-|-|-|-|-|\n"); + no_printable_data++; } } - junitxml_testsuite_free(&testsuite); - fclose(resultfp); - } else { - fprintf(stderr, "bad test suite: %s: %s\n", strerror(errno), filename); - } - } else { - if (!no_printable_data) { - // Triggering for reasons unknown - //fprintf(indexfp, "|No data|-|-|-|-|-|-|\n"); - no_printable_data++; } } + fprintf(indexfp, "\n"); + no_printable_data = 0; } fprintf(indexfp, "\n"); - no_printable_data = 0; - } - fprintf(indexfp, "\n"); } guard_strlist_free(&archs); guard_strlist_free(&platforms); diff --git a/src/cli/stasis_indexer/readmes.c b/src/cli/stasis_indexer/readmes.c index c98ecfc..df3614e 100644 --- a/src/cli/stasis_indexer/readmes.c +++ b/src/cli/stasis_indexer/readmes.c @@ -36,9 +36,6 @@ int indexer_readmes(struct Delivery ctx[], const size_t nelem) { continue; } fprintf(indexfp, "### %s-%s\n\n", platform, arch); - - fprintf(indexfp, "|Release|Info|Receipt|\n"); - fprintf(indexfp, "|:----:|:----:|:----:|\n"); for (size_t i = 0; i < nelem; i++) { char link_name[PATH_MAX]; char readme_name[PATH_MAX]; @@ -52,13 +49,26 @@ int indexer_readmes(struct Delivery ctx[], const size_t nelem) { sprintf(conf_name, "%s.ini", latest[i].info.release_name); sprintf(conf_name_relative, "../config/%s.ini", latest[i].info.release_name); if (strstr(link_name, platform) && strstr(link_name, arch)) { - fprintf(indexfp, "|[%s](%s)|[%s](%s)|[%s](%s)|\n", link_name, link_name, readme_name, readme_name, conf_name, conf_name_relative); + fprintf(indexfp, "- Info: [README](%s)\n", readme_name); + fprintf(indexfp, "- Release: [Conda Environment YAML](%s)\n", link_name); + fprintf(indexfp, "- Receipt: [STASIS input file](%s)\n", conf_name_relative); } } fprintf(indexfp, "\n"); } fprintf(indexfp, "\n"); } + + fprintf(indexfp, "## Releases\n"); + for (size_t i = 0; ctx[i].meta.name != NULL; i++) { + struct Delivery *current = &ctx[i]; + fprintf(indexfp, "### %s\n", current->info.release_name); + fprintf(indexfp, "- Info: [README](README-%s.html)\n", current->info.release_name); + fprintf(indexfp, "- Release: [Conda Environment YAML](%s.yml)\n", current->info.release_name); + fprintf(indexfp, "- Receipt: [STASIS input file](%s.ini)\n", current->info.release_name); + } + fprintf(indexfp, "\n"); + guard_strlist_free(&archs); guard_strlist_free(&platforms); fclose(indexfp); -- cgit From c2fd3e850a3e1fb7ba0b694e8d2baa6ede1f170d Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 6 Dec 2024 08:46:26 -0500 Subject: The indexer now installs conda into ~/.stasis/indexer/conda instead of installing it from scratch every time. * We only need conda's indexer anyway * Same applies to ~/.stasis/indexer/tools assuming we ever need to use `jf` directly (not yet) --- src/cli/stasis_indexer/stasis_indexer_main.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/cli/stasis_indexer/stasis_indexer_main.c b/src/cli/stasis_indexer/stasis_indexer_main.c index 016b26e..4649c7a 100644 --- a/src/cli/stasis_indexer/stasis_indexer_main.c +++ b/src/cli/stasis_indexer/stasis_indexer_main.c @@ -133,9 +133,15 @@ void indexer_init_dirs(struct Delivery *ctx, const char *workdir) { fprintf(stderr, "Failed to configure temporary storage directory\n"); exit(1); } + + char *user_dir = expandpath("~/.stasis/indexer"); + if (!user_dir) { + SYSERROR("%s", "expandpath failed"); + } + path_store(&ctx->storage.output_dir, PATH_MAX, ctx->storage.root, ""); - path_store(&ctx->storage.tools_dir, PATH_MAX, ctx->storage.output_dir, "tools"); - path_store(&globals.conda_install_prefix, PATH_MAX, ctx->storage.tools_dir, "conda"); + path_store(&ctx->storage.tools_dir, PATH_MAX, user_dir, "tools"); + path_store(&globals.conda_install_prefix, PATH_MAX, user_dir, "conda"); path_store(&ctx->storage.cfgdump_dir, PATH_MAX, ctx->storage.output_dir, "config"); path_store(&ctx->storage.meta_dir, PATH_MAX, ctx->storage.output_dir, "meta"); path_store(&ctx->storage.delivery_dir, PATH_MAX, ctx->storage.output_dir, "delivery"); @@ -143,6 +149,7 @@ void indexer_init_dirs(struct Delivery *ctx, const char *workdir) { path_store(&ctx->storage.results_dir, PATH_MAX, ctx->storage.output_dir, "results"); path_store(&ctx->storage.wheel_artifact_dir, PATH_MAX, ctx->storage.package_dir, "wheels"); path_store(&ctx->storage.conda_artifact_dir, PATH_MAX, ctx->storage.package_dir, "conda"); + guard_free(user_dir); char newpath[PATH_MAX] = {0}; if (getenv("PATH")) { -- cgit From b72ea77ed6d06699c2268137cae39ed36d67a6a8 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Sat, 7 Dec 2024 18:05:16 -0500 Subject: get_files: empty pattern becomes '*' --- src/cli/stasis_indexer/helpers.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src') diff --git a/src/cli/stasis_indexer/helpers.c b/src/cli/stasis_indexer/helpers.c index 8279896..f67292f 100644 --- a/src/cli/stasis_indexer/helpers.c +++ b/src/cli/stasis_indexer/helpers.c @@ -220,6 +220,10 @@ int get_files(struct StrList **out, const char *path, const char *pattern, ...) char userpattern[PATH_MAX] = {0}; vsprintf(userpattern, pattern, args); va_end(args); + if (!strlen(userpattern)) { + userpattern[0] = '*'; + userpattern[1] = 0; + } struct StrList *list = listdir(path); if (!list) { return -1; -- cgit From 83bd300fcd006ee14314c26569c47716dfff2352 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Sat, 7 Dec 2024 18:06:08 -0500 Subject: indexer_junitxml_report: Toggle no_printable_data instead of incrementing --- src/cli/stasis_indexer/junitxml_report.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/cli/stasis_indexer/junitxml_report.c b/src/cli/stasis_indexer/junitxml_report.c index 10c336d..9b85a3f 100644 --- a/src/cli/stasis_indexer/junitxml_report.c +++ b/src/cli/stasis_indexer/junitxml_report.c @@ -136,7 +136,7 @@ int indexer_junitxml_report(struct Delivery ctx[], const size_t nelem) { if (!no_printable_data) { // Triggering for reasons unknown //fprintf(indexfp, "|No data|-|-|-|-|-|-|\n"); - no_printable_data++; + no_printable_data = 1; } } } -- cgit From 3ae9311dd7961756f6e22c2a77dd3f5690208148 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Sat, 7 Dec 2024 18:07:45 -0500 Subject: Add get_docker_images() helper function --- src/cli/stasis_indexer/helpers.c | 17 +++++++++++++++++ src/cli/stasis_indexer/helpers.h | 1 + 2 files changed, 18 insertions(+) (limited to 'src') diff --git a/src/cli/stasis_indexer/helpers.c b/src/cli/stasis_indexer/helpers.c index f67292f..6d2fdd0 100644 --- a/src/cli/stasis_indexer/helpers.c +++ b/src/cli/stasis_indexer/helpers.c @@ -255,6 +255,23 @@ int get_files(struct StrList **out, const char *path, const char *pattern, ...) return 0; } +struct StrList *get_docker_images(struct Delivery *ctx, char *pattern) { + char *tarball = NULL; + asprintf(&tarball, "%s*.tar*", pattern); + if (!tarball) { + SYSERROR("%s", "Unable to allocate bytes for docker image wildcard pattern"); + return NULL; + } + tolower_s(tarball); + replace_text(tarball, "+", "-", 0); + + struct StrList *files = NULL; + get_files(&files, ctx->storage.docker_artifact_dir, tarball); + guard_free(tarball); + + return files; +} + int load_metadata(struct Delivery *ctx, const char *filename) { char line[STASIS_NAME_MAX] = {0}; diff --git a/src/cli/stasis_indexer/helpers.h b/src/cli/stasis_indexer/helpers.h index 733691d..d493f75 100644 --- a/src/cli/stasis_indexer/helpers.h +++ b/src/cli/stasis_indexer/helpers.h @@ -20,6 +20,7 @@ int pandoc_exec(const char *in_file, const char *out_file, const char *css_file, int get_latest_rc(struct Delivery ctx[], size_t nelem); struct Delivery *get_latest_deliveries(struct Delivery ctx[], size_t nelem); int get_files(struct StrList **out, const char *path, const char *pattern, ...); +struct StrList *get_docker_images(struct Delivery *ctx, char *pattern); int load_metadata(struct Delivery *ctx, const char *filename); int micromamba_configure(const struct Delivery *ctx, struct MicromambaInfo *m); -- cgit From 3475952db1f6e22f861077a0aac5435175df3975 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Sat, 7 Dec 2024 18:08:15 -0500 Subject: Initialize docker artifact directory --- src/cli/stasis_indexer/stasis_indexer_main.c | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/cli/stasis_indexer/stasis_indexer_main.c b/src/cli/stasis_indexer/stasis_indexer_main.c index 4649c7a..ef39394 100644 --- a/src/cli/stasis_indexer/stasis_indexer_main.c +++ b/src/cli/stasis_indexer/stasis_indexer_main.c @@ -149,6 +149,7 @@ void indexer_init_dirs(struct Delivery *ctx, const char *workdir) { path_store(&ctx->storage.results_dir, PATH_MAX, ctx->storage.output_dir, "results"); path_store(&ctx->storage.wheel_artifact_dir, PATH_MAX, ctx->storage.package_dir, "wheels"); path_store(&ctx->storage.conda_artifact_dir, PATH_MAX, ctx->storage.package_dir, "conda"); + path_store(&ctx->storage.docker_artifact_dir, PATH_MAX, ctx->storage.package_dir, "docker"); guard_free(user_dir); char newpath[PATH_MAX] = {0}; -- cgit From b677e21d0b996f8aef760f4c8330712107dad217 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Sat, 7 Dec 2024 18:09:20 -0500 Subject: Show current release followed by all available releases * Include link to archived docker image (i.e. curl -L [..] | docker load) --- src/cli/stasis_indexer/readmes.c | 147 +++++++++++++++++++++++---------------- 1 file changed, 87 insertions(+), 60 deletions(-) (limited to 'src') diff --git a/src/cli/stasis_indexer/readmes.c b/src/cli/stasis_indexer/readmes.c index df3614e..77b5178 100644 --- a/src/cli/stasis_indexer/readmes.c +++ b/src/cli/stasis_indexer/readmes.c @@ -2,84 +2,111 @@ #include "readmes.h" int indexer_readmes(struct Delivery ctx[], const size_t nelem) { - struct Delivery *latest = NULL; - latest = get_latest_deliveries(ctx, nelem); + struct Delivery *latest_deliveries = get_latest_deliveries(ctx, nelem); + if (!latest_deliveries) { + if (errno) { + return -1; + } + return 0; + } char indexfile[PATH_MAX] = {0}; sprintf(indexfile, "%s/README.md", ctx->storage.delivery_dir); - if (!pushd(ctx->storage.delivery_dir)) { - FILE *indexfp = fopen(indexfile, "w+"); - if (!indexfp) { - fprintf(stderr, "Unable to open %s for writing\n", indexfile); - return -1; - } - struct StrList *archs = get_architectures(latest, nelem); - struct StrList *platforms = get_platforms(latest, nelem); + FILE *indexfp = fopen(indexfile, "w+"); + if (!indexfp) { + fprintf(stderr, "Unable to open %s for writing\n", indexfile); + return -1; + } + struct StrList *archs = get_architectures(latest_deliveries, nelem); + struct StrList *platforms = get_platforms(latest_deliveries, nelem); - fprintf(indexfp, "# %s-%s\n\n", ctx->meta.name, ctx->meta.version); - fprintf(indexfp, "## Current Release\n\n"); - for (size_t p = 0; p < strlist_count(platforms); p++) { - char *platform = strlist_item(platforms, p); - for (size_t a = 0; a < strlist_count(archs); a++) { - char *arch = strlist_item(archs, a); - int have_combo = 0; - for (size_t i = 0; i < nelem; i++) { - if (latest[i].system.platform) { - if (strstr(latest[i].system.platform[DELIVERY_PLATFORM_RELEASE], platform) && - strstr(latest[i].system.arch, arch)) { - have_combo = 1; - } + fprintf(indexfp, "# %s-%s\n\n", ctx->meta.name, ctx->meta.version); + fprintf(indexfp, "## Current Release\n\n"); + for (size_t p = 0; p < strlist_count(platforms); p++) { + char *platform = strlist_item(platforms, p); + for (size_t a = 0; a < strlist_count(archs); a++) { + char *arch = strlist_item(archs, a); + int have_combo = 0; + for (size_t i = 0; i < nelem; i++) { + if (latest_deliveries[i].system.platform) { + if (strstr(latest_deliveries[i].system.platform[DELIVERY_PLATFORM_RELEASE], platform) && + strstr(latest_deliveries[i].system.arch, arch)) { + have_combo = 1; } } - if (!have_combo) { + } + if (!have_combo) { + continue; + } + fprintf(indexfp, "### %s-%s\n\n", platform, arch); + for (size_t i = 0; i < nelem; i++) { + char link_name[PATH_MAX] = {0}; + char readme_name[PATH_MAX] = {0}; + char conf_name[PATH_MAX] = {0}; + char conf_name_relative[PATH_MAX] = {0}; + if (!latest_deliveries[i].meta.name) { continue; } - fprintf(indexfp, "### %s-%s\n\n", platform, arch); - for (size_t i = 0; i < nelem; i++) { - char link_name[PATH_MAX]; - char readme_name[PATH_MAX]; - char conf_name[PATH_MAX]; - char conf_name_relative[PATH_MAX]; - if (!latest[i].meta.name) { - continue; - } - sprintf(link_name, "latest-py%s-%s-%s.yml", latest[i].meta.python_compact, latest[i].system.platform[DELIVERY_PLATFORM_RELEASE], latest[i].system.arch); - sprintf(readme_name, "README-py%s-%s-%s.md", latest[i].meta.python_compact, latest[i].system.platform[DELIVERY_PLATFORM_RELEASE], latest[i].system.arch); - sprintf(conf_name, "%s.ini", latest[i].info.release_name); - sprintf(conf_name_relative, "../config/%s.ini", latest[i].info.release_name); - if (strstr(link_name, platform) && strstr(link_name, arch)) { - fprintf(indexfp, "- Info: [README](%s)\n", readme_name); - fprintf(indexfp, "- Release: [Conda Environment YAML](%s)\n", link_name); - fprintf(indexfp, "- Receipt: [STASIS input file](%s)\n", conf_name_relative); + sprintf(link_name, "latest-py%s-%s-%s.yml", latest_deliveries[i].meta.python_compact, latest_deliveries[i].system.platform[DELIVERY_PLATFORM_RELEASE], latest_deliveries[i].system.arch); + sprintf(readme_name, "README-py%s-%s-%s.md", latest_deliveries[i].meta.python_compact, latest_deliveries[i].system.platform[DELIVERY_PLATFORM_RELEASE], latest_deliveries[i].system.arch); + sprintf(conf_name, "%s.ini", latest_deliveries[i].info.release_name); + sprintf(conf_name_relative, "../config/%s.ini", latest_deliveries[i].info.release_name); + if (strstr(link_name, platform) && strstr(link_name, arch)) { + fprintf(indexfp, "- Info: [README](%s)\n", readme_name); + fprintf(indexfp, "- Release: [Conda Environment YAML](%s)\n", link_name); + fprintf(indexfp, "- Receipt: [STASIS input file](%s)\n", conf_name_relative); + fprintf(indexfp, "- Docker: "); + struct StrList *docker_images = get_docker_images(&latest_deliveries[i], ""); + if (docker_images + && strlist_count(docker_images) + && !strcmp(latest_deliveries[i].system.platform[DELIVERY_PLATFORM_RELEASE], "linux")) { + fprintf(indexfp, "[Archive](../packages/docker/%s)\n", path_basename(strlist_item(docker_images, 0))); + guard_free(docker_images); + } else { + fprintf(indexfp, "N/A\n"); } } - fprintf(indexfp, "\n"); } fprintf(indexfp, "\n"); } + fprintf(indexfp, "\n"); + } - fprintf(indexfp, "## Releases\n"); - for (size_t i = 0; ctx[i].meta.name != NULL; i++) { - struct Delivery *current = &ctx[i]; - fprintf(indexfp, "### %s\n", current->info.release_name); - fprintf(indexfp, "- Info: [README](README-%s.html)\n", current->info.release_name); - fprintf(indexfp, "- Release: [Conda Environment YAML](%s.yml)\n", current->info.release_name); - fprintf(indexfp, "- Receipt: [STASIS input file](%s.ini)\n", current->info.release_name); + fprintf(indexfp, "## Releases\n"); + for (size_t i = 0; ctx[i].meta.name != NULL; i++) { + struct Delivery *current = &ctx[i]; + fprintf(indexfp, "### %s\n", current->info.release_name); + fprintf(indexfp, "- Info: [README](README-%s.html)\n", current->info.release_name); + fprintf(indexfp, "- Release: [Conda Environment YAML](%s.yml)\n", current->info.release_name); + fprintf(indexfp, "- Receipt: [STASIS input file](../config/%s.ini)\n", current->info.release_name); + fprintf(indexfp, "- Docker: \n"); + + char *pattern = NULL; + asprintf(&pattern, "*%s*", current->info.build_number); + if (!pattern) { + SYSERROR("%s", "Unable to allocate bytes for pattern"); + return -1; } - fprintf(indexfp, "\n"); - guard_strlist_free(&archs); - guard_strlist_free(&platforms); - fclose(indexfp); - popd(); - } else { - fprintf(stderr, "Unable to enter delivery directory: %s\n", ctx->storage.delivery_dir); - guard_free(latest); - return -1; + struct StrList *docker_images = get_docker_images(current, pattern); + if (docker_images + && strlist_count(docker_images) + && !strcmp(current->system.platform[DELIVERY_PLATFORM_RELEASE], "linux")) { + fprintf(indexfp, "[Archive](../packages/docker/%s)\n", path_basename(strlist_item(docker_images, 0))); + guard_free(docker_images); + } else { + fprintf(indexfp, "N/A\n"); + } + guard_free(pattern); } + fprintf(indexfp, "\n"); + + guard_strlist_free(&archs); + guard_strlist_free(&platforms); + fclose(indexfp); - // "latest" is an array of pointers to ctxs[]. Do not free the contents of the array. - guard_free(latest); + // "latest_deliveries" is an array of pointers to ctxs[]. Do not free the contents of the array. + guard_free(latest_deliveries); return 0; } -- cgit From 4c403d1f1318a163b017605c2af6d1a14c579f99 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Sat, 7 Dec 2024 18:09:34 -0500 Subject: Remove dead code --- src/cli/stasis_indexer/junitxml_report.c | 5 ----- 1 file changed, 5 deletions(-) (limited to 'src') diff --git a/src/cli/stasis_indexer/junitxml_report.c b/src/cli/stasis_indexer/junitxml_report.c index 9b85a3f..b9d185c 100644 --- a/src/cli/stasis_indexer/junitxml_report.c +++ b/src/cli/stasis_indexer/junitxml_report.c @@ -79,11 +79,6 @@ int indexer_junitxml_report(struct Delivery ctx[], const size_t nelem) { guard_free(bname_tmp); char result_outfile[PATH_MAX] = {0}; - - char result_path[PATH_MAX] = {0}; - //snprintf(result_path, sizeof(result_path) -1 , "%s/%s", platform, arch); - //mkdirs(result_path, 0755); - char *short_name_pattern = NULL; asprintf(&short_name_pattern, "-%s", current->info.release_name); -- cgit