aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJoseph Hunkeler <jhunkeler@users.noreply.github.com>2024-12-07 19:53:44 -0500
committerGitHub <noreply@github.com>2024-12-07 19:53:44 -0500
commit456c5a481a7dabb53434a696488ac6eecb962d5b (patch)
tree8f4743a4cfdad61f9aeac0dffce050e5bd9eef1d
parentbdadebfceffad22179b33948113b2bf82f02c1f7 (diff)
parent4c403d1f1318a163b017605c2af6d1a14c579f99 (diff)
downloadstasis-456c5a481a7dabb53434a696488ac6eecb962d5b.tar.gz
Merge pull request #74 from jhunkeler/with-indexer-tlc
Combined: indexer-tlc and clone-existing-directory
-rw-r--r--README.md2
-rw-r--r--include/core.h1
-rw-r--r--include/delivery.h12
-rw-r--r--include/utils.h5
-rw-r--r--src/cli/stasis/args.c2
-rw-r--r--src/cli/stasis/args.h13
-rw-r--r--src/cli/stasis/stasis_main.c28
-rw-r--r--src/cli/stasis_indexer/CMakeLists.txt11
-rw-r--r--src/cli/stasis_indexer/args.c38
-rw-r--r--src/cli/stasis_indexer/args.h9
-rw-r--r--src/cli/stasis_indexer/callbacks.c39
-rw-r--r--src/cli/stasis_indexer/callbacks.h9
-rw-r--r--src/cli/stasis_indexer/helpers.c337
-rw-r--r--src/cli/stasis_indexer/helpers.h27
-rw-r--r--src/cli/stasis_indexer/junitxml_report.c159
-rw-r--r--src/cli/stasis_indexer/junitxml_report.h8
-rw-r--r--src/cli/stasis_indexer/readmes.c112
-rw-r--r--src/cli/stasis_indexer/readmes.h8
-rw-r--r--src/cli/stasis_indexer/stasis_indexer.c942
-rw-r--r--src/cli/stasis_indexer/stasis_indexer_main.c398
-rw-r--r--src/cli/stasis_indexer/website.c68
-rw-r--r--src/cli/stasis_indexer/website.h8
-rw-r--r--src/lib/core/delivery_artifactory.c17
-rw-r--r--src/lib/core/delivery_docker.c2
-rw-r--r--src/lib/core/delivery_init.c30
-rw-r--r--src/lib/core/globals.c1
-rw-r--r--src/lib/core/str.c32
-rw-r--r--src/lib/core/utils.c22
-rw-r--r--stasis_pandoc.css6
29 files changed, 1359 insertions, 987 deletions
diff --git a/README.md b/README.md
index f1d198e..196f653 100644
--- a/README.md
+++ b/README.md
@@ -163,6 +163,8 @@ stasis mydelivery.ini
| --overwrite | n/a | Overwrite an existing release |
| --no-docker | n/a | Do not build docker images |
| --no-artifactory | n/a | Do not upload artifacts to Artifactory |
+| --no-artifactory-build-info| n/a | Do not upload build info objects to Artifactory |
+| --no-artifactory-upload | n/a | Do not upload artifacts to Artifactory (dry-run) |
| --no-testing | n/a | Do not execute test scripts |
| --no-parallel | n/a | Do not execute tests in parallel |
| --no-rewrite | n/a | Do not rewrite paths and URLs in output files |
diff --git a/include/core.h b/include/core.h
index b8b047f..362ac8d 100644
--- a/include/core.h
+++ b/include/core.h
@@ -40,6 +40,7 @@ struct STASIS_GLOBAL {
bool enable_docker; //!< Enable docker image builds
bool enable_artifactory; //!< Enable artifactory uploads
bool enable_artifactory_build_info; //!< Enable build info (best disabled for pure test runs)
+ bool enable_artifactory_upload; //!< Enable artifactory file upload (dry-run when false)
bool enable_testing; //!< Enable package testing
bool enable_overwrite; //!< Enable release file clobbering
bool enable_rewrite_spec_stage_2; //!< Enable automatic @STR@ replacement in output files
diff --git a/include/delivery.h b/include/delivery.h
index 2ab25d1..40ca3e6 100644
--- a/include/delivery.h
+++ b/include/delivery.h
@@ -424,6 +424,9 @@ int populate_mission_ini(struct Delivery **ctx, int render_mode);
void validate_delivery_ini(struct INIFILE *ini);
int filter_repo_tags(char *repo, struct StrList *patterns);
+
+#define DELIVERY_NOT_FOUND 0
+#define DELIVERY_FOUND 1
/**
* Determine whether a release on-disk matches the release name in use
* @param ctx Delivery context
@@ -433,4 +436,13 @@ int delivery_exists(struct Delivery *ctx);
int delivery_overlay_packages_from_env(struct Delivery *ctx, const char *env_name);
+/**
+ * Retrieve remote deliveries associated with the current version series
+ * @param ctx Delivery context
+ * @return -1 on error
+ * @return 1 on failure
+ * @return 0 on success
+ */
+int delivery_series_sync(struct Delivery *ctx);
+
#endif //STASIS_DELIVERY_H
diff --git a/include/utils.h b/include/utils.h
index e26b3c5..87f28cc 100644
--- a/include/utils.h
+++ b/include/utils.h
@@ -408,4 +408,9 @@ char *find_version_spec(char *package_name);
*/
int env_manipulate_pathstr(const char *key, char *path, int mode);
+/**
+* Append or replace a file extension
+*/
+int gen_file_extension_str(char *filename, const char *extension);
+
#endif //STASIS_UTILS_H
diff --git a/src/cli/stasis/args.c b/src/cli/stasis/args.c
index ed11ab9..f3ce823 100644
--- a/src/cli/stasis/args.c
+++ b/src/cli/stasis/args.c
@@ -17,6 +17,7 @@ struct option long_options[] = {
{"no-docker", no_argument, 0, OPT_NO_DOCKER},
{"no-artifactory", no_argument, 0, OPT_NO_ARTIFACTORY},
{"no-artifactory-build-info", no_argument, 0, OPT_NO_ARTIFACTORY_BUILD_INFO},
+ {"no-artifactory-upload", no_argument, 0, OPT_NO_ARTIFACTORY_UPLOAD},
{"no-testing", no_argument, 0, OPT_NO_TESTING},
{"no-parallel", no_argument, 0, OPT_NO_PARALLEL},
{"no-rewrite", no_argument, 0, OPT_NO_REWRITE_SPEC_STAGE_2},
@@ -39,6 +40,7 @@ const char *long_options_help[] = {
"Do not build docker images",
"Do not upload artifacts to Artifactory",
"Do not upload build info objects to Artifactory",
+ "Do not upload artifacts to Artifactory (dry-run)",
"Do not execute test scripts",
"Do not execute tests in parallel",
"Do not rewrite paths and URLs in output files",
diff --git a/src/cli/stasis/args.h b/src/cli/stasis/args.h
index 932eac7..5bad752 100644
--- a/src/cli/stasis/args.h
+++ b/src/cli/stasis/args.h
@@ -10,12 +10,13 @@
#define OPT_NO_DOCKER 1001
#define OPT_NO_ARTIFACTORY 1002
#define OPT_NO_ARTIFACTORY_BUILD_INFO 1003
-#define OPT_NO_TESTING 1004
-#define OPT_OVERWRITE 1005
-#define OPT_NO_REWRITE_SPEC_STAGE_2 1006
-#define OPT_FAIL_FAST 1007
-#define OPT_NO_PARALLEL 1008
-#define OPT_POOL_STATUS_INTERVAL 1009
+#define OPT_NO_ARTIFACTORY_UPLOAD 1004
+#define OPT_NO_TESTING 1005
+#define OPT_OVERWRITE 1006
+#define OPT_NO_REWRITE_SPEC_STAGE_2 1007
+#define OPT_FAIL_FAST 1009
+#define OPT_NO_PARALLEL 1010
+#define OPT_POOL_STATUS_INTERVAL 1011
extern struct option long_options[];
void usage(char *progname);
diff --git a/src/cli/stasis/stasis_main.c b/src/cli/stasis/stasis_main.c
index 093e32e..dc4e2d1 100644
--- a/src/cli/stasis/stasis_main.c
+++ b/src/cli/stasis/stasis_main.c
@@ -102,6 +102,10 @@ int main(int argc, char *argv[]) {
case OPT_NO_ARTIFACTORY_BUILD_INFO:
globals.enable_artifactory_build_info = false;
break;
+ case OPT_NO_ARTIFACTORY_UPLOAD:
+ globals.enable_artifactory_build_info = false;
+ globals.enable_artifactory_upload = false;
+ break;
case OPT_NO_TESTING:
globals.enable_testing = false;
break;
@@ -220,11 +224,31 @@ int main(int argc, char *argv[]) {
// Safety gate: Avoid clobbering a delivered release unless the user wants that behavior
msg(STASIS_MSG_L1, "Checking release history\n");
- if (delivery_exists(&ctx)) {
+ if (!globals.enable_overwrite && delivery_exists(&ctx) == DELIVERY_FOUND) {
msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Refusing to overwrite release: %s\nUse --overwrite to enable release clobbering.\n", ctx.info.release_name);
exit(1);
}
+ if (globals.enable_artifactory) {
+ // We need to download previous revisions to ensure processed packages are available at build-time
+ // This is also a docker requirement. Python wheels must exist locally.
+ if (ctx.meta.rc > 1) {
+ msg(STASIS_MSG_L1, "Syncing delivery artifacts for %s\n", ctx.info.build_name);
+ if (delivery_series_sync(&ctx) != 0) {
+ msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Unable to sync artifacts for %s\n", ctx.info.build_name);
+ msg(STASIS_MSG_L3, "Case #1:\n"
+ "\tIf this is a new 'version', and 'rc' is greater "
+ "than 1, then no previous deliveries exist remotely. "
+ "Reset 'rc' to 1.\n");
+ msg(STASIS_MSG_L3, "Case #2:\n"
+ "\tThe Artifactory server %s is unreachable, or the credentials used "
+ "are invalid.\n", globals.jfrog.url);
+ // No continue-on-error check. Without the previous delivery nothing can be done.
+ exit(1);
+ }
+ }
+ }
+
// Unlikely to occur: this should help prevent rmtree() from destroying your entire filesystem
// if path is "/" then, die
// or if empty string, die
@@ -541,7 +565,7 @@ int main(int argc, char *argv[]) {
}
if (want_artifactory) {
- if (globals.enable_artifactory) {
+ if (globals.enable_artifactory && globals.enable_artifactory_upload) {
msg(STASIS_MSG_L1, "Uploading artifacts\n");
delivery_artifact_upload(&ctx);
} else {
diff --git a/src/cli/stasis_indexer/CMakeLists.txt b/src/cli/stasis_indexer/CMakeLists.txt
index eae1394..1d4c658 100644
--- a/src/cli/stasis_indexer/CMakeLists.txt
+++ b/src/cli/stasis_indexer/CMakeLists.txt
@@ -1,6 +1,15 @@
add_executable(stasis_indexer
- stasis_indexer.c
+ args.c
+ stasis_indexer_main.c
+ callbacks.c
+ helpers.c
+ junitxml_report.c
+ website.c
+ website.h
+ readmes.c
+ readmes.h
)
+target_include_directories(stasis_indexer PRIVATE ${CMAKE_SOURCE_DIR})
target_link_libraries(stasis_indexer PRIVATE stasis_core)
install(TARGETS stasis_indexer RUNTIME)
diff --git a/src/cli/stasis_indexer/args.c b/src/cli/stasis_indexer/args.c
new file mode 100644
index 0000000..2d92ab0
--- /dev/null
+++ b/src/cli/stasis_indexer/args.c
@@ -0,0 +1,38 @@
+#include "core.h"
+#include "args.h"
+
+struct option long_options[] = {
+ {"help", no_argument, 0, 'h'},
+ {"destdir", required_argument, 0, 'd'},
+ {"verbose", no_argument, 0, 'v'},
+ {"unbuffered", no_argument, 0, 'U'},
+ {"web", no_argument, 0, 'w'},
+ {0, 0, 0, 0},
+};
+
+const char *long_options_help[] = {
+ "Display this usage statement",
+ "Destination directory",
+ "Increase output verbosity",
+ "Disable line buffering",
+ "Generate HTML indexes (requires pandoc)",
+ NULL,
+};
+
+void usage(char *name) {
+ const int maxopts = sizeof(long_options) / sizeof(long_options[0]);
+ char *opts = calloc(maxopts + 1, sizeof(char));
+ for (int i = 0; i < maxopts; i++) {
+ opts[i] = (char) long_options[i].val;
+ }
+ printf("usage: %s [-%s] {{STASIS_ROOT}...}\n", name, opts);
+ guard_free(opts);
+
+ for (int i = 0; i < maxopts - 1; i++) {
+ char line[255] = {0};
+ sprintf(line, " --%s -%c %-20s", long_options[i].name, long_options[i].val, long_options_help[i]);
+ puts(line);
+ }
+
+}
+
diff --git a/src/cli/stasis_indexer/args.h b/src/cli/stasis_indexer/args.h
new file mode 100644
index 0000000..543aa4b
--- /dev/null
+++ b/src/cli/stasis_indexer/args.h
@@ -0,0 +1,9 @@
+#ifndef STASIS_ARGS_H
+#define STASIS_ARGS_H
+
+#include <getopt.h>
+
+extern struct option long_options[];
+void usage(char *name);
+
+#endif //STASIS_ARGS_H
diff --git a/src/cli/stasis_indexer/callbacks.c b/src/cli/stasis_indexer/callbacks.c
new file mode 100644
index 0000000..0186e1c
--- /dev/null
+++ b/src/cli/stasis_indexer/callbacks.c
@@ -0,0 +1,39 @@
+//
+// Created by jhunk on 11/15/24.
+//
+
+#include "core.h"
+#include "callbacks.h"
+
+// qsort callback to sort delivery contexts by compact python version
+int callback_sort_deliveries_cmpfn(const void *a, const void *b) {
+ const struct Delivery *delivery1 = (struct Delivery *) a;
+ const size_t delivery1_python = strtoul(delivery1->meta.python_compact, NULL, 10);
+ const struct Delivery *delivery2 = (struct Delivery *) b;
+ const size_t delivery2_python = strtoul(delivery2->meta.python_compact, NULL, 10);
+
+ if (delivery2_python > delivery1_python) {
+ return 1;
+ }
+ if (delivery2_python < delivery1_python) {
+ return -1;
+ }
+ return 0;
+}
+
+// qsort callback to sort dynamically allocated delivery contexts by compact python version
+int callback_sort_deliveries_dynamic_cmpfn(const void *a, const void *b) {
+ const struct Delivery *delivery1 = a;
+ const size_t delivery1_python = strtoul(delivery1->meta.python_compact, NULL, 10);
+ const struct Delivery *delivery2 = b;
+ const size_t delivery2_python = strtoul(delivery2->meta.python_compact, NULL, 10);
+
+ if (delivery2_python > delivery1_python) {
+ return 1;
+ }
+ if (delivery2_python < delivery1_python) {
+ return -1;
+ }
+ return 0;
+}
+
diff --git a/src/cli/stasis_indexer/callbacks.h b/src/cli/stasis_indexer/callbacks.h
new file mode 100644
index 0000000..7d95cbb
--- /dev/null
+++ b/src/cli/stasis_indexer/callbacks.h
@@ -0,0 +1,9 @@
+#ifndef CALLBACKS_H
+#define CALLBACKS_H
+
+#include "delivery.h"
+
+int callback_sort_deliveries_cmpfn(const void *a, const void *b);
+int callback_sort_deliveries_dynamic_cmpfn(const void *a, const void *b);
+
+#endif //CALLBACKS_H
diff --git a/src/cli/stasis_indexer/helpers.c b/src/cli/stasis_indexer/helpers.c
new file mode 100644
index 0000000..6d2fdd0
--- /dev/null
+++ b/src/cli/stasis_indexer/helpers.c
@@ -0,0 +1,337 @@
+//
+// Created by jhunk on 11/15/24.
+//
+
+#include "core.h"
+#include "helpers.h"
+
+struct StrList *get_architectures(struct Delivery ctx[], const size_t nelem) {
+ struct StrList *architectures = strlist_init();
+ for (size_t i = 0; i < nelem; i++) {
+ if (ctx[i].system.arch) {
+ if (!strstr_array(architectures->data, ctx[i].system.arch)) {
+ strlist_append(&architectures, ctx[i].system.arch);
+ }
+ }
+ }
+ return architectures;
+}
+
+struct StrList *get_platforms(struct Delivery ctx[], const size_t nelem) {
+ struct StrList *platforms = strlist_init();
+ for (size_t i = 0; i < nelem; i++) {
+ if (ctx[i].system.platform) {
+ if (!strstr_array(platforms->data, ctx[i].system.platform[DELIVERY_PLATFORM_RELEASE])) {
+ strlist_append(&platforms, ctx[i].system.platform[DELIVERY_PLATFORM_RELEASE]);
+ }
+ }
+ }
+ return platforms;
+}
+
+int get_pandoc_version(size_t *result) {
+ *result = 0;
+ int state = 0;
+ char *version_str = shell_output("pandoc --version", &state);
+ if (state || !version_str) {
+ // an error occurred
+ return -1;
+ }
+
+ // Verify that we're looking at pandoc
+ if (strlen(version_str) > 7 && !strncmp(version_str, "pandoc ", 7)) {
+ // we have pandoc
+ char *v_begin = &version_str[7];
+ if (!v_begin) {
+ SYSERROR("unexpected pandoc output: %s", version_str);
+ return -1;
+ }
+ char *v_end = strchr(version_str, '\n');
+ if (v_end) {
+ *v_end = 0;
+ }
+
+ char **parts = split(v_begin, ".", 0);
+ if (!parts) {
+ SYSERROR("unable to split pandoc version string, '%s': %s", version_str, strerror(errno));
+ return -1;
+ }
+
+ size_t parts_total;
+ for (parts_total = 0; parts[parts_total] != NULL; parts_total++) {}
+
+ // generate the version as an integer
+ // note: pandoc version scheme never exceeds four elements (or bytes in this case)
+ for (size_t i = 0; i < 4; i++) {
+ unsigned char tmp = 0;
+ if (i < parts_total) {
+ // only process version elements we have. the rest will be zeros.
+ tmp = strtoul(parts[i], NULL, 10);
+ }
+ // pack version element into result
+ *result = *result << 8 | tmp;
+ }
+ } else {
+ // invalid version string
+ return 1;
+ }
+
+ return 0;
+}
+
+int pandoc_exec(const char *in_file, const char *out_file, const char *css_file, const char *title) {
+ if (!find_program("pandoc")) {
+ fprintf(stderr, "pandoc is not installed: unable to generate HTML indexes\n");
+ return 0;
+ }
+
+ char pandoc_versioned_args[255] = {0};
+ size_t pandoc_version = 0;
+ if (!get_pandoc_version(&pandoc_version)) {
+ // < 2.19
+ if (pandoc_version < 0x02130000) {
+ strcat(pandoc_versioned_args, "--self-contained ");
+ } else {
+ // >= 2.19
+ strcat(pandoc_versioned_args, "--embed-resources ");
+ }
+
+ // >= 1.15.0.4
+ if (pandoc_version >= 0x010f0004) {
+ strcat(pandoc_versioned_args, "--standalone ");
+ }
+
+ // >= 1.10.0.1
+ if (pandoc_version >= 0x010a0001) {
+ strcat(pandoc_versioned_args, "-f gfm+autolink_bare_uris ");
+ }
+
+ // > 3.1.9
+ if (pandoc_version > 0x03010900) {
+ strcat(pandoc_versioned_args, "-f gfm+alerts ");
+ }
+ }
+
+ // Converts a markdown file to html
+ char cmd[STASIS_BUFSIZ] = {0};
+ strcpy(cmd, "pandoc ");
+ strcat(cmd, pandoc_versioned_args);
+ if (css_file && strlen(css_file)) {
+ strcat(cmd, "--css ");
+ strcat(cmd, css_file);
+ }
+ strcat(cmd, " ");
+ strcat(cmd, "--metadata title=\"");
+ strcat(cmd, title);
+ strcat(cmd, "\" ");
+ strcat(cmd, "-o ");
+ strcat(cmd, out_file);
+ strcat(cmd, " ");
+ strcat(cmd, in_file);
+
+ if (globals.verbose) {
+ puts(cmd);
+ }
+
+ // This might be negative when killed by a signal.
+ return system(cmd);
+}
+
+
+int micromamba_configure(const struct Delivery *ctx, struct MicromambaInfo *m) {
+ int status = 0;
+ char *micromamba_prefix = NULL;
+ if (asprintf(&micromamba_prefix, "%s/bin", ctx->storage.tools_dir) < 0) {
+ return -1;
+ }
+ m->conda_prefix = globals.conda_install_prefix;
+ m->micromamba_prefix = micromamba_prefix;
+
+ const size_t pathvar_len = strlen(getenv("PATH")) + strlen(m->micromamba_prefix) + strlen(m->conda_prefix) + 3 + 4 + 1;
+ // ^^^^^^^^^^^^^^^^^^
+ // 3 = separators
+ // 4 = chars (/bin)
+ // 1 = nul terminator
+ char *pathvar = calloc(pathvar_len, sizeof(*pathvar));
+ if (!pathvar) {
+ SYSERROR("%s", "Unable to allocate bytes for temporary path string");
+ exit(1);
+ }
+ snprintf(pathvar, pathvar_len, "%s/bin:%s:%s", m->conda_prefix, m->micromamba_prefix, getenv("PATH"));
+ setenv("PATH", pathvar, 1);
+ guard_free(pathvar);
+
+ status += micromamba(m, "config prepend --env channels conda-forge");
+ if (!globals.verbose) {
+ status += micromamba(m, "config set --env quiet true");
+ }
+ status += micromamba(m, "config set --env always_yes true");
+ status += micromamba(m, "install conda-build pandoc");
+
+ return status;
+}
+
+int get_latest_rc(struct Delivery ctx[], const size_t nelem) {
+ int result = 0;
+ for (size_t i = 0; i < nelem; i++) {
+ if (ctx[i].meta.rc > result) {
+ result = ctx[i].meta.rc;
+ }
+ }
+ return result;
+}
+
+int sort_by_latest_rc(const void *a, const void *b) {
+ const struct Delivery *aa = a;
+ const struct Delivery *bb = b;
+ if (aa->meta.rc > bb->meta.rc) {
+ return -1;
+ }
+ if (aa->meta.rc < bb->meta.rc) {
+ return 1;
+ }
+ return 0;
+}
+
+struct Delivery *get_latest_deliveries(struct Delivery ctx[], size_t nelem) {
+ int latest = 0;
+ size_t n = 0;
+
+ struct Delivery *result = calloc(nelem + 1, sizeof(*result));
+ if (!result) {
+ fprintf(stderr, "Unable to allocate %zu bytes for result delivery array: %s\n", nelem * sizeof(*result), strerror(errno));
+ return NULL;
+ }
+
+ latest = get_latest_rc(ctx, nelem);
+ qsort(ctx, nelem, sizeof(*ctx), sort_by_latest_rc);
+ for (size_t i = 0; i < nelem; i++) {
+ if (ctx[i].meta.rc == latest) {
+ result[n] = ctx[i];
+ n++;
+ }
+ }
+ return result;
+}
+
+int get_files(struct StrList **out, const char *path, const char *pattern, ...) {
+ va_list args;
+ va_start(args, pattern);
+ char userpattern[PATH_MAX] = {0};
+ vsprintf(userpattern, pattern, args);
+ va_end(args);
+ if (!strlen(userpattern)) {
+ userpattern[0] = '*';
+ userpattern[1] = 0;
+ }
+ struct StrList *list = listdir(path);
+ if (!list) {
+ return -1;
+ }
+
+ if (!*out) {
+ *out = strlist_init();
+ if (!*out) {
+ guard_strlist_free(&list);
+ return -1;
+ }
+ }
+
+ size_t no_match = 0;
+ for (size_t i = 0; i < strlist_count(list); i++) {
+ char *item = strlist_item(list, i);
+ if (fnmatch(userpattern, item, 0)) {
+ no_match++;
+ } else {
+ strlist_append(out, item);
+ }
+ }
+ if (no_match >= strlist_count(list)) {
+ fprintf(stderr, "no files matching the pattern: %s\n", userpattern);
+ guard_strlist_free(&list);
+ return -1;
+ }
+ guard_strlist_free(&list);
+ return 0;
+}
+
+struct StrList *get_docker_images(struct Delivery *ctx, char *pattern) {
+ char *tarball = NULL;
+ asprintf(&tarball, "%s*.tar*", pattern);
+ if (!tarball) {
+ SYSERROR("%s", "Unable to allocate bytes for docker image wildcard pattern");
+ return NULL;
+ }
+ tolower_s(tarball);
+ replace_text(tarball, "+", "-", 0);
+
+ struct StrList *files = NULL;
+ get_files(&files, ctx->storage.docker_artifact_dir, tarball);
+ guard_free(tarball);
+
+ return files;
+}
+
+int load_metadata(struct Delivery *ctx, const char *filename) {
+ char line[STASIS_NAME_MAX] = {0};
+
+ FILE *fp = fopen(filename, "r");
+ if (!fp) {
+ return -1;
+ }
+
+ while (fgets(line, sizeof(line) - 1, fp) != NULL) {
+ char **parts = split(line, " ", 1);
+ const char *name = parts[0];
+ char *value = parts[1];
+
+ strip(value);
+ if (!strcmp(name, "name")) {
+ ctx->meta.name = strdup(value);
+ } else if (!strcmp(name, "version")) {
+ ctx->meta.version = strdup(value);
+ } else if (!strcmp(name, "rc")) {
+ ctx->meta.rc = (int) strtol(value, NULL, 10);
+ } else if (!strcmp(name, "python")) {
+ ctx->meta.python = strdup(value);
+ } else if (!strcmp(name, "python_compact")) {
+ ctx->meta.python_compact = strdup(value);
+ } else if (!strcmp(name, "mission")) {
+ ctx->meta.mission = strdup(value);
+ } else if (!strcmp(name, "codename")) {
+ ctx->meta.codename = strdup(value);
+ } else if (!strcmp(name, "platform")) {
+ ctx->system.platform = split(value, " ", 0);
+ } else if (!strcmp(name, "arch")) {
+ ctx->system.arch = strdup(value);
+ } else if (!strcmp(name, "time")) {
+ ctx->info.time_str_epoch = strdup(value);
+ } else if (!strcmp(name, "release_fmt")) {
+ ctx->rules.release_fmt = strdup(value);
+ } else if (!strcmp(name, "release_name")) {
+ ctx->info.release_name = strdup(value);
+ } else if (!strcmp(name, "build_name_fmt")) {
+ ctx->rules.build_name_fmt = strdup(value);
+ } else if (!strcmp(name, "build_name")) {
+ ctx->info.build_name = strdup(value);
+ } else if (!strcmp(name, "build_number_fmt")) {
+ ctx->rules.build_number_fmt = strdup(value);
+ } else if (!strcmp(name, "build_number")) {
+ ctx->info.build_number = strdup(value);
+ } else if (!strcmp(name, "conda_installer_baseurl")) {
+ ctx->conda.installer_baseurl = strdup(value);
+ } else if (!strcmp(name, "conda_installer_name")) {
+ ctx->conda.installer_name = strdup(value);
+ } else if (!strcmp(name, "conda_installer_version")) {
+ ctx->conda.installer_version = strdup(value);
+ } else if (!strcmp(name, "conda_installer_platform")) {
+ ctx->conda.installer_platform = strdup(value);
+ } else if (!strcmp(name, "conda_installer_arch")) {
+ ctx->conda.installer_arch = strdup(value);
+ }
+ GENERIC_ARRAY_FREE(parts);
+ }
+ fclose(fp);
+
+ return 0;
+}
diff --git a/src/cli/stasis_indexer/helpers.h b/src/cli/stasis_indexer/helpers.h
new file mode 100644
index 0000000..d493f75
--- /dev/null
+++ b/src/cli/stasis_indexer/helpers.h
@@ -0,0 +1,27 @@
+#ifndef HELPERS_H
+#define HELPERS_H
+
+#include "delivery.h"
+
+#define ARRAY_COUNT_DYNAMIC(X, COUNTER) \
+ do { \
+ for (COUNTER = 0; X && X[COUNTER] != NULL; COUNTER++) {} \
+ } while(0)
+
+#define ARRAY_COUNT_BY_STRUCT_MEMBER(X, MEMBER, COUNTER) \
+ do { \
+ for (COUNTER = 0; X[COUNTER].MEMBER != NULL; COUNTER++) {} \
+ } while(0)
+
+struct StrList *get_architectures(struct Delivery ctx[], size_t nelem);
+struct StrList *get_platforms(struct Delivery ctx[], size_t nelem);
+int get_pandoc_version(size_t *result);
+int pandoc_exec(const char *in_file, const char *out_file, const char *css_file, const char *title);
+int get_latest_rc(struct Delivery ctx[], size_t nelem);
+struct Delivery *get_latest_deliveries(struct Delivery ctx[], size_t nelem);
+int get_files(struct StrList **out, const char *path, const char *pattern, ...);
+struct StrList *get_docker_images(struct Delivery *ctx, char *pattern);
+int load_metadata(struct Delivery *ctx, const char *filename);
+int micromamba_configure(const struct Delivery *ctx, struct MicromambaInfo *m);
+
+#endif //HELPERS_H
diff --git a/src/cli/stasis_indexer/junitxml_report.c b/src/cli/stasis_indexer/junitxml_report.c
new file mode 100644
index 0000000..b9d185c
--- /dev/null
+++ b/src/cli/stasis_indexer/junitxml_report.c
@@ -0,0 +1,159 @@
+//
+// Created by jhunk on 11/15/24.
+//
+
+#include "core.h"
+#include "callbacks.h"
+#include "junitxml.h"
+#include "junitxml_report.h"
+
+int indexer_junitxml_report(struct Delivery ctx[], const size_t nelem) {
+ struct Delivery *latest = get_latest_deliveries(ctx, nelem);
+ if (!latest) {
+ return -1;
+ }
+ size_t latest_count;
+ ARRAY_COUNT_BY_STRUCT_MEMBER(latest, meta.name, latest_count);
+
+ char indexfile[PATH_MAX] = {0};
+ sprintf(indexfile, "%s/README.md", ctx->storage.results_dir);
+
+ struct StrList *file_listing = listdir(ctx->storage.results_dir);
+ if (!file_listing) {
+ // no test results to process
+ return 0;
+ }
+
+ if (!pushd(ctx->storage.results_dir)) {
+ FILE *indexfp = fopen(indexfile, "w+");
+ if (!indexfp) {
+ fprintf(stderr, "Unable to open %s for writing\n", indexfile);
+ return -1;
+ }
+
+ struct StrList *archs = get_architectures(latest, nelem);
+ struct StrList *platforms = get_platforms(latest, nelem);
+ qsort(latest, latest_count, sizeof(*latest), callback_sort_deliveries_dynamic_cmpfn);
+ fprintf(indexfp, "# %s-%s Test Report\n\n", ctx->meta.name, ctx->meta.version);
+ size_t no_printable_data = 0;
+
+ size_t delivery_count = get_latest_rc(latest, latest_count);
+ for (size_t p = 0; p < strlist_count(platforms); p++) {
+ char *platform = strlist_item(platforms, p);
+ for (size_t a = 0; a < strlist_count(archs); a++) {
+ char *arch = strlist_item(archs, a);
+
+ fprintf(indexfp, "## %s-%s\n\n", platform, arch);
+ for (size_t d = 0; d < delivery_count; d++) {
+ struct Delivery *current = &ctx[d];
+ if (current->meta.rc == (int) d + 1
+ && strcmp(current->system.arch, arch) != 0
+ && strcmp(current->system.platform[DELIVERY_PLATFORM_RELEASE], platform) != 0) {
+ continue;
+ }
+
+ fprintf(indexfp, "### %s\n", current->info.release_name);
+ fprintf(indexfp, "\n|Suite|Duration|Fail |Skip |Error |\n");
+ fprintf(indexfp, "|:----|:------:|:------:|:---:|:----:|\n");
+ for (size_t f = 0; f < strlist_count(file_listing); f++) {
+ char *filename = strlist_item(file_listing, f);
+ if (!endswith(filename, ".xml")) {
+ continue;
+ }
+
+ char pattern[PATH_MAX] = {0};
+ snprintf(pattern, sizeof(pattern) - 1, "*%s*", current->info.release_name);
+ if (!fnmatch(pattern, filename, 0) && strstr(filename, platform) &&
+ strstr(filename, arch)) {
+ struct JUNIT_Testsuite *testsuite = junitxml_testsuite_read(filename);
+ if (testsuite) {
+ if (globals.verbose) {
+ printf("%s: duration: %0.4f, failed: %d, skipped: %d, errors: %d\n", filename,
+ testsuite->time, testsuite->failures, testsuite->skipped,
+ testsuite->errors);
+ }
+
+ char *bname_tmp = strdup(filename);
+ char *bname = path_basename(bname_tmp);
+ bname[strlen(bname) - 4] = 0;
+ guard_free(bname_tmp);
+
+ char result_outfile[PATH_MAX] = {0};
+ char *short_name_pattern = NULL;
+ asprintf(&short_name_pattern, "-%s", current->info.release_name);
+
+ char short_name[PATH_MAX] = {0};
+ strncpy(short_name, bname, sizeof(short_name) - 1);
+ replace_text(short_name, short_name_pattern, "", 0);
+ replace_text(short_name, "results-", "", 0);
+ guard_free(short_name_pattern);
+
+ fprintf(indexfp, "|[%s](%s.html)|%0.4f|%d|%d|%d|\n", short_name,
+ bname,
+ testsuite->time, testsuite->failures, testsuite->skipped,
+ testsuite->errors);
+
+ snprintf(result_outfile, sizeof(result_outfile) - strlen(bname) - 3, "%s.md",
+ bname);
+ FILE *resultfp = fopen(result_outfile, "w+");
+ if (!resultfp) {
+ SYSERROR("Unable to open %s for writing", result_outfile);
+ return -1;
+ }
+
+ for (size_t i = 0; i < testsuite->_tc_inuse; i++) {
+ if (testsuite->testcase[i]->tc_result_state_type) {
+ const char *type_str = NULL;
+ const int state = testsuite->testcase[i]->tc_result_state_type;
+ const char *message = NULL;
+ if (state == JUNIT_RESULT_STATE_FAILURE) {
+ message = testsuite->testcase[i]->result_state.failure->message;
+ type_str = "[FAILED]";
+ } else if (state == JUNIT_RESULT_STATE_ERROR) {
+ message = testsuite->testcase[i]->result_state.error->message;
+ type_str = "[ERROR]";
+ } else if (state == JUNIT_RESULT_STATE_SKIPPED) {
+ message = testsuite->testcase[i]->result_state.skipped->message;
+ type_str = "[SKIPPED]";
+ }
+ fprintf(resultfp, "### %s %s :: %s\n", type_str,
+ testsuite->testcase[i]->classname, testsuite->testcase[i]->name);
+ fprintf(resultfp, "\nDuration: %0.04fs\n", testsuite->testcase[i]->time);
+ fprintf(resultfp, "\n```\n%s\n```\n", message);
+ }
+ }
+ junitxml_testsuite_free(&testsuite);
+ fclose(resultfp);
+ } else {
+ fprintf(stderr, "bad test suite: %s: %s\n", strerror(errno), filename);
+ }
+ } else {
+ if (!no_printable_data) {
+ // Triggering for reasons unknown
+ //fprintf(indexfp, "|No data|-|-|-|-|-|-|\n");
+ no_printable_data = 1;
+ }
+ }
+ }
+ }
+ fprintf(indexfp, "\n");
+ no_printable_data = 0;
+ }
+ fprintf(indexfp, "\n");
+ }
+ guard_strlist_free(&archs);
+ guard_strlist_free(&platforms);
+ fclose(indexfp);
+ popd();
+ } else {
+ fprintf(stderr, "Unable to enter delivery directory: %s\n", ctx->storage.delivery_dir);
+ guard_free(latest);
+ return -1;
+ }
+
+ // "latest" is an array of pointers to ctxs[]. Do not free the contents of the array.
+ guard_free(latest);
+ return 0;
+}
+
+
diff --git a/src/cli/stasis_indexer/junitxml_report.h b/src/cli/stasis_indexer/junitxml_report.h
new file mode 100644
index 0000000..6d2a248
--- /dev/null
+++ b/src/cli/stasis_indexer/junitxml_report.h
@@ -0,0 +1,8 @@
+#ifndef JUNITXML_REPORT_H
+#define JUNITXML_REPORT_H
+
+#include "helpers.h"
+
+int indexer_junitxml_report(struct Delivery ctx[], size_t nelem);
+
+#endif //JUNITXML_REPORT_H
diff --git a/src/cli/stasis_indexer/readmes.c b/src/cli/stasis_indexer/readmes.c
new file mode 100644
index 0000000..77b5178
--- /dev/null
+++ b/src/cli/stasis_indexer/readmes.c
@@ -0,0 +1,112 @@
+#include "core.h"
+#include "readmes.h"
+
+int indexer_readmes(struct Delivery ctx[], const size_t nelem) {
+ struct Delivery *latest_deliveries = get_latest_deliveries(ctx, nelem);
+ if (!latest_deliveries) {
+ if (errno) {
+ return -1;
+ }
+ return 0;
+ }
+
+ char indexfile[PATH_MAX] = {0};
+ sprintf(indexfile, "%s/README.md", ctx->storage.delivery_dir);
+
+ FILE *indexfp = fopen(indexfile, "w+");
+ if (!indexfp) {
+ fprintf(stderr, "Unable to open %s for writing\n", indexfile);
+ return -1;
+ }
+ struct StrList *archs = get_architectures(latest_deliveries, nelem);
+ struct StrList *platforms = get_platforms(latest_deliveries, nelem);
+
+ fprintf(indexfp, "# %s-%s\n\n", ctx->meta.name, ctx->meta.version);
+ fprintf(indexfp, "## Current Release\n\n");
+ for (size_t p = 0; p < strlist_count(platforms); p++) {
+ char *platform = strlist_item(platforms, p);
+ for (size_t a = 0; a < strlist_count(archs); a++) {
+ char *arch = strlist_item(archs, a);
+ int have_combo = 0;
+ for (size_t i = 0; i < nelem; i++) {
+ if (latest_deliveries[i].system.platform) {
+ if (strstr(latest_deliveries[i].system.platform[DELIVERY_PLATFORM_RELEASE], platform) &&
+ strstr(latest_deliveries[i].system.arch, arch)) {
+ have_combo = 1;
+ }
+ }
+ }
+ if (!have_combo) {
+ continue;
+ }
+ fprintf(indexfp, "### %s-%s\n\n", platform, arch);
+ for (size_t i = 0; i < nelem; i++) {
+ char link_name[PATH_MAX] = {0};
+ char readme_name[PATH_MAX] = {0};
+ char conf_name[PATH_MAX] = {0};
+ char conf_name_relative[PATH_MAX] = {0};
+ if (!latest_deliveries[i].meta.name) {
+ continue;
+ }
+ sprintf(link_name, "latest-py%s-%s-%s.yml", latest_deliveries[i].meta.python_compact, latest_deliveries[i].system.platform[DELIVERY_PLATFORM_RELEASE], latest_deliveries[i].system.arch);
+ sprintf(readme_name, "README-py%s-%s-%s.md", latest_deliveries[i].meta.python_compact, latest_deliveries[i].system.platform[DELIVERY_PLATFORM_RELEASE], latest_deliveries[i].system.arch);
+ sprintf(conf_name, "%s.ini", latest_deliveries[i].info.release_name);
+ sprintf(conf_name_relative, "../config/%s.ini", latest_deliveries[i].info.release_name);
+ if (strstr(link_name, platform) && strstr(link_name, arch)) {
+ fprintf(indexfp, "- Info: [README](%s)\n", readme_name);
+ fprintf(indexfp, "- Release: [Conda Environment YAML](%s)\n", link_name);
+ fprintf(indexfp, "- Receipt: [STASIS input file](%s)\n", conf_name_relative);
+ fprintf(indexfp, "- Docker: ");
+ struct StrList *docker_images = get_docker_images(&latest_deliveries[i], "");
+ if (docker_images
+ && strlist_count(docker_images)
+ && !strcmp(latest_deliveries[i].system.platform[DELIVERY_PLATFORM_RELEASE], "linux")) {
+ fprintf(indexfp, "[Archive](../packages/docker/%s)\n", path_basename(strlist_item(docker_images, 0)));
+ guard_free(docker_images);
+ } else {
+ fprintf(indexfp, "N/A\n");
+ }
+ }
+ }
+ fprintf(indexfp, "\n");
+ }
+ fprintf(indexfp, "\n");
+ }
+
+ fprintf(indexfp, "## Releases\n");
+ for (size_t i = 0; ctx[i].meta.name != NULL; i++) {
+ struct Delivery *current = &ctx[i];
+ fprintf(indexfp, "### %s\n", current->info.release_name);
+ fprintf(indexfp, "- Info: [README](README-%s.html)\n", current->info.release_name);
+ fprintf(indexfp, "- Release: [Conda Environment YAML](%s.yml)\n", current->info.release_name);
+ fprintf(indexfp, "- Receipt: [STASIS input file](../config/%s.ini)\n", current->info.release_name);
+ fprintf(indexfp, "- Docker: \n");
+
+ char *pattern = NULL;
+ asprintf(&pattern, "*%s*", current->info.build_number);
+ if (!pattern) {
+ SYSERROR("%s", "Unable to allocate bytes for pattern");
+ return -1;
+ }
+
+ struct StrList *docker_images = get_docker_images(current, pattern);
+ if (docker_images
+ && strlist_count(docker_images)
+ && !strcmp(current->system.platform[DELIVERY_PLATFORM_RELEASE], "linux")) {
+ fprintf(indexfp, "[Archive](../packages/docker/%s)\n", path_basename(strlist_item(docker_images, 0)));
+ guard_free(docker_images);
+ } else {
+ fprintf(indexfp, "N/A\n");
+ }
+ guard_free(pattern);
+ }
+ fprintf(indexfp, "\n");
+
+ guard_strlist_free(&archs);
+ guard_strlist_free(&platforms);
+ fclose(indexfp);
+
+ // "latest_deliveries" is an array of pointers to ctxs[]. Do not free the contents of the array.
+ guard_free(latest_deliveries);
+ return 0;
+}
diff --git a/src/cli/stasis_indexer/readmes.h b/src/cli/stasis_indexer/readmes.h
new file mode 100644
index 0000000..d4fa7ac
--- /dev/null
+++ b/src/cli/stasis_indexer/readmes.h
@@ -0,0 +1,8 @@
+#ifndef READMES_H
+#define READMES_H
+
+#include "helpers.h"
+
+int indexer_readmes(struct Delivery ctx[], size_t nelem);
+
+#endif //READMES_H
diff --git a/src/cli/stasis_indexer/stasis_indexer.c b/src/cli/stasis_indexer/stasis_indexer.c
deleted file mode 100644
index fddf18c..0000000
--- a/src/cli/stasis_indexer/stasis_indexer.c
+++ /dev/null
@@ -1,942 +0,0 @@
-#include <getopt.h>
-#include <fnmatch.h>
-#include "delivery.h"
-#include "junitxml.h"
-
-static struct option long_options[] = {
- {"help", no_argument, 0, 'h'},
- {"destdir", required_argument, 0, 'd'},
- {"verbose", no_argument, 0, 'v'},
- {"unbuffered", no_argument, 0, 'U'},
- {"web", no_argument, 0, 'w'},
- {0, 0, 0, 0},
-};
-
-const char *long_options_help[] = {
- "Display this usage statement",
- "Destination directory",
- "Increase output verbosity",
- "Disable line buffering",
- "Generate HTML indexes (requires pandoc)",
- NULL,
-};
-
-static void usage(char *name) {
- int maxopts = sizeof(long_options) / sizeof(long_options[0]);
- char *opts = calloc(maxopts + 1, sizeof(char));
- for (int i = 0; i < maxopts; i++) {
- opts[i] = (char) long_options[i].val;
- }
- printf("usage: %s [-%s] {{STASIS_ROOT}...}\n", name, opts);
- guard_free(opts);
-
- for (int i = 0; i < maxopts - 1; i++) {
- char line[255] = {0};
- sprintf(line, " --%s -%c %-20s", long_options[i].name, long_options[i].val, long_options_help[i]);
- puts(line);
- }
-
-}
-
-int indexer_combine_rootdirs(const char *dest, char **rootdirs, const size_t rootdirs_total) {
- char cmd[PATH_MAX];
- char destdir_bare[PATH_MAX];
- char destdir_with_output[PATH_MAX];
- char *destdir = destdir_bare;
-
- memset(cmd, 0, sizeof(cmd));
- memset(destdir_bare, 0, sizeof(destdir_bare));
- memset(destdir_with_output, 0, sizeof(destdir_bare));
-
- strcpy(destdir_bare, dest);
- strcpy(destdir_with_output, dest);
- strcat(destdir_with_output, "/output");
-
- if (!access(destdir_with_output, F_OK)) {
- destdir = destdir_with_output;
- }
-
- sprintf(cmd, "rsync -ah%s --delete --exclude 'tools/' --exclude 'tmp/' --exclude 'build/' ", globals.verbose ? "v" : "q");
- for (size_t i = 0; i < rootdirs_total; i++) {
- char srcdir_bare[PATH_MAX] = {0};
- char srcdir_with_output[PATH_MAX] = {0};
- char *srcdir = srcdir_bare;
- strcpy(srcdir_bare, rootdirs[i]);
- strcpy(srcdir_with_output, rootdirs[i]);
- strcat(srcdir_with_output, "/output");
-
- if (access(srcdir_bare, F_OK)) {
- fprintf(stderr, "%s does not exist\n", srcdir_bare);
- continue;
- }
-
- if (!access(srcdir_with_output, F_OK)) {
- srcdir = srcdir_with_output;
- }
- snprintf(cmd + strlen(cmd), sizeof(srcdir) - strlen(srcdir) + 4, "'%s'/ ", srcdir);
- }
- snprintf(cmd + strlen(cmd), sizeof(cmd) - strlen(destdir) + 1, " %s/", destdir);
-
- if (globals.verbose) {
- puts(cmd);
- }
-
- if (system(cmd)) {
- return -1;
- }
- return 0;
-}
-
-int indexer_wheels(struct Delivery *ctx) {
- return delivery_index_wheel_artifacts(ctx);
-}
-
-int indexer_load_metadata(struct Delivery *ctx, const char *filename) {
- char line[STASIS_NAME_MAX] = {0};
-
- FILE *fp = fopen(filename, "r");
- if (!fp) {
- return -1;
- }
-
- while (fgets(line, sizeof(line) - 1, fp) != NULL) {
- char **parts = split(line, " ", 1);
- char *name = parts[0];
- char *value = parts[1];
- strip(value);
- if (!strcmp(name, "name")) {
- ctx->meta.name = strdup(value);
- } else if (!strcmp(name, "version")) {
- ctx->meta.version = strdup(value);
- } else if (!strcmp(name, "rc")) {
- ctx->meta.rc = (int) strtol(value, NULL, 10);
- } else if (!strcmp(name, "python")) {
- ctx->meta.python = strdup(value);
- } else if (!strcmp(name, "python_compact")) {
- ctx->meta.python_compact = strdup(value);
- } else if (!strcmp(name, "mission")) {
- ctx->meta.mission = strdup(value);
- } else if (!strcmp(name, "codename")) {
- ctx->meta.codename = strdup(value);
- } else if (!strcmp(name, "platform")) {
- ctx->system.platform = split(value, " ", 0);
- } else if (!strcmp(name, "arch")) {
- ctx->system.arch = strdup(value);
- } else if (!strcmp(name, "time")) {
- ctx->info.time_str_epoch = strdup(value);
- } else if (!strcmp(name, "release_fmt")) {
- ctx->rules.release_fmt = strdup(value);
- } else if (!strcmp(name, "release_name")) {
- ctx->info.release_name = strdup(value);
- } else if (!strcmp(name, "build_name_fmt")) {
- ctx->rules.build_name_fmt = strdup(value);
- } else if (!strcmp(name, "build_name")) {
- ctx->info.build_name = strdup(value);
- } else if (!strcmp(name, "build_number_fmt")) {
- ctx->rules.build_number_fmt = strdup(value);
- } else if (!strcmp(name, "build_number")) {
- ctx->info.build_number = strdup(value);
- } else if (!strcmp(name, "conda_installer_baseurl")) {
- ctx->conda.installer_baseurl = strdup(value);
- } else if (!strcmp(name, "conda_installer_name")) {
- ctx->conda.installer_name = strdup(value);
- } else if (!strcmp(name, "conda_installer_version")) {
- ctx->conda.installer_version = strdup(value);
- } else if (!strcmp(name, "conda_installer_platform")) {
- ctx->conda.installer_platform = strdup(value);
- } else if (!strcmp(name, "conda_installer_arch")) {
- ctx->conda.installer_arch = strdup(value);
- }
- GENERIC_ARRAY_FREE(parts);
- }
- fclose(fp);
-
- return 0;
-}
-
-int indexer_get_files(struct StrList **out, const char *path, const char *pattern, ...) {
- va_list args;
- va_start(args, pattern);
- char userpattern[PATH_MAX] = {0};
- vsprintf(userpattern, pattern, args);
- va_end(args);
- struct StrList *list = listdir(path);
- if (!list) {
- return -1;
- }
-
- if (!(*out)) {
- (*out) = strlist_init();
- if (!(*out)) {
- guard_strlist_free(&list);
- return -1;
- }
- }
-
- size_t no_match = 0;
- for (size_t i = 0; i < strlist_count(list); i++) {
- char *item = strlist_item(list, i);
- if (fnmatch(userpattern, item, 0)) {
- no_match++;
- } else {
- strlist_append((out), item);
- }
- }
- if (no_match >= strlist_count(list)) {
- fprintf(stderr, "no files matching the pattern: %s\n", userpattern);
- guard_strlist_free(&list);
- return -1;
- }
- guard_strlist_free(&list);
- return 0;
-}
-
-int get_latest_rc(struct Delivery ctx[], size_t nelem) {
- int result = 0;
- for (size_t i = 0; i < nelem; i++) {
- if (ctx[i].meta.rc > result) {
- result = ctx[i].meta.rc;
- }
- }
- return result;
-}
-
-struct Delivery **get_latest_deliveries(struct Delivery ctx[], size_t nelem) {
- struct Delivery **result = NULL;
- int latest = 0;
- size_t n = 0;
-
- result = calloc(nelem + 1, sizeof(result));
- if (!result) {
- fprintf(stderr, "Unable to allocate %zu bytes for result delivery array: %s\n", nelem * sizeof(result), strerror(errno));
- return NULL;
- }
-
- latest = get_latest_rc(ctx, nelem);
- for (size_t i = 0; i < nelem; i++) {
- if (ctx[i].meta.rc == latest) {
- result[n] = &ctx[i];
- n++;
- }
- }
-
- return result;
-}
-
-int get_pandoc_version(size_t *result) {
- *result = 0;
- int state = 0;
- char *version_str = shell_output("pandoc --version", &state);
- if (state || !version_str) {
- // an error occurred
- return -1;
- }
-
- // Verify that we're looking at pandoc
- if (strlen(version_str) > 7 && !strncmp(version_str, "pandoc ", 7)) {
- // we have pandoc
- char *v_begin = &version_str[7];
- if (!v_begin) {
- SYSERROR("unexpected pandoc output: %s", version_str);
- return -1;
- }
- char *v_end = strchr(version_str, '\n');
- if (v_end) {
- *v_end = 0;
- }
-
- char **parts = split(v_begin, ".", 0);
- if (!parts) {
- SYSERROR("unable to split pandoc version string, '%s': %s", version_str, strerror(errno));
- return -1;
- }
-
- size_t parts_total;
- for (parts_total = 0; parts[parts_total] != NULL; parts_total++) {}
-
- // generate the version as an integer
- // note: pandoc version scheme never exceeds four elements (or bytes in this case)
- for (size_t i = 0; i < 4; i++) {
- unsigned char tmp = 0;
- if (i < parts_total) {
- // only process version elements we have. the rest will be zeros.
- tmp = strtoul(parts[i], NULL, 10);
- }
- // pack version element into result
- *result = (*result << 8) | tmp;
- }
- } else {
- // invalid version string
- return 1;
- }
-
- return 0;
-}
-
-int indexer_make_website(struct Delivery *ctx) {
- const char *pattern = "*.md";
-
- if (!find_program("pandoc")) {
- fprintf(stderr, "pandoc is not installed: unable to generate HTML indexes\n");
- return 0;
- }
-
- char *css_filename = calloc(PATH_MAX, sizeof(*css_filename));
- if (!css_filename) {
- SYSERROR("unable to allocate string for CSS file path: %s", strerror(errno));
- return -1;
- }
-
- sprintf(css_filename, "%s/%s", globals.sysconfdir, "stasis_pandoc.css");
- int have_css = access(css_filename, F_OK | R_OK) == 0;
-
- char pandoc_versioned_args[255] = {0};
- size_t pandoc_version = 0;
-
- if (!get_pandoc_version(&pandoc_version)) {
- // < 2.19
- if (pandoc_version < 0x02130000) {
- strcat(pandoc_versioned_args, "--self-contained ");
- } else {
- // >= 2.19
- strcat(pandoc_versioned_args, "--embed-resources ");
- }
-
- // >= 1.15.0.4
- if (pandoc_version >= 0x010f0004) {
- strcat(pandoc_versioned_args, "--standalone ");
- }
-
- // >= 1.10.0.1
- if (pandoc_version >= 0x010a0001) {
- strcat(pandoc_versioned_args, "-f gfm+autolink_bare_uris ");
- }
-
- // > 3.1.9
- if (pandoc_version > 0x03010900) {
- strcat(pandoc_versioned_args, "-f gfm+alerts ");
- }
- }
-
- struct StrList *dirs = strlist_init();
- strlist_append(&dirs, ctx->storage.delivery_dir);
- strlist_append(&dirs, ctx->storage.results_dir);
-
- struct StrList *inputs = NULL;
- for (size_t i = 0; i < strlist_count(dirs); i++) {
- if (indexer_get_files(&inputs, ctx->storage.delivery_dir, pattern)) {
- SYSERROR("%s does not contain files with pattern: %s", ctx->storage.delivery_dir, pattern);
- guard_strlist_free(&inputs);
- continue;
- }
- char *root = strlist_item(dirs, i);
- for (size_t x = 0; x < strlist_count(inputs); x++) {
- char cmd[PATH_MAX] = {0};
- char *filename = strlist_item(inputs, x);
- char fullpath_src[PATH_MAX] = {0};
- char fullpath_dest[PATH_MAX] = {0};
- sprintf(fullpath_src, "%s/%s", root, filename);
- if (access(fullpath_src, F_OK)) {
- continue;
- }
-
- // Replace *.md extension with *.html.
- strcpy(fullpath_dest, fullpath_src);
- char *ext = strrchr(fullpath_dest, '.');
- if (ext) {
- *ext = '\0';
- }
- strcat(fullpath_dest, ".html");
-
- // Converts a markdown file to html
- strcpy(cmd, "pandoc ");
- strcat(cmd, pandoc_versioned_args);
- if (have_css) {
- strcat(cmd, "--css ");
- strcat(cmd, css_filename);
- }
- strcat(cmd, " ");
- strcat(cmd, "--metadata title=\"STASIS\" ");
- strcat(cmd, "-o ");
- strcat(cmd, fullpath_dest);
- strcat(cmd, " ");
- strcat(cmd, fullpath_src);
- if (globals.verbose) {
- puts(cmd);
- }
- // This might be negative when killed by a signal.
- // Otherwise, the return code is not critical to us.
- if (system(cmd) < 0) {
- guard_free(css_filename);
- guard_strlist_free(&dirs);
- return 1;
- }
- if (file_replace_text(fullpath_dest, ".md", ".html", 0)) {
- // inform-only
- SYSERROR("%s: failed to rewrite *.md urls with *.html extension", fullpath_dest);
- }
-
- // Link the nearest README.html to index.html
- if (!strcmp(filename, "README.md")) {
- char link_from[PATH_MAX] = {0};
- char link_dest[PATH_MAX] = {0};
- strcpy(link_from, "README.html");
- sprintf(link_dest, "%s/%s", root, "index.html");
- if (symlink(link_from, link_dest)) {
- SYSERROR("Warning: symlink(%s, %s) failed: %s", link_from, link_dest, strerror(errno));
- }
- }
- }
- guard_strlist_free(&inputs);
- }
- guard_free(css_filename);
- guard_strlist_free(&dirs);
-
- return 0;
-}
-
-static int micromamba_configure(const struct Delivery *ctx, struct MicromambaInfo *m) {
- int status = 0;
- char *micromamba_prefix = NULL;
- if (asprintf(&micromamba_prefix, "%s/bin", ctx->storage.tools_dir) < 0) {
- return -1;
- }
- m->conda_prefix = globals.conda_install_prefix;
- m->micromamba_prefix = micromamba_prefix;
-
- size_t pathvar_len = (strlen(getenv("PATH")) + strlen(m->micromamba_prefix) + strlen(m->conda_prefix)) + 3 + 4 + 1;
- // ^^^^^^^^^^^^^^^^^^
- // 3 = separators
- // 4 = chars (/bin)
- // 1 = nul terminator
- char *pathvar = calloc(pathvar_len, sizeof(*pathvar));
- if (!pathvar) {
- SYSERROR("%s", "Unable to allocate bytes for temporary path string");
- exit(1);
- }
- snprintf(pathvar, pathvar_len, "%s/bin:%s:%s", m->conda_prefix, m->micromamba_prefix, getenv("PATH"));
- setenv("PATH", pathvar, 1);
- guard_free(pathvar);
-
- status += micromamba(m, "config prepend --env channels conda-forge");
- if (!globals.verbose) {
- status += micromamba(m, "config set --env quiet true");
- }
- status += micromamba(m, "config set --env always_yes true");
- status += micromamba(m, "install conda-build pandoc");
-
- return status;
-}
-
-int indexer_conda(struct Delivery *ctx, struct MicromambaInfo m) {
- int status = 0;
-
- status += micromamba(&m, "run conda index %s", ctx->storage.conda_artifact_dir);
- return status;
-}
-
-static struct StrList *get_architectures(struct Delivery ctx[], size_t nelem) {
- struct StrList *architectures = strlist_init();
- for (size_t i = 0; i < nelem; i++) {
- if (!strstr_array(architectures->data, ctx[i].system.arch)) {
- strlist_append(&architectures, ctx[i].system.arch);
- }
- }
- return architectures;
-}
-
-static struct StrList *get_platforms(struct Delivery ctx[], size_t nelem) {
- struct StrList *platforms = strlist_init();
- for (size_t i = 0; i < nelem; i++) {
- if (!strstr_array(platforms->data, ctx[i].system.platform[DELIVERY_PLATFORM_RELEASE])) {
- strlist_append(&platforms, ctx[i].system.platform[DELIVERY_PLATFORM_RELEASE]);
- }
- }
- return platforms;
-}
-
-int indexer_symlinks(struct Delivery ctx[], size_t nelem) {
- struct Delivery **data = NULL;
- data = get_latest_deliveries(ctx, nelem);
- //int latest = get_latest_rc(ctx, nelem);
-
- if (!pushd(ctx->storage.delivery_dir)) {
- for (size_t i = 0; i < nelem; i++) {
- char link_name_spec[PATH_MAX];
- char link_name_readme[PATH_MAX];
-
- char file_name_spec[PATH_MAX];
- char file_name_readme[PATH_MAX];
-
- if (!data[i]) {
- continue;
- }
- sprintf(link_name_spec, "latest-py%s-%s-%s.yml", data[i]->meta.python_compact, data[i]->system.platform[DELIVERY_PLATFORM_RELEASE], data[i]->system.arch);
- sprintf(file_name_spec, "%s.yml", data[i]->info.release_name);
-
- sprintf(link_name_readme, "README-py%s-%s-%s.md", data[i]->meta.python_compact, data[i]->system.platform[DELIVERY_PLATFORM_RELEASE], data[i]->system.arch);
- sprintf(file_name_readme, "README-%s.md", data[i]->info.release_name);
-
- if (!access(link_name_spec, F_OK)) {
- if (unlink(link_name_spec)) {
- fprintf(stderr, "Unable to remove spec link: %s\n", link_name_spec);
- }
- }
- if (!access(link_name_readme, F_OK)) {
- if (unlink(link_name_readme)) {
- fprintf(stderr, "Unable to remove readme link: %s\n", link_name_readme);
- }
- }
-
- if (globals.verbose) {
- printf("%s -> %s\n", file_name_spec, link_name_spec);
- }
- if (symlink(file_name_spec, link_name_spec)) {
- fprintf(stderr, "Unable to link %s as %s\n", file_name_spec, link_name_spec);
- }
-
- if (globals.verbose) {
- printf("%s -> %s\n", file_name_readme, link_name_readme);
- }
- if (symlink(file_name_readme, link_name_readme)) {
- fprintf(stderr, "Unable to link %s as %s\n", file_name_readme, link_name_readme);
- }
- }
- popd();
- } else {
- fprintf(stderr, "Unable to enter delivery directory: %s\n", ctx->storage.delivery_dir);
- guard_free(data);
- return -1;
- }
-
- // "latest" is an array of pointers to ctx[]. Do not free the contents of the array.
- guard_free(data);
- return 0;
-}
-
-int indexer_readmes(struct Delivery ctx[], size_t nelem) {
- struct Delivery **latest = NULL;
- latest = get_latest_deliveries(ctx, nelem);
-
- char indexfile[PATH_MAX] = {0};
- sprintf(indexfile, "%s/README.md", ctx->storage.delivery_dir);
-
- if (!pushd(ctx->storage.delivery_dir)) {
- FILE *indexfp = fopen(indexfile, "w+");
- if (!indexfp) {
- fprintf(stderr, "Unable to open %s for writing\n", indexfile);
- return -1;
- }
- struct StrList *archs = get_architectures(*latest, nelem);
- struct StrList *platforms = get_platforms(*latest, nelem);
-
- fprintf(indexfp, "# %s-%s\n\n", ctx->meta.name, ctx->meta.version);
- fprintf(indexfp, "## Current Release\n\n");
- for (size_t p = 0; p < strlist_count(platforms); p++) {
- char *platform = strlist_item(platforms, p);
- for (size_t a = 0; a < strlist_count(archs); a++) {
- char *arch = strlist_item(archs, a);
- int have_combo = 0;
- for (size_t i = 0; i < nelem; i++) {
- if (latest[i] && latest[i]->system.platform) {
- if (strstr(latest[i]->system.platform[DELIVERY_PLATFORM_RELEASE], platform) &&
- strstr(latest[i]->system.arch, arch)) {
- have_combo = 1;
- }
- }
- }
- if (!have_combo) {
- continue;
- }
- fprintf(indexfp, "### %s-%s\n\n", platform, arch);
-
- fprintf(indexfp, "|Release|Info|Receipt|\n");
- fprintf(indexfp, "|:----:|:----:|:----:|\n");
- for (size_t i = 0; i < nelem; i++) {
- char link_name[PATH_MAX];
- char readme_name[PATH_MAX];
- char conf_name[PATH_MAX];
- char conf_name_relative[PATH_MAX];
- if (!latest[i]) {
- continue;
- }
- sprintf(link_name, "latest-py%s-%s-%s.yml", latest[i]->meta.python_compact, latest[i]->system.platform[DELIVERY_PLATFORM_RELEASE], latest[i]->system.arch);
- sprintf(readme_name, "README-py%s-%s-%s.md", latest[i]->meta.python_compact, latest[i]->system.platform[DELIVERY_PLATFORM_RELEASE], latest[i]->system.arch);
- sprintf(conf_name, "%s.ini", latest[i]->info.release_name);
- sprintf(conf_name_relative, "../config/%s-rendered.ini", latest[i]->info.release_name);
- if (strstr(link_name, platform) && strstr(link_name, arch)) {
- fprintf(indexfp, "|[%s](%s)|[%s](%s)|[%s](%s)|\n", link_name, link_name, readme_name, readme_name, conf_name, conf_name_relative);
- }
- }
- fprintf(indexfp, "\n");
- }
- fprintf(indexfp, "\n");
- }
- guard_strlist_free(&archs);
- guard_strlist_free(&platforms);
- fclose(indexfp);
- popd();
- } else {
- fprintf(stderr, "Unable to enter delivery directory: %s\n", ctx->storage.delivery_dir);
- guard_free(latest);
- return -1;
- }
-
- // "latest" is an array of pointers to ctxs[]. Do not free the contents of the array.
- guard_free(latest);
- return 0;
-}
-
-int indexer_junitxml_report(struct Delivery ctx[], size_t nelem) {
- struct Delivery **latest = NULL;
- latest = get_latest_deliveries(ctx, nelem);
-
- char indexfile[PATH_MAX] = {0};
- sprintf(indexfile, "%s/README.md", ctx->storage.results_dir);
-
- struct StrList *file_listing = listdir(ctx->storage.results_dir);
- if (!file_listing) {
- // no test results to process
- return 0;
- }
-
- if (!pushd(ctx->storage.results_dir)) {
- FILE *indexfp = fopen(indexfile, "w+");
- if (!indexfp) {
- fprintf(stderr, "Unable to open %s for writing\n", indexfile);
- return -1;
- }
- struct StrList *archs = get_architectures(*latest, nelem);
- struct StrList *platforms = get_platforms(*latest, nelem);
-
- fprintf(indexfp, "# %s-%s Test Report\n\n", ctx->meta.name, ctx->meta.version);
- fprintf(indexfp, "## Current Release\n\n");
- for (size_t p = 0; p < strlist_count(platforms); p++) {
- char *platform = strlist_item(platforms, p);
- for (size_t a = 0; a < strlist_count(archs); a++) {
- char *arch = strlist_item(archs, a);
- int have_combo = 0;
- for (size_t i = 0; i < nelem; i++) {
- if (latest[i] && latest[i]->system.platform) {
- if (strstr(latest[i]->system.platform[DELIVERY_PLATFORM_RELEASE], platform) &&
- strstr(latest[i]->system.arch, arch)) {
- have_combo = 1;
- break;
- }
- }
- }
- if (!have_combo) {
- continue;
- }
- fprintf(indexfp, "### %s-%s\n\n", platform, arch);
-
- fprintf(indexfp, "|Suite|Duration|Fail |Skip |Error |\n");
- fprintf(indexfp, "|:----|:------:|:------:|:---:|:----:|\n");
- for (size_t f = 0; f < strlist_count(file_listing); f++) {
- char *filename = strlist_item(file_listing, f);
- if (!endswith(filename, ".xml")) {
- continue;
- }
-
- if (strstr(filename, platform) && strstr(filename, arch)) {
- struct JUNIT_Testsuite *testsuite = junitxml_testsuite_read(filename);
- if (testsuite) {
- if (globals.verbose) {
- printf("%s: duration: %0.4f, failed: %d, skipped: %d, errors: %d\n", filename, testsuite->time, testsuite->failures, testsuite->skipped, testsuite->errors);
- }
- fprintf(indexfp, "|[%s](%s)|%0.4f|%d|%d|%d|\n", filename, filename, testsuite->time, testsuite->failures, testsuite->skipped, testsuite->errors);
- /*
- * TODO: Display failure/skip/error output.
- *
- for (size_t i = 0; i < testsuite->_tc_inuse; i++) {
- if (testsuite->testcase[i]->tc_result_state_type) {
- printf("testcase: %s :: %s\n", testsuite->testcase[i]->classname, testsuite->testcase[i]->name);
- if (testsuite->testcase[i]->tc_result_state_type == JUNIT_RESULT_STATE_FAILURE) {
- printf("failure: %s\n", testsuite->testcase[i]->result_state.failure->message);
- } else if (testsuite->testcase[i]->tc_result_state_type == JUNIT_RESULT_STATE_SKIPPED) {
- printf("skipped: %s\n", testsuite->testcase[i]->result_state.skipped->message);
- }
- }
- }
- */
- junitxml_testsuite_free(&testsuite);
- } else {
- fprintf(stderr, "bad test suite: %s: %s\n", strerror(errno), filename);
- continue;
- }
- }
- }
- fprintf(indexfp, "\n");
- }
- fprintf(indexfp, "\n");
- }
- guard_strlist_free(&archs);
- guard_strlist_free(&platforms);
- fclose(indexfp);
- popd();
- } else {
- fprintf(stderr, "Unable to enter delivery directory: %s\n", ctx->storage.delivery_dir);
- guard_free(latest);
- return -1;
- }
-
- // "latest" is an array of pointers to ctxs[]. Do not free the contents of the array.
- guard_free(latest);
- return 0;
-}
-
-void indexer_init_dirs(struct Delivery *ctx, const char *workdir) {
- path_store(&ctx->storage.root, PATH_MAX, workdir, "");
- path_store(&ctx->storage.tmpdir, PATH_MAX, ctx->storage.root, "tmp");
- if (delivery_init_tmpdir(ctx)) {
- fprintf(stderr, "Failed to configure temporary storage directory\n");
- exit(1);
- }
- path_store(&ctx->storage.output_dir, PATH_MAX, ctx->storage.root, "");
- path_store(&ctx->storage.tools_dir, PATH_MAX, ctx->storage.output_dir, "tools");
- path_store(&globals.conda_install_prefix, PATH_MAX, ctx->storage.tools_dir, "conda");
- path_store(&ctx->storage.cfgdump_dir, PATH_MAX, ctx->storage.output_dir, "config");
- path_store(&ctx->storage.meta_dir, PATH_MAX, ctx->storage.output_dir, "meta");
- path_store(&ctx->storage.delivery_dir, PATH_MAX, ctx->storage.output_dir, "delivery");
- path_store(&ctx->storage.package_dir, PATH_MAX, ctx->storage.output_dir, "packages");
- path_store(&ctx->storage.results_dir, PATH_MAX, ctx->storage.output_dir, "results");
- path_store(&ctx->storage.wheel_artifact_dir, PATH_MAX, ctx->storage.package_dir, "wheels");
- path_store(&ctx->storage.conda_artifact_dir, PATH_MAX, ctx->storage.package_dir, "conda");
-
- char newpath[PATH_MAX] = {0};
- if (getenv("PATH")) {
- sprintf(newpath, "%s/bin:%s", ctx->storage.tools_dir, getenv("PATH"));
- setenv("PATH", newpath, 1);
- } else {
- SYSERROR("%s", "environment variable PATH is undefined. Unable to continue.");
- exit(1);
- }
-}
-
-int main(int argc, char *argv[]) {
- size_t rootdirs_total = 0;
- char *destdir = NULL;
- char **rootdirs = NULL;
- int do_html = 0;
- int c = 0;
- int option_index = 0;
- while ((c = getopt_long(argc, argv, "hd:vUw", long_options, &option_index)) != -1) {
- switch (c) {
- case 'h':
- usage(path_basename(argv[0]));
- exit(0);
- case 'd':
- if (mkdir(optarg, 0755)) {
- if (errno != 0 && errno != EEXIST) {
- SYSERROR("Unable to create destination directory, '%s': %s", optarg, strerror(errno));
- exit(1);
- }
- }
- destdir = realpath(optarg, NULL);
- break;
- case 'U':
- fflush(stdout);
- fflush(stderr);
- setvbuf(stdout, NULL, _IONBF, 0);
- setvbuf(stderr, NULL, _IONBF, 0);
- break;
- case 'v':
- globals.verbose = 1;
- break;
- case 'w':
- do_html = 1;
- break;
- case '?':
- default:
- exit(1);
- }
- }
-
- int current_index = optind;
- if (optind < argc) {
- rootdirs_total = argc - current_index;
- rootdirs = calloc(rootdirs_total + 1, sizeof(**rootdirs));
-
- int i = 0;
- while (optind < argc) {
- if (argv[optind]) {
- if (access(argv[optind], F_OK) < 0) {
- fprintf(stderr, "%s: %s\n", argv[optind], strerror(errno));
- exit(1);
- }
- }
- // use first positional argument
- rootdirs[i] = realpath(argv[optind], NULL);
- optind++;
- break;
- }
- }
-
- if (isempty(destdir)) {
- if (mkdir("output", 0755)) {
- if (errno != 0 && errno != EEXIST) {
- SYSERROR("Unable to create destination directory, '%s': %s", "output", strerror(errno));
- exit(1);
- }
- }
- destdir = realpath("output", NULL);
- }
-
- if (!rootdirs || !rootdirs_total) {
- fprintf(stderr, "You must specify at least one STASIS root directory to index\n");
- exit(1);
- } else {
- for (size_t i = 0; i < rootdirs_total; i++) {
- if (isempty(rootdirs[i]) || !strcmp(rootdirs[i], "/") || !strcmp(rootdirs[i], "\\")) {
- SYSERROR("Unsafe directory: %s", rootdirs[i]);
- exit(1);
- } else if (access(rootdirs[i], F_OK)) {
- SYSERROR("%s: %s", rootdirs[i], strerror(errno));
- exit(1);
- }
- }
- }
-
- char stasis_sysconfdir_tmp[PATH_MAX];
- if (getenv("STASIS_SYSCONFDIR")) {
- strncpy(stasis_sysconfdir_tmp, getenv("STASIS_SYSCONFDIR"), sizeof(stasis_sysconfdir_tmp) - 1);
- } else {
- strncpy(stasis_sysconfdir_tmp, STASIS_SYSCONFDIR, sizeof(stasis_sysconfdir_tmp) - 1);
- }
-
- globals.sysconfdir = realpath(stasis_sysconfdir_tmp, NULL);
- if (!globals.sysconfdir) {
- msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Unable to resolve path to configuration directory: %s\n", stasis_sysconfdir_tmp);
- exit(1);
- }
-
- char workdir_template[PATH_MAX] = {0};
- char *system_tmp = getenv("TMPDIR");
- if (system_tmp) {
- strcat(workdir_template, system_tmp);
- } else {
- strcat(workdir_template, "/tmp");
- }
- strcat(workdir_template, "/stasis-combine.XXXXXX");
- char *workdir = mkdtemp(workdir_template);
- if (!workdir) {
- SYSERROR("Unable to create temporary directory: %s", workdir_template);
- exit(1);
- } else if (isempty(workdir) || !strcmp(workdir, "/") || !strcmp(workdir, "\\")) {
- SYSERROR("Unsafe directory: %s", workdir);
- exit(1);
- }
-
- struct Delivery ctx = {0};
-
- printf(BANNER, VERSION, AUTHOR);
-
- indexer_init_dirs(&ctx, workdir);
-
- msg(STASIS_MSG_L1, "%s delivery root %s\n",
- rootdirs_total > 1 ? "Merging" : "Indexing",
- rootdirs_total > 1 ? "directories" : "directory");
- if (indexer_combine_rootdirs(workdir, rootdirs, rootdirs_total)) {
- SYSERROR("%s", "Copy operation failed");
- rmtree(workdir);
- exit(1);
- }
-
- if (access(ctx.storage.conda_artifact_dir, F_OK)) {
- mkdirs(ctx.storage.conda_artifact_dir, 0755);
- }
-
- if (access(ctx.storage.wheel_artifact_dir, F_OK)) {
- mkdirs(ctx.storage.wheel_artifact_dir, 0755);
- }
-
- struct MicromambaInfo m;
- if (micromamba_configure(&ctx, &m)) {
- SYSERROR("%s", "Unable to configure micromamba");
- exit(1);
- }
-
- msg(STASIS_MSG_L1, "Indexing conda packages\n");
- if (indexer_conda(&ctx, m)) {
- SYSERROR("%s", "Conda package indexing operation failed");
- exit(1);
- }
-
- msg(STASIS_MSG_L1, "Indexing wheel packages\n");
- if (indexer_wheels(&ctx)) {
- SYSERROR("%s", "Python package indexing operation failed");
- exit(1);
- }
-
- msg(STASIS_MSG_L1, "Loading metadata\n");
- struct StrList *metafiles = NULL;
- indexer_get_files(&metafiles, ctx.storage.meta_dir, "*.stasis");
- strlist_sort(metafiles, STASIS_SORT_LEN_ASCENDING);
- struct Delivery local[strlist_count(metafiles)];
-
- for (size_t i = 0; i < strlist_count(metafiles); i++) {
- char *item = strlist_item(metafiles, i);
- memset(&local[i], 0, sizeof(ctx));
- memcpy(&local[i], &ctx, sizeof(ctx));
- char path[PATH_MAX];
- sprintf(path, "%s/%s", ctx.storage.meta_dir, item);
- if (globals.verbose) {
- puts(path);
- }
- indexer_load_metadata(&local[i], path);
- }
-
- msg(STASIS_MSG_L1, "Generating links to latest release iteration\n");
- if (indexer_symlinks(local, strlist_count(metafiles))) {
- SYSERROR("%s", "Link generation failed");
- exit(1);
- }
-
- msg(STASIS_MSG_L1, "Generating README.md\n");
- if (indexer_readmes(local, strlist_count(metafiles))) {
- SYSERROR("%s", "README indexing operation failed");
- exit(1);
- }
-
- msg(STASIS_MSG_L1, "Indexing test results\n");
- if (indexer_junitxml_report(local, strlist_count(metafiles))) {
- SYSERROR("%s", "Test result indexing operation failed");
- exit(1);
- }
-
- if (do_html) {
- msg(STASIS_MSG_L1, "Generating HTML indexes\n");
- if (indexer_make_website(local)) {
- SYSERROR("%s", "Site creation failed");
- exit(1);
- }
- }
-
- msg(STASIS_MSG_L1, "Copying indexed delivery to '%s'\n", destdir);
- char cmd[PATH_MAX] = {0};
- sprintf(cmd, "rsync -ah%s --delete --exclude 'tmp/' --exclude 'tools/' '%s/' '%s/'", globals.verbose ? "v" : "q", workdir, destdir);
- guard_free(destdir);
-
- if (globals.verbose) {
- puts(cmd);
- }
-
- if (system(cmd)) {
- SYSERROR("%s", "Copy operation failed");
- rmtree(workdir);
- exit(1);
- }
-
- msg(STASIS_MSG_L1, "Removing work directory: %s\n", workdir);
- if (rmtree(workdir)) {
- SYSERROR("Failed to remove work directory: %s", strerror(errno));
- }
-
- guard_free(destdir);
- GENERIC_ARRAY_FREE(rootdirs);
- guard_strlist_free(&metafiles);
- delivery_free(&ctx);
- globals_free();
- msg(STASIS_MSG_L1, "Done!\n");
- return 0;
-}
diff --git a/src/cli/stasis_indexer/stasis_indexer_main.c b/src/cli/stasis_indexer/stasis_indexer_main.c
new file mode 100644
index 0000000..ef39394
--- /dev/null
+++ b/src/cli/stasis_indexer/stasis_indexer_main.c
@@ -0,0 +1,398 @@
+#include <getopt.h>
+#include "args.h"
+#include "callbacks.h"
+#include "helpers.h"
+#include "junitxml_report.h"
+#include "website.h"
+#include "readmes.h"
+#include "delivery.h"
+
+int indexer_combine_rootdirs(const char *dest, char **rootdirs, const size_t rootdirs_total) {
+ char cmd[PATH_MAX];
+ char destdir_bare[PATH_MAX];
+ char destdir_with_output[PATH_MAX];
+ char *destdir = destdir_bare;
+
+ memset(cmd, 0, sizeof(cmd));
+ memset(destdir_bare, 0, sizeof(destdir_bare));
+ memset(destdir_with_output, 0, sizeof(destdir_bare));
+
+ strcpy(destdir_bare, dest);
+ strcpy(destdir_with_output, dest);
+ strcat(destdir_with_output, "/output");
+
+ if (!access(destdir_with_output, F_OK)) {
+ destdir = destdir_with_output;
+ }
+
+ sprintf(cmd, "rsync -ah%s --delete --exclude 'tools/' --exclude 'tmp/' --exclude 'build/' ", globals.verbose ? "v" : "q");
+ for (size_t i = 0; i < rootdirs_total; i++) {
+ char srcdir_bare[PATH_MAX] = {0};
+ char srcdir_with_output[PATH_MAX] = {0};
+ char *srcdir = srcdir_bare;
+ strcpy(srcdir_bare, rootdirs[i]);
+ strcpy(srcdir_with_output, rootdirs[i]);
+ strcat(srcdir_with_output, "/output");
+
+ if (access(srcdir_bare, F_OK)) {
+ fprintf(stderr, "%s does not exist\n", srcdir_bare);
+ continue;
+ }
+
+ if (!access(srcdir_with_output, F_OK)) {
+ srcdir = srcdir_with_output;
+ }
+ snprintf(cmd + strlen(cmd), sizeof(srcdir) - strlen(srcdir) + 4, "'%s'/ ", srcdir);
+ }
+ snprintf(cmd + strlen(cmd), sizeof(cmd) - strlen(destdir) + 1, " %s/", destdir);
+
+ if (globals.verbose) {
+ puts(cmd);
+ }
+
+ if (system(cmd)) {
+ return -1;
+ }
+ return 0;
+}
+
+int indexer_wheels(struct Delivery *ctx) {
+ return delivery_index_wheel_artifacts(ctx);
+}
+
+int indexer_conda(const struct Delivery *ctx, struct MicromambaInfo m) {
+ int status = 0;
+
+ status += micromamba(&m, "run conda index %s", ctx->storage.conda_artifact_dir);
+ return status;
+}
+
+int indexer_symlinks(struct Delivery *ctx, const size_t nelem) {
+ struct Delivery *data = NULL;
+ data = get_latest_deliveries(ctx, nelem);
+ //int latest = get_latest_rc(ctx, nelem);
+
+ if (!pushd(ctx->storage.delivery_dir)) {
+ for (size_t i = 0; i < nelem; i++) {
+ char link_name_spec[PATH_MAX];
+ char link_name_readme[PATH_MAX];
+
+ char file_name_spec[PATH_MAX];
+ char file_name_readme[PATH_MAX];
+
+ if (!data[i].meta.name) {
+ continue;
+ }
+ sprintf(link_name_spec, "latest-py%s-%s-%s.yml", data[i].meta.python_compact, data[i].system.platform[DELIVERY_PLATFORM_RELEASE], data[i].system.arch);
+ sprintf(file_name_spec, "%s.yml", data[i].info.release_name);
+
+ sprintf(link_name_readme, "README-py%s-%s-%s.md", data[i].meta.python_compact, data[i].system.platform[DELIVERY_PLATFORM_RELEASE], data[i].system.arch);
+ sprintf(file_name_readme, "README-%s.md", data[i].info.release_name);
+
+ if (!access(link_name_spec, F_OK)) {
+ if (unlink(link_name_spec)) {
+ fprintf(stderr, "Unable to remove spec link: %s\n", link_name_spec);
+ }
+ }
+ if (!access(link_name_readme, F_OK)) {
+ if (unlink(link_name_readme)) {
+ fprintf(stderr, "Unable to remove readme link: %s\n", link_name_readme);
+ }
+ }
+
+ if (globals.verbose) {
+ printf("%s -> %s\n", file_name_spec, link_name_spec);
+ }
+ if (symlink(file_name_spec, link_name_spec)) {
+ fprintf(stderr, "Unable to link %s as %s\n", file_name_spec, link_name_spec);
+ }
+
+ if (globals.verbose) {
+ printf("%s -> %s\n", file_name_readme, link_name_readme);
+ }
+ if (symlink(file_name_readme, link_name_readme)) {
+ fprintf(stderr, "Unable to link %s as %s\n", file_name_readme, link_name_readme);
+ }
+ }
+ popd();
+ } else {
+ fprintf(stderr, "Unable to enter delivery directory: %s\n", ctx->storage.delivery_dir);
+ guard_free(data);
+ return -1;
+ }
+
+ // "latest" is an array of pointers to ctx[]. Do not free the contents of the array.
+ guard_free(data);
+ return 0;
+}
+
+void indexer_init_dirs(struct Delivery *ctx, const char *workdir) {
+ path_store(&ctx->storage.root, PATH_MAX, workdir, "");
+ path_store(&ctx->storage.tmpdir, PATH_MAX, ctx->storage.root, "tmp");
+ if (delivery_init_tmpdir(ctx)) {
+ fprintf(stderr, "Failed to configure temporary storage directory\n");
+ exit(1);
+ }
+
+ char *user_dir = expandpath("~/.stasis/indexer");
+ if (!user_dir) {
+ SYSERROR("%s", "expandpath failed");
+ }
+
+ path_store(&ctx->storage.output_dir, PATH_MAX, ctx->storage.root, "");
+ path_store(&ctx->storage.tools_dir, PATH_MAX, user_dir, "tools");
+ path_store(&globals.conda_install_prefix, PATH_MAX, user_dir, "conda");
+ path_store(&ctx->storage.cfgdump_dir, PATH_MAX, ctx->storage.output_dir, "config");
+ path_store(&ctx->storage.meta_dir, PATH_MAX, ctx->storage.output_dir, "meta");
+ path_store(&ctx->storage.delivery_dir, PATH_MAX, ctx->storage.output_dir, "delivery");
+ path_store(&ctx->storage.package_dir, PATH_MAX, ctx->storage.output_dir, "packages");
+ path_store(&ctx->storage.results_dir, PATH_MAX, ctx->storage.output_dir, "results");
+ path_store(&ctx->storage.wheel_artifact_dir, PATH_MAX, ctx->storage.package_dir, "wheels");
+ path_store(&ctx->storage.conda_artifact_dir, PATH_MAX, ctx->storage.package_dir, "conda");
+ path_store(&ctx->storage.docker_artifact_dir, PATH_MAX, ctx->storage.package_dir, "docker");
+ guard_free(user_dir);
+
+ char newpath[PATH_MAX] = {0};
+ if (getenv("PATH")) {
+ sprintf(newpath, "%s/bin:%s", ctx->storage.tools_dir, getenv("PATH"));
+ setenv("PATH", newpath, 1);
+ } else {
+ SYSERROR("%s", "environment variable PATH is undefined. Unable to continue.");
+ exit(1);
+ }
+}
+
+int main(const int argc, char *argv[]) {
+ size_t rootdirs_total = 0;
+ char *destdir = NULL;
+ char **rootdirs = NULL;
+ int do_html = 0;
+ int c = 0;
+ int option_index = 0;
+ while ((c = getopt_long(argc, argv, "hd:vUw", long_options, &option_index)) != -1) {
+ switch (c) {
+ case 'h':
+ usage(path_basename(argv[0]));
+ exit(0);
+ case 'd':
+ if (mkdir(optarg, 0755)) {
+ if (errno != 0 && errno != EEXIST) {
+ SYSERROR("Unable to create destination directory, '%s': %s", optarg, strerror(errno));
+ exit(1);
+ }
+ }
+ destdir = realpath(optarg, NULL);
+ break;
+ case 'U':
+ fflush(stdout);
+ fflush(stderr);
+ setvbuf(stdout, NULL, _IONBF, 0);
+ setvbuf(stderr, NULL, _IONBF, 0);
+ break;
+ case 'v':
+ globals.verbose = 1;
+ break;
+ case 'w':
+ do_html = 1;
+ break;
+ case '?':
+ default:
+ exit(1);
+ }
+ }
+
+ const int current_index = optind;
+ if (optind < argc) {
+ rootdirs_total = argc - current_index;
+ rootdirs = calloc(rootdirs_total + 1, sizeof(**rootdirs));
+
+ int i = 0;
+ while (optind < argc) {
+ if (argv[optind]) {
+ if (access(argv[optind], F_OK) < 0) {
+ fprintf(stderr, "%s: %s\n", argv[optind], strerror(errno));
+ exit(1);
+ }
+ }
+ // use first positional argument
+ rootdirs[i] = realpath(argv[optind], NULL);
+ optind++;
+ break;
+ }
+ }
+
+ if (isempty(destdir)) {
+ if (mkdir("output", 0755)) {
+ if (errno != 0 && errno != EEXIST) {
+ SYSERROR("Unable to create destination directory, '%s': %s", "output", strerror(errno));
+ exit(1);
+ }
+ }
+ destdir = realpath("output", NULL);
+ }
+
+ if (!rootdirs || !rootdirs_total) {
+ fprintf(stderr, "You must specify at least one STASIS root directory to index\n");
+ exit(1);
+ }
+
+ for (size_t i = 0; i < rootdirs_total; i++) {
+ if (isempty(rootdirs[i]) || !strcmp(rootdirs[i], "/") || !strcmp(rootdirs[i], "\\")) {
+ SYSERROR("Unsafe directory: %s", rootdirs[i]);
+ exit(1);
+ }
+
+ if (access(rootdirs[i], F_OK)) {
+ SYSERROR("%s: %s", rootdirs[i], strerror(errno));
+ exit(1);
+ }
+ }
+
+ char stasis_sysconfdir_tmp[PATH_MAX];
+ if (getenv("STASIS_SYSCONFDIR")) {
+ strncpy(stasis_sysconfdir_tmp, getenv("STASIS_SYSCONFDIR"), sizeof(stasis_sysconfdir_tmp) - 1);
+ } else {
+ strncpy(stasis_sysconfdir_tmp, STASIS_SYSCONFDIR, sizeof(stasis_sysconfdir_tmp) - 1);
+ }
+
+ globals.sysconfdir = realpath(stasis_sysconfdir_tmp, NULL);
+ if (!globals.sysconfdir) {
+ msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Unable to resolve path to configuration directory: %s\n", stasis_sysconfdir_tmp);
+ exit(1);
+ }
+
+ char workdir_template[PATH_MAX] = {0};
+ const char *system_tmp = getenv("TMPDIR");
+ if (system_tmp) {
+ strcat(workdir_template, system_tmp);
+ } else {
+ strcat(workdir_template, "/tmp");
+ }
+ strcat(workdir_template, "/stasis-combine.XXXXXX");
+ char *workdir = mkdtemp(workdir_template);
+ if (!workdir) {
+ SYSERROR("Unable to create temporary directory: %s", workdir_template);
+ exit(1);
+ }
+ if (isempty(workdir) || !strcmp(workdir, "/") || !strcmp(workdir, "\\")) {
+ SYSERROR("Unsafe directory: %s", workdir);
+ exit(1);
+ }
+
+ struct Delivery ctx = {0};
+
+ printf(BANNER, VERSION, AUTHOR);
+
+ indexer_init_dirs(&ctx, workdir);
+
+ msg(STASIS_MSG_L1, "%s delivery root %s\n",
+ rootdirs_total > 1 ? "Merging" : "Indexing",
+ rootdirs_total > 1 ? "directories" : "directory");
+ if (indexer_combine_rootdirs(workdir, rootdirs, rootdirs_total)) {
+ SYSERROR("%s", "Copy operation failed");
+ rmtree(workdir);
+ exit(1);
+ }
+
+ if (access(ctx.storage.conda_artifact_dir, F_OK)) {
+ mkdirs(ctx.storage.conda_artifact_dir, 0755);
+ }
+
+ if (access(ctx.storage.wheel_artifact_dir, F_OK)) {
+ mkdirs(ctx.storage.wheel_artifact_dir, 0755);
+ }
+
+ struct MicromambaInfo m;
+ if (micromamba_configure(&ctx, &m)) {
+ SYSERROR("%s", "Unable to configure micromamba");
+ exit(1);
+ }
+
+ msg(STASIS_MSG_L1, "Indexing conda packages\n");
+ if (indexer_conda(&ctx, m)) {
+ SYSERROR("%s", "Conda package indexing operation failed");
+ exit(1);
+ }
+
+ msg(STASIS_MSG_L1, "Indexing wheel packages\n");
+ if (indexer_wheels(&ctx)) {
+ SYSERROR("%s", "Python package indexing operation failed");
+ exit(1);
+ }
+
+ msg(STASIS_MSG_L1, "Loading metadata\n");
+ struct StrList *metafiles = NULL;
+ get_files(&metafiles, ctx.storage.meta_dir, "*.stasis");
+ strlist_sort(metafiles, STASIS_SORT_LEN_ASCENDING);
+
+ struct Delivery *local = calloc(strlist_count(metafiles) + 1, sizeof(*local));
+ if (!local) {
+ SYSERROR("%s", "Unable to allocate bytes for local delivery context array");
+ exit(1);
+ }
+
+ for (size_t i = 0; i < strlist_count(metafiles); i++) {
+ char *item = strlist_item(metafiles, i);
+ // Copy the pre-filled contents of the main delivery context
+ memcpy(&local[i], &ctx, sizeof(ctx));
+ if (globals.verbose) {
+ puts(item);
+ }
+ load_metadata(&local[i], item);
+ }
+ qsort(local, strlist_count(metafiles), sizeof(*local), callback_sort_deliveries_cmpfn);
+
+ msg(STASIS_MSG_L1, "Generating links to latest release iteration\n");
+ if (indexer_symlinks(local, strlist_count(metafiles))) {
+ SYSERROR("%s", "Link generation failed");
+ exit(1);
+ }
+
+ msg(STASIS_MSG_L1, "Generating README.md\n");
+ if (indexer_readmes(local, strlist_count(metafiles))) {
+ SYSERROR("%s", "README indexing operation failed");
+ exit(1);
+ }
+
+ msg(STASIS_MSG_L1, "Indexing test results\n");
+ if (indexer_junitxml_report(local, strlist_count(metafiles))) {
+ SYSERROR("%s", "Test result indexing operation failed");
+ exit(1);
+ }
+
+ if (do_html) {
+ msg(STASIS_MSG_L1, "Generating HTML indexes\n");
+ if (indexer_make_website(local)) {
+ SYSERROR("%s", "Site creation failed");
+ exit(1);
+ }
+ }
+
+ msg(STASIS_MSG_L1, "Copying indexed delivery to '%s'\n", destdir);
+ char cmd[PATH_MAX] = {0};
+ sprintf(cmd, "rsync -ah%s --delete --exclude 'tmp/' --exclude 'tools/' '%s/' '%s/'", globals.verbose ? "v" : "q", workdir, destdir);
+ guard_free(destdir);
+
+ if (globals.verbose) {
+ puts(cmd);
+ }
+
+ if (system(cmd)) {
+ SYSERROR("%s", "Copy operation failed");
+ rmtree(workdir);
+ exit(1);
+ }
+
+ msg(STASIS_MSG_L1, "Removing work directory: %s\n", workdir);
+ if (rmtree(workdir)) {
+ SYSERROR("Failed to remove work directory: %s", strerror(errno));
+ }
+
+ guard_free(destdir);
+ GENERIC_ARRAY_FREE(rootdirs);
+ guard_strlist_free(&metafiles);
+ delivery_free(&ctx);
+ globals_free();
+ msg(STASIS_MSG_L1, "Done!\n");
+ return 0;
+}
diff --git a/src/cli/stasis_indexer/website.c b/src/cli/stasis_indexer/website.c
new file mode 100644
index 0000000..55f0c45
--- /dev/null
+++ b/src/cli/stasis_indexer/website.c
@@ -0,0 +1,68 @@
+#include "core.h"
+#include "website.h"
+
+int indexer_make_website(const struct Delivery *ctx) {
+ char *css_filename = calloc(PATH_MAX, sizeof(*css_filename));
+ if (!css_filename) {
+ SYSERROR("unable to allocate string for CSS file path: %s", strerror(errno));
+ return -1;
+ }
+
+ sprintf(css_filename, "%s/%s", globals.sysconfdir, "stasis_pandoc.css");
+ const int have_css = access(css_filename, F_OK | R_OK) == 0;
+
+ struct StrList *dirs = strlist_init();
+ strlist_append(&dirs, ctx->storage.delivery_dir);
+ strlist_append(&dirs, ctx->storage.results_dir);
+
+ struct StrList *inputs = NULL;
+ for (size_t i = 0; i < strlist_count(dirs); i++) {
+ const char *pattern = "*.md";
+ char *dirpath = strlist_item(dirs, i);
+ if (get_files(&inputs, dirpath, pattern)) {
+ SYSERROR("%s does not contain files with pattern: %s", dirpath, pattern);
+ continue;
+ }
+
+ char *root = strlist_item(dirs, i);
+ for (size_t x = 0; x < strlist_count(inputs); x++) {
+ char *filename = path_basename(strlist_item(inputs, x));
+ char fullpath_src[PATH_MAX] = {0};
+ char fullpath_dest[PATH_MAX] = {0};
+ sprintf(fullpath_src, "%s/%s", root, filename);
+ if (access(fullpath_src, F_OK)) {
+ continue;
+ }
+
+ // Replace *.md extension with *.html.
+ strcpy(fullpath_dest, fullpath_src);
+ gen_file_extension_str(fullpath_dest, ".html");
+
+ // Convert markdown to html
+ if (pandoc_exec(fullpath_src, fullpath_dest, have_css ? css_filename : NULL, "STASIS")) {
+ msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "Unable to convert %s\n", fullpath_src);
+ }
+
+ if (file_replace_text(fullpath_dest, ".md", ".html", 0)) {
+ // inform-only
+ SYSERROR("%s: failed to rewrite *.md urls with *.html extension", fullpath_dest);
+ }
+
+ // Link the nearest README.html to index.html
+ if (!strcmp(filename, "README.md")) {
+ char link_from[PATH_MAX] = {0};
+ char link_dest[PATH_MAX] = {0};
+ strcpy(link_from, "README.html");
+ sprintf(link_dest, "%s/%s", root, "index.html");
+ if (symlink(link_from, link_dest)) {
+ SYSERROR("Warning: symlink(%s, %s) failed: %s", link_from, link_dest, strerror(errno));
+ }
+ }
+ }
+ guard_strlist_free(&inputs);
+ }
+ guard_free(css_filename);
+ guard_strlist_free(&dirs);
+
+ return 0;
+}
diff --git a/src/cli/stasis_indexer/website.h b/src/cli/stasis_indexer/website.h
new file mode 100644
index 0000000..e67d58b
--- /dev/null
+++ b/src/cli/stasis_indexer/website.h
@@ -0,0 +1,8 @@
+#ifndef WEBSITE_H
+#define WEBSITE_H
+
+#include "helpers.h"
+
+int indexer_make_website(const struct Delivery *ctx);
+
+#endif //WEBSITE_H
diff --git a/src/lib/core/delivery_artifactory.c b/src/lib/core/delivery_artifactory.c
index b69615e..9ad5829 100644
--- a/src/lib/core/delivery_artifactory.c
+++ b/src/lib/core/delivery_artifactory.c
@@ -185,3 +185,20 @@ int delivery_mission_render_files(struct Delivery *ctx) {
return 0;
}
+int delivery_series_sync(struct Delivery *ctx) {
+ struct JFRT_Download dl = {0};
+
+ char *remote_dir = NULL;
+ if (asprintf(&remote_dir, "%s/%s/%s/(*)", globals.jfrog.repo, ctx->meta.mission, ctx->info.build_name) < 0) {
+ SYSERROR("%s", "Unable to allocate bytes for remote directory path");
+ return -1;
+ }
+
+ char *dest_dir = NULL;
+ if (asprintf(&dest_dir, "%s/{1}", ctx->storage.output_dir) < 0) {
+ SYSERROR("%s", "Unable to allocate bytes for destination directory path");
+ return -1;
+ }
+
+ return jfrog_cli_rt_download(&ctx->deploy.jfrog_auth, &dl, remote_dir, dest_dir);
+}
diff --git a/src/lib/core/delivery_docker.c b/src/lib/core/delivery_docker.c
index c170082..57015ad 100644
--- a/src/lib/core/delivery_docker.c
+++ b/src/lib/core/delivery_docker.c
@@ -92,7 +92,7 @@ int delivery_docker(struct Delivery *ctx) {
memset(rsync_cmd, 0, sizeof(rsync_cmd));
sprintf(rsync_cmd, "rsync -avi --progress '%s' '%s'", ctx->storage.wheel_artifact_dir, dest);
if (system(rsync_cmd)) {
- fprintf(stderr, "Failed to copy wheel artifactory to docker build directory\n");
+ fprintf(stderr, "Failed to copy wheel artifacts to docker build directory\n");
}
if (docker_build(ctx->storage.build_docker_dir, args, ctx->deploy.docker.capabilities.build)) {
diff --git a/src/lib/core/delivery_init.c b/src/lib/core/delivery_init.c
index 356a8ce..2fced03 100644
--- a/src/lib/core/delivery_init.c
+++ b/src/lib/core/delivery_init.c
@@ -309,7 +309,7 @@ int bootstrap_build_info(struct Delivery *ctx) {
}
int delivery_exists(struct Delivery *ctx) {
- int release_exists = 0;
+ int release_exists = DELIVERY_NOT_FOUND;
char release_pattern[PATH_MAX] = {0};
sprintf(release_pattern, "*%s*", ctx->info.release_name);
@@ -320,25 +320,27 @@ int delivery_exists(struct Delivery *ctx) {
}
struct JFRT_Search search = {.fail_no_op = true};
- release_exists = jfrog_cli_rt_search(&ctx->deploy.jfrog_auth, &search, globals.jfrog.repo, release_pattern);
- if (release_exists != 2) {
- if (!globals.enable_overwrite && !release_exists) {
- // --fail_no_op returns 2 on failure
- // without: it returns an empty list "[]" and exit code 0
- return 1; // found
- }
+ // release_exists error states:
+ // `jf rt search --fail_no_op` returns 2 on failure
+ // otherwise, search returns an empty list "[]" and returns 0
+ const int match = jfrog_cli_rt_search(&ctx->deploy.jfrog_auth, &search, globals.jfrog.repo, release_pattern);
+ if (!match) {
+ release_exists = DELIVERY_FOUND;
}
} else {
struct StrList *files = listdir(ctx->storage.delivery_dir);
- for (size_t i = 0; i < strlist_count(files); i++) {
+ const size_t files_count = strlist_count(files);
+
+ for (size_t i = 0; i < files_count; i++) {
char *filename = strlist_item(files, i);
- release_exists = fnmatch(release_pattern, filename, FNM_PATHNAME);
- if (!globals.enable_overwrite && !release_exists) {
- guard_strlist_free(&files);
- return 1; // found
+ const int match = fnmatch(release_pattern, filename, FNM_PATHNAME);
+ if (match == 0) {
+ release_exists = DELIVERY_FOUND;
+ break;
}
}
guard_strlist_free(&files);
}
- return 0; // not found
+
+ return release_exists;
}
diff --git a/src/lib/core/globals.c b/src/lib/core/globals.c
index 83465f1..0f0941a 100644
--- a/src/lib/core/globals.c
+++ b/src/lib/core/globals.c
@@ -37,6 +37,7 @@ struct STASIS_GLOBAL globals = {
.enable_docker = true, ///< Toggle docker usage
.enable_artifactory = true, ///< Toggle artifactory server usage
.enable_artifactory_build_info = true, ///< Toggle build-info uploads
+ .enable_artifactory_upload = true, ///< Toggle artifactory file uploads
.enable_testing = true, ///< Toggle [test] block "script" execution. "script_setup" always executes.
.enable_rewrite_spec_stage_2 = true, ///< Leave template stings in output files
.enable_parallel = true, ///< Toggle testing in parallel
diff --git a/src/lib/core/str.c b/src/lib/core/str.c
index 45fb60a..d774e72 100644
--- a/src/lib/core/str.c
+++ b/src/lib/core/str.c
@@ -162,7 +162,7 @@ char *join(char **arr, const char *separator) {
}
char *join_ex(char *separator, ...) {
- va_list ap; // Variadic argument list
+ va_list ap = {0}; // Variadic argument list
size_t separator_len = 0; // Length of separator string
size_t size = 0; // Length of output string
size_t argc = 0; // Number of arguments ^ "..."
@@ -174,13 +174,6 @@ char *join_ex(char *separator, ...) {
return NULL;
}
- // Initialize array
- argv = calloc(argc + 1, sizeof(char **));
- if (argv == NULL) {
- perror("join_ex calloc failed");
- return NULL;
- }
-
// Get length of the separator
separator_len = strlen(separator);
@@ -192,16 +185,21 @@ char *join_ex(char *separator, ...) {
// 5. Append `current` string to `argv` array
// 6. Update argument counter `argc`
va_start(ap, separator);
- for(argc = 0; (current = va_arg(ap, char *)) != NULL; argc++) {
- char **tmp = realloc(argv, (argc + 1) * sizeof(char *));
- if (tmp == NULL) {
- perror("join_ex realloc failed");
- GENERIC_ARRAY_FREE(argv);
- return NULL;
- }
- argv = tmp;
+ va_list ap_tmp = {0};
+ va_copy(ap_tmp, ap);
+ for(argc = 0; (current = va_arg(ap_tmp, char *)) != NULL; argc++) {}
+ va_end(ap_tmp);
+
+ // Initialize array
+ argv = calloc(argc + 1, sizeof(char **));
+ if (argv == NULL) {
+ perror("join_ex calloc failed");
+ return NULL;
+ }
+
+ for(size_t i = 0; i < argc && (current = va_arg(ap, char *)); i++) {
size += strlen(current) + separator_len;
- argv[argc] = strdup(current);
+ argv[i] = strdup(current);
}
va_end(ap);
diff --git a/src/lib/core/utils.c b/src/lib/core/utils.c
index 18731e6..aa4173c 100644
--- a/src/lib/core/utils.c
+++ b/src/lib/core/utils.c
@@ -766,7 +766,15 @@ struct StrList *listdir(const char *path) {
if (!strcmp(rec->d_name, ".") || !strcmp(rec->d_name, "..")) {
continue;
}
- strlist_append(&node, rec->d_name);
+ char *fullpath = join_ex("/", path, rec->d_name, NULL);
+ if (!fullpath) {
+ SYSERROR("%s", "Unable to allocate bytes to construct full path");
+ guard_strlist_free(&node);
+ closedir(dp);
+ return NULL;
+ }
+ strlist_append(&node, fullpath);
+ guard_free(fullpath);
}
closedir(dp);
return node;
@@ -791,8 +799,6 @@ int mkdirs(const char *_path, mode_t mode) {
char result[PATH_MAX] = {0};
int status = 0;
while ((token = strsep(&path, "/")) != NULL && !status) {
- if (token[0] == '.')
- continue;
strcat(result, token);
strcat(result, "/");
status = mkdir(result, mode);
@@ -850,3 +856,13 @@ int env_manipulate_pathstr(const char *key, char *path, int mode) {
return 0;
}
+int gen_file_extension_str(char *filename, const char *extension) {
+ char *ext_orig = strrchr(filename, '.');
+ if (!ext_orig) {
+ strcat(filename, extension);
+ return 0;
+ }
+
+ return replace_text(ext_orig, ext_orig, extension, 0);
+}
+
diff --git a/stasis_pandoc.css b/stasis_pandoc.css
index 456c3e9..634939f 100644
--- a/stasis_pandoc.css
+++ b/stasis_pandoc.css
@@ -7,7 +7,7 @@ html {
body {
margin: 0 auto;
- max-width: 50%;
+ max-width: 100%;
padding-left: 50px;
padding-right: 50px;
padding-top: 50px;
@@ -94,6 +94,10 @@ h6 {
font-weight: normal;
}
+h1.title {
+ text-align: left;
+}
+
ol,
ul {
padding-left: 1.7em;