diff options
author | Joseph Hunkeler <jhunkeler@gmail.com> | 2025-02-14 12:54:24 -0500 |
---|---|---|
committer | Joseph Hunkeler <jhunkeler@gmail.com> | 2025-02-14 12:54:24 -0500 |
commit | 190d3fbe0f99feb8b00cef34d834c4ef5abb5262 (patch) | |
tree | b9dd8520d2e8e3a97ea6267bc06f89b25353d69b | |
parent | 952de22b54c85aac5eef77d9aafe6587ad555fce (diff) | |
download | stasis-190d3fbe0f99feb8b00cef34d834c4ef5abb5262.tar.gz |
Add library stasis_index.atestable-entrypoint
-rw-r--r-- | CMakeLists.txt | 1 | ||||
-rw-r--r-- | src/cli/stasis_indexer/CMakeLists.txt | 9 | ||||
-rw-r--r-- | src/cli/stasis_indexer/stasis_indexer_main.c | 403 | ||||
-rw-r--r-- | src/lib/CMakeLists.txt | 1 | ||||
-rw-r--r-- | src/lib/index/CMakeLists.txt | 19 | ||||
-rw-r--r-- | src/lib/index/args.c (renamed from src/cli/stasis_indexer/args.c) | 0 | ||||
-rw-r--r-- | src/lib/index/helpers.c (renamed from src/cli/stasis_indexer/helpers.c) | 0 | ||||
-rw-r--r-- | src/lib/index/include/args.h (renamed from src/cli/stasis_indexer/include/args.h) | 0 | ||||
-rw-r--r-- | src/lib/index/include/helpers.h (renamed from src/cli/stasis_indexer/include/helpers.h) | 0 | ||||
-rw-r--r-- | src/lib/index/include/index.h | 14 | ||||
-rw-r--r-- | src/lib/index/include/index_callbacks.h (renamed from src/cli/stasis_indexer/include/callbacks.h) | 0 | ||||
-rw-r--r-- | src/lib/index/include/junitxml_report.h (renamed from src/cli/stasis_indexer/include/junitxml_report.h) | 0 | ||||
-rw-r--r-- | src/lib/index/include/readmes.h (renamed from src/cli/stasis_indexer/include/readmes.h) | 0 | ||||
-rw-r--r-- | src/lib/index/include/website.h (renamed from src/cli/stasis_indexer/include/website.h) | 0 | ||||
-rw-r--r-- | src/lib/index/index_callbacks.c (renamed from src/cli/stasis_indexer/callbacks.c) | 2 | ||||
-rw-r--r-- | src/lib/index/junitxml_report.c (renamed from src/cli/stasis_indexer/junitxml_report.c) | 2 | ||||
-rw-r--r-- | src/lib/index/readmes.c (renamed from src/cli/stasis_indexer/readmes.c) | 0 | ||||
-rw-r--r-- | src/lib/index/stasis_index_entrypoint.c | 402 | ||||
-rw-r--r-- | src/lib/index/website.c (renamed from src/cli/stasis_indexer/website.c) | 0 |
19 files changed, 444 insertions, 409 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index bf1d934..16dde4b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -41,6 +41,7 @@ message(CHECK_START "Compiler flags: ${nix_cflags}") set(core_INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/src/lib/core/include) set(delivery_INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/src/lib/delivery/include) set(entrypoint_INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/src/lib/entrypoint/include) +set(index_INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/src/lib/index/include) set(SYSCONFDIR "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_SYSCONFDIR}") configure_file(${PROJECT_SOURCE_DIR}/include/config.h.in ${CMAKE_CURRENT_BINARY_DIR}/include/config.h @ONLY) include_directories(${PROJECT_BINARY_DIR}/include) diff --git a/src/cli/stasis_indexer/CMakeLists.txt b/src/cli/stasis_indexer/CMakeLists.txt index 68e4ae1..8f92ac4 100644 --- a/src/cli/stasis_indexer/CMakeLists.txt +++ b/src/cli/stasis_indexer/CMakeLists.txt @@ -1,19 +1,14 @@ add_executable(stasis_indexer - args.c stasis_indexer_main.c - callbacks.c - helpers.c - junitxml_report.c - website.c - readmes.c ) target_include_directories(stasis_indexer PRIVATE ${core_INCLUDE} ${delivery_INCLUDE} + ${index_INCLUDE} ${CMAKE_CURRENT_SOURCE_DIR}/include ) target_link_libraries(stasis_indexer PRIVATE - stasis_delivery + stasis_index ) install(TARGETS stasis_indexer RUNTIME) diff --git a/src/cli/stasis_indexer/stasis_indexer_main.c b/src/cli/stasis_indexer/stasis_indexer_main.c index 86f7834..c97ca4d 100644 --- a/src/cli/stasis_indexer/stasis_indexer_main.c +++ b/src/cli/stasis_indexer/stasis_indexer_main.c @@ -1,402 +1,5 @@ -#include <getopt.h> -#include "args.h" -#include "callbacks.h" -#include "helpers.h" -#include "junitxml_report.h" -#include "website.h" -#include "readmes.h" -#include "delivery.h" +#include "index.h" -int indexer_combine_rootdirs(const char *dest, char **rootdirs, const size_t rootdirs_total) { - char cmd[PATH_MAX]; - char destdir_bare[PATH_MAX]; - char destdir_with_output[PATH_MAX]; - char *destdir = destdir_bare; - - memset(cmd, 0, sizeof(cmd)); - memset(destdir_bare, 0, sizeof(destdir_bare)); - memset(destdir_with_output, 0, sizeof(destdir_bare)); - - strcpy(destdir_bare, dest); - strcpy(destdir_with_output, dest); - strcat(destdir_with_output, "/output"); - - if (!access(destdir_with_output, F_OK)) { - destdir = destdir_with_output; - } - - sprintf(cmd, "rsync -ah%s --delete --exclude 'tools/' --exclude 'tmp/' --exclude 'build/' ", globals.verbose ? "v" : "q"); - for (size_t i = 0; i < rootdirs_total; i++) { - char srcdir_bare[PATH_MAX] = {0}; - char srcdir_with_output[PATH_MAX] = {0}; - char *srcdir = srcdir_bare; - strcpy(srcdir_bare, rootdirs[i]); - strcpy(srcdir_with_output, rootdirs[i]); - strcat(srcdir_with_output, "/output"); - - if (access(srcdir_bare, F_OK)) { - fprintf(stderr, "%s does not exist\n", srcdir_bare); - continue; - } - - if (!access(srcdir_with_output, F_OK)) { - srcdir = srcdir_with_output; - } - snprintf(cmd + strlen(cmd), sizeof(srcdir) - strlen(srcdir) + 4, "'%s'/ ", srcdir); - } - snprintf(cmd + strlen(cmd), sizeof(cmd) - strlen(destdir) + 1, " %s/", destdir); - - if (globals.verbose) { - puts(cmd); - } - - if (system(cmd)) { - return -1; - } - return 0; -} - -int indexer_wheels(struct Delivery *ctx) { - return delivery_index_wheel_artifacts(ctx); -} - -int indexer_conda(const struct Delivery *ctx, struct MicromambaInfo m) { - int status = 0; - - status += micromamba(&m, "run conda index %s", ctx->storage.conda_artifact_dir); - return status; -} - -int indexer_symlinks(struct Delivery *ctx, const size_t nelem) { - struct Delivery *data = NULL; - data = get_latest_deliveries(ctx, nelem); - //int latest = get_latest_rc(ctx, nelem); - - if (!pushd(ctx->storage.delivery_dir)) { - for (size_t i = 0; i < nelem; i++) { - char link_name_spec[PATH_MAX]; - char link_name_readme[PATH_MAX]; - - char file_name_spec[PATH_MAX]; - char file_name_readme[PATH_MAX]; - - if (!data[i].meta.name) { - continue; - } - sprintf(link_name_spec, "latest-py%s-%s-%s.yml", data[i].meta.python_compact, data[i].system.platform[DELIVERY_PLATFORM_RELEASE], data[i].system.arch); - sprintf(file_name_spec, "%s.yml", data[i].info.release_name); - - sprintf(link_name_readme, "README-py%s-%s-%s.md", data[i].meta.python_compact, data[i].system.platform[DELIVERY_PLATFORM_RELEASE], data[i].system.arch); - sprintf(file_name_readme, "README-%s.md", data[i].info.release_name); - - if (!access(link_name_spec, F_OK)) { - if (unlink(link_name_spec)) { - fprintf(stderr, "Unable to remove spec link: %s\n", link_name_spec); - } - } - if (!access(link_name_readme, F_OK)) { - if (unlink(link_name_readme)) { - fprintf(stderr, "Unable to remove readme link: %s\n", link_name_readme); - } - } - - if (globals.verbose) { - printf("%s -> %s\n", file_name_spec, link_name_spec); - } - if (symlink(file_name_spec, link_name_spec)) { - fprintf(stderr, "Unable to link %s as %s\n", file_name_spec, link_name_spec); - } - - if (globals.verbose) { - printf("%s -> %s\n", file_name_readme, link_name_readme); - } - if (symlink(file_name_readme, link_name_readme)) { - fprintf(stderr, "Unable to link %s as %s\n", file_name_readme, link_name_readme); - } - } - popd(); - } else { - fprintf(stderr, "Unable to enter delivery directory: %s\n", ctx->storage.delivery_dir); - guard_free(data); - return -1; - } - - // "latest" is an array of pointers to ctx[]. Do not free the contents of the array. - guard_free(data); - return 0; -} - -void indexer_init_dirs(struct Delivery *ctx, const char *workdir) { - path_store(&ctx->storage.root, PATH_MAX, workdir, ""); - path_store(&ctx->storage.tmpdir, PATH_MAX, ctx->storage.root, "tmp"); - if (delivery_init_tmpdir(ctx)) { - fprintf(stderr, "Failed to configure temporary storage directory\n"); - exit(1); - } - - char *user_dir = expandpath("~/.stasis/indexer"); - if (!user_dir) { - SYSERROR("%s", "expandpath failed"); - } - - path_store(&ctx->storage.output_dir, PATH_MAX, ctx->storage.root, ""); - path_store(&ctx->storage.tools_dir, PATH_MAX, user_dir, "tools"); - path_store(&globals.conda_install_prefix, PATH_MAX, user_dir, "conda"); - path_store(&ctx->storage.cfgdump_dir, PATH_MAX, ctx->storage.output_dir, "config"); - path_store(&ctx->storage.meta_dir, PATH_MAX, ctx->storage.output_dir, "meta"); - path_store(&ctx->storage.delivery_dir, PATH_MAX, ctx->storage.output_dir, "delivery"); - path_store(&ctx->storage.package_dir, PATH_MAX, ctx->storage.output_dir, "packages"); - path_store(&ctx->storage.results_dir, PATH_MAX, ctx->storage.output_dir, "results"); - path_store(&ctx->storage.wheel_artifact_dir, PATH_MAX, ctx->storage.package_dir, "wheels"); - path_store(&ctx->storage.conda_artifact_dir, PATH_MAX, ctx->storage.package_dir, "conda"); - path_store(&ctx->storage.docker_artifact_dir, PATH_MAX, ctx->storage.package_dir, "docker"); - guard_free(user_dir); - - char newpath[PATH_MAX] = {0}; - if (getenv("PATH")) { - sprintf(newpath, "%s/bin:%s", ctx->storage.tools_dir, getenv("PATH")); - setenv("PATH", newpath, 1); - } else { - SYSERROR("%s", "environment variable PATH is undefined. Unable to continue."); - exit(1); - } -} - -int main(const int argc, char *argv[]) { - size_t rootdirs_total = 0; - char *destdir = NULL; - char **rootdirs = NULL; - int do_html = 0; - int c = 0; - int option_index = 0; - while ((c = getopt_long(argc, argv, "hd:vUw", long_options, &option_index)) != -1) { - switch (c) { - case 'h': - usage(path_basename(argv[0])); - exit(0); - case 'd': - if (mkdir(optarg, 0755)) { - if (errno != 0 && errno != EEXIST) { - SYSERROR("Unable to create destination directory, '%s': %s", optarg, strerror(errno)); - exit(1); - } - } - destdir = realpath(optarg, NULL); - break; - case 'U': - fflush(stdout); - fflush(stderr); - setvbuf(stdout, NULL, _IONBF, 0); - setvbuf(stderr, NULL, _IONBF, 0); - break; - case 'v': - globals.verbose = 1; - break; - case 'w': - do_html = 1; - break; - case '?': - default: - exit(1); - } - } - - const int current_index = optind; - if (optind < argc) { - rootdirs_total = argc - current_index; - rootdirs = calloc(rootdirs_total + 1, sizeof(*rootdirs)); - - int i = 0; - while (optind < argc) { - if (argv[optind]) { - if (access(argv[optind], F_OK) < 0) { - fprintf(stderr, "%s: %s\n", argv[optind], strerror(errno)); - exit(1); - } - } - // use first positional argument - rootdirs[i] = realpath(argv[optind], NULL); - optind++; - break; - } - } - - if (isempty(destdir)) { - if (mkdir("output", 0755)) { - if (errno != 0 && errno != EEXIST) { - SYSERROR("Unable to create destination directory, '%s': %s", "output", strerror(errno)); - exit(1); - } - } - destdir = realpath("output", NULL); - } - - if (!rootdirs || !rootdirs_total) { - fprintf(stderr, "You must specify at least one STASIS root directory to index\n"); - exit(1); - } - - for (size_t i = 0; i < rootdirs_total; i++) { - if (isempty(rootdirs[i]) || !strcmp(rootdirs[i], "/") || !strcmp(rootdirs[i], "\\")) { - SYSERROR("Unsafe directory: %s", rootdirs[i]); - exit(1); - } - - if (access(rootdirs[i], F_OK)) { - SYSERROR("%s: %s", rootdirs[i], strerror(errno)); - exit(1); - } - } - - char stasis_sysconfdir_tmp[PATH_MAX]; - if (getenv("STASIS_SYSCONFDIR")) { - strncpy(stasis_sysconfdir_tmp, getenv("STASIS_SYSCONFDIR"), sizeof(stasis_sysconfdir_tmp) - 1); - } else { - strncpy(stasis_sysconfdir_tmp, STASIS_SYSCONFDIR, sizeof(stasis_sysconfdir_tmp) - 1); - } - - globals.sysconfdir = realpath(stasis_sysconfdir_tmp, NULL); - if (!globals.sysconfdir) { - msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Unable to resolve path to configuration directory: %s\n", stasis_sysconfdir_tmp); - exit(1); - } - - char workdir_template[PATH_MAX] = {0}; - const char *system_tmp = getenv("TMPDIR"); - if (system_tmp) { - strcat(workdir_template, system_tmp); - } else { - strcat(workdir_template, "/tmp"); - } - strcat(workdir_template, "/stasis-combine.XXXXXX"); - char *workdir = mkdtemp(workdir_template); - if (!workdir) { - SYSERROR("Unable to create temporary directory: %s", workdir_template); - exit(1); - } - if (isempty(workdir) || !strcmp(workdir, "/") || !strcmp(workdir, "\\")) { - SYSERROR("Unsafe directory: %s", workdir); - exit(1); - } - - struct Delivery ctx = {0}; - - printf(BANNER, VERSION, AUTHOR); - - indexer_init_dirs(&ctx, workdir); - - msg(STASIS_MSG_L1, "%s delivery root %s\n", - rootdirs_total > 1 ? "Merging" : "Indexing", - rootdirs_total > 1 ? "directories" : "directory"); - if (indexer_combine_rootdirs(workdir, rootdirs, rootdirs_total)) { - SYSERROR("%s", "Copy operation failed"); - rmtree(workdir); - exit(1); - } - - if (access(ctx.storage.conda_artifact_dir, F_OK)) { - mkdirs(ctx.storage.conda_artifact_dir, 0755); - } - - if (access(ctx.storage.wheel_artifact_dir, F_OK)) { - mkdirs(ctx.storage.wheel_artifact_dir, 0755); - } - - struct MicromambaInfo m; - if (micromamba_configure(&ctx, &m)) { - SYSERROR("%s", "Unable to configure micromamba"); - exit(1); - } - - msg(STASIS_MSG_L1, "Indexing conda packages\n"); - if (indexer_conda(&ctx, m)) { - SYSERROR("%s", "Conda package indexing operation failed"); - exit(1); - } - - msg(STASIS_MSG_L1, "Indexing wheel packages\n"); - if (indexer_wheels(&ctx)) { - SYSERROR("%s", "Python package indexing operation failed"); - exit(1); - } - - msg(STASIS_MSG_L1, "Loading metadata\n"); - struct StrList *metafiles = NULL; - get_files(&metafiles, ctx.storage.meta_dir, "*.stasis"); - strlist_sort(metafiles, STASIS_SORT_LEN_ASCENDING); - - struct Delivery *local = calloc(strlist_count(metafiles) + 1, sizeof(*local)); - if (!local) { - SYSERROR("%s", "Unable to allocate bytes for local delivery context array"); - exit(1); - } - - for (size_t i = 0; i < strlist_count(metafiles); i++) { - char *item = strlist_item(metafiles, i); - // Copy the pre-filled contents of the main delivery context - memcpy(&local[i], &ctx, sizeof(ctx)); - if (globals.verbose) { - puts(item); - } - load_metadata(&local[i], item); - } - qsort(local, strlist_count(metafiles), sizeof(*local), callback_sort_deliveries_cmpfn); - - msg(STASIS_MSG_L1, "Generating links to latest release iteration\n"); - if (indexer_symlinks(local, strlist_count(metafiles))) { - SYSERROR("%s", "Link generation failed"); - exit(1); - } - - msg(STASIS_MSG_L1, "Generating README.md\n"); - if (indexer_readmes(local, strlist_count(metafiles))) { - SYSERROR("%s", "README indexing operation failed"); - exit(1); - } - - msg(STASIS_MSG_L1, "Indexing test results\n"); - if (indexer_junitxml_report(local, strlist_count(metafiles))) { - SYSERROR("%s", "Test result indexing operation failed"); - exit(1); - } - - if (do_html) { - msg(STASIS_MSG_L1, "Generating HTML indexes\n"); - if (indexer_make_website(local)) { - SYSERROR("%s", "Site creation failed"); - exit(1); - } - } - - msg(STASIS_MSG_L1, "Copying indexed delivery to '%s'\n", destdir); - char cmd[PATH_MAX] = {0}; - sprintf(cmd, "rsync -ah%s --delete --exclude 'tmp/' --exclude 'tools/' '%s/' '%s/'", globals.verbose ? "v" : "q", workdir, destdir); - guard_free(destdir); - - if (globals.verbose) { - puts(cmd); - } - - if (system(cmd)) { - SYSERROR("%s", "Copy operation failed"); - rmtree(workdir); - exit(1); - } - - msg(STASIS_MSG_L1, "Removing work directory: %s\n", workdir); - if (rmtree(workdir)) { - SYSERROR("Failed to remove work directory: %s", strerror(errno)); - } - - guard_free(destdir); - GENERIC_ARRAY_FREE(rootdirs); - guard_strlist_free(&metafiles); - guard_free(m.micromamba_prefix); - delivery_free(&ctx); - guard_free(local); - globals_free(); - - msg(STASIS_MSG_L1, "Done!\n"); - - return 0; +int main(int argc, char *argv[]) { + return stasis_index_entrypoint(argc, argv); } diff --git a/src/lib/CMakeLists.txt b/src/lib/CMakeLists.txt index e08acfb..3cf815e 100644 --- a/src/lib/CMakeLists.txt +++ b/src/lib/CMakeLists.txt @@ -1,3 +1,4 @@ add_subdirectory(core) add_subdirectory(delivery) add_subdirectory(entrypoint) +add_subdirectory(index) diff --git a/src/lib/index/CMakeLists.txt b/src/lib/index/CMakeLists.txt new file mode 100644 index 0000000..a647d0d --- /dev/null +++ b/src/lib/index/CMakeLists.txt @@ -0,0 +1,19 @@ +add_library(stasis_index STATIC + args.c + helpers.c + index_callbacks.c + junitxml_report.c + readmes.c + stasis_index_entrypoint.c + website.c + +) +target_link_libraries(stasis_index PRIVATE + stasis_delivery +) +target_include_directories(stasis_index PRIVATE + ${core_INCLUDE} + ${delivery_INCLUDE} + ${CMAKE_CURRENT_SOURCE_DIR}/include +) +target_link_libraries(stasis_index PUBLIC LibXml2::LibXml2) diff --git a/src/cli/stasis_indexer/args.c b/src/lib/index/args.c index 2d92ab0..2d92ab0 100644 --- a/src/cli/stasis_indexer/args.c +++ b/src/lib/index/args.c diff --git a/src/cli/stasis_indexer/helpers.c b/src/lib/index/helpers.c index 5ae01ca..5ae01ca 100644 --- a/src/cli/stasis_indexer/helpers.c +++ b/src/lib/index/helpers.c diff --git a/src/cli/stasis_indexer/include/args.h b/src/lib/index/include/args.h index 543aa4b..543aa4b 100644 --- a/src/cli/stasis_indexer/include/args.h +++ b/src/lib/index/include/args.h diff --git a/src/cli/stasis_indexer/include/helpers.h b/src/lib/index/include/helpers.h index 46705d2..46705d2 100644 --- a/src/cli/stasis_indexer/include/helpers.h +++ b/src/lib/index/include/helpers.h diff --git a/src/lib/index/include/index.h b/src/lib/index/include/index.h new file mode 100644 index 0000000..c57fc7b --- /dev/null +++ b/src/lib/index/include/index.h @@ -0,0 +1,14 @@ +#ifndef STASIS_INDEX_H +#define STASIS_INDEX_H + +#include "args.h" +#include "helpers.h" +#include "index_callbacks.h" +#include "index.h" +#include "junitxml_report.h" +#include "readmes.h" +#include "website.h" + +int stasis_index_entrypoint(int argc, char *argv[]); + +#endif diff --git a/src/cli/stasis_indexer/include/callbacks.h b/src/lib/index/include/index_callbacks.h index 7d95cbb..7d95cbb 100644 --- a/src/cli/stasis_indexer/include/callbacks.h +++ b/src/lib/index/include/index_callbacks.h diff --git a/src/cli/stasis_indexer/include/junitxml_report.h b/src/lib/index/include/junitxml_report.h index 6d2a248..6d2a248 100644 --- a/src/cli/stasis_indexer/include/junitxml_report.h +++ b/src/lib/index/include/junitxml_report.h diff --git a/src/cli/stasis_indexer/include/readmes.h b/src/lib/index/include/readmes.h index d4fa7ac..d4fa7ac 100644 --- a/src/cli/stasis_indexer/include/readmes.h +++ b/src/lib/index/include/readmes.h diff --git a/src/cli/stasis_indexer/include/website.h b/src/lib/index/include/website.h index e67d58b..e67d58b 100644 --- a/src/cli/stasis_indexer/include/website.h +++ b/src/lib/index/include/website.h diff --git a/src/cli/stasis_indexer/callbacks.c b/src/lib/index/index_callbacks.c index 603aef9..1e60863 100644 --- a/src/cli/stasis_indexer/callbacks.c +++ b/src/lib/index/index_callbacks.c @@ -3,7 +3,7 @@ // #include "core.h" -#include "callbacks.h" +#include "index_callbacks.h" // qsort callback to sort delivery contexts by compact python version int callback_sort_deliveries_cmpfn(const void *a, const void *b) { diff --git a/src/cli/stasis_indexer/junitxml_report.c b/src/lib/index/junitxml_report.c index fbb36af..f774c7c 100644 --- a/src/cli/stasis_indexer/junitxml_report.c +++ b/src/lib/index/junitxml_report.c @@ -3,7 +3,7 @@ // #include "core.h" -#include "callbacks.h" +#include "index_callbacks.h" #include "junitxml.h" #include "junitxml_report.h" diff --git a/src/cli/stasis_indexer/readmes.c b/src/lib/index/readmes.c index 7daf261..7daf261 100644 --- a/src/cli/stasis_indexer/readmes.c +++ b/src/lib/index/readmes.c diff --git a/src/lib/index/stasis_index_entrypoint.c b/src/lib/index/stasis_index_entrypoint.c new file mode 100644 index 0000000..87347a3 --- /dev/null +++ b/src/lib/index/stasis_index_entrypoint.c @@ -0,0 +1,402 @@ +#include <getopt.h> +#include "args.h" +#include "index_callbacks.h" +#include "helpers.h" +#include "junitxml_report.h" +#include "website.h" +#include "readmes.h" +#include "delivery.h" + +int indexer_combine_rootdirs(const char *dest, char **rootdirs, const size_t rootdirs_total) { + char cmd[PATH_MAX]; + char destdir_bare[PATH_MAX]; + char destdir_with_output[PATH_MAX]; + char *destdir = destdir_bare; + + memset(cmd, 0, sizeof(cmd)); + memset(destdir_bare, 0, sizeof(destdir_bare)); + memset(destdir_with_output, 0, sizeof(destdir_bare)); + + strcpy(destdir_bare, dest); + strcpy(destdir_with_output, dest); + strcat(destdir_with_output, "/output"); + + if (!access(destdir_with_output, F_OK)) { + destdir = destdir_with_output; + } + + sprintf(cmd, "rsync -ah%s --delete --exclude 'tools/' --exclude 'tmp/' --exclude 'build/' ", globals.verbose ? "v" : "q"); + for (size_t i = 0; i < rootdirs_total; i++) { + char srcdir_bare[PATH_MAX] = {0}; + char srcdir_with_output[PATH_MAX] = {0}; + char *srcdir = srcdir_bare; + strcpy(srcdir_bare, rootdirs[i]); + strcpy(srcdir_with_output, rootdirs[i]); + strcat(srcdir_with_output, "/output"); + + if (access(srcdir_bare, F_OK)) { + fprintf(stderr, "%s does not exist\n", srcdir_bare); + continue; + } + + if (!access(srcdir_with_output, F_OK)) { + srcdir = srcdir_with_output; + } + snprintf(cmd + strlen(cmd), sizeof(srcdir) - strlen(srcdir) + 4, "'%s'/ ", srcdir); + } + snprintf(cmd + strlen(cmd), sizeof(cmd) - strlen(destdir) + 1, " %s/", destdir); + + if (globals.verbose) { + puts(cmd); + } + + if (system(cmd)) { + return -1; + } + return 0; +} + +int indexer_wheels(struct Delivery *ctx) { + return delivery_index_wheel_artifacts(ctx); +} + +int indexer_conda(const struct Delivery *ctx, struct MicromambaInfo m) { + int status = 0; + + status += micromamba(&m, "run conda index %s", ctx->storage.conda_artifact_dir); + return status; +} + +int indexer_symlinks(struct Delivery *ctx, const size_t nelem) { + struct Delivery *data = NULL; + data = get_latest_deliveries(ctx, nelem); + //int latest = get_latest_rc(ctx, nelem); + + if (!pushd(ctx->storage.delivery_dir)) { + for (size_t i = 0; i < nelem; i++) { + char link_name_spec[PATH_MAX]; + char link_name_readme[PATH_MAX]; + + char file_name_spec[PATH_MAX]; + char file_name_readme[PATH_MAX]; + + if (!data[i].meta.name) { + continue; + } + sprintf(link_name_spec, "latest-py%s-%s-%s.yml", data[i].meta.python_compact, data[i].system.platform[DELIVERY_PLATFORM_RELEASE], data[i].system.arch); + sprintf(file_name_spec, "%s.yml", data[i].info.release_name); + + sprintf(link_name_readme, "README-py%s-%s-%s.md", data[i].meta.python_compact, data[i].system.platform[DELIVERY_PLATFORM_RELEASE], data[i].system.arch); + sprintf(file_name_readme, "README-%s.md", data[i].info.release_name); + + if (!access(link_name_spec, F_OK)) { + if (unlink(link_name_spec)) { + fprintf(stderr, "Unable to remove spec link: %s\n", link_name_spec); + } + } + if (!access(link_name_readme, F_OK)) { + if (unlink(link_name_readme)) { + fprintf(stderr, "Unable to remove readme link: %s\n", link_name_readme); + } + } + + if (globals.verbose) { + printf("%s -> %s\n", file_name_spec, link_name_spec); + } + if (symlink(file_name_spec, link_name_spec)) { + fprintf(stderr, "Unable to link %s as %s\n", file_name_spec, link_name_spec); + } + + if (globals.verbose) { + printf("%s -> %s\n", file_name_readme, link_name_readme); + } + if (symlink(file_name_readme, link_name_readme)) { + fprintf(stderr, "Unable to link %s as %s\n", file_name_readme, link_name_readme); + } + } + popd(); + } else { + fprintf(stderr, "Unable to enter delivery directory: %s\n", ctx->storage.delivery_dir); + guard_free(data); + return -1; + } + + // "latest" is an array of pointers to ctx[]. Do not free the contents of the array. + guard_free(data); + return 0; +} + +void indexer_init_dirs(struct Delivery *ctx, const char *workdir) { + path_store(&ctx->storage.root, PATH_MAX, workdir, ""); + path_store(&ctx->storage.tmpdir, PATH_MAX, ctx->storage.root, "tmp"); + if (delivery_init_tmpdir(ctx)) { + fprintf(stderr, "Failed to configure temporary storage directory\n"); + exit(1); + } + + char *user_dir = expandpath("~/.stasis/indexer"); + if (!user_dir) { + SYSERROR("%s", "expandpath failed"); + } + + path_store(&ctx->storage.output_dir, PATH_MAX, ctx->storage.root, ""); + path_store(&ctx->storage.tools_dir, PATH_MAX, user_dir, "tools"); + path_store(&globals.conda_install_prefix, PATH_MAX, user_dir, "conda"); + path_store(&ctx->storage.cfgdump_dir, PATH_MAX, ctx->storage.output_dir, "config"); + path_store(&ctx->storage.meta_dir, PATH_MAX, ctx->storage.output_dir, "meta"); + path_store(&ctx->storage.delivery_dir, PATH_MAX, ctx->storage.output_dir, "delivery"); + path_store(&ctx->storage.package_dir, PATH_MAX, ctx->storage.output_dir, "packages"); + path_store(&ctx->storage.results_dir, PATH_MAX, ctx->storage.output_dir, "results"); + path_store(&ctx->storage.wheel_artifact_dir, PATH_MAX, ctx->storage.package_dir, "wheels"); + path_store(&ctx->storage.conda_artifact_dir, PATH_MAX, ctx->storage.package_dir, "conda"); + path_store(&ctx->storage.docker_artifact_dir, PATH_MAX, ctx->storage.package_dir, "docker"); + guard_free(user_dir); + + char newpath[PATH_MAX] = {0}; + if (getenv("PATH")) { + sprintf(newpath, "%s/bin:%s", ctx->storage.tools_dir, getenv("PATH")); + setenv("PATH", newpath, 1); + } else { + SYSERROR("%s", "environment variable PATH is undefined. Unable to continue."); + exit(1); + } +} + +int stasis_index_entrypoint(const int argc, char *argv[]) { + size_t rootdirs_total = 0; + char *destdir = NULL; + char **rootdirs = NULL; + int do_html = 0; + int c = 0; + int option_index = 0; + while ((c = getopt_long(argc, argv, "hd:vUw", long_options, &option_index)) != -1) { + switch (c) { + case 'h': + usage(path_basename(argv[0])); + exit(0); + case 'd': + if (mkdir(optarg, 0755)) { + if (errno != 0 && errno != EEXIST) { + SYSERROR("Unable to create destination directory, '%s': %s", optarg, strerror(errno)); + exit(1); + } + } + destdir = realpath(optarg, NULL); + break; + case 'U': + fflush(stdout); + fflush(stderr); + setvbuf(stdout, NULL, _IONBF, 0); + setvbuf(stderr, NULL, _IONBF, 0); + break; + case 'v': + globals.verbose = 1; + break; + case 'w': + do_html = 1; + break; + case '?': + default: + exit(1); + } + } + + const int current_index = optind; + if (optind < argc) { + rootdirs_total = argc - current_index; + rootdirs = calloc(rootdirs_total + 1, sizeof(*rootdirs)); + + int i = 0; + while (optind < argc) { + if (argv[optind]) { + if (access(argv[optind], F_OK) < 0) { + fprintf(stderr, "%s: %s\n", argv[optind], strerror(errno)); + exit(1); + } + } + // use first positional argument + rootdirs[i] = realpath(argv[optind], NULL); + optind++; + break; + } + } + + if (isempty(destdir)) { + if (mkdir("output", 0755)) { + if (errno != 0 && errno != EEXIST) { + SYSERROR("Unable to create destination directory, '%s': %s", "output", strerror(errno)); + exit(1); + } + } + destdir = realpath("output", NULL); + } + + if (!rootdirs || !rootdirs_total) { + fprintf(stderr, "You must specify at least one STASIS root directory to index\n"); + exit(1); + } + + for (size_t i = 0; i < rootdirs_total; i++) { + if (isempty(rootdirs[i]) || !strcmp(rootdirs[i], "/") || !strcmp(rootdirs[i], "\\")) { + SYSERROR("Unsafe directory: %s", rootdirs[i]); + exit(1); + } + + if (access(rootdirs[i], F_OK)) { + SYSERROR("%s: %s", rootdirs[i], strerror(errno)); + exit(1); + } + } + + char stasis_sysconfdir_tmp[PATH_MAX]; + if (getenv("STASIS_SYSCONFDIR")) { + strncpy(stasis_sysconfdir_tmp, getenv("STASIS_SYSCONFDIR"), sizeof(stasis_sysconfdir_tmp) - 1); + } else { + strncpy(stasis_sysconfdir_tmp, STASIS_SYSCONFDIR, sizeof(stasis_sysconfdir_tmp) - 1); + } + + globals.sysconfdir = realpath(stasis_sysconfdir_tmp, NULL); + if (!globals.sysconfdir) { + msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Unable to resolve path to configuration directory: %s\n", stasis_sysconfdir_tmp); + exit(1); + } + + char workdir_template[PATH_MAX] = {0}; + const char *system_tmp = getenv("TMPDIR"); + if (system_tmp) { + strcat(workdir_template, system_tmp); + } else { + strcat(workdir_template, "/tmp"); + } + strcat(workdir_template, "/stasis-combine.XXXXXX"); + char *workdir = mkdtemp(workdir_template); + if (!workdir) { + SYSERROR("Unable to create temporary directory: %s", workdir_template); + exit(1); + } + if (isempty(workdir) || !strcmp(workdir, "/") || !strcmp(workdir, "\\")) { + SYSERROR("Unsafe directory: %s", workdir); + exit(1); + } + + struct Delivery ctx = {0}; + + printf(BANNER, VERSION, AUTHOR); + + indexer_init_dirs(&ctx, workdir); + + msg(STASIS_MSG_L1, "%s delivery root %s\n", + rootdirs_total > 1 ? "Merging" : "Indexing", + rootdirs_total > 1 ? "directories" : "directory"); + if (indexer_combine_rootdirs(workdir, rootdirs, rootdirs_total)) { + SYSERROR("%s", "Copy operation failed"); + rmtree(workdir); + exit(1); + } + + if (access(ctx.storage.conda_artifact_dir, F_OK)) { + mkdirs(ctx.storage.conda_artifact_dir, 0755); + } + + if (access(ctx.storage.wheel_artifact_dir, F_OK)) { + mkdirs(ctx.storage.wheel_artifact_dir, 0755); + } + + struct MicromambaInfo m; + if (micromamba_configure(&ctx, &m)) { + SYSERROR("%s", "Unable to configure micromamba"); + exit(1); + } + + msg(STASIS_MSG_L1, "Indexing conda packages\n"); + if (indexer_conda(&ctx, m)) { + SYSERROR("%s", "Conda package indexing operation failed"); + exit(1); + } + + msg(STASIS_MSG_L1, "Indexing wheel packages\n"); + if (indexer_wheels(&ctx)) { + SYSERROR("%s", "Python package indexing operation failed"); + exit(1); + } + + msg(STASIS_MSG_L1, "Loading metadata\n"); + struct StrList *metafiles = NULL; + get_files(&metafiles, ctx.storage.meta_dir, "*.stasis"); + strlist_sort(metafiles, STASIS_SORT_LEN_ASCENDING); + + struct Delivery *local = calloc(strlist_count(metafiles) + 1, sizeof(*local)); + if (!local) { + SYSERROR("%s", "Unable to allocate bytes for local delivery context array"); + exit(1); + } + + for (size_t i = 0; i < strlist_count(metafiles); i++) { + char *item = strlist_item(metafiles, i); + // Copy the pre-filled contents of the main delivery context + memcpy(&local[i], &ctx, sizeof(ctx)); + if (globals.verbose) { + puts(item); + } + load_metadata(&local[i], item); + } + qsort(local, strlist_count(metafiles), sizeof(*local), callback_sort_deliveries_cmpfn); + + msg(STASIS_MSG_L1, "Generating links to latest release iteration\n"); + if (indexer_symlinks(local, strlist_count(metafiles))) { + SYSERROR("%s", "Link generation failed"); + exit(1); + } + + msg(STASIS_MSG_L1, "Generating README.md\n"); + if (indexer_readmes(local, strlist_count(metafiles))) { + SYSERROR("%s", "README indexing operation failed"); + exit(1); + } + + msg(STASIS_MSG_L1, "Indexing test results\n"); + if (indexer_junitxml_report(local, strlist_count(metafiles))) { + SYSERROR("%s", "Test result indexing operation failed"); + exit(1); + } + + if (do_html) { + msg(STASIS_MSG_L1, "Generating HTML indexes\n"); + if (indexer_make_website(local)) { + SYSERROR("%s", "Site creation failed"); + exit(1); + } + } + + msg(STASIS_MSG_L1, "Copying indexed delivery to '%s'\n", destdir); + char cmd[PATH_MAX] = {0}; + sprintf(cmd, "rsync -ah%s --delete --exclude 'tmp/' --exclude 'tools/' '%s/' '%s/'", globals.verbose ? "v" : "q", workdir, destdir); + guard_free(destdir); + + if (globals.verbose) { + puts(cmd); + } + + if (system(cmd)) { + SYSERROR("%s", "Copy operation failed"); + rmtree(workdir); + exit(1); + } + + msg(STASIS_MSG_L1, "Removing work directory: %s\n", workdir); + if (rmtree(workdir)) { + SYSERROR("Failed to remove work directory: %s", strerror(errno)); + } + + guard_free(destdir); + GENERIC_ARRAY_FREE(rootdirs); + guard_strlist_free(&metafiles); + guard_free(m.micromamba_prefix); + delivery_free(&ctx); + guard_free(local); + globals_free(); + + msg(STASIS_MSG_L1, "Done!\n"); + + return 0; +} diff --git a/src/cli/stasis_indexer/website.c b/src/lib/index/website.c index 55f0c45..55f0c45 100644 --- a/src/cli/stasis_indexer/website.c +++ b/src/lib/index/website.c |