From a9d975389aaf5d79d738517b98161e375e757cba Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Mon, 18 Nov 2024 09:27:35 -0500 Subject: Break down indexer into independent source files * Generate test result output * Add helper function to manage changing file extensions --- src/lib/core/utils.c | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'src/lib/core') diff --git a/src/lib/core/utils.c b/src/lib/core/utils.c index 18731e6..c03f8fa 100644 --- a/src/lib/core/utils.c +++ b/src/lib/core/utils.c @@ -850,3 +850,13 @@ int env_manipulate_pathstr(const char *key, char *path, int mode) { return 0; } +int gen_file_extension_str(char *filename, const char *extension) { + char *ext_orig = strrchr(filename, '.'); + if (!ext_orig) { + strcat(filename, extension); + return 0; + } + + return replace_text(ext_orig, ext_orig, extension, 0); +} + -- cgit From 93ad037af095d490e836a16b92e23f637f1f61ab Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Wed, 20 Nov 2024 22:32:01 -0500 Subject: pip_packages now accepts name[extra1,extra2]==version strings --- src/lib/core/delivery.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) (limited to 'src/lib/core') diff --git a/src/lib/core/delivery.c b/src/lib/core/delivery.c index 1ceb8b7..aa3e51a 100644 --- a/src/lib/core/delivery.c +++ b/src/lib/core/delivery.c @@ -207,6 +207,15 @@ void delivery_defer_packages(struct Delivery *ctx, int type) { strncpy(package_name, name, sizeof(package_name) - 1); } + char *extra_begin = strchr(package_name, '['); + char *extra_end = NULL; + if (extra_begin) { + extra_end = strchr(extra_begin, ']'); + if (extra_end) { + *extra_begin = '\0'; + } + } + msg(STASIS_MSG_L3, "package '%s': ", package_name); // When spec is present in name, set tests->version to the version detected in the name @@ -214,11 +223,7 @@ void delivery_defer_packages(struct Delivery *ctx, int type) { struct Test *test = &ctx->tests[x]; char nametmp[1024] = {0}; - if (spec_end != NULL && spec_begin != NULL) { - strncpy(nametmp, name, spec_begin - name); - } else { - strcpy(nametmp, name); - } + strncpy(nametmp, package_name, sizeof(nametmp) - 1); // Is the [test:NAME] in the package name? if (!strcmp(nametmp, test->name)) { // Override test->version when a version is provided by the (pip|conda)_package list item @@ -260,7 +265,9 @@ void delivery_defer_packages(struct Delivery *ctx, int type) { fprintf(stderr, "%s's existence command failed for '%s': %s\n", mode, name, pkg_index_provides_strerror(upstream_exists)); exit(1); - } else if (upstream_exists == PKG_NOT_FOUND) { + } + + if (upstream_exists == PKG_NOT_FOUND) { build_for_host = 1; } else { build_for_host = 0; -- cgit From 9173fe4a472500b3f87780d116ff7a54d4729c78 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 22 Nov 2024 15:46:56 -0500 Subject: Add basic unindent function --- src/lib/core/str.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) (limited to 'src/lib/core') diff --git a/src/lib/core/str.c b/src/lib/core/str.c index a7dbab1..45fb60a 100644 --- a/src/lib/core/str.c +++ b/src/lib/core/str.c @@ -649,3 +649,28 @@ char *to_short_version(const char *s) { strchrdel(result, "."); return result; } + +void unindent(char *s) { + char *pos = NULL; + size_t leading_spaces; + + // Set position to beginning of string + pos = s; + + while (pos != NULL) { + const size_t len = strlen(s); + for (leading_spaces = 0; isspace(pos[leading_spaces]); leading_spaces++) {} + + // For each new line strip an indent + if (leading_spaces >= 4 && len >= 4) { + leading_spaces = 4; // remove first level of indentation + memmove(pos, pos + leading_spaces, len - leading_spaces); + pos[len - leading_spaces] = '\0'; + } + + pos = strchr(pos, '\n'); + if (pos && strlen(pos)) { + pos++; + } + } +} \ No newline at end of file -- cgit From ff4e8b219792684d2f09d865fab7aad1eab3a957 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 22 Nov 2024 16:23:45 -0500 Subject: Unindent script --- src/lib/core/delivery_test.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/lib/core') diff --git a/src/lib/core/delivery_test.c b/src/lib/core/delivery_test.c index 0bcf04d..8dbcd78 100644 --- a/src/lib/core/delivery_test.c +++ b/src/lib/core/delivery_test.c @@ -118,6 +118,9 @@ void delivery_tests_run(struct Delivery *ctx) { SYSERROR("An error occurred while rendering the following:\n%s", cmd); exit(1); } + // Move indents + // HEREDOCs will not work otherwise + unindent(cmd); if (test->disable) { msg(STASIS_MSG_L2, "Script execution disabled by configuration\n", test->name); -- cgit From 506269820329f019d1ff26eb681fbaa70bcb9674 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 22 Nov 2024 16:26:40 -0500 Subject: Unindent script_setup --- src/lib/core/delivery_test.c | 1 + 1 file changed, 1 insertion(+) (limited to 'src/lib/core') diff --git a/src/lib/core/delivery_test.c b/src/lib/core/delivery_test.c index 8dbcd78..e80e0ec 100644 --- a/src/lib/core/delivery_test.c +++ b/src/lib/core/delivery_test.c @@ -192,6 +192,7 @@ void delivery_tests_run(struct Delivery *ctx) { SYSERROR("An error occurred while rendering the following:\n%s", cmd); exit(1); } + unindent(cmd); struct MultiProcessingTask *task = NULL; char *runner_cmd = NULL; -- cgit From 080866ee9400dbc1cf82a71621e55cb20861f6e8 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 3 Dec 2024 00:57:34 -0500 Subject: Add DELIVERY_[NOT_]FOUND defines * Add delivery_series_sync function to download previously delivered files from artifactory --- src/lib/core/delivery_artifactory.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'src/lib/core') diff --git a/src/lib/core/delivery_artifactory.c b/src/lib/core/delivery_artifactory.c index b69615e..9ad5829 100644 --- a/src/lib/core/delivery_artifactory.c +++ b/src/lib/core/delivery_artifactory.c @@ -185,3 +185,20 @@ int delivery_mission_render_files(struct Delivery *ctx) { return 0; } +int delivery_series_sync(struct Delivery *ctx) { + struct JFRT_Download dl = {0}; + + char *remote_dir = NULL; + if (asprintf(&remote_dir, "%s/%s/%s/(*)", globals.jfrog.repo, ctx->meta.mission, ctx->info.build_name) < 0) { + SYSERROR("%s", "Unable to allocate bytes for remote directory path"); + return -1; + } + + char *dest_dir = NULL; + if (asprintf(&dest_dir, "%s/{1}", ctx->storage.output_dir) < 0) { + SYSERROR("%s", "Unable to allocate bytes for destination directory path"); + return -1; + } + + return jfrog_cli_rt_download(&ctx->deploy.jfrog_auth, &dl, remote_dir, dest_dir); +} -- cgit From cc6458bc5efffc9712bb0c731957dfe75d1ebca5 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 3 Dec 2024 00:58:40 -0500 Subject: Fix error message wording --- src/lib/core/delivery_docker.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib/core') diff --git a/src/lib/core/delivery_docker.c b/src/lib/core/delivery_docker.c index c170082..57015ad 100644 --- a/src/lib/core/delivery_docker.c +++ b/src/lib/core/delivery_docker.c @@ -92,7 +92,7 @@ int delivery_docker(struct Delivery *ctx) { memset(rsync_cmd, 0, sizeof(rsync_cmd)); sprintf(rsync_cmd, "rsync -avi --progress '%s' '%s'", ctx->storage.wheel_artifact_dir, dest); if (system(rsync_cmd)) { - fprintf(stderr, "Failed to copy wheel artifactory to docker build directory\n"); + fprintf(stderr, "Failed to copy wheel artifacts to docker build directory\n"); } if (docker_build(ctx->storage.build_docker_dir, args, ctx->deploy.docker.capabilities.build)) { -- cgit From 38e4789530a38ac9e7dfddfc0afcff178bd1647f Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 3 Dec 2024 01:00:22 -0500 Subject: Simplify delivery_exists() function * Returns DELIVERY_[NOT_]FOUND, or -1 on error --- src/lib/core/delivery_init.c | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) (limited to 'src/lib/core') diff --git a/src/lib/core/delivery_init.c b/src/lib/core/delivery_init.c index 356a8ce..7e51f24 100644 --- a/src/lib/core/delivery_init.c +++ b/src/lib/core/delivery_init.c @@ -320,25 +320,27 @@ int delivery_exists(struct Delivery *ctx) { } struct JFRT_Search search = {.fail_no_op = true}; + // release_exists error states: + // `jf rt search --fail_no_op` returns 2 on failure + // otherwise, search returns an empty list "[]" and returns 0 release_exists = jfrog_cli_rt_search(&ctx->deploy.jfrog_auth, &search, globals.jfrog.repo, release_pattern); - if (release_exists != 2) { - if (!globals.enable_overwrite && !release_exists) { - // --fail_no_op returns 2 on failure - // without: it returns an empty list "[]" and exit code 0 - return 1; // found - } - } } else { struct StrList *files = listdir(ctx->storage.delivery_dir); for (size_t i = 0; i < strlist_count(files); i++) { char *filename = strlist_item(files, i); release_exists = fnmatch(release_pattern, filename, FNM_PATHNAME); - if (!globals.enable_overwrite && !release_exists) { - guard_strlist_free(&files); - return 1; // found + if (!release_exists) { + break; } } guard_strlist_free(&files); } - return 0; // not found + + if (release_exists < 0) { + return -1; // error + } + if (release_exists >= 1) { + return DELIVERY_NOT_FOUND; + } + return DELIVERY_FOUND; } -- cgit From 9098abc13882e6b665e46361721d3bcba7da55eb Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 3 Dec 2024 09:26:32 -0500 Subject: delivery_exists() returns DELIVERY_NOT_FOUND by default --- src/lib/core/delivery_init.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) (limited to 'src/lib/core') diff --git a/src/lib/core/delivery_init.c b/src/lib/core/delivery_init.c index 7e51f24..2fced03 100644 --- a/src/lib/core/delivery_init.c +++ b/src/lib/core/delivery_init.c @@ -309,7 +309,7 @@ int bootstrap_build_info(struct Delivery *ctx) { } int delivery_exists(struct Delivery *ctx) { - int release_exists = 0; + int release_exists = DELIVERY_NOT_FOUND; char release_pattern[PATH_MAX] = {0}; sprintf(release_pattern, "*%s*", ctx->info.release_name); @@ -323,24 +323,24 @@ int delivery_exists(struct Delivery *ctx) { // release_exists error states: // `jf rt search --fail_no_op` returns 2 on failure // otherwise, search returns an empty list "[]" and returns 0 - release_exists = jfrog_cli_rt_search(&ctx->deploy.jfrog_auth, &search, globals.jfrog.repo, release_pattern); + const int match = jfrog_cli_rt_search(&ctx->deploy.jfrog_auth, &search, globals.jfrog.repo, release_pattern); + if (!match) { + release_exists = DELIVERY_FOUND; + } } else { struct StrList *files = listdir(ctx->storage.delivery_dir); - for (size_t i = 0; i < strlist_count(files); i++) { + const size_t files_count = strlist_count(files); + + for (size_t i = 0; i < files_count; i++) { char *filename = strlist_item(files, i); - release_exists = fnmatch(release_pattern, filename, FNM_PATHNAME); - if (!release_exists) { + const int match = fnmatch(release_pattern, filename, FNM_PATHNAME); + if (match == 0) { + release_exists = DELIVERY_FOUND; break; } } guard_strlist_free(&files); } - if (release_exists < 0) { - return -1; // error - } - if (release_exists >= 1) { - return DELIVERY_NOT_FOUND; - } - return DELIVERY_FOUND; + return release_exists; } -- cgit From 5796ce9338c7fe2aa8a26766ff9e01448d785c99 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 3 Dec 2024 10:47:37 -0500 Subject: Add ability to use artifactory without uploading any artifacts at the end. * New option: --no-artifactory-upload * Implies --no-artifactory-build-info * Updated README.md --- src/lib/core/globals.c | 1 + 1 file changed, 1 insertion(+) (limited to 'src/lib/core') diff --git a/src/lib/core/globals.c b/src/lib/core/globals.c index 83465f1..0f0941a 100644 --- a/src/lib/core/globals.c +++ b/src/lib/core/globals.c @@ -37,6 +37,7 @@ struct STASIS_GLOBAL globals = { .enable_docker = true, ///< Toggle docker usage .enable_artifactory = true, ///< Toggle artifactory server usage .enable_artifactory_build_info = true, ///< Toggle build-info uploads + .enable_artifactory_upload = true, ///< Toggle artifactory file uploads .enable_testing = true, ///< Toggle [test] block "script" execution. "script_setup" always executes. .enable_rewrite_spec_stage_2 = true, ///< Leave template stings in output files .enable_parallel = true, ///< Toggle testing in parallel -- cgit From 2a15ece791252c07ddf2fe9868709bd80eecf489 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 6 Dec 2024 08:21:58 -0500 Subject: Fix segfault in join_ex * Calculate the number of function arguments with va_copy() * Remove realloc() usage. No point. --- src/lib/core/str.c | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) (limited to 'src/lib/core') diff --git a/src/lib/core/str.c b/src/lib/core/str.c index 45fb60a..d774e72 100644 --- a/src/lib/core/str.c +++ b/src/lib/core/str.c @@ -162,7 +162,7 @@ char *join(char **arr, const char *separator) { } char *join_ex(char *separator, ...) { - va_list ap; // Variadic argument list + va_list ap = {0}; // Variadic argument list size_t separator_len = 0; // Length of separator string size_t size = 0; // Length of output string size_t argc = 0; // Number of arguments ^ "..." @@ -174,13 +174,6 @@ char *join_ex(char *separator, ...) { return NULL; } - // Initialize array - argv = calloc(argc + 1, sizeof(char **)); - if (argv == NULL) { - perror("join_ex calloc failed"); - return NULL; - } - // Get length of the separator separator_len = strlen(separator); @@ -192,16 +185,21 @@ char *join_ex(char *separator, ...) { // 5. Append `current` string to `argv` array // 6. Update argument counter `argc` va_start(ap, separator); - for(argc = 0; (current = va_arg(ap, char *)) != NULL; argc++) { - char **tmp = realloc(argv, (argc + 1) * sizeof(char *)); - if (tmp == NULL) { - perror("join_ex realloc failed"); - GENERIC_ARRAY_FREE(argv); - return NULL; - } - argv = tmp; + va_list ap_tmp = {0}; + va_copy(ap_tmp, ap); + for(argc = 0; (current = va_arg(ap_tmp, char *)) != NULL; argc++) {} + va_end(ap_tmp); + + // Initialize array + argv = calloc(argc + 1, sizeof(char **)); + if (argv == NULL) { + perror("join_ex calloc failed"); + return NULL; + } + + for(size_t i = 0; i < argc && (current = va_arg(ap, char *)); i++) { size += strlen(current) + separator_len; - argv[argc] = strdup(current); + argv[i] = strdup(current); } va_end(ap); -- cgit From a2bcfc37f3634179b4e75fff38d4c36a1ab2a812 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 6 Dec 2024 08:26:35 -0500 Subject: Fix listdir() * Now returns the absolute path(s) to the file(s) * Remove restriction on reading hidden files --- src/lib/core/utils.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'src/lib/core') diff --git a/src/lib/core/utils.c b/src/lib/core/utils.c index c03f8fa..aa4173c 100644 --- a/src/lib/core/utils.c +++ b/src/lib/core/utils.c @@ -766,7 +766,15 @@ struct StrList *listdir(const char *path) { if (!strcmp(rec->d_name, ".") || !strcmp(rec->d_name, "..")) { continue; } - strlist_append(&node, rec->d_name); + char *fullpath = join_ex("/", path, rec->d_name, NULL); + if (!fullpath) { + SYSERROR("%s", "Unable to allocate bytes to construct full path"); + guard_strlist_free(&node); + closedir(dp); + return NULL; + } + strlist_append(&node, fullpath); + guard_free(fullpath); } closedir(dp); return node; @@ -791,8 +799,6 @@ int mkdirs(const char *_path, mode_t mode) { char result[PATH_MAX] = {0}; int status = 0; while ((token = strsep(&path, "/")) != NULL && !status) { - if (token[0] == '.') - continue; strcat(result, token); strcat(result, "/"); status = mkdir(result, mode); -- cgit