diff options
| -rw-r--r-- | src/lib/core/template_func_proto.c | 4 | ||||
| -rw-r--r-- | src/lib/delivery/delivery.c | 58 | ||||
| -rw-r--r-- | src/lib/delivery/delivery_build.c | 78 | ||||
| -rw-r--r-- | src/lib/delivery/delivery_install.c | 6 | ||||
| -rw-r--r-- | src/lib/delivery/delivery_populate.c | 16 | ||||
| -rw-r--r-- | src/lib/delivery/delivery_show.c | 10 | ||||
| -rw-r--r-- | src/lib/delivery/delivery_test.c | 64 |
7 files changed, 153 insertions, 83 deletions
diff --git a/src/lib/core/template_func_proto.c b/src/lib/core/template_func_proto.c index 3e1cd99..52a11b5 100644 --- a/src/lib/core/template_func_proto.c +++ b/src/lib/core/template_func_proto.c @@ -28,9 +28,9 @@ int get_github_release_notes_auto_tplfunc_entrypoint(void *frame, void *data_out const struct Delivery *ctx = (struct Delivery *) f->data_in; struct StrList *notes_list = strlist_init(); - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(*ctx->tests); i++) { + for (size_t i = 0; i < ctx->tests->num_used; i++) { // Get test context - const struct Test *test = &ctx->tests[i]; + const struct Test *test = ctx->tests->test[i]; if (test->name && test->version && test->repository) { char *repository = strdup(test->repository); char *match = strstr(repository, "spacetelescope/"); diff --git a/src/lib/delivery/delivery.c b/src/lib/delivery/delivery.c index 600ddf9..4409e69 100644 --- a/src/lib/delivery/delivery.c +++ b/src/lib/delivery/delivery.c @@ -153,21 +153,21 @@ struct Delivery *delivery_duplicate(const struct Delivery *ctx) { result->deploy.jfrog_auth.url = strdup_maybe(ctx->deploy.jfrog_auth.url); result->deploy.jfrog_auth.user = strdup_maybe(ctx->deploy.jfrog_auth.user); - for (size_t i = 0; i < sizeof(result->tests) / sizeof(result->tests[0]); i++) { - result->tests[i].disable = ctx->tests[i].disable; - result->tests[i].parallel = ctx->tests[i].parallel; - result->tests[i].build_recipe = strdup_maybe(ctx->tests[i].build_recipe); - result->tests[i].name = strdup_maybe(ctx->tests[i].name); - result->tests[i].version = strdup_maybe(ctx->tests[i].version); - result->tests[i].repository = strdup_maybe(ctx->tests[i].repository); - result->tests[i].repository_info_ref = strdup_maybe(ctx->tests[i].repository_info_ref); - result->tests[i].repository_info_tag = strdup_maybe(ctx->tests[i].repository_info_tag); - result->tests[i].repository_remove_tags = strlist_copy(ctx->tests[i].repository_remove_tags); - if (ctx->tests[i].runtime.environ) { - result->tests[i].runtime.environ = runtime_copy(ctx->tests[i].runtime.environ->data); + for (size_t i = 0; i < result->tests->num_used; i++) { + result->tests->test[i]->disable = ctx->tests->test[i]->disable; + result->tests->test[i]->parallel = ctx->tests->test[i]->parallel; + result->tests->test[i]->build_recipe = strdup_maybe(ctx->tests->test[i]->build_recipe); + result->tests->test[i]->name = strdup_maybe(ctx->tests->test[i]->name); + result->tests->test[i]->version = strdup_maybe(ctx->tests->test[i]->version); + result->tests->test[i]->repository = strdup_maybe(ctx->tests->test[i]->repository); + result->tests->test[i]->repository_info_ref = strdup_maybe(ctx->tests->test[i]->repository_info_ref); + result->tests->test[i]->repository_info_tag = strdup_maybe(ctx->tests->test[i]->repository_info_tag); + result->tests->test[i]->repository_remove_tags = strlist_copy(ctx->tests->test[i]->repository_remove_tags); + if (ctx->tests->test[i]->runtime->environ) { + result->tests->test[i]->runtime->environ = runtime_copy(ctx->tests->test[i]->runtime->environ->data); } - result->tests[i].script = strdup_maybe(ctx->tests[i].script); - result->tests[i].script_setup = strdup_maybe(ctx->tests[i].script_setup); + result->tests->test[i]->script = strdup_maybe(ctx->tests->test[i]->script); + result->tests->test[i]->script_setup = strdup_maybe(ctx->tests->test[i]->script_setup); } return result; @@ -230,19 +230,23 @@ void delivery_free(struct Delivery *ctx) { guard_strlist_free(&ctx->conda.pip_packages_purge); guard_strlist_free(&ctx->conda.wheels_packages); - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { - guard_free(ctx->tests[i].name); - guard_free(ctx->tests[i].version); - guard_free(ctx->tests[i].repository); - guard_free(ctx->tests[i].repository_info_ref); - guard_free(ctx->tests[i].repository_info_tag); - guard_strlist_free(&ctx->tests[i].repository_remove_tags); - guard_free(ctx->tests[i].script); - guard_free(ctx->tests[i].script_setup); - guard_free(ctx->tests[i].build_recipe); + for (size_t i = 0; i < ctx->tests->num_used; i++) { + guard_free(ctx->tests->test[i]->name); + guard_free(ctx->tests->test[i]->version); + guard_free(ctx->tests->test[i]->repository); + guard_free(ctx->tests->test[i]->repository_info_ref); + guard_free(ctx->tests->test[i]->repository_info_tag); + guard_strlist_free(&ctx->tests->test[i]->repository_remove_tags); + guard_free(ctx->tests->test[i]->script); + guard_free(ctx->tests->test[i]->script_setup); + guard_free(ctx->tests->test[i]->build_recipe); // test-specific runtime variables - guard_runtime_free(ctx->tests[i].runtime.environ); + guard_runtime_free(ctx->tests->test[i]->runtime->environ); + guard_free(ctx->tests->test[i]->runtime); + guard_free(ctx->tests->test[i]); } + guard_free(ctx->tests->test); + guard_free(ctx->tests); guard_free(ctx->rules.release_fmt); guard_free(ctx->rules.build_name_fmt); @@ -388,8 +392,8 @@ void delivery_defer_packages(struct Delivery *ctx, int type) { msg(STASIS_MSG_L3, "package '%s': ", package_name); // When spec is present in name, set tests->version to the version detected in the name - for (size_t x = 0; x < sizeof(ctx->tests) / sizeof(ctx->tests[0]) && ctx->tests[x].name != NULL; x++) { - struct Test *test = &ctx->tests[x]; + for (size_t x = 0; x < ctx->tests->num_used; x++) { + struct Test *test = ctx->tests->test[x]; char nametmp[1024] = {0}; strncpy(nametmp, package_name, sizeof(nametmp) - 1); diff --git a/src/lib/delivery/delivery_build.c b/src/lib/delivery/delivery_build.c index 86555bd..0013e96 100644 --- a/src/lib/delivery/delivery_build.c +++ b/src/lib/delivery/delivery_build.c @@ -1,11 +1,11 @@ #include "delivery.h" int delivery_build_recipes(struct Delivery *ctx) { - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { + for (size_t i = 0; i < ctx->tests->num_used; i++) { char *recipe_dir = NULL; - if (ctx->tests[i].build_recipe) { // build a conda recipe - if (recipe_clone(ctx->storage.build_recipes_dir, ctx->tests[i].build_recipe, NULL, &recipe_dir)) { - fprintf(stderr, "Encountered an issue while cloning recipe for: %s\n", ctx->tests[i].name); + if (ctx->tests->test[i]->build_recipe) { // build a conda recipe + if (recipe_clone(ctx->storage.build_recipes_dir, ctx->tests->test[i]->build_recipe, NULL, &recipe_dir)) { + fprintf(stderr, "Encountered an issue while cloning recipe for: %s\n", ctx->tests->test[i]->name); return -1; } if (!recipe_dir) { @@ -15,44 +15,48 @@ int delivery_build_recipes(struct Delivery *ctx) { int recipe_type = recipe_get_type(recipe_dir); if(!pushd(recipe_dir)) { if (RECIPE_TYPE_ASTROCONDA == recipe_type) { - pushd(path_basename(ctx->tests[i].repository)); + pushd(path_basename(ctx->tests->test[i]->repository)); } else if (RECIPE_TYPE_CONDA_FORGE == recipe_type) { pushd("recipe"); } - char recipe_version[100]; - char recipe_buildno[100]; + char recipe_version[200]; + char recipe_buildno[200]; char recipe_git_url[PATH_MAX]; char recipe_git_rev[PATH_MAX]; - char tag[100]; - if (ctx->tests[i].repository_info_tag) { - const int is_long_tag = num_chars(ctx->tests[i].repository_info_tag, '-') > 1; + char tag[100] = {0}; + if (ctx->tests->test[i]->repository_info_tag) { + const int is_long_tag = num_chars(ctx->tests->test[i]->repository_info_tag, '-') > 1; if (is_long_tag) { - const size_t len = strcspn(ctx->tests[i].repository_info_tag, "-"); - strncpy(tag, ctx->tests[i].repository_info_tag, len); + const size_t len = strcspn(ctx->tests->test[i]->repository_info_tag, "-"); + strncpy(tag, ctx->tests->test[i]->repository_info_tag, len); tag[len] = '\0'; } else { - strcpy(tag, ctx->tests[i].repository_info_tag); - tag[strlen(ctx->tests[i].repository_info_tag)] = '\0'; + strncpy(tag, ctx->tests->test[i]->repository_info_tag, sizeof(tag) - 1); + tag[strlen(ctx->tests->test[i]->repository_info_tag)] = '\0'; } } else { - strcpy(tag, ctx->tests[i].version); - } + strcpy(tag, ctx->tests->test[i]->version); + } //sprintf(recipe_version, "{%% set version = GIT_DESCRIBE_TAG ~ \".dev\" ~ GIT_DESCRIBE_NUMBER ~ \"+\" ~ GIT_DESCRIBE_HASH %%}"); - //sprintf(recipe_git_url, " git_url: %s", ctx->tests[i].repository); - //sprintf(recipe_git_rev, " git_rev: %s", ctx->tests[i].version); + //sprintf(recipe_git_url, " git_url: %s", ctx->tests->test[i]->repository); + //sprintf(recipe_git_rev, " git_rev: %s", ctx->tests->test[i]->version); // TODO: Conditionally download archives if github.com is the origin. Else, use raw git_* keys ^^^ + // 03/2026 - How can we know if the repository URL supports archive downloads? + // Perhaps we can key it to the recipe type, because the archive is a requirement imposed + // by conda-forge. Hmm. + sprintf(recipe_version, "{%% set version = \"%s\" %%}", tag); - sprintf(recipe_git_url, " url: %s/archive/refs/tags/{{ version }}.tar.gz", ctx->tests[i].repository); + sprintf(recipe_git_url, " url: %s/archive/refs/tags/{{ version }}.tar.gz", ctx->tests->test[i]->repository); strcpy(recipe_git_rev, ""); sprintf(recipe_buildno, " number: 0"); unsigned flags = REPLACE_TRUNCATE_AFTER_MATCH; //file_replace_text("meta.yaml", "{% set version = ", recipe_version); if (ctx->meta.final) { // remove this. i.e. statis cannot deploy a release to conda-forge - sprintf(recipe_version, "{%% set version = \"%s\" %%}", ctx->tests[i].version); + sprintf(recipe_version, "{%% set version = \"%s\" %%}", ctx->tests->test[i]->version); // TODO: replace sha256 of tagged archive // TODO: leave the recipe unchanged otherwise. in theory this should produce the same conda package hash as conda forge. // For now, remove the sha256 requirement @@ -206,7 +210,7 @@ int manylinux_exec(const char *image, const char *script, const char *copy_to_co goto manylinux_fail; } - if (asprintf(&wheel_paths_filename, "wheel_paths_%s.txt", container_name) < 0) { + if (asprintf(&wheel_paths_filename, "%s/wheel_paths_%s.txt", globals.tmpdir, container_name) < 0) { SYSERROR("%s", "unable to allocate memory for wheel paths file name"); goto manylinux_fail; } @@ -387,28 +391,28 @@ struct StrList *delivery_build_wheels(struct Delivery *ctx) { *spec = '\0'; } - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { - if ((ctx->tests[i].name && !strcmp(name, ctx->tests[i].name)) && (!ctx->tests[i].build_recipe && ctx->tests[i].repository)) { // build from source + for (size_t i = 0; i < ctx->tests->num_used; i++) { + if ((ctx->tests->test[i]->name && !strcmp(name, ctx->tests->test[i]->name)) && (!ctx->tests->test[i]->build_recipe && ctx->tests->test[i]->repository)) { // build from source char srcdir[PATH_MAX]; char wheeldir[PATH_MAX]; memset(srcdir, 0, sizeof(srcdir)); memset(wheeldir, 0, sizeof(wheeldir)); - sprintf(srcdir, "%s/%s", ctx->storage.build_sources_dir, ctx->tests[i].name); - if (git_clone(&proc, ctx->tests[i].repository, srcdir, ctx->tests[i].version)) { + sprintf(srcdir, "%s/%s", ctx->storage.build_sources_dir, ctx->tests->test[i]->name); + if (git_clone(&proc, ctx->tests->test[i]->repository, srcdir, ctx->tests->test[i]->version)) { SYSERROR("Unable to checkout tag '%s' for package '%s' from repository '%s'\n", - ctx->tests[i].version, ctx->tests[i].name, ctx->tests[i].repository); + ctx->tests->test[i]->version, ctx->tests->test[i]->name, ctx->tests->test[i]->repository); return NULL; } - if (!ctx->tests[i].repository_info_tag) { - ctx->tests[i].repository_info_tag = strdup(git_describe(srcdir)); + if (!ctx->tests->test[i]->repository_info_tag) { + ctx->tests->test[i]->repository_info_tag = strdup(git_describe(srcdir)); } - if (!ctx->tests[i].repository_info_ref) { - ctx->tests[i].repository_info_ref = strdup(git_rev_parse(srcdir, ctx->tests[i].version)); + if (!ctx->tests->test[i]->repository_info_ref) { + ctx->tests->test[i]->repository_info_ref = strdup(git_rev_parse(srcdir, ctx->tests->test[i]->version)); } - if (ctx->tests[i].repository_remove_tags && strlist_count(ctx->tests[i].repository_remove_tags)) { - filter_repo_tags(srcdir, ctx->tests[i].repository_remove_tags); + if (ctx->tests->test[i]->repository_remove_tags && strlist_count(ctx->tests->test[i]->repository_remove_tags)) { + filter_repo_tags(srcdir, ctx->tests->test[i]->repository_remove_tags); } if (!pushd(srcdir)) { @@ -430,7 +434,7 @@ struct StrList *delivery_build_wheels(struct Delivery *ctx) { COE_CHECK_ABORT(dep_status, "Unreproducible delivery"); } - strcpy(dname, ctx->tests[i].name); + strcpy(dname, ctx->tests->test[i]->name); tolower_s(dname); sprintf(outdir, "%s/%s", ctx->storage.wheel_artifact_dir, dname); if (mkdirs(outdir, 0755)) { @@ -440,8 +444,8 @@ struct StrList *delivery_build_wheels(struct Delivery *ctx) { } if (use_builder_manylinux) { if (delivery_build_wheels_manylinux(ctx, outdir)) { - fprintf(stderr, "failed to generate wheel package for %s-%s\n", ctx->tests[i].name, - ctx->tests[i].version); + fprintf(stderr, "failed to generate wheel package for %s-%s\n", ctx->tests->test[i]->name, + ctx->tests->test[i]->version); guard_strlist_free(&result); guard_free(cmd); return NULL; @@ -461,8 +465,8 @@ struct StrList *delivery_build_wheels(struct Delivery *ctx) { } if (python_exec(cmd)) { - fprintf(stderr, "failed to generate wheel package for %s-%s\n", ctx->tests[i].name, - ctx->tests[i].version); + fprintf(stderr, "failed to generate wheel package for %s-%s\n", ctx->tests->test[i]->name, + ctx->tests->test[i]->version); guard_strlist_free(&result); guard_free(cmd); return NULL; diff --git a/src/lib/delivery/delivery_install.c b/src/lib/delivery/delivery_install.c index f40a509..02b9dff 100644 --- a/src/lib/delivery/delivery_install.c +++ b/src/lib/delivery/delivery_install.c @@ -2,7 +2,7 @@ static struct Test *requirement_from_test(struct Delivery *ctx, const char *name) { struct Test *result = NULL; - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { + for (size_t i = 0; i < ctx->tests->num_used; i++) { char *package_name = strdup(name); if (package_name) { char *spec = find_version_spec(package_name); @@ -11,8 +11,8 @@ static struct Test *requirement_from_test(struct Delivery *ctx, const char *name } remove_extras(package_name); - if (ctx->tests[i].name && !strcmp(package_name, ctx->tests[i].name)) { - result = &ctx->tests[i]; + if (ctx->tests->test[i]->name && !strcmp(package_name, ctx->tests->test[i]->name)) { + result = ctx->tests->test[i]; guard_free(package_name); break; } diff --git a/src/lib/delivery/delivery_populate.c b/src/lib/delivery/delivery_populate.c index 4ea93c1..d41e3a4 100644 --- a/src/lib/delivery/delivery_populate.c +++ b/src/lib/delivery/delivery_populate.c @@ -193,6 +193,7 @@ static void normalize_ini_list(struct INIFILE **inip, struct StrList **listp, ch (*inip) = ini; (*listp) = list; } + int populate_delivery_ini(struct Delivery *ctx, int render_mode) { struct INIFILE *ini = ctx->_stasis_ini_fp.delivery; struct INIData *rtdata; @@ -277,11 +278,17 @@ int populate_delivery_ini(struct Delivery *ctx, int render_mode) { ctx->conda.pip_packages_defer = strlist_init(); } - for (size_t z = 0, i = 0; i < ini->section_count; i++) { + ctx->tests = tests_init(TEST_NUM_ALLOC_INITIAL); + for (size_t i = 0; i < ini->section_count; i++) { char *section_name = ini->section[i]->key; if (startswith(section_name, "test:")) { union INIVal val; - struct Test *test = &ctx->tests[z]; + struct Test *test = test_init(); + if (!test) { + SYSERROR("%s", "unable to allocate memory for test structure"); + return -1; + } + val.as_char_p = strchr(ini->section[i]->key, ':') + 1; if (val.as_char_p && isempty(val.as_char_p)) { return 1; @@ -299,7 +306,8 @@ int populate_delivery_ini(struct Delivery *ctx, int render_mode) { } test->repository_remove_tags = ini_getval_strlist(ini, section_name, "repository_remove_tags", LINE_SEP, render_mode, &err); test->build_recipe = ini_getval_str(ini, section_name, "build_recipe", render_mode, &err); - test->runtime.environ = ini_getval_strlist(ini, section_name, "runtime", LINE_SEP, render_mode, &err); + + test->runtime->environ = ini_getval_strlist(ini, section_name, "runtime", LINE_SEP, render_mode, &err); const char *timeout_str = ini_getval_str(ini, section_name, "timeout", render_mode, &err); if (timeout_str) { test->timeout = str_to_timeout((char *) timeout_str); @@ -312,7 +320,7 @@ int populate_delivery_ini(struct Delivery *ctx, int render_mode) { return 1; } } - z++; + tests_add(ctx->tests, test); } } diff --git a/src/lib/delivery/delivery_show.c b/src/lib/delivery/delivery_show.c index adfa1be..f4ac825 100644 --- a/src/lib/delivery/delivery_show.c +++ b/src/lib/delivery/delivery_show.c @@ -84,13 +84,13 @@ void delivery_conda_show(struct Delivery *ctx) { void delivery_tests_show(struct Delivery *ctx) { printf("\n====TESTS====\n"); - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { - if (!ctx->tests[i].name) { + for (size_t i = 0; i < ctx->tests->num_used; i++) { + if (!ctx->tests->test[i]->name) { continue; } - printf("%-20s %-20s %s\n", ctx->tests[i].name, - ctx->tests[i].version, - ctx->tests[i].repository); + printf("%-20s %-20s %s\n", ctx->tests->test[i]->name, + ctx->tests->test[i]->version, + ctx->tests->test[i]->repository); } } diff --git a/src/lib/delivery/delivery_test.c b/src/lib/delivery/delivery_test.c index 500ade9..3ba9d56 100644 --- a/src/lib/delivery/delivery_test.c +++ b/src/lib/delivery/delivery_test.c @@ -1,5 +1,59 @@ #include "delivery.h" +struct Tests *tests_init(const size_t num_tests) { + struct Tests *tests = calloc(1, sizeof(*tests)); + if (!tests) { + return NULL; + } + + tests->test = calloc(num_tests, sizeof(*tests->test)); + if (!tests->test) { + return NULL; + } + tests->num_used = 0; + tests->num_alloc = num_tests; + + return tests; +} + +int tests_add(struct Tests *tests, struct Test *x) { + if (tests->num_used >= tests->num_alloc) { +#ifdef DEBUG + const size_t old_alloc = tests->num_alloc; +#endif + struct Test **tmp = realloc(tests->test, tests->num_alloc++ * sizeof(*tests->test)); + SYSDEBUG("Increasing size of test array: %zu -> %zu", old_alloc, tests->num_alloc); + if (!tmp) { + SYSDEBUG("Failed to allocate %zu bytes for test array", tests->num_alloc * sizeof(*tests->test)); + return -1; + } + tests->test = tmp; + } + + SYSDEBUG("Adding test: '%s'", x->name); + tests->test[tests->num_used++] = x; + return 0; +} + +struct Test *test_init() { + struct Test *result = calloc(1, sizeof(*result)); + result->runtime = calloc(1, sizeof(*result->runtime)); + + return result; +} + +void test_free(struct Test **x) { + struct Test *test = *x; + guard_free(test); +} + +void tests_free(struct Tests **x) { + for (size_t i = 0; i < (*x)->num_alloc; i++) { + test_free(&(*x)->test[i]); + } + guard_free((*x)->test); +} + void delivery_tests_run(struct Delivery *ctx) { static const int SETUP = 0; static const int PARALLEL = 1; @@ -16,7 +70,7 @@ void delivery_tests_run(struct Delivery *ctx) { // amount of debug information. snprintf(globals.workaround.conda_reactivate, PATH_MAX - 1, "\nset +x; mamba activate ${CONDA_DEFAULT_ENV}; set -x\n"); - if (!ctx->tests[0].name) { + if (!ctx->tests || !ctx->tests->num_used) { msg(STASIS_MSG_WARN | STASIS_MSG_L2, "no tests are defined!\n"); } else { pool[PARALLEL] = mp_pool_init("parallel", ctx->storage.tmpdir); @@ -60,8 +114,8 @@ void delivery_tests_run(struct Delivery *ctx) { // Iterate over our test records, retrieving the source code for each package, and assigning its scripted tasks // to the appropriate processing pool - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { - struct Test *test = &ctx->tests[i]; + for (size_t i = 0; i < ctx->tests->num_used; i++) { + struct Test *test = ctx->tests->test[i]; if (!test->name && !test->repository && !test->script) { // skip unused test records continue; @@ -181,8 +235,8 @@ void delivery_tests_run(struct Delivery *ctx) { // Configure "script_setup" tasks // Directories should exist now, so no need to go through initializing everything all over again. - for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) { - struct Test *test = &ctx->tests[i]; + for (size_t i = 0; i < ctx->tests->num_used; i++) { + const struct Test *test = ctx->tests->test[i]; if (test->script_setup) { char destdir[PATH_MAX]; sprintf(destdir, "%s/%s", ctx->storage.build_sources_dir, path_basename(test->repository)); |