aboutsummaryrefslogtreecommitdiff
path: root/src/lib/delivery
diff options
context:
space:
mode:
Diffstat (limited to 'src/lib/delivery')
-rw-r--r--src/lib/delivery/CMakeLists.txt1
-rw-r--r--src/lib/delivery/delivery.c290
-rw-r--r--src/lib/delivery/delivery_artifactory.c50
-rw-r--r--src/lib/delivery/delivery_build.c388
-rw-r--r--src/lib/delivery/delivery_conda.c47
-rw-r--r--src/lib/delivery/delivery_docker.c44
-rw-r--r--src/lib/delivery/delivery_export.c77
-rw-r--r--src/lib/delivery/delivery_init.c165
-rw-r--r--src/lib/delivery/delivery_install.c303
-rw-r--r--src/lib/delivery/delivery_populate.c105
-rw-r--r--src/lib/delivery/delivery_postprocess.c70
-rw-r--r--src/lib/delivery/delivery_show.c12
-rw-r--r--src/lib/delivery/delivery_test.c143
-rw-r--r--src/lib/delivery/include/delivery.h140
14 files changed, 1501 insertions, 334 deletions
diff --git a/src/lib/delivery/CMakeLists.txt b/src/lib/delivery/CMakeLists.txt
index 78ed20f..559b2dc 100644
--- a/src/lib/delivery/CMakeLists.txt
+++ b/src/lib/delivery/CMakeLists.txt
@@ -1,4 +1,5 @@
add_library(stasis_delivery STATIC
+ delivery_export.c
delivery_postprocess.c
delivery_conda.c
delivery_docker.c
diff --git a/src/lib/delivery/delivery.c b/src/lib/delivery/delivery.c
index d480ab4..dc9e2ce 100644
--- a/src/lib/delivery/delivery.c
+++ b/src/lib/delivery/delivery.c
@@ -1,8 +1,203 @@
#include "delivery.h"
+#include "conda.h"
+
+struct Delivery *delivery_duplicate(struct Delivery *ctx) {
+ struct Delivery *result = calloc(1, sizeof(*result));
+ if (!result) {
+ return NULL;
+ }
+ // Conda
+ result->conda.conda_packages = strlist_copy(ctx->conda.conda_packages);
+ result->conda.conda_packages_defer = strlist_copy(ctx->conda.conda_packages_defer);
+ result->conda.conda_packages_purge = strlist_copy(ctx->conda.conda_packages_purge);
+ result->conda.pip_packages = strlist_copy(ctx->conda.pip_packages);
+ result->conda.pip_packages_defer = strlist_copy(ctx->conda.pip_packages_defer);
+ result->conda.pip_packages_purge = strlist_copy(ctx->conda.pip_packages_purge);
+ result->conda.wheels_packages = strlist_copy(ctx->conda.wheels_packages);
+ result->conda.installer_arch = strdup_maybe(ctx->conda.installer_arch);
+ result->conda.installer_baseurl = strdup_maybe(ctx->conda.installer_baseurl);
+ result->conda.installer_name = strdup_maybe(ctx->conda.installer_name);
+ result->conda.installer_path = strdup_maybe(ctx->conda.installer_path);
+ result->conda.installer_platform = strdup_maybe(ctx->conda.installer_platform);
+ result->conda.installer_version = strdup_maybe(ctx->conda.installer_version);
+ result->conda.tool_build_version = strdup_maybe(ctx->conda.tool_build_version);
+ result->conda.tool_version = strdup_maybe(ctx->conda.tool_version);
+
+ // Info
+ result->info.build_name = strdup_maybe(ctx->info.build_name);
+ result->info.build_number = strdup_maybe(ctx->info.build_number);
+ result->info.release_name = strdup_maybe(ctx->info.release_name);
+ result->info.time_info = ctx->info.time_info;
+ result->info.time_now = ctx->info.time_now;
+ result->info.time_str_epoch = strdup_maybe(ctx->info.time_str_epoch);
+
+ // Meta
+ result->meta.name = strdup_maybe(ctx->meta.name);
+ result->meta.based_on = strdup_maybe(ctx->meta.based_on);
+ result->meta.codename = strdup_maybe(ctx->meta.codename);
+ result->meta.mission = strdup_maybe(ctx->meta.mission);
+ result->meta.final = ctx->meta.final;
+ result->meta.python = strdup_maybe(ctx->meta.python);
+ result->meta.python_compact = strdup_maybe(ctx->meta.python_compact);
+ result->meta.rc = ctx->meta.rc;
+ result->meta.version = strdup_maybe(ctx->meta.version);
+
+ // Rules
+ result->rules.build_name_fmt = strdup_maybe(ctx->rules.build_name_fmt);
+ result->rules.build_number_fmt = strdup_maybe(ctx->rules.build_number_fmt);
+ // Unused member?
+ result->rules.enable_final = ctx->rules.enable_final;
+ result->rules.release_fmt = ctx->rules.release_fmt;
+ // TODO: need content duplication function
+ memcpy(&result->rules.content, &ctx->rules.content, sizeof(ctx->rules.content));
+
+ if (ctx->rules._handle) {
+ SYSDEBUG("duplicating INIFILE handle - BEGIN");
+ result->rules._handle = malloc(sizeof(*result->rules._handle));
+ if (!result->rules._handle) {
+ SYSERROR("unable to allocate space for INIFILE handle");
+ SYSERROR("%s", "unable to allocate space for INIFILE handle");
+ delivery_free(ctx);
+ return NULL;
+ }
+ result->rules._handle->section = malloc(ctx->rules._handle->section_count * sizeof(**ctx->rules._handle->section));
+ if (!result->rules._handle->section) {
+ guard_free(result->rules._handle);
+ SYSERROR("unable to allocate space for INIFILE section");
+ SYSERROR("%s", "unable to allocate space for INIFILE section");
+ delivery_free(ctx);
+ return NULL;
+ }
+ memcpy(result->rules._handle, &ctx->rules._handle, sizeof(*ctx->rules._handle));
+ SYSDEBUG("duplicating INIFILE handle - END");
+ }
+
+ // Runtime
+ if (ctx->runtime.environ) {
+ result->runtime.environ = runtime_copy(ctx->runtime.environ->data);
+ }
+
+ // Storage
+ result->storage.tools_dir = strdup_maybe(ctx->storage.tools_dir);
+ result->storage.package_dir = strdup_maybe(ctx->storage.package_dir);
+ result->storage.results_dir = strdup_maybe(ctx->storage.results_dir);
+ result->storage.output_dir = strdup_maybe(ctx->storage.output_dir);
+ result->storage.cfgdump_dir = strdup_maybe(ctx->storage.cfgdump_dir);
+ result->storage.delivery_dir = strdup_maybe(ctx->storage.delivery_dir);
+ result->storage.meta_dir = strdup_maybe(ctx->storage.meta_dir);
+ result->storage.mission_dir = strdup_maybe(ctx->storage.mission_dir);
+ result->storage.root = strdup_maybe(ctx->storage.root);
+ result->storage.tmpdir = strdup_maybe(ctx->storage.tmpdir);
+ result->storage.build_dir = strdup_maybe(ctx->storage.build_dir);
+ result->storage.build_docker_dir = strdup_maybe(ctx->storage.build_docker_dir);
+ result->storage.build_recipes_dir = strdup_maybe(ctx->storage.build_recipes_dir);
+ result->storage.build_sources_dir = strdup_maybe(ctx->storage.build_sources_dir);
+ result->storage.build_testing_dir = strdup_maybe(ctx->storage.build_testing_dir);
+ result->storage.conda_artifact_dir = strdup_maybe(ctx->storage.conda_artifact_dir);
+ result->storage.conda_install_prefix = strdup_maybe(ctx->storage.conda_install_prefix);
+ result->storage.conda_staging_dir = strdup_maybe(ctx->storage.conda_staging_dir);
+ result->storage.conda_staging_url = strdup_maybe(ctx->storage.conda_staging_url);
+ result->storage.docker_artifact_dir = strdup_maybe(ctx->storage.docker_artifact_dir);
+ result->storage.wheel_artifact_dir = strdup_maybe(ctx->storage.wheel_artifact_dir);
+ result->storage.wheel_staging_url = strdup_maybe(ctx->storage.wheel_staging_url);
+
+ result->system.arch = strdup_maybe(ctx->system.arch);
+ if (ctx->system.platform) {
+ result->system.platform = malloc(DELIVERY_PLATFORM_MAX * sizeof(*result->system.platform));
+ if (!result->system.platform) {
+ SYSERROR("unable to allocate space for system platform array");
+ SYSERROR("%s", "unable to allocate space for system platform array");
+ delivery_free(ctx);
+ return NULL;
+ }
+ for (size_t i = 0; i < DELIVERY_PLATFORM_MAX; i++) {
+ result->system.platform[i] = strdup_maybe(ctx->system.platform[i]);
+ if (!result->system.platform[i]) {
+ SYSERROR("%s", "unable to allocate record in system platform array");
+ guard_array_n_free(result->system.platform, DELIVERY_PLATFORM_MAX);
+ delivery_free(ctx);
+ return NULL;
+ }
+ }
+ }
+
+ // Docker
+ result->deploy.docker.build_args = strlist_copy(ctx->deploy.docker.build_args);
+ result->deploy.docker.tags = strlist_copy(ctx->deploy.docker.tags);
+ result->deploy.docker.capabilities = ctx->deploy.docker.capabilities;
+ result->deploy.docker.dockerfile = strdup_maybe(ctx->deploy.docker.dockerfile);
+ result->deploy.docker.image_compression = strdup_maybe(ctx->deploy.docker.image_compression);
+ result->deploy.docker.registry = strdup_maybe(ctx->deploy.docker.registry);
+ result->deploy.docker.test_script = strdup_maybe(ctx->deploy.docker.test_script);
+
+ // Jfrog
+ // TODO: break out into a separate a function
+ for (size_t i = 0; i < sizeof(ctx->deploy.jfrog) / sizeof(ctx->deploy.jfrog[0]); i++) {
+ result->deploy.jfrog[i].dest = strdup_maybe(ctx->deploy.jfrog[i].dest);
+ result->deploy.jfrog[i].files = strlist_copy(ctx->deploy.jfrog[i].files);
+ result->deploy.jfrog[i].repo = strdup_maybe(ctx->deploy.jfrog[i].repo);
+ result->deploy.jfrog[i].upload_ctx.ant = ctx->deploy.jfrog[i].upload_ctx.ant;
+ result->deploy.jfrog[i].upload_ctx.archive = ctx->deploy.jfrog[i].upload_ctx.archive;
+ result->deploy.jfrog[i].upload_ctx.build_name = ctx->deploy.jfrog[i].upload_ctx.build_name;
+ result->deploy.jfrog[i].upload_ctx.build_number = ctx->deploy.jfrog[i].upload_ctx.build_number;
+ result->deploy.jfrog[i].upload_ctx.deb = ctx->deploy.jfrog[i].upload_ctx.deb;
+ result->deploy.jfrog[i].upload_ctx.detailed_summary = ctx->deploy.jfrog[i].upload_ctx.detailed_summary;
+ result->deploy.jfrog[i].upload_ctx.dry_run = ctx->deploy.jfrog[i].upload_ctx.dry_run;
+ result->deploy.jfrog[i].upload_ctx.exclusions = strdup_maybe(ctx->deploy.jfrog[i].upload_ctx.exclusions);
+ result->deploy.jfrog[i].upload_ctx.explode = ctx->deploy.jfrog[i].upload_ctx.explode;
+ result->deploy.jfrog[i].upload_ctx.fail_no_op = ctx->deploy.jfrog[i].upload_ctx.fail_no_op;
+ result->deploy.jfrog[i].upload_ctx.flat = ctx->deploy.jfrog[i].upload_ctx.flat;
+ result->deploy.jfrog[i].upload_ctx.include_dirs = ctx->deploy.jfrog[i].upload_ctx.include_dirs;
+ result->deploy.jfrog[i].upload_ctx.module = strdup_maybe(ctx->deploy.jfrog[i].upload_ctx.module);
+ result->deploy.jfrog[i].upload_ctx.project = strdup_maybe(ctx->deploy.jfrog[i].upload_ctx.project);
+ result->deploy.jfrog[i].upload_ctx.quiet = ctx->deploy.jfrog[i].upload_ctx.quiet;
+ result->deploy.jfrog[i].upload_ctx.recursive = ctx->deploy.jfrog[i].upload_ctx.recursive;
+ result->deploy.jfrog[i].upload_ctx.regexp = ctx->deploy.jfrog[i].upload_ctx.regexp;
+ result->deploy.jfrog[i].upload_ctx.retries = ctx->deploy.jfrog[i].upload_ctx.retries;
+ result->deploy.jfrog[i].upload_ctx.retry_wait_time = ctx->deploy.jfrog[i].upload_ctx.retry_wait_time;
+ result->deploy.jfrog[i].upload_ctx.spec = strdup_maybe(ctx->deploy.jfrog[i].upload_ctx.spec);
+ result->deploy.jfrog[i].upload_ctx.spec_vars = strdup_maybe(ctx->deploy.jfrog[i].upload_ctx.spec_vars);
+ result->deploy.jfrog[i].upload_ctx.symlinks = ctx->deploy.jfrog[i].upload_ctx.symlinks;
+ result->deploy.jfrog[i].upload_ctx.sync_deletes = ctx->deploy.jfrog[i].upload_ctx.sync_deletes;
+ result->deploy.jfrog[i].upload_ctx.target_props = strdup_maybe(ctx->deploy.jfrog[i].upload_ctx.target_props);
+ result->deploy.jfrog[i].upload_ctx.threads = ctx->deploy.jfrog[i].upload_ctx.threads;
+ result->deploy.jfrog[i].upload_ctx.workaround_parent_only = ctx->deploy.jfrog[i].upload_ctx.workaround_parent_only;
+ }
+
+ result->deploy.jfrog_auth.access_token = strdup_maybe(ctx->deploy.jfrog_auth.access_token);
+ result->deploy.jfrog_auth.client_cert_key_path = strdup_maybe(ctx->deploy.jfrog_auth.client_cert_key_path);
+ result->deploy.jfrog_auth.client_cert_path = strdup_maybe(ctx->deploy.jfrog_auth.client_cert_path);
+ result->deploy.jfrog_auth.insecure_tls = ctx->deploy.jfrog_auth.insecure_tls;
+ result->deploy.jfrog_auth.password = strdup_maybe(ctx->deploy.jfrog_auth.password);
+ result->deploy.jfrog_auth.server_id = strdup_maybe(ctx->deploy.jfrog_auth.server_id);
+ result->deploy.jfrog_auth.ssh_key_path = strdup_maybe(ctx->deploy.jfrog_auth.ssh_key_path);
+ result->deploy.jfrog_auth.ssh_passphrase = strdup_maybe(ctx->deploy.jfrog_auth.ssh_passphrase);
+ result->deploy.jfrog_auth.url = strdup_maybe(ctx->deploy.jfrog_auth.url);
+ result->deploy.jfrog_auth.user = strdup_maybe(ctx->deploy.jfrog_auth.user);
+
+ for (size_t i = 0; result->tests && i < result->tests->num_used; i++) {
+ result->tests->test[i]->disable = ctx->tests->test[i]->disable;
+ result->tests->test[i]->parallel = ctx->tests->test[i]->parallel;
+ result->tests->test[i]->build_recipe = strdup_maybe(ctx->tests->test[i]->build_recipe);
+ result->tests->test[i]->name = strdup_maybe(ctx->tests->test[i]->name);
+ result->tests->test[i]->version = strdup_maybe(ctx->tests->test[i]->version);
+ result->tests->test[i]->repository = strdup_maybe(ctx->tests->test[i]->repository);
+ result->tests->test[i]->repository_info_ref = strdup_maybe(ctx->tests->test[i]->repository_info_ref);
+ result->tests->test[i]->repository_info_tag = strdup_maybe(ctx->tests->test[i]->repository_info_tag);
+ result->tests->test[i]->repository_remove_tags = strlist_copy(ctx->tests->test[i]->repository_remove_tags);
+ if (ctx->tests->test[i]->runtime->environ) {
+ result->tests->test[i]->runtime->environ = runtime_copy(ctx->tests->test[i]->runtime->environ->data);
+ }
+ result->tests->test[i]->script = strdup_maybe(ctx->tests->test[i]->script);
+ result->tests->test[i]->script_setup = strdup_maybe(ctx->tests->test[i]->script_setup);
+ }
+
+ return result;
+}
void delivery_free(struct Delivery *ctx) {
guard_free(ctx->system.arch);
- guard_array_free(ctx->system.platform);
+ guard_array_n_free(ctx->system.platform, DELIVERY_PLATFORM_MAX);
guard_free(ctx->meta.name);
guard_free(ctx->meta.version);
guard_free(ctx->meta.codename);
@@ -57,23 +252,14 @@ void delivery_free(struct Delivery *ctx) {
guard_strlist_free(&ctx->conda.pip_packages_purge);
guard_strlist_free(&ctx->conda.wheels_packages);
- for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) {
- guard_free(ctx->tests[i].name);
- guard_free(ctx->tests[i].version);
- guard_free(ctx->tests[i].repository);
- guard_free(ctx->tests[i].repository_info_ref);
- guard_free(ctx->tests[i].repository_info_tag);
- guard_strlist_free(&ctx->tests[i].repository_remove_tags);
- guard_free(ctx->tests[i].script);
- guard_free(ctx->tests[i].script_setup);
- guard_free(ctx->tests[i].build_recipe);
- // test-specific runtime variables
- guard_runtime_free(ctx->tests[i].runtime.environ);
- }
+ tests_free(&ctx->tests);
guard_free(ctx->rules.release_fmt);
guard_free(ctx->rules.build_name_fmt);
guard_free(ctx->rules.build_number_fmt);
+ if (ctx->rules._handle) {
+ ini_free(&ctx->rules._handle);
+ }
guard_free(ctx->deploy.docker.test_script);
guard_free(ctx->deploy.docker.registry);
@@ -104,8 +290,11 @@ void delivery_free(struct Delivery *ctx) {
guard_free(ctx->_stasis_ini_fp.mission_path);
}
-int delivery_format_str(struct Delivery *ctx, char **dest, const char *fmt) {
- size_t fmt_len = strlen(fmt);
+int delivery_format_str(struct Delivery *ctx, char **dest, size_t maxlen, const char *fmt) {
+ const size_t fmt_len = strlen(fmt);
+ if (maxlen < 1) {
+ maxlen = 1;
+ }
if (!*dest) {
*dest = calloc(STASIS_NAME_MAX, sizeof(**dest));
@@ -119,47 +308,47 @@ int delivery_format_str(struct Delivery *ctx, char **dest, const char *fmt) {
i++;
switch (fmt[i]) {
case 'n': // name
- strcat(*dest, ctx->meta.name);
+ strncat(*dest, ctx->meta.name, maxlen - 1);
break;
case 'c': // codename
- strcat(*dest, ctx->meta.codename);
+ strncat(*dest, ctx->meta.codename, maxlen - 1);
break;
case 'm': // mission
- strcat(*dest, ctx->meta.mission);
+ strncat(*dest, ctx->meta.mission, maxlen - 1);
break;
case 'r': // revision
- sprintf(*dest + strlen(*dest), "%d", ctx->meta.rc);
+ snprintf(*dest + strlen(*dest), maxlen - strlen(*dest), "%d", ctx->meta.rc);
break;
case 'R': // "final"-aware revision
if (ctx->meta.final)
- strcat(*dest, "final");
+ strncat(*dest, "final", maxlen);
else
- sprintf(*dest + strlen(*dest), "%d", ctx->meta.rc);
+ snprintf(*dest + strlen(*dest), maxlen - strlen(*dest), "%d", ctx->meta.rc);
break;
case 'v': // version
- strcat(*dest, ctx->meta.version);
+ strncat(*dest, ctx->meta.version, maxlen - 1);
break;
case 'P': // python version
- strcat(*dest, ctx->meta.python);
+ strncat(*dest, ctx->meta.python, maxlen - 1);
break;
case 'p': // python version major/minor
- strcat(*dest, ctx->meta.python_compact);
+ strncat(*dest, ctx->meta.python_compact, maxlen - 1);
break;
case 'a': // system architecture name
- strcat(*dest, ctx->system.arch);
+ strncat(*dest, ctx->system.arch, maxlen - 1);
break;
case 'o': // system platform (OS) name
- strcat(*dest, ctx->system.platform[DELIVERY_PLATFORM_RELEASE]);
+ strncat(*dest, ctx->system.platform[DELIVERY_PLATFORM_RELEASE], maxlen - 1);
break;
case 't': // unix epoch
- sprintf(*dest + strlen(*dest), "%ld", ctx->info.time_now);
+ snprintf(*dest + strlen(*dest), maxlen - strlen(*dest), "%ld", ctx->info.time_now);
break;
default: // unknown formatter, write as-is
- sprintf(*dest + strlen(*dest), "%c%c", fmt[i - 1], fmt[i]);
+ snprintf(*dest + strlen(*dest), maxlen - strlen(*dest), "%c%c", fmt[i - 1], fmt[i]);
break;
}
} else { // write non-format text
- sprintf(*dest + strlen(*dest), "%c", fmt[i]);
+ snprintf(*dest + strlen(*dest), maxlen - strlen(*dest), "%c", fmt[i]);
}
}
return 0;
@@ -174,15 +363,17 @@ void delivery_defer_packages(struct Delivery *ctx, int type) {
if (DEFER_CONDA == type) {
dataptr = ctx->conda.conda_packages;
deferred = ctx->conda.conda_packages_defer;
- strcpy(mode, "conda");
+ strncpy(mode, "conda", sizeof(mode) - 1);
} else if (DEFER_PIP == type) {
dataptr = ctx->conda.pip_packages;
deferred = ctx->conda.pip_packages_defer;
- strcpy(mode, "pip");
+ strncpy(mode, "pip", sizeof(mode) - 1);
} else {
- SYSERROR("BUG: type %d does not map to a supported package manager!\n", type);
+ SYSERROR("BUG: type %d does not map to a supported package manager!", type);
exit(1);
}
+ mode[sizeof(mode) - 1] = '\0';
+
msg(STASIS_MSG_L2, "Filtering %s packages by test definition...\n", mode);
struct StrList *filtered = NULL;
@@ -199,7 +390,7 @@ void delivery_defer_packages(struct Delivery *ctx, int type) {
// Compile a list of packages that are *also* to be tested.
char *spec_begin = strpbrk(name, "@~=<>!");
char *spec_end = spec_begin;
- char package_name[255] = {0};
+ char package_name[STASIS_NAME_MAX] = {0};
if (spec_end) {
// A version is present in the package name. Jump past operator(s).
@@ -207,25 +398,37 @@ void delivery_defer_packages(struct Delivery *ctx, int type) {
spec_end++;
}
strncpy(package_name, name, spec_begin - name);
+ package_name[spec_begin - name] = '\0';
} else {
strncpy(package_name, name, sizeof(package_name) - 1);
+ package_name[sizeof(package_name) - 1] = '\0';
}
remove_extras(package_name);
msg(STASIS_MSG_L3, "package '%s': ", package_name);
// When spec is present in name, set tests->version to the version detected in the name
- for (size_t x = 0; x < sizeof(ctx->tests) / sizeof(ctx->tests[0]) && ctx->tests[x].name != NULL; x++) {
- struct Test *test = &ctx->tests[x];
- char nametmp[1024] = {0};
+ for (size_t x = 0; x < ctx->tests->num_used; x++) {
+ struct Test *test = ctx->tests->test[x];
+ char nametmp[STASIS_NAME_MAX] = {0};
strncpy(nametmp, package_name, sizeof(nametmp) - 1);
+ nametmp[sizeof(nametmp) - 1] = '\0';
+
// Is the [test:NAME] in the package name?
if (!strcmp(nametmp, test->name)) {
// Override test->version when a version is provided by the (pip|conda)_package list item
guard_free(test->version);
if (spec_begin && spec_end) {
- test->version = strdup(spec_end);
+ char *version_at = strrchr(spec_end, '@');
+ if (version_at) {
+ if (strlen(version_at)) {
+ version_at++;
+ }
+ test->version = strdup(version_at);
+ } else {
+ test->version = strdup(spec_end);
+ }
} else {
// There are too many possible default branches nowadays: master, main, develop, xyz, etc.
// HEAD is a safe bet.
@@ -233,6 +436,9 @@ void delivery_defer_packages(struct Delivery *ctx, int type) {
}
// Is the list item a git+schema:// URL?
+ // TODO: nametmp is just the name so this will never work. but do we want it to? this looks like
+ // TODO: an unsafe feature. We shouldn't be able to change what's in the config. we should
+ // TODO: be getting what we asked for, or exit the program with an error.
if (strstr(nametmp, "git+") && strstr(nametmp, "://")) {
char *xrepo = strstr(nametmp, "+");
if (xrepo) {
@@ -252,13 +458,13 @@ void delivery_defer_packages(struct Delivery *ctx, int type) {
int upstream_exists = 0;
if (DEFER_PIP == type) {
- upstream_exists = pkg_index_provides(PKG_USE_PIP, PYPI_INDEX_DEFAULT, name);
+ upstream_exists = pkg_index_provides(PKG_USE_PIP, PYPI_INDEX_DEFAULT, name, ctx->storage.tmpdir);
} else if (DEFER_CONDA == type) {
- upstream_exists = pkg_index_provides(PKG_USE_CONDA, NULL, name);
+ upstream_exists = pkg_index_provides(PKG_USE_CONDA, NULL, name, ctx->storage.tmpdir);
}
if (PKG_INDEX_PROVIDES_FAILED(upstream_exists)) {
- fprintf(stderr, "%s's existence command failed for '%s': %s\n",
+ SYSERROR("%s's existence command failed for '%s': %s",
mode, name, pkg_index_provides_strerror(upstream_exists));
exit(1);
}
@@ -283,7 +489,7 @@ void delivery_defer_packages(struct Delivery *ctx, int type) {
}
if (!strlist_count(deferred)) {
- msg(STASIS_MSG_WARN | STASIS_MSG_L2, "No %s packages were filtered by test definitions\n", mode);
+ SYSWARN("No %s packages were filtered by test definitions", mode);
} else {
if (DEFER_CONDA == type) {
guard_strlist_free(&ctx->conda.conda_packages);
diff --git a/src/lib/delivery/delivery_artifactory.c b/src/lib/delivery/delivery_artifactory.c
index 97db752..d7a5457 100644
--- a/src/lib/delivery/delivery_artifactory.c
+++ b/src/lib/delivery/delivery_artifactory.c
@@ -4,8 +4,12 @@ int delivery_init_artifactory(struct Delivery *ctx) {
int status = 0;
char dest[PATH_MAX] = {0};
char filepath[PATH_MAX] = {0};
- snprintf(dest, sizeof(dest) - 1, "%s/bin", ctx->storage.tools_dir);
- snprintf(filepath, sizeof(dest) - 1, "%s/bin/jf", ctx->storage.tools_dir);
+
+ SYSDEBUG("Initializing artifactory tools");
+ snprintf(dest, sizeof(dest), "%s/bin", ctx->storage.tools_dir);
+ SYSDEBUG("dest=%s", dest);
+ snprintf(filepath, sizeof(dest), "%s/bin/jf", ctx->storage.tools_dir);
+ SYSDEBUG("filepath=%s", filepath);
if (!access(filepath, F_OK)) {
// already have it
@@ -13,6 +17,7 @@ int delivery_init_artifactory(struct Delivery *ctx) {
goto delivery_init_artifactory_envsetup;
}
+ SYSDEBUG("Assign platform");
char *platform = ctx->system.platform[DELIVERY_PLATFORM];
msg(STASIS_MSG_L3, "Downloading %s for %s %s\n", globals.jfrog.remote_filename, platform, ctx->system.arch);
if ((status = artifactory_download_cli(dest,
@@ -32,7 +37,7 @@ int delivery_init_artifactory(struct Delivery *ctx) {
// JFROG_CLI_HOME_DIR is where .jfrog is stored
char path[PATH_MAX] = {0};
- snprintf(path, sizeof(path) - 1, "%s/.jfrog", ctx->storage.build_dir);
+ snprintf(path, sizeof(path), "%s/.jfrog", ctx->storage.build_dir);
setenv("JFROG_CLI_HOME_DIR", path, 1);
// JFROG_CLI_TEMP_DIR is where the obvious is stored
@@ -44,7 +49,7 @@ int delivery_artifact_upload(struct Delivery *ctx) {
int status = 0;
if (jfrt_auth_init(&ctx->deploy.jfrog_auth)) {
- fprintf(stderr, "Failed to initialize Artifactory authentication context\n");
+ SYSERROR("Failed to initialize Artifactory authentication context");
return -1;
}
@@ -55,9 +60,10 @@ int delivery_artifact_upload(struct Delivery *ctx) {
jfrt_upload_init(&ctx->deploy.jfrog[i].upload_ctx);
if (!globals.jfrog.repo) {
- msg(STASIS_MSG_WARN, "Artifactory repository path is not configured!\n");
- fprintf(stderr, "set STASIS_JF_REPO environment variable...\nOr append to configuration file:\n\n");
- fprintf(stderr, "[deploy:artifactory]\nrepo = example/generic/repo/path\n\n");
+ SYSWARN("Artifactory repository path is not configured!");
+ SYSWARN("set STASIS_JF_REPO environment variable...\nOr append to configuration file:");
+ SYSWARN("");
+ SYSWARN("[deploy:artifactory]\nrepo = example/generic/repo/path\n");
status++;
break;
} else if (!ctx->deploy.jfrog[i].repo) {
@@ -66,7 +72,7 @@ int delivery_artifact_upload(struct Delivery *ctx) {
if (!ctx->deploy.jfrog[i].repo || isempty(ctx->deploy.jfrog[i].repo) || !strlen(ctx->deploy.jfrog[i].repo)) {
// Unlikely to trigger if the config parser is working correctly
- msg(STASIS_MSG_ERROR, "Artifactory repository path is empty. Cannot continue.\n");
+ SYSERROR("Artifactory repository path is empty. Cannot continue.");
status++;
break;
}
@@ -76,7 +82,7 @@ int delivery_artifact_upload(struct Delivery *ctx) {
ctx->deploy.jfrog[i].upload_ctx.build_number = ctx->info.build_number;
if (jfrog_cli_rt_ping(&ctx->deploy.jfrog_auth)) {
- msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Unable to contact artifactory server: %s\n", ctx->deploy.jfrog_auth.url);
+ SYSERROR("Unable to contact artifactory server: %s", ctx->deploy.jfrog_auth.url);
return -1;
}
@@ -84,8 +90,8 @@ int delivery_artifact_upload(struct Delivery *ctx) {
for (size_t f = 0; f < strlist_count(ctx->deploy.jfrog[i].files); f++) {
char dest[PATH_MAX] = {0};
char files[PATH_MAX] = {0};
- snprintf(dest, sizeof(dest) - 1, "%s/%s", ctx->deploy.jfrog[i].repo, ctx->deploy.jfrog[i].dest);
- snprintf(files, sizeof(files) - 1, "%s", strlist_item(ctx->deploy.jfrog[i].files, f));
+ snprintf(dest, sizeof(dest), "%s/%s", ctx->deploy.jfrog[i].repo, ctx->deploy.jfrog[i].dest);
+ snprintf(files, sizeof(files), "%s", strlist_item(ctx->deploy.jfrog[i].files, f));
status += jfrog_cli_rt_upload(&ctx->deploy.jfrog_auth, &ctx->deploy.jfrog[i].upload_ctx, files, dest);
}
}
@@ -99,7 +105,7 @@ int delivery_artifact_upload(struct Delivery *ctx) {
ctx->deploy.jfrog[0].upload_ctx.build_number);
}
} else {
- msg(STASIS_MSG_WARN | STASIS_MSG_L2, "Artifactory build info upload is disabled by CLI argument\n");
+ SYSWARN("Artifactory build info upload is disabled by CLI argument");
}
return status;
@@ -107,7 +113,7 @@ int delivery_artifact_upload(struct Delivery *ctx) {
int delivery_mission_render_files(struct Delivery *ctx) {
if (!ctx->storage.mission_dir) {
- fprintf(stderr, "Mission directory is not configured. Context not initialized?\n");
+ SYSERROR("Mission directory is not configured. Context not initialized?");
return -1;
}
struct Data {
@@ -119,7 +125,7 @@ int delivery_mission_render_files(struct Delivery *ctx) {
memset(&data, 0, sizeof(data));
data.src = calloc(PATH_MAX, sizeof(*data.src));
if (!data.src) {
- perror("data.src");
+ SYSERROR("unable to allocate memory for data.src: %s", strerror(errno));
return -1;
}
@@ -134,7 +140,7 @@ int delivery_mission_render_files(struct Delivery *ctx) {
guard_free(data.src);
return 1;
}
- sprintf(data.src, "%s/%s/%s", ctx->storage.mission_dir, ctx->meta.mission, val.as_char_p);
+ snprintf(data.src, PATH_MAX, "%s/%s/%s", ctx->storage.mission_dir, ctx->meta.mission, val.as_char_p);
msg(STASIS_MSG_L2, "%s\n", data.src);
int err = 0;
@@ -149,14 +155,14 @@ int delivery_mission_render_files(struct Delivery *ctx) {
char *contents = calloc(st.st_size + 1, sizeof(*contents));
if (!contents) {
- perror("template file contents");
+ SYSERROR("unable to allocate memory for template file contents: %s", strerror(errno));
guard_free(data.dest);
continue;
}
FILE *fp = fopen(data.src, "rb");
if (!fp) {
- perror(data.src);
+ SYSERROR("unable to open source template file: %s", strerror(errno));
guard_free(contents);
guard_free(data.dest);
continue;
@@ -189,13 +195,13 @@ int delivery_series_sync(struct Delivery *ctx) {
struct JFRT_Download dl = {0};
if (jfrt_auth_init(&ctx->deploy.jfrog_auth)) {
- fprintf(stderr, "Failed to initialize Artifactory authentication context\n");
+ SYSERROR("Failed to initialize Artifactory authentication context");
return -1; // error
}
char *r_fmt = strdup(ctx->rules.release_fmt);
if (!r_fmt) {
- SYSERROR("%s", "Unable to allocate bytes for release format string");
+ SYSERROR("Unable to allocate bytes for release format string");
return -1;
}
@@ -210,7 +216,7 @@ int delivery_series_sync(struct Delivery *ctx) {
}
char *release_pattern = NULL;
- if (delivery_format_str(ctx, &release_pattern, r_fmt) < 0) {
+ if (delivery_format_str(ctx, &release_pattern, STASIS_NAME_MAX, r_fmt) < 0) {
SYSERROR("Unable to render delivery format string: %s", r_fmt);
guard_free(r_fmt);
return -1;
@@ -223,7 +229,7 @@ int delivery_series_sync(struct Delivery *ctx) {
ctx->meta.mission,
ctx->info.build_name,
release_pattern) < 0) {
- SYSERROR("%s", "Unable to allocate bytes for remote directory path");
+ SYSERROR("Unable to allocate bytes for remote directory path");
guard_free(release_pattern);
return -1;
}
@@ -231,7 +237,7 @@ int delivery_series_sync(struct Delivery *ctx) {
char *dest_dir = NULL;
if (asprintf(&dest_dir, "%s/{1}", ctx->storage.output_dir) < 0) {
- SYSERROR("%s", "Unable to allocate bytes for destination directory path");
+ SYSERROR("Unable to allocate bytes for destination directory path");
return -1;
}
diff --git a/src/lib/delivery/delivery_build.c b/src/lib/delivery/delivery_build.c
index 2d891d2..66f9126 100644
--- a/src/lib/delivery/delivery_build.c
+++ b/src/lib/delivery/delivery_build.c
@@ -1,43 +1,68 @@
+#include <fnmatch.h>
+
#include "delivery.h"
+#include "conda.h"
+#include "recipe.h"
int delivery_build_recipes(struct Delivery *ctx) {
- for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) {
+ for (size_t i = 0; i < ctx->tests->num_used; i++) {
char *recipe_dir = NULL;
- if (ctx->tests[i].build_recipe) { // build a conda recipe
- if (recipe_clone(ctx->storage.build_recipes_dir, ctx->tests[i].build_recipe, NULL, &recipe_dir)) {
- fprintf(stderr, "Encountered an issue while cloning recipe for: %s\n", ctx->tests[i].name);
+ if (ctx->tests->test[i]->build_recipe) { // build a conda recipe
+ if (recipe_clone(ctx->storage.build_recipes_dir, ctx->tests->test[i]->build_recipe, NULL, &recipe_dir)) {
+ SYSERROR("Encountered an issue while cloning recipe for: %s", ctx->tests->test[i]->name);
return -1;
}
if (!recipe_dir) {
- fprintf(stderr, "BUG: recipe_clone() succeeded but recipe_dir is NULL: %s\n", strerror(errno));
+ SYSERROR("BUG: recipe_clone() succeeded but recipe_dir is NULL: %s", strerror(errno));
return -1;
}
int recipe_type = recipe_get_type(recipe_dir);
if(!pushd(recipe_dir)) {
if (RECIPE_TYPE_ASTROCONDA == recipe_type) {
- pushd(path_basename(ctx->tests[i].repository));
+ pushd(path_basename(ctx->tests->test[i]->repository));
} else if (RECIPE_TYPE_CONDA_FORGE == recipe_type) {
pushd("recipe");
}
- char recipe_version[100];
- char recipe_buildno[100];
+ char recipe_version[200];
+ char recipe_buildno[200];
char recipe_git_url[PATH_MAX];
char recipe_git_rev[PATH_MAX];
+ char tag[100] = {0};
+ if (ctx->tests->test[i]->repository_info_tag) {
+ const int is_long_tag = num_chars(ctx->tests->test[i]->repository_info_tag, '-') > 1;
+ if (is_long_tag) {
+ const size_t len = strcspn(ctx->tests->test[i]->repository_info_tag, "-");
+ strncpy(tag, ctx->tests->test[i]->repository_info_tag, len);
+ tag[len] = '\0';
+ } else {
+ strncpy(tag, ctx->tests->test[i]->repository_info_tag, sizeof(tag) - 1);
+ tag[sizeof(tag) - 1] = '\0';
+ }
+ } else {
+ strncpy(tag, ctx->tests->test[i]->version, sizeof(tag) - 1);
+ tag[sizeof(tag) - 1] = '\0';
+ }
+
//sprintf(recipe_version, "{%% set version = GIT_DESCRIBE_TAG ~ \".dev\" ~ GIT_DESCRIBE_NUMBER ~ \"+\" ~ GIT_DESCRIBE_HASH %%}");
- //sprintf(recipe_git_url, " git_url: %s", ctx->tests[i].repository);
- //sprintf(recipe_git_rev, " git_rev: %s", ctx->tests[i].version);
+ //sprintf(recipe_git_url, " git_url: %s", ctx->tests->test[i]->repository);
+ //sprintf(recipe_git_rev, " git_rev: %s", ctx->tests->test[i]->version);
// TODO: Conditionally download archives if github.com is the origin. Else, use raw git_* keys ^^^
- sprintf(recipe_version, "{%% set version = \"%s\" %%}", ctx->tests[i].repository_info_tag ? ctx->tests[i].repository_info_tag : ctx->tests[i].version);
- sprintf(recipe_git_url, " url: %s/archive/refs/tags/{{ version }}.tar.gz", ctx->tests[i].repository);
- strcpy(recipe_git_rev, "");
- sprintf(recipe_buildno, " number: 0");
+ // 03/2026 - How can we know if the repository URL supports archive downloads?
+ // Perhaps we can key it to the recipe type, because the archive is a requirement imposed
+ // by conda-forge. Hmm.
+
+ snprintf(recipe_version, sizeof(recipe_version), "{%% set version = \"%s\" %%}", tag);
+ snprintf(recipe_git_url, sizeof(recipe_git_url), " url: %s/archive/refs/tags/{{ version }}.tar.gz", ctx->tests->test[i]->repository);
+ strncpy(recipe_git_rev, "", sizeof(recipe_git_rev) - 1);
+ recipe_git_rev[sizeof(recipe_git_rev) - 1] = '\0';
+ snprintf(recipe_buildno, sizeof(recipe_buildno), " number: 0");
unsigned flags = REPLACE_TRUNCATE_AFTER_MATCH;
//file_replace_text("meta.yaml", "{% set version = ", recipe_version);
if (ctx->meta.final) { // remove this. i.e. statis cannot deploy a release to conda-forge
- sprintf(recipe_version, "{%% set version = \"%s\" %%}", ctx->tests[i].version);
+ snprintf(recipe_version, sizeof(recipe_version), "{%% set version = \"%s\" %%}", ctx->tests->test[i]->version);
// TODO: replace sha256 of tagged archive
// TODO: leave the recipe unchanged otherwise. in theory this should produce the same conda package hash as conda forge.
// For now, remove the sha256 requirement
@@ -55,25 +80,28 @@ int delivery_build_recipes(struct Delivery *ctx) {
char arch[STASIS_NAME_MAX] = {0};
char platform[STASIS_NAME_MAX] = {0};
- strcpy(platform, ctx->system.platform[DELIVERY_PLATFORM]);
+ strncpy(platform, ctx->system.platform[DELIVERY_PLATFORM], sizeof(platform) - 1);
if (strstr(platform, "Darwin")) {
memset(platform, 0, sizeof(platform));
- strcpy(platform, "osx");
+ strncpy(platform, "osx", sizeof(platform) - 1);
}
+ platform[sizeof(platform) - 1] = '\0';
tolower_s(platform);
+
if (strstr(ctx->system.arch, "arm64")) {
- strcpy(arch, "arm64");
+ strncpy(arch, "arm64", sizeof(arch) - 1);
} else if (strstr(ctx->system.arch, "64")) {
- strcpy(arch, "64");
+ strncpy(arch, "64", sizeof(arch) - 1);
} else {
- strcat(arch, "32"); // blind guess
+ strncat(arch, "32", sizeof(arch) - strlen(arch) - 1); // blind guess
}
+ arch[sizeof(arch) - 1] = '\0';
tolower_s(arch);
- sprintf(command, "mambabuild --python=%s -m ../.ci_support/%s_%s_.yaml .",
+ snprintf(command, sizeof(command), "mambabuild --python=%s -m ../.ci_support/%s_%s_.yaml .",
ctx->meta.python, platform, arch);
} else {
- sprintf(command, "mambabuild --python=%s .", ctx->meta.python);
+ snprintf(command, sizeof(command), "mambabuild --python=%s .", ctx->meta.python);
}
int status = conda_exec(command);
if (status) {
@@ -86,7 +114,7 @@ int delivery_build_recipes(struct Delivery *ctx) {
}
popd();
} else {
- fprintf(stderr, "Unable to enter recipe directory %s: %s\n", recipe_dir, strerror(errno));
+ SYSERROR("Unable to enter recipe directory %s: %s", recipe_dir, strerror(errno));
guard_free(recipe_dir);
return -1;
}
@@ -112,7 +140,7 @@ int filter_repo_tags(char *repo, struct StrList *patterns) {
int match = fnmatch(pattern, tag, 0);
if (!match) {
char cmd[PATH_MAX] = {0};
- sprintf(cmd, "git tag -d %s", tag);
+ snprintf(cmd, sizeof(cmd), "git tag -d %s", tag);
result += system(cmd);
break;
}
@@ -127,13 +155,238 @@ int filter_repo_tags(char *repo, struct StrList *patterns) {
return result;
}
+static int read_without_line_endings(const size_t line, char ** arg) {
+ (void) line;
+ if (*arg) {
+ strip(*arg);
+ if (isempty(*arg)) {
+ return 1; // skip
+ }
+ }
+ return 0;
+}
+
+int manylinux_exec(const char *image, const char *script, const char *copy_to_container_dir, const char *copy_from_container_dir, const char *copy_to_host_dir) {
+ int result = -1; // fail by default
+ char *container_name = NULL;
+ char *source_copy_command = NULL;
+ char *copy_command = NULL;
+ char *rm_command = NULL;
+ char *nop_create_command = NULL;
+ char *nop_rm_command = NULL;
+ char *volume_rm_command = NULL;
+ char *find_command = NULL;
+ char *wheel_paths_filename = NULL;
+ char *args = NULL;
+
+ const uid_t uid = geteuid();
+ char suffix[7] = {0};
+
+ // setup
+
+ if (get_random_bytes(suffix, sizeof(suffix))) {
+ SYSERROR("unable to acquire value from random generator");
+ goto manylinux_fail;
+ }
+
+ if (asprintf(&container_name, "manylinux_build_%d_%zd_%s", uid, time(NULL), suffix) < 0) {
+ SYSERROR("unable to allocate memory for container name");
+ goto manylinux_fail;
+ }
+
+ if (asprintf(&args, "--name %s -w /build -v %s:/build", container_name, container_name) < 0) {
+ SYSERROR("unable to allocate memory for docker arguments");
+ goto manylinux_fail;
+ }
+
+ if (!strstr(image, "manylinux")) {
+ SYSERROR("expected a manylinux image, but got %s", image);
+ goto manylinux_fail;
+ }
+
+ if (asprintf(&nop_create_command, "run --name nop_%s -v %s:/build busybox", container_name, container_name) < 0) {
+ SYSERROR("unable to allocate memory for nop container command");
+ goto manylinux_fail;
+ }
+
+ if (asprintf(&source_copy_command, "cp %s nop_%s:/build", copy_to_container_dir, container_name) < 0) {
+ SYSERROR("unable to allocate memory for source copy command");
+ goto manylinux_fail;
+ }
+
+ if (asprintf(&nop_rm_command, "rm nop_%s", container_name) < 0) {
+ SYSERROR("unable to allocate memory for nop container command");
+ goto manylinux_fail;
+ }
+
+ if (asprintf(&wheel_paths_filename, "%s/wheel_paths_%s.txt", globals.tmpdir, container_name) < 0) {
+ SYSERROR("unable to allocate memory for wheel paths file name");
+ goto manylinux_fail;
+ }
+
+ if (asprintf(&find_command, "run --rm -t -v %s:/build busybox sh -c 'find %s -name \"*.whl\"' > %s", container_name, copy_from_container_dir, wheel_paths_filename) < 0) {
+ SYSERROR("unable to allocate memory for find command");
+ goto manylinux_fail;
+ }
+
+ // execute
+
+ if (docker_exec(nop_create_command, 0)) {
+ SYSERROR("docker nop container creation failed");
+ goto manylinux_fail;
+ }
+
+ if (docker_exec(source_copy_command, 0)) {
+ SYSERROR("docker source copy operation failed");
+ goto manylinux_fail;
+ }
+
+ if (docker_exec(nop_rm_command, STASIS_DOCKER_QUIET)) {
+ SYSERROR("docker nop container removal failed");
+ goto manylinux_fail;
+ }
+
+ if (docker_script(image, args, (char *) script, 0)) {
+ SYSERROR("manylinux execution failed");
+ goto manylinux_fail;
+ }
+
+ if (docker_exec(find_command, 0)) {
+ SYSERROR("docker find command failed");
+ goto manylinux_fail;
+ }
+
+ struct StrList *wheel_paths = strlist_init();
+ if (!wheel_paths) {
+ SYSERROR("wheel_paths not initialized");
+ goto manylinux_fail;
+ }
+
+ if (strlist_append_file(wheel_paths, wheel_paths_filename, read_without_line_endings)) {
+ SYSERROR("wheel_paths append failed");
+ goto manylinux_fail;
+ }
+
+ for (size_t i = 0; i < strlist_count(wheel_paths); i++) {
+ const char *item = strlist_item(wheel_paths, i);
+ if (asprintf(&copy_command, "cp %s:%s %s", container_name, item, copy_to_host_dir) < 0) {
+ SYSERROR("unable to allocate memory for docker copy command");
+ goto manylinux_fail;
+ }
+
+ if (docker_exec(copy_command, 0)) {
+ SYSERROR("docker copy operation failed");
+ goto manylinux_fail;
+ }
+ guard_free(copy_command);
+ }
+
+ // Success
+ result = 0;
+
+ manylinux_fail:
+ if (wheel_paths_filename) {
+ remove(wheel_paths_filename);
+ }
+
+ if (container_name) {
+ // Keep going on failure unless memory related.
+ // We don't want build debris everywhere.
+ if (asprintf(&rm_command, "rm %s", container_name) < 0) {
+ SYSERROR("unable to allocate memory for rm command");
+ goto late_fail;
+ }
+
+ if (docker_exec(rm_command, STASIS_DOCKER_QUIET)) {
+ SYSERROR("docker container removal operation failed");
+ }
+
+ if (asprintf(&volume_rm_command, "volume rm -f %s", container_name) < 0) {
+ SYSERROR("unable to allocate memory for docker volume removal command");
+ goto late_fail;
+ }
+
+ if (docker_exec(volume_rm_command, STASIS_DOCKER_QUIET)) {
+ SYSERROR("docker volume removal operation failed");
+ }
+ }
+
+ late_fail:
+ guard_free(container_name);
+ guard_free(args);
+ guard_free(copy_command);
+ guard_free(rm_command);
+ guard_free(volume_rm_command);
+ guard_free(source_copy_command);
+ guard_free(nop_create_command);
+ guard_free(nop_rm_command);
+ guard_free(find_command);
+ guard_free(wheel_paths_filename);
+ guard_strlist_free(&wheel_paths);
+ return result;
+}
+
+int delivery_build_wheels_manylinux(struct Delivery *ctx, const char *outdir) {
+ msg(STASIS_MSG_L1, "Building wheels\n");
+
+ const char *manylinux_image = globals.wheel_builder_manylinux_image;
+ if (!manylinux_image) {
+ SYSERROR("manylinux_image not initialized");
+ return -1;
+ }
+
+ int manylinux_build_status = 0;
+
+ msg(STASIS_MSG_L2, "Using: %s\n", manylinux_image);
+ const struct Meta *meta = &ctx->meta;
+ const char *script_fmt =
+ "set -e -x\n"
+ "git config --global --add safe.directory /build\n"
+ "python%s -m pip install auditwheel build\n"
+ "python%s -m build -w .\n"
+ "auditwheel show --allow-pure-python-wheel dist/*.whl\n"
+ "auditwheel repair --allow-pure-python-wheel dist/*.whl\n";
+ char *script = NULL;
+ if (asprintf(&script, script_fmt,
+ meta->python, meta->python) < 0) {
+ SYSERROR("unable to allocate memory for build script");
+ return -1;
+ }
+ manylinux_build_status = manylinux_exec(
+ manylinux_image,
+ script,
+ "./",
+ "/build/wheelhouse",
+ outdir);
+
+ if (manylinux_build_status) {
+ SYSERROR("manylinux build failed (%d)", manylinux_build_status);
+ guard_free(script);
+ return -1;
+ }
+ guard_free(script);
+ return 0;
+}
+
struct StrList *delivery_build_wheels(struct Delivery *ctx) {
+ const int on_linux = strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Linux") == 0;
+ const int docker_usable = ctx->deploy.docker.capabilities.usable;
+ int use_builder_build = strcmp(globals.wheel_builder, "native") == 0;
+ const int use_builder_cibuildwheel = strcmp(globals.wheel_builder, "cibuildwheel") == 0 && on_linux && docker_usable;
+ const int use_builder_manylinux = strcmp(globals.wheel_builder, "manylinux") == 0 && on_linux && docker_usable;
+
+ if (!use_builder_build && !use_builder_cibuildwheel && !use_builder_manylinux) {
+ SYSWARN("Cannot build wheel for platform using: %s", globals.wheel_builder);
+ SYSWARN("Falling back to native toolchain.", globals.wheel_builder);
+ use_builder_build = 1;
+ }
+
struct StrList *result = NULL;
struct Process proc = {0};
result = strlist_init();
if (!result) {
- perror("unable to allocate memory for string list");
+ SYSERROR("unable to allocate memory for string list");
return NULL;
}
@@ -141,57 +394,103 @@ struct StrList *delivery_build_wheels(struct Delivery *ctx) {
char name[100] = {0};
char *fullspec = strlist_item(ctx->conda.pip_packages_defer, p);
strncpy(name, fullspec, sizeof(name) - 1);
+ name[sizeof(name) - 1] = '\0';
remove_extras(name);
char *spec = find_version_spec(name);
if (spec) {
*spec = '\0';
}
- for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) {
- if ((ctx->tests[i].name && !strcmp(name, ctx->tests[i].name)) && (!ctx->tests[i].build_recipe && ctx->tests[i].repository)) { // build from source
+ for (size_t i = 0; i < ctx->tests->num_used; i++) {
+ if ((ctx->tests->test[i]->name && !strcmp(name, ctx->tests->test[i]->name)) && (!ctx->tests->test[i]->build_recipe && ctx->tests->test[i]->repository)) { // build from source
char srcdir[PATH_MAX];
char wheeldir[PATH_MAX];
memset(srcdir, 0, sizeof(srcdir));
memset(wheeldir, 0, sizeof(wheeldir));
- sprintf(srcdir, "%s/%s", ctx->storage.build_sources_dir, ctx->tests[i].name);
- if (git_clone(&proc, ctx->tests[i].repository, srcdir, ctx->tests[i].version)) {
- SYSERROR("Unable to checkout tag '%s' for package '%s' from repository '%s'\n",
- ctx->tests[i].version, ctx->tests[i].name, ctx->tests[i].repository);
+ snprintf(srcdir, sizeof(srcdir), "%s/%s", ctx->storage.build_sources_dir, ctx->tests->test[i]->name);
+ if (git_clone(&proc, ctx->tests->test[i]->repository, srcdir, ctx->tests->test[i]->version)) {
+ SYSERROR("Unable to checkout tag '%s' for package '%s' from repository '%s'",
+ ctx->tests->test[i]->version, ctx->tests->test[i]->name, ctx->tests->test[i]->repository);
return NULL;
}
- if (ctx->tests[i].repository_remove_tags && strlist_count(ctx->tests[i].repository_remove_tags)) {
- filter_repo_tags(srcdir, ctx->tests[i].repository_remove_tags);
+ if (!ctx->tests->test[i]->repository_info_tag) {
+ ctx->tests->test[i]->repository_info_tag = strdup(git_describe(srcdir));
+ }
+ if (!ctx->tests->test[i]->repository_info_ref) {
+ ctx->tests->test[i]->repository_info_ref = strdup(git_rev_parse(srcdir, ctx->tests->test[i]->version));
+ }
+ if (ctx->tests->test[i]->repository_remove_tags && strlist_count(ctx->tests->test[i]->repository_remove_tags)) {
+ filter_repo_tags(srcdir, ctx->tests->test[i]->repository_remove_tags);
}
if (!pushd(srcdir)) {
char dname[NAME_MAX];
char outdir[PATH_MAX];
- char cmd[PATH_MAX * 2];
+ char *cmd = NULL;
memset(dname, 0, sizeof(dname));
memset(outdir, 0, sizeof(outdir));
- memset(cmd, 0, sizeof(outdir));
- strcpy(dname, ctx->tests[i].name);
+ const int dep_status = check_python_package_dependencies(".");
+ if (dep_status) {
+ SYSERROR("Please replace all occurrences above with standard package specs:\n"
+ "\n"
+ " package==x.y.z\n"
+ " package>=x.y.z\n"
+ " package<=x.y.z\n"
+ " ...\n"
+ "\n");
+ COE_CHECK_ABORT(true, "Unreproducible delivery");
+ }
+
+ strncpy(dname, ctx->tests->test[i]->name, sizeof(dname) - 1);
+ dname[sizeof(dname) - 1] = '\0';
tolower_s(dname);
- sprintf(outdir, "%s/%s", ctx->storage.wheel_artifact_dir, dname);
+ snprintf(outdir, sizeof(outdir), "%s/%s", ctx->storage.wheel_artifact_dir, dname);
if (mkdirs(outdir, 0755)) {
- fprintf(stderr, "failed to create output directory: %s\n", outdir);
+ SYSERROR("failed to create output directory: %s", outdir);
guard_strlist_free(&result);
return NULL;
}
+ if (use_builder_manylinux) {
+ if (delivery_build_wheels_manylinux(ctx, outdir)) {
+ SYSERROR("failed to generate wheel package for %s-%s", ctx->tests->test[i]->name,
+ ctx->tests->test[i]->version);
+ guard_strlist_free(&result);
+ guard_free(cmd);
+ return NULL;
+ }
+ } else if (use_builder_build || use_builder_cibuildwheel) {
+ if (use_builder_build) {
+ if (asprintf(&cmd, "-m build -w -o %s", outdir) < 0) {
+ SYSERROR("Unable to allocate memory for build command");
+ return NULL;
+ }
+ } else if (use_builder_cibuildwheel) {
+ if (asprintf(&cmd, "-m cibuildwheel --output-dir %s --only cp%s-manylinux_%s",
+ outdir, ctx->meta.python_compact, ctx->system.arch) < 0) {
+ SYSERROR("Unable to allocate memory for cibuildwheel command");
+ return NULL;
+ }
+ }
- sprintf(cmd, "-m build -w -o %s", outdir);
- if (python_exec(cmd)) {
- fprintf(stderr, "failed to generate wheel package for %s-%s\n", ctx->tests[i].name,
- ctx->tests[i].version);
- guard_strlist_free(&result);
+ if (python_exec(cmd)) {
+ SYSERROR("failed to generate wheel package for %s-%s", ctx->tests->test[i]->name,
+ ctx->tests->test[i]->version);
+ guard_strlist_free(&result);
+ guard_free(cmd);
+ return NULL;
+ }
+ } else {
+ SYSERROR("unknown wheel builder backend: %s", globals.wheel_builder);
return NULL;
}
+
+ guard_free(cmd);
popd();
} else {
- fprintf(stderr, "Unable to enter source directory %s: %s\n", srcdir, strerror(errno));
+ SYSERROR("Unable to enter source directory %s: %s", srcdir, strerror(errno));
guard_strlist_free(&result);
return NULL;
}
@@ -200,4 +499,3 @@ struct StrList *delivery_build_wheels(struct Delivery *ctx) {
}
return result;
}
-
diff --git a/src/lib/delivery/delivery_conda.c b/src/lib/delivery/delivery_conda.c
index 8974ae8..117e6c9 100644
--- a/src/lib/delivery/delivery_conda.c
+++ b/src/lib/delivery/delivery_conda.c
@@ -1,21 +1,37 @@
#include "delivery.h"
+#include "conda.h"
-void delivery_get_conda_installer_url(struct Delivery *ctx, char *result) {
+void delivery_get_conda_installer_url(struct Delivery *ctx, char *result, size_t maxlen) {
+ int len = 0;
if (ctx->conda.installer_version) {
// Use version specified by configuration file
- sprintf(result, "%s/%s-%s-%s-%s.sh", ctx->conda.installer_baseurl,
+ len = snprintf(NULL, 0, "%s/%s-%s-%s-%s.sh",
+ ctx->conda.installer_baseurl,
+ ctx->conda.installer_name,
+ ctx->conda.installer_version,
+ ctx->conda.installer_platform,
+ ctx->conda.installer_arch);
+
+ snprintf(result, maxlen - len, "%s/%s-%s-%s-%s.sh",
+ ctx->conda.installer_baseurl,
ctx->conda.installer_name,
ctx->conda.installer_version,
ctx->conda.installer_platform,
ctx->conda.installer_arch);
} else {
// Use latest installer
- sprintf(result, "%s/%s-%s-%s.sh", ctx->conda.installer_baseurl,
+ len = snprintf(NULL, 0, "%s/%s-%s-%s.sh",
+ ctx->conda.installer_baseurl,
ctx->conda.installer_name,
ctx->conda.installer_platform,
ctx->conda.installer_arch);
- }
+ snprintf(result, maxlen - len, "%s/%s-%s-%s.sh",
+ ctx->conda.installer_baseurl,
+ ctx->conda.installer_name,
+ ctx->conda.installer_platform,
+ ctx->conda.installer_arch);
+ }
}
int delivery_get_conda_installer(struct Delivery *ctx, char *installer_url) {
@@ -23,12 +39,15 @@ int delivery_get_conda_installer(struct Delivery *ctx, char *installer_url) {
char *installer = path_basename(installer_url);
memset(script_path, 0, sizeof(script_path));
- sprintf(script_path, "%s/%s", ctx->storage.tmpdir, installer);
+ snprintf(script_path, sizeof(script_path), "%s/%s", ctx->storage.tmpdir, installer);
if (access(script_path, F_OK)) {
// Script doesn't exist
- long fetch_status = download(installer_url, script_path, NULL);
+ char *errmsg = NULL;
+ long fetch_status = download(installer_url, script_path, &errmsg);
if (HTTP_ERROR(fetch_status) || fetch_status < 0) {
// download failed
+ SYSERROR("download failed: %s: %s", errmsg, installer_url);
+ guard_free(errmsg);
return -1;
}
} else {
@@ -51,31 +70,31 @@ void delivery_install_conda(char *install_script, char *conda_install_dir) {
if (!access(conda_install_dir, F_OK)) {
// directory exists so remove it
if (rmtree(conda_install_dir)) {
- perror("unable to remove previous installation");
+ SYSERROR("unable to remove previous installation: %s", strerror(errno));
exit(1);
}
// Proceed with the installation
// -b = batch mode (non-interactive)
char cmd[PATH_MAX] = {0};
- snprintf(cmd, sizeof(cmd) - 1, "%s %s -b -p %s",
+ snprintf(cmd, sizeof(cmd), "%s %s -b -p %s",
find_program("bash"),
install_script,
conda_install_dir);
if (shell_safe(&proc, cmd)) {
- fprintf(stderr, "conda installation failed\n");
+ SYSERROR("conda installation failed");
exit(1);
}
} else {
// Proceed with the installation
// -b = batch mode (non-interactive)
char cmd[PATH_MAX] = {0};
- snprintf(cmd, sizeof(cmd) - 1, "%s %s -b -p %s",
+ snprintf(cmd, sizeof(cmd), "%s %s -b -p %s",
find_program("bash"),
install_script,
conda_install_dir);
if (shell_safe(&proc, cmd)) {
- fprintf(stderr, "conda installation failed\n");
+ SYSERROR("conda installation failed");
exit(1);
}
}
@@ -86,7 +105,7 @@ void delivery_install_conda(char *install_script, char *conda_install_dir) {
void delivery_conda_enable(struct Delivery *ctx, char *conda_install_dir) {
if (conda_activate(conda_install_dir, "base")) {
- fprintf(stderr, "conda activation failed\n");
+ SYSERROR("conda activation failed");
exit(1);
}
@@ -94,10 +113,10 @@ void delivery_conda_enable(struct Delivery *ctx, char *conda_install_dir) {
// way to make sure the file is used. Not setting this variable leads to strange
// behavior, especially if a conda environment is already active when STASIS is loaded.
char rcpath[PATH_MAX];
- sprintf(rcpath, "%s/%s", conda_install_dir, ".condarc");
+ snprintf(rcpath, sizeof(rcpath), "%s/%s", conda_install_dir, ".condarc");
setenv("CONDARC", rcpath, 1);
if (runtime_replace(&ctx->runtime.environ, __environ)) {
- perror("unable to replace runtime environment after activating conda");
+ SYSERROR("unable to replace runtime environment after activating conda");
exit(1);
}
diff --git a/src/lib/delivery/delivery_docker.c b/src/lib/delivery/delivery_docker.c
index 57015ad..79e9729 100644
--- a/src/lib/delivery/delivery_docker.c
+++ b/src/lib/delivery/delivery_docker.c
@@ -11,15 +11,15 @@ int delivery_docker(struct Delivery *ctx) {
size_t total_build_args = strlist_count(ctx->deploy.docker.build_args);
if (!has_registry) {
- msg(STASIS_MSG_WARN | STASIS_MSG_L2, "No docker registry defined. You will need to manually re-tag the resulting image.\n");
+ SYSWARN("No docker registry defined. You will need to manually re-tag the resulting image.");
}
if (!total_tags) {
char default_tag[PATH_MAX];
- msg(STASIS_MSG_WARN | STASIS_MSG_L2, "No docker tags defined by configuration. Generating default tag(s).\n");
+ SYSWARN("No docker tags defined by configuration. Generating default tag(s).");
// generate local tag
memset(default_tag, 0, sizeof(default_tag));
- sprintf(default_tag, "%s:%s-py%s", ctx->meta.name, ctx->info.build_name, ctx->meta.python_compact);
+ snprintf(default_tag, sizeof(default_tag), "%s:%s-py%s", ctx->meta.name, ctx->info.build_name, ctx->meta.python_compact);
tolower_s(default_tag);
// Add tag
@@ -29,7 +29,7 @@ int delivery_docker(struct Delivery *ctx) {
if (has_registry) {
// generate tag for target registry
memset(default_tag, 0, sizeof(default_tag));
- sprintf(default_tag, "%s/%s:%s-py%s", ctx->deploy.docker.registry, ctx->meta.name, ctx->info.build_number, ctx->meta.python_compact);
+ snprintf(default_tag, sizeof(default_tag), "%s/%s:%s-py%s", ctx->deploy.docker.registry, ctx->meta.name, ctx->info.build_number, ctx->meta.python_compact);
tolower_s(default_tag);
// Add tag
@@ -44,9 +44,10 @@ int delivery_docker(struct Delivery *ctx) {
// Append image tags to command
for (size_t i = 0; i < total_tags; i++) {
char *tag_orig = strlist_item(ctx->deploy.docker.tags, i);
- strcpy(tag, tag_orig);
+ strncpy(tag, tag_orig, sizeof(tag) - 1);
+ tag[sizeof(tag) - 1] = '\0';
docker_sanitize_tag(tag);
- sprintf(args + strlen(args), " -t \"%s\" ", tag);
+ snprintf(args + strlen(args), sizeof(args) - strlen(args), " -t \"%s\" ", tag);
}
// Append build arguments to command (i.e. --build-arg "key=value"
@@ -55,7 +56,7 @@ int delivery_docker(struct Delivery *ctx) {
if (!build_arg) {
break;
}
- sprintf(args + strlen(args), " --build-arg \"%s\" ", build_arg);
+ snprintf(args + strlen(args), sizeof(args) - strlen(args), " --build-arg \"%s\" ", build_arg);
}
// Build the image
@@ -65,34 +66,34 @@ int delivery_docker(struct Delivery *ctx) {
memset(delivery_file, 0, sizeof(delivery_file));
memset(dest, 0, sizeof(dest));
- sprintf(delivery_file, "%s/%s.yml", ctx->storage.delivery_dir, ctx->info.release_name);
+ snprintf(delivery_file, sizeof(delivery_file), "%s/%s.yml", ctx->storage.delivery_dir, ctx->info.release_name);
if (access(delivery_file, F_OK) < 0) {
- fprintf(stderr, "docker build cannot proceed without delivery file: %s\n", delivery_file);
+ SYSERROR("docker build cannot proceed without delivery file: %s", delivery_file);
return -1;
}
- sprintf(dest, "%s/%s.yml", ctx->storage.build_docker_dir, ctx->info.release_name);
+ snprintf(dest, sizeof(dest), "%s/%s.yml", ctx->storage.build_docker_dir, ctx->info.release_name);
if (copy2(delivery_file, dest, CT_PERM)) {
- fprintf(stderr, "Failed to copy delivery file to %s: %s\n", dest, strerror(errno));
+ SYSERROR("Failed to copy delivery file to %s: %s", dest, strerror(errno));
return -1;
}
memset(dest, 0, sizeof(dest));
- sprintf(dest, "%s/packages", ctx->storage.build_docker_dir);
+ snprintf(dest, sizeof(dest), "%s/packages", ctx->storage.build_docker_dir);
msg(STASIS_MSG_L2, "Copying conda packages\n");
memset(rsync_cmd, 0, sizeof(rsync_cmd));
- sprintf(rsync_cmd, "rsync -avi --progress '%s' '%s'", ctx->storage.conda_artifact_dir, dest);
+ snprintf(rsync_cmd, sizeof(rsync_cmd), "rsync -avi --progress '%s' '%s'", ctx->storage.conda_artifact_dir, dest);
if (system(rsync_cmd)) {
- fprintf(stderr, "Failed to copy conda artifacts to docker build directory\n");
+ SYSERROR("Failed to copy conda artifacts to docker build directory");
return -1;
}
msg(STASIS_MSG_L2, "Copying wheel packages\n");
memset(rsync_cmd, 0, sizeof(rsync_cmd));
- sprintf(rsync_cmd, "rsync -avi --progress '%s' '%s'", ctx->storage.wheel_artifact_dir, dest);
+ snprintf(rsync_cmd, sizeof(rsync_cmd), "rsync -avi --progress '%s' '%s'", ctx->storage.wheel_artifact_dir, dest);
if (system(rsync_cmd)) {
- fprintf(stderr, "Failed to copy wheel artifacts to docker build directory\n");
+ SYSWARN("Failed to copy wheel artifacts to docker build directory. No wheels produced?");
}
if (docker_build(ctx->storage.build_docker_dir, args, ctx->deploy.docker.capabilities.build)) {
@@ -102,23 +103,24 @@ int delivery_docker(struct Delivery *ctx) {
// Test the image
// All tags point back to the same image so test the first one we see
// regardless of how many are defined
- strcpy(tag, strlist_item(ctx->deploy.docker.tags, 0));
+ strncpy(tag, strlist_item(ctx->deploy.docker.tags, 0), sizeof(tag) - 1);
+ tag[sizeof(tag) - 1] = '\0';
docker_sanitize_tag(tag);
msg(STASIS_MSG_L2, "Executing image test script for %s\n", tag);
if (ctx->deploy.docker.test_script) {
if (isempty(ctx->deploy.docker.test_script)) {
- msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "Image test script has no content\n");
+ SYSWARN("Image test script has no content");
} else {
int state;
- if ((state = docker_script(tag, ctx->deploy.docker.test_script, 0))) {
- msg(STASIS_MSG_L2 | STASIS_MSG_ERROR, "Non-zero exit (%d) from test script. %s image archive will not be generated.\n", state >> 8, tag);
+ if ((state = docker_script(tag, "--rm", ctx->deploy.docker.test_script, 0))) {
+ SYSERROR("Non-zero exit (%d) from test script. %s image archive will not be generated.", state >> 8, tag);
// test failed -- don't save the image
return -1;
}
}
} else {
- msg(STASIS_MSG_L2 | STASIS_MSG_WARN, "No image test script defined\n");
+ SYSWARN("No image test script defined");
}
// Test successful, save image
diff --git a/src/lib/delivery/delivery_export.c b/src/lib/delivery/delivery_export.c
new file mode 100644
index 0000000..0321050
--- /dev/null
+++ b/src/lib/delivery/delivery_export.c
@@ -0,0 +1,77 @@
+#include "delivery.h"
+#include "conda.h"
+
+static void delivery_export_configuration(const struct Delivery *ctx) {
+ msg(STASIS_MSG_L2, "Exporting delivery configuration\n");
+
+ SYSDEBUG("Entering configuration directory: %s", ctx->storage.delivery_dir);
+ if (!pushd(ctx->storage.cfgdump_dir)) {
+ char filename[PATH_MAX] = {0};
+ SYSDEBUG("Populating filename");
+ snprintf(filename, sizeof(filename), "%s.ini", ctx->info.release_name);
+ SYSDEBUG("filename: %s", filename);
+
+ SYSDEBUG("%s: opening", filename);
+ FILE *spec = fopen(filename, "w+");
+ if (!spec) {
+ SYSERROR("open failed %s", filename);
+ exit(1);
+ }
+ SYSDEBUG("%s: writing", filename);
+ ini_write(ctx->_stasis_ini_fp.delivery, &spec, INI_WRITE_RAW);
+ SYSDEBUG("%s: writing done", filename);
+ fclose(spec);
+ SYSDEBUG("%s: closing", filename);
+
+ SYSDEBUG("Zeroing filename");
+ memset(filename, 0, sizeof(filename));
+ SYSDEBUG("Populating rendered filename");
+ snprintf(filename, sizeof(filename), "%s-rendered.ini", ctx->info.release_name);
+ SYSDEBUG("filename: %s", filename);
+
+ SYSDEBUG("%s: opening", filename);
+ spec = fopen(filename, "w+");
+ if (!spec) {
+ SYSERROR("open failed %s", filename);
+ exit(1);
+ }
+ SYSDEBUG("%s: writing", filename);
+ ini_write(ctx->_stasis_ini_fp.delivery, &spec, INI_WRITE_PRESERVE);
+ SYSDEBUG("%s: writing done", filename);
+ SYSDEBUG("%s: closing", filename);
+ fclose(spec);
+ SYSDEBUG("Returning from %s", ctx->storage.cfgdump_dir);
+ popd();
+ } else {
+ SYSERROR("Failed to enter directory: %s", ctx->storage.delivery_dir);
+ exit(1);
+ }
+}
+
+void delivery_export(const struct Delivery *ctx, char *envs[]) {
+ delivery_export_configuration(ctx);
+
+ for (size_t i = 0; envs[i] != NULL; i++) {
+ char *name = envs[i];
+ msg(STASIS_MSG_L2, "Exporting %s\n", name);
+ if (conda_env_export(name, ctx->storage.delivery_dir, name)) {
+ SYSERROR("export failed %s", name);
+ exit(1);
+ }
+ }
+}
+
+void delivery_rewrite_stage1(struct Delivery *ctx, char *specfile) {
+ // Rewrite release environment output (i.e. set package origin(s) to point to the deployment server, etc.)
+ msg(STASIS_MSG_L3, "Rewriting release spec file (stage 1): %s\n", path_basename(specfile));
+ delivery_rewrite_spec(ctx, specfile, DELIVERY_REWRITE_SPEC_STAGE_1);
+
+ msg(STASIS_MSG_L1, "Rendering mission templates\n");
+ delivery_mission_render_files(ctx);
+}
+
+void delivery_rewrite_stage2(struct Delivery *ctx, char *specfile) {
+ msg(STASIS_MSG_L3, "Rewriting release spec file (stage 2): %s\n", path_basename(specfile));
+ delivery_rewrite_spec(ctx, specfile, DELIVERY_REWRITE_SPEC_STAGE_2);
+}
+
diff --git a/src/lib/delivery/delivery_init.c b/src/lib/delivery/delivery_init.c
index 56c591a..5bc326d 100644
--- a/src/lib/delivery/delivery_init.c
+++ b/src/lib/delivery/delivery_init.c
@@ -1,3 +1,6 @@
+#include <fnmatch.h>
+#include <sys/utsname.h>
+
#include "delivery.h"
int has_mount_flags(const char *mount_point, const unsigned long flags) {
@@ -11,34 +14,54 @@ int has_mount_flags(const char *mount_point, const unsigned long flags) {
int delivery_init_tmpdir(struct Delivery *ctx) {
char *tmpdir = NULL;
- char *x = NULL;
- int unusable = 0;
+ int unusable = 1;
errno = 0;
- x = getenv("TMPDIR");
+ //int need_setenv = 0;
+ const char *x = getenv("TMPDIR");
if (x) {
guard_free(ctx->storage.tmpdir);
tmpdir = strdup(x);
+ if (!tmpdir) {
+ // memory error
+ SYSERROR("unable to allocate tmpdir");
+ goto l_delivery_init_tmpdir_fatal;
+ }
} else {
- tmpdir = ctx->storage.tmpdir;
+ tmpdir = strdup("/tmp/stasis");
+ if (!tmpdir) {
+ SYSERROR("unable to allocate tmpdir");
+ goto l_delivery_init_tmpdir_fatal;
+ }
+ //need_setenv = 1;
}
- if (!tmpdir) {
- // memory error
- return -1;
+ if (!ctx->storage.tmpdir) {
+ ctx->storage.tmpdir = strdup(tmpdir);
+ if (!ctx->storage.tmpdir) {
+ SYSERROR("unable to allocate ctx->storage.tmpdir");
+ goto l_delivery_init_tmpdir_fatal;
+ }
+ } else {
+ // we already have a temp directory to use
+ guard_free(tmpdir);
+ tmpdir = strdup(ctx->storage.tmpdir);
+ if (!tmpdir) {
+ SYSERROR("unable to allocate tmpdir");
+ goto l_delivery_init_tmpdir_fatal;
+ }
}
-
// If the directory doesn't exist, create it
if (access(tmpdir, F_OK) < 0) {
if (mkdirs(tmpdir, 0755) < 0) {
- msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Unable to create temporary storage directory: %s (%s)\n", tmpdir, strerror(errno));
+ SYSERROR("Unable to create temporary storage directory: %s (%s)", tmpdir, strerror(errno));
goto l_delivery_init_tmpdir_fatal;
}
}
// If we can't read, write, or execute, then die
if (access(tmpdir, R_OK | W_OK | X_OK) < 0) {
- msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "%s requires at least 0755 permissions.\n");
+ SYSERROR("%s requires at least 0755 permissions.");
goto l_delivery_init_tmpdir_fatal;
}
@@ -50,26 +73,38 @@ int delivery_init_tmpdir(struct Delivery *ctx) {
#if defined(STASIS_OS_LINUX)
// If we can't execute programs, or write data to the file system at all, then die
if ((st.f_flag & ST_NOEXEC) != 0) {
- msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "%s is mounted with noexec\n", tmpdir);
+ SYSERROR("%s is mounted with noexec", tmpdir);
goto l_delivery_init_tmpdir_fatal;
}
#endif
if ((st.f_flag & ST_RDONLY) != 0) {
- msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "%s is mounted read-only\n", tmpdir);
+ SYSERROR("%s is mounted read-only", tmpdir);
goto l_delivery_init_tmpdir_fatal;
}
- if (!globals.tmpdir) {
+ if (!globals.tmpdir || strcmp(globals.tmpdir, ctx->storage.tmpdir) != 0) {
globals.tmpdir = strdup(tmpdir);
+ if (!globals.tmpdir) {
+ SYSERROR("unable to allocate globals.tmpdir");
+ goto l_delivery_init_tmpdir_fatal;
+ }
}
if (!ctx->storage.tmpdir) {
ctx->storage.tmpdir = strdup(globals.tmpdir);
+ if (!ctx->storage.tmpdir) {
+ SYSERROR("unable to allocate globals.tmpdir");
+ goto l_delivery_init_tmpdir_fatal;
+ }
}
- return unusable;
+ unusable = 0;
+ // TODO: Figure out why this breaks EVERYTHING
+ //if (need_setenv) {
+ // setenv("TMPDIR", ctx->storage.tmpdir, 1);
+ //}
l_delivery_init_tmpdir_fatal:
- unusable = 1;
+ guard_free(tmpdir);
return unusable;
}
@@ -94,7 +129,7 @@ void delivery_init_dirs_stage1(struct Delivery *ctx) {
char *rootdir = getenv("STASIS_ROOT");
if (rootdir) {
if (isempty(rootdir)) {
- fprintf(stderr, "STASIS_ROOT is set, but empty. Please assign a file system path to this environment variable.\n");
+ SYSERROR("STASIS_ROOT is set, but empty. Please assign a file system path to this environment variable.");
exit(1);
}
path_store(&ctx->storage.root, PATH_MAX, rootdir, ctx->info.build_name);
@@ -105,9 +140,10 @@ void delivery_init_dirs_stage1(struct Delivery *ctx) {
path_store(&ctx->storage.tools_dir, PATH_MAX, ctx->storage.root, "tools");
path_store(&ctx->storage.tmpdir, PATH_MAX, ctx->storage.root, "tmp");
if (delivery_init_tmpdir(ctx)) {
- msg(STASIS_MSG_ERROR | STASIS_MSG_L1, "Set $TMPDIR to a location other than %s\n", globals.tmpdir);
- if (globals.tmpdir)
+ SYSERROR("Set $TMPDIR to a location other than %s", globals.tmpdir);
+ if (globals.tmpdir) {
guard_free(globals.tmpdir);
+ }
exit(1);
}
@@ -119,7 +155,7 @@ void delivery_init_dirs_stage1(struct Delivery *ctx) {
}
if (access(ctx->storage.mission_dir, F_OK)) {
- msg(STASIS_MSG_L1, "%s: %s\n", ctx->storage.mission_dir, strerror(errno));
+ msg(STASIS_MSG_L1, "%s: %s: mission directory does not exist\n", ctx->storage.mission_dir, strerror(errno));
exit(1);
}
@@ -150,60 +186,77 @@ void delivery_init_dirs_stage1(struct Delivery *ctx) {
}
int delivery_init_platform(struct Delivery *ctx) {
- msg(STASIS_MSG_L2, "Setting architecture\n");
+ SYSDEBUG("Setting architecture");
char archsuffix[20];
struct utsname uts;
if (uname(&uts)) {
- msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "uname() failed: %s\n", strerror(errno));
+ SYSERROR("uname() failed: %s", strerror(errno));
return -1;
}
ctx->system.platform = calloc(DELIVERY_PLATFORM_MAX + 1, sizeof(*ctx->system.platform));
if (!ctx->system.platform) {
- SYSERROR("Unable to allocate %d records for platform array\n", DELIVERY_PLATFORM_MAX);
+ SYSERROR("Unable to allocate %d records for platform array", DELIVERY_PLATFORM_MAX);
return -1;
}
for (size_t i = 0; i < DELIVERY_PLATFORM_MAX; i++) {
ctx->system.platform[i] = calloc(DELIVERY_PLATFORM_MAXLEN, sizeof(*ctx->system.platform[0]));
+ if (!ctx->system.platform[i]) {
+ SYSERROR("Unable to allocate record %zu in platform array", i);
+ guard_array_n_free(ctx->system.platform, i);
+ return -1;
+ }
}
ctx->system.arch = strdup(uts.machine);
if (!ctx->system.arch) {
// memory error
+ guard_array_n_free(ctx->system.platform, DELIVERY_PLATFORM_MAX);
+ ctx->system.platform = NULL;
return -1;
}
if (!strcmp(ctx->system.arch, "x86_64")) {
- strcpy(archsuffix, "64");
+ strncpy(archsuffix, "64", sizeof(archsuffix) - 1);
} else {
- strcpy(archsuffix, ctx->system.arch);
+ strncpy(archsuffix, ctx->system.arch, sizeof(archsuffix) - 1);
}
+ archsuffix[sizeof(archsuffix) - 1] = '\0';
+
+ SYSDEBUG("Setting platform");
+ strncpy(ctx->system.platform[DELIVERY_PLATFORM], uts.sysname, DELIVERY_PLATFORM_MAXLEN - 1);
+ ctx->system.platform[DELIVERY_PLATFORM][DELIVERY_PLATFORM_MAXLEN - 1] = '\0';
- msg(STASIS_MSG_L2, "Setting platform\n");
- strcpy(ctx->system.platform[DELIVERY_PLATFORM], uts.sysname);
if (!strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Darwin")) {
- sprintf(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], "osx-%s", archsuffix);
- strcpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], "MacOSX");
- strcpy(ctx->system.platform[DELIVERY_PLATFORM_RELEASE], "macos");
+ snprintf(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], DELIVERY_PLATFORM_MAXLEN, "osx-%s", archsuffix);
+ strncpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], "MacOSX", DELIVERY_PLATFORM_MAXLEN - 1);
+ ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER][DELIVERY_PLATFORM_MAXLEN - 1] = '\0';
+ strncpy(ctx->system.platform[DELIVERY_PLATFORM_RELEASE], "macos", DELIVERY_PLATFORM_MAXLEN - 1);
+ ctx->system.platform[DELIVERY_PLATFORM_RELEASE][DELIVERY_PLATFORM_MAXLEN - 1] = '\0';
} else if (!strcmp(ctx->system.platform[DELIVERY_PLATFORM], "Linux")) {
- sprintf(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], "linux-%s", archsuffix);
- strcpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], "Linux");
- strcpy(ctx->system.platform[DELIVERY_PLATFORM_RELEASE], "linux");
+ snprintf(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], DELIVERY_PLATFORM_MAXLEN, "linux-%s", archsuffix);
+ strncpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], "Linux", DELIVERY_PLATFORM_MAXLEN - 1);
+ ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER][DELIVERY_PLATFORM_MAXLEN - 1] = '\0';
+ strncpy(ctx->system.platform[DELIVERY_PLATFORM_RELEASE], "linux", DELIVERY_PLATFORM_MAXLEN - 1);
+ ctx->system.platform[DELIVERY_PLATFORM_RELEASE][DELIVERY_PLATFORM_MAXLEN - 1] = '\0';
} else {
// Not explicitly supported systems
- strcpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], ctx->system.platform[DELIVERY_PLATFORM]);
- strcpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], ctx->system.platform[DELIVERY_PLATFORM]);
- strcpy(ctx->system.platform[DELIVERY_PLATFORM_RELEASE], ctx->system.platform[DELIVERY_PLATFORM]);
+ strncpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR], ctx->system.platform[DELIVERY_PLATFORM], DELIVERY_PLATFORM_MAXLEN - 1);
+ ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR][DELIVERY_PLATFORM_MAXLEN - 1] = '\0';
+ strncpy(ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER], ctx->system.platform[DELIVERY_PLATFORM], DELIVERY_PLATFORM_MAXLEN - 1);
+ ctx->system.platform[DELIVERY_PLATFORM_CONDA_INSTALLER][DELIVERY_PLATFORM_MAXLEN - 1] = '\0';
+ strncpy(ctx->system.platform[DELIVERY_PLATFORM_RELEASE], ctx->system.platform[DELIVERY_PLATFORM], DELIVERY_PLATFORM_MAXLEN - 1);
+ ctx->system.platform[DELIVERY_PLATFORM_RELEASE][DELIVERY_PLATFORM_MAXLEN - 1] = '\0';
tolower_s(ctx->system.platform[DELIVERY_PLATFORM_RELEASE]);
}
long cpu_count = get_cpu_count();
if (!cpu_count) {
- fprintf(stderr, "Unable to determine CPU count. Falling back to 1.\n");
+ SYSERROR("Unable to determine CPU count. Falling back to 1.");
cpu_count = 1;
}
char ncpus[100] = {0};
- sprintf(ncpus, "%ld", cpu_count);
+ snprintf(ncpus, sizeof(ncpus), "%ld", cpu_count);
// Declare some important bits as environment variables
setenv("CPU_COUNT", ncpus, 1);
@@ -246,22 +299,33 @@ int delivery_init(struct Delivery *ctx, int render_mode) {
}
// Configure architecture and platform information
- delivery_init_platform(ctx);
+ if (delivery_init_platform(ctx)) {
+ // memory error
+ return -1;
+ }
// Create STASIS directory structure
delivery_init_dirs_stage1(ctx);
char config_local[PATH_MAX];
- sprintf(config_local, "%s/%s", ctx->storage.tmpdir, "config");
+ snprintf(config_local, sizeof(config_local), "%s/%s", ctx->storage.tmpdir, "config");
setenv("XDG_CONFIG_HOME", config_local, 1);
+ if (mkdirs(config_local, 0755)) {
+ SYSERROR("%s: unable to create directory", config_local);
+ // fall through because XDG doesn't _really_ need to be there
+ }
char cache_local[PATH_MAX];
- sprintf(cache_local, "%s/%s", ctx->storage.tmpdir, "cache");
+ snprintf(cache_local, sizeof(cache_local), "%s/%s", ctx->storage.tmpdir, "cache");
setenv("XDG_CACHE_HOME", cache_local, 1);
+ if (mkdirs(cache_local, 0755)) {
+ SYSERROR("%s: unable to create directory", cache_local);
+ // fall through because XDG doesn't _really_ need to be there
+ }
// add tools to PATH
char pathvar_tmp[STASIS_BUFSIZ];
- sprintf(pathvar_tmp, "%s/bin:%s", ctx->storage.tools_dir, getenv("PATH"));
+ snprintf(pathvar_tmp, sizeof(pathvar_tmp), "%s/bin:%s", ctx->storage.tools_dir, getenv("PATH"));
setenv("PATH", pathvar_tmp, 1);
// Prevent git from paginating output
@@ -287,18 +351,31 @@ int delivery_init(struct Delivery *ctx, int render_mode) {
int bootstrap_build_info(struct Delivery *ctx) {
struct Delivery local = {0};
+ memcpy(&local.deploy.docker.capabilities, &ctx->deploy.docker.capabilities, sizeof(local.deploy.docker.capabilities));
+
+ SYSDEBUG("ini_open(%s)", ctx->_stasis_ini_fp.cfg_path);
local._stasis_ini_fp.cfg = ini_open(ctx->_stasis_ini_fp.cfg_path);
+ SYSDEBUG("ini_open(%s)", ctx->_stasis_ini_fp.delivery_path);
local._stasis_ini_fp.delivery = ini_open(ctx->_stasis_ini_fp.delivery_path);
+
if (delivery_init_platform(&local)) {
+ SYSDEBUG("delivery_init_platform failed");
+ delivery_free(&local);
return -1;
}
if (populate_delivery_cfg(&local, INI_READ_RENDER)) {
+ SYSDEBUG("populate_delivery_cfg failed");
+ delivery_free(&local);
return -1;
}
if (populate_delivery_ini(&local, INI_READ_RENDER)) {
+ SYSDEBUG("populate_delivery_ini failed");
+ delivery_free(&local);
return -1;
}
if (populate_info(&local)) {
+ SYSDEBUG("populate_info failed");
+ delivery_free(&local);
return -1;
}
ctx->info.build_name = strdup(local.info.build_name);
@@ -308,12 +385,14 @@ int bootstrap_build_info(struct Delivery *ctx) {
ctx->info.time_info = malloc(sizeof(*ctx->info.time_info));
if (!ctx->info.time_info) {
SYSERROR("Unable to allocate %zu bytes for tm struct: %s", sizeof(*local.info.time_info), strerror(errno));
+ delivery_free(&local);
return -1;
}
}
memcpy(ctx->info.time_info, local.info.time_info, sizeof(*local.info.time_info));
ctx->info.time_now = local.info.time_now;
ctx->info.time_str_epoch = strdup(local.info.time_str_epoch);
+ SYSDEBUG("delivery_free local resources");
delivery_free(&local);
return 0;
}
@@ -321,11 +400,11 @@ int bootstrap_build_info(struct Delivery *ctx) {
int delivery_exists(struct Delivery *ctx) {
int release_exists = DELIVERY_NOT_FOUND;
char release_pattern[PATH_MAX] = {0};
- sprintf(release_pattern, "*%s*", ctx->info.release_name);
+ snprintf(release_pattern, sizeof(release_pattern), "*%s*", ctx->info.release_name);
if (globals.enable_artifactory) {
if (jfrt_auth_init(&ctx->deploy.jfrog_auth)) {
- fprintf(stderr, "Failed to initialize Artifactory authentication context\n");
+ SYSERROR("Failed to initialize Artifactory authentication context");
return -1; // error
}
diff --git a/src/lib/delivery/delivery_install.c b/src/lib/delivery/delivery_install.c
index 246c604..efdb819 100644
--- a/src/lib/delivery/delivery_install.c
+++ b/src/lib/delivery/delivery_install.c
@@ -1,8 +1,11 @@
#include "delivery.h"
+#include "conda.h"
+#include "wheelinfo.h"
+#include "version_compare.h"
static struct Test *requirement_from_test(struct Delivery *ctx, const char *name) {
struct Test *result = NULL;
- for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) {
+ for (size_t i = 0; i < ctx->tests->num_used; i++) {
char *package_name = strdup(name);
if (package_name) {
char *spec = find_version_spec(package_name);
@@ -11,8 +14,8 @@ static struct Test *requirement_from_test(struct Delivery *ctx, const char *name
}
remove_extras(package_name);
- if (ctx->tests[i].name && !strcmp(package_name, ctx->tests[i].name)) {
- result = &ctx->tests[i];
+ if (ctx->tests->test[i]->name && !strcmp(package_name, ctx->tests->test[i]->name)) {
+ result = ctx->tests->test[i];
guard_free(package_name);
break;
}
@@ -32,11 +35,13 @@ static char *have_spec_in_config(const struct Delivery *ctx, const char *name) {
char package[255] = {0};
if (op) {
strncpy(package, config_spec, op - config_spec);
+ package[op - config_spec] = '\0';
} else {
strncpy(package, config_spec, sizeof(package) - 1);
+ package[sizeof(package) - 1] = '\0';
}
remove_extras(package);
- if (strncmp(package, name, strlen(package)) == 0) {
+ if (strncmp(package, name, strlen(name)) == 0) {
return config_spec;
}
}
@@ -81,8 +86,10 @@ int delivery_overlay_packages_from_env(struct Delivery *ctx, const char *env_nam
char *op = find_version_spec(spec);
if (op) {
strncpy(spec_name, spec, op - spec);
+ spec_name[op - spec] = '\0';
} else {
strncpy(spec_name, spec, sizeof(spec_name) - 1);
+ spec_name[sizeof(spec_name) - 1] = '\0';
}
struct Test *test_block = requirement_from_test(ctx, spec_name);
@@ -102,8 +109,10 @@ int delivery_overlay_packages_from_env(struct Delivery *ctx, const char *env_nam
// we only care about packages with specs here. if something else arrives, ignore it
if (op) {
strncpy(frozen_name, frozen_spec, op - frozen_spec);
+ frozen_name[op - frozen_spec] = '\0';
} else {
strncpy(frozen_name, frozen_spec, sizeof(frozen_name) - 1);
+ frozen_name[sizeof(frozen_name) - 1] = '\0';
}
struct Test *test = requirement_from_test(ctx, frozen_name);
if (test && strcmp(test->name, frozen_name) == 0) {
@@ -128,6 +137,159 @@ int delivery_overlay_packages_from_env(struct Delivery *ctx, const char *env_nam
return 0;
}
+int delivery_conda_enforce_package_version(struct Delivery *ctx, const char *env_name, const char *name) {
+ char *spec_installed = NULL;
+ char *spec_request = NULL;
+ int status = 0;
+
+ if (isempty((char *) env_name)) {
+ SYSERROR("environment name cannot be NULL or empty");
+ return -1;
+ }
+ if (isempty((char *) name)) {
+ SYSERROR("name cannot be NULL or empty");
+ return -1;
+ }
+
+ int proc_status = 0;
+ char cmd[PATH_MAX] = {0};
+ snprintf(cmd, PATH_MAX, "conda list --name %s", env_name);
+
+ char *output = shell_output(cmd, &proc_status);
+ if (!output || proc_status) {
+ SYSERROR("unable to retreive list of installed packages (exit: %d)", proc_status);
+ guard_free(output);
+ return -1;
+ }
+
+ struct StrList *lines = strlist_init();
+ if (!lines) {
+ SYSERROR("unable to allocate memory for installed package list");
+ guard_free(output);
+ status = -1;
+ goto cleanup;
+ }
+
+ if (strlist_append_tokenize(lines, output, LINE_SEP)) {
+ SYSERROR("unable to tokenize installed package list");
+ guard_free(output);
+ strlist_free(&lines);
+ status = -1;
+ goto cleanup;
+ }
+
+ for (size_t i = 0; i < strlist_count(lines); i++) {
+ char *line = strlist_item(lines, i);
+ if (!line) {
+ SYSERROR("line is NULL");
+ status = -1;
+ goto cleanup;
+ }
+ if (startswith(line, "#") || isempty(line)) {
+ continue;
+ }
+ collapse_whitespace(&line);
+ strip(line);
+
+ struct StrList *tokens = strlist_init();
+ if (!tokens) {
+ SYSERROR("unable to allocate memory for tokenized installed package list");
+ status = -1;
+ goto cleanup;
+ }
+
+ if (strlist_append_tokenize(tokens, line, " ")) {
+ SYSERROR("unable to tokenize installed package list");
+ status = -1;
+ goto cleanup;
+ }
+
+ const char *installed_version = strlist_item(tokens, 1);
+ if (!installed_version) {
+ SYSERROR("not enough data in line (name and version not found)");
+ guard_strlist_free(&tokens);
+ status = -1;
+ goto cleanup;
+ }
+
+ if (strstr(line, name)) {
+ spec_installed = strdup(installed_version);
+ if (!spec_installed) {
+ SYSERROR("unable to allocated memory for installed package version");
+ guard_strlist_free(&tokens);
+ status = -1;
+ goto cleanup;
+ }
+ guard_strlist_free(&tokens);
+ break;
+ }
+
+ guard_strlist_free(&tokens);
+ }
+
+ for (size_t i = 0; i < strlist_count(ctx->conda.conda_packages); i++) {
+ const char *item = strlist_item(ctx->conda.conda_packages, i);
+ if (!item) {
+ SYSERROR("conda_packages list record %zu is NULL", i);
+ status = -1;
+ goto cleanup;
+ }
+ if (strstr(item, name)) {
+ const char *spec_tmp = find_version_spec((char *) item);
+ if (spec_tmp) {
+ while (!isalnum(*spec_tmp)) {
+ spec_tmp++;
+ }
+ spec_request = strdup(spec_tmp);
+ } else {
+ spec_request = strdup(item);
+ }
+
+ if (!spec_request) {
+ SYSERROR("unable to allocate memory for conda package spec request");
+ status = -1;
+ goto cleanup;
+ }
+ break;
+ }
+ }
+
+ const int stop = version_compare(NOT | EQ, spec_request, spec_installed);
+ if (stop < 0) {
+ SYSERROR("version comparison failed (spec_request: %s, spec_installed: %s)", spec_request, spec_installed);
+ status = -1;
+ goto cleanup;
+ }
+ if (stop == 0) {
+ goto cleanup;
+ }
+
+ snprintf(cmd, PATH_MAX, "remove --name %s %s", env_name, name);
+ if (conda_exec(cmd)) {
+ SYSERROR("unable to remove package %s from %s", name, env_name);
+ status = -1;
+ goto cleanup;
+ }
+ snprintf(cmd, PATH_MAX, "install --name %s %s=%s", env_name, name, spec_request);
+ if (conda_exec(cmd)) {
+ SYSERROR("unable to install package %s into %s", name, env_name);
+ status = -1;
+ goto cleanup;
+ }
+
+ cleanup:
+ guard_free(spec_request);
+ guard_free(spec_installed);
+ strlist_free(&lines);
+ guard_free(output);
+ return status;
+}
+
+static int fn_nop(const char *command) {
+ (void) command;
+ return 1;
+}
+
int delivery_purge_packages(struct Delivery *ctx, const char *env_name, int use_pkg_manager) {
int status = 0;
char subcommand[100] = {0};
@@ -145,19 +307,24 @@ int delivery_purge_packages(struct Delivery *ctx, const char *env_name, int use_
case PKG_USE_CONDA:
fn = conda_exec;
list = ctx->conda.conda_packages_purge;
- strcpy(package_manager, "conda");
+ strncpy(package_manager, "conda", sizeof(package_manager) - 1);
+ package_manager[sizeof(package_manager) - 1] = '\0';
// conda is already configured for "always_yes"
- strcpy(subcommand, "remove");
+ strncpy(subcommand, "remove", sizeof(subcommand) - 1);
+ subcommand[sizeof(subcommand) - 1] = '\0';
break;
case PKG_USE_PIP:
fn = pip_exec;
list = ctx->conda.pip_packages_purge;
- strcpy(package_manager, "pip");
+ strncpy(package_manager, "pip", sizeof(package_manager) - 1);
+ package_manager[sizeof(package_manager) - 1] = '\0';
// avoid user prompt to remove packages
- strcpy(subcommand, "uninstall -y");
+ strncpy(subcommand, "uninstall -y", sizeof(subcommand) - 1);
+ subcommand[sizeof(subcommand) - 1] = '\0';
break;
default:
SYSERROR("Unknown package manager: %d", use_pkg_manager);
+ fn = fn_nop;
status = -1;
break;
}
@@ -166,7 +333,7 @@ int delivery_purge_packages(struct Delivery *ctx, const char *env_name, int use_
char *package = strlist_item(list, i);
char *command = NULL;
if (asprintf(&command, "%s '%s'", subcommand, package) < 0) {
- SYSERROR("%s", "Unable to allocate bytes for removal command");
+ SYSERROR("Unable to allocate bytes for removal command");
status = -1;
break;
}
@@ -175,11 +342,12 @@ int delivery_purge_packages(struct Delivery *ctx, const char *env_name, int use_
SYSERROR("%s removal operation failed", package_manager);
guard_free(command);
status = 1;
- break;
+ goto cleanup;
}
guard_free(command);
}
+ cleanup:
if (current_env) {
conda_activate(ctx->storage.conda_install_prefix, current_env);
guard_free(current_env);
@@ -189,8 +357,7 @@ int delivery_purge_packages(struct Delivery *ctx, const char *env_name, int use_
}
int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, char *env_name, int type, struct StrList **manifest) {
- char cmd[PATH_MAX];
- char pkgs[STASIS_BUFSIZ];
+ char command_base[PATH_MAX];
const char *env_current = getenv("CONDA_DEFAULT_ENV");
if (env_current) {
@@ -203,9 +370,8 @@ int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, cha
}
}
- memset(cmd, 0, sizeof(cmd));
- memset(pkgs, 0, sizeof(pkgs));
- strcat(cmd, "install");
+ memset(command_base, 0, sizeof(command_base));
+ strncat(command_base, "install", sizeof(command_base) - strlen(command_base) - 1);
typedef int (*Runner)(const char *);
Runner runner = NULL;
@@ -215,18 +381,31 @@ int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, cha
runner = pip_exec;
}
+ if (!runner) {
+ SYSERROR("Invalid callback runner of type: %d", type);
+ return -1;
+ }
+
if (INSTALL_PKG_CONDA_DEFERRED & type) {
- strcat(cmd, " --use-local");
+ strncat(command_base, " --use-local", sizeof(command_base) - strlen(command_base) - 1);
+ command_base[sizeof(command_base) - 1] = '\0';
} else if (INSTALL_PKG_PIP_DEFERRED & type) {
// Don't change the baseline package set unless we're working with a
// new build. Release candidates will need to keep packages as stable
// as possible between releases.
if (!ctx->meta.based_on) {
- strcat(cmd, " --upgrade");
+ strncat(command_base, " --upgrade", sizeof(command_base) - strlen(command_base) - 1);
+ command_base[sizeof(command_base) - 1] = '\0';
}
+ snprintf(command_base + strlen(command_base), sizeof(command_base) - strlen(command_base), " --extra-index-url 'file://%s'", ctx->storage.wheel_artifact_dir);
}
- sprintf(cmd + strlen(cmd), " --extra-index-url 'file://%s'", ctx->storage.wheel_artifact_dir);
+ size_t args_alloc_len = STASIS_BUFSIZ;
+ char *args = calloc(args_alloc_len + 1, sizeof(*args));
+ if (!args) {
+ SYSERROR("Unable to allocate bytes for command arguments");
+ return -1;
+ }
for (size_t x = 0; manifest[x] != NULL; x++) {
char *name = NULL;
@@ -237,79 +416,131 @@ int delivery_install_packages(struct Delivery *ctx, char *conda_install_dir, cha
continue;
}
if (INSTALL_PKG_PIP_DEFERRED & type) {
+ SYSDEBUG("Getting requirements for test: %s", name);
struct Test *info = requirement_from_test(ctx, name);
if (info) {
- if (!strcmp(info->version, "HEAD")) {
+ if (!strcmp(info->version, "HEAD") || is_git_sha(info->version)) {
+ SYSDEBUG("Using version: %s", info->version);
struct StrList *tag_data = strlist_init();
if (!tag_data) {
- SYSERROR("%s", "Unable to allocate memory for tag data\n");
+ SYSERROR("Unable to allocate memory for tag data");
+ guard_free(args);
return -1;
}
+ SYSDEBUG("Tokenizing repository info tag: %s", info->repository_info_tag);
strlist_append_tokenize(tag_data, info->repository_info_tag, "-");
- struct Wheel *whl = NULL;
+ struct WheelInfo *whl = NULL;
char *post_commit = NULL;
char *hash = NULL;
if (strlist_count(tag_data) > 1) {
post_commit = strlist_item(tag_data, 1);
+ SYSDEBUG("post_commit: %s", post_commit);
hash = strlist_item(tag_data, 2);
+ SYSDEBUG("hash: %s", hash);
}
// We can't match on version here (index 0). The wheel's version is not guaranteed to be
// equal to the tag; setuptools_scm auto-increments the value, the user can change it manually,
// etc.
errno = 0;
- whl = get_wheel_info(ctx->storage.wheel_artifact_dir, info->name,
+ SYSDEBUG("%s", "Getting wheel information");
+ whl = wheelinfo_get(ctx->storage.wheel_artifact_dir, info->name,
(char *[]) {ctx->meta.python_compact, ctx->system.arch,
"none", "any",
post_commit, hash,
NULL}, WHEEL_MATCH_ANY);
if (!whl && errno) {
// error
- SYSERROR("Unable to read Python wheel info: %s\n", strerror(errno));
+ SYSERROR("Unable to read Python wheel info: %s", strerror(errno));
exit(1);
} else if (!whl) {
// not found
- fprintf(stderr, "No wheel packages found that match the description of '%s'", info->name);
+ SYSERROR("No wheel packages found that match the description of '%s'", info->name);
} else {
- // found
- guard_strlist_free(&tag_data);
+ // found, replace the original version with newly detected version
+ SYSDEBUG("Replacing version: %s", whl->version);
+ guard_free(info->version);
info->version = strdup(whl->version);
+ SYSDEBUG("Version replaced with: %s", whl->version);
}
- wheel_free(&whl);
+ guard_strlist_free(&tag_data);
+ wheelinfo_free(&whl);
}
char req[255] = {0};
if (!strcmp(name, info->name)) {
- strcpy(req, info->name);
+ strncpy(req, info->name, sizeof(req) - 1);
+ req[sizeof(req) - 1] = '\0';
} else {
- strcpy(req, name);
+ strncpy(req, name, sizeof(req) - 1);
+ req[sizeof(req) - 1] = '\0';
char *spec = find_version_spec(req);
if (spec) {
*spec = 0;
}
}
- snprintf(cmd + strlen(cmd),
- sizeof(cmd) - strlen(cmd) - strlen(info->name) - strlen(info->version) + 5,
- " '%s==%s'", req, info->version);
+ const char *fmt_append = "%s '%s==%s'";
+ const char *fmt = " '%s==%s'";
+ const int required_len = snprintf(NULL, 0, fmt_append, args, req, info->version);
+ if (required_len > (int) args_alloc_len) {
+ if (grow(required_len, &args_alloc_len, &args)) {
+ SYSERROR("Unable to allocate %d bytes for command arguments", required_len);
+ guard_free(args);
+ return -1;
+ }
+ }
+ snprintf(args + strlen(args), args_alloc_len - strlen(args), fmt, req, info->version);
} else {
- fprintf(stderr, "Deferred package '%s' is not present in the tested package list!\n", name);
+ SYSERROR("Deferred package '%s' is not present in the tested package list!", name);
+ guard_free(args);
return -1;
}
} else {
if (startswith(name, "--") || startswith(name, "-")) {
- sprintf(cmd + strlen(cmd), " %s", name);
+ const char *fmt_append = "%s %s";
+ const char *fmt = " %s";
+ const int required_len = snprintf(NULL, 0, fmt_append, args, name);
+ if (required_len > (int) args_alloc_len) {
+ if (grow(required_len, &args_alloc_len, &args)) {
+ SYSERROR("Unable to allocate %d bytes for command arguments", required_len);
+ guard_free(args);
+ return -1;
+ }
+ }
+ snprintf(args + strlen(args), args_alloc_len - strlen(args), fmt, name);
} else {
- sprintf(cmd + strlen(cmd), " '%s'", name);
+ const char *fmt_append = "%s '%s'";
+ const char *fmt = " '%s'";
+ const int required_len = snprintf(NULL, 0, fmt_append, args, name);
+ if (required_len > (int) args_alloc_len) {
+ if (grow(required_len, &args_alloc_len, &args)) {
+ SYSERROR("Unable to allocate %d bytes for command arguments", required_len);
+ guard_free(args);
+ return -1;
+ }
+ }
+ snprintf(args + strlen(args), args_alloc_len - strlen(args), fmt, name);
}
}
}
- int status = runner(cmd);
+ char *command = NULL;
+ if (asprintf(&command, "%s %s", command_base, args) < 0) {
+ SYSERROR("Unable to allocate bytes for command");
+ guard_free(args);
+ return -1;
+ }
+
+ int status = runner(command);
+ guard_free(args);
+ guard_free(command);
if (status) {
+ // fail quickly
return status;
}
}
+ guard_free(args);
return 0;
}
diff --git a/src/lib/delivery/delivery_populate.c b/src/lib/delivery/delivery_populate.c
index 84676f1..cfa3da2 100644
--- a/src/lib/delivery/delivery_populate.c
+++ b/src/lib/delivery/delivery_populate.c
@@ -33,18 +33,18 @@ int populate_info(struct Delivery *ctx) {
if (!ctx->info.time_info) {
ctx->info.time_info = malloc(sizeof(*ctx->info.time_info));
if (!ctx->info.time_info) {
- msg(STASIS_MSG_ERROR, "%s: Unable to allocate memory for time_info\n", strerror(errno));
+ SYSERROR("%s: Unable to allocate memory for time_info", strerror(errno));
return -1;
}
if (!localtime_r(&ctx->info.time_now, ctx->info.time_info)) {
- msg(STASIS_MSG_ERROR, "%s: localtime_r failed\n", strerror(errno));
+ SYSERROR("%s: localtime_r failed", strerror(errno));
return -1;
}
}
ctx->info.time_str_epoch = calloc(STASIS_TIME_STR_MAX, sizeof(*ctx->info.time_str_epoch));
if (!ctx->info.time_str_epoch) {
- msg(STASIS_MSG_ERROR, "%s: Unable to allocate memory for Unix epoch string\n", strerror(errno));
+ SYSERROR("%s: Unable to allocate memory for Unix epoch string", strerror(errno));
return -1;
}
snprintf(ctx->info.time_str_epoch, STASIS_TIME_STR_MAX - 1, "%li", ctx->info.time_now);
@@ -55,6 +55,7 @@ int populate_info(struct Delivery *ctx) {
int populate_delivery_cfg(struct Delivery *ctx, int render_mode) {
struct INIFILE *cfg = ctx->_stasis_ini_fp.cfg;
if (!cfg) {
+ SYSDEBUG("cfg is NULL");
return -1;
}
int err = 0;
@@ -84,6 +85,45 @@ int populate_delivery_cfg(struct Delivery *ctx, int render_mode) {
}
globals.pip_packages = ini_getval_strlist(cfg, "default", "pip_packages", LINE_SEP, render_mode, &err);
+ err = 0;
+ if (!globals.wheel_builder) {
+ globals.wheel_builder = ini_getval_str(cfg, "default", "wheel_builder", render_mode, &err);
+ if (err) {
+ SYSWARN("wheel_builder is undefined. Falling back to system toolchain: 'build'.");
+ globals.wheel_builder = strdup("build");
+ if (!globals.wheel_builder) {
+ SYSERROR("unable to allocate memory for default wheel_builder value");
+ return -1;
+ }
+ }
+ }
+
+ err = 0;
+ if (!globals.wheel_builder_manylinux_image) {
+ globals.wheel_builder_manylinux_image = ini_getval_str(cfg, "default", "wheel_builder_manylinux_image", render_mode, &err);
+ }
+
+ if (err && globals.wheel_builder && strcmp(globals.wheel_builder, "manylinux") == 0) {
+ SYSERROR("default:wheel_builder is set to 'manylinux', however default:wheel_builder_manylinux_image is not configured");
+ return -1;
+ }
+
+ if (strcmp(globals.wheel_builder, "manylinux") == 0) {
+ char *manifest_inspect_cmd = NULL;
+ if (asprintf(&manifest_inspect_cmd, "manifest inspect '%s'", globals.wheel_builder_manylinux_image) < 0) {
+ SYSERROR("unable to allocate memory for docker command");
+ guard_free(manifest_inspect_cmd);
+ return -1;
+ }
+ if (ctx->deploy.docker.capabilities.usable && docker_exec(manifest_inspect_cmd, STASIS_DOCKER_QUIET_STDOUT)) {
+ SYSERROR("Image provided by default:wheel_builder_manylinux_image does not exist: %s", globals.wheel_builder_manylinux_image);
+ guard_free(manifest_inspect_cmd);
+ return -1;
+ }
+ guard_free(manifest_inspect_cmd);
+ }
+
+
if (globals.jfrog.jfrog_artifactory_base_url) {
guard_free(globals.jfrog.jfrog_artifactory_base_url);
}
@@ -153,6 +193,7 @@ static void normalize_ini_list(struct INIFILE **inip, struct StrList **listp, ch
(*inip) = ini;
(*listp) = list;
}
+
int populate_delivery_ini(struct Delivery *ctx, int render_mode) {
struct INIFILE *ini = ctx->_stasis_ini_fp.delivery;
struct INIData *rtdata;
@@ -162,8 +203,6 @@ int populate_delivery_ini(struct Delivery *ctx, int render_mode) {
// keys in the configuration
RuntimeEnv *rt = runtime_copy(__environ);
while ((rtdata = ini_getall(ini, "runtime")) != NULL) {
- char rec[STASIS_BUFSIZ];
- sprintf(rec, "%s=%s", lstrip(strip(rtdata->key)), lstrip(strip(rtdata->value)));
runtime_set(rt, rtdata->key, rtdata->value);
}
runtime_apply(rt);
@@ -201,7 +240,9 @@ int populate_delivery_ini(struct Delivery *ctx, int render_mode) {
normalize_ini_list(&ini, &ctx->conda.pip_packages_purge, "conda", "pip_packages_purge", render_mode);
// Delivery metadata consumed
- populate_mission_ini(&ctx, render_mode);
+ if (populate_mission_ini(&ctx, render_mode)) {
+ return -1;
+ }
if (ctx->info.release_name) {
guard_free(ctx->info.release_name);
@@ -215,16 +256,16 @@ int populate_delivery_ini(struct Delivery *ctx, int render_mode) {
return -1;
}
- if (delivery_format_str(ctx, &ctx->info.release_name, ctx->rules.release_fmt)) {
- fprintf(stderr, "Failed to generate release name. Format used: %s\n", ctx->rules.release_fmt);
+ if (delivery_format_str(ctx, &ctx->info.release_name, STASIS_NAME_MAX, ctx->rules.release_fmt)) {
+ SYSERROR("Failed to generate release name. Format used: %s", ctx->rules.release_fmt);
return -1;
}
if (!ctx->info.build_name) {
- delivery_format_str(ctx, &ctx->info.build_name, ctx->rules.build_name_fmt);
+ delivery_format_str(ctx, &ctx->info.build_name, STASIS_NAME_MAX, ctx->rules.build_name_fmt);
}
if (!ctx->info.build_number) {
- delivery_format_str(ctx, &ctx->info.build_number, ctx->rules.build_number_fmt);
+ delivery_format_str(ctx, &ctx->info.build_number, STASIS_NAME_MAX, ctx->rules.build_number_fmt);
}
// Best I can do to make output directories unique. Annoying.
@@ -237,11 +278,17 @@ int populate_delivery_ini(struct Delivery *ctx, int render_mode) {
ctx->conda.pip_packages_defer = strlist_init();
}
- for (size_t z = 0, i = 0; i < ini->section_count; i++) {
+ ctx->tests = tests_init(TEST_NUM_ALLOC_INITIAL);
+ for (size_t i = 0; i < ini->section_count; i++) {
char *section_name = ini->section[i]->key;
if (startswith(section_name, "test:")) {
union INIVal val;
- struct Test *test = &ctx->tests[z];
+ struct Test *test = test_init();
+ if (!test) {
+ SYSERROR("unable to allocate memory for test structure");
+ return -1;
+ }
+
val.as_char_p = strchr(ini->section[i]->key, ':') + 1;
if (val.as_char_p && isempty(val.as_char_p)) {
return 1;
@@ -259,8 +306,21 @@ int populate_delivery_ini(struct Delivery *ctx, int render_mode) {
}
test->repository_remove_tags = ini_getval_strlist(ini, section_name, "repository_remove_tags", LINE_SEP, render_mode, &err);
test->build_recipe = ini_getval_str(ini, section_name, "build_recipe", render_mode, &err);
- test->runtime.environ = ini_getval_strlist(ini, section_name, "runtime", LINE_SEP, render_mode, &err);
- z++;
+
+ test->runtime->environ = ini_getval_strlist(ini, section_name, "runtime", LINE_SEP, render_mode, &err);
+ const char *timeout_str = ini_getval_str(ini, section_name, "timeout", render_mode, &err);
+ if (timeout_str) {
+ test->timeout = str_to_timeout((char *) timeout_str);
+ if (test->timeout == STR_TO_TIMEOUT_INVALID_TIME_SCALE) {
+ SYSERROR("In 'test:%s', invalid time scale format: %s. Use n[hms].", test->name, timeout_str);
+ return 1;
+ }
+ if (test->timeout == STR_TO_TIMEOUT_NEGATIVE) {
+ SYSERROR("In 'test:%s', timeout cannot be negative: %s", test->name, timeout_str);
+ return 1;
+ }
+ }
+ tests_add(ctx->tests, test);
}
}
@@ -309,25 +369,26 @@ int populate_mission_ini(struct Delivery **ctx, int render_mode) {
int err = 0;
if ((*ctx)->_stasis_ini_fp.mission) {
+ // mission configurations are optional
return 0;
}
// Now populate the rules
char missionfile[PATH_MAX] = {0};
if (getenv("STASIS_SYSCONFDIR")) {
- sprintf(missionfile, "%s/%s/%s/%s.ini",
+ snprintf(missionfile, sizeof(missionfile), "%s/%s/%s/%s.ini",
getenv("STASIS_SYSCONFDIR"), "mission", (*ctx)->meta.mission, (*ctx)->meta.mission);
} else {
- sprintf(missionfile, "%s/%s/%s/%s.ini",
+ snprintf(missionfile, sizeof(missionfile), "%s/%s/%s/%s.ini",
globals.sysconfdir, "mission", (*ctx)->meta.mission, (*ctx)->meta.mission);
}
- msg(STASIS_MSG_L2, "Reading mission configuration: %s\n", missionfile);
+ SYSDEBUG("Reading mission configuration: %s", missionfile);
(*ctx)->_stasis_ini_fp.mission = ini_open(missionfile);
struct INIFILE *ini = (*ctx)->_stasis_ini_fp.mission;
if (!ini) {
- msg(STASIS_MSG_ERROR | STASIS_MSG_L2, "Failed to read mission configuration: %s, %s\n", missionfile, strerror(errno));
- exit(1);
+ SYSERROR("Failed to read mission configuration: %s, %s", missionfile, strerror(errno));
+ return -1;
}
(*ctx)->_stasis_ini_fp.mission_path = strdup(missionfile);
@@ -343,7 +404,7 @@ int populate_mission_ini(struct Delivery **ctx, int render_mode) {
void validate_delivery_ini(struct INIFILE *ini) {
if (!ini) {
- SYSERROR("%s", "INIFILE is NULL!");
+ SYSERROR("INIFILE is NULL!");
exit(1);
}
if (ini_section_search(&ini, INI_SEARCH_EXACT, "meta")) {
@@ -353,7 +414,7 @@ void validate_delivery_ini(struct INIFILE *ini) {
ini_has_key_required(ini, "meta", "mission");
ini_has_key_required(ini, "meta", "python");
} else {
- SYSERROR("%s", "[meta] configuration section is required");
+ SYSERROR("[meta] configuration section is required");
exit(1);
}
@@ -363,7 +424,7 @@ void validate_delivery_ini(struct INIFILE *ini) {
ini_has_key_required(ini, "conda", "installer_platform");
ini_has_key_required(ini, "conda", "installer_arch");
} else {
- SYSERROR("%s", "[conda] configuration section is required");
+ SYSERROR("[conda] configuration section is required");
exit(1);
}
diff --git a/src/lib/delivery/delivery_postprocess.c b/src/lib/delivery/delivery_postprocess.c
index 5029e02..63093b3 100644
--- a/src/lib/delivery/delivery_postprocess.c
+++ b/src/lib/delivery/delivery_postprocess.c
@@ -1,8 +1,12 @@
#include "delivery.h"
+#include "log.h"
+#include "conda.h"
const char *release_header = "# delivery_name: %s\n"
"# delivery_fmt: %s\n"
+ "# stasis_version: %s\n"
+ "# stasis_branch: %s\n"
"# creation_time: %s\n"
"# conda_ident: %s\n"
"# conda_build_ident: %s\n";
@@ -11,9 +15,11 @@ char *delivery_get_release_header(struct Delivery *ctx) {
char output[STASIS_BUFSIZ];
char stamp[100];
strftime(stamp, sizeof(stamp) - 1, "%c", ctx->info.time_info);
- sprintf(output, release_header,
+ snprintf(output, sizeof(output), release_header,
ctx->info.release_name,
ctx->rules.release_fmt,
+ STASIS_VERSION,
+ STASIS_VERSION_BRANCH,
stamp,
ctx->conda.tool_version,
ctx->conda.tool_build_version);
@@ -22,14 +28,16 @@ char *delivery_get_release_header(struct Delivery *ctx) {
int delivery_dump_metadata(struct Delivery *ctx) {
char filename[PATH_MAX];
- sprintf(filename, "%s/meta-%s.stasis", ctx->storage.meta_dir, ctx->info.release_name);
+ snprintf(filename, sizeof(filename), "%s/meta-%s.stasis", ctx->storage.meta_dir, ctx->info.release_name);
FILE *fp = fopen(filename, "w+");
if (!fp) {
return -1;
}
if (globals.verbose) {
- printf("%s\n", filename);
+ msg(STASIS_MSG_L2, "%s\n", filename);
}
+ fprintf(fp, "stasis_version %s\n", STASIS_VERSION);
+ fprintf(fp, "stasis_version_branch %s\n", STASIS_VERSION_BRANCH);
fprintf(fp, "name %s\n", ctx->meta.name);
fprintf(fp, "version %s\n", ctx->meta.version);
fprintf(fp, "rc %d\n", ctx->meta.rc);
@@ -66,16 +74,16 @@ void delivery_rewrite_spec(struct Delivery *ctx, char *filename, unsigned stage)
FILE *tp = NULL;
if (stage == DELIVERY_REWRITE_SPEC_STAGE_1) {
- SYSDEBUG("%s", "Entering stage 1");
+ SYSDEBUG("Entering stage 1");
header = delivery_get_release_header(ctx);
SYSDEBUG("Release header:\n%s", header);
if (!header) {
- msg(STASIS_MSG_ERROR, "failed to generate release header string\n", filename);
+ SYSERROR("failed to generate release header string", filename);
exit(1);
}
tempfile = xmkstemp(&tp, "w+");
if (!tempfile || !tp) {
- msg(STASIS_MSG_ERROR, "%s: unable to create temporary file\n", strerror(errno));
+ SYSERROR("%s: unable to create temporary file", strerror(errno));
exit(1);
}
SYSDEBUG("Writing header to temporary file: %s", tempfile);
@@ -84,7 +92,7 @@ void delivery_rewrite_spec(struct Delivery *ctx, char *filename, unsigned stage)
// Read the original file
char **contents = file_readlines(filename, 0, 0, NULL);
if (!contents) {
- msg(STASIS_MSG_ERROR, "%s: unable to read %s", filename);
+ SYSERROR("%s: unable to read %s", filename);
exit(1);
}
@@ -128,45 +136,45 @@ void delivery_rewrite_spec(struct Delivery *ctx, char *filename, unsigned stage)
// Replace the original file with our temporary data
if (copy2(tempfile, filename, CT_PERM) < 0) {
- fprintf(stderr, "%s: could not rename '%s' to '%s'\n", strerror(errno), tempfile, filename);
+ SYSERROR("%s: could not rename '%s' to '%s'", strerror(errno), tempfile, filename);
exit(1);
}
SYSDEBUG("Removing file: %s", tempfile);
remove(tempfile);
guard_free(tempfile);
} else if (globals.enable_rewrite_spec_stage_2 && stage == DELIVERY_REWRITE_SPEC_STAGE_2) {
- SYSDEBUG("%s", "Entering stage 2");
+ SYSDEBUG("Entering stage 2");
char output[PATH_MAX] = {0};
// Replace "local" channel with the staging URL
if (ctx->storage.conda_staging_url) {
- SYSDEBUG("%s", "Will replace conda channel with staging area url");
+ SYSDEBUG("Will replace conda channel with staging area url");
file_replace_text(filename, "@CONDA_CHANNEL@", ctx->storage.conda_staging_url, 0);
} else if (globals.jfrog.repo) {
- SYSDEBUG("%s", "Will replace conda channel with artifactory repo packages/conda url");
- sprintf(output, "%s/%s/%s/%s/packages/conda", globals.jfrog.url, globals.jfrog.repo, ctx->meta.mission, ctx->info.build_name);
+ SYSDEBUG("Will replace conda channel with artifactory repo packages/conda url");
+ snprintf(output, sizeof(output), "%s/%s/%s/%s/packages/conda", globals.jfrog.url, globals.jfrog.repo, ctx->meta.mission, ctx->info.build_name);
file_replace_text(filename, "@CONDA_CHANNEL@", output, 0);
} else {
- SYSDEBUG("%s", "Will replace conda channel with local conda artifact directory");
- msg(STASIS_MSG_WARN, "conda_staging_dir is not configured. Using fallback: '%s'\n", ctx->storage.conda_artifact_dir);
+ SYSDEBUG("Will replace conda channel with local conda artifact directory");
+ SYSWARN("conda_staging_dir is not configured. Using fallback: '%s'", ctx->storage.conda_artifact_dir);
file_replace_text(filename, "@CONDA_CHANNEL@", ctx->storage.conda_artifact_dir, 0);
}
if (ctx->storage.wheel_staging_url) {
- SYSDEBUG("%s", "Will replace pip arguments with wheel staging url");
- sprintf(output, "--extra-index-url %s/%s/%s/packages/wheels", ctx->storage.wheel_staging_url, ctx->meta.mission, ctx->info.build_name);
+ SYSDEBUG("Will replace pip arguments with wheel staging url");
+ snprintf(output, sizeof(output), "--extra-index-url %s/%s/%s/packages/wheels", ctx->storage.wheel_staging_url, ctx->meta.mission, ctx->info.build_name);
file_replace_text(filename, "@PIP_ARGUMENTS@", ctx->storage.wheel_staging_url, 0);
} else if (globals.enable_artifactory && globals.jfrog.url && globals.jfrog.repo) {
- SYSDEBUG("%s", "Will replace pip arguments with artifactory repo packages/wheel url");
- sprintf(output, "--extra-index-url %s/%s/%s/%s/packages/wheels", globals.jfrog.url, globals.jfrog.repo, ctx->meta.mission, ctx->info.build_name);
+ SYSDEBUG("Will replace pip arguments with artifactory repo packages/wheel url");
+ snprintf(output, sizeof(output), "--extra-index-url %s/%s/%s/%s/packages/wheels", globals.jfrog.url, globals.jfrog.repo, ctx->meta.mission, ctx->info.build_name);
file_replace_text(filename, "@PIP_ARGUMENTS@", output, 0);
} else {
- SYSDEBUG("%s", "Will replace pip arguments with local wheel artifact directory");
- msg(STASIS_MSG_WARN, "wheel_staging_dir is not configured. Using fallback: '%s'\n", ctx->storage.wheel_artifact_dir);
- sprintf(output, "--extra-index-url file://%s", ctx->storage.wheel_artifact_dir);
+ SYSDEBUG("Will replace pip arguments with local wheel artifact directory");
+ SYSWARN("wheel_staging_dir is not configured. Using fallback: '%s'", ctx->storage.wheel_artifact_dir);
+ snprintf(output, sizeof(output), "--extra-index-url file://%s", ctx->storage.wheel_artifact_dir);
file_replace_text(filename, "@PIP_ARGUMENTS@", output, 0);
}
}
- SYSDEBUG("%s", "Rewriting finished");
+ SYSDEBUG("Rewriting finished");
}
int delivery_copy_conda_artifacts(struct Delivery *ctx) {
@@ -177,16 +185,15 @@ int delivery_copy_conda_artifacts(struct Delivery *ctx) {
memset(conda_build_dir, 0, sizeof(conda_build_dir));
memset(subdir, 0, sizeof(subdir));
- sprintf(conda_build_dir, "%s/%s", ctx->storage.conda_install_prefix, "conda-bld");
+ snprintf(conda_build_dir, sizeof(conda_build_dir), "%s/%s", ctx->storage.conda_install_prefix, "conda-bld");
// One must run conda build at least once to create the "conda-bld" directory.
// When this directory is missing there can be no build artifacts.
if (access(conda_build_dir, F_OK) < 0) {
- msg(STASIS_MSG_RESTRICT | STASIS_MSG_WARN | STASIS_MSG_L3,
- "Skipped: 'conda build' has never been executed.\n");
+ SYSWARN("Skipped: 'conda build' has never been executed.");
return 0;
}
- snprintf(cmd, sizeof(cmd) - 1, "rsync -avi --progress %s/%s %s",
+ snprintf(cmd, sizeof(cmd), "rsync -avi --progress %s/%s %s",
conda_build_dir,
ctx->system.platform[DELIVERY_PLATFORM_CONDA_SUBDIR],
ctx->storage.conda_artifact_dir);
@@ -200,7 +207,7 @@ int delivery_index_conda_artifacts(struct Delivery *ctx) {
int delivery_copy_wheel_artifacts(struct Delivery *ctx) {
char cmd[PATH_MAX] = {0};
- snprintf(cmd, sizeof(cmd) - 1, "rsync -avi --progress %s/*/dist/*.whl %s",
+ snprintf(cmd, sizeof(cmd), "rsync -avi --progress %s/*/dist/*.whl %s",
ctx->storage.build_sources_dir,
ctx->storage.wheel_artifact_dir);
return system(cmd);
@@ -217,7 +224,7 @@ int delivery_index_wheel_artifacts(struct Delivery *ctx) {
// Generate a "dumb" local pypi index that is compatible with:
// pip install --extra-index-url
char top_index[PATH_MAX] = {0};
- sprintf(top_index, "%s/index.html", ctx->storage.wheel_artifact_dir);
+ snprintf(top_index, sizeof(top_index),"%s/index.html", ctx->storage.wheel_artifact_dir);
SYSDEBUG("Opening top-level index for writing: %s", top_index);
FILE *top_fp = fopen(top_index, "w+");
if (!top_fp) {
@@ -232,11 +239,12 @@ int delivery_index_wheel_artifacts(struct Delivery *ctx) {
}
char bottom_index[PATH_MAX * 2] = {0};
- sprintf(bottom_index, "%s/%s/index.html", ctx->storage.wheel_artifact_dir, rec->d_name);
+ snprintf(bottom_index, sizeof(bottom_index), "%s/%s/index.html", ctx->storage.wheel_artifact_dir, rec->d_name);
SYSDEBUG("Opening bottom-level for writing: %s", bottom_index);
FILE *bottom_fp = fopen(bottom_index, "w+");
if (!bottom_fp) {
closedir(dp);
+ fclose(top_fp);
return -3;
}
@@ -248,7 +256,7 @@ int delivery_index_wheel_artifacts(struct Delivery *ctx) {
fprintf(top_fp, "<a href=\"%s/\">%s</a><br/>\n", rec->d_name, rec->d_name);
char dpath[PATH_MAX * 2] = {0};
- sprintf(dpath, "%s/%s", ctx->storage.wheel_artifact_dir, rec->d_name);
+ snprintf(dpath, sizeof(dpath), "%s/%s", ctx->storage.wheel_artifact_dir, rec->d_name);
struct StrList *packages = listdir(dpath);
if (!packages) {
closedir(dp);
@@ -275,6 +283,6 @@ int delivery_index_wheel_artifacts(struct Delivery *ctx) {
}
closedir(dp);
fclose(top_fp);
- SYSDEBUG("%s", "Wheel indexing complete");
+ SYSDEBUG("Wheel indexing complete");
return 0;
}
diff --git a/src/lib/delivery/delivery_show.c b/src/lib/delivery/delivery_show.c
index adfa1be..1740688 100644
--- a/src/lib/delivery/delivery_show.c
+++ b/src/lib/delivery/delivery_show.c
@@ -84,13 +84,13 @@ void delivery_conda_show(struct Delivery *ctx) {
void delivery_tests_show(struct Delivery *ctx) {
printf("\n====TESTS====\n");
- for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) {
- if (!ctx->tests[i].name) {
+ for (size_t i = 0; i < ctx->tests->num_used; i++) {
+ if (!ctx->tests->test[i]->name) {
continue;
}
- printf("%-20s %-20s %s\n", ctx->tests[i].name,
- ctx->tests[i].version,
- ctx->tests[i].repository);
+ printf("%-20s %-20s %s\n", ctx->tests->test[i]->name,
+ ctx->tests->test[i]->version,
+ ctx->tests->test[i]->repository);
}
}
@@ -108,7 +108,7 @@ void delivery_runtime_show(struct Delivery *ctx) {
char *item = strlist_item(rt, i);
if (!item) {
// not supposed to occur
- msg(STASIS_MSG_WARN | STASIS_MSG_L1, "Encountered unexpected NULL at record %zu of %zu of runtime array.\n", i);
+ SYSWARN("Encountered unexpected NULL in record %zu of %zu in runtime array.", i, total);
return;
}
printf("%s\n", item);
diff --git a/src/lib/delivery/delivery_test.c b/src/lib/delivery/delivery_test.c
index e80e0ec..f59a62e 100644
--- a/src/lib/delivery/delivery_test.c
+++ b/src/lib/delivery/delivery_test.c
@@ -1,5 +1,87 @@
#include "delivery.h"
+struct Tests *tests_init(const size_t num_tests) {
+ struct Tests *tests = calloc(1, sizeof(*tests));
+ if (!tests) {
+ return NULL;
+ }
+
+ tests->test = calloc(num_tests, sizeof(*tests->test));
+ if (!tests->test) {
+ guard_free(tests);
+ return NULL;
+ }
+ tests->num_used = 0;
+ tests->num_alloc = num_tests;
+
+ return tests;
+}
+
+int tests_add(struct Tests *tests, struct Test *x) {
+ if (tests->num_used >= tests->num_alloc) {
+ const size_t old_alloc = tests->num_alloc;
+ struct Test **tmp = realloc(tests->test, tests->num_alloc++ * sizeof(**tests->test));
+ SYSDEBUG("Increasing size of test array: %zu -> %zu", old_alloc, tests->num_alloc);
+ if (!tmp) {
+ SYSDEBUG("Failed to allocate %zu bytes for test array", tests->num_alloc * sizeof(**tests->test));
+ return -1;
+ }
+ tests->test = tmp;
+ }
+
+ SYSDEBUG("Adding test: '%s'", x->name);
+ tests->test[tests->num_used++] = x;
+ return 0;
+}
+
+struct Test *test_init() {
+ struct Test *result = calloc(1, sizeof(*result));
+ if (!result) {
+ return NULL;
+ }
+
+ result->runtime = calloc(1, sizeof(*result->runtime));
+ if (!result->runtime) {
+ guard_free(result);
+ return NULL;
+ }
+
+ return result;
+}
+
+void test_free(struct Test **x) {
+ struct Test *test = *x;
+ if (!test) {
+ return;
+ }
+ guard_free(test->name);
+ guard_free(test->version);
+ guard_free(test->repository);
+ guard_free(test->repository_info_ref);
+ guard_free(test->repository_info_tag);
+ guard_strlist_free(&test->repository_remove_tags);
+ guard_free(test->script);
+ guard_free(test->script_setup);
+ guard_free(test->build_recipe);
+ // test-specific runtime variables
+ guard_runtime_free(test->runtime->environ);
+ guard_free(test->runtime);
+ guard_free(test);
+}
+
+void tests_free(struct Tests **x) {
+ struct Tests *tests = *x;
+ if (!tests) {
+ return;
+ }
+
+ for (size_t i = 0; i < tests->num_alloc; i++) {
+ test_free(&tests->test[i]);
+ }
+ guard_free(tests->test);
+ guard_free(tests);
+}
+
void delivery_tests_run(struct Delivery *ctx) {
static const int SETUP = 0;
static const int PARALLEL = 1;
@@ -16,26 +98,26 @@ void delivery_tests_run(struct Delivery *ctx) {
// amount of debug information.
snprintf(globals.workaround.conda_reactivate, PATH_MAX - 1, "\nset +x; mamba activate ${CONDA_DEFAULT_ENV}; set -x\n");
- if (!ctx->tests[0].name) {
- msg(STASIS_MSG_WARN | STASIS_MSG_L2, "no tests are defined!\n");
+ if (!ctx->tests || !ctx->tests->num_used) {
+ SYSWARN("no tests are defined!");
} else {
pool[PARALLEL] = mp_pool_init("parallel", ctx->storage.tmpdir);
if (!pool[PARALLEL]) {
- perror("mp_pool_init/parallel");
+ SYSERROR("mp_pool_init/parallel initialization failed");
exit(1);
}
pool[PARALLEL]->status_interval = globals.pool_status_interval;
pool[SERIAL] = mp_pool_init("serial", ctx->storage.tmpdir);
if (!pool[SERIAL]) {
- perror("mp_pool_init/serial");
+ SYSERROR("mp_pool_init/serial initialization failed");
exit(1);
}
pool[SERIAL]->status_interval = globals.pool_status_interval;
pool[SETUP] = mp_pool_init("setup", ctx->storage.tmpdir);
if (!pool[SETUP]) {
- perror("mp_pool_init/setup");
+ SYSERROR("mp_pool_init/setup initialization failed");
exit(1);
}
pool[SETUP]->status_interval = globals.pool_status_interval;
@@ -60,21 +142,20 @@ void delivery_tests_run(struct Delivery *ctx) {
// Iterate over our test records, retrieving the source code for each package, and assigning its scripted tasks
// to the appropriate processing pool
- for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) {
- struct Test *test = &ctx->tests[i];
+ for (size_t i = 0; i < ctx->tests->num_used; i++) {
+ struct Test *test = ctx->tests->test[i];
if (!test->name && !test->repository && !test->script) {
// skip unused test records
continue;
}
msg(STASIS_MSG_L2, "Loading tests for %s %s\n", test->name, test->version);
if (!test->script || !strlen(test->script)) {
- msg(STASIS_MSG_WARN | STASIS_MSG_L3, "Nothing to do. To fix, declare a 'script' in section: [test:%s]\n",
- test->name);
+ SYSWARN("Nothing to do. To fix, declare a 'script' in section: [test:%s]", test->name);
continue;
}
char destdir[PATH_MAX];
- sprintf(destdir, "%s/%s", ctx->storage.build_sources_dir, path_basename(test->repository));
+ snprintf(destdir, sizeof(destdir), "%s/%s", ctx->storage.build_sources_dir, path_basename(test->repository));
if (!access(destdir, F_OK)) {
msg(STASIS_MSG_L3, "Purging repository %s\n", destdir);
@@ -97,6 +178,18 @@ void delivery_tests_run(struct Delivery *ctx) {
if (pushd(destdir)) {
COE_CHECK_ABORT(1, "Unable to enter repository directory\n");
} else {
+ const int dep_status = check_python_package_dependencies(".");
+ if (dep_status) {
+ SYSERROR("Please replace all occurrences above with standard package specs:\n"
+ "\n"
+ " package==x.y.z\n"
+ " package>=x.y.z\n"
+ " package<=x.y.z\n"
+ " ...\n"
+ "\n");
+ COE_CHECK_ABORT(true, "Unreproducible delivery");
+ }
+
char *cmd = calloc(strlen(test->script) + STASIS_BUFSIZ, sizeof(*cmd));
if (!cmd) {
SYSERROR("Unable to allocate test script buffer: %s", strerror(errno));
@@ -106,11 +199,12 @@ void delivery_tests_run(struct Delivery *ctx) {
msg(STASIS_MSG_L3, "Queuing task for %s\n", test->name);
memset(&proc, 0, sizeof(proc));
- strcpy(cmd, test->script);
+ strncpy(cmd, test->script, strlen(test->script) + STASIS_BUFSIZ - 1);
+ cmd[strlen(test->script) + STASIS_BUFSIZ - 1] = '\0';
char *cmd_rendered = tpl_render(cmd);
if (cmd_rendered) {
if (strcmp(cmd_rendered, cmd) != 0) {
- strcpy(cmd, cmd_rendered);
+ strncpy(cmd, cmd_rendered, strlen(test->script) + STASIS_BUFSIZ - 1);
cmd[strlen(cmd_rendered) ? strlen(cmd_rendered) - 1 : 0] = 0;
}
guard_free(cmd_rendered);
@@ -135,7 +229,8 @@ void delivery_tests_run(struct Delivery *ctx) {
if (!globals.enable_parallel || !test->parallel) {
selected = SERIAL;
memset(pool_name, 0, sizeof(pool_name));
- strcpy(pool_name, "serial");
+ strncpy(pool_name, "serial", sizeof(pool_name) - 1);
+ pool_name[sizeof(pool_name) - 1] = '\0';
}
if (asprintf(&runner_cmd, runner_cmd_fmt, cmd) < 0) {
@@ -154,6 +249,12 @@ void delivery_tests_run(struct Delivery *ctx) {
}
exit(1);
}
+
+ // Apply timeout from test block
+ if (test->timeout) {
+ task->timeout = test->timeout;
+ }
+
guard_free(runner_cmd);
guard_free(cmd);
popd();
@@ -163,11 +264,11 @@ void delivery_tests_run(struct Delivery *ctx) {
// Configure "script_setup" tasks
// Directories should exist now, so no need to go through initializing everything all over again.
- for (size_t i = 0; i < sizeof(ctx->tests) / sizeof(ctx->tests[0]); i++) {
- struct Test *test = &ctx->tests[i];
+ for (size_t i = 0; i < ctx->tests->num_used; i++) {
+ const struct Test *test = ctx->tests->test[i];
if (test->script_setup) {
char destdir[PATH_MAX];
- sprintf(destdir, "%s/%s", ctx->storage.build_sources_dir, path_basename(test->repository));
+ snprintf(destdir, sizeof(destdir), "%s/%s", ctx->storage.build_sources_dir, path_basename(test->repository));
if (access(destdir, F_OK)) {
SYSERROR("%s: %s", destdir, strerror(errno));
exit(1);
@@ -181,11 +282,13 @@ void delivery_tests_run(struct Delivery *ctx) {
}
strncpy(cmd, test->script_setup, cmd_len - 1);
+ cmd[cmd_len - 1] = '\0';
+
char *cmd_rendered = tpl_render(cmd);
if (cmd_rendered) {
if (strcmp(cmd_rendered, cmd) != 0) {
strncpy(cmd, cmd_rendered, cmd_len - 1);
- cmd[strlen(cmd_rendered) ? strlen(cmd_rendered) - 1 : 0] = 0;
+ cmd[strlen(cmd_rendered) ? strlen(cmd_rendered) - 1 : 0] = '\0';
}
guard_free(cmd_rendered);
} else {
@@ -217,7 +320,7 @@ void delivery_tests_run(struct Delivery *ctx) {
guard_free(cmd);
popd();
} else {
- SYSERROR("Failed to change directory: %s\n", destdir);
+ SYSERROR("Failed to change directory: %s", destdir);
exit(1);
}
}
@@ -282,10 +385,10 @@ int delivery_fixup_test_results(struct Delivery *ctx) {
continue;
}
- sprintf(path, "%s/%s", ctx->storage.results_dir, rec->d_name);
+ snprintf(path, sizeof(path), "%s/%s", ctx->storage.results_dir, rec->d_name);
msg(STASIS_MSG_L3, "%s\n", rec->d_name);
if (xml_pretty_print_in_place(path, STASIS_XML_PRETTY_PRINT_PROG, STASIS_XML_PRETTY_PRINT_ARGS)) {
- msg(STASIS_MSG_L3 | STASIS_MSG_WARN, "Failed to rewrite file '%s'\n", rec->d_name);
+ SYSWARN("Failed to rewrite file '%s'", rec->d_name);
}
}
diff --git a/src/lib/delivery/include/delivery.h b/src/lib/delivery/include/delivery.h
index 26a5499..3103a86 100644
--- a/src/lib/delivery/include/delivery.h
+++ b/src/lib/delivery/include/delivery.h
@@ -5,20 +5,11 @@
#include <string.h>
#include <stdbool.h>
-#include <unistd.h>
-#include <sys/utsname.h>
-#include <fnmatch.h>
-#include <sys/statvfs.h>
#include "artifactory.h"
-#include "conda.h"
-#include "copy.h"
-#include "core.h"
#include "docker.h"
#include "environment.h"
#include "ini.h"
#include "multiprocessing.h"
-#include "recipe.h"
-#include "wheel.h"
#define DELIVERY_PLATFORM_MAX 4
#define DELIVERY_PLATFORM_MAXLEN 65
@@ -44,6 +35,28 @@ struct Content {
char *data;
};
+//! Number of test records to allocate (grows dynamically)
+#define TEST_NUM_ALLOC_INITIAL 10
+
+/*! \struct Test
+ * \brief Test information
+ */
+struct Test {
+ char *name; ///< Name of package
+ char *version; ///< Version of package
+ char *repository; ///< Git repository of package
+ char *script_setup; ///< Commands to execute before the main script
+ char *script; ///< Commands to execute
+ bool disable; ///< Toggle a test block
+ bool parallel; ///< Toggle parallel or serial execution
+ char *build_recipe; ///< Conda recipe to build (optional)
+ char *repository_info_ref; ///< Git commit hash
+ char *repository_info_tag; ///< Git tag (first parent)
+ struct StrList *repository_remove_tags; ///< Git tags to remove (to fix duplicate commit tags)
+ struct Runtime *runtime; ///< Environment variables specific to the test context
+ int timeout; ///< Timeout in seconds
+}; ///< An array of tests
+
/*! \struct Delivery
* \brief A structure describing a full delivery object
*/
@@ -64,10 +77,8 @@ struct Delivery {
* \brief System information
*/
struct System {
- char *arch;
- ///< System CPU architecture ident
- char **platform;
- ///< System platform name
+ char *arch; ///< System CPU architecture ident
+ char **platform; ///< System platform name
} system;
/*! \struct Storage
* \brief Storage paths
@@ -155,23 +166,11 @@ struct Delivery {
RuntimeEnv *environ; ///< Environment variables
} runtime;
- /*! \struct Test
- * \brief Test information
- */
- struct Test {
- char *name; ///< Name of package
- char *version; ///< Version of package
- char *repository; ///< Git repository of package
- char *script_setup; ///< Commands to execute before the main script
- char *script; ///< Commands to execute
- bool disable; ///< Toggle a test block
- bool parallel; ///< Toggle parallel or serial execution
- char *build_recipe; ///< Conda recipe to build (optional)
- char *repository_info_ref; ///< Git commit hash
- char *repository_info_tag; ///< Git tag (first parent)
- struct StrList *repository_remove_tags; ///< Git tags to remove (to fix duplicate commit tags)
- struct Runtime runtime; ///< Environment variables specific to the test context
- } tests[1000]; ///< An array of tests
+ struct Tests {
+ struct Test **test;
+ size_t num_used;
+ size_t num_alloc;
+ } *tests;
struct Deploy {
struct JFRT_Auth jfrog_auth;
@@ -311,9 +310,10 @@ int delivery_get_conda_installer(struct Delivery *ctx, char *installer_url);
* Generate URL based on Delivery context
* @param ctx pointer to Delivery context
* @param result pointer to char
+ * @param maxlen
* @return in result
*/
-void delivery_get_conda_installer_url(struct Delivery *ctx, char *result);
+void delivery_get_conda_installer_url(struct Delivery *ctx, char *result, size_t maxlen);
/**
* Install packages based on Delivery context
@@ -384,10 +384,11 @@ void delivery_install_conda(char *install_script, char *conda_install_dir);
*
* @param ctx pointer to Delivery context
* @param dest NULL pointer to string, or initialized string
+ * @param maxlen
* @param fmt release format string
* @return 0 on success, -1 on error
*/
-int delivery_format_str(struct Delivery *ctx, char **dest, const char *fmt);
+int delivery_format_str(struct Delivery *ctx, char **dest, size_t maxlen, const char *fmt);
// helper function
int delivery_gather_tool_versions(struct Delivery *ctx);
@@ -439,6 +440,16 @@ int delivery_exists(struct Delivery *ctx);
int delivery_overlay_packages_from_env(struct Delivery *ctx, const char *env_name);
/**
+ * Conda does not handle version suffixes well, if at all. For example, if pkg-1.2.3rc1 is installed Conda will
+ * silently ignore a request to install pkg-1.2.3. This function serves as a workaround by comparing the version
+ * on-disk, and the requested version from the package list, and if the versions are not equal the on-disk package
+ * is replaced by the one in the package list.
+ *
+ * When a package is present in the list without a pinned version it will be reinstalled with whatever is available
+ */
+int delivery_conda_enforce_package_version(struct Delivery *ctx, const char *env_name, const char *name);
+
+/**
* Retrieve remote deliveries associated with the current version series
* @param ctx Delivery context
* @return -1 on error
@@ -459,4 +470,69 @@ int delivery_series_sync(struct Delivery *ctx);
*/
int delivery_purge_packages(struct Delivery *ctx, const char *env_name, int use_pkg_manager);
+/**
+ * Export delivery environments
+ *
+ * @param ctx Delivery context
+ * @param envs array of conda environment names
+ */
+void delivery_export(const struct Delivery *ctx, char *envs[]);
+
+/**
+ * STAGE 1: Rewrite delivery-related strings in specfile
+ *
+ * @param ctx Delivery context
+ * @param specfile path to YAML spec file
+ */
+void delivery_rewrite_stage1(struct Delivery *ctx, char *specfile);
+
+/**
+ * STAGE 2: Rewrite delivery-related strings in specfile
+ *
+ * @param ctx Delivery context
+ * @param specfile path to YAML spec file
+ */
+void delivery_rewrite_stage2(struct Delivery *ctx, char *specfile);
+
+/**
+ * Return a copy of a delivery context
+ * @param ctx Delivery context
+ * @return a copy
+ */
+struct Delivery *delivery_duplicate(struct Delivery *ctx);
+
+/**
+ * Initialize a `Tests` structure
+ * @param num_tests number of test records
+ * @return a an initialized `Tests` structure
+ */
+struct Tests *tests_init(size_t num_tests);
+
+/**
+ * Add a `Test` structure to `Tests`
+ * @param tests list to add to
+ * @param x test to add to list
+ * @return 0=success, -1=error
+ */
+int tests_add(struct Tests *tests, struct Test *x);
+
+/**
+ * Free a `Test` structure
+ * @param x pointer to `Test`
+ */
+void test_free(struct Test **x);
+
+/**
+ * Free a `Tests` structure
+ * @param x pointer to `Tests`
+ */
+void tests_free(struct Tests **x);
+
+/**
+ * Initialize a `Test` structure
+ * @return an initialized `Test` structure
+ */
+struct Test *test_init();
+
+
#endif //STASIS_DELIVERY_H
generated by cgit (git 2.34.1) at 2026-05-19 00:27:33 -0400