diff options
| -rw-r--r-- | include/deliverable.h | 36 | ||||
| -rw-r--r-- | include/system.h | 5 | ||||
| -rw-r--r-- | src/deliverable.c | 123 | ||||
| -rw-r--r-- | src/main.c | 61 | ||||
| -rw-r--r-- | src/system.c | 83 | ||||
| -rw-r--r-- | src/utils.c | 6 |
6 files changed, 212 insertions, 102 deletions
diff --git a/include/deliverable.h b/include/deliverable.h index 135a14c..e699703 100644 --- a/include/deliverable.h +++ b/include/deliverable.h @@ -28,6 +28,12 @@ #define DEFER_CONDA 0 ///< Build conda packages #define DEFER_PIP 1 ///< Build python packages +struct Content { + unsigned type; + char *filename; + char *data; +}; + /*! \struct Delivery * \brief A structure describing a full delivery object */ @@ -49,6 +55,7 @@ struct Delivery { char *tmpdir; ///< Temporary storage area (within root) char *delivery_dir; ///< Delivery artifact output directory char *tools_dir; ///< Tools storage + char *mission_dir; ///< Mission data storage char *conda_install_prefix; ///< Path to install Conda char *conda_artifact_dir; ///< Base path to store compiled conda packages char *conda_staging_dir; ///< Base path to copy compiled conda packages @@ -123,6 +130,12 @@ struct Delivery { char *build_recipe; ///< Conda recipe to build (optional) struct Runtime runtime; ///< Environment variables specific to the test context } tests[1000]; ///< An array of tests + + struct Rule { + bool enable_final; ///< true=allow rc value replacement, false=keep rc value even if final release + char *release_fmt; ///< Release generator format string + struct Content content[1000]; + } rules; }; /** @@ -284,6 +297,29 @@ void delivery_conda_enable(struct Delivery *ctx, char *conda_install_dir); */ void delivery_install_conda(char *install_script, char *conda_install_dir); +/** + * Generate a formatted release string + * + * Formatters: + * %n = Delivery Name + * %c = Delivery Codename (HST mission, only) + * %m = Mission + * %R = Delivery Revision number (or "final") + * %r = Delivery Revision number + * %v = Delivery Version + * %P = Python version (i.e. 3.9.1) + * %p = Compact Python version (i.e. 3.9.1 -> 39) + * %a = System architecture name + * %o = System platform name + * %t = Delivery timestamp (Unix Epoch) + * + * @param ctx pointer to Delivery context + * @param dest NULL pointer to string, or initialized string + * @param fmt release format string + * @return 0 on success, -1 on error + */ +int delivery_format_str(struct Delivery *ctx, char **dest, const char *fmt); + // helper function void delivery_gather_tool_versions(struct Delivery *ctx); diff --git a/include/system.h b/include/system.h index 32c7c79..8ad613f 100644 --- a/include/system.h +++ b/include/system.h @@ -21,9 +21,8 @@ struct Process { int returncode; }; -int shell(struct Process *proc, char *args[]); -int shell2(struct Process *proc, char *args); -int shell_safe(struct Process *proc, char *args[]); +int shell(struct Process *proc, char *args); +int shell_safe(struct Process *proc, char *args); char *shell_output(const char *command, int *status); #endif //OMC_SYSTEM_H diff --git a/src/deliverable.c b/src/deliverable.c index e6704d9..b77502c 100644 --- a/src/deliverable.c +++ b/src/deliverable.c @@ -103,8 +103,8 @@ int delivery_init_tmpdir(struct Delivery *ctx) { goto l_delivery_init_tmpdir_fatal; } - ctx->storage.tmpdir = strdup(tmpdir); globals.tmpdir = strdup(tmpdir); + ctx->storage.tmpdir = globals.tmpdir; return unusable; l_delivery_init_tmpdir_fatal: @@ -407,6 +407,8 @@ int delivery_init(struct Delivery *ctx, struct INIFILE *ini, struct INIFILE *cfg getter(ini, ini->section[i]->key, "build_recipe", INIVAL_TYPE_STR) conv_str(ctx, tests[z].build_recipe) + getter(ini, ini->section[i]->key, "runtime", INIVAL_TO_LIST) + conv_strlist(ctx, tests[z].runtime.environ, "\n") z++; } } @@ -438,6 +440,67 @@ int delivery_init(struct Delivery *ctx, struct INIFILE *ini, struct INIFILE *cfg return 0; } +int delivery_format_str(struct Delivery *ctx, char **dest, const char *fmt) { + size_t fmt_len = strlen(fmt); + + if (!*dest) { + *dest = calloc(OMC_NAME_MAX, sizeof(**dest)); + if (!*dest) { + return -1; + } + } + + for (size_t i = 0; i < fmt_len; i++) { + if (fmt[i] == '%' && strlen(&fmt[i])) { + i++; + switch (fmt[i]) { + case 'n': // name + strcat(*dest, ctx->meta.name); + break; + case 'c': // codename + strcat(*dest, ctx->meta.codename); + break; + case 'm': // mission + strcat(*dest, ctx->meta.mission); + break; + case 'r': // revision + sprintf(*dest + strlen(*dest), "%d", ctx->meta.rc); + break; + case 'R': // "final"-aware revision + if (ctx->meta.final) + strcat(*dest, "final"); + else + sprintf(*dest + strlen(*dest), "%d", ctx->meta.rc); + break; + case 'v': // version + strcat(*dest, ctx->meta.version); + break; + case 'P': // python version + strcat(*dest, ctx->meta.python); + break; + case 'p': // python version major/minor + strcat(*dest, ctx->meta.python_compact); + break; + case 'a': // system architecture name + strcat(*dest, ctx->system.arch); + break; + case 'o': // system platform (OS) name + strcat(*dest, ctx->system.platform[DELIVERY_PLATFORM_RELEASE]); + break; + case 't': // unix epoch + sprintf(*dest + strlen(*dest), "%ld", ctx->info.time_now); + break; + default: // unknown formatter, write as-is + sprintf(*dest + strlen(*dest), "%c%c", fmt[i - 1], fmt[i]); + break; + } + } else { // write non-format text + sprintf(*dest + strlen(*dest), "%c", fmt[i]); + } + } + return 0; +} + void delivery_meta_show(struct Delivery *ctx) { printf("\n====DELIVERY====\n"); printf("%-20s %-10s\n", "Target Python:", ctx->meta.python); @@ -461,21 +524,29 @@ void delivery_conda_show(struct Delivery *ctx) { printf("%-20s %-10s\n", "Prefix:", ctx->storage.conda_install_prefix); puts("Native Packages:"); - for (size_t i = 0; i < strlist_count(ctx->conda.conda_packages); i++) { - char *token = strlist_item(ctx->conda.conda_packages, i); - if (isempty(token) || isblank(*token) || startswith(token, "-")) { - continue; + if (strlist_count(ctx->conda.conda_packages)) { + for (size_t i = 0; i < strlist_count(ctx->conda.conda_packages); i++) { + char *token = strlist_item(ctx->conda.conda_packages, i); + if (isempty(token) || isblank(*token) || startswith(token, "-")) { + continue; + } + printf("%21s%s\n", "", token); } - printf("%21s%s\n", "", token); + } else { + printf("%21s%s\n", "", "N/A"); } puts("Python Packages:"); - for (size_t i = 0; i < strlist_count(ctx->conda.pip_packages); i++) { - char *token = strlist_item(ctx->conda.pip_packages, i); - if (isempty(token) || isblank(*token) || startswith(token, "-")) { - continue; + if (strlist_count(ctx->conda.pip_packages)) { + for (size_t i = 0; i < strlist_count(ctx->conda.pip_packages); i++) { + char *token = strlist_item(ctx->conda.pip_packages, i); + if (isempty(token) || isblank(*token) || startswith(token, "-")) { + continue; + } + printf("%21s%s\n", "", token); } - printf("%21s%s\n", "", token); + } else { + printf("%21s%s\n", "", "N/A"); } } @@ -851,7 +922,12 @@ void delivery_install_conda(char *install_script, char *conda_install_dir) { // Proceed with the installation // -b = batch mode (non-interactive) - if (shell_safe(&proc, (char *[]) {find_program("bash"), install_script, "-b", "-p", conda_install_dir, NULL})) { + char cmd[255] = {0}; + snprintf(cmd, sizeof(cmd) - 1, "%s %s -b -p %s", + find_program("bash"), + install_script, + conda_install_dir); + if (shell_safe(&proc, cmd)) { fprintf(stderr, "conda installation failed\n"); exit(1); } @@ -1116,7 +1192,7 @@ void delivery_tests_run(struct Delivery *ctx) { // enable trace mode before executing each test script memset(cmd, 0, sizeof(cmd)); sprintf(cmd, "set -x ; %s", ctx->tests[i].script); - status = shell2(&proc, cmd); + status = shell(&proc, cmd); if (status) { COE_CHECK_ABORT(!globals.continue_on_error, "Test failure") } @@ -1188,6 +1264,9 @@ int delivery_artifact_upload(struct Delivery *ctx) { char *password = getenv("OMC_JF_PASSWORD"); char *ssh_key_path = getenv("OMC_JF_SSH_KEY_PATH"); char *ssh_passphrase = getenv("OMC_JF_SSH_PASSPHRASE"); + char *client_cert_key_path = getenv("OMC_JF_CLIENT_CERT_KEY_PATH"); + char *client_cert_path = getenv("OMC_JF_CLIENT_CERT_PATH"); + char *repo = getenv("OMC_JF_REPO"); if (!url) { fprintf(stderr, "Artifactory URL is not configured:\n"); @@ -1214,11 +1293,25 @@ int delivery_artifact_upload(struct Delivery *ctx) { } auth_ctx.password = NULL; auth_ctx.access_token = NULL; + } else if (client_cert_key_path && client_cert_path) { + auth_ctx.user = NULL; + auth_ctx.password = NULL; + auth_ctx.access_token = NULL; + auth_ctx.ssh_key_path = NULL; + auth_ctx.client_cert_key_path = client_cert_key_path; + auth_ctx.client_cert_path = client_cert_path; } else { fprintf(stderr, "Artifactory authentication is not configured:\n"); fprintf(stderr, "set OMC_JF_USER and OMC_JF_PASSWORD\n"); - fprintf(stderr, "set OMC_JF_ACCESS_TOKEN\nor\n"); - fprintf(stderr, "set OMC_JF_SSH_KEY_PATH and OMC_JF_SSH_KEY_PASSPHRASE\n"); + fprintf(stderr, "or, set OMC_JF_ACCESS_TOKEN\n"); + fprintf(stderr, "or, set OMC_JF_SSH_KEY_PATH and OMC_JF_SSH_KEY_PASSPHRASE\n"); + fprintf(stderr, "or, set OMC_JF_CLIENT_CERT_KEY_PATH and OMC_JF_CLIENT_CERT_PATH\n"); + return -1; + } + + if (!repo) { + fprintf(stderr, "Artifactory destination reppsitory is not configured:\n"); + fprintf(stderr, "set OMC_JF_REPO to a remote repository path\n"); return -1; } @@ -18,7 +18,7 @@ const char *BANNER = "---------------------------------------------------------- " Delivery Generator \n" " v%s \n" "---------------------------------------------------------------------\n" - "Copyright (C) 2023 %s,\n" + "Copyright (C) 2023-2024 %s,\n" "Association of Universities for Research in Astronomy (AURA)\n"; struct OMC_GLOBAL globals = { @@ -190,7 +190,7 @@ int main(int argc, char *argv[], char *arge[]) { msg(OMC_MSG_L2, "Reading OMC global configuration: %s\n", config_input); cfg = ini_open(config_input); if (!cfg) { - msg(OMC_MSG_ERROR | OMC_MSG_L2, "Failed to read config file: %s, %s", delivery_input, strerror(errno)); + msg(OMC_MSG_ERROR | OMC_MSG_L2, "Failed to read config file: %s, %s\n", delivery_input, strerror(errno)); exit(1); } } @@ -198,7 +198,7 @@ int main(int argc, char *argv[], char *arge[]) { msg(OMC_MSG_L2, "Reading OMC delivery configuration: %s\n", delivery_input); ini = ini_open(delivery_input); if (!ini) { - msg(OMC_MSG_ERROR | OMC_MSG_L2, "Failed to read delivery file: %s, %s", delivery_input, strerror(errno)); + msg(OMC_MSG_ERROR | OMC_MSG_L2, "Failed to read delivery file: %s, %s\n", delivery_input, strerror(errno)); exit(1); } @@ -224,6 +224,58 @@ int main(int argc, char *argv[], char *arge[]) { exit(1); } + // Expose variables for use with the template engine + tpl_register("meta.name", ctx.meta.name); + tpl_register("meta.version", ctx.meta.version); + tpl_register("meta.codename", ctx.meta.codename); + tpl_register("meta.mission", ctx.meta.mission); + tpl_register("meta.python", ctx.meta.python); + tpl_register("meta.python_compact", ctx.meta.python_compact); + tpl_register("info.release_name", ctx.info.release_name); + tpl_register("conda.installer_baseurl", ctx.conda.installer_baseurl); + tpl_register("conda.installer_name", ctx.conda.installer_name); + tpl_register("conda.installer_version", ctx.conda.installer_version); + tpl_register("conda.installer_arch", ctx.conda.installer_arch); + tpl_register("conda.installer_platform", ctx.conda.installer_platform); + tpl_register("system.arch", ctx.system.arch); + tpl_register("system.platform", ctx.system.platform[DELIVERY_PLATFORM_RELEASE]); + + char missionfile[PATH_MAX] = {0}; + if (getenv("OMC_SYSCONFDIR")) { + globals.sysconfdir = calloc(PATH_MAX, sizeof(*globals.sysconfdir)); + } else { + globals.sysconfdir = strdup(OMC_SYSCONFDIR); + } + if (!globals.sysconfdir) { + msg(OMC_MSG_ERROR | OMC_MSG_L1, "Unable to allocate memory for system configuration directory path\n"); + exit(1); + } + if (getenv("OMC_SYSCONFDIR")) { + sprintf(missionfile, "%s/%s/%s/%s.ini", getenv("OMC_SYSCONFDIR"), "mission", ctx.meta.mission, ctx.meta.mission); + } else { + sprintf(missionfile, "%s/%s/%s/%s.ini", OMC_SYSCONFDIR, "mission", ctx.meta.mission, ctx.meta.mission); + } + + struct INIFILE *cfg_mission; + cfg_mission = ini_open(missionfile); + if (!cfg_mission) { + msg(OMC_MSG_ERROR | OMC_MSG_L2, "Failed to read misson configuration: %s, %s\n", missionfile, strerror(errno)); + exit(1); + } + + /* + union INIVal x; + ini_getval(cfg_mission, "template:readme.md.in", "destination", INIVAL_TYPE_STR, &x); + char *output = tpl_render(x.as_char_p); + if (!output) { + fprintf(stderr, "Render failed!\n"); + exit(1); + } + puts(output); + tpl_free(); + + exit(0); + */ runtime_apply(ctx.runtime.environ); snprintf(env_name, sizeof(env_name_testing) - 1, "%s", ctx.info.release_name); snprintf(env_name_testing, sizeof(env_name) - 1, "%s_test", env_name); @@ -406,10 +458,13 @@ int main(int argc, char *argv[], char *arge[]) { msg(OMC_MSG_L1, "Cleaning up\n"); ini_free(&ini); if (cfg) { + // optional extras ini_free(&cfg); } + ini_free(&cfg_mission); delivery_free(&ctx); globals_free(); + tpl_free(); msg(OMC_MSG_L1, "Done!\n"); return 0; diff --git a/src/system.c b/src/system.c index 52e354a..2a5302e 100644 --- a/src/system.c +++ b/src/system.c @@ -5,78 +5,7 @@ #include "system.h" #include "omc.h" -int shell(struct Process *proc, char *args[]) { - FILE *fp_out, *fp_err; - pid_t pid; - pid_t status; - status = 0; - errno = 0; - - pid = fork(); - if (pid == -1) { - fprintf(stderr, "fork failed\n"); - exit(1); - } else if (pid == 0) { - int retval; - if (proc != NULL) { - if (strlen(proc->stdout)) { - fp_out = freopen(proc->stdout, "w+", stdout); - } - - if (strlen(proc->stderr)) { - fp_err = freopen(proc->stderr, "w+", stderr); - } - - if (proc->redirect_stderr) { - if (fp_err) { - fclose(fp_err); - fclose(stderr); - } - dup2(fileno(stdout), fileno(stderr)); - } - } - - retval = execv(args[0], args); - fprintf(stderr, "# executing: "); - for (size_t x = 0; args[x] != NULL; x++) { - fprintf(stderr, "%s ", args[x]); - } - - if (proc != NULL && strlen(proc->stdout)) { - fflush(fp_out); - fclose(fp_out); - fflush(stdout); - fclose(stdout); - } - if (proc != NULL && strlen(proc->stderr)) { - fflush(fp_err); - fclose(fp_err); - fflush(stderr); - fclose(stderr); - } - exit(retval); - } else { - if (waitpid(pid, &status, WUNTRACED) > 0) { - if (WIFEXITED(status) && WEXITSTATUS(status)) { - if (WEXITSTATUS(status) == 127) { - fprintf(stderr, "execv failed\n"); - } - } else if (WIFSIGNALED(status)) { - fprintf(stderr, "signal received: %d\n", WIFSIGNALED(status)); - } - } else { - fprintf(stderr, "waitpid() failed\n"); - } - } - - - if (proc != NULL) { - proc->returncode = status; - } - return WEXITSTATUS(status); -} - -int shell2(struct Process *proc, char *args) { +int shell(struct Process *proc, char *args) { FILE *fp_out = NULL; FILE *fp_err = NULL; pid_t pid; @@ -166,16 +95,14 @@ int shell2(struct Process *proc, char *args) { return WEXITSTATUS(status); } -int shell_safe(struct Process *proc, char *args[]) { +int shell_safe(struct Process *proc, char *args) { FILE *fp; char buf[1024] = {0}; int result; - for (size_t i = 0; args[i] != NULL; i++) { - if (strpbrk(args[i], ";&|()")) { - args[i] = NULL; - break; - } + char *invalid_ch = strpbrk(args, ";&|()"); + if (invalid_ch) { + args = NULL; } result = shell(proc, args); diff --git a/src/utils.c b/src/utils.c index 85a7a74..9e941f1 100644 --- a/src/utils.c +++ b/src/utils.c @@ -347,7 +347,7 @@ int git_clone(struct Process *proc, char *url, char *destdir, char *gitref) { sprintf(command, "%s clone --recursive %s", program, url); if (destdir && access(destdir, F_OK) < 0) { sprintf(command + strlen(command), " %s", destdir); - result = shell2(proc, command); + result = shell(proc, command); } if (destdir) { @@ -360,12 +360,12 @@ int git_clone(struct Process *proc, char *url, char *destdir, char *gitref) { { memset(command, 0, sizeof(command)); sprintf(command, "%s fetch --all", program); - result += shell2(proc, command); + result += shell(proc, command); if (gitref != NULL) { memset(command, 0, sizeof(command)); sprintf(command, "%s checkout %s", program, gitref); - result += shell2(proc, command); + result += shell(proc, command); } popd(); } |