diff options
author | Joseph Hunkeler <jhunkeler@users.noreply.github.com> | 2025-02-12 17:04:01 -0500 |
---|---|---|
committer | GitHub <noreply@github.com> | 2025-02-12 17:04:01 -0500 |
commit | 09de456a667a771348c3f1c9b0bd56afe4b8d3bd (patch) | |
tree | aedfcbb0b2876e3d7aad28a7cc09354a611c156e | |
parent | 2bd06b22e455fa154e7db27677421c8b53cbf43c (diff) | |
parent | 4e22506e086e02ffe3056c6fd39ccb201fd358c8 (diff) | |
download | stasis-09de456a667a771348c3f1c9b0bd56afe4b8d3bd.tar.gz |
Merge pull request #89 from jhunkeler/huge-changeset
Huge changeset
-rw-r--r-- | CMakeLists.txt | 21 | ||||
-rw-r--r-- | src/cli/stasis_indexer/include/helpers.h | 4 | ||||
-rw-r--r-- | src/lib/core/copy.c | 6 | ||||
-rw-r--r-- | src/lib/core/download.c | 6 | ||||
-rw-r--r-- | src/lib/core/environment.c | 2 | ||||
-rw-r--r-- | src/lib/core/include/conda.h | 2 | ||||
-rw-r--r-- | src/lib/core/include/core.h | 8 | ||||
-rw-r--r-- | src/lib/core/include/core_mem.h | 2 | ||||
-rw-r--r-- | src/lib/core/include/core_message.h | 19 | ||||
-rw-r--r-- | src/lib/core/include/multiprocessing.h | 4 | ||||
-rw-r--r-- | src/lib/core/ini.c | 12 | ||||
-rw-r--r-- | src/lib/core/multiprocessing.c | 18 | ||||
-rw-r--r-- | src/lib/core/str.c | 2 | ||||
-rw-r--r-- | src/lib/core/template.c | 44 | ||||
-rw-r--r-- | src/lib/delivery/delivery_install.c | 4 | ||||
-rw-r--r-- | src/lib/delivery/delivery_populate.c | 38 | ||||
-rw-r--r-- | src/lib/delivery/delivery_postprocess.c | 24 | ||||
-rw-r--r-- | tests/CMakeLists.txt | 8 | ||||
-rw-r--r-- | tests/include/testing.h | 15 | ||||
-rw-r--r-- | tests/test_artifactory.c | 5 | ||||
-rw-r--r-- | tests/test_conda.c | 6 | ||||
-rw-r--r-- | tests/test_environment.c | 4 | ||||
-rw-r--r-- | tests/test_template.c | 17 |
23 files changed, 197 insertions, 74 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 76b7186..8f5dfa9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,6 +23,12 @@ if (FORTIFY_SOURCE) set(nix_cflags ${nix_cflags} -O -D_FORTIFY_SOURCE=1) endif () +# Toggle extremely verbose output +option(DEBUG_MESSAGES OFF) +if (DEBUG_MESSAGES) + set(nix_cflags ${nix_cflags} -DDEBUG) +endif() + if (CMAKE_C_COMPILER_ID STREQUAL "GNU") add_compile_options(${nix_cflags}) elseif (CMAKE_C_COMPILER_ID MATCHES "Clang") @@ -30,6 +36,7 @@ elseif (CMAKE_C_COMPILER_ID MATCHES "Clang") elseif (CMAKE_C_COMPILER_ID STREQUAL "MSVC") add_compile_options(${win_cflags}) endif() +message(CHECK_START "Compiler flags: ${nix_cflags}") set(core_INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/src/lib/core/include) set(delivery_INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/src/lib/delivery/include) @@ -38,23 +45,17 @@ configure_file(${PROJECT_SOURCE_DIR}/include/config.h.in ${CMAKE_CURRENT_BINARY_ include_directories(${PROJECT_BINARY_DIR}/include) add_subdirectory(src) -# Toggle extremely verbose output -option(BUILD_TESTING_DEBUG OFF) -if (BUILD_TESTING_DEBUG) - add_compile_options(-DDEBUG) -endif() - # Toggle regression testing on/off -option(BUILD_TESTING_RT ON) +option(TESTS_RT ON) # Toggle testing -option(BUILD_TESTING OFF) +option(TESTS OFF) message(CHECK_START "Run unit tests") -if (BUILD_TESTING) +if (TESTS) message(CHECK_PASS "yes") enable_testing() message(CHECK_START "Run regression tests") - if (BUILD_TESTING_RT) + if (TESTS_RT) message(CHECK_PASS "yes") else() message(CHECK_PASS "no") diff --git a/src/cli/stasis_indexer/include/helpers.h b/src/cli/stasis_indexer/include/helpers.h index d493f75..46705d2 100644 --- a/src/cli/stasis_indexer/include/helpers.h +++ b/src/cli/stasis_indexer/include/helpers.h @@ -5,12 +5,12 @@ #define ARRAY_COUNT_DYNAMIC(X, COUNTER) \ do { \ - for (COUNTER = 0; X && X[COUNTER] != NULL; COUNTER++) {} \ + for ((COUNTER) = 0; (X) && (X)[COUNTER] != NULL; (COUNTER)++) {} \ } while(0) #define ARRAY_COUNT_BY_STRUCT_MEMBER(X, MEMBER, COUNTER) \ do { \ - for (COUNTER = 0; X[COUNTER].MEMBER != NULL; COUNTER++) {} \ + for ((COUNTER) = 0; (X)[COUNTER].MEMBER != NULL; (COUNTER)++) {} \ } while(0) struct StrList *get_architectures(struct Delivery ctx[], size_t nelem); diff --git a/src/lib/core/copy.c b/src/lib/core/copy.c index 928bc40..25eede3 100644 --- a/src/lib/core/copy.c +++ b/src/lib/core/copy.c @@ -3,6 +3,7 @@ int copy2(const char *src, const char *dest, unsigned int op) { struct stat src_stat, dnamest; + SYSDEBUG("Stat source file: %s", src); if (lstat(src, &src_stat) < 0) { perror(src); return -1; @@ -20,7 +21,9 @@ int copy2(const char *src, const char *dest, unsigned int op) { *dname_endptr = '\0'; } + SYSDEBUG("Stat destination file: %s", dname); stat(dname, &dnamest); + if (S_ISLNK(src_stat.st_mode)) { char lpath[1024] = {0}; if (readlink(src, lpath, sizeof(lpath)) < 0) { @@ -44,12 +47,14 @@ int copy2(const char *src, const char *dest, unsigned int op) { } else if (S_ISREG(src_stat.st_mode)) { char buf[STASIS_BUFSIZ] = {0}; size_t bytes_read; + SYSDEBUG("%s", "Opening source file for reading"); FILE *fp1 = fopen(src, "rb"); if (!fp1) { perror(src); return -1; } + SYSDEBUG("%s", "Opening destination file for writing"); FILE *fp2 = fopen(dest, "w+b"); if (!fp2) { perror(dest); @@ -79,5 +84,6 @@ int copy2(const char *src, const char *dest, unsigned int op) { errno = EOPNOTSUPP; return -1; } + SYSDEBUG("%s", "Data copied"); return 0; } diff --git a/src/lib/core/download.c b/src/lib/core/download.c index f07a850..c3f8dca 100644 --- a/src/lib/core/download.c +++ b/src/lib/core/download.c @@ -3,6 +3,7 @@ // #include "download.h" +#include "core.h" size_t download_writer(void *fp, size_t size, size_t nmemb, void *stream) { size_t bytes = fwrite(fp, size, nmemb, (FILE *) stream); @@ -10,7 +11,6 @@ size_t download_writer(void *fp, size_t size, size_t nmemb, void *stream) { } long download(char *url, const char *filename, char **errmsg) { - extern char *VERSION; long http_code = -1; char user_agent[20]; sprintf(user_agent, "stasis/%s", VERSION); @@ -37,7 +37,9 @@ long download(char *url, const char *filename, char **errmsg) { } curl_easy_setopt(c, CURLOPT_CONNECTTIMEOUT, timeout); + SYSDEBUG("curl_easy_perform(): \n\turl=%s\n\tfilename=%s\n\tuser agent=%s\n\ttimeout=%zu", url, filename, user_agent, timeout); CURLcode curl_code = curl_easy_perform(c); + SYSDEBUG("curl status code: %d", curl_code); if (curl_code != CURLE_OK) { if (errmsg) { strcpy(*errmsg, curl_easy_strerror(curl_code)); @@ -47,8 +49,8 @@ long download(char *url, const char *filename, char **errmsg) { goto failed; } curl_easy_getinfo(c, CURLINFO_RESPONSE_CODE, &http_code); - failed: + SYSDEBUG("HTTP code: %li", http_code); fclose(fp); curl_easy_cleanup(c); curl_global_cleanup(); diff --git a/src/lib/core/environment.c b/src/lib/core/environment.c index cc72b8d..45fef2b 100644 --- a/src/lib/core/environment.c +++ b/src/lib/core/environment.c @@ -145,7 +145,7 @@ void runtime_export(RuntimeEnv *env, char **keys) { RuntimeEnv *runtime_copy(char **env) { RuntimeEnv *rt = NULL; size_t env_count; - for (env_count = 0; env[env_count] != NULL; env_count++) {}; + for (env_count = 0; env[env_count] != NULL; env_count++) {} rt = strlist_init(); for (size_t i = 0; i < env_count; i++) { diff --git a/src/lib/core/include/conda.h b/src/lib/core/include/conda.h index b8d0caa..8b4a0bd 100644 --- a/src/lib/core/include/conda.h +++ b/src/lib/core/include/conda.h @@ -24,7 +24,7 @@ #define PKG_INDEX_PROVIDES_E_MANAGER_RUNTIME (PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET + 3) #define PKG_INDEX_PROVIDES_E_MANAGER_SIGNALED (PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET + 4) #define PKG_INDEX_PROVIDES_E_MANAGER_EXEC (PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET + 5) -#define PKG_INDEX_PROVIDES_FAILED(ECODE) (ECODE <= PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET) +#define PKG_INDEX_PROVIDES_FAILED(ECODE) ((ECODE) <= PKG_INDEX_PROVIDES_ERROR_MESSAGE_OFFSET) struct MicromambaInfo { char *micromamba_prefix; //!< Path to write micromamba binary diff --git a/src/lib/core/include/core.h b/src/lib/core/include/core.h index 362ac8d..35a9506 100644 --- a/src/lib/core/include/core.h +++ b/src/lib/core/include/core.h @@ -11,10 +11,6 @@ #include <time.h> #include <sys/statvfs.h> -#define SYSERROR(MSG, ...) do { \ - fprintf(stderr, "%s:%s:%d:%s - ", path_basename(__FILE__), __FUNCTION__, __LINE__, (errno > 0) ? strerror(errno) : "info"); \ - fprintf(stderr, MSG LINE_SEP, __VA_ARGS__); \ -} while (0) #define STASIS_BUFSIZ 8192 #define STASIS_NAME_MAX 255 #define STASIS_DIRSTACK_MAX 1024 @@ -23,10 +19,11 @@ #include "config.h" #include "core_mem.h" +#include "core_message.h" #define COE_CHECK_ABORT(COND, MSG) \ do {\ - if (!globals.continue_on_error && COND) { \ + if (!globals.continue_on_error && (COND)) { \ msg(STASIS_MSG_ERROR, MSG ": Aborting execution (--continue-on-error/-C is not enabled)\n"); \ exit(1); \ } \ @@ -76,7 +73,6 @@ extern const char *VERSION; extern const char *AUTHOR; extern const char *BANNER; - /** * Free memory allocated in global configuration structure */ diff --git a/src/lib/core/include/core_mem.h b/src/lib/core/include/core_mem.h index bd50e9d..362715f 100644 --- a/src/lib/core/include/core_mem.h +++ b/src/lib/core/include/core_mem.h @@ -5,7 +5,7 @@ #include "environment.h" #include "strlist.h" -#define guard_runtime_free(X) do { if (X) { runtime_free(X); X = NULL; } } while (0) +#define guard_runtime_free(X) do { if (X) { runtime_free(X); (X) = NULL; } } while (0) #define guard_strlist_free(X) do { if ((*X)) { strlist_free(X); (*X) = NULL; } } while (0) #define guard_free(X) do { if (X) { free(X); X = NULL; } } while (0) #define GENERIC_ARRAY_FREE(ARR) do { \ diff --git a/src/lib/core/include/core_message.h b/src/lib/core/include/core_message.h new file mode 100644 index 0000000..1ffa846 --- /dev/null +++ b/src/lib/core/include/core_message.h @@ -0,0 +1,19 @@ + +#ifndef STASIS_CORE_MESSAGE_H +#define STASIS_CORE_MESSAGE_H + +#define SYSERROR(MSG, ...) do { \ + fprintf(stderr, "%s:%d:%s():%s - ", path_basename(__FILE__), __LINE__, __FUNCTION__, (errno > 0) ? strerror(errno) : "info"); \ + fprintf(stderr, MSG LINE_SEP, __VA_ARGS__); \ +} while (0) + +#ifdef DEBUG +#define SYSDEBUG(MSG, ...) do { \ + fprintf(stderr, "DEBUG: %s:%d:%s(): ", path_basename(__FILE__), __LINE__, __FUNCTION__); \ + fprintf(stderr, MSG LINE_SEP, __VA_ARGS__); \ +} while (0) +#else +#define SYSDEBUG(MSG, ...) do {} while (0) +#endif + +#endif //STASIS_CORE_MESSAGE_H diff --git a/src/lib/core/include/multiprocessing.h b/src/lib/core/include/multiprocessing.h index ec7c1ad..ff674e9 100644 --- a/src/lib/core/include/multiprocessing.h +++ b/src/lib/core/include/multiprocessing.h @@ -16,7 +16,9 @@ struct MultiProcessingTask { int status; ///< Child process exit status int signaled_by; ///< Last signal received, if any time_t _now; ///< Current time - time_t _seconds; ///< Time elapsed (used by MultiprocessingPool.status_interval) + time_t _seconds; ///< Time elapsed since status interval (used by MultiprocessingPool.status_interval) + time_t _startup; ///< Time elapsed since task started + long elapsed; ///< Total time elapsed in seconds char ident[255]; ///< Identity of the pool task char *cmd; ///< Shell command(s) to be executed size_t cmd_len; ///< Length of command string (for mmap/munmap) diff --git a/src/lib/core/ini.c b/src/lib/core/ini.c index 14f337d..4f449c6 100644 --- a/src/lib/core/ini.c +++ b/src/lib/core/ini.c @@ -490,18 +490,12 @@ char *unquote(char *s) { void ini_free(struct INIFILE **ini) { for (size_t section = 0; section < (*ini)->section_count; section++) { -#ifdef DEBUG - SYSERROR("freeing section: %s", (*ini)->section[section]->key); -#endif + SYSDEBUG("freeing section: %s", (*ini)->section[section]->key); for (size_t data = 0; data < (*ini)->section[section]->data_count; data++) { if ((*ini)->section[section]->data[data]) { -#ifdef DEBUG - SYSERROR("freeing data key: %s", (*ini)->section[section]->data[data]->key); -#endif + SYSDEBUG("freeing data key: %s", (*ini)->section[section]->data[data]->key); guard_free((*ini)->section[section]->data[data]->key); -#ifdef DEBUG - SYSERROR("freeing data value: %s", (*ini)->section[section]->data[data]->value); -#endif + SYSDEBUG("freeing data value: %s", (*ini)->section[section]->data[data]->value); guard_free((*ini)->section[section]->data[data]->value); guard_free((*ini)->section[section]->data[data]); } diff --git a/src/lib/core/multiprocessing.c b/src/lib/core/multiprocessing.c index 252bab9..0cf251e 100644 --- a/src/lib/core/multiprocessing.c +++ b/src/lib/core/multiprocessing.c @@ -78,6 +78,7 @@ int parent(struct MultiProcessingPool *pool, struct MultiProcessingTask *task, p } static int mp_task_fork(struct MultiProcessingPool *pool, struct MultiProcessingTask *task) { + SYSDEBUG("Preparing to fork() child task %s:%s", pool->ident, task->ident); pid_t pid = fork(); int child_status = 0; if (pid == -1) { @@ -89,8 +90,10 @@ static int mp_task_fork(struct MultiProcessingPool *pool, struct MultiProcessing } struct MultiProcessingTask *mp_pool_task(struct MultiProcessingPool *pool, const char *ident, char *working_dir, char *cmd) { + SYSDEBUG("%s", "Finding next available slot"); struct MultiProcessingTask *slot = mp_pool_next_available(pool); if (pool->num_used != pool->num_alloc) { + SYSDEBUG("Using slot %zu of %zu", pool->num_used, pool->num_alloc); pool->num_used++; } else { fprintf(stderr, "Maximum number of tasks reached\n"); @@ -119,6 +122,7 @@ struct MultiProcessingTask *mp_pool_task(struct MultiProcessingPool *pool, const // Create a temporary file to act as our intermediate command script FILE *tp = NULL; char *t_name = NULL; + t_name = xmkstemp(&tp, "w"); if (!t_name || !tp) { return NULL; @@ -135,11 +139,13 @@ struct MultiProcessingTask *mp_pool_task(struct MultiProcessingPool *pool, const guard_free(t_name); // Populate the script + SYSDEBUG("Generating runner script: %s", slot->parent_script); fprintf(tp, "#!/bin/bash\n%s\n", cmd); fflush(tp); fclose(tp); // Record the command(s) + SYSDEBUG("%s", "mmap() slot command"); slot->cmd_len = (strlen(cmd) * sizeof(*cmd)) + 1; slot->cmd = mmap(NULL, slot->cmd_len, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); memset(slot->cmd, 0, slot->cmd_len); @@ -230,14 +236,17 @@ int mp_pool_kill(struct MultiProcessingPool *pool, int signum) { exit(1); } // We are short-circuiting the normal flow, and the process is now dead, so mark it as such + SYSDEBUG("Marking slot %zu: UNUSED", i); slot->pid = MP_POOL_PID_UNUSED; } } } if (!access(slot->log_file, F_OK)) { + SYSDEBUG("Removing log file: %s", slot->log_file); remove(slot->log_file); } if (!access(slot->parent_script, F_OK)) { + SYSDEBUG("Removing runner script: %s", slot->parent_script); remove(slot->parent_script); } } @@ -261,6 +270,7 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) { for (size_t i = lower_i; i < upper_i; i++) { struct MultiProcessingTask *slot = &pool->task[i]; if (slot->status == MP_POOL_TASK_STATUS_INITIAL) { + slot->_startup = time(NULL); if (mp_task_fork(pool, slot)) { fprintf(stderr, "%s: mp_task_fork failed\n", slot->ident); kill(0, SIGTERM); @@ -271,6 +281,7 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) { if (slot->pid == MP_POOL_PID_UNUSED) { // Child is already used up, skip it hang_check++; + SYSDEBUG("slot %zu: hang_check=%zu", i, hang_check); if (hang_check >= pool->num_used) { // If you join a pool that's already finished it will spin // forever. This protects the program from entering an @@ -331,7 +342,7 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) { } if (status >> 8 != 0 || (status & 0xff) != 0) { - fprintf(stderr, "%s Task failed\n", progress); + fprintf(stderr, "%s Task failed after %lus\n", progress, slot->elapsed); failures++; if (flags & MP_POOL_FAIL_FAST && pool->num_used > 1) { @@ -339,7 +350,7 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) { return -2; } } else { - printf("%s Task finished\n", progress); + printf("%s Task finished after %lus\n", progress, slot->elapsed); } // Clean up logs and scripts left behind by the task @@ -365,9 +376,10 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags) { slot->_seconds = 0; } if (slot->_seconds == 0) { - printf("[%s:%s] Task is running (pid: %d)\n", pool->ident, slot->ident, slot->parent_pid); + printf("[%s:%s] Task is running (pid: %d, elapsed: %lus)\n", pool->ident, slot->ident, slot->parent_pid, slot->elapsed); } } + slot->elapsed = time(NULL) - slot->_startup; } if (tasks_complete == pool->num_used) { diff --git a/src/lib/core/str.c b/src/lib/core/str.c index d774e72..6457afe 100644 --- a/src/lib/core/str.c +++ b/src/lib/core/str.c @@ -526,7 +526,7 @@ char *normalize_space(char *s) { return NULL; } - if ((tmp = calloc(strlen(s) + 1, sizeof(char))) == NULL) { + if (!(tmp = strdup(s))) { perror("could not allocate memory for temporary string"); return NULL; } diff --git a/src/lib/core/template.c b/src/lib/core/template.c index 60ed91e..f3eab62 100644 --- a/src/lib/core/template.c +++ b/src/lib/core/template.c @@ -20,6 +20,7 @@ struct tplfunc_frame *tpl_pool_func[1024] = {0}; unsigned tpl_pool_func_used = 0; extern void tpl_reset() { + SYSDEBUG("%s", "Resetting template engine"); tpl_free(); tpl_pool_used = 0; tpl_pool_func_used = 0; @@ -31,19 +32,23 @@ void tpl_register_func(char *key, void *tplfunc_ptr, int argc, void *data_in) { frame->argc = argc; frame->func = tplfunc_ptr; frame->data_in = data_in; + SYSDEBUG("Registering function:\n\tkey=%s\n\targc=%d\n\tfunc=%p\n\tdata_in=%p", frame->key, frame->argc, frame->func, frame->data_in); tpl_pool_func[tpl_pool_func_used] = frame; tpl_pool_func_used++; } int tpl_key_exists(char *key) { + SYSDEBUG("Key '%s' exists?", key); for (size_t i = 0; i < tpl_pool_used; i++) { if (tpl_pool[i]->key) { if (!strcmp(tpl_pool[i]->key, key)) { + SYSDEBUG("%s", "YES"); return true; } } } + SYSDEBUG("%s", "NO"); return false; } @@ -51,6 +56,7 @@ void tpl_register(char *key, char **ptr) { struct tpl_item *item = NULL; int replacing = 0; + SYSDEBUG("Registering string:\n\tkey=%s\n\tptr=%s", key, *ptr ? *ptr : "NOT SET"); if (tpl_key_exists(key)) { for (size_t i = 0; i < tpl_pool_used; i++) { if (tpl_pool[i]->key) { @@ -61,7 +67,9 @@ void tpl_register(char *key, char **ptr) { } } replacing = 1; + SYSDEBUG("%s", "Item will be replaced"); } else { + SYSDEBUG("%s", "Creating new item"); item = calloc(1, sizeof(*item)); item->key = strdup(key); } @@ -73,6 +81,7 @@ void tpl_register(char *key, char **ptr) { item->ptr = ptr; if (!replacing) { + SYSDEBUG("Registered tpl_item at index %u:\n\tkey=%s\n\tptr=%s", tpl_pool_used, item->key, *item->ptr); tpl_pool[tpl_pool_used] = item; tpl_pool_used++; } @@ -83,27 +92,26 @@ void tpl_free() { struct tpl_item *item = tpl_pool[i]; if (item) { if (item->key) { -#ifdef DEBUG - SYSERROR("freeing template item key: %s", item->key); -#endif + SYSDEBUG("freeing template item key: %s", item->key); guard_free(item->key); } -#ifdef DEBUG - SYSERROR("freeing template item: %p", item); -#endif + SYSDEBUG("freeing template item: %p", item); item->ptr = NULL; } guard_free(item); } for (unsigned i = 0; i < tpl_pool_func_used; i++) { struct tplfunc_frame *item = tpl_pool_func[i]; + SYSDEBUG("freeing template function key: %s", item->key); guard_free(item->key); + SYSDEBUG("freeing template item: %p", item); guard_free(item); } } char *tpl_getval(char *key) { char *result = NULL; + SYSDEBUG("Getting value of template string: %s", key); for (size_t i = 0; i < tpl_pool_used; i++) { if (tpl_pool[i]->key) { if (!strcmp(tpl_pool[i]->key, key)) { @@ -116,6 +124,7 @@ char *tpl_getval(char *key) { } struct tplfunc_frame *tpl_getfunc(char *key) { + SYSDEBUG("Getting function frame: %s", key); struct tplfunc_frame *result = NULL; for (size_t i = 0; i < tpl_pool_func_used; i++) { if (tpl_pool_func[i]->key) { @@ -131,9 +140,8 @@ struct tplfunc_frame *tpl_getfunc(char *key) { static int grow(size_t z, size_t *output_bytes, char **output) { if (z >= *output_bytes) { size_t new_size = *output_bytes + z + 1; -#ifdef DEBUG - fprintf(stderr, "template output buffer new size: %zu\n", new_size); -#endif + SYSDEBUG("template output buffer new size: %zu\n", new_size); + char *tmp = realloc(*output, new_size); if (!tmp) { perror("realloc failed"); @@ -178,6 +186,7 @@ char *tpl_render(char *str) { } // Read key name + SYSDEBUG("%s", "Reading key"); size_t key_len = 0; while (isalnum(pos[off]) || pos[off] != '}') { if (isspace(pos[off]) || isblank(pos[off])) { @@ -189,6 +198,7 @@ char *tpl_render(char *str) { key_len++; off++; } + SYSDEBUG("Key is %s", key); char *type_stop = NULL; type_stop = strchr(key, ':'); @@ -197,8 +207,10 @@ char *tpl_render(char *str) { int do_func = 0; if (type_stop) { if (!strncmp(key, "env", type_stop - key)) { + SYSDEBUG("%s", "Will render as value of environment variable"); do_env = 1; } else if (!strncmp(key, "func", type_stop - key)) { + SYSDEBUG("%s", "Will render as output from function"); do_func = 1; } } @@ -207,6 +219,7 @@ char *tpl_render(char *str) { b_close = strstr(&pos[off], "}}"); if (!b_close) { fprintf(stderr, "error while templating '%s'\n\nunbalanced brace at position %zu\n", str, z); + guard_free(output); return NULL; } else { // Jump past closing brace @@ -259,12 +272,14 @@ char *tpl_render(char *str) { fprintf(stderr, "%s returned non-zero status: %d\n", frame->key, func_status); } value = strdup(func_result ? func_result : ""); + SYSDEBUG("Returned from function: %s (status: %d)\nData OUT\n--------\n'%s'", k, func_status, value); guard_free(func_result); } GENERIC_ARRAY_FREE(params); } else { // Read replacement value value = strdup(tpl_getval(key) ? tpl_getval(key) : ""); + SYSDEBUG("Rendered:\nData\n----\n'%s'", value); } } @@ -279,16 +294,11 @@ char *tpl_render(char *str) { output[z] = 0; } -#ifdef DEBUG - fprintf(stderr, "z=%zu, output_bytes=%zu\n", z, output_bytes); -#endif output[z] = pos[off]; z++; } -#ifdef DEBUG - fprintf(stderr, "template output length: %zu\n", strlen(output)); - fprintf(stderr, "template output bytes: %zu\n", output_bytes); -#endif + SYSDEBUG("template output length: %zu", strlen(output)); + SYSDEBUG("template output bytes: %zu", output_bytes); return output; } @@ -300,6 +310,7 @@ int tpl_render_to_file(char *str, const char *filename) { } // Open the destination file for writing + SYSDEBUG("Rendering to %s", filename); FILE *fp = fopen(filename, "w+"); if (!fp) { guard_free(result); @@ -309,6 +320,7 @@ int tpl_render_to_file(char *str, const char *filename) { // Write rendered string to file fprintf(fp, "%s", result); fclose(fp); + SYSDEBUG("%s", "Rendered successfully"); guard_free(result); return 0; diff --git a/src/lib/delivery/delivery_install.c b/src/lib/delivery/delivery_install.c index d3aab01..15ad7e0 100644 --- a/src/lib/delivery/delivery_install.c +++ b/src/lib/delivery/delivery_install.c @@ -128,7 +128,7 @@ int delivery_purge_packages(struct Delivery *ctx, const char *env_name, int use_ char package_manager[100] = {0}; typedef int (fnptr)(const char *); - const char *current_env = conda_get_active_environment(); + char *current_env = conda_get_active_environment(); if (current_env) { conda_activate(ctx->storage.conda_install_prefix, env_name); } @@ -171,10 +171,12 @@ int delivery_purge_packages(struct Delivery *ctx, const char *env_name, int use_ status = 1; break; } + guard_free(command); } if (current_env) { conda_activate(ctx->storage.conda_install_prefix, current_env); + guard_free(current_env); } return status; diff --git a/src/lib/delivery/delivery_populate.c b/src/lib/delivery/delivery_populate.c index 63df9fd..f242328 100644 --- a/src/lib/delivery/delivery_populate.c +++ b/src/lib/delivery/delivery_populate.c @@ -69,16 +69,54 @@ int populate_delivery_cfg(struct Delivery *ctx, int render_mode) { if (!globals.always_update_base_environment) { globals.always_update_base_environment = ini_getval_bool(cfg, "default", "always_update_base_environment", render_mode, &err); } + if (globals.conda_install_prefix) { + guard_free(globals.conda_install_prefix); + } globals.conda_install_prefix = ini_getval_str(cfg, "default", "conda_install_prefix", render_mode, &err); + + if (globals.conda_packages) { + guard_strlist_free(&globals.conda_packages); + } globals.conda_packages = ini_getval_strlist(cfg, "default", "conda_packages", LINE_SEP, render_mode, &err); + + if (globals.pip_packages) { + guard_strlist_free(&globals.pip_packages); + } globals.pip_packages = ini_getval_strlist(cfg, "default", "pip_packages", LINE_SEP, render_mode, &err); + if (globals.jfrog.jfrog_artifactory_base_url) { + guard_free(globals.jfrog.jfrog_artifactory_base_url); + } globals.jfrog.jfrog_artifactory_base_url = ini_getval_str(cfg, "jfrog_cli_download", "url", render_mode, &err); + + if (globals.jfrog.jfrog_artifactory_product) { + guard_free(globals.jfrog.jfrog_artifactory_product); + } globals.jfrog.jfrog_artifactory_product = ini_getval_str(cfg, "jfrog_cli_download", "product", render_mode, &err); + + if (globals.jfrog.cli_major_ver) { + guard_free(globals.jfrog.cli_major_ver); + } globals.jfrog.cli_major_ver = ini_getval_str(cfg, "jfrog_cli_download", "version_series", render_mode, &err); + + if (globals.jfrog.version) { + guard_free(globals.jfrog.version); + } globals.jfrog.version = ini_getval_str(cfg, "jfrog_cli_download", "version", render_mode, &err); + + if (globals.jfrog.remote_filename) { + guard_free(globals.jfrog.remote_filename); + } globals.jfrog.remote_filename = ini_getval_str(cfg, "jfrog_cli_download", "filename", render_mode, &err); + + if (globals.jfrog.url) { + guard_free(globals.jfrog.url); + } globals.jfrog.url = ini_getval_str(cfg, "deploy:artifactory", "url", render_mode, &err); + + if (globals.jfrog.repo) { + guard_free(globals.jfrog.repo); + } globals.jfrog.repo = ini_getval_str(cfg, "deploy:artifactory", "repo", render_mode, &err); return 0; diff --git a/src/lib/delivery/delivery_postprocess.c b/src/lib/delivery/delivery_postprocess.c index d8ac58b..b43e247 100644 --- a/src/lib/delivery/delivery_postprocess.c +++ b/src/lib/delivery/delivery_postprocess.c @@ -66,7 +66,9 @@ void delivery_rewrite_spec(struct Delivery *ctx, char *filename, unsigned stage) FILE *tp = NULL; if (stage == DELIVERY_REWRITE_SPEC_STAGE_1) { + SYSDEBUG("%s", "Entering stage 1"); header = delivery_get_release_header(ctx); + SYSDEBUG("Release header:\n%s", header); if (!header) { msg(STASIS_MSG_ERROR, "failed to generate release header string\n", filename); exit(1); @@ -76,6 +78,7 @@ void delivery_rewrite_spec(struct Delivery *ctx, char *filename, unsigned stage) msg(STASIS_MSG_ERROR, "%s: unable to create temporary file\n", strerror(errno)); exit(1); } + SYSDEBUG("Writing header to temporary file: %s", tempfile); fprintf(tp, "%s", header); // Read the original file @@ -88,19 +91,22 @@ void delivery_rewrite_spec(struct Delivery *ctx, char *filename, unsigned stage) // Write temporary data for (size_t i = 0; contents[i] != NULL; i++) { if (startswith(contents[i], "channels:")) { - // Allow for additional conda channel injection if (ctx->conda.conda_packages_defer && strlist_count(ctx->conda.conda_packages_defer)) { + // Allow for additional conda channel injection + SYSDEBUG("Appending conda channel template on line %zu", i); fprintf(tp, "%s - @CONDA_CHANNEL@\n", contents[i]); continue; } } else if (strstr(contents[i], "- pip:")) { if (ctx->conda.pip_packages_defer && strlist_count(ctx->conda.pip_packages_defer)) { // Allow for additional pip argument injection + SYSDEBUG("Appending pip argument template on line %zu", i); fprintf(tp, "%s - @PIP_ARGUMENTS@\n", contents[i]); continue; } } else if (startswith(contents[i], "prefix:")) { // Remove the prefix key + SYSDEBUG("Removing prefix key on line %zu", i); if (strstr(contents[i], "/") || strstr(contents[i], "\\")) { // path is on the same line as the key continue; @@ -118,38 +124,49 @@ void delivery_rewrite_spec(struct Delivery *ctx, char *filename, unsigned stage) guard_free(header); fflush(tp); fclose(tp); + SYSDEBUG("Done writing temporary file: %s", tempfile); // Replace the original file with our temporary data if (copy2(tempfile, filename, CT_PERM) < 0) { fprintf(stderr, "%s: could not rename '%s' to '%s'\n", strerror(errno), tempfile, filename); exit(1); } + SYSDEBUG("Removing file: %s", tempfile); remove(tempfile); guard_free(tempfile); } else if (globals.enable_rewrite_spec_stage_2 && stage == DELIVERY_REWRITE_SPEC_STAGE_2) { + SYSDEBUG("%s", "Entering stage 2"); char output[PATH_MAX] = {0}; // Replace "local" channel with the staging URL if (ctx->storage.conda_staging_url) { + SYSDEBUG("%s", "Will replace conda channel with staging area url"); file_replace_text(filename, "@CONDA_CHANNEL@", ctx->storage.conda_staging_url, 0); } else if (globals.jfrog.repo) { + SYSDEBUG("%s", "Will replace conda channel with artifactory repo packages/conda url"); sprintf(output, "%s/%s/%s/%s/packages/conda", globals.jfrog.url, globals.jfrog.repo, ctx->meta.mission, ctx->info.build_name); file_replace_text(filename, "@CONDA_CHANNEL@", output, 0); } else { + SYSDEBUG("%s", "Will replace conda channel with local conda artifact directory"); msg(STASIS_MSG_WARN, "conda_staging_dir is not configured. Using fallback: '%s'\n", ctx->storage.conda_artifact_dir); file_replace_text(filename, "@CONDA_CHANNEL@", ctx->storage.conda_artifact_dir, 0); } if (ctx->storage.wheel_staging_url) { + SYSDEBUG("%s", "Will replace pip arguments with wheel staging url"); + sprintf(output, "--extra-index-url %s/%s/%s/packages/wheels", ctx->storage.wheel_staging_url, ctx->meta.mission, ctx->info.build_name); file_replace_text(filename, "@PIP_ARGUMENTS@", ctx->storage.wheel_staging_url, 0); } else if (globals.enable_artifactory && globals.jfrog.url && globals.jfrog.repo) { + SYSDEBUG("%s", "Will replace pip arguments with artifactory repo packages/wheel url"); sprintf(output, "--extra-index-url %s/%s/%s/%s/packages/wheels", globals.jfrog.url, globals.jfrog.repo, ctx->meta.mission, ctx->info.build_name); file_replace_text(filename, "@PIP_ARGUMENTS@", output, 0); } else { + SYSDEBUG("%s", "Will replace pip arguments with local wheel artifact directory"); msg(STASIS_MSG_WARN, "wheel_staging_dir is not configured. Using fallback: '%s'\n", ctx->storage.wheel_artifact_dir); sprintf(output, "--extra-index-url file://%s", ctx->storage.wheel_artifact_dir); file_replace_text(filename, "@PIP_ARGUMENTS@", output, 0); } } + SYSDEBUG("%s", "Rewriting finished"); } int delivery_copy_conda_artifacts(struct Delivery *ctx) { @@ -201,6 +218,7 @@ int delivery_index_wheel_artifacts(struct Delivery *ctx) { // pip install --extra-index-url char top_index[PATH_MAX] = {0}; sprintf(top_index, "%s/index.html", ctx->storage.wheel_artifact_dir); + SYSDEBUG("Opening top-level index for writing: %s", top_index); FILE *top_fp = fopen(top_index, "w+"); if (!top_fp) { closedir(dp); @@ -215,6 +233,7 @@ int delivery_index_wheel_artifacts(struct Delivery *ctx) { char bottom_index[PATH_MAX * 2] = {0}; sprintf(bottom_index, "%s/%s/index.html", ctx->storage.wheel_artifact_dir, rec->d_name); + SYSDEBUG("Opening bottom-level for writing: %s", bottom_index); FILE *bottom_fp = fopen(bottom_index, "w+"); if (!bottom_fp) { closedir(dp); @@ -225,6 +244,7 @@ int delivery_index_wheel_artifacts(struct Delivery *ctx) { printf("+ %s\n", rec->d_name); } // Add record to top level index + SYSDEBUG("Appending top-level link for %s", rec->d_name); fprintf(top_fp, "<a href=\"%s/\">%s</a><br/>\n", rec->d_name, rec->d_name); char dpath[PATH_MAX * 2] = {0}; @@ -246,6 +266,7 @@ int delivery_index_wheel_artifacts(struct Delivery *ctx) { printf("`- %s\n", package); } // Write record to bottom level index + SYSDEBUG("Appending bottom-level link for %s", package); fprintf(bottom_fp, "<a href=\"%s\">%s</a><br/>\n", package, package); } fclose(bottom_fp); @@ -254,5 +275,6 @@ int delivery_index_wheel_artifacts(struct Delivery *ctx) { } closedir(dp); fclose(top_fp); + SYSDEBUG("%s", "Wheel indexing complete"); return 0; } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index bcc05ce..2b09e9e 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,9 +1,9 @@ find_program(BASH_PROGRAM bash) set(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/tests) set(CTEST_BINARY_DIRECTORY ${PROJECT_BINARY_DIR}/tests) -set(nix_gnu_cflags -Wno-format-truncation -Wno-error -Wno-unused-parameter -Wno-unused-result -Wno-discarded-qualifiers) -set(nix_clang_cflags -Wno-format-truncation -Wno-unused-parameter -Wno-unused-result -Wno-incompatible-pointer-types-discards-qualifiers) -set(win_msvc_cflags /Wall) +set(nix_gnu_cflags ${CMAKE_C_FLAGS} -Wno-format-truncation -Wno-error -Wno-unused-parameter -Wno-unused-result -Wno-discarded-qualifiers) +set(nix_clang_cflags ${CMAKE_C_FLAGS} -Wno-format-truncation -Wno-unused-parameter -Wno-unused-result -Wno-incompatible-pointer-types-discards-qualifiers) +set(win_msvc_cflags ${CMAKE_C_FLAGS} /Wall) file(GLOB source_files "test_*.c") file(GLOB rt_files "rt_*.sh") @@ -12,7 +12,7 @@ set(ext_pattern "(^.*/|\\.[^.]*$)") file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/data DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) -if (BASH_PROGRAM AND BUILD_TESTING_RT) +if (BASH_PROGRAM AND TESTS_RT) foreach(rt_file ${rt_files}) file(REAL_PATH ${rt_file} rt_name) string(REGEX REPLACE ${ext_pattern} "" rt_name ${rt_file}) diff --git a/tests/include/testing.h b/tests/include/testing.h index 4c97bf2..ab24115 100644 --- a/tests/include/testing.h +++ b/tests/include/testing.h @@ -92,9 +92,10 @@ inline char *stasis_testing_read_ascii(const char *filename) { return NULL; } - char *result; + char *result = NULL; result = calloc(st.st_size + 1, sizeof(*result)); if (fread(result, sizeof(*result), st.st_size, fp) < 1) { + guard_free(result); perror(filename); fclose(fp); return NULL; @@ -211,7 +212,7 @@ inline void stasis_testing_teardown_workspace() { .lineno = __LINE__, \ .status = (COND), \ .msg_assertion = "ASSERT(" #COND ")", \ - .msg_reason = REASON } ); \ + .msg_reason = (REASON) } ); \ } while (0) #define STASIS_ASSERT_FATAL(COND, REASON) do { \ @@ -221,7 +222,7 @@ inline void stasis_testing_teardown_workspace() { .lineno = __LINE__, \ .status = (COND), \ .msg_assertion = "ASSERT FATAL (" #COND ")", \ - .msg_reason = REASON } \ + .msg_reason = (REASON) } \ ); \ if (stasis_test_results[stasis_test_results_i ? stasis_test_results_i - 1 : stasis_test_results_i].status == false) {\ exit(STASIS_TEST_SUITE_FATAL); \ @@ -236,7 +237,7 @@ inline void stasis_testing_teardown_workspace() { .status = true, \ .skip = (COND), \ .msg_assertion = "SKIP (" #COND ")", \ - .msg_reason = REASON } \ + .msg_reason = (REASON) } \ ); \ if (stasis_test_results[stasis_test_results_i ? stasis_test_results_i - 1 : stasis_test_results_i].skip == true) {\ return; \ @@ -244,9 +245,9 @@ inline void stasis_testing_teardown_workspace() { } while (0) #define STASIS_TEST_RUN(X) do { \ - for (size_t i = 0; i < sizeof(X) / sizeof(*X); i++) { \ - if (X[i]) { \ - X[i](); \ + for (size_t i = 0; i < sizeof(X) / sizeof(*(X)); i++) { \ + if ((X)[i]) { \ + (X)[i](); \ } \ } \ } while (0) diff --git a/tests/test_artifactory.c b/tests/test_artifactory.c index 4f41543..4af7eec 100644 --- a/tests/test_artifactory.c +++ b/tests/test_artifactory.c @@ -2,9 +2,7 @@ #include "artifactory.h" #include "delivery.h" -// Import private functions from core -extern int delivery_init_platform(struct Delivery *ctx); -extern int populate_delivery_cfg(struct Delivery *ctx, int render_mode); + struct JFRT_Auth gauth; struct JFRT_Auth gnoauth; @@ -78,6 +76,7 @@ int main(int argc, char *argv[]) { mkdir(ws, 0755); if (pushd(ws)) { SYSERROR("failed to change directory to %s", ws); + guard_free(basedir); STASIS_ASSERT_FATAL(true, "workspace creation failed"); } diff --git a/tests/test_conda.c b/tests/test_conda.c index 63a2781..81f593f 100644 --- a/tests/test_conda.c +++ b/tests/test_conda.c @@ -43,6 +43,8 @@ void test_conda_installation() { delivery_get_conda_installer_url(&ctx, install_url); delivery_get_conda_installer(&ctx, install_url); delivery_install_conda(ctx.conda.installer_path, ctx.storage.conda_install_prefix); + + guard_free(install_url); STASIS_ASSERT_FATAL(access(ctx.storage.conda_install_prefix, F_OK) == 0, "conda was not installed correctly"); STASIS_ASSERT_FATAL(conda_activate(ctx.storage.conda_install_prefix, "base") == 0, "unable to activate base environment"); STASIS_ASSERT_FATAL(conda_exec("info") == 0, "conda was not installed correctly"); @@ -155,7 +157,9 @@ void test_pip_index_provides() { void test_conda_get_active_environment() { conda_activate(ctx.storage.conda_install_prefix, "base"); - STASIS_ASSERT(strcmp(conda_get_active_environment(), "base") == 0, "base environment not active"); + char *active_env = conda_get_active_environment(); + STASIS_ASSERT(strcmp(active_env, "base") == 0, "base environment not active"); + guard_free(active_env); } void test_conda_provides() { diff --git a/tests/test_environment.c b/tests/test_environment.c index 134d061..43c3672 100644 --- a/tests/test_environment.c +++ b/tests/test_environment.c @@ -1,6 +1,6 @@ #include "testing.h" -extern char **environ; + void test_runtime_copy() { RuntimeEnv *env = runtime_copy(environ); @@ -27,7 +27,7 @@ void test_runtime_copy_empty() { void test_runtime() { RuntimeEnv *env = runtime_copy(environ); runtime_set(env, "CUSTOM_KEY", "Very custom"); - ssize_t idx = -1; + ssize_t idx; STASIS_ASSERT((idx = runtime_contains(env, "CUSTOM_KEY")) >= 0, "CUSTOM_KEY should exist in object"); STASIS_ASSERT(strcmp(strlist_item(env, idx), "CUSTOM_KEY=Very custom") == 0, "Incorrect index returned by runtime_contains"); diff --git a/tests/test_template.c b/tests/test_template.c index 79fce5e..596c2b7 100644 --- a/tests/test_template.c +++ b/tests/test_template.c @@ -47,9 +47,15 @@ void test_tpl_workflow() { tpl_reset(); tpl_register("hello_message", &data); - STASIS_ASSERT(strcmp(tpl_render("I said, \"{{ hello_message }}\""), "I said, \"Hello world!\"") == 0, "stored value in key is incorrect"); + char *result = NULL; + result = tpl_render("I said, \"{{ hello_message }}\""); + STASIS_ASSERT(strcmp(result, "I said, \"Hello world!\"") == 0, "stored value in key is incorrect"); + guard_free(result); + setenv("HELLO", "Hello environment!", 1); - STASIS_ASSERT(strcmp(tpl_render("{{ env:HELLO }}"), "Hello environment!") == 0, "environment variable content mismatch"); + result = tpl_render("{{ env:HELLO }}"); + STASIS_ASSERT(strcmp(result, "Hello environment!") == 0, "environment variable content mismatch"); + guard_free(result); unsetenv("HELLO"); guard_free(data); } @@ -87,18 +93,25 @@ void test_tpl_register_func() { char *result = NULL; result = tpl_render("{{ func:add(0,3) }}"); STASIS_ASSERT(result != NULL && strcmp(result, "3") == 0, "Answer was not 3"); + guard_free(result); result = tpl_render("{{ func:add(1,2) }}"); STASIS_ASSERT(result != NULL && strcmp(result, "3") == 0, "Answer was not 3"); + guard_free(result); result = tpl_render("{{ func:sub(6,3) }}"); STASIS_ASSERT(result != NULL && strcmp(result, "3") == 0, "Answer was not 3"); + guard_free(result); result = tpl_render("{{ func:sub(4,1) }}"); STASIS_ASSERT(result != NULL && strcmp(result, "3") == 0, "Answer was not 3"); + guard_free(result); result = tpl_render("{{ func:mul(1, 3) }}"); STASIS_ASSERT(result != NULL && strcmp(result, "3") == 0, "Answer was not 3"); + guard_free(result); result = tpl_render("{{ func:div(6,2) }}"); STASIS_ASSERT(result != NULL && strcmp(result, "3") == 0, "Answer was not 3"); + guard_free(result); result = tpl_render("{{ func:div(3,1) }}"); STASIS_ASSERT(result != NULL && strcmp(result, "3") == 0, "Answer was not 3"); + guard_free(result); } int main(int argc, char *argv[]) { |