From b7251ce3bf65bcbec7ecbb98a0eb0b3c9abde507 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 13 Sep 2024 09:35:34 -0400 Subject: Move guard_ macros to core_mem.h * Move core_mem.h below config.h --- include/core.h | 13 ++--- include/core_mem.h | 18 +++++++ include/multiprocessing.h | 120 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 141 insertions(+), 10 deletions(-) create mode 100644 include/core_mem.h create mode 100644 include/multiprocessing.h (limited to 'include') diff --git a/include/core.h b/include/core.h index ef90e96..614a894 100644 --- a/include/core.h +++ b/include/core.h @@ -21,6 +21,7 @@ #define HTTP_ERROR(X) X >= 400 #include "config.h" +#include "core_mem.h" #include "envctl.h" #include "template.h" #include "utils.h" @@ -42,16 +43,6 @@ #include "github.h" #include "template_func_proto.h" -#define guard_runtime_free(X) do { if (X) { runtime_free(X); X = NULL; } } while (0) -#define guard_strlist_free(X) do { if ((*X)) { strlist_free(X); (*X) = NULL; } } while (0) -#define guard_free(X) do { if (X) { free(X); X = NULL; } } while (0) -#define GENERIC_ARRAY_FREE(ARR) do { \ - for (size_t ARR_I = 0; ARR && ARR[ARR_I] != NULL; ARR_I++) { \ - guard_free(ARR[ARR_I]); \ - } \ - guard_free(ARR); \ -} while (0) - #define COE_CHECK_ABORT(COND, MSG) \ do {\ if (!globals.continue_on_error && COND) { \ @@ -71,6 +62,8 @@ struct STASIS_GLOBAL { bool enable_testing; //!< Enable package testing bool enable_overwrite; //!< Enable release file clobbering bool enable_rewrite_spec_stage_2; //!< Enable automatic @STR@ replacement in output files + long cpu_limit; + long parallel_fail_fast; struct StrList *conda_packages; //!< Conda packages to install after initial activation struct StrList *pip_packages; //!< Pip packages to install after initial activation char *tmpdir; //!< Path to temporary storage directory diff --git a/include/core_mem.h b/include/core_mem.h new file mode 100644 index 0000000..ef07a00 --- /dev/null +++ b/include/core_mem.h @@ -0,0 +1,18 @@ +// +// Created by jhunk on 9/13/24. +// + +#ifndef STASIS_CORE_MEM_H +#define STASIS_CORE_MEM_H + +#define guard_runtime_free(X) do { if (X) { runtime_free(X); X = NULL; } } while (0) +#define guard_strlist_free(X) do { if ((*X)) { strlist_free(X); (*X) = NULL; } } while (0) +#define guard_free(X) do { if (X) { free(X); X = NULL; } } while (0) +#define GENERIC_ARRAY_FREE(ARR) do { \ + for (size_t ARR_I = 0; ARR && ARR[ARR_I] != NULL; ARR_I++) { \ + guard_free(ARR[ARR_I]); \ + } \ + guard_free(ARR); \ +} while (0) + +#endif //STASIS_CORE_MEM_H diff --git a/include/multiprocessing.h b/include/multiprocessing.h new file mode 100644 index 0000000..6bcf18e --- /dev/null +++ b/include/multiprocessing.h @@ -0,0 +1,120 @@ +/// @file multiprocessing.h +#ifndef STASIS_MULTIPROCESSING_H +#define STASIS_MULTIPROCESSING_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct MultiProcessingTask { + sem_t *gate; ///< Child process startup lock + pid_t pid; ///< Program PID + pid_t parent_pid; ///< Program PID (parent process) + int status; ///< Child process exit status + char ident[NAME_MAX]; ///< Identity of the pool task + char log_file[NAME_MAX]; ///< Path to stdout/stderr log file + char parent_script[PATH_MAX]; ///< Path to temporary script executing the task +}; + +struct MultiProcessingPool { + struct MultiProcessingTask *task; ///< Array of tasks to execute + size_t num_used; ///< Number of tasks populated in the task array + size_t num_alloc; ///< Number of tasks allocated by the task array + const char *ident; ///< Identity of task pool + const char *log_root; ///< Base directory to store stderr/stdout log files +}; + +///!< Maximum number of multiprocessing tasks STASIS can execute +#define MP_POOL_TASK_MAX 1000 + +///!< Value signifies a process is unused or finished executing +#define MP_POOL_PID_UNUSED 0 + +// Option flags for mp_pool_join() +#define MP_POOL_FAIL_FAST 1 << 1 + +/** + * Create a multiprocessing pool + * + * ```c + * #include "multiprocessing.h" + * #include "utils.h" // for get_cpu_count() + * + * int main(int argc, char *argv[]) { + * struct MultiProcessingPool *mp; + * mp = mp_pool_init("mypool", "/tmp/mypool_logs"); + * if (mp) { + * char *commands[] = { + * "/bin/echo hello world", + * "/bin/echo world hello", + * NULL + * } + * for (size_t i = 0; commands[i] != NULL); i++) { + * struct MultiProcessingTask *task; + * char task_name[100]; + * + * sprintf(task_name, "mytask%zu", i); + * task = mp_task(mp, task_name, commands[i]); + * if (!task) { + * // handle task creation error + * } + * } + * if (mp_pool_join(mp, get_cpu_count())) { + * // handle pool execution error + * } + * mp_pool_free(&mp); + * } else { + * // handle pool initialization error + * } + * } + * ``` + * + * @param ident a name to identify the pool + * @param log_root the path to store program output + * @return pointer to initialized MultiProcessingPool + * @return NULL on error + */ +struct MultiProcessingPool *mp_pool_init(const char *ident, const char *log_root); + +/** + * Create a multiprocessing pool task + * + * @param pool a pointer to MultiProcessingPool + * @param ident a name to identify the task + * @param cmd a command to execute + * @return pointer to MultiProcessingTask structure + * @return NULL on error + */ +struct MultiProcessingTask *mp_task(struct MultiProcessingPool *pool, const char *ident, char *cmd); + +/** + * Execute all tasks in a pool + * + * @param pool a pointer to MultiProcessingPool + * @param jobs the number of processes to spawn at once (for serial execution use `1`) + * @param flags option to be OR'd (MP_POOL_FAIL_FAST) + * @return 0 on success + * @return >0 on failure + * @return <0 on error + */ +int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags); + +/** + * Release resources allocated by mp_pool_init() + * + * @param a pointer to MultiProcessingPool + */ +void mp_pool_free(struct MultiProcessingPool **pool); + + +#endif //STASIS_MULTIPROCESSING_H -- cgit From 8f17199d16bcdb29516d34514f95d1a117f6bd26 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 13 Sep 2024 09:58:17 -0400 Subject: Implement multiprocessing pool(s) * Adds --cpu-limit and --parallel-fail-fast arguments * Adds disable, parallel, and setup_script keys to [test] blocks --- include/delivery.h | 3 +++ include/template_func_proto.h | 1 + 2 files changed, 4 insertions(+) (limited to 'include') diff --git a/include/delivery.h b/include/delivery.h index 067cd0b..6dd6cc4 100644 --- a/include/delivery.h +++ b/include/delivery.h @@ -149,7 +149,10 @@ struct Delivery { char *name; ///< Name of package char *version; ///< Version of package char *repository; ///< Git repository of package + char *script_setup; ///< Commands to execute before the main script char *script; ///< Commands to execute + bool disable; ///< Toggle a test block + bool parallel; ///< Toggle parallel or serial execution char *build_recipe; ///< Conda recipe to build (optional) char *repository_info_ref; ///< Git commit hash char *repository_info_tag; ///< Git tag (first parent) diff --git a/include/template_func_proto.h b/include/template_func_proto.h index 7778a11..0c8bbb7 100644 --- a/include/template_func_proto.h +++ b/include/template_func_proto.h @@ -7,5 +7,6 @@ int get_github_release_notes_tplfunc_entrypoint(void *frame, void *data_out); int get_github_release_notes_auto_tplfunc_entrypoint(void *frame, void *data_out); int get_junitxml_file_entrypoint(void *frame, void *data_out); int get_basetemp_dir_entrypoint(void *frame, void *data_out); +int tox_run_entrypoint(void *frame, void *data_out); #endif //TEMPLATE_FUNC_PROTO_H \ No newline at end of file -- cgit From 87971633eea3306a0b85a55d7d581841b9dbf905 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 13 Sep 2024 10:10:21 -0400 Subject: Fixing headers --- include/multiprocessing.h | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) (limited to 'include') diff --git a/include/multiprocessing.h b/include/multiprocessing.h index 6bcf18e..9e0c429 100644 --- a/include/multiprocessing.h +++ b/include/multiprocessing.h @@ -2,27 +2,20 @@ #ifndef STASIS_MULTIPROCESSING_H #define STASIS_MULTIPROCESSING_H -#include -#include +#include "core.h" #include -#include -#include -#include #include #include -#include #include -#include #include -#include struct MultiProcessingTask { sem_t *gate; ///< Child process startup lock pid_t pid; ///< Program PID pid_t parent_pid; ///< Program PID (parent process) int status; ///< Child process exit status - char ident[NAME_MAX]; ///< Identity of the pool task - char log_file[NAME_MAX]; ///< Path to stdout/stderr log file + char ident[255]; ///< Identity of the pool task + char log_file[255]; ///< Path to stdout/stderr log file char parent_script[PATH_MAX]; ///< Path to temporary script executing the task }; -- cgit From 1fe385d782ae117d2a68266e14777d890eddf4e0 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 13 Sep 2024 12:16:41 -0400 Subject: Darwin portability: Use sem_open and sem_close instead of sem_init and sem_destroy --- include/multiprocessing.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include') diff --git a/include/multiprocessing.h b/include/multiprocessing.h index 9e0c429..648bd80 100644 --- a/include/multiprocessing.h +++ b/include/multiprocessing.h @@ -3,6 +3,7 @@ #define STASIS_MULTIPROCESSING_H #include "core.h" +#include #include #include #include -- cgit From 17d3d0517123f5f07b4ac6bb9f1dec73c1c8ce4c Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 13 Sep 2024 13:36:09 -0400 Subject: Fix sem_open initial state * Move slot->gate assignment to mp_pool_task() * Remove mmap() to slot->gate. * Change type of ident and log_root variables for the sake of easy (fewer maps) --- include/multiprocessing.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/include/multiprocessing.h b/include/multiprocessing.h index 648bd80..f0e6e5c 100644 --- a/include/multiprocessing.h +++ b/include/multiprocessing.h @@ -24,8 +24,8 @@ struct MultiProcessingPool { struct MultiProcessingTask *task; ///< Array of tasks to execute size_t num_used; ///< Number of tasks populated in the task array size_t num_alloc; ///< Number of tasks allocated by the task array - const char *ident; ///< Identity of task pool - const char *log_root; ///< Base directory to store stderr/stdout log files + char ident[255]; ///< Identity of task pool + char log_root[PATH_MAX]; ///< Base directory to store stderr/stdout log files }; ///!< Maximum number of multiprocessing tasks STASIS can execute -- cgit From 8573ad716839caf34a2c9c016d5ffbcbaab5ba2e Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 13 Sep 2024 14:44:09 -0400 Subject: Fix mp_pool_join example --- include/multiprocessing.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include') diff --git a/include/multiprocessing.h b/include/multiprocessing.h index f0e6e5c..e4ddfdf 100644 --- a/include/multiprocessing.h +++ b/include/multiprocessing.h @@ -63,7 +63,7 @@ struct MultiProcessingPool { * // handle task creation error * } * } - * if (mp_pool_join(mp, get_cpu_count())) { + * if (mp_pool_join(mp, get_cpu_count(), MP_POOL_FAIL_FAST)) { * // handle pool execution error * } * mp_pool_free(&mp); -- cgit From 63cf3148cc3f60edf5c604a562ec95b40b8cb010 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 13 Sep 2024 15:01:29 -0400 Subject: Add multiprocessing.h to core.h * Remove multiprocessing.h from other files --- include/core.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include') diff --git a/include/core.h b/include/core.h index 614a894..5011dea 100644 --- a/include/core.h +++ b/include/core.h @@ -22,6 +22,7 @@ #include "config.h" #include "core_mem.h" +#include "multiprocessing.h" #include "envctl.h" #include "template.h" #include "utils.h" -- cgit From 1e320e21492d3c09bae269b10d8faf0d0d131bb9 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 13 Sep 2024 15:02:00 -0400 Subject: Fix doxygen comments --- include/multiprocessing.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'include') diff --git a/include/multiprocessing.h b/include/multiprocessing.h index e4ddfdf..4e89722 100644 --- a/include/multiprocessing.h +++ b/include/multiprocessing.h @@ -28,13 +28,13 @@ struct MultiProcessingPool { char log_root[PATH_MAX]; ///< Base directory to store stderr/stdout log files }; -///!< Maximum number of multiprocessing tasks STASIS can execute +/// Maximum number of multiprocessing tasks STASIS can execute #define MP_POOL_TASK_MAX 1000 -///!< Value signifies a process is unused or finished executing +/// Value signifies a process is unused or finished executing #define MP_POOL_PID_UNUSED 0 -// Option flags for mp_pool_join() +/// Option flags for mp_pool_join() #define MP_POOL_FAIL_FAST 1 << 1 /** -- cgit From db1a3056296ea3ed13c5a425cf1f11602b43a6c7 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 17 Sep 2024 09:37:14 -0400 Subject: Add pool summary and elapsed time output * Add get_task_duration() * Add get_pool_show_summary() * Add signaled_by member to MultiProcessingTask * Add time_data member to MultiProcessingTask for duration tracking --- include/multiprocessing.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'include') diff --git a/include/multiprocessing.h b/include/multiprocessing.h index 4e89722..2356403 100644 --- a/include/multiprocessing.h +++ b/include/multiprocessing.h @@ -15,9 +15,14 @@ struct MultiProcessingTask { pid_t pid; ///< Program PID pid_t parent_pid; ///< Program PID (parent process) int status; ///< Child process exit status + int signaled_by; ///< Last signal received, if any char ident[255]; ///< Identity of the pool task char log_file[255]; ///< Path to stdout/stderr log file char parent_script[PATH_MAX]; ///< Path to temporary script executing the task + struct { + struct timespec t_start; + struct timespec t_stop; + } time_data; ///< Wall-time counters }; struct MultiProcessingPool { @@ -103,6 +108,13 @@ struct MultiProcessingTask *mp_task(struct MultiProcessingPool *pool, const char */ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags); +/** + * Show summary of pool tasks + * + * @pararm pool a pointer to MultiProcessingPool + */ +void mp_pool_show_summary(struct MultiProcessingPool *pool); + /** * Release resources allocated by mp_pool_init() * -- cgit From 8b47235f7c81e04fa5efef492974509789f40273 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Wed, 18 Sep 2024 10:04:21 -0400 Subject: Rename mp_task to mp_pool_task --- include/multiprocessing.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include') diff --git a/include/multiprocessing.h b/include/multiprocessing.h index 2356403..2db33df 100644 --- a/include/multiprocessing.h +++ b/include/multiprocessing.h @@ -94,7 +94,7 @@ struct MultiProcessingPool *mp_pool_init(const char *ident, const char *log_root * @return pointer to MultiProcessingTask structure * @return NULL on error */ -struct MultiProcessingTask *mp_task(struct MultiProcessingPool *pool, const char *ident, char *cmd); +struct MultiProcessingTask *mp_pool_task(struct MultiProcessingPool *pool, const char *ident, char *cmd); /** * Execute all tasks in a pool -- cgit From c8c0802c98db5e3d6de6b79887133e0d15567fd8 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Wed, 18 Sep 2024 10:44:13 -0400 Subject: Fix a likely buffer overflow * The log_root is appended to log_file, and might exceed the defined length. --- include/multiprocessing.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include') diff --git a/include/multiprocessing.h b/include/multiprocessing.h index 2db33df..1974e41 100644 --- a/include/multiprocessing.h +++ b/include/multiprocessing.h @@ -17,7 +17,7 @@ struct MultiProcessingTask { int status; ///< Child process exit status int signaled_by; ///< Last signal received, if any char ident[255]; ///< Identity of the pool task - char log_file[255]; ///< Path to stdout/stderr log file + char log_file[PATH_MAX]; ///< Full path to stdout/stderr log file char parent_script[PATH_MAX]; ///< Path to temporary script executing the task struct { struct timespec t_start; -- cgit From 1999612a69a7947fb1b6fc45705299fe6db650ba Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 20 Sep 2024 08:36:26 -0400 Subject: Refactor structure * Break delivery.c into smaller components --- include/delivery.h | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) (limited to 'include') diff --git a/include/delivery.h b/include/delivery.h index 6dd6cc4..6da890a 100644 --- a/include/delivery.h +++ b/include/delivery.h @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include "core.h" #define DELIVERY_PLATFORM_MAX 4 @@ -289,7 +291,7 @@ int delivery_copy_conda_artifacts(struct Delivery *ctx); * Retrieve Conda installer * @param installer_url URL to installation script */ -int delivery_get_installer(struct Delivery *ctx, char *installer_url); +int delivery_get_conda_installer(struct Delivery *ctx, char *installer_url); /** * Generate URL based on Delivery context @@ -297,7 +299,7 @@ int delivery_get_installer(struct Delivery *ctx, char *installer_url); * @param result pointer to char * @return in result */ -void delivery_get_installer_url(struct Delivery *ctx, char *result); +void delivery_get_conda_installer_url(struct Delivery *ctx, char *result); /** * Install packages based on Delivery context @@ -379,6 +381,12 @@ void delivery_gather_tool_versions(struct Delivery *ctx); // helper function int delivery_init_tmpdir(struct Delivery *ctx); +void delivery_init_dirs_stage1(struct Delivery *ctx); + +void delivery_init_dirs_stage2(struct Delivery *ctx); + +int delivery_init_platform(struct Delivery *ctx); + int delivery_init_artifactory(struct Delivery *ctx); int delivery_artifact_upload(struct Delivery *ctx); @@ -389,10 +397,21 @@ int delivery_docker(struct Delivery *ctx); int delivery_fixup_test_results(struct Delivery *ctx); -int *bootstrap_build_info(struct Delivery *ctx); +int bootstrap_build_info(struct Delivery *ctx); int delivery_dump_metadata(struct Delivery *ctx); +int populate_info(struct Delivery *ctx); + +int populate_delivery_cfg(struct Delivery *ctx, int render_mode); + +int populate_delivery_ini(struct Delivery *ctx, int render_mode); + +int populate_mission_ini(struct Delivery **ctx, int render_mode); + +void validate_delivery_ini(struct INIFILE *ini); + +int filter_repo_tags(char *repo, struct StrList *patterns); /** * Determine whether a release on-disk matches the release name in use * @param ctx Delivery context -- cgit From fb4a7d1b212c9c6a51551e5043f6941a2371d075 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 26 Sep 2024 09:51:59 -0400 Subject: Remove usage of POSIX semaphores --- include/multiprocessing.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include') diff --git a/include/multiprocessing.h b/include/multiprocessing.h index 1974e41..f1cc815 100644 --- a/include/multiprocessing.h +++ b/include/multiprocessing.h @@ -11,7 +11,6 @@ #include struct MultiProcessingTask { - sem_t *gate; ///< Child process startup lock pid_t pid; ///< Program PID pid_t parent_pid; ///< Program PID (parent process) int status; ///< Child process exit status -- cgit From 71b45aa6ba60a651185cf47792fbf61b7e2b07a8 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 26 Sep 2024 09:54:52 -0400 Subject: Add cmd and working_dir members to MultiProcessingTask structure --- include/multiprocessing.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include') diff --git a/include/multiprocessing.h b/include/multiprocessing.h index f1cc815..aac6d18 100644 --- a/include/multiprocessing.h +++ b/include/multiprocessing.h @@ -16,6 +16,8 @@ struct MultiProcessingTask { int status; ///< Child process exit status int signaled_by; ///< Last signal received, if any char ident[255]; ///< Identity of the pool task + char *cmd; ///< Shell command(s) to be executed + char working_dir[PATH_MAX]; ///< Path to directory `cmd` should be executed in char log_file[PATH_MAX]; ///< Full path to stdout/stderr log file char parent_script[PATH_MAX]; ///< Path to temporary script executing the task struct { -- cgit From 3e610e935858995f411df7bb0a77e2efeeae3d66 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 26 Sep 2024 09:56:25 -0400 Subject: Implement cmd and working_dir --- include/multiprocessing.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include') diff --git a/include/multiprocessing.h b/include/multiprocessing.h index aac6d18..d4213d2 100644 --- a/include/multiprocessing.h +++ b/include/multiprocessing.h @@ -95,7 +95,7 @@ struct MultiProcessingPool *mp_pool_init(const char *ident, const char *log_root * @return pointer to MultiProcessingTask structure * @return NULL on error */ -struct MultiProcessingTask *mp_pool_task(struct MultiProcessingPool *pool, const char *ident, char *cmd); +struct MultiProcessingTask *mp_pool_task(struct MultiProcessingPool *pool, const char *ident, char *working_dir, char *cmd); /** * Execute all tasks in a pool -- cgit From 108242ce16fc7d7d9e81a1a6e9783fd9bda8b60c Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 27 Sep 2024 13:23:07 -0400 Subject: mp_pool_init(): return NULL when ident argument is NULL * reported by @kmacdonald-stsci --- include/multiprocessing.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include') diff --git a/include/multiprocessing.h b/include/multiprocessing.h index d4213d2..af8b90a 100644 --- a/include/multiprocessing.h +++ b/include/multiprocessing.h @@ -17,6 +17,7 @@ struct MultiProcessingTask { int signaled_by; ///< Last signal received, if any char ident[255]; ///< Identity of the pool task char *cmd; ///< Shell command(s) to be executed + size_t cmd_len; ///< Length of command string (for mmap/munmap) char working_dir[PATH_MAX]; ///< Path to directory `cmd` should be executed in char log_file[PATH_MAX]; ///< Full path to stdout/stderr log file char parent_script[PATH_MAX]; ///< Path to temporary script executing the task -- cgit From 31db9bb6e4434dc888fc724090d8f0d9d8eea619 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Mon, 30 Sep 2024 11:44:13 -0400 Subject: Fix typo * pararm -> param --- include/multiprocessing.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include') diff --git a/include/multiprocessing.h b/include/multiprocessing.h index af8b90a..c126999 100644 --- a/include/multiprocessing.h +++ b/include/multiprocessing.h @@ -113,7 +113,7 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags); /** * Show summary of pool tasks * - * @pararm pool a pointer to MultiProcessingPool + * @param pool a pointer to MultiProcessingPool */ void mp_pool_show_summary(struct MultiProcessingPool *pool); -- cgit From f6c504630ecfa38c1516123019bdf67b921f1107 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Wed, 2 Oct 2024 14:56:59 -0400 Subject: Allow user to define the time interval for "task is running" message --- include/core.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include') diff --git a/include/core.h b/include/core.h index 5011dea..ae0e8fe 100644 --- a/include/core.h +++ b/include/core.h @@ -65,6 +65,7 @@ struct STASIS_GLOBAL { bool enable_rewrite_spec_stage_2; //!< Enable automatic @STR@ replacement in output files long cpu_limit; long parallel_fail_fast; + int pool_status_interval; //!< Report "Task is running" every n seconds struct StrList *conda_packages; //!< Conda packages to install after initial activation struct StrList *pip_packages; //!< Pip packages to install after initial activation char *tmpdir; //!< Path to temporary storage directory -- cgit From 04cf9ee4f65c1d0b2b60b9ac87cd91c0a333889e Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Wed, 2 Oct 2024 14:57:48 -0400 Subject: Allow user to define the time interval for "task is running" message --- include/multiprocessing.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include') diff --git a/include/multiprocessing.h b/include/multiprocessing.h index c126999..5919462 100644 --- a/include/multiprocessing.h +++ b/include/multiprocessing.h @@ -15,6 +15,8 @@ struct MultiProcessingTask { pid_t parent_pid; ///< Program PID (parent process) int status; ///< Child process exit status int signaled_by; ///< Last signal received, if any + time_t _now; ///< Current time + time_t _seconds; ///< Time elapsed (used by MultiprocessingPool.status_interval) char ident[255]; ///< Identity of the pool task char *cmd; ///< Shell command(s) to be executed size_t cmd_len; ///< Length of command string (for mmap/munmap) @@ -33,6 +35,7 @@ struct MultiProcessingPool { size_t num_alloc; ///< Number of tasks allocated by the task array char ident[255]; ///< Identity of task pool char log_root[PATH_MAX]; ///< Base directory to store stderr/stdout log files + int status_interval; ///< Report a pooled task is "running" every n seconds }; /// Maximum number of multiprocessing tasks STASIS can execute -- cgit From 6fe8c2572fbf73cee3936ab241fcbfbdd54fe633 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Wed, 2 Oct 2024 15:00:12 -0400 Subject: Allow user to disable parallel mode (shortcut for --cpu-limit=1) --- include/core.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/include/core.h b/include/core.h index ae0e8fe..e09b212 100644 --- a/include/core.h +++ b/include/core.h @@ -63,8 +63,9 @@ struct STASIS_GLOBAL { bool enable_testing; //!< Enable package testing bool enable_overwrite; //!< Enable release file clobbering bool enable_rewrite_spec_stage_2; //!< Enable automatic @STR@ replacement in output files - long cpu_limit; - long parallel_fail_fast; + bool enable_parallel; //!< Enable testing in parallel + long cpu_limit; //!< Limit parallel processing to n cores (default: max - 1) + long parallel_fail_fast; //!< Fail immediately on error int pool_status_interval; //!< Report "Task is running" every n seconds struct StrList *conda_packages; //!< Conda packages to install after initial activation struct StrList *pip_packages; //!< Pip packages to install after initial activation -- cgit