From b7251ce3bf65bcbec7ecbb98a0eb0b3c9abde507 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 13 Sep 2024 09:35:34 -0400 Subject: Move guard_ macros to core_mem.h * Move core_mem.h below config.h --- include/multiprocessing.h | 120 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 include/multiprocessing.h (limited to 'include/multiprocessing.h') diff --git a/include/multiprocessing.h b/include/multiprocessing.h new file mode 100644 index 0000000..6bcf18e --- /dev/null +++ b/include/multiprocessing.h @@ -0,0 +1,120 @@ +/// @file multiprocessing.h +#ifndef STASIS_MULTIPROCESSING_H +#define STASIS_MULTIPROCESSING_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct MultiProcessingTask { + sem_t *gate; ///< Child process startup lock + pid_t pid; ///< Program PID + pid_t parent_pid; ///< Program PID (parent process) + int status; ///< Child process exit status + char ident[NAME_MAX]; ///< Identity of the pool task + char log_file[NAME_MAX]; ///< Path to stdout/stderr log file + char parent_script[PATH_MAX]; ///< Path to temporary script executing the task +}; + +struct MultiProcessingPool { + struct MultiProcessingTask *task; ///< Array of tasks to execute + size_t num_used; ///< Number of tasks populated in the task array + size_t num_alloc; ///< Number of tasks allocated by the task array + const char *ident; ///< Identity of task pool + const char *log_root; ///< Base directory to store stderr/stdout log files +}; + +///!< Maximum number of multiprocessing tasks STASIS can execute +#define MP_POOL_TASK_MAX 1000 + +///!< Value signifies a process is unused or finished executing +#define MP_POOL_PID_UNUSED 0 + +// Option flags for mp_pool_join() +#define MP_POOL_FAIL_FAST 1 << 1 + +/** + * Create a multiprocessing pool + * + * ```c + * #include "multiprocessing.h" + * #include "utils.h" // for get_cpu_count() + * + * int main(int argc, char *argv[]) { + * struct MultiProcessingPool *mp; + * mp = mp_pool_init("mypool", "/tmp/mypool_logs"); + * if (mp) { + * char *commands[] = { + * "/bin/echo hello world", + * "/bin/echo world hello", + * NULL + * } + * for (size_t i = 0; commands[i] != NULL); i++) { + * struct MultiProcessingTask *task; + * char task_name[100]; + * + * sprintf(task_name, "mytask%zu", i); + * task = mp_task(mp, task_name, commands[i]); + * if (!task) { + * // handle task creation error + * } + * } + * if (mp_pool_join(mp, get_cpu_count())) { + * // handle pool execution error + * } + * mp_pool_free(&mp); + * } else { + * // handle pool initialization error + * } + * } + * ``` + * + * @param ident a name to identify the pool + * @param log_root the path to store program output + * @return pointer to initialized MultiProcessingPool + * @return NULL on error + */ +struct MultiProcessingPool *mp_pool_init(const char *ident, const char *log_root); + +/** + * Create a multiprocessing pool task + * + * @param pool a pointer to MultiProcessingPool + * @param ident a name to identify the task + * @param cmd a command to execute + * @return pointer to MultiProcessingTask structure + * @return NULL on error + */ +struct MultiProcessingTask *mp_task(struct MultiProcessingPool *pool, const char *ident, char *cmd); + +/** + * Execute all tasks in a pool + * + * @param pool a pointer to MultiProcessingPool + * @param jobs the number of processes to spawn at once (for serial execution use `1`) + * @param flags option to be OR'd (MP_POOL_FAIL_FAST) + * @return 0 on success + * @return >0 on failure + * @return <0 on error + */ +int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags); + +/** + * Release resources allocated by mp_pool_init() + * + * @param a pointer to MultiProcessingPool + */ +void mp_pool_free(struct MultiProcessingPool **pool); + + +#endif //STASIS_MULTIPROCESSING_H -- cgit From 87971633eea3306a0b85a55d7d581841b9dbf905 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 13 Sep 2024 10:10:21 -0400 Subject: Fixing headers --- include/multiprocessing.h | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) (limited to 'include/multiprocessing.h') diff --git a/include/multiprocessing.h b/include/multiprocessing.h index 6bcf18e..9e0c429 100644 --- a/include/multiprocessing.h +++ b/include/multiprocessing.h @@ -2,27 +2,20 @@ #ifndef STASIS_MULTIPROCESSING_H #define STASIS_MULTIPROCESSING_H -#include -#include +#include "core.h" #include -#include -#include -#include #include #include -#include #include -#include #include -#include struct MultiProcessingTask { sem_t *gate; ///< Child process startup lock pid_t pid; ///< Program PID pid_t parent_pid; ///< Program PID (parent process) int status; ///< Child process exit status - char ident[NAME_MAX]; ///< Identity of the pool task - char log_file[NAME_MAX]; ///< Path to stdout/stderr log file + char ident[255]; ///< Identity of the pool task + char log_file[255]; ///< Path to stdout/stderr log file char parent_script[PATH_MAX]; ///< Path to temporary script executing the task }; -- cgit From 1fe385d782ae117d2a68266e14777d890eddf4e0 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 13 Sep 2024 12:16:41 -0400 Subject: Darwin portability: Use sem_open and sem_close instead of sem_init and sem_destroy --- include/multiprocessing.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/multiprocessing.h') diff --git a/include/multiprocessing.h b/include/multiprocessing.h index 9e0c429..648bd80 100644 --- a/include/multiprocessing.h +++ b/include/multiprocessing.h @@ -3,6 +3,7 @@ #define STASIS_MULTIPROCESSING_H #include "core.h" +#include #include #include #include -- cgit From 17d3d0517123f5f07b4ac6bb9f1dec73c1c8ce4c Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 13 Sep 2024 13:36:09 -0400 Subject: Fix sem_open initial state * Move slot->gate assignment to mp_pool_task() * Remove mmap() to slot->gate. * Change type of ident and log_root variables for the sake of easy (fewer maps) --- include/multiprocessing.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/multiprocessing.h') diff --git a/include/multiprocessing.h b/include/multiprocessing.h index 648bd80..f0e6e5c 100644 --- a/include/multiprocessing.h +++ b/include/multiprocessing.h @@ -24,8 +24,8 @@ struct MultiProcessingPool { struct MultiProcessingTask *task; ///< Array of tasks to execute size_t num_used; ///< Number of tasks populated in the task array size_t num_alloc; ///< Number of tasks allocated by the task array - const char *ident; ///< Identity of task pool - const char *log_root; ///< Base directory to store stderr/stdout log files + char ident[255]; ///< Identity of task pool + char log_root[PATH_MAX]; ///< Base directory to store stderr/stdout log files }; ///!< Maximum number of multiprocessing tasks STASIS can execute -- cgit From 8573ad716839caf34a2c9c016d5ffbcbaab5ba2e Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 13 Sep 2024 14:44:09 -0400 Subject: Fix mp_pool_join example --- include/multiprocessing.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/multiprocessing.h') diff --git a/include/multiprocessing.h b/include/multiprocessing.h index f0e6e5c..e4ddfdf 100644 --- a/include/multiprocessing.h +++ b/include/multiprocessing.h @@ -63,7 +63,7 @@ struct MultiProcessingPool { * // handle task creation error * } * } - * if (mp_pool_join(mp, get_cpu_count())) { + * if (mp_pool_join(mp, get_cpu_count(), MP_POOL_FAIL_FAST)) { * // handle pool execution error * } * mp_pool_free(&mp); -- cgit From 1e320e21492d3c09bae269b10d8faf0d0d131bb9 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 13 Sep 2024 15:02:00 -0400 Subject: Fix doxygen comments --- include/multiprocessing.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'include/multiprocessing.h') diff --git a/include/multiprocessing.h b/include/multiprocessing.h index e4ddfdf..4e89722 100644 --- a/include/multiprocessing.h +++ b/include/multiprocessing.h @@ -28,13 +28,13 @@ struct MultiProcessingPool { char log_root[PATH_MAX]; ///< Base directory to store stderr/stdout log files }; -///!< Maximum number of multiprocessing tasks STASIS can execute +/// Maximum number of multiprocessing tasks STASIS can execute #define MP_POOL_TASK_MAX 1000 -///!< Value signifies a process is unused or finished executing +/// Value signifies a process is unused or finished executing #define MP_POOL_PID_UNUSED 0 -// Option flags for mp_pool_join() +/// Option flags for mp_pool_join() #define MP_POOL_FAIL_FAST 1 << 1 /** -- cgit From db1a3056296ea3ed13c5a425cf1f11602b43a6c7 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 17 Sep 2024 09:37:14 -0400 Subject: Add pool summary and elapsed time output * Add get_task_duration() * Add get_pool_show_summary() * Add signaled_by member to MultiProcessingTask * Add time_data member to MultiProcessingTask for duration tracking --- include/multiprocessing.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'include/multiprocessing.h') diff --git a/include/multiprocessing.h b/include/multiprocessing.h index 4e89722..2356403 100644 --- a/include/multiprocessing.h +++ b/include/multiprocessing.h @@ -15,9 +15,14 @@ struct MultiProcessingTask { pid_t pid; ///< Program PID pid_t parent_pid; ///< Program PID (parent process) int status; ///< Child process exit status + int signaled_by; ///< Last signal received, if any char ident[255]; ///< Identity of the pool task char log_file[255]; ///< Path to stdout/stderr log file char parent_script[PATH_MAX]; ///< Path to temporary script executing the task + struct { + struct timespec t_start; + struct timespec t_stop; + } time_data; ///< Wall-time counters }; struct MultiProcessingPool { @@ -103,6 +108,13 @@ struct MultiProcessingTask *mp_task(struct MultiProcessingPool *pool, const char */ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags); +/** + * Show summary of pool tasks + * + * @pararm pool a pointer to MultiProcessingPool + */ +void mp_pool_show_summary(struct MultiProcessingPool *pool); + /** * Release resources allocated by mp_pool_init() * -- cgit From 8b47235f7c81e04fa5efef492974509789f40273 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Wed, 18 Sep 2024 10:04:21 -0400 Subject: Rename mp_task to mp_pool_task --- include/multiprocessing.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/multiprocessing.h') diff --git a/include/multiprocessing.h b/include/multiprocessing.h index 2356403..2db33df 100644 --- a/include/multiprocessing.h +++ b/include/multiprocessing.h @@ -94,7 +94,7 @@ struct MultiProcessingPool *mp_pool_init(const char *ident, const char *log_root * @return pointer to MultiProcessingTask structure * @return NULL on error */ -struct MultiProcessingTask *mp_task(struct MultiProcessingPool *pool, const char *ident, char *cmd); +struct MultiProcessingTask *mp_pool_task(struct MultiProcessingPool *pool, const char *ident, char *cmd); /** * Execute all tasks in a pool -- cgit From c8c0802c98db5e3d6de6b79887133e0d15567fd8 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Wed, 18 Sep 2024 10:44:13 -0400 Subject: Fix a likely buffer overflow * The log_root is appended to log_file, and might exceed the defined length. --- include/multiprocessing.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/multiprocessing.h') diff --git a/include/multiprocessing.h b/include/multiprocessing.h index 2db33df..1974e41 100644 --- a/include/multiprocessing.h +++ b/include/multiprocessing.h @@ -17,7 +17,7 @@ struct MultiProcessingTask { int status; ///< Child process exit status int signaled_by; ///< Last signal received, if any char ident[255]; ///< Identity of the pool task - char log_file[255]; ///< Path to stdout/stderr log file + char log_file[PATH_MAX]; ///< Full path to stdout/stderr log file char parent_script[PATH_MAX]; ///< Path to temporary script executing the task struct { struct timespec t_start; -- cgit From fb4a7d1b212c9c6a51551e5043f6941a2371d075 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 26 Sep 2024 09:51:59 -0400 Subject: Remove usage of POSIX semaphores --- include/multiprocessing.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/multiprocessing.h') diff --git a/include/multiprocessing.h b/include/multiprocessing.h index 1974e41..f1cc815 100644 --- a/include/multiprocessing.h +++ b/include/multiprocessing.h @@ -11,7 +11,6 @@ #include struct MultiProcessingTask { - sem_t *gate; ///< Child process startup lock pid_t pid; ///< Program PID pid_t parent_pid; ///< Program PID (parent process) int status; ///< Child process exit status -- cgit From 71b45aa6ba60a651185cf47792fbf61b7e2b07a8 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 26 Sep 2024 09:54:52 -0400 Subject: Add cmd and working_dir members to MultiProcessingTask structure --- include/multiprocessing.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/multiprocessing.h') diff --git a/include/multiprocessing.h b/include/multiprocessing.h index f1cc815..aac6d18 100644 --- a/include/multiprocessing.h +++ b/include/multiprocessing.h @@ -16,6 +16,8 @@ struct MultiProcessingTask { int status; ///< Child process exit status int signaled_by; ///< Last signal received, if any char ident[255]; ///< Identity of the pool task + char *cmd; ///< Shell command(s) to be executed + char working_dir[PATH_MAX]; ///< Path to directory `cmd` should be executed in char log_file[PATH_MAX]; ///< Full path to stdout/stderr log file char parent_script[PATH_MAX]; ///< Path to temporary script executing the task struct { -- cgit From 3e610e935858995f411df7bb0a77e2efeeae3d66 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 26 Sep 2024 09:56:25 -0400 Subject: Implement cmd and working_dir --- include/multiprocessing.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/multiprocessing.h') diff --git a/include/multiprocessing.h b/include/multiprocessing.h index aac6d18..d4213d2 100644 --- a/include/multiprocessing.h +++ b/include/multiprocessing.h @@ -95,7 +95,7 @@ struct MultiProcessingPool *mp_pool_init(const char *ident, const char *log_root * @return pointer to MultiProcessingTask structure * @return NULL on error */ -struct MultiProcessingTask *mp_pool_task(struct MultiProcessingPool *pool, const char *ident, char *cmd); +struct MultiProcessingTask *mp_pool_task(struct MultiProcessingPool *pool, const char *ident, char *working_dir, char *cmd); /** * Execute all tasks in a pool -- cgit From 108242ce16fc7d7d9e81a1a6e9783fd9bda8b60c Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 27 Sep 2024 13:23:07 -0400 Subject: mp_pool_init(): return NULL when ident argument is NULL * reported by @kmacdonald-stsci --- include/multiprocessing.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/multiprocessing.h') diff --git a/include/multiprocessing.h b/include/multiprocessing.h index d4213d2..af8b90a 100644 --- a/include/multiprocessing.h +++ b/include/multiprocessing.h @@ -17,6 +17,7 @@ struct MultiProcessingTask { int signaled_by; ///< Last signal received, if any char ident[255]; ///< Identity of the pool task char *cmd; ///< Shell command(s) to be executed + size_t cmd_len; ///< Length of command string (for mmap/munmap) char working_dir[PATH_MAX]; ///< Path to directory `cmd` should be executed in char log_file[PATH_MAX]; ///< Full path to stdout/stderr log file char parent_script[PATH_MAX]; ///< Path to temporary script executing the task -- cgit From 31db9bb6e4434dc888fc724090d8f0d9d8eea619 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Mon, 30 Sep 2024 11:44:13 -0400 Subject: Fix typo * pararm -> param --- include/multiprocessing.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/multiprocessing.h') diff --git a/include/multiprocessing.h b/include/multiprocessing.h index af8b90a..c126999 100644 --- a/include/multiprocessing.h +++ b/include/multiprocessing.h @@ -113,7 +113,7 @@ int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags); /** * Show summary of pool tasks * - * @pararm pool a pointer to MultiProcessingPool + * @param pool a pointer to MultiProcessingPool */ void mp_pool_show_summary(struct MultiProcessingPool *pool); -- cgit From 04cf9ee4f65c1d0b2b60b9ac87cd91c0a333889e Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Wed, 2 Oct 2024 14:57:48 -0400 Subject: Allow user to define the time interval for "task is running" message --- include/multiprocessing.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/multiprocessing.h') diff --git a/include/multiprocessing.h b/include/multiprocessing.h index c126999..5919462 100644 --- a/include/multiprocessing.h +++ b/include/multiprocessing.h @@ -15,6 +15,8 @@ struct MultiProcessingTask { pid_t parent_pid; ///< Program PID (parent process) int status; ///< Child process exit status int signaled_by; ///< Last signal received, if any + time_t _now; ///< Current time + time_t _seconds; ///< Time elapsed (used by MultiprocessingPool.status_interval) char ident[255]; ///< Identity of the pool task char *cmd; ///< Shell command(s) to be executed size_t cmd_len; ///< Length of command string (for mmap/munmap) @@ -33,6 +35,7 @@ struct MultiProcessingPool { size_t num_alloc; ///< Number of tasks allocated by the task array char ident[255]; ///< Identity of task pool char log_root[PATH_MAX]; ///< Base directory to store stderr/stdout log files + int status_interval; ///< Report a pooled task is "running" every n seconds }; /// Maximum number of multiprocessing tasks STASIS can execute -- cgit