diff options
Diffstat (limited to 'include')
| -rw-r--r-- | include/artifactory.h | 1 | ||||
| -rw-r--r-- | include/conda.h | 16 | ||||
| -rw-r--r-- | include/core.h | 38 | ||||
| -rw-r--r-- | include/core_mem.h | 18 | ||||
| -rw-r--r-- | include/delivery.h | 39 | ||||
| -rw-r--r-- | include/docker.h | 2 | ||||
| -rw-r--r-- | include/download.h | 2 | ||||
| -rw-r--r-- | include/envctl.h | 2 | ||||
| -rw-r--r-- | include/github.h | 1 | ||||
| -rw-r--r-- | include/ini.h | 1 | ||||
| -rw-r--r-- | include/multiprocessing.h | 131 | ||||
| -rw-r--r-- | include/package.h | 30 | ||||
| -rw-r--r-- | include/str.h | 1 | ||||
| -rw-r--r-- | include/strlist.h | 5 | ||||
| -rw-r--r-- | include/template_func_proto.h | 2 | ||||
| -rw-r--r-- | include/utils.h | 31 | ||||
| -rw-r--r-- | include/wheel.h | 36 |
17 files changed, 305 insertions, 51 deletions
diff --git a/include/artifactory.h b/include/artifactory.h index c6e5c2b..e580886 100644 --- a/include/artifactory.h +++ b/include/artifactory.h @@ -5,6 +5,7 @@ #include <stdio.h> #include <stdlib.h> #include "core.h" +#include "download.h" //! JFrog Artifactory Authentication struct struct JFRT_Auth { diff --git a/include/conda.h b/include/conda.h index c546672..1eb42f4 100644 --- a/include/conda.h +++ b/include/conda.h @@ -4,7 +4,9 @@ #include <stdio.h> #include <string.h> +#include <sys/utsname.h> #include "core.h" +#include "download.h" #define CONDA_INSTALL_PREFIX "conda" #define PYPI_INDEX_DEFAULT "https://pypi.org/simple" @@ -186,10 +188,18 @@ int conda_index(const char *path); /** * Determine whether a simple index contains a package * @param index_url a file system path or url pointing to a simple index - * @param name package name (required) - * @param version package version (may be NULL) + * @param spec a pip package specification (e.g. `name==1.2.3`) * @return not found = 0, found = 1, error = -1 */ -int pip_index_provides(const char *index_url, const char *name, const char *version); +int pip_index_provides(const char *index_url, const char *spec); + +/** + * Determine whether conda can find a package in its channel list + * @param spec a conda package specification (e.g. `name=1.2.3`) + * @return not found = 0, found = 1, error = -1 + */ +int conda_provides(const char *spec); + +char *conda_get_active_environment(); #endif //STASIS_CONDA_H diff --git a/include/core.h b/include/core.h index ef90e96..b0a1a11 100644 --- a/include/core.h +++ b/include/core.h @@ -1,9 +1,10 @@ -//! @file stasis.h +//! @file core.h #ifndef STASIS_CORE_H #define STASIS_CORE_H #include <stdio.h> #include <stdlib.h> +#include <stdbool.h> #include <string.h> #include <limits.h> #include <unistd.h> @@ -21,36 +22,7 @@ #define HTTP_ERROR(X) X >= 400 #include "config.h" -#include "envctl.h" -#include "template.h" -#include "utils.h" -#include "copy.h" -#include "ini.h" -#include "conda.h" -#include "environment.h" -#include "artifactory.h" -#include "docker.h" -#include "delivery.h" -#include "str.h" -#include "strlist.h" -#include "system.h" -#include "download.h" -#include "recipe.h" -#include "relocation.h" -#include "wheel.h" -#include "junitxml.h" -#include "github.h" -#include "template_func_proto.h" - -#define guard_runtime_free(X) do { if (X) { runtime_free(X); X = NULL; } } while (0) -#define guard_strlist_free(X) do { if ((*X)) { strlist_free(X); (*X) = NULL; } } while (0) -#define guard_free(X) do { if (X) { free(X); X = NULL; } } while (0) -#define GENERIC_ARRAY_FREE(ARR) do { \ - for (size_t ARR_I = 0; ARR && ARR[ARR_I] != NULL; ARR_I++) { \ - guard_free(ARR[ARR_I]); \ - } \ - guard_free(ARR); \ -} while (0) +#include "core_mem.h" #define COE_CHECK_ABORT(COND, MSG) \ do {\ @@ -71,6 +43,10 @@ struct STASIS_GLOBAL { bool enable_testing; //!< Enable package testing bool enable_overwrite; //!< Enable release file clobbering bool enable_rewrite_spec_stage_2; //!< Enable automatic @STR@ replacement in output files + bool enable_parallel; //!< Enable testing in parallel + long cpu_limit; //!< Limit parallel processing to n cores (default: max - 1) + long parallel_fail_fast; //!< Fail immediately on error + int pool_status_interval; //!< Report "Task is running" every n seconds struct StrList *conda_packages; //!< Conda packages to install after initial activation struct StrList *pip_packages; //!< Pip packages to install after initial activation char *tmpdir; //!< Path to temporary storage directory diff --git a/include/core_mem.h b/include/core_mem.h new file mode 100644 index 0000000..bd50e9d --- /dev/null +++ b/include/core_mem.h @@ -0,0 +1,18 @@ +//! @file core_mem.h +#ifndef STASIS_CORE_MEM_H +#define STASIS_CORE_MEM_H + +#include "environment.h" +#include "strlist.h" + +#define guard_runtime_free(X) do { if (X) { runtime_free(X); X = NULL; } } while (0) +#define guard_strlist_free(X) do { if ((*X)) { strlist_free(X); (*X) = NULL; } } while (0) +#define guard_free(X) do { if (X) { free(X); X = NULL; } } while (0) +#define GENERIC_ARRAY_FREE(ARR) do { \ + for (size_t ARR_I = 0; ARR && ARR[ARR_I] != NULL; ARR_I++) { \ + guard_free(ARR[ARR_I]); \ + } \ + guard_free(ARR); \ +} while (0) + +#endif //STASIS_CORE_MEM_H diff --git a/include/delivery.h b/include/delivery.h index 067cd0b..bd5137c 100644 --- a/include/delivery.h +++ b/include/delivery.h @@ -7,7 +7,18 @@ #include <stdbool.h> #include <unistd.h> #include <sys/utsname.h> +#include <fnmatch.h> +#include <sys/statvfs.h> #include "core.h" +#include "copy.h" +#include "environment.h" +#include "conda.h" +#include "ini.h" +#include "artifactory.h" +#include "docker.h" +#include "wheel.h" +#include "multiprocessing.h" +#include "recipe.h" #define DELIVERY_PLATFORM_MAX 4 #define DELIVERY_PLATFORM_MAXLEN 65 @@ -149,7 +160,10 @@ struct Delivery { char *name; ///< Name of package char *version; ///< Version of package char *repository; ///< Git repository of package + char *script_setup; ///< Commands to execute before the main script char *script; ///< Commands to execute + bool disable; ///< Toggle a test block + bool parallel; ///< Toggle parallel or serial execution char *build_recipe; ///< Conda recipe to build (optional) char *repository_info_ref; ///< Git commit hash char *repository_info_tag; ///< Git tag (first parent) @@ -286,7 +300,7 @@ int delivery_copy_conda_artifacts(struct Delivery *ctx); * Retrieve Conda installer * @param installer_url URL to installation script */ -int delivery_get_installer(struct Delivery *ctx, char *installer_url); +int delivery_get_conda_installer(struct Delivery *ctx, char *installer_url); /** * Generate URL based on Delivery context @@ -294,7 +308,7 @@ int delivery_get_installer(struct Delivery *ctx, char *installer_url); * @param result pointer to char * @return in result */ -void delivery_get_installer_url(struct Delivery *ctx, char *result); +void delivery_get_conda_installer_url(struct Delivery *ctx, char *result); /** * Install packages based on Delivery context @@ -376,6 +390,12 @@ void delivery_gather_tool_versions(struct Delivery *ctx); // helper function int delivery_init_tmpdir(struct Delivery *ctx); +void delivery_init_dirs_stage1(struct Delivery *ctx); + +void delivery_init_dirs_stage2(struct Delivery *ctx); + +int delivery_init_platform(struct Delivery *ctx); + int delivery_init_artifactory(struct Delivery *ctx); int delivery_artifact_upload(struct Delivery *ctx); @@ -386,10 +406,21 @@ int delivery_docker(struct Delivery *ctx); int delivery_fixup_test_results(struct Delivery *ctx); -int *bootstrap_build_info(struct Delivery *ctx); +int bootstrap_build_info(struct Delivery *ctx); int delivery_dump_metadata(struct Delivery *ctx); +int populate_info(struct Delivery *ctx); + +int populate_delivery_cfg(struct Delivery *ctx, int render_mode); + +int populate_delivery_ini(struct Delivery *ctx, int render_mode); + +int populate_mission_ini(struct Delivery **ctx, int render_mode); + +void validate_delivery_ini(struct INIFILE *ini); + +int filter_repo_tags(char *repo, struct StrList *patterns); /** * Determine whether a release on-disk matches the release name in use * @param ctx Delivery context @@ -397,4 +428,6 @@ int delivery_dump_metadata(struct Delivery *ctx); */ int delivery_exists(struct Delivery *ctx); +int delivery_overlay_packages_from_env(struct Delivery *ctx, const char *env_name); + #endif //STASIS_DELIVERY_H diff --git a/include/docker.h b/include/docker.h index ff8a8d5..7585d86 100644 --- a/include/docker.h +++ b/include/docker.h @@ -2,6 +2,8 @@ #ifndef STASIS_DOCKER_H #define STASIS_DOCKER_H +#include "core.h" + //! Flag to squelch output from docker_exec() #define STASIS_DOCKER_QUIET 1 << 1 diff --git a/include/download.h b/include/download.h index 058812e..0b6311e 100644 --- a/include/download.h +++ b/include/download.h @@ -2,6 +2,8 @@ #ifndef STASIS_DOWNLOAD_H #define STASIS_DOWNLOAD_H +#include <stdlib.h> +#include <string.h> #include <curl/curl.h> size_t download_writer(void *fp, size_t size, size_t nmemb, void *stream); diff --git a/include/envctl.h b/include/envctl.h index c8ef357..659cae3 100644 --- a/include/envctl.h +++ b/include/envctl.h @@ -1,7 +1,9 @@ +//! @file envctl.h #ifndef STASIS_ENVCTL_H #define STASIS_ENVCTL_H #include <stdlib.h> +#include "core.h" #define STASIS_ENVCTL_PASSTHRU 0 #define STASIS_ENVCTL_REQUIRED 1 << 1 diff --git a/include/github.h b/include/github.h index cebeabf..f9b47a3 100644 --- a/include/github.h +++ b/include/github.h @@ -1,3 +1,4 @@ +//! @file github.h #ifndef STASIS_GITHUB_H #define STASIS_GITHUB_H diff --git a/include/ini.h b/include/ini.h index 3d0565b..557f157 100644 --- a/include/ini.h +++ b/include/ini.h @@ -5,6 +5,7 @@ #include <stdio.h> #include <stddef.h> #include <stdbool.h> +#include "template.h" #define INI_WRITE_RAW 0 ///< Dump INI data. Contents are not modified. #define INI_WRITE_PRESERVE 1 ///< Dump INI data. Template strings are diff --git a/include/multiprocessing.h b/include/multiprocessing.h new file mode 100644 index 0000000..5919462 --- /dev/null +++ b/include/multiprocessing.h @@ -0,0 +1,131 @@ +/// @file multiprocessing.h +#ifndef STASIS_MULTIPROCESSING_H +#define STASIS_MULTIPROCESSING_H + +#include "core.h" +#include <signal.h> +#include <sys/wait.h> +#include <semaphore.h> +#include <sys/mman.h> +#include <fcntl.h> +#include <sys/stat.h> + +struct MultiProcessingTask { + pid_t pid; ///< Program PID + pid_t parent_pid; ///< Program PID (parent process) + int status; ///< Child process exit status + int signaled_by; ///< Last signal received, if any + time_t _now; ///< Current time + time_t _seconds; ///< Time elapsed (used by MultiprocessingPool.status_interval) + char ident[255]; ///< Identity of the pool task + char *cmd; ///< Shell command(s) to be executed + size_t cmd_len; ///< Length of command string (for mmap/munmap) + char working_dir[PATH_MAX]; ///< Path to directory `cmd` should be executed in + char log_file[PATH_MAX]; ///< Full path to stdout/stderr log file + char parent_script[PATH_MAX]; ///< Path to temporary script executing the task + struct { + struct timespec t_start; + struct timespec t_stop; + } time_data; ///< Wall-time counters +}; + +struct MultiProcessingPool { + struct MultiProcessingTask *task; ///< Array of tasks to execute + size_t num_used; ///< Number of tasks populated in the task array + size_t num_alloc; ///< Number of tasks allocated by the task array + char ident[255]; ///< Identity of task pool + char log_root[PATH_MAX]; ///< Base directory to store stderr/stdout log files + int status_interval; ///< Report a pooled task is "running" every n seconds +}; + +/// Maximum number of multiprocessing tasks STASIS can execute +#define MP_POOL_TASK_MAX 1000 + +/// Value signifies a process is unused or finished executing +#define MP_POOL_PID_UNUSED 0 + +/// Option flags for mp_pool_join() +#define MP_POOL_FAIL_FAST 1 << 1 + +/** + * Create a multiprocessing pool + * + * ```c + * #include "multiprocessing.h" + * #include "utils.h" // for get_cpu_count() + * + * int main(int argc, char *argv[]) { + * struct MultiProcessingPool *mp; + * mp = mp_pool_init("mypool", "/tmp/mypool_logs"); + * if (mp) { + * char *commands[] = { + * "/bin/echo hello world", + * "/bin/echo world hello", + * NULL + * } + * for (size_t i = 0; commands[i] != NULL); i++) { + * struct MultiProcessingTask *task; + * char task_name[100]; + * + * sprintf(task_name, "mytask%zu", i); + * task = mp_task(mp, task_name, commands[i]); + * if (!task) { + * // handle task creation error + * } + * } + * if (mp_pool_join(mp, get_cpu_count(), MP_POOL_FAIL_FAST)) { + * // handle pool execution error + * } + * mp_pool_free(&mp); + * } else { + * // handle pool initialization error + * } + * } + * ``` + * + * @param ident a name to identify the pool + * @param log_root the path to store program output + * @return pointer to initialized MultiProcessingPool + * @return NULL on error + */ +struct MultiProcessingPool *mp_pool_init(const char *ident, const char *log_root); + +/** + * Create a multiprocessing pool task + * + * @param pool a pointer to MultiProcessingPool + * @param ident a name to identify the task + * @param cmd a command to execute + * @return pointer to MultiProcessingTask structure + * @return NULL on error + */ +struct MultiProcessingTask *mp_pool_task(struct MultiProcessingPool *pool, const char *ident, char *working_dir, char *cmd); + +/** + * Execute all tasks in a pool + * + * @param pool a pointer to MultiProcessingPool + * @param jobs the number of processes to spawn at once (for serial execution use `1`) + * @param flags option to be OR'd (MP_POOL_FAIL_FAST) + * @return 0 on success + * @return >0 on failure + * @return <0 on error + */ +int mp_pool_join(struct MultiProcessingPool *pool, size_t jobs, size_t flags); + +/** + * Show summary of pool tasks + * + * @param pool a pointer to MultiProcessingPool + */ +void mp_pool_show_summary(struct MultiProcessingPool *pool); + +/** + * Release resources allocated by mp_pool_init() + * + * @param a pointer to MultiProcessingPool + */ +void mp_pool_free(struct MultiProcessingPool **pool); + + +#endif //STASIS_MULTIPROCESSING_H diff --git a/include/package.h b/include/package.h new file mode 100644 index 0000000..eff1874 --- /dev/null +++ b/include/package.h @@ -0,0 +1,30 @@ +#ifndef STASIS_PACKAGE_H +#define STASIS_PACKAGE_H + +struct Package { + struct { + const char *name; + const char *version_spec; + const char *version; + } meta; + struct { + const char *uri; + unsigned handler; + } source; + struct { + struct Test *test; + size_t pass; + size_t fail; + size_t skip; + }; + unsigned state; +}; + +struct Package *stasis_package_init(void); +void stasis_package_set_name(struct Package *pkg, const char *name); +void stasis_package_set_version(struct Package *pkg, const char *version); +void stasis_package_set_version_spec(struct Package *pkg, const char *version_spec); +void stasis_package_set_uri(struct Package *pkg, const char *uri); +void stasis_package_set_handler(struct Package *pkg, unsigned handler); + +#endif //STASIS_PACKAGE_H diff --git a/include/str.h b/include/str.h index 4cf221d..7254225 100644 --- a/include/str.h +++ b/include/str.h @@ -9,6 +9,7 @@ #include <string.h> #include <stdarg.h> #include <ctype.h> +#include "relocation.h" #include "core.h" #define STASIS_SORT_ALPHA 1 << 0 diff --git a/include/strlist.h b/include/strlist.h index dd22a0a..cdbfc01 100644 --- a/include/strlist.h +++ b/include/strlist.h @@ -4,10 +4,15 @@ */ #ifndef STASIS_STRLIST_H #define STASIS_STRLIST_H + +typedef int (ReaderFn)(size_t line, char **); + #include <stdlib.h> +#include "core.h" #include "utils.h" #include "str.h" + struct StrList { size_t num_alloc; size_t num_inuse; diff --git a/include/template_func_proto.h b/include/template_func_proto.h index 7778a11..286ccfb 100644 --- a/include/template_func_proto.h +++ b/include/template_func_proto.h @@ -1,3 +1,4 @@ +//! @file template_func_proto.h #ifndef TEMPLATE_FUNC_PROTO_H #define TEMPLATE_FUNC_PROTO_H @@ -7,5 +8,6 @@ int get_github_release_notes_tplfunc_entrypoint(void *frame, void *data_out); int get_github_release_notes_auto_tplfunc_entrypoint(void *frame, void *data_out); int get_junitxml_file_entrypoint(void *frame, void *data_out); int get_basetemp_dir_entrypoint(void *frame, void *data_out); +int tox_run_entrypoint(void *frame, void *data_out); #endif //TEMPLATE_FUNC_PROTO_H
\ No newline at end of file diff --git a/include/utils.h b/include/utils.h index a3d244a..4ade817 100644 --- a/include/utils.h +++ b/include/utils.h @@ -8,7 +8,12 @@ #include <unistd.h> #include <limits.h> #include <errno.h> +#include "core.h" +#include "copy.h" #include "system.h" +#include "strlist.h" +#include "utils.h" +#include "ini.h" #if defined(STASIS_OS_WINDOWS) #define PATH_ENV_VAR "path" @@ -25,8 +30,6 @@ #define STASIS_XML_PRETTY_PRINT_PROG "xmllint" #define STASIS_XML_PRETTY_PRINT_ARGS "--format" -typedef int (ReaderFn)(size_t line, char **); - /** * Change directory. Push path on directory stack. * @@ -365,4 +368,28 @@ long get_cpu_count(); */ int mkdirs(const char *_path, mode_t mode); +/** + * Return pointer to a (possible) version specifier + * + * ```c + * char s[] = "abc==1.2.3"; + * char *spec_begin = find_version_spec(s); + * // spec_begin is "==1.2.3" + * + * char package_name[255]; + * char s[] = "abc"; + * char *spec_pos = find_version_spec(s); + * if (spec_pos) { + * strncpy(package_name, spec_pos - s); + * // use spec + * } else { + * // spec not found + * } + * + * @param str a pointer to a buffer containing a package spec (i.e. abc==1.2.3, abc>=1.2.3, abc) + * @return a pointer to the first occurrence of a version spec character + * @return NULL if not found + */ +char *find_version_spec(char *package_name); + #endif //STASIS_UTILS_H diff --git a/include/wheel.h b/include/wheel.h index 619e0f7..1a689e9 100644 --- a/include/wheel.h +++ b/include/wheel.h @@ -1,3 +1,4 @@ +//! @file wheel.h #ifndef STASIS_WHEEL_H #define STASIS_WHEEL_H @@ -5,20 +6,31 @@ #include <string.h> #include <stdio.h> #include "str.h" - -#define WHEEL_MATCH_EXACT 0 -#define WHEEL_MATCH_ANY 1 +#define WHEEL_MATCH_EXACT 0 ///< Match when all patterns are present +#define WHEEL_MATCH_ANY 1 ///< Match when any patterns are present struct Wheel { - char *distribution; - char *version; - char *build_tag; - char *python_tag; - char *abi_tag; - char *platform_tag; - char *path_name; - char *file_name; + char *distribution; ///< Package name + char *version; ///< Package version + char *build_tag; ///< Package build tag (optional) + char *python_tag; ///< Package Python tag (pyXY) + char *abi_tag; ///< Package ABI tag (cpXY, abiX, none) + char *platform_tag; ///< Package platform tag (linux_x86_64, any) + char *path_name; ///< Path to package on-disk + char *file_name; ///< Name of package on-disk }; -struct Wheel *get_wheel_file(const char *basepath, const char *name, char *to_match[], unsigned match_mode); +/** + * Extract metadata from a Python Wheel file name + * + * @param basepath directory containing a wheel file + * @param name of wheel file + * @param to_match a NULL terminated array of patterns (i.e. platform, arch, version, etc) + * @param match_mode WHEEL_MATCH_EXACT + * @param match_mode WHEEL_MATCH ANY + * @return pointer to populated Wheel on success + * @return NULL on error + */ +struct Wheel *get_wheel_info(const char *basepath, const char *name, char *to_match[], unsigned match_mode); +void wheel_free(struct Wheel **wheel); #endif //STASIS_WHEEL_H |