From 8f17199d16bcdb29516d34514f95d1a117f6bd26 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 13 Sep 2024 09:58:17 -0400 Subject: Implement multiprocessing pool(s) * Adds --cpu-limit and --parallel-fail-fast arguments * Adds disable, parallel, and setup_script keys to [test] blocks --- src/stasis_main.c | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) (limited to 'src/stasis_main.c') diff --git a/src/stasis_main.c b/src/stasis_main.c index 7ea465c..164a9ca 100644 --- a/src/stasis_main.c +++ b/src/stasis_main.c @@ -12,15 +12,18 @@ #define OPT_NO_TESTING 1004 #define OPT_OVERWRITE 1005 #define OPT_NO_REWRITE_SPEC_STAGE_2 1006 +#define OPT_PARALLEL_FAIL_FAST 1007 static struct option long_options[] = { {"help", no_argument, 0, 'h'}, {"version", no_argument, 0, 'V'}, {"continue-on-error", no_argument, 0, 'C'}, {"config", required_argument, 0, 'c'}, + {"cpu-limit", required_argument, 0, 'l'}, {"python", required_argument, 0, 'p'}, {"verbose", no_argument, 0, 'v'}, {"unbuffered", no_argument, 0, 'U'}, {"update-base", no_argument, 0, OPT_ALWAYS_UPDATE_BASE}, + {"parallel-fail-fast", no_argument, 0, OPT_PARALLEL_FAIL_FAST}, {"overwrite", no_argument, 0, OPT_OVERWRITE}, {"no-docker", no_argument, 0, OPT_NO_DOCKER}, {"no-artifactory", no_argument, 0, OPT_NO_ARTIFACTORY}, @@ -35,10 +38,12 @@ const char *long_options_help[] = { "Display program version", "Allow tests to fail", "Read configuration file", + "Number of processes to spawn concurrently (default: cpus - 1)", "Override version of Python in configuration", "Increase output verbosity", "Disable line buffering", "Update conda installation prior to STASIS environment creation", + "On test error, terminate all concurrent tasks", "Overwrite an existing release", "Do not build docker images", "Do not upload artifacts to Artifactory", @@ -201,6 +206,13 @@ static void check_requirements(struct Delivery *ctx) { } int main(int argc, char *argv[]) { + /* + extern int exmain(int argc, char *argv[]); + exmain(argc, argv); + printf("ending program\n"); + exit(0); + */ + struct Delivery ctx; struct Process proc = { .f_stdout = "", @@ -214,6 +226,10 @@ int main(int argc, char *argv[]) { char installer_url[PATH_MAX]; char python_override_version[STASIS_NAME_MAX]; int user_disabled_docker = false; + globals.cpu_limit = get_cpu_count(); + if (globals.cpu_limit > 1) { + globals.cpu_limit--; + } memset(env_name, 0, sizeof(env_name)); memset(env_name_testing, 0, sizeof(env_name_testing)); @@ -241,9 +257,18 @@ int main(int argc, char *argv[]) { case 'p': strcpy(python_override_version, optarg); break; + case 'l': + globals.cpu_limit = strtol(optarg, NULL, 10); + if (globals.cpu_limit < 1) { + globals.cpu_limit = 1; + } + break; case OPT_ALWAYS_UPDATE_BASE: globals.always_update_base_environment = true; break; + case OPT_PARALLEL_FAIL_FAST: + globals.parallel_fail_fast = true; + break; case 'U': setenv("PYTHONUNBUFFERED", "1", 1); fflush(stdout); @@ -327,7 +352,6 @@ int main(int argc, char *argv[]) { tpl_register("deploy.jfrog.repo", &globals.jfrog.repo); tpl_register("deploy.jfrog.url", &globals.jfrog.url); tpl_register("deploy.docker.registry", &ctx.deploy.docker.registry); - tpl_register("workaround.tox_posargs", &globals.workaround.tox_posargs); tpl_register("workaround.conda_reactivate", &globals.workaround.conda_reactivate); // Expose function(s) to the template engine @@ -336,6 +360,7 @@ int main(int argc, char *argv[]) { tpl_register_func("get_github_release_notes_auto", &get_github_release_notes_auto_tplfunc_entrypoint, 1, &ctx); tpl_register_func("junitxml_file", &get_junitxml_file_entrypoint, 1, &ctx); tpl_register_func("basetemp_dir", &get_basetemp_dir_entrypoint, 1, &ctx); + tpl_register_func("tox_run", &tox_run_entrypoint, 2, &ctx); // Set up PREFIX/etc directory information // The user may manipulate the base directory path with STASIS_SYSCONFDIR -- cgit From 6f7cf6e1094e54e593801cc452b65ae9c7b4824a Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Tue, 17 Sep 2024 10:56:41 -0400 Subject: Remove short circuit test code * Remove exmain() and dead comments from main() --- src/stasis_main.c | 7 ------- 1 file changed, 7 deletions(-) (limited to 'src/stasis_main.c') diff --git a/src/stasis_main.c b/src/stasis_main.c index 164a9ca..72b7cdd 100644 --- a/src/stasis_main.c +++ b/src/stasis_main.c @@ -206,13 +206,6 @@ static void check_requirements(struct Delivery *ctx) { } int main(int argc, char *argv[]) { - /* - extern int exmain(int argc, char *argv[]); - exmain(argc, argv); - printf("ending program\n"); - exit(0); - */ - struct Delivery ctx; struct Process proc = { .f_stdout = "", -- cgit From 1999612a69a7947fb1b6fc45705299fe6db650ba Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Fri, 20 Sep 2024 08:36:26 -0400 Subject: Refactor structure * Break delivery.c into smaller components --- src/stasis_main.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/stasis_main.c') diff --git a/src/stasis_main.c b/src/stasis_main.c index 72b7cdd..da050f2 100644 --- a/src/stasis_main.c +++ b/src/stasis_main.c @@ -441,9 +441,9 @@ int main(int argc, char *argv[]) { } msg(STASIS_MSG_L1, "Conda setup\n"); - delivery_get_installer_url(&ctx, installer_url); + delivery_get_conda_installer_url(&ctx, installer_url); msg(STASIS_MSG_L2, "Downloading: %s\n", installer_url); - if (delivery_get_installer(&ctx, installer_url)) { + if (delivery_get_conda_installer(&ctx, installer_url)) { msg(STASIS_MSG_ERROR, "download failed: %s\n", installer_url); exit(1); } -- cgit From f6c504630ecfa38c1516123019bdf67b921f1107 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Wed, 2 Oct 2024 14:56:59 -0400 Subject: Allow user to define the time interval for "task is running" message --- src/stasis_main.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'src/stasis_main.c') diff --git a/src/stasis_main.c b/src/stasis_main.c index da050f2..8f14e5f 100644 --- a/src/stasis_main.c +++ b/src/stasis_main.c @@ -13,12 +13,14 @@ #define OPT_OVERWRITE 1005 #define OPT_NO_REWRITE_SPEC_STAGE_2 1006 #define OPT_PARALLEL_FAIL_FAST 1007 +#define OPT_POOL_STATUS_INTERVAL 1009 static struct option long_options[] = { {"help", no_argument, 0, 'h'}, {"version", no_argument, 0, 'V'}, {"continue-on-error", no_argument, 0, 'C'}, {"config", required_argument, 0, 'c'}, {"cpu-limit", required_argument, 0, 'l'}, + {"pool-status-interval", required_argument, 0, OPT_POOL_STATUS_INTERVAL}, {"python", required_argument, 0, 'p'}, {"verbose", no_argument, 0, 'v'}, {"unbuffered", no_argument, 0, 'U'}, @@ -39,6 +41,7 @@ const char *long_options_help[] = { "Allow tests to fail", "Read configuration file", "Number of processes to spawn concurrently (default: cpus - 1)", + "Report task status every n seconds (default: 30)", "Override version of Python in configuration", "Increase output verbosity", "Disable line buffering", @@ -262,6 +265,16 @@ int main(int argc, char *argv[]) { case OPT_PARALLEL_FAIL_FAST: globals.parallel_fail_fast = true; break; + case OPT_POOL_STATUS_INTERVAL: + globals.pool_status_interval = (int) strtol(optarg, NULL, 10); + if (globals.pool_status_interval < 1) { + globals.pool_status_interval = 1; + } else if (globals.pool_status_interval > 60 * 10) { + // Possible poor choice alert + fprintf(stderr, "Caution: Excessive pausing between status updates may cause third-party CI/CD" + " jobs to fail if the stdout/stderr streams are idle for too long!\n"); + } + break; case 'U': setenv("PYTHONUNBUFFERED", "1", 1); fflush(stdout); -- cgit From 6fe8c2572fbf73cee3936ab241fcbfbdd54fe633 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Wed, 2 Oct 2024 15:00:12 -0400 Subject: Allow user to disable parallel mode (shortcut for --cpu-limit=1) --- src/stasis_main.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'src/stasis_main.c') diff --git a/src/stasis_main.c b/src/stasis_main.c index 8f14e5f..2fcfcaf 100644 --- a/src/stasis_main.c +++ b/src/stasis_main.c @@ -13,7 +13,9 @@ #define OPT_OVERWRITE 1005 #define OPT_NO_REWRITE_SPEC_STAGE_2 1006 #define OPT_PARALLEL_FAIL_FAST 1007 +#define OPT_NO_PARALLEL 1008 #define OPT_POOL_STATUS_INTERVAL 1009 + static struct option long_options[] = { {"help", no_argument, 0, 'h'}, {"version", no_argument, 0, 'V'}, @@ -31,6 +33,7 @@ static struct option long_options[] = { {"no-artifactory", no_argument, 0, OPT_NO_ARTIFACTORY}, {"no-artifactory-build-info", no_argument, 0, OPT_NO_ARTIFACTORY_BUILD_INFO}, {"no-testing", no_argument, 0, OPT_NO_TESTING}, + {"no-parallel", no_argument, 0, OPT_NO_PARALLEL}, {"no-rewrite", no_argument, 0, OPT_NO_REWRITE_SPEC_STAGE_2}, {0, 0, 0, 0}, }; @@ -52,6 +55,7 @@ const char *long_options_help[] = { "Do not upload artifacts to Artifactory", "Do not upload build info objects to Artifactory", "Do not execute test scripts", + "Do not execute tests in parallel", "Do not rewrite paths and URLs in output files", NULL, }; @@ -224,7 +228,7 @@ int main(int argc, char *argv[]) { int user_disabled_docker = false; globals.cpu_limit = get_cpu_count(); if (globals.cpu_limit > 1) { - globals.cpu_limit--; + globals.cpu_limit--; // max - 1 } memset(env_name, 0, sizeof(env_name)); @@ -255,8 +259,9 @@ int main(int argc, char *argv[]) { break; case 'l': globals.cpu_limit = strtol(optarg, NULL, 10); - if (globals.cpu_limit < 1) { + if (globals.cpu_limit <= 1) { globals.cpu_limit = 1; + globals.enable_parallel = false; // No point } break; case OPT_ALWAYS_UPDATE_BASE: @@ -304,6 +309,9 @@ int main(int argc, char *argv[]) { case OPT_NO_REWRITE_SPEC_STAGE_2: globals.enable_rewrite_spec_stage_2 = false; break; + case OPT_NO_PARALLEL: + globals.enable_parallel = false; + break; case '?': default: exit(1); -- cgit From b041e8216c05940f5b3676ef5c7e0ce2b4441bc8 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Wed, 2 Oct 2024 15:00:57 -0400 Subject: "Task started" is more accurate than "queued" when this is printed --- src/stasis_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/stasis_main.c') diff --git a/src/stasis_main.c b/src/stasis_main.c index 2fcfcaf..47579a9 100644 --- a/src/stasis_main.c +++ b/src/stasis_main.c @@ -12,7 +12,7 @@ #define OPT_NO_TESTING 1004 #define OPT_OVERWRITE 1005 #define OPT_NO_REWRITE_SPEC_STAGE_2 1006 -#define OPT_PARALLEL_FAIL_FAST 1007 +#define OPT_FAIL_FAST 1007 #define OPT_NO_PARALLEL 1008 #define OPT_POOL_STATUS_INTERVAL 1009 -- cgit From f0ba8cd378a460f927644e41f49be95d0e956f81 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Wed, 2 Oct 2024 15:01:54 -0400 Subject: Rename argument --parallel-fail-fast to --fail-fast * All tasks are executed by the same machinery under the hood. So have them all react the same way. --- src/stasis_main.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/stasis_main.c') diff --git a/src/stasis_main.c b/src/stasis_main.c index 47579a9..b5d43e3 100644 --- a/src/stasis_main.c +++ b/src/stasis_main.c @@ -27,7 +27,7 @@ static struct option long_options[] = { {"verbose", no_argument, 0, 'v'}, {"unbuffered", no_argument, 0, 'U'}, {"update-base", no_argument, 0, OPT_ALWAYS_UPDATE_BASE}, - {"parallel-fail-fast", no_argument, 0, OPT_PARALLEL_FAIL_FAST}, + {"fail-fast", no_argument, 0, OPT_FAIL_FAST}, {"overwrite", no_argument, 0, OPT_OVERWRITE}, {"no-docker", no_argument, 0, OPT_NO_DOCKER}, {"no-artifactory", no_argument, 0, OPT_NO_ARTIFACTORY}, @@ -49,7 +49,7 @@ const char *long_options_help[] = { "Increase output verbosity", "Disable line buffering", "Update conda installation prior to STASIS environment creation", - "On test error, terminate all concurrent tasks", + "On error, immediately terminate all tasks", "Overwrite an existing release", "Do not build docker images", "Do not upload artifacts to Artifactory", @@ -267,7 +267,7 @@ int main(int argc, char *argv[]) { case OPT_ALWAYS_UPDATE_BASE: globals.always_update_base_environment = true; break; - case OPT_PARALLEL_FAIL_FAST: + case OPT_FAIL_FAST: globals.parallel_fail_fast = true; break; case OPT_POOL_STATUS_INTERVAL: -- cgit