From 4a8008359f38db1b3d7acaf7013a66a5c5394922 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Wed, 11 Feb 2026 15:05:08 -0500 Subject: Add a retry loop to download * Configurable with env var: STASIS_DOWNLOAD_RETRIES --- README.md | 2 ++ src/lib/core/download.c | 86 ++++++++++++++++++++++++++++++++----------------- 2 files changed, 58 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index c1a490d..67ebf0d 100644 --- a/README.md +++ b/README.md @@ -200,6 +200,8 @@ stasis mydelivery.ini | STASIS_JF_CLIENT_CERT_CERT_PATH | Path to OpenSSL cert files | | STASIS_JF_CLIENT_CERT_KEY_PATH | OpenSSL key file (in cert path) | | STASIS_JF_REPO | Artifactory "generic" repository to write to | +| STASIS_DOWNLOAD_TIMEOUT | Number of seconds before timing out a remote file download | +| STASIS_DOWNLOAD_RETRIES | Number of retries before giving up on a remote file download | ## Main configuration (stasis.ini) diff --git a/src/lib/core/download.c b/src/lib/core/download.c index b021860..12921c6 100644 --- a/src/lib/core/download.c +++ b/src/lib/core/download.c @@ -14,44 +14,70 @@ long download(char *url, const char *filename, char **errmsg) { long http_code = -1; char user_agent[20]; sprintf(user_agent, "stasis/%s", VERSION); + long timeout = 30L; - char *timeout_str = getenv("STASIS_DOWNLOAD_TIMEOUT"); + const char *timeout_str = getenv("STASIS_DOWNLOAD_TIMEOUT"); + if (timeout_str) { + timeout = strtol(timeout_str, NULL, 10); + if (timeout <= 0L) { + timeout = 1L; + } + } + + size_t max_retries = 5; + const char *max_retries_str = getenv("STASIS_DOWNLOAD_RETRIES"); + if (max_retries_str) { + max_retries = strtol(timeout_str, NULL, 10); + if (max_retries <= 0) { + max_retries = 1; + } + } curl_global_init(CURL_GLOBAL_ALL); CURL *c = curl_easy_init(); - curl_easy_setopt(c, CURLOPT_URL, url); - curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, download_writer); - FILE *fp = fopen(filename, "wb"); - if (!fp) { - return -1; - } + for (size_t retry = 0; retry < max_retries; retry++) { + if (retry) { + fprintf(stderr, "\n[ATTEMPT %zu/%zu]\n", retry + 1, max_retries); + } + curl_easy_setopt(c, CURLOPT_URL, url); + curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, download_writer); + FILE *fp = fopen(filename, "wb"); + if (!fp) { + return -1; + } - curl_easy_setopt(c, CURLOPT_VERBOSE, 0L); - curl_easy_setopt(c, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(c, CURLOPT_USERAGENT, user_agent); - curl_easy_setopt(c, CURLOPT_NOPROGRESS, 0L); - curl_easy_setopt(c, CURLOPT_WRITEDATA, fp); + curl_easy_setopt(c, CURLOPT_VERBOSE, 0L); + curl_easy_setopt(c, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(c, CURLOPT_USERAGENT, user_agent); + curl_easy_setopt(c, CURLOPT_NOPROGRESS, 0L); + curl_easy_setopt(c, CURLOPT_WRITEDATA, fp); + curl_easy_setopt(c, CURLOPT_CONNECTTIMEOUT, timeout); - if (timeout_str) { - timeout = strtol(timeout_str, NULL, 10); - } - curl_easy_setopt(c, CURLOPT_CONNECTTIMEOUT, timeout); - - SYSDEBUG("curl_easy_perform(): \n\turl=%s\n\tfilename=%s\n\tuser agent=%s\n\ttimeout=%zu", url, filename, user_agent, timeout); - CURLcode curl_code = curl_easy_perform(c); - SYSDEBUG("curl status code: %d", curl_code); - if (curl_code != CURLE_OK) { - if (!*errmsg) { - *errmsg = strdup(curl_easy_strerror(curl_code)); - } else { - strncpy(*errmsg, curl_easy_strerror(curl_code), strlen(curl_easy_strerror(curl_code) + 1)); + SYSDEBUG("curl_easy_perform(): \n\turl=%s\n\tfilename=%s\n\tuser agent=%s\n\ttimeout=%zu", url, filename, user_agent, timeout); + CURLcode curl_code = curl_easy_perform(c); + SYSDEBUG("curl status code: %d", curl_code); + + curl_easy_getinfo(c, CURLINFO_RESPONSE_CODE, &http_code); + SYSDEBUG("HTTP code: %li", http_code); + + if (curl_code != CURLE_OK) { + const size_t errmsg_maxlen = 256; + if (!*errmsg) { + *errmsg = calloc(errmsg_maxlen, sizeof(char)); + } + snprintf(*errmsg, errmsg_maxlen, "%s", curl_easy_strerror(curl_code)); + curl_easy_reset(c); + fclose(fp); + continue; + } + + fclose(fp); + if (CURLE_OK && *errmsg) { + // Retry loop succeeded, no error + *errmsg[0] = '\0'; } - goto failed; + break; } - curl_easy_getinfo(c, CURLINFO_RESPONSE_CODE, &http_code); - failed: - SYSDEBUG("HTTP code: %li", http_code); - fclose(fp); curl_easy_cleanup(c); curl_global_cleanup(); return http_code; -- cgit From 6ec8c3e753917f9f5e2404d6d7928b32a7bd7a59 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 12 Feb 2026 09:21:16 -0500 Subject: Rename STASIS_DOWNLOAD_RETRIES to STASIS_DOWNLOAD_RETRY_MAX --- src/lib/core/download.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/core/download.c b/src/lib/core/download.c index 12921c6..a1b14e0 100644 --- a/src/lib/core/download.c +++ b/src/lib/core/download.c @@ -25,7 +25,7 @@ long download(char *url, const char *filename, char **errmsg) { } size_t max_retries = 5; - const char *max_retries_str = getenv("STASIS_DOWNLOAD_RETRIES"); + const char *max_retries_str = getenv("STASIS_DOWNLOAD_RETRY_MAX"); if (max_retries_str) { max_retries = strtol(timeout_str, NULL, 10); if (max_retries <= 0) { -- cgit From 91aaab5fca6562999a44da9b03a0cc8cd3fea696 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 12 Feb 2026 09:21:45 -0500 Subject: Use snprintf for user_agent string --- src/lib/core/download.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/core/download.c b/src/lib/core/download.c index a1b14e0..1b84f39 100644 --- a/src/lib/core/download.c +++ b/src/lib/core/download.c @@ -13,7 +13,7 @@ size_t download_writer(void *fp, size_t size, size_t nmemb, void *stream) { long download(char *url, const char *filename, char **errmsg) { long http_code = -1; char user_agent[20]; - sprintf(user_agent, "stasis/%s", VERSION); + snprintf(user_agent, sizeof(user_agent), "stasis/%s", VERSION); long timeout = 30L; const char *timeout_str = getenv("STASIS_DOWNLOAD_TIMEOUT"); -- cgit From a1a581431c6341eed53fafed51e9af4a025ea5c8 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 12 Feb 2026 09:22:40 -0500 Subject: max_retries should use value of max_retries_str --- src/lib/core/download.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/core/download.c b/src/lib/core/download.c index 1b84f39..25271d8 100644 --- a/src/lib/core/download.c +++ b/src/lib/core/download.c @@ -27,7 +27,7 @@ long download(char *url, const char *filename, char **errmsg) { size_t max_retries = 5; const char *max_retries_str = getenv("STASIS_DOWNLOAD_RETRY_MAX"); if (max_retries_str) { - max_retries = strtol(timeout_str, NULL, 10); + max_retries = strtol(max_retries_str, NULL, 10); if (max_retries <= 0) { max_retries = 1; } -- cgit From 3c7264b379e5290dd50c106a5c268c024670c476 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 12 Feb 2026 09:23:34 -0500 Subject: Change "ATTEMPT" to "RETRY" and emit vital information --- src/lib/core/download.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/core/download.c b/src/lib/core/download.c index 25271d8..699be54 100644 --- a/src/lib/core/download.c +++ b/src/lib/core/download.c @@ -37,7 +37,7 @@ long download(char *url, const char *filename, char **errmsg) { CURL *c = curl_easy_init(); for (size_t retry = 0; retry < max_retries; retry++) { if (retry) { - fprintf(stderr, "\n[ATTEMPT %zu/%zu]\n", retry + 1, max_retries); + fprintf(stderr, "[RETRY %zu/%zu] %s: %s\n", retry + 1, max_retries, *errmsg, url); } curl_easy_setopt(c, CURLOPT_URL, url); curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, download_writer); -- cgit From 3461cb98d5e8bb8be5fb1407da609b78a76b417c Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 12 Feb 2026 09:24:45 -0500 Subject: Only set http_code on success. Return -1 otherise. --- src/lib/core/download.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/lib/core/download.c b/src/lib/core/download.c index 699be54..dea6aa4 100644 --- a/src/lib/core/download.c +++ b/src/lib/core/download.c @@ -57,9 +57,6 @@ long download(char *url, const char *filename, char **errmsg) { CURLcode curl_code = curl_easy_perform(c); SYSDEBUG("curl status code: %d", curl_code); - curl_easy_getinfo(c, CURLINFO_RESPONSE_CODE, &http_code); - SYSDEBUG("HTTP code: %li", http_code); - if (curl_code != CURLE_OK) { const size_t errmsg_maxlen = 256; if (!*errmsg) { @@ -76,6 +73,10 @@ long download(char *url, const char *filename, char **errmsg) { // Retry loop succeeded, no error *errmsg[0] = '\0'; } + + curl_easy_getinfo(c, CURLINFO_RESPONSE_CODE, &http_code); + SYSDEBUG("HTTP code: %li", http_code); + break; } curl_easy_cleanup(c); -- cgit From 33a5cdae6c903ac6ac4caced2e3a114fbeb0ab0d Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 12 Feb 2026 09:27:14 -0500 Subject: Implement STASIS_DOWNLOAD_RETRY_SECONDS --- src/lib/core/download.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/lib/core/download.c b/src/lib/core/download.c index dea6aa4..68a5a23 100644 --- a/src/lib/core/download.c +++ b/src/lib/core/download.c @@ -33,6 +33,16 @@ long download(char *url, const char *filename, char **errmsg) { } } + size_t max_retry_seconds = 3; + const char *max_retry_seconds_str = getenv("STASIS_DOWNLOAD_RETRY_SECONDS"); + if (max_retry_seconds_str) { + max_retry_seconds = strtol(max_retry_seconds_str, NULL, 10); + if (max_retry_seconds < 0) { + max_retry_seconds = 0; + } + } + + curl_global_init(CURL_GLOBAL_ALL); CURL *c = curl_easy_init(); for (size_t retry = 0; retry < max_retries; retry++) { @@ -65,6 +75,7 @@ long download(char *url, const char *filename, char **errmsg) { snprintf(*errmsg, errmsg_maxlen, "%s", curl_easy_strerror(curl_code)); curl_easy_reset(c); fclose(fp); + sleep(max_retry_seconds); continue; } -- cgit From 6ba221e2432b64cd5ebced8f006aaab5022cc2e6 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 12 Feb 2026 09:27:31 -0500 Subject: Comment --- src/lib/core/download.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/core/download.c b/src/lib/core/download.c index 68a5a23..ff97ee6 100644 --- a/src/lib/core/download.c +++ b/src/lib/core/download.c @@ -79,7 +79,9 @@ long download(char *url, const char *filename, char **errmsg) { continue; } + // Data written. Clean up. fclose(fp); + if (CURLE_OK && *errmsg) { // Retry loop succeeded, no error *errmsg[0] = '\0'; -- cgit From 608600b8d1b8c4bf759e6246cc2a442a58f94bdc Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 12 Feb 2026 09:29:02 -0500 Subject: Document env vars in README: * STASIS_DOWNLOAD_RETRY_MAX * STASIS_DOWNLOAD_RETRY_SECONDS --- README.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 67ebf0d..e72247f 100644 --- a/README.md +++ b/README.md @@ -201,7 +201,8 @@ stasis mydelivery.ini | STASIS_JF_CLIENT_CERT_KEY_PATH | OpenSSL key file (in cert path) | | STASIS_JF_REPO | Artifactory "generic" repository to write to | | STASIS_DOWNLOAD_TIMEOUT | Number of seconds before timing out a remote file download | -| STASIS_DOWNLOAD_RETRIES | Number of retries before giving up on a remote file download | +| STASIS_DOWNLOAD_RETRY_MAX | Number of retries before giving up on a remote file download | +| STASIS_DOWNLOAD_RETRY_SECONDS | Number of seconds to wait before retrying a remote file download | ## Main configuration (stasis.ini) @@ -288,14 +289,14 @@ Sections starting with `test:` will be used during the testing phase of the stas | Key | Type | Purpose | Required | |--------------|---------|----------------------------------------------------------|----------| | disable | Boolean | Disable `script` execution (`script_setup` always executes) | N | -| parallel | Boolean | Execute test block in parallel (default) or sequentially | N | -| timeout | String | Kill test script after `n[hms]` | N | -| build_recipe | String | Git repository path to package's conda recipe | N | -| repository | String | Git repository path or URL to clone | Y | -| version | String | Git commit or tag to check out | Y | -| runtime | List | Export environment variables specific to test context | Y | -| script_setup | List | Body of a shell script that will install dependencies | N | -| script | List | Body of a shell script that will execute the tests | Y | +| parallel | Boolean | Execute test block in parallel (default) or sequentially | N | +| timeout | String | Kill test script after `n[hms]` | N | +| build_recipe | String | Git repository path to package's conda recipe | N | +| repository | String | Git repository path or URL to clone | Y | +| version | String | Git commit or tag to check out | Y | +| runtime | List | Export environment variables specific to test context | Y | +| script_setup | List | Body of a shell script that will install dependencies | N | +| script | List | Body of a shell script that will execute the tests | Y | ### deploy:artifactory:_name_ -- cgit From 635ec182af3d281c27fe5297a042d68516fa3058 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 12 Feb 2026 09:29:11 -0500 Subject: Fix table format --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e72247f..a8c72d6 100644 --- a/README.md +++ b/README.md @@ -286,8 +286,8 @@ Environment variables exported are _global_ to all programs executed by stasis. Sections starting with `test:` will be used during the testing phase of the stasis pipeline. Where the value of `name` following the colon is an arbitrary value, and only used for reporting which test-run is executing. Section names must be unique. -| Key | Type | Purpose | Required | -|--------------|---------|----------------------------------------------------------|----------| +| Key | Type | Purpose | Required | +|--------------|---------|-------------------------------------------------------------|----------| | disable | Boolean | Disable `script` execution (`script_setup` always executes) | N | | parallel | Boolean | Execute test block in parallel (default) or sequentially | N | | timeout | String | Kill test script after `n[hms]` | N | -- cgit From b192fdbf220acc88cb4b9ebb859537048f2fcf9b Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 12 Feb 2026 09:29:39 -0500 Subject: Remove dead code --- tests/test_download.c | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_download.c b/tests/test_download.c index 714e614..31e9792 100644 --- a/tests/test_download.c +++ b/tests/test_download.c @@ -33,7 +33,6 @@ void test_download() { } STASIS_ASSERT(http_code == tc[i].http_code, "expecting non-error HTTP code"); - //char **data = file_readlines(filename, 0, 100, NULL); char *data = stasis_testing_read_ascii(filename); if (http_code >= 0) { STASIS_ASSERT(data != NULL, "data should not be null"); -- cgit From 835903affdbe8b807fabfa8b991cd02ff4ba547d Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 12 Feb 2026 09:41:31 -0500 Subject: Test against the tc struct's expected value --- tests/test_strlist.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_strlist.c b/tests/test_strlist.c index ce38ff6..47722c0 100644 --- a/tests/test_strlist.c +++ b/tests/test_strlist.c @@ -115,6 +115,7 @@ void test_strlist_append_file() { const char *local_filename = "test_strlist_append_file.txt"; struct testcase tc[] = { + {.origin = "https://this-will-never-work.tld/remote.txt", .expected = (const char *[]){NULL}}, {.origin = "https://ssb.stsci.edu/jhunk/stasis_test/test_strlist_append_file_from_remote.txt", .expected = expected}, {.origin = local_filename, .expected = expected}, }; @@ -141,10 +142,10 @@ void test_strlist_append_file() { const char *left; const char *right; left = strlist_item(list, z); - right = expected[z]; + right = tc[i].expected[z]; STASIS_ASSERT(strcmp(left, right) == 0, "file content is different than expected"); } - STASIS_ASSERT(strcmp_array((const char **) list->data, expected) == 0, "file contents does not match expected values"); + STASIS_ASSERT(strcmp_array((const char **) list->data, tc[i].expected) == 0, "file contents does not match expected values"); guard_strlist_free(&list); } } -- cgit From 3e176019323a4cfd61dfa04ab72096506c4395a5 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 12 Feb 2026 09:45:33 -0500 Subject: Error message pointer must be valid --- src/lib/core/strlist.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/lib/core/strlist.c b/src/lib/core/strlist.c index 5655da9..a0db5f3 100644 --- a/src/lib/core/strlist.c +++ b/src/lib/core/strlist.c @@ -92,8 +92,11 @@ int strlist_append_file(struct StrList *pStrList, char *_path, ReaderFn *readerF } close(fd); filename = strdup(tempfile); - long http_code = download(path, filename, NULL); + char *errmsg = NULL; + long http_code = download(path, filename, &errmsg); if (HTTP_ERROR(http_code)) { + SYSERROR("%s: %s", errmsg, filename); + guard_free(errmsg); retval = -1; goto fatal; } -- cgit From 34e6d866b79199218fa4dddafc45b3c9f046806f Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 12 Feb 2026 09:47:17 -0500 Subject: Error message pointer must be valid --- src/lib/delivery/delivery_conda.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/lib/delivery/delivery_conda.c b/src/lib/delivery/delivery_conda.c index 8974ae8..191d93f 100644 --- a/src/lib/delivery/delivery_conda.c +++ b/src/lib/delivery/delivery_conda.c @@ -26,9 +26,12 @@ int delivery_get_conda_installer(struct Delivery *ctx, char *installer_url) { sprintf(script_path, "%s/%s", ctx->storage.tmpdir, installer); if (access(script_path, F_OK)) { // Script doesn't exist - long fetch_status = download(installer_url, script_path, NULL); + char *errmsg = NULL; + long fetch_status = download(installer_url, script_path, &errmsg); if (HTTP_ERROR(fetch_status) || fetch_status < 0) { // download failed + SYSERROR("download failed: %s: %s\n", errmsg, installer_url); + guard_free(errmsg); return -1; } } else { -- cgit From 4458f2e9da2ef474c63457a96ee0f637fa89ad14 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 12 Feb 2026 09:51:58 -0500 Subject: Use ssize_t to avoid rollover from strtol on user input --- src/lib/core/download.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/core/download.c b/src/lib/core/download.c index ff97ee6..817e576 100644 --- a/src/lib/core/download.c +++ b/src/lib/core/download.c @@ -24,7 +24,7 @@ long download(char *url, const char *filename, char **errmsg) { } } - size_t max_retries = 5; + ssize_t max_retries = 5; const char *max_retries_str = getenv("STASIS_DOWNLOAD_RETRY_MAX"); if (max_retries_str) { max_retries = strtol(max_retries_str, NULL, 10); @@ -33,7 +33,7 @@ long download(char *url, const char *filename, char **errmsg) { } } - size_t max_retry_seconds = 3; + ssize_t max_retry_seconds = 3; const char *max_retry_seconds_str = getenv("STASIS_DOWNLOAD_RETRY_SECONDS"); if (max_retry_seconds_str) { max_retry_seconds = strtol(max_retry_seconds_str, NULL, 10); @@ -45,7 +45,7 @@ long download(char *url, const char *filename, char **errmsg) { curl_global_init(CURL_GLOBAL_ALL); CURL *c = curl_easy_init(); - for (size_t retry = 0; retry < max_retries; retry++) { + for (ssize_t retry = 0; retry < max_retries; retry++) { if (retry) { fprintf(stderr, "[RETRY %zu/%zu] %s: %s\n", retry + 1, max_retries, *errmsg, url); } -- cgit From 2cd2d2c47593941f002ed45f44f0e0c1072738d7 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 12 Feb 2026 09:53:21 -0500 Subject: errmsg argument must be a valid pointer * Remove redundant status check already performed by HTTP_ERROR() --- src/lib/core/artifactory.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/lib/core/artifactory.c b/src/lib/core/artifactory.c index eedaf43..d5457e7 100644 --- a/src/lib/core/artifactory.c +++ b/src/lib/core/artifactory.c @@ -61,9 +61,10 @@ int artifactory_download_cli(char *dest, } sprintf(path + strlen(path), "/%s", remote_filename); - long fetch_status = download(url, path, NULL); - if (HTTP_ERROR(fetch_status) || fetch_status < 0) { - fprintf(stderr, "%s: download failed: %s\n", __FUNCTION__, url); + char *errmsg = NULL; + long fetch_status = download(url, path, &errmsg); + if (HTTP_ERROR(fetch_status)) { + SYSERROR("download failed: %s: %s\n", errmsg, url); return -1; } chmod(path, 0755); -- cgit