From 4a8008359f38db1b3d7acaf7013a66a5c5394922 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Wed, 11 Feb 2026 15:05:08 -0500 Subject: Add a retry loop to download * Configurable with env var: STASIS_DOWNLOAD_RETRIES --- src/lib/core/download.c | 86 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 56 insertions(+), 30 deletions(-) (limited to 'src/lib') diff --git a/src/lib/core/download.c b/src/lib/core/download.c index b021860..12921c6 100644 --- a/src/lib/core/download.c +++ b/src/lib/core/download.c @@ -14,44 +14,70 @@ long download(char *url, const char *filename, char **errmsg) { long http_code = -1; char user_agent[20]; sprintf(user_agent, "stasis/%s", VERSION); + long timeout = 30L; - char *timeout_str = getenv("STASIS_DOWNLOAD_TIMEOUT"); + const char *timeout_str = getenv("STASIS_DOWNLOAD_TIMEOUT"); + if (timeout_str) { + timeout = strtol(timeout_str, NULL, 10); + if (timeout <= 0L) { + timeout = 1L; + } + } + + size_t max_retries = 5; + const char *max_retries_str = getenv("STASIS_DOWNLOAD_RETRIES"); + if (max_retries_str) { + max_retries = strtol(timeout_str, NULL, 10); + if (max_retries <= 0) { + max_retries = 1; + } + } curl_global_init(CURL_GLOBAL_ALL); CURL *c = curl_easy_init(); - curl_easy_setopt(c, CURLOPT_URL, url); - curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, download_writer); - FILE *fp = fopen(filename, "wb"); - if (!fp) { - return -1; - } + for (size_t retry = 0; retry < max_retries; retry++) { + if (retry) { + fprintf(stderr, "\n[ATTEMPT %zu/%zu]\n", retry + 1, max_retries); + } + curl_easy_setopt(c, CURLOPT_URL, url); + curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, download_writer); + FILE *fp = fopen(filename, "wb"); + if (!fp) { + return -1; + } - curl_easy_setopt(c, CURLOPT_VERBOSE, 0L); - curl_easy_setopt(c, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(c, CURLOPT_USERAGENT, user_agent); - curl_easy_setopt(c, CURLOPT_NOPROGRESS, 0L); - curl_easy_setopt(c, CURLOPT_WRITEDATA, fp); + curl_easy_setopt(c, CURLOPT_VERBOSE, 0L); + curl_easy_setopt(c, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(c, CURLOPT_USERAGENT, user_agent); + curl_easy_setopt(c, CURLOPT_NOPROGRESS, 0L); + curl_easy_setopt(c, CURLOPT_WRITEDATA, fp); + curl_easy_setopt(c, CURLOPT_CONNECTTIMEOUT, timeout); - if (timeout_str) { - timeout = strtol(timeout_str, NULL, 10); - } - curl_easy_setopt(c, CURLOPT_CONNECTTIMEOUT, timeout); - - SYSDEBUG("curl_easy_perform(): \n\turl=%s\n\tfilename=%s\n\tuser agent=%s\n\ttimeout=%zu", url, filename, user_agent, timeout); - CURLcode curl_code = curl_easy_perform(c); - SYSDEBUG("curl status code: %d", curl_code); - if (curl_code != CURLE_OK) { - if (!*errmsg) { - *errmsg = strdup(curl_easy_strerror(curl_code)); - } else { - strncpy(*errmsg, curl_easy_strerror(curl_code), strlen(curl_easy_strerror(curl_code) + 1)); + SYSDEBUG("curl_easy_perform(): \n\turl=%s\n\tfilename=%s\n\tuser agent=%s\n\ttimeout=%zu", url, filename, user_agent, timeout); + CURLcode curl_code = curl_easy_perform(c); + SYSDEBUG("curl status code: %d", curl_code); + + curl_easy_getinfo(c, CURLINFO_RESPONSE_CODE, &http_code); + SYSDEBUG("HTTP code: %li", http_code); + + if (curl_code != CURLE_OK) { + const size_t errmsg_maxlen = 256; + if (!*errmsg) { + *errmsg = calloc(errmsg_maxlen, sizeof(char)); + } + snprintf(*errmsg, errmsg_maxlen, "%s", curl_easy_strerror(curl_code)); + curl_easy_reset(c); + fclose(fp); + continue; + } + + fclose(fp); + if (CURLE_OK && *errmsg) { + // Retry loop succeeded, no error + *errmsg[0] = '\0'; } - goto failed; + break; } - curl_easy_getinfo(c, CURLINFO_RESPONSE_CODE, &http_code); - failed: - SYSDEBUG("HTTP code: %li", http_code); - fclose(fp); curl_easy_cleanup(c); curl_global_cleanup(); return http_code; -- cgit From 6ec8c3e753917f9f5e2404d6d7928b32a7bd7a59 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 12 Feb 2026 09:21:16 -0500 Subject: Rename STASIS_DOWNLOAD_RETRIES to STASIS_DOWNLOAD_RETRY_MAX --- src/lib/core/download.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib') diff --git a/src/lib/core/download.c b/src/lib/core/download.c index 12921c6..a1b14e0 100644 --- a/src/lib/core/download.c +++ b/src/lib/core/download.c @@ -25,7 +25,7 @@ long download(char *url, const char *filename, char **errmsg) { } size_t max_retries = 5; - const char *max_retries_str = getenv("STASIS_DOWNLOAD_RETRIES"); + const char *max_retries_str = getenv("STASIS_DOWNLOAD_RETRY_MAX"); if (max_retries_str) { max_retries = strtol(timeout_str, NULL, 10); if (max_retries <= 0) { -- cgit From 91aaab5fca6562999a44da9b03a0cc8cd3fea696 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 12 Feb 2026 09:21:45 -0500 Subject: Use snprintf for user_agent string --- src/lib/core/download.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib') diff --git a/src/lib/core/download.c b/src/lib/core/download.c index a1b14e0..1b84f39 100644 --- a/src/lib/core/download.c +++ b/src/lib/core/download.c @@ -13,7 +13,7 @@ size_t download_writer(void *fp, size_t size, size_t nmemb, void *stream) { long download(char *url, const char *filename, char **errmsg) { long http_code = -1; char user_agent[20]; - sprintf(user_agent, "stasis/%s", VERSION); + snprintf(user_agent, sizeof(user_agent), "stasis/%s", VERSION); long timeout = 30L; const char *timeout_str = getenv("STASIS_DOWNLOAD_TIMEOUT"); -- cgit From a1a581431c6341eed53fafed51e9af4a025ea5c8 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 12 Feb 2026 09:22:40 -0500 Subject: max_retries should use value of max_retries_str --- src/lib/core/download.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib') diff --git a/src/lib/core/download.c b/src/lib/core/download.c index 1b84f39..25271d8 100644 --- a/src/lib/core/download.c +++ b/src/lib/core/download.c @@ -27,7 +27,7 @@ long download(char *url, const char *filename, char **errmsg) { size_t max_retries = 5; const char *max_retries_str = getenv("STASIS_DOWNLOAD_RETRY_MAX"); if (max_retries_str) { - max_retries = strtol(timeout_str, NULL, 10); + max_retries = strtol(max_retries_str, NULL, 10); if (max_retries <= 0) { max_retries = 1; } -- cgit From 3c7264b379e5290dd50c106a5c268c024670c476 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 12 Feb 2026 09:23:34 -0500 Subject: Change "ATTEMPT" to "RETRY" and emit vital information --- src/lib/core/download.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib') diff --git a/src/lib/core/download.c b/src/lib/core/download.c index 25271d8..699be54 100644 --- a/src/lib/core/download.c +++ b/src/lib/core/download.c @@ -37,7 +37,7 @@ long download(char *url, const char *filename, char **errmsg) { CURL *c = curl_easy_init(); for (size_t retry = 0; retry < max_retries; retry++) { if (retry) { - fprintf(stderr, "\n[ATTEMPT %zu/%zu]\n", retry + 1, max_retries); + fprintf(stderr, "[RETRY %zu/%zu] %s: %s\n", retry + 1, max_retries, *errmsg, url); } curl_easy_setopt(c, CURLOPT_URL, url); curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, download_writer); -- cgit From 3461cb98d5e8bb8be5fb1407da609b78a76b417c Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 12 Feb 2026 09:24:45 -0500 Subject: Only set http_code on success. Return -1 otherise. --- src/lib/core/download.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src/lib') diff --git a/src/lib/core/download.c b/src/lib/core/download.c index 699be54..dea6aa4 100644 --- a/src/lib/core/download.c +++ b/src/lib/core/download.c @@ -57,9 +57,6 @@ long download(char *url, const char *filename, char **errmsg) { CURLcode curl_code = curl_easy_perform(c); SYSDEBUG("curl status code: %d", curl_code); - curl_easy_getinfo(c, CURLINFO_RESPONSE_CODE, &http_code); - SYSDEBUG("HTTP code: %li", http_code); - if (curl_code != CURLE_OK) { const size_t errmsg_maxlen = 256; if (!*errmsg) { @@ -76,6 +73,10 @@ long download(char *url, const char *filename, char **errmsg) { // Retry loop succeeded, no error *errmsg[0] = '\0'; } + + curl_easy_getinfo(c, CURLINFO_RESPONSE_CODE, &http_code); + SYSDEBUG("HTTP code: %li", http_code); + break; } curl_easy_cleanup(c); -- cgit From 33a5cdae6c903ac6ac4caced2e3a114fbeb0ab0d Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 12 Feb 2026 09:27:14 -0500 Subject: Implement STASIS_DOWNLOAD_RETRY_SECONDS --- src/lib/core/download.c | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'src/lib') diff --git a/src/lib/core/download.c b/src/lib/core/download.c index dea6aa4..68a5a23 100644 --- a/src/lib/core/download.c +++ b/src/lib/core/download.c @@ -33,6 +33,16 @@ long download(char *url, const char *filename, char **errmsg) { } } + size_t max_retry_seconds = 3; + const char *max_retry_seconds_str = getenv("STASIS_DOWNLOAD_RETRY_SECONDS"); + if (max_retry_seconds_str) { + max_retry_seconds = strtol(max_retry_seconds_str, NULL, 10); + if (max_retry_seconds < 0) { + max_retry_seconds = 0; + } + } + + curl_global_init(CURL_GLOBAL_ALL); CURL *c = curl_easy_init(); for (size_t retry = 0; retry < max_retries; retry++) { @@ -65,6 +75,7 @@ long download(char *url, const char *filename, char **errmsg) { snprintf(*errmsg, errmsg_maxlen, "%s", curl_easy_strerror(curl_code)); curl_easy_reset(c); fclose(fp); + sleep(max_retry_seconds); continue; } -- cgit From 6ba221e2432b64cd5ebced8f006aaab5022cc2e6 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 12 Feb 2026 09:27:31 -0500 Subject: Comment --- src/lib/core/download.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/lib') diff --git a/src/lib/core/download.c b/src/lib/core/download.c index 68a5a23..ff97ee6 100644 --- a/src/lib/core/download.c +++ b/src/lib/core/download.c @@ -79,7 +79,9 @@ long download(char *url, const char *filename, char **errmsg) { continue; } + // Data written. Clean up. fclose(fp); + if (CURLE_OK && *errmsg) { // Retry loop succeeded, no error *errmsg[0] = '\0'; -- cgit From 3e176019323a4cfd61dfa04ab72096506c4395a5 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 12 Feb 2026 09:45:33 -0500 Subject: Error message pointer must be valid --- src/lib/core/strlist.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src/lib') diff --git a/src/lib/core/strlist.c b/src/lib/core/strlist.c index 5655da9..a0db5f3 100644 --- a/src/lib/core/strlist.c +++ b/src/lib/core/strlist.c @@ -92,8 +92,11 @@ int strlist_append_file(struct StrList *pStrList, char *_path, ReaderFn *readerF } close(fd); filename = strdup(tempfile); - long http_code = download(path, filename, NULL); + char *errmsg = NULL; + long http_code = download(path, filename, &errmsg); if (HTTP_ERROR(http_code)) { + SYSERROR("%s: %s", errmsg, filename); + guard_free(errmsg); retval = -1; goto fatal; } -- cgit From 34e6d866b79199218fa4dddafc45b3c9f046806f Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 12 Feb 2026 09:47:17 -0500 Subject: Error message pointer must be valid --- src/lib/delivery/delivery_conda.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src/lib') diff --git a/src/lib/delivery/delivery_conda.c b/src/lib/delivery/delivery_conda.c index 8974ae8..191d93f 100644 --- a/src/lib/delivery/delivery_conda.c +++ b/src/lib/delivery/delivery_conda.c @@ -26,9 +26,12 @@ int delivery_get_conda_installer(struct Delivery *ctx, char *installer_url) { sprintf(script_path, "%s/%s", ctx->storage.tmpdir, installer); if (access(script_path, F_OK)) { // Script doesn't exist - long fetch_status = download(installer_url, script_path, NULL); + char *errmsg = NULL; + long fetch_status = download(installer_url, script_path, &errmsg); if (HTTP_ERROR(fetch_status) || fetch_status < 0) { // download failed + SYSERROR("download failed: %s: %s\n", errmsg, installer_url); + guard_free(errmsg); return -1; } } else { -- cgit From 4458f2e9da2ef474c63457a96ee0f637fa89ad14 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 12 Feb 2026 09:51:58 -0500 Subject: Use ssize_t to avoid rollover from strtol on user input --- src/lib/core/download.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/lib') diff --git a/src/lib/core/download.c b/src/lib/core/download.c index ff97ee6..817e576 100644 --- a/src/lib/core/download.c +++ b/src/lib/core/download.c @@ -24,7 +24,7 @@ long download(char *url, const char *filename, char **errmsg) { } } - size_t max_retries = 5; + ssize_t max_retries = 5; const char *max_retries_str = getenv("STASIS_DOWNLOAD_RETRY_MAX"); if (max_retries_str) { max_retries = strtol(max_retries_str, NULL, 10); @@ -33,7 +33,7 @@ long download(char *url, const char *filename, char **errmsg) { } } - size_t max_retry_seconds = 3; + ssize_t max_retry_seconds = 3; const char *max_retry_seconds_str = getenv("STASIS_DOWNLOAD_RETRY_SECONDS"); if (max_retry_seconds_str) { max_retry_seconds = strtol(max_retry_seconds_str, NULL, 10); @@ -45,7 +45,7 @@ long download(char *url, const char *filename, char **errmsg) { curl_global_init(CURL_GLOBAL_ALL); CURL *c = curl_easy_init(); - for (size_t retry = 0; retry < max_retries; retry++) { + for (ssize_t retry = 0; retry < max_retries; retry++) { if (retry) { fprintf(stderr, "[RETRY %zu/%zu] %s: %s\n", retry + 1, max_retries, *errmsg, url); } -- cgit From 2cd2d2c47593941f002ed45f44f0e0c1072738d7 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 12 Feb 2026 09:53:21 -0500 Subject: errmsg argument must be a valid pointer * Remove redundant status check already performed by HTTP_ERROR() --- src/lib/core/artifactory.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src/lib') diff --git a/src/lib/core/artifactory.c b/src/lib/core/artifactory.c index eedaf43..d5457e7 100644 --- a/src/lib/core/artifactory.c +++ b/src/lib/core/artifactory.c @@ -61,9 +61,10 @@ int artifactory_download_cli(char *dest, } sprintf(path + strlen(path), "/%s", remote_filename); - long fetch_status = download(url, path, NULL); - if (HTTP_ERROR(fetch_status) || fetch_status < 0) { - fprintf(stderr, "%s: download failed: %s\n", __FUNCTION__, url); + char *errmsg = NULL; + long fetch_status = download(url, path, &errmsg); + if (HTTP_ERROR(fetch_status)) { + SYSERROR("download failed: %s: %s\n", errmsg, url); return -1; } chmod(path, 0755); -- cgit