Merge pull request #147 from jhunkeler/conda-updatesHEAD1.8.0master

Support modern versions of conda/mamba

5 files changed, 164 insertions, 50 deletions

diff --git a/src/lib/delivery/delivery.c b/src/lib/delivery/delivery.c
index dc9e2ce..d256681 100644
--- a/src/lib/delivery/delivery.c
+++ b/src/lib/delivery/delivery.c
@@ -236,6 +236,8 @@ void delivery_free(struct Delivery *ctx) {
     guard_free(ctx->info.build_number);
     guard_free(ctx->info.release_name);
     guard_free(ctx->info.time_info);
+
+    conda_capable_free(&ctx->conda.capabilities);
     guard_free(ctx->conda.installer_baseurl);
     guard_free(ctx->conda.installer_name);
     guard_free(ctx->conda.installer_version);
@@ -308,37 +310,38 @@ int delivery_format_str(struct Delivery *ctx, char **dest, size_t maxlen, const
             i++;
             switch (fmt[i]) {
                 case 'n':   // name
-                    strncat(*dest, ctx->meta.name, maxlen - 1);
+                    safe_strncat(*dest, ctx->meta.name, maxlen);
                     break;
                 case 'c':   // codename
-                    strncat(*dest, ctx->meta.codename, maxlen - 1);
+                    safe_strncat(*dest, ctx->meta.codename, maxlen);
                     break;
                 case 'm':   // mission
-                    strncat(*dest, ctx->meta.mission, maxlen - 1);
+                    safe_strncat(*dest, ctx->meta.mission, maxlen);
                     break;
                 case 'r':   // revision
                     snprintf(*dest + strlen(*dest), maxlen - strlen(*dest), "%d", ctx->meta.rc);
                     break;
                 case 'R':   // "final"-aware revision
-                    if (ctx->meta.final)
-                        strncat(*dest, "final", maxlen);
-                    else
+                    if (ctx->meta.final) {
+                        safe_strncat(*dest, "final", maxlen);
+                    } else {
                         snprintf(*dest + strlen(*dest), maxlen - strlen(*dest), "%d", ctx->meta.rc);
+                    }
                     break;
                 case 'v':   // version
-                    strncat(*dest, ctx->meta.version, maxlen - 1);
+                    safe_strncat(*dest, ctx->meta.version, maxlen);
                     break;
                 case 'P':   // python version
-                    strncat(*dest, ctx->meta.python, maxlen - 1);
+                    safe_strncat(*dest, ctx->meta.python, maxlen);
                     break;
                 case 'p':   // python version major/minor
-                    strncat(*dest, ctx->meta.python_compact, maxlen - 1);
+                    safe_strncat(*dest, ctx->meta.python_compact, maxlen);
                     break;
                 case 'a':   // system architecture name
-                    strncat(*dest, ctx->system.arch, maxlen - 1);
+                    safe_strncat(*dest, ctx->system.arch, maxlen);
                     break;
                 case 'o':   // system platform (OS) name
-                    strncat(*dest, ctx->system.platform[DELIVERY_PLATFORM_RELEASE], maxlen - 1);
+                    safe_strncat(*dest, ctx->system.platform[DELIVERY_PLATFORM_RELEASE], maxlen);
                     break;
                 case 't':   // unix epoch
                     snprintf(*dest + strlen(*dest), maxlen - strlen(*dest), "%ld", ctx->info.time_now);
@@ -363,11 +366,11 @@ void delivery_defer_packages(struct Delivery *ctx, int type) {
     if (DEFER_CONDA == type) {
         dataptr = ctx->conda.conda_packages;
         deferred = ctx->conda.conda_packages_defer;
-        strncpy(mode, "conda", sizeof(mode) - 1);
+        safe_strncpy(mode, "conda", sizeof(mode));
     } else if (DEFER_PIP == type) {
         dataptr = ctx->conda.pip_packages;
         deferred = ctx->conda.pip_packages_defer;
-        strncpy(mode, "pip", sizeof(mode) - 1);
+        safe_strncpy(mode, "pip", sizeof(mode));
     } else {
         SYSERROR("BUG: type %d does not map to a supported package manager!", type);
         exit(1);
@@ -397,11 +400,10 @@ void delivery_defer_packages(struct Delivery *ctx, int type) {
             while (*spec_end != '\0' && !isalnum(*spec_end)) {
                 spec_end++;
             }
-            strncpy(package_name, name, spec_begin - name);
-            package_name[spec_begin - name] = '\0';
+            size_t spec_len = spec_begin - name;
+            safe_strncpy(package_name, name, spec_len ? spec_len + 1 : sizeof(package_name));
         } else {
-            strncpy(package_name, name, sizeof(package_name) - 1);
-            package_name[sizeof(package_name) - 1] = '\0';
+            safe_strncpy(package_name, name, sizeof(package_name));
         }
         remove_extras(package_name);
 
@@ -412,8 +414,7 @@ void delivery_defer_packages(struct Delivery *ctx, int type) {
             struct Test *test = ctx->tests->test[x];
             char nametmp[STASIS_NAME_MAX] = {0};
 
-            strncpy(nametmp, package_name, sizeof(nametmp) - 1);
-            nametmp[sizeof(nametmp) - 1] = '\0';
+            safe_strncpy(nametmp, package_name, sizeof(nametmp));
 
             // Is the [test:NAME] in the package name?
             if (!strcmp(nametmp, test->name)) {
@@ -479,6 +480,10 @@ void delivery_defer_packages(struct Delivery *ctx, int type) {
             }
         }
 
+        if (getenv("STASIS_ALWAYS_BUILD_FOR_HOST")) {
+            build_for_host = 1;
+        }
+
         if (build_for_host) {
             printf("BUILD FOR HOST\n");
             strlist_append(&deferred, name);
diff --git a/src/lib/delivery/delivery_build.c b/src/lib/delivery/delivery_build.c
index d8674e0..9ef5d92 100644
--- a/src/lib/delivery/delivery_build.c
+++ b/src/lib/delivery/delivery_build.c
@@ -16,11 +16,13 @@ int delivery_build_recipes(struct Delivery *ctx) {
                 SYSERROR("BUG: recipe_clone() succeeded but recipe_dir is NULL: %s", strerror(errno));
                 return -1;
             }
-            int recipe_type = recipe_get_type(recipe_dir);
+            const int recipe_style = recipe_get_style(recipe_dir);
+            const int recipe_build_system = recipe_get_build_system(recipe_dir, recipe_style);
+
             if(!pushd(recipe_dir)) {
-                if (RECIPE_TYPE_ASTROCONDA == recipe_type) {
+                if (RECIPE_STYLE_ASTROCONDA == recipe_style) {
                     pushd(path_basename(ctx->tests->test[i]->repository));
-                } else if (RECIPE_TYPE_CONDA_FORGE == recipe_type) {
+                } else if (RECIPE_STYLE_CONDA_FORGE == recipe_style) {
                     pushd("recipe");
                 }
 
@@ -56,23 +58,28 @@ int delivery_build_recipes(struct Delivery *ctx) {
                 snprintf(recipe_buildno, sizeof(recipe_buildno), "  number: 0");
 
                 unsigned flags = REPLACE_TRUNCATE_AFTER_MATCH;
-                //file_replace_text("meta.yaml", "{% set version = ", recipe_version);
-                if (ctx->meta.final) { // remove this. i.e. statis cannot deploy a release to conda-forge
-                    snprintf(recipe_version, sizeof(recipe_version), "{%% set version = \"%s\" %%}", ctx->tests->test[i]->version);
-                    // TODO: replace sha256 of tagged archive
-                    // TODO: leave the recipe unchanged otherwise. in theory this should produce the same conda package hash as conda forge.
-                    // For now, remove the sha256 requirement
-                    file_replace_text("meta.yaml", "sha256:", "\n", flags);
-                } else {
-                    file_replace_text("meta.yaml", "{% set version = ", recipe_version, flags);
-                    file_replace_text("meta.yaml", "  url:", recipe_git_url, flags);
-                    //file_replace_text("meta.yaml", "sha256:", recipe_git_rev);
-                    file_replace_text("meta.yaml", "  sha256:", "\n", flags);
-                    file_replace_text("meta.yaml", "  number:", recipe_buildno, flags);
+                if (recipe_build_system == RECIPE_BUILD_CONDA_BUILD) {
+                    if (ctx->meta.final) { // remove this. i.e. statis cannot deploy a release to conda-forge
+                        snprintf(recipe_version, sizeof(recipe_version), "{%% set version = \"%s\" %%}", ctx->tests->test[i]->version);
+                        // TODO: replace sha256 of tagged archive
+                        // TODO: leave the recipe unchanged otherwise. in theory this should produce the same conda package hash as conda forge.
+                        // For now, remove the sha256 requirement
+                        file_replace_text("meta.yaml", "sha256:", "\n", flags);
+                    } else {
+                        file_replace_text("meta.yaml", "{% set version = ", recipe_version, flags);
+                        file_replace_text("meta.yaml", "  url:", recipe_git_url, flags);
+                        file_replace_text("meta.yaml", "  sha256:", "\n", flags);
+                        file_replace_text("meta.yaml", "  number:", recipe_buildno, flags);
+                    }
+                } else if (recipe_build_system == RECIPE_BUILD_RATTLER) {
+                    file_replace_text("recipe.yaml", "  version:", ctx->tests->test[i]->version, flags);
+                    file_replace_text("recipe.yaml", "  url:", recipe_git_url, flags);
+                    file_replace_text("recipe.yaml", "  sha256:", "\n", flags);
+                    file_replace_text("recipe.yaml", "  number:", recipe_buildno, flags);
                 }
 
                 char command[PATH_MAX];
-                if (RECIPE_TYPE_CONDA_FORGE == recipe_type) {
+                if (RECIPE_STYLE_CONDA_FORGE == recipe_style) {
                     char arch[STASIS_NAME_MAX] = {0};
                     char platform[STASIS_NAME_MAX] = {0};
 
@@ -92,18 +99,34 @@ int delivery_build_recipes(struct Delivery *ctx) {
                     }
                     tolower_s(arch);
 
-                    snprintf(command, sizeof(command), "mambabuild --python=%s -m ../.ci_support/%s_%s_.yaml .",
-                            ctx->meta.python, platform, arch);
+                    // default build tool is "conda build" aka "build"
+                    char tool[STASIS_NAME_MAX] = "build";
+                    if (recipe_build_system == RECIPE_BUILD_CONDA_BUILD) {
+                        if (strlist_contains(globals.conda_packages, "boa", NULL)) {
+                            safe_strncpy(tool, "mambabuild", sizeof(tool));
+                        }
+                    } else if (recipe_build_system == RECIPE_BUILD_RATTLER) {
+                        snprintf(tool, sizeof(tool), "rattler-build");
+                    }
+                    snprintf(command, sizeof(command), "%s --python=%s -m ../.ci_support/%s_%s_.yaml .",
+                            tool, ctx->meta.python, platform, arch);
+                } else {
+                    snprintf(command, sizeof(command), "build --python=%s .", ctx->meta.python);
+                }
+
+                int status = 0;
+                if (recipe_build_system == RECIPE_BUILD_RATTLER) {
+                    // rattler-build is a standalone program, not a conda sub-command
+                    status = system(command);
                 } else {
-                    snprintf(command, sizeof(command), "mambabuild --python=%s .", ctx->meta.python);
+                    status = conda_exec(command);
                 }
-                int status = conda_exec(command);
                 if (status) {
                     guard_free(recipe_dir);
                     return -1;
                 }
 
-                if (RECIPE_TYPE_GENERIC != recipe_type) {
+                if (RECIPE_STYLE_GENERIC != recipe_style) {
                     popd();
                 }
                 popd();
diff --git a/src/lib/delivery/delivery_conda.c b/src/lib/delivery/delivery_conda.c
index 117e6c9..4f2920e 100644
--- a/src/lib/delivery/delivery_conda.c
+++ b/src/lib/delivery/delivery_conda.c
@@ -103,24 +103,61 @@ void delivery_install_conda(char *install_script, char *conda_install_dir) {
     }
 }
 
-void delivery_conda_enable(struct Delivery *ctx, char *conda_install_dir) {
-    if (conda_activate(conda_install_dir, "base")) {
-        SYSERROR("conda activation failed");
-        exit(1);
-    }
+void delivery_conda_enable(struct Delivery *ctx) {
+    setenv("MAMBA_ROOT_PREFIX", ctx->storage.conda_install_prefix, 1);
 
     // Setting the CONDARC environment variable appears to be the only consistent
     // way to make sure the file is used. Not setting this variable leads to strange
     // behavior, especially if a conda environment is already active when STASIS is loaded.
     char rcpath[PATH_MAX];
-    snprintf(rcpath, sizeof(rcpath), "%s/%s", conda_install_dir, ".condarc");
+    snprintf(rcpath, sizeof(rcpath), "%s/%s", ctx->storage.conda_install_prefix, ".condarc");
     setenv("CONDARC", rcpath, 1);
+    setenv("MAMBARC", rcpath, 1);
     if (runtime_replace(&ctx->runtime.environ, __environ)) {
         SYSERROR("unable to replace runtime environment after activating conda");
         exit(1);
     }
 
-    if (conda_setup_headless()) {
+    if (conda_activate(ctx->storage.conda_install_prefix, "base")) {
+        SYSERROR("conda activation failed");
+        exit(1);
+    }
+
+    char pinned[PATH_MAX];
+    snprintf(pinned, sizeof(pinned), "%s/conda-meta/pinned", ctx->storage.conda_install_prefix);
+    touch(pinned);
+    if (errno == ENOENT) {
+        errno = 0;
+    }
+
+    char *conda_version = strdup(ctx->conda.installer_version);
+    if (conda_version) {
+        char *rev = strpbrk(conda_version, "-");
+        if (rev) {
+            *rev = '\0';
+        }
+
+        FILE *pinned_fp = fopen(pinned, "w+");
+        if (!pinned_fp) {
+            SYSERROR("unable to open conda-meta/pinned file for writing: %s", strerror(errno));
+            exit(1);
+        }
+        fprintf(pinned_fp, "conda=%s\n", conda_version);
+        fclose(pinned_fp);
+        guard_free(conda_version);
+    }
+
+    if (conda_capable(&ctx->conda.capabilities, ctx->storage.conda_install_prefix)) {
+        SYSERROR("Conda capability check failed");
+        exit(1);
+    }
+
+    if (!ctx->conda.capabilities.usable) {
+        SYSERROR("Conda is broken");
+        exit(1);
+    }
+
+    if (conda_setup_headless(&ctx->conda.capabilities)) {
         // no COE check. this call must succeed.
         exit(1);
     }
diff --git a/src/lib/delivery/delivery_populate.c b/src/lib/delivery/delivery_populate.c
index 5ce11d7..314ef46 100644
--- a/src/lib/delivery/delivery_populate.c
+++ b/src/lib/delivery/delivery_populate.c
@@ -199,6 +199,44 @@ static void normalize_ini_list(struct INIFILE **inip, struct StrList **listp, ch
     (*listp) = list;
 }
 
+static int check_package_spec_list(struct StrList *list, const char *ini_section, const char *ini_key) {
+    if (!list) {
+        // empty lists are OK
+        return 0;
+    }
+    for (size_t i = 0; i < strlist_count(list); i++) {
+        const char *item = strlist_item(list, i);
+        if (!item) {
+            continue;
+        }
+        const char *invalid_chars = "@:/<>!~";
+        const char *invalid_spec = strpbrk(item, invalid_chars);
+        if (invalid_spec) {
+            SYSERROR("Invalid version specification detected at %s:%s[%zu], \"%s\"", ini_section, ini_key, i, item);
+            SYSERROR("Package specification may not contain the operator '%c' (or any of \"%s\")", *invalid_spec, invalid_chars);
+            SYSERROR("");
+            SYSERROR("%s:%s supports:", ini_section, ini_key);
+            SYSERROR("    {package}[=={version|tag|branch|ref}]");
+            SYSERROR("");
+            char *reported_name = strdup(item);
+            if (!reported_name) {
+                SYSERROR("unable to allocate memory for reported_name");
+                return -1;
+            }
+            const char *reported_name_end = reported_name;
+            while (!ispunct(*reported_name_end)) {
+                reported_name_end++;
+            }
+            reported_name[reported_name_end - reported_name] = '\0';
+            SYSERROR("Set test:%s.repository to point to a valid Git repository URL to enable [tag|branch|ref]", reported_name);
+
+            guard_free(reported_name);
+            return -1;
+        }
+    }
+    return 0;
+}
+
 int populate_delivery_ini(struct Delivery *ctx, int render_mode) {
     struct INIFILE *ini = ctx->_stasis_ini_fp.delivery;
     struct INIData *rtdata;
@@ -235,12 +273,22 @@ int populate_delivery_ini(struct Delivery *ctx, int render_mode) {
     ctx->conda.installer_platform = ini_getval_str(ini, "conda", "installer_platform", render_mode, &err);
     ctx->conda.installer_arch = ini_getval_str(ini, "conda", "installer_arch", render_mode, &err);
     ctx->conda.installer_baseurl = ini_getval_str(ini, "conda", "installer_baseurl", render_mode, &err);
+
     ctx->conda.conda_packages = ini_getval_strlist(ini, "conda", "conda_packages", LINE_SEP, render_mode, &err);
     normalize_ini_list(&ini, &ctx->conda.conda_packages, "conda", "conda_packages", render_mode);
+    if (check_package_spec_list(ctx->conda.conda_packages, "conda", "conda_packages")) {
+        return -1;
+    }
+
     ctx->conda.conda_packages_purge = ini_getval_strlist(ini, "conda", "conda_packages_purge", LINE_SEP, render_mode, &err);
     normalize_ini_list(&ini, &ctx->conda.conda_packages_purge, "conda", "conda_package_purge", render_mode);
+
     ctx->conda.pip_packages = ini_getval_strlist(ini, "conda", "pip_packages", LINE_SEP, render_mode, &err);
     normalize_ini_list(&ini, &ctx->conda.pip_packages, "conda", "pip_packages", render_mode);
+    if (check_package_spec_list(ctx->conda.pip_packages, "conda", "pip_packages")) {
+        return -1;
+    }
+
     ctx->conda.pip_packages_purge = ini_getval_strlist(ini, "conda", "pip_packages_purge", LINE_SEP, render_mode, &err);
     normalize_ini_list(&ini, &ctx->conda.pip_packages_purge, "conda", "pip_packages_purge", render_mode);
 
diff --git a/src/lib/delivery/include/delivery.h b/src/lib/delivery/include/delivery.h
index 3103a86..7d846ab 100644
--- a/src/lib/delivery/include/delivery.h
+++ b/src/lib/delivery/include/delivery.h
@@ -10,6 +10,7 @@
 #include "environment.h"
 #include "ini.h"
 #include "multiprocessing.h"
+#include "conda.h"
 
 #define DELIVERY_PLATFORM_MAX 4
 #define DELIVERY_PLATFORM_MAXLEN 65
@@ -157,6 +158,7 @@ struct Delivery {
         struct StrList *pip_packages_defer;     ///< Python packages to be built for delivery
         struct StrList *pip_packages_purge;     ///< Python packages to remove from a delivery (for: based_on)
         struct StrList *wheels_packages;        ///< Wheel packages built for delivery
+        struct CondaCapabilities capabilities; ///< Capability information
     } conda;
 
     /*! \struct Runtime
@@ -354,9 +356,8 @@ void delivery_defer_packages(struct Delivery *ctx, int type);
 /**
  * Configure and activate a Conda installation based on Delivery context
  * @param ctx pointer to Delivery context
- * @param conda_install_dir path to Conda installation
  */
-void delivery_conda_enable(struct Delivery *ctx, char *conda_install_dir);
+void delivery_conda_enable(struct Delivery *ctx);
 
 /**
  * Install Conda