Merge pull request #152 from jhunkeler/force-repeatable1.9.0

Implement --force-repeatable option

10 files changed, 185 insertions, 27 deletions

diff --git a/src/cli/stasis/args.c b/src/cli/stasis/args.c
index c1bf031..d4dec0c 100644
--- a/src/cli/stasis/args.c
+++ b/src/cli/stasis/args.c
@@ -17,6 +17,7 @@ struct option long_options[] = {
     {"overwrite", no_argument, 0, OPT_OVERWRITE},
     {"wheel-builder", required_argument, 0, OPT_WHEEL_BUILDER},
     {"wheel-builder-manylinux-image", required_argument, 0, OPT_WHEEL_BUILDER_MANYLINUX_IMAGE},
+    {"force-repeatable", no_argument, 0, OPT_FORCE_REPEATABLE},
     {"no-docker", no_argument, 0, OPT_NO_DOCKER},
     {"no-artifactory", no_argument, 0, OPT_NO_ARTIFACTORY},
     {"no-artifactory-build-info", no_argument, 0, OPT_NO_ARTIFACTORY_BUILD_INFO},
@@ -44,6 +45,7 @@ const char *long_options_help[] = {
     "Overwrite an existing release",
     "Wheel building backend (build, cibuildwheel, manylinux)",
     "Manylinux image name",
+    "Adapt package source(s) and settings to reduce changes",
     "Do not build docker images",
     "Do not upload artifacts to Artifactory",
     "Do not upload build info objects to Artifactory",
diff --git a/src/cli/stasis/include/args.h b/src/cli/stasis/include/args.h
index e789261..ecb20c3 100644
--- a/src/cli/stasis/include/args.h
+++ b/src/cli/stasis/include/args.h
@@ -21,6 +21,7 @@
 #define OPT_TASK_TIMEOUT 1013
 #define OPT_WHEEL_BUILDER 1014
 #define OPT_WHEEL_BUILDER_MANYLINUX_IMAGE 1015
+#define OPT_FORCE_REPEATABLE 1016
 
 extern struct option long_options[];
 void usage(char *progname);
diff --git a/src/cli/stasis/stasis_main.c b/src/cli/stasis/stasis_main.c
index c5c1f00..5a9f694 100644
--- a/src/cli/stasis/stasis_main.c
+++ b/src/cli/stasis/stasis_main.c
@@ -638,6 +638,9 @@ int main(const int argc, char *argv[]) {
             case OPT_WHEEL_BUILDER_MANYLINUX_IMAGE:
                 globals.wheel_builder_manylinux_image = strdup(optarg);
                 break;
+            case OPT_FORCE_REPEATABLE:
+                globals.force_repeatable = true;
+                break;
             case '?':
             default:
                 exit(1);
diff --git a/src/lib/core/include/core.h b/src/lib/core/include/core.h
index ed523be..e3e54c9 100644
--- a/src/lib/core/include/core.h
+++ b/src/lib/core/include/core.h
@@ -57,6 +57,7 @@ struct STASIS_GLOBAL {
     int task_timeout; ///!< Time in seconds before task is terminated
     char *wheel_builder; ///!< Backend to build wheels (build, cibuildwheel, manylinux)
     char *wheel_builder_manylinux_image; ///!< Image to use for a Manylinux build
+    bool force_repeatable; ///!< Reduces surface area of random changes between builds
     struct {
         char *tox_posargs;
         char *conda_reactivate;
diff --git a/src/lib/core/include/utils.h b/src/lib/core/include/utils.h
index e75995a..f9937b5 100644
--- a/src/lib/core/include/utils.h
+++ b/src/lib/core/include/utils.h
@@ -180,6 +180,15 @@ char *git_describe(const char *path);
 char *git_rev_parse(const char *path, char *args);
 
 /**
+ * Check if a file in a git repository is flagged with 'assumed-unchanged'.
+ *
+ * @param filename path to file
+ * @return -1 on error, 0 if
+ */
+
+int is_git_assumed_unchanged(char *filename);
+
+/**
  * Helper function to initialize simple STASIS internal path strings
  *
  * ```c
@@ -473,7 +482,7 @@ int in_ascii_range(char c, char lower, char upper);
 #define GIT_HASH_LEN 40
 int is_git_sha(char const *hash);
 
-int check_python_package_dependencies(const char *srcdir);
+int check_python_package_dependencies(const char *srcdir, struct StrList **out_files, struct StrList **out_matches);
 
 void seconds_to_human_readable(int v, char *result, size_t maxlen);
 
diff --git a/src/lib/core/utils.c b/src/lib/core/utils.c
index 462604d..eee8bec 100644
--- a/src/lib/core/utils.c
+++ b/src/lib/core/utils.c
@@ -1078,7 +1078,7 @@ static int read_vcs_records(const size_t line, char **data) {
     // no match, continue
     return 1;
 }
-int check_python_package_dependencies(const char *srcdir) {
+int check_python_package_dependencies(const char *srcdir, struct StrList **out_files, struct StrList **out_matches) {
     const char *configs[] = {
         "pyproject.toml",
         "setup.cfg",
@@ -1103,12 +1103,23 @@ int check_python_package_dependencies(const char *srcdir) {
         }
         const size_t count = strlist_count(data);
         if (count) {
-            printf("\nERROR: VCS requirement(s) detected in %s:\n", configfile);
+            if (out_files) {
+                strlist_append(out_files, (char *) configs[i]);
+            }
             for (size_t j = 0; j < count; j++) {
                 char *record = strlist_item(data, j);
                 lstrip(record);
                 strip(record);
-                printf("[%zu] %s\n", j, record);
+                char *match = substring_between(record, "\"\"");
+                if (!match) {
+                    SYSERROR("unable to allocate bytes for matched sub-string");
+                    guard_strlist_free(&data);
+                    return -1;
+                }
+                if (out_matches) {
+                    strlist_append(out_matches, match);
+                }
+                guard_free(match);
             }
             guard_strlist_free(&data);
             return 1;
@@ -1306,3 +1317,44 @@ int is_file_compressed(const char *filename) {
     return 0;
 }
 
+/**
+ * Check if a file in a git repository is flagged as 'assumed-unchanged'.
+ *
+ * @param filename path to file
+ * @return -1 on error, 0 if
+ */
+int is_git_assumed_unchanged(char *filename) {
+    char *cmd = NULL;
+    if (asprintf(&cmd, "git ls-files -v -- '%s'", filename) < 0) {
+        SYSERROR("unable to allocate memory for file path");
+        return -1;
+    }
+    int status = 0;
+    char *output = shell_output(cmd, &status);
+    if (!output) {
+        SYSERROR("unable to allocate memory for git ls-files output");
+        guard_free(cmd);
+        return -1;
+    }
+    if (status) {
+        SYSERROR("git ls-files command failed to run");
+        guard_free(cmd);
+        guard_free(output);
+        return -1;
+    }
+    guard_free(cmd);
+
+    char *token = NULL;
+    char *data = output;
+    while ((token = strsep(&data, LINE_SEP)) != NULL) {
+        char *bname = path_basename(filename);
+        // "h" indicates assume-unchanged has been applied (see: man git-ls-files and option '-v')
+        if (strstr(token, bname) && startswith(token, "h")) {
+            guard_free(output);
+            return 1;
+        }
+    }
+
+    guard_free(output);
+    return 0;
+}
diff --git a/src/lib/delivery/delivery.c b/src/lib/delivery/delivery.c
index d256681..befa55a 100644
--- a/src/lib/delivery/delivery.c
+++ b/src/lib/delivery/delivery.c
@@ -1,6 +1,109 @@
 #include "delivery.h"
 #include "conda.h"
 
+
+const char *DELIVERY_MESSAGES[] = {
+    "",
+    "To resolve, please replace all occurrences with standard package specs:\n" \
+        "\n" \
+        "    package==x.y.z\n" \
+        "    package>=x.y.z\n" \
+        "    package<=x.y.z\n" \
+        "    ...\n",
+    NULL
+};
+
+void delivery_autoresolve_vcs_urls(const char *srcdir) {
+    msg(STASIS_MSG_L3, "Checking for VCS requirements in build scripts... ");
+
+    struct StrList *setup_files = strlist_init();
+    if (!setup_files) {
+        SYSERROR("unable to to allocate bytes for setup file list");
+        exit(1);
+    }
+
+    struct StrList *matches = strlist_init();
+    if (!matches) {
+        SYSERROR("unable to allocate bytes for matches list");
+        guard_strlist_free(&setup_files);
+        exit(1);
+    }
+
+    const int dep_status = check_python_package_dependencies(srcdir, &setup_files, &matches);
+    if (dep_status) {
+        fprintf(stdout, STASIS_COLOR_RED "FOUND" STASIS_COLOR_RESET "\n");
+
+        if (strlist_count(setup_files) && strlist_count(matches)) {
+            for (size_t f = 0; f < strlist_count(setup_files); f++) {
+                const char *setup_file = strlist_item(setup_files, f);
+                for (size_t m = 0; m < strlist_count(matches); m++) {
+                    const char *match = strlist_item(matches, m);
+                    fprintf(stdout, "[%s] %s\n", setup_file, match);
+                }
+            }
+        }
+
+        if (globals.force_repeatable) {
+            SYSWARN("%s", DELIVERY_MESSAGES[DM_VCS_URL_IN_SETUP]);
+        } else {
+            // Having VCS URLs in a project's setup files is illegal
+            SYSERROR("%s", DELIVERY_MESSAGES[DM_VCS_URL_IN_SETUP]);
+            COE_CHECK_ABORT(true, "Unreproducible delivery");
+        }
+
+        // VCS records have been found, force_repeatable or COE is enabled, so we don't care about integrity
+        // Replace the records in the file(s)
+        if (strlist_count(setup_files) && strlist_count(matches)) {
+            SYSDEBUG("Replacing VCS URLs");
+            for (size_t f = 0; f < strlist_count(setup_files); f++) {
+                char gitcmd[PATH_MAX] = {0};
+                const char *setup_file = strlist_item(setup_files, f);
+
+                SYSDEBUG("Processing %s", setup_file);
+                if (is_git_assumed_unchanged((char *) setup_file)) {
+                    SYSDEBUG("%s is already marked assume-unchanged, skipping");
+                    continue;
+                }
+
+                SYSDEBUG("Setting assume-unchanged flag");
+                snprintf(gitcmd, sizeof(gitcmd), "git update-index --assume-unchanged -- %s", setup_file);
+                if (shell(NULL, gitcmd)) {
+                    SYSERROR("unable to set assume-unchanged flag on file: %s", setup_file);
+                    exit(1);
+                }
+
+                for (size_t m = 0; m < strlist_count(matches); m++) {
+                    const char *match = strlist_item(matches, m);
+                    // copy the original match string
+                    char *replacement = strdup(match);
+                    if (!replacement) {
+                        SYSERROR("unable to allocate bytes for replacement value");
+                        exit(1);
+                    }
+
+                    // Truncate the replacement string at the first space character or @ symbol
+                    char *stop = strpbrk(replacement, " @");
+                    SYSDEBUG("Found delimiter '%c'", *stop);
+
+                    if (stop) {
+                        *stop = '\0';
+                        printf("[%s] Replacing '%s' with '%s'\n", setup_file, match, replacement);
+                        if (file_replace_text(setup_file, match, replacement, 0)) {
+                            SYSERROR("%s: replacement failed", setup_file);
+                            exit(1);
+                        }
+                    }
+                    guard_free(replacement);
+                }
+            }
+        }
+    } else {
+        fprintf(stdout, STASIS_COLOR_GREEN "NONE" STASIS_COLOR_RESET "\n");
+    }
+    guard_strlist_free(&setup_files);
+    guard_strlist_free(&matches);
+}
+
 struct Delivery *delivery_duplicate(struct Delivery *ctx) {
     struct Delivery *result = calloc(1, sizeof(*result));
     if (!result) {
diff --git a/src/lib/delivery/delivery_build.c b/src/lib/delivery/delivery_build.c
index 7ea5b29..82c7824 100644
--- a/src/lib/delivery/delivery_build.c
+++ b/src/lib/delivery/delivery_build.c
@@ -448,17 +448,7 @@ struct StrList *delivery_build_wheels(struct Delivery *ctx) {
                     memset(dname, 0, sizeof(dname));
                     memset(outdir, 0, sizeof(outdir));
 
-                    const int dep_status = check_python_package_dependencies(".");
-                    if (dep_status) {
-                        SYSERROR("Please replace all occurrences above with standard package specs:\n"
-                                        "\n"
-                                        "    package==x.y.z\n"
-                                        "    package>=x.y.z\n"
-                                        "    package<=x.y.z\n"
-                                        "    ...\n"
-                                        "\n");
-                        COE_CHECK_ABORT(true, "Unreproducible delivery");
-                    }
+                    delivery_autoresolve_vcs_urls(".");
 
                     safe_strncpy(dname, ctx->tests->test[i]->name, sizeof(dname));
                     tolower_s(dname);
diff --git a/src/lib/delivery/delivery_test.c b/src/lib/delivery/delivery_test.c
index 4ea3b3d..d3f7a17 100644
--- a/src/lib/delivery/delivery_test.c
+++ b/src/lib/delivery/delivery_test.c
@@ -178,17 +178,7 @@ void delivery_tests_run(struct Delivery *ctx) {
             if (pushd(destdir)) {
                 COE_CHECK_ABORT(1, "Unable to enter repository directory");
             } else {
-                const int dep_status = check_python_package_dependencies(".");
-                if (dep_status) {
-                    SYSERROR("Please replace all occurrences above with standard package specs:\n"
-                                    "\n"
-                                    "    package==x.y.z\n"
-                                    "    package>=x.y.z\n"
-                                    "    package<=x.y.z\n"
-                                    "    ...\n"
-                                    "\n");
-                    COE_CHECK_ABORT(true, "Unreproducible delivery");
-                }
+                delivery_autoresolve_vcs_urls(".");
 
                 char *cmd = calloc(strlen(test->script) + STASIS_BUFSIZ, sizeof(*cmd));
                 if (!cmd) {
@@ -256,7 +246,6 @@ void delivery_tests_run(struct Delivery *ctx) {
                 guard_free(runner_cmd);
                 guard_free(cmd);
                 popd();
-
             }
         }
 
diff --git a/src/lib/delivery/include/delivery.h b/src/lib/delivery/include/delivery.h
index 7d846ab..822cc03 100644
--- a/src/lib/delivery/include/delivery.h
+++ b/src/lib/delivery/include/delivery.h
@@ -30,6 +30,12 @@
 #define DEFER_CONDA 0                       ///< Build conda packages
 #define DEFER_PIP 1                         ///< Build python packages
 
+enum {
+    DM_NONE = 0,
+    DM_VCS_URL_IN_SETUP,
+};
+extern const char *DELIVERY_MESSAGES[];  // see delivery.c
+
 struct Content {
     unsigned type;
     char *filename;
@@ -429,6 +435,8 @@ void validate_delivery_ini(struct INIFILE *ini);
 
 int filter_repo_tags(char *repo, struct StrList *patterns);
 
+void delivery_autoresolve_vcs_urls(const char *srcdir);
+
 #define DELIVERY_NOT_FOUND 0
 #define DELIVERY_FOUND 1
 /**