Merge pull request #80 from jhunkeler/fix-index-junitxml-report

Fix index junitxml report

6 files changed, 117 insertions, 127 deletions

diff --git a/src/cli/stasis_indexer/callbacks.c b/src/cli/stasis_indexer/callbacks.c
index 0186e1c..603aef9 100644
--- a/src/cli/stasis_indexer/callbacks.c
+++ b/src/cli/stasis_indexer/callbacks.c
@@ -25,13 +25,15 @@ int callback_sort_deliveries_cmpfn(const void *a, const void *b) {
 int callback_sort_deliveries_dynamic_cmpfn(const void *a, const void *b) {
     const struct Delivery *delivery1 = a;
     const size_t delivery1_python = strtoul(delivery1->meta.python_compact, NULL, 10);
+    const int delivery1_rc = delivery1->meta.rc;
     const struct Delivery *delivery2 = b;
     const size_t delivery2_python = strtoul(delivery2->meta.python_compact, NULL, 10);
+    const int delivery2_rc = delivery2->meta.rc;
 
-    if (delivery2_python > delivery1_python) {
+    if (delivery2_python > delivery1_python && delivery2_rc > delivery1_rc) {
         return 1;
     }
-    if (delivery2_python < delivery1_python) {
+    if (delivery2_python < delivery1_python && delivery2_rc < delivery1_rc) {
         return -1;
     }
     return 0;
diff --git a/src/cli/stasis_indexer/junitxml_report.c b/src/cli/stasis_indexer/junitxml_report.c
index b9d185c..65ec4f5 100644
--- a/src/cli/stasis_indexer/junitxml_report.c
+++ b/src/cli/stasis_indexer/junitxml_report.c
@@ -7,14 +7,85 @@
 #include "junitxml.h"
 #include "junitxml_report.h"
 
-int indexer_junitxml_report(struct Delivery ctx[], const size_t nelem) {
-    struct Delivery *latest = get_latest_deliveries(ctx, nelem);
-    if (!latest) {
-        return -1;
+static int is_file_in_listing(struct StrList *list, const char *pattern) {
+    for (size_t i = 0; i < strlist_count(list); i++) {
+        char const *path = strlist_item(list, i);
+        if (!fnmatch(pattern, path, 0)) {
+            return 1;
+        }
     }
-    size_t latest_count;
-    ARRAY_COUNT_BY_STRUCT_MEMBER(latest, meta.name, latest_count);
+    return 0;
+}
 
+static int write_report_output(struct Delivery *ctx, FILE *destfp, const char *xmlfilename) {
+    struct JUNIT_Testsuite *testsuite = junitxml_testsuite_read(xmlfilename);
+    if (testsuite) {
+        if (globals.verbose) {
+            printf("%s: duration: %0.4f, total: %d, passed: %d, failed: %d, skipped: %d, errors: %d\n", xmlfilename,
+                   testsuite->time, testsuite->tests,
+                   testsuite->passed, testsuite->failures,
+                   testsuite->skipped, testsuite->errors);
+        }
+
+        char *bname_tmp = strdup(xmlfilename);
+        char *bname = path_basename(bname_tmp);
+        bname[strlen(bname) - 4] = 0;
+        guard_free(bname_tmp);
+
+        char result_outfile[PATH_MAX] = {0};
+        char *short_name_pattern = NULL;
+        asprintf(&short_name_pattern, "-%s", ctx->info.release_name);
+
+        char short_name[PATH_MAX] = {0};
+        strncpy(short_name, bname, sizeof(short_name) - 1);
+        replace_text(short_name, short_name_pattern, "", 0);
+        replace_text(short_name, "results-", "", 0);
+        guard_free(short_name_pattern);
+
+        fprintf(destfp, "|[%s](%s.html)|%0.4f|%d|%d|%d|%d|%d|\n", short_name, bname,
+                testsuite->time, testsuite->tests,
+                testsuite->passed, testsuite->failures,
+                testsuite->skipped, testsuite->errors);
+
+        snprintf(result_outfile, sizeof(result_outfile) - strlen(bname) - 3, "%s.md",
+                 bname);
+        FILE *resultfp = fopen(result_outfile, "w+");
+        if (!resultfp) {
+            SYSERROR("Unable to open %s for writing", result_outfile);
+            return -1;
+        }
+
+        for (size_t i = 0; i < testsuite->_tc_inuse; i++) {
+            const char *type_str = NULL;
+            const int state = testsuite->testcase[i]->tc_result_state_type;
+            const char *message = NULL;
+            if (state == JUNIT_RESULT_STATE_FAILURE) {
+                message = testsuite->testcase[i]->result_state.failure->message;
+                type_str = "[FAILED]";
+            } else if (state == JUNIT_RESULT_STATE_ERROR) {
+                message = testsuite->testcase[i]->result_state.error->message;
+                type_str = "[ERROR]";
+            } else if (state == JUNIT_RESULT_STATE_SKIPPED) {
+                message = testsuite->testcase[i]->result_state.skipped->message;
+                type_str = "[SKIPPED]";
+            } else {
+                message = testsuite->testcase[i]->message ? testsuite->testcase[i]->message : "";
+                type_str = "[PASSED]";
+            }
+            fprintf(resultfp, "### %s %s :: %s\n", type_str,
+                    testsuite->testcase[i]->classname, testsuite->testcase[i]->name);
+            fprintf(resultfp, "\nDuration: %0.04fs\n", testsuite->testcase[i]->time);
+            fprintf(resultfp, "\n```\n%s\n```\n\n", message);
+        }
+        junitxml_testsuite_free(&testsuite);
+        fclose(resultfp);
+    } else {
+        fprintf(stderr, "bad test suite: %s: %s\n", strerror(errno), xmlfilename);
+    }
+    return 0;
+}
+
+int indexer_junitxml_report(struct Delivery ctx[], const size_t nelem) {
     char indexfile[PATH_MAX] = {0};
     sprintf(indexfile, "%s/README.md", ctx->storage.results_dir);
 
@@ -30,130 +101,42 @@ int indexer_junitxml_report(struct Delivery ctx[], const size_t nelem) {
             fprintf(stderr, "Unable to open %s for writing\n", indexfile);
             return -1;
         }
-
-        struct StrList *archs = get_architectures(latest, nelem);
-        struct StrList *platforms = get_platforms(latest, nelem);
-        qsort(latest, latest_count, sizeof(*latest), callback_sort_deliveries_dynamic_cmpfn);
-        fprintf(indexfp, "# %s-%s Test Report\n\n", ctx->meta.name, ctx->meta.version);
-        size_t no_printable_data = 0;
-
-        size_t delivery_count = get_latest_rc(latest, latest_count);
-            for (size_t p = 0; p < strlist_count(platforms); p++) {
-                char *platform = strlist_item(platforms, p);
-                for (size_t a = 0; a < strlist_count(archs); a++) {
-                    char *arch = strlist_item(archs, a);
-
-                    fprintf(indexfp, "## %s-%s\n\n", platform, arch);
-                    for (size_t d = 0; d < delivery_count; d++) {
-                        struct Delivery *current = &ctx[d];
-                        if (current->meta.rc == (int) d + 1
-                            && strcmp(current->system.arch, arch) != 0
-                            && strcmp(current->system.platform[DELIVERY_PLATFORM_RELEASE], platform) != 0) {
-                            continue;
-                        }
-
-                        fprintf(indexfp, "### %s\n", current->info.release_name);
-                        fprintf(indexfp, "\n|Suite|Duration|Fail    |Skip |Error |\n");
-                        fprintf(indexfp, "|:----|:------:|:------:|:---:|:----:|\n");
-                        for (size_t f = 0; f < strlist_count(file_listing); f++) {
-                            char *filename = strlist_item(file_listing, f);
-                            if (!endswith(filename, ".xml")) {
-                                continue;
-                            }
-
-                            char pattern[PATH_MAX] = {0};
-                            snprintf(pattern, sizeof(pattern) - 1, "*%s*", current->info.release_name);
-                            if (!fnmatch(pattern, filename, 0) && strstr(filename, platform) &&
-                                strstr(filename, arch)) {
-                                struct JUNIT_Testsuite *testsuite = junitxml_testsuite_read(filename);
-                                if (testsuite) {
-                                    if (globals.verbose) {
-                                        printf("%s: duration: %0.4f, failed: %d, skipped: %d, errors: %d\n", filename,
-                                               testsuite->time, testsuite->failures, testsuite->skipped,
-                                               testsuite->errors);
-                                    }
-
-                                    char *bname_tmp = strdup(filename);
-                                    char *bname = path_basename(bname_tmp);
-                                    bname[strlen(bname) - 4] = 0;
-                                    guard_free(bname_tmp);
-
-                                    char result_outfile[PATH_MAX] = {0};
-                                    char *short_name_pattern = NULL;
-                                    asprintf(&short_name_pattern, "-%s", current->info.release_name);
-
-                                    char short_name[PATH_MAX] = {0};
-                                    strncpy(short_name, bname, sizeof(short_name) - 1);
-                                    replace_text(short_name, short_name_pattern, "", 0);
-                                    replace_text(short_name, "results-", "", 0);
-                                    guard_free(short_name_pattern);
-
-                                    fprintf(indexfp, "|[%s](%s.html)|%0.4f|%d|%d|%d|\n", short_name,
-                                            bname,
-                                            testsuite->time, testsuite->failures, testsuite->skipped,
-                                            testsuite->errors);
-
-                                    snprintf(result_outfile, sizeof(result_outfile) - strlen(bname) - 3, "%s.md",
-                                             bname);
-                                    FILE *resultfp = fopen(result_outfile, "w+");
-                                    if (!resultfp) {
-                                        SYSERROR("Unable to open %s for writing", result_outfile);
-                                        return -1;
-                                    }
-
-                                    for (size_t i = 0; i < testsuite->_tc_inuse; i++) {
-                                        if (testsuite->testcase[i]->tc_result_state_type) {
-                                            const char *type_str = NULL;
-                                            const int state = testsuite->testcase[i]->tc_result_state_type;
-                                            const char *message = NULL;
-                                            if (state == JUNIT_RESULT_STATE_FAILURE) {
-                                                message = testsuite->testcase[i]->result_state.failure->message;
-                                                type_str = "[FAILED]";
-                                            } else if (state == JUNIT_RESULT_STATE_ERROR) {
-                                                message = testsuite->testcase[i]->result_state.error->message;
-                                                type_str = "[ERROR]";
-                                            } else if (state == JUNIT_RESULT_STATE_SKIPPED) {
-                                                message = testsuite->testcase[i]->result_state.skipped->message;
-                                                type_str = "[SKIPPED]";
-                                            }
-                                            fprintf(resultfp, "### %s %s :: %s\n", type_str,
-                                                    testsuite->testcase[i]->classname, testsuite->testcase[i]->name);
-                                            fprintf(resultfp, "\nDuration: %0.04fs\n", testsuite->testcase[i]->time);
-                                            fprintf(resultfp, "\n```\n%s\n```\n", message);
-                                        }
-                                    }
-                                    junitxml_testsuite_free(&testsuite);
-                                    fclose(resultfp);
-                                } else {
-                                    fprintf(stderr, "bad test suite: %s: %s\n", strerror(errno), filename);
-                                }
-                            } else {
-                                if (!no_printable_data) {
-                                    // Triggering for reasons unknown
-                                    //fprintf(indexfp, "|No data|-|-|-|-|-|-|\n");
-                                    no_printable_data = 1;
-                                }
-                            }
-                        }
+        printf("index %s opened for writing", indexfile);
+
+        for (size_t d = 0; d < nelem; d++) {
+            char pattern[PATH_MAX] = {0};
+            snprintf(pattern, sizeof(pattern) - 1, "*%s*", ctx[d].info.release_name);
+
+            // if result directory contains this release name, print it
+            fprintf(indexfp, "### %s\n", ctx[d].info.release_name);
+            if (!is_file_in_listing(file_listing, pattern)) {
+                fprintf(indexfp, "No test results\n");
+                continue;
+            }
+            fprintf(indexfp, "\n|Suite|Duration|Total|Pass|Fail|Skip|Error|\n");
+            fprintf(indexfp, "|:----|:------:|:---:|:--:|:--:|:--:|:---:|\n");
+
+            for (size_t i = 0; i < strlist_count(file_listing); i++) {
+                const char *filename = strlist_item(file_listing, i);
+                // if not a xml file, skip it
+                if (!endswith(filename, ".xml")) {
+                    continue;
+                }
+                if (!fnmatch(pattern, filename, 0)) {
+                    if (write_report_output(&ctx[d], indexfp, filename)) {
+                        // warn only
+                        SYSERROR("Unable to write xml report file using %s", filename);
                     }
-                    fprintf(indexfp, "\n");
-                    no_printable_data = 0;
                 }
-                fprintf(indexfp, "\n");
+            }
+            fprintf(indexfp, "\n");
         }
-        guard_strlist_free(&archs);
-        guard_strlist_free(&platforms);
         fclose(indexfp);
         popd();
     } else {
         fprintf(stderr, "Unable to enter delivery directory: %s\n", ctx->storage.delivery_dir);
-        guard_free(latest);
         return -1;
     }
 
-    // "latest" is an array of pointers to ctxs[]. Do not free the contents of the array.
-    guard_free(latest);
     return 0;
 }
-
-
diff --git a/src/cli/stasis_indexer/readmes.c b/src/cli/stasis_indexer/readmes.c
index 77b5178..7daf261 100644
--- a/src/cli/stasis_indexer/readmes.c
+++ b/src/cli/stasis_indexer/readmes.c
@@ -53,10 +53,11 @@ int indexer_readmes(struct Delivery ctx[], const size_t nelem) {
                 sprintf(conf_name, "%s.ini", latest_deliveries[i].info.release_name);
                 sprintf(conf_name_relative, "../config/%s.ini", latest_deliveries[i].info.release_name);
                 if (strstr(link_name, platform) && strstr(link_name, arch)) {
-                    fprintf(indexfp, "- Info: [README](%s)\n", readme_name);
-                    fprintf(indexfp, "- Release: [Conda Environment YAML](%s)\n", link_name);
-                    fprintf(indexfp, "- Receipt: [STASIS input file](%s)\n", conf_name_relative);
-                    fprintf(indexfp, "- Docker: ");
+                    fprintf(indexfp, "- Python %s\n", latest_deliveries[i].meta.python);
+                    fprintf(indexfp, "  - Info: [README](%s)\n", readme_name);
+                    fprintf(indexfp, "  - Release: [Conda Environment YAML](%s)\n", link_name);
+                    fprintf(indexfp, "  - Receipt: [STASIS input file](%s)\n", conf_name_relative);
+                    fprintf(indexfp, "  - Docker: ");
                     struct StrList *docker_images = get_docker_images(&latest_deliveries[i], "");
                     if (docker_images
                         && strlist_count(docker_images)
diff --git a/src/lib/core/include/junitxml.h b/src/lib/core/include/junitxml.h
index 777ee27..a6fca19 100644
--- a/src/lib/core/include/junitxml.h
+++ b/src/lib/core/include/junitxml.h
@@ -68,6 +68,8 @@ struct JUNIT_Testsuite {
     int failures;
     /// Total number of skipped tests
     int skipped;
+    /// Total number of successful tests
+    int passed;
     /// Total number of tests
     int tests;
     /// Total duration in fractional seconds
diff --git a/src/lib/core/junitxml.c b/src/lib/core/junitxml.c
index 8ab231b..f747224 100644
--- a/src/lib/core/junitxml.c
+++ b/src/lib/core/junitxml.c
@@ -192,6 +192,7 @@ static int read_xml_data(xmlTextReaderPtr reader, struct JUNIT_Testsuite **tests
             (*testsuite)->testcase[cur_tc]->result_state.skipped = skipped;
         }
     }
+    (*testsuite)->passed = (*testsuite)->tests - (*testsuite)->failures - (*testsuite)->errors - (*testsuite)->skipped;
     guard_strlist_free(&attrs);
     return 0;
 }
diff --git a/src/lib/core/utils.c b/src/lib/core/utils.c
index 621831c..a248f58 100644
--- a/src/lib/core/utils.c
+++ b/src/lib/core/utils.c
@@ -801,6 +801,7 @@ struct StrList *listdir(const char *path) {
         strlist_append(&node, fullpath);
         guard_free(fullpath);
     }
+    strlist_sort(node, STASIS_SORT_ALPHA);
     closedir(dp);
     return node;
 }