From 8e0934853a374eb340ee0dd96fd21a3d7c74183d Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 2 Jul 2020 10:50:16 -0400 Subject: Fixed up a few problems: * Add mkdirs() * Create the output directory if it does not exist * Use the basename of the input path(s) to create the output path(s) * Verify the program received enough input files to proceed --- splitfits.c | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 52 insertions(+), 5 deletions(-) diff --git a/splitfits.c b/splitfits.c index 7493ed9..dd0e6d3 100644 --- a/splitfits.c +++ b/splitfits.c @@ -4,12 +4,42 @@ #include #include #include +#include #define OP_SPLIT 0 #define OP_COMBINE 1 #define FITS_BLOCK 2880 #define FITS_RECORD 80 +static int mkdirs(const char *_path, mode_t mode) { + const char *delim; + char *parts; + char *token; + char path[PATH_MAX]; + int status; + + delim = "/"; + parts = strdup(_path); + if (parts == NULL) { + perror(parts); + exit(1); + } + + memset(path, '\0', PATH_MAX); + + status = -1; + token = NULL; + while ((token = strsep(&parts, delim)) != NULL) { + strcat(path, token); + strcat(path, delim); + status = mkdir(path, mode); + if (status < 0) { + break; + } + } + return status; +} + /** * Check for FITS header keyword in data block * @param block of memory of size FITS_BLOCK @@ -151,6 +181,8 @@ int split_file(const char *_filename, const char *dest) { FILE *fp_out; FILE *map_out; char outfile[PATH_MAX]; + char path[PATH_MAX]; + char filename[PATH_MAX]; char _mapfile[PATH_MAX]; char *mapfile; char *block; @@ -176,7 +208,8 @@ int split_file(const char *_filename, const char *dest) { } mapfile = _mapfile; - sprintf(mapfile, "%s/%s.part_map", dest ? dest : ".", _filename); + strcpy(filename, _filename); + sprintf(mapfile, "%s/%s.part_map", dest ? dest : ".", get_basename(filename)); block_size = FITS_BLOCK; dataFrame = dataframe_init(); @@ -241,8 +274,6 @@ int split_file(const char *_filename, const char *dest) { // Read offset from the input files and write it to its respective .part_N file for (off = 0; off < dataFrame->num_inuse; off++) { - char path[PATH_MAX]; - char filename[PATH_MAX]; char *ext; // Get dirname of input path @@ -263,7 +294,7 @@ int split_file(const char *_filename, const char *dest) { } // Create output file name - sprintf(outfile, "%s/%s", path, filename); + sprintf(outfile, "%s/%s", path, get_basename(filename)); // Strip file extension from output file if ((ext = strrchr(outfile, '.')) == NULL) { @@ -485,8 +516,17 @@ int main(int argc, char *argv[]) { exit(0); } else if (strcmp(argv[inputs], "-o") == 0 || strcmp(argv[inputs], "--outdir") == 0) { inputs++; + if (argv[inputs] == NULL) { + fprintf(stderr, "-o requires an argument\n"); + usage(prog); + exit(1); + } if (access(argv[inputs], R_OK | W_OK | X_OK) != 0) { - fprintf(stderr, "%s: output directory does not exist or is not writable\n", argv[inputs]); + printf("Creating output directory: %s\n", argv[inputs]); + if (mkdirs(argv[inputs], 0755) < 0) { + perror(outdir); + exit(1); + } } outdir = strdup(argv[inputs]); } else if (strcmp(argv[inputs], "-c") == 0 || strcmp(argv[inputs], "--combine") == 0) { @@ -498,6 +538,13 @@ int main(int argc, char *argv[]) { } } + // Make sure we have at least one file to process + if (argc - inputs == 0) { + fprintf(stderr, "At least one input FILE is required\n"); + usage(prog); + exit(1); + } + // Make sure all input files exist bad_files = 0; for (size_t i = inputs; i < argc; i++) { -- cgit From 5803dc0fdb911a67ad9ca1ef9600edc58331d55a Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 2 Jul 2020 10:53:55 -0400 Subject: Add basic test framework and a few tests --- CMakeLists.txt | 6 ++ test.sh | 184 ++++++++++++++++++++++++++++++++++++++++++++++++++++ tests/test_split.sh | 35 ++++++++++ tests/test_usage.sh | 9 +++ 4 files changed, 234 insertions(+) create mode 100755 test.sh create mode 100644 tests/test_split.sh create mode 100644 tests/test_usage.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index ea12358..5287d06 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,5 +5,11 @@ set(CMAKE_C_STANDARD 99) add_executable(splitfits splitfits.c) +file(GLOB TEST_FRAMEWORK "tests") +file(COPY "test.sh" DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) +file(COPY ${TEST_FRAMEWORK} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) + +add_custom_target(check "${CMAKE_CURRENT_BINARY_DIR}/test.sh") + install(TARGETS splitfits RUNTIME) diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..56f6abb --- /dev/null +++ b/test.sh @@ -0,0 +1,184 @@ +#!/usr/bin/env bash +set -o pipefail + +RTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +test_program=${RTDIR}/splitfits +test_data=${RTDIR}/data +test_data_remote=https://nx.astroconda.org/repository +test_data_remote_auth=${test_data_remote_auth:-} +test_data_upload=generic/spb-splitfits/$(git describe --always) + +get_data_exists() { + local url + local response + + url="${test_data_remote}/$1" + if [[ -z $url ]]; then + echo "get_data_exists: requires a path relative to $test_data_remote" + exit 1 + fi + + response=$(curl -s --head "$url" | head -n 1 | awk '{print $2}') + if (( $response != 200 )); then + return 1; + fi + return 0; +} + +get_data() { + local url + local filename + + if [[ ! -d "${test_data}" ]]; then + mkdir -p "${test_data}" + fi + + url="${test_data_remote}/$1" + if [[ -z $url ]]; then + echo "get_data: requires a path relative to $test_data_remote" + exit 1 + fi + + filename=$(basename $url) + if ! get_data_exists "$1"; then + echo "${url}: not found on remote server" >&2 + return 1 + fi + + if ! (cd $test_data && curl -L -O "$url"); then + echo "Could not download data" >&2 + return 1 + fi + + echo "$test_data/$filename"; +} + +put_data() { + local url + local filename + + url="${test_data_remote}/$2" + filename="$1" + + if [[ -z $url ]]; then + echo "put_data: requires a path relative to $test_data_remote" + return 1 + fi + + if ! curl -s --user "${test_data_remote_auth}" --upload-file "${filename}" "${url}/${filename}"; then + echo "Could not upload '${filename}' to '${url}'" >&2 + return 1 + fi + + return 0 +} + +put_result() { + local src; + local dest; + src="$1" + dest="$2" + + if [[ ! -d "$src" ]]; then + echo "push_result: source directory does not exist: ${src}" >&2 + return 1 + fi + + if [[ -z "$dest" ]]; then + echo "push_result: requires a destination relative to ${test_data_remote}" >&2 + return 1 + fi + + for f in $(find "${src}" -type f); do + echo "Uploading '$f' to '$dest'" + if ! put_data "$f" "${dest}"; then + echo "Failed uploading '$src' to '$dest'" >&2 + return 1 + fi + done; + return 0 +} + +read_tests() { + for f in "${RTDIR}"/tests/test_*; do + source "$f" || return 0 + done + return 1 +} + +export_tests() { + tests=( + $(declare -f | cut -d ' ' -f 1 | grep -E '^test_') + ) + total_passed=0 + total_failed=0 + total_tests=${#tests[@]} +} + +run_tests() { + for (( i=0; i < ${total_tests}; i++ )); do + just_failed=0 + test_case="${tests[$i]}" + rm -rf "${test_case}" + mkdir -p "${test_case}" + pushd "${test_case}" &>/dev/null + log_file="output_${test_case}.log" + + /bin/echo -n -e "[$i] ${test_case}... " + "$test_case" 2>&1 | tee ${log_file} &>/dev/null + retval=$? + + if [[ "$retval" -ne "0" ]]; then + echo "FAILED ($retval)" + (( just_failed++ )) + (( total_failed++ )) + else + echo "PASSED" + (( total_passed++ )) + fi + + if [[ ${just_failed} -ne 0 ]] && [[ -s ${log_file} ]]; then + echo "# OUTPUT:" + cat "${log_file}" + fi + popd &>/dev/null + done + + # Upload all artifacts in all test directories + if [[ -n "$test_data_remote_auth" ]]; then + echo + for (( i=0; i < ${total_tests}; i++ )); do + test_case="${tests[i]}" + put_result ${test_case} ${test_data_upload} + done + fi +} + +check_runtime() { + if [[ ! -f "${test_program}" ]]; then + echo "Tests cannot run without: ${test_program}" >&2 + exit 1 + fi +} + +show_stats() { + printf "\n%d test(s): %d passed, %d failed\n" ${total_tests} ${total_passed} ${total_failed} +} + +# main program +check_runtime + +if read_tests; then + echo "Failed to aggregate tests." >&2 + exit 1 +fi + +export_tests +run_tests +show_stats + +if (( total_failed )); then + exit 1 +fi + +exit 0 diff --git a/tests/test_split.sh b/tests/test_split.sh new file mode 100644 index 0000000..141e320 --- /dev/null +++ b/tests/test_split.sh @@ -0,0 +1,35 @@ +test_splitfits_data1() { + local datafile + local retval + + retval=0 + datafile=$(get_data generic/fits/sample.fits) + [[ ! -f ${datafile} ]] && return 1 + + if ! ${test_program} -o ${test_case} ${datafile}; then + return 1 + fi + + set +x + return $retval +} + +test_splitfits_combine_data1() { + local datafile + local retval + + retval=0 + datafile=$(get_data generic/fits/sample.fits) + [[ ! -f ${datafile} ]] && return 1 + + if ! ${test_program} -o ${test_case} ${datafile}; then + return 1 + fi + + if ! ${test_program} -c ${test_case}/$(basename ${datafile}).part_map; then + return 1 + fi + + set +x + return $retval +} diff --git a/tests/test_usage.sh b/tests/test_usage.sh new file mode 100644 index 0000000..2a257d9 --- /dev/null +++ b/tests/test_usage.sh @@ -0,0 +1,9 @@ +test_splitfits_usage_longopt() { + ${test_program} --help + return $? +} + +test_splitfits_usage_shortopt() { + ${test_program} -h + return $? +} -- cgit From 19416faaede8cbbb4802417de858e9d7563c9e7d Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 2 Jul 2020 10:54:36 -0400 Subject: Add circleci config --- .circleci/config.yml | 34 ++++++++++++++++++++++++++++++++++ CMakeLists.txt | 4 ++-- 2 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..e412a11 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,34 @@ +version: 2 +jobs: + build: + machine: + image: ubuntu-1604:202004-01 + + steps: + - checkout + - run: | + set -xe + mkdir build && cd build + cmake .. + make + + test: + machine: + image: ubuntu-1604:202004-01 + + steps: + - checkout + - run: | + set -xe + mkdir build && cd build + cmake .. + make + make check + +workflows: + version: 2 + + build-and-test: + jobs: + - build + - test diff --git a/CMakeLists.txt b/CMakeLists.txt index 5287d06..0f3a91d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.16) +cmake_minimum_required(VERSION 3.1...3.16) project(splitfits C) set(CMAKE_C_STANDARD 99) @@ -12,4 +12,4 @@ file(COPY ${TEST_FRAMEWORK} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) add_custom_target(check "${CMAKE_CURRENT_BINARY_DIR}/test.sh") install(TARGETS splitfits - RUNTIME) + RUNTIME DESTINATION bin) -- cgit From 675e46d1f1ff672ebe6f8903f4b249db2456ede4 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 2 Jul 2020 12:27:51 -0400 Subject: Update README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index e4ff981..066c951 100644 --- a/README.md +++ b/README.md @@ -13,15 +13,19 @@ CFITSIO is a full-featured library and has quite a few system dependencies. `spl $ mkdir build $ cd build $ cmake .. -DCMAKE_INSTALL_PREFIX=/usr/local +$ make +$ make check $ make install ``` ## Without cmake ```sh $ cc -o splitfits splitfits.c +$ ./test.sh $ install -m 755 splitfits /usr/local/bin ``` + # Usage ``` -- cgit From 0fff9db32efe3bfaeb561843e604024d08cea1b0 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 2 Jul 2020 12:32:50 -0400 Subject: Use better git describe arguments --- test.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test.sh b/test.sh index 56f6abb..48216ff 100755 --- a/test.sh +++ b/test.sh @@ -2,11 +2,12 @@ set -o pipefail RTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -test_program=${RTDIR}/splitfits -test_data=${RTDIR}/data +test_program="${RTDIR}/splitfits" +test_program_version="$(git describe --always --tags --long)" +test_data="${RTDIR}/data" test_data_remote=https://nx.astroconda.org/repository test_data_remote_auth=${test_data_remote_auth:-} -test_data_upload=generic/spb-splitfits/$(git describe --always) +test_data_upload="generic/spb-splitfits/${test_program_version}" get_data_exists() { local url -- cgit From 5dd87260434756f3c974a6c75d373645d84f7b30 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 2 Jul 2020 12:37:28 -0400 Subject: Break up configuration from test script --- CMakeLists.txt | 3 ++- test.sh | 8 ++------ test_config.sh | 6 ++++++ 3 files changed, 10 insertions(+), 7 deletions(-) create mode 100644 test_config.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 0f3a91d..db56622 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,8 +5,9 @@ set(CMAKE_C_STANDARD 99) add_executable(splitfits splitfits.c) +file(GLOB TEST_RUNNER "test*.sh") +file(COPY ${TEST_RUNNER} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) file(GLOB TEST_FRAMEWORK "tests") -file(COPY "test.sh" DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) file(COPY ${TEST_FRAMEWORK} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) add_custom_target(check "${CMAKE_CURRENT_BINARY_DIR}/test.sh") diff --git a/test.sh b/test.sh index 48216ff..99a78de 100755 --- a/test.sh +++ b/test.sh @@ -2,12 +2,8 @@ set -o pipefail RTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -test_program="${RTDIR}/splitfits" -test_program_version="$(git describe --always --tags --long)" -test_data="${RTDIR}/data" -test_data_remote=https://nx.astroconda.org/repository -test_data_remote_auth=${test_data_remote_auth:-} -test_data_upload="generic/spb-splitfits/${test_program_version}" + +source "${RTDIR}"/test_config.sh get_data_exists() { local url diff --git a/test_config.sh b/test_config.sh new file mode 100644 index 0000000..da541b3 --- /dev/null +++ b/test_config.sh @@ -0,0 +1,6 @@ +test_program="${RTDIR}/splitfits" +test_program_version="$(git describe --always --tags --long)" +test_data="${RTDIR}/data" +test_data_remote=https://nx.astroconda.org/repository +test_data_remote_auth=${test_data_remote_auth:-} +test_data_upload="generic/spb-splitfits/${test_program_version}" -- cgit From 0c6b6035c3318e95e5d417639a3d7854cce72464 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 2 Jul 2020 14:22:11 -0400 Subject: Add version information --- CMakeLists.txt | 15 ++++++++++++++- splitfits.c | 6 ++++++ version.h.in | 6 ++++++ 3 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 version.h.in diff --git a/CMakeLists.txt b/CMakeLists.txt index db56622..43f002c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,15 +1,28 @@ cmake_minimum_required(VERSION 3.1...3.16) project(splitfits C) - set(CMAKE_C_STANDARD 99) +find_package(Git) +if(GIT_FOUND AND EXISTS "${CMAKE_SOURCE_DIR}/.git") + execute_process(COMMAND ${GIT_EXECUTABLE} describe --always --long --tags --dirty + OUTPUT_VARIABLE GIT_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE) +else() + message(WARNING "git not found or directory is not a repository") + set(GIT_VERSION, "unknown") +endif() + add_executable(splitfits splitfits.c) +include_directories("${CMAKE_CURRENT_BINARY_DIR}") +configure_file("${CMAKE_CURRENT_SOURCE_DIR}/version.h.in" + "${CMAKE_CURRENT_BINARY_DIR}/version.h" @ONLY) file(GLOB TEST_RUNNER "test*.sh") file(COPY ${TEST_RUNNER} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) file(GLOB TEST_FRAMEWORK "tests") file(COPY ${TEST_FRAMEWORK} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) + add_custom_target(check "${CMAKE_CURRENT_BINARY_DIR}/test.sh") install(TARGETS splitfits diff --git a/splitfits.c b/splitfits.c index dd0e6d3..6347c4a 100644 --- a/splitfits.c +++ b/splitfits.c @@ -5,6 +5,7 @@ #include #include #include +#include "version.h" #define OP_SPLIT 0 #define OP_COMBINE 1 @@ -486,6 +487,7 @@ void usage(char *program_name) { printf(" -h --help This message\n"); printf(" -c --combine Reconstruct original file using .part_map data\n"); printf(" -o --outdir Path where output files are stored\n"); + printf(" -V --version Display version\n"); } int main(int argc, char *argv[]) { @@ -532,6 +534,10 @@ int main(int argc, char *argv[]) { } else if (strcmp(argv[inputs], "-c") == 0 || strcmp(argv[inputs], "--combine") == 0) { // User wants to reconstruct a FITS file using a .part_map op_mode = OP_COMBINE; + } else if (strcmp(argv[inputs], "-V") == 0 || strcmp(argv[inputs], "--version") == 0) { + // Dump version and exit + puts(SPLITFITS_VERSION); + exit(0); } else { // Arguments beyond this point are considered input file paths break; diff --git a/version.h.in b/version.h.in new file mode 100644 index 0000000..581156e --- /dev/null +++ b/version.h.in @@ -0,0 +1,6 @@ +#ifndef SPLITFITS_VERSION_H +#define SPLITFITS_VERSION_H + +#define SPLITFITS_VERSION "@GIT_VERSION@" + +#endif -- cgit From 288ce22e256f71ba01cea63aa03a4880e5b1612d Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Thu, 2 Jul 2020 14:59:46 -0400 Subject: Incorporate circleci into artifact upload --- test_config.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test_config.sh b/test_config.sh index da541b3..7746a6d 100644 --- a/test_config.sh +++ b/test_config.sh @@ -3,4 +3,9 @@ test_program_version="$(git describe --always --tags --long)" test_data="${RTDIR}/data" test_data_remote=https://nx.astroconda.org/repository test_data_remote_auth=${test_data_remote_auth:-} -test_data_upload="generic/spb-splitfits/${test_program_version}" +test_data_upload="generic/spb-splitfits" +if [[ $CIRCLECI == "true" ]]; then + test_data_upload="${test_data_upload}/ci/${CIRCLE_BRANCH}_${CIRCLE_JOB}/${CIRCLE_BUILD_NUM}/${test_program_version}" +else + test_data_upload="${test_data_upload}/user/${test_program_version}" +fi -- cgit