From 4361cdd8c038c258683fc266554bf8c4f05fa0d6 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Mon, 28 Oct 2024 13:20:33 -0400 Subject: Add environment setup script to test directory --- tests/setup.sh | 186 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 186 insertions(+) create mode 100644 tests/setup.sh (limited to 'tests') diff --git a/tests/setup.sh b/tests/setup.sh new file mode 100644 index 0000000..78c710d --- /dev/null +++ b/tests/setup.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +set -o pipefail + +export LOGFILE_STASIS="stasis.log" +export LOGFILE_INDEXER="stasis_indexer.log" +export CHECK_OUTPUT_PATTERNS=() + +unset STASIS_SYSCONFDIR +if [ -n "$GITHUB_TOKEN" ] && [ -z "$STASIS_GH_TOKEN"]; then + export STASIS_GH_TOKEN="$GITHUB_TOKEN" +else + export STASIS_GH_TOKEN="anonymous" +fi + +if [[ -z "$PYTHON_VERSIONS" ]]; then + PYTHON_VERSIONS=( + 3.10 + 3.11 + 3.12 + ) +fi + +setup_script_dir="$(dirname ${BASH_SOURCE[0]})" +TOPDIR=$(pwd) + +WS_DEFAULT=rt_workspace_ +setup_workspace() { + if [ -z "$1" ]; then + echo "setup_workspace requires a name argument" >&2 + return 1 + fi + WORKSPACE="${WS_DEFAULT}$1" + rm -rf "$WORKSPACE" + if ! mkdir -p "$WORKSPACE"; then + echo "directory creation failed. cannot continue" >&2 + return 1; + fi + WORKSPACE="$(realpath $WORKSPACE)" + + export PREFIX="$WORKSPACE"/local + if ! mkdir -p "$PREFIX"; then + echo "directory creation failed. cannot continue" >&2 + return 1; + fi + + export BUILD_DIR="$WORKSPACE"/build + if ! mkdir -p "$BUILD_DIR"; then + echo "directory creation failed. cannot continue" >&2 + return 1; + fi + + pushd "$WORKSPACE" + export LANG="C" + export HOME="$WORKSPACE" + . /etc/profile +} + +teardown_workspace() { + if [ -z "$1" ]; then + echo "teardown_workspace requires a workspace path" >&2 + return 1 + elif ! [[ ${WS_DEFAULT}$1 =~ ${WS_DEFAULT}.* ]]; then + echo "$1 is not a valid workspace" >&2 + return 1 + fi + popd + clean_up +} + +install_stasis() { + pushd "$BUILD_DIR" + if ! cmake -DCMAKE_INSTALL_PREFIX="$PREFIX" -DCMAKE_BUILD_TYPE=Debug "${TOPDIR}"/../..; then + echo "cmake failed" >&2 + return 1 + fi + + if ! make install; then + echo "make failed" >&2 + return 1 + fi + + export PATH="$PREFIX/bin:$PATH" + hash -r + if ! type -P stasis; then + echo "stasis program not on PATH" >&2 + return 1 + fi + + if ! type -P stasis_indexer; then + echo "stasis_indexer program not on PATH" >&2 + return 1 + fi + popd +} + +run_stasis() { + local logfile="$LOGFILE_STASIS" + $(type -P stasis) --unbuffered -v $@ 2>&1 | tee "$logfile" +} + +run_stasis_indexer() { + local logfile="$LOGFILE_INDEXER" + local root="$1" + if [ -z "$root" ]; then + echo "run_stasis_indexer root directory cannot be empty" >&2 + exit 1 + fi + $(type -P stasis_indexer) --web --unbuffered -v "$root"/* 2>&1 | tee "$logfile" +} + +check_output_add() { + local pattern="$1" + CHECK_OUTPUT_PATTERNS+=("$pattern") +} + +check_output_reset() { + CHECK_OUTPUT_PATTERNS=() +} + +check_output_stasis_dir() { + local startdir="$1" + local logfile="$LOGFILE_STASIS" + + echo "#### Files ####" + find $startdir | sort + echo + + echo "#### Contents ####" + files=$(find $startdir -type f \( -name "$logfile" -o -name '*.yml' -o -name '*.md' -o -name '*.stasis' -o -name '*.ini' \) | sort) + for x in $files; do + echo + echo "FILENAME: $x" + echo + if [ "$x" == "$logfile" ]; then + # do not print thousands of lines of output we _just_ sat through + echo "Output omitted" + else + cat "$x" + echo "[EOF]" + fi + echo + + for cond in "${CHECK_OUTPUT_PATTERNS[@]}"; do + if grep --color -H -n "$cond" "$x" >&2; then + echo "ERROR DETECTED IN $x!" >&2 + retcode+=1 + fi + done + done + + return $retcode +} + +check_output_indexed_dir() { + local retcode=0 + local startdir="$1" + local logfile="$2" + + echo "#### Files ####" + find $startdir | sort + + for cond in "${CHECK_OUTPUT_PATTERNS[@]}"; do + if grep --color -H -n "$cond" "$logfile" >&2; then + echo "ERROR DETECTED IN INDEX OPERATION!" >&2 + retcode+=1 + fi + done + + echo "#### Contents ####" + files=$(find $startdir -type f \( -name '*.html' \) | sort) + for x in $files; do + echo + echo "FILENAME: $x" + echo + cat "$x" + echo "[EOF]" + echo + done + return $retcode +} + +clean_up() { + if [ -z "$RT_KEEP_WORKSPACE" ] && [ -d "$WORKSPACE" ]; then + rm -rf "$WORKSPACE" + fi +} \ No newline at end of file -- cgit From 4f24068ea3319f3e156431753064527cb695995c Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Mon, 28 Oct 2024 13:21:11 -0400 Subject: rt_generic: Use setup script --- tests/rt_generic.sh | 135 +++++++++++----------------------------------------- 1 file changed, 27 insertions(+), 108 deletions(-) (limited to 'tests') diff --git a/tests/rt_generic.sh b/tests/rt_generic.sh index 6e4454c..d19a46b 100644 --- a/tests/rt_generic.sh +++ b/tests/rt_generic.sh @@ -1,110 +1,29 @@ -#!/usr/bin/env bash -set -x -unset STASIS_SYSCONFDIR -if [ -n "$GITHUB_TOKEN" ] && [ -z "$STASIS_GH_TOKEN"]; then - export STASIS_GH_TOKEN="$GITHUB_TOKEN" -else - export STASIS_GH_TOKEN="anonymous" -fi -python_versions=( - 3.10 - 3.11 - 3.12 -) - -topdir=$(pwd) - -ws="rt_workspace" -rm -rf "$ws" -mkdir -p "$ws" -ws="$(realpath $ws)" - -prefix="$ws"/local -mkdir -p "$prefix" - -bdir="$ws"/build -mkdir -p "$bdir" - -pushd "$bdir" -cmake -DCMAKE_INSTALL_PREFIX="$prefix" -DCMAKE_BUILD_TYPE=Debug "${topdir}"/../.. -make install -export PATH="$prefix/bin:$PATH" -popd - -pushd "$ws" - type -P stasis - type -P stasis_indexer - retcode=0 - - for py_version in "${python_versions[@]}"; do - stasis --python "$py_version" --no-docker --no-artifactory --unbuffered -v "$topdir"/generic.ini - retcode+=$? - done - - set +x +set -e - echo "#### Files ####" - find stasis/*/output | sort - echo +here="$(dirname ${BASH_SOURCE[0]})" +source $here/setup.sh - echo "#### Contents ####" - files=$(find stasis/*/output -type f \( -name '*.yml' -o -name '*.md' -o -name '*.stasis' -o -name '*.ini' \) | sort) - for x in $files; do - echo - echo "FILENAME: $x" - echo - cat "$x" - echo "[EOF]" - echo - - fail_on_main=( - "(null)" - ) - for cond in "${fail_on_main[@]}"; do - if grep --color -H -n "$cond" "$x" >&2; then - echo "ERROR DETECTED IN $x!" >&2 - retcode+=1 - fi - done - done - - # Something above failed, so drop out. Don't bother indexing. - # Don't clean up either. - (( retcode )) && exit $retcode - - fail_on_indexer=( - "(null)" - ) - logfile="stasis_indexer.log" - set -x - stasis_indexer --web --unbuffered -v stasis/* 2>&1 | tee "$logfile" - retcode=$? - - set +x - echo "#### Files ####" - find output - - for cond in "${fail_on_indexer[@]}"; do - if grep --color -H -n "$cond" "$logfile" >&2; then - echo "ERROR DETECTED IN INDEX OPERATION!" >&2 - exit 1 - fi - done - - echo "#### Contents ####" - files=$(find output -type f \( -name '*.html' \) | sort) - for x in $files; do - echo - echo "FILENAME: $x" - echo - cat "$x" - echo "[EOF]" - echo - done -popd - -if [ -z "$RT_KEEP_WORKSPACE" ]; then - rm -rf "$ws" -fi - -exit $retcode \ No newline at end of file +TEST_NAME=generic +PYTHON_VERSIONS=( + 3.11 +) +setup_workspace "$TEST_NAME" +install_stasis +for py_version in "${PYTHON_VERSIONS[@]}"; do + run_stasis --python "$py_version" \ + --no-docker \ + --no-artifactory \ + "$TOPDIR"/"$TEST_NAME".ini +done + +check_output_add "(null)" +check_output_stasis_dir stasis/*/output +check_output_reset + +# NOTE: indexer default output directory is "output" +check_output_add "(null)" +run_stasis_indexer stasis +check_output_indexed_dir output +check_output_reset + +teardown_workspace "$TEST_NAME" -- cgit From 87840f958af165e0d8ac65446c3d9a3a5266902e Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Mon, 28 Oct 2024 13:24:27 -0400 Subject: Add rt_generic_based_on.sh * Tests the "based_on" key --- tests/CMakeLists.txt | 3 ++ tests/data/generic_based_on.ini | 65 +++++++++++++++++++++++++++++++++++++++++ tests/data/generic_based_on.yml | 9 ++++++ tests/rt_generic_based_on.sh | 29 ++++++++++++++++++ 4 files changed, 106 insertions(+) create mode 100644 tests/data/generic_based_on.ini create mode 100644 tests/data/generic_based_on.yml create mode 100644 tests/rt_generic_based_on.sh (limited to 'tests') diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 0da290f..343f838 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -10,11 +10,14 @@ set(nix_clang_cflags -Wno-format-truncation -Wno-unused-parameter -Wno-unused-re set(win_msvc_cflags /Wall) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/data/generic.ini ${CMAKE_CURRENT_BINARY_DIR} COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/data/generic_based_on.ini ${CMAKE_CURRENT_BINARY_DIR} COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/data/generic_based_on.yml ${CMAKE_CURRENT_BINARY_DIR} COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/data/result.xml ${CMAKE_CURRENT_BINARY_DIR} COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/data/result_error.xml ${CMAKE_CURRENT_BINARY_DIR} COPYONLY) file(GLOB source_files "test_*.c") if (BASH_PROGRAM AND BUILD_TESTING_RT) + add_test (rt_generic_based_on ${BASH_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/rt_generic_based_on.sh) add_test (rt_generic ${BASH_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/rt_generic.sh) endif() diff --git a/tests/data/generic_based_on.ini b/tests/data/generic_based_on.ini new file mode 100644 index 0000000..9e6ac5d --- /dev/null +++ b/tests/data/generic_based_on.ini @@ -0,0 +1,65 @@ +[meta] +mission = generic +name = GENERIC +version = 1.2.3 +rc = 1 +final = false +based_on = generic_based_on.yml +python = 3.11 + + +[conda] +installer_name = Miniforge3 +installer_version = 24.3.0-0 +installer_platform = {{env:STASIS_CONDA_PLATFORM}} +installer_arch = {{env:STASIS_CONDA_ARCH}} +installer_baseurl = https://github.com/conda-forge/miniforge/releases/download/{{conda.installer_version}} +;conda_packages = +pip_packages = + firewatch==0.0.4 + tweakwcs==0.8.8 + + +[runtime] +CPPFLAGS = ${CPPFLAGS} -fpermissive +PYTHONUNBUFFERED = 1 + + +[test:firewatch] +repository = https://github.com/astroconda/firewatch +script_setup = + pip install -e '.' +script = + firewatch -c conda-forge -p ${STASIS_CONDA_PLATFORM_SUBDIR} | grep -E ' python-[0-9]' + + +[test:tweakwcs] +repository = https://github.com/spacetelescope/tweakwcs +script_setup = + pip install -e '.[test]' +script = + pytest \ + -r fEsx \ + --basetemp="{{ func:basetemp_dir() }}" \ + --junitxml="{{ func:junitxml_file() }}" + + +[deploy:artifactory:delivery] +files = + {{ storage.output_dir }}/** +dest = {{ meta.mission }}/{{ info.build_name }}/ + + +[deploy:docker] +registry = bytesalad.stsci.edu +image_compression = zstd -v -9 -c +build_args = + SNAPSHOT_INPUT={{ info.release_name }}.yml + SNAPSHOT_PKGDIR=packages +tags = + {{ meta.name }}:{{ info.build_number }}-py{{ meta.python_compact }} + {{ deploy.docker.registry }}/{{ meta.name }}:{{ info.build_number }}-py{{ meta.python_compact }} +test_script = + source /etc/profile + python -m pip freeze + mamba info diff --git a/tests/data/generic_based_on.yml b/tests/data/generic_based_on.yml new file mode 100644 index 0000000..3ab97a7 --- /dev/null +++ b/tests/data/generic_based_on.yml @@ -0,0 +1,9 @@ +channels: + - conda-forge +dependencies: + - pip + - python + - setuptools + - pip: + - firewatch==0.0.3 + - tweakwcs==0.8.7 \ No newline at end of file diff --git a/tests/rt_generic_based_on.sh b/tests/rt_generic_based_on.sh new file mode 100644 index 0000000..7ca11ce --- /dev/null +++ b/tests/rt_generic_based_on.sh @@ -0,0 +1,29 @@ +here="$(dirname ${BASH_SOURCE[0]})" +source $here/setup.sh + +TEST_NAME=generic_based_on +PYTHON_VERSIONS=( + 3.11 +) +setup_workspace "$TEST_NAME" +install_stasis + +ln -s "$TOPDIR"/"$TEST_NAME".yml +for py_version in "${PYTHON_VERSIONS[@]}"; do + run_stasis --python "$py_version" \ + --no-docker \ + --no-artifactory \ + "$TOPDIR"/"$TEST_NAME".ini +done + +check_output_add "(null)" +check_output_stasis_dir stasis/*/output +check_output_reset + +# NOTE: indexer default output directory is "output" +check_output_add "(null)" +run_stasis_indexer stasis +check_output_indexed_dir output +check_output_reset + +teardown_workspace "$TEST_NAME" -- cgit From 8fc617004e7db4a3dcc07516e96456eddb05d045 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Wed, 30 Oct 2024 09:15:05 -0400 Subject: Copy test data in bulk * Detect RT scripts * Generalize file extension pattern use --- tests/CMakeLists.txt | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'tests') diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 343f838..e35d88d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -9,20 +9,25 @@ set(nix_gnu_cflags -Wno-format-truncation -Wno-error -Wno-unused-parameter -Wno- set(nix_clang_cflags -Wno-format-truncation -Wno-unused-parameter -Wno-unused-result -Wno-incompatible-pointer-types-discards-qualifiers) set(win_msvc_cflags /Wall) -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/data/generic.ini ${CMAKE_CURRENT_BINARY_DIR} COPYONLY) -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/data/generic_based_on.ini ${CMAKE_CURRENT_BINARY_DIR} COPYONLY) -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/data/generic_based_on.yml ${CMAKE_CURRENT_BINARY_DIR} COPYONLY) -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/data/result.xml ${CMAKE_CURRENT_BINARY_DIR} COPYONLY) -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/data/result_error.xml ${CMAKE_CURRENT_BINARY_DIR} COPYONLY) file(GLOB source_files "test_*.c") +file(GLOB rt_files "rt_*.sh") +set(ext_pattern "(^.*/|\\.[^.]*$)") + +file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/data + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) if (BASH_PROGRAM AND BUILD_TESTING_RT) + foreach(rt_file ${rt_files}) + file(REAL_PATH ${rt_file} rt_name) + string(REGEX REPLACE ${ext_pattern} "" rt_name ${rt_file}) + add_test (${rt_name} ${BASH_PROGRAM} ${rt_file}) + endforeach() add_test (rt_generic_based_on ${BASH_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/rt_generic_based_on.sh) add_test (rt_generic ${BASH_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/rt_generic.sh) endif() foreach(source_file ${source_files}) - string(REGEX REPLACE "(^.*/|\\.[^.]*$)" "" test_executable ${source_file}) + string(REGEX REPLACE ${ext_pattern} "" test_executable ${source_file}) add_executable(${test_executable} ${source_file}) if (CMAKE_C_COMPILER_ID STREQUAL "GNU") target_compile_options(${test_executable} PRIVATE ${nix_cflags} ${nix_gnu_cflags}) -- cgit From ac564ea340771b6dbf10ebdb64d5fede55da342d Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Wed, 30 Oct 2024 09:15:46 -0400 Subject: generic_based_on.ini using TEST_DATA variable from the environment to find yaml config --- tests/data/generic_based_on.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/data/generic_based_on.ini b/tests/data/generic_based_on.ini index 9e6ac5d..1c993ea 100644 --- a/tests/data/generic_based_on.ini +++ b/tests/data/generic_based_on.ini @@ -4,7 +4,7 @@ name = GENERIC version = 1.2.3 rc = 1 final = false -based_on = generic_based_on.yml +based_on = {{ env:TEST_DATA }}/generic_based_on.yml python = 3.11 -- cgit From 9aa6d8c681bcf6d6c0df25b2d0624e6ee6abd3c5 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Wed, 30 Oct 2024 09:16:55 -0400 Subject: Exposes TOPDIR and TEST_DATA variables to test script(s) --- tests/setup.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/setup.sh b/tests/setup.sh index 78c710d..014f7fc 100644 --- a/tests/setup.sh +++ b/tests/setup.sh @@ -21,7 +21,8 @@ if [[ -z "$PYTHON_VERSIONS" ]]; then fi setup_script_dir="$(dirname ${BASH_SOURCE[0]})" -TOPDIR=$(pwd) +export TOPDIR=$(pwd) +export TEST_DATA="$TOPDIR"/data WS_DEFAULT=rt_workspace_ setup_workspace() { -- cgit From 7a77598e0355c8d1df97745f1678363f050fc607 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Wed, 30 Oct 2024 09:18:14 -0400 Subject: Implement a runner (run_command) and generate a summary (run_summary) when the script ends --- tests/setup.sh | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 72 insertions(+), 2 deletions(-) (limited to 'tests') diff --git a/tests/setup.sh b/tests/setup.sh index 014f7fc..743fafd 100644 --- a/tests/setup.sh +++ b/tests/setup.sh @@ -94,6 +94,36 @@ install_stasis() { popd } + +STASIS_TEST_RESULT_FAIL=0 +STASIS_TEST_RESULT_PASS=0 +STASIS_TEST_RESULT_SKIP=0 +run_command() { + local logfile="$(mktemp).log" + local cmd="${@}" + local lines_on_error=100 + /bin/echo "Testing: $cmd " + if ! $cmd &>"$logfile"; then + echo "... FAIL" + echo "#" + echo "# Last $lines_on_error line(s) follow:" + echo "#" + tail -n $lines_on_error "$logfile" + (( STASIS_TEST_RESULT_FAIL++ )) + else + echo "... PASS" + (( STASIS_TEST_RESULT_PASS++ )) + fi + rm -f "$logfile" +} + +run_summary() { + local total=$(( STASIS_TEST_RESULT_PASS + STASIS_TEST_RESULT_FAIL + STASIS_TEST_RESULT_SKIP)) + echo + echo "[RT] ${STASIS_TEST_RESULT_PASS} tests passed, ${STASIS_TEST_RESULT_FAIL} failed, ${STASIS_TEST_RESULT_SKIP} skipped out of ${total}" + echo +} + run_stasis() { local logfile="$LOGFILE_STASIS" $(type -P stasis) --unbuffered -v $@ 2>&1 | tee "$logfile" @@ -119,6 +149,7 @@ check_output_reset() { } check_output_stasis_dir() { + local retcode=0 local startdir="$1" local logfile="$LOGFILE_STASIS" @@ -149,7 +180,11 @@ check_output_stasis_dir() { done done - return $retcode + if (( retcode )); then + return 1 + else + return 0 + fi } check_output_indexed_dir() { @@ -177,11 +212,46 @@ check_output_indexed_dir() { echo "[EOF]" echo done - return $retcode + if (( retcode )); then + return 1 + else + return 0 + fi +} + +assert_eq() { + local a="$1" + local b="$2" + local msg="$3" + if [[ "$a" == "$b" ]]; then + return 0 + else + [[ -n "$msg" ]] && echo "'$a' != '$b' :: $msg" >&2 + return 1 + fi +} + +assert_file_contains() { + local file="$1" + local str="$2" + local msg="$3" + if grep -E "$str" "$file" &>/dev/null; then + return 0 + else + [[ -n "$msg" ]] && echo "'$str' not in file '$file' :: $msg" >&2 + return 1 + fi } clean_up() { if [ -z "$RT_KEEP_WORKSPACE" ] && [ -d "$WORKSPACE" ]; then rm -rf "$WORKSPACE" fi + + run_summary + if (( STASIS_TEST_RESULT_FAIL )); then + exit 1 + else + exit 0 + fi } \ No newline at end of file -- cgit From 5f29a25a76f9dfb06a97e921f1d428c683970831 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Wed, 30 Oct 2024 09:37:37 -0400 Subject: Update RT tests to use run_command function --- tests/rt_generic.sh | 20 +++++++++++--------- tests/rt_generic_based_on.sh | 15 ++++++++------- 2 files changed, 19 insertions(+), 16 deletions(-) (limited to 'tests') diff --git a/tests/rt_generic.sh b/tests/rt_generic.sh index d19a46b..674f6f0 100644 --- a/tests/rt_generic.sh +++ b/tests/rt_generic.sh @@ -1,5 +1,4 @@ -set -e - +#!/usr/bin/env bash here="$(dirname ${BASH_SOURCE[0]})" source $here/setup.sh @@ -7,23 +6,26 @@ TEST_NAME=generic PYTHON_VERSIONS=( 3.11 ) + setup_workspace "$TEST_NAME" -install_stasis +run_command install_stasis for py_version in "${PYTHON_VERSIONS[@]}"; do - run_stasis --python "$py_version" \ + run_command run_stasis --python "$py_version" \ --no-docker \ --no-artifactory \ - "$TOPDIR"/"$TEST_NAME".ini + "$TEST_DATA"/"$TEST_NAME".ini done check_output_add "(null)" -check_output_stasis_dir stasis/*/output +run_command check_output_stasis_dir stasis/*/output check_output_reset # NOTE: indexer default output directory is "output" check_output_add "(null)" -run_stasis_indexer stasis -check_output_indexed_dir output +run_command run_stasis_indexer stasis +run_command check_output_indexed_dir output check_output_reset -teardown_workspace "$TEST_NAME" +run_command assert_file_contains "$LOGFILE_STASIS" "USE EXTERNAL" "External packages should have been used" + +teardown_workspace "$TEST_NAME" \ No newline at end of file diff --git a/tests/rt_generic_based_on.sh b/tests/rt_generic_based_on.sh index 7ca11ce..7d78399 100644 --- a/tests/rt_generic_based_on.sh +++ b/tests/rt_generic_based_on.sh @@ -1,3 +1,4 @@ +#!/usr/bin/env bash here="$(dirname ${BASH_SOURCE[0]})" source $here/setup.sh @@ -6,24 +7,24 @@ PYTHON_VERSIONS=( 3.11 ) setup_workspace "$TEST_NAME" -install_stasis +run_command install_stasis -ln -s "$TOPDIR"/"$TEST_NAME".yml +ln -s "$TEST_DATA"/"$TEST_NAME".yml for py_version in "${PYTHON_VERSIONS[@]}"; do - run_stasis --python "$py_version" \ + run_command run_stasis --python "$py_version" \ --no-docker \ --no-artifactory \ - "$TOPDIR"/"$TEST_NAME".ini + "$TEST_DATA"/"$TEST_NAME".ini done check_output_add "(null)" -check_output_stasis_dir stasis/*/output +run_command check_output_stasis_dir stasis/*/output check_output_reset # NOTE: indexer default output directory is "output" check_output_add "(null)" -run_stasis_indexer stasis -check_output_indexed_dir output +run_command run_stasis_indexer stasis +run_command check_output_indexed_dir output check_output_reset teardown_workspace "$TEST_NAME" -- cgit From 5898e7d5bef98c2c7515fe23c4160ad3dbbaf6ad Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Wed, 30 Oct 2024 09:38:00 -0400 Subject: Update location of test data --- tests/test_junitxml.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'tests') diff --git a/tests/test_junitxml.c b/tests/test_junitxml.c index 7111249..e222b56 100644 --- a/tests/test_junitxml.c +++ b/tests/test_junitxml.c @@ -3,7 +3,7 @@ void test_junitxml_testsuite_read() { struct JUNIT_Testsuite *testsuite; - STASIS_ASSERT_FATAL((testsuite = junitxml_testsuite_read("result.xml")) != NULL, "failed to load testsuite data"); + STASIS_ASSERT_FATAL((testsuite = junitxml_testsuite_read("data/result.xml")) != NULL, "failed to load testsuite data"); STASIS_ASSERT(testsuite->name != NULL, "Test suite must be named"); STASIS_ASSERT(testsuite->skipped > 0, "missed skipped tests"); STASIS_ASSERT(testsuite->failures > 0, "missed failed tests"); @@ -44,7 +44,7 @@ void test_junitxml_testsuite_read() { void test_junitxml_testsuite_read_error() { struct JUNIT_Testsuite *testsuite; - STASIS_ASSERT_FATAL((testsuite = junitxml_testsuite_read("result_error.xml")) != NULL, "failed to load testsuite data"); + STASIS_ASSERT_FATAL((testsuite = junitxml_testsuite_read("data/result_error.xml")) != NULL, "failed to load testsuite data"); STASIS_ASSERT(testsuite->name != NULL, "test suite must be named"); STASIS_ASSERT(testsuite->skipped == 0, "should not have any skipped tests"); -- cgit From 252f9425fc9a5fc8a7e152772b7af67ff7869652 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Wed, 30 Oct 2024 10:48:51 -0400 Subject: Fixes download test * NGINX updated and changed the default error page source code * Look for "404" in the page contents --- tests/test_download.c | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) (limited to 'tests') diff --git a/tests/test_download.c b/tests/test_download.c index ad8724e..714e614 100644 --- a/tests/test_download.c +++ b/tests/test_download.c @@ -2,15 +2,20 @@ #include "download.h" void test_download() { + enum MATCH_STYLE { + match_begins=0, + match_contains, + }; struct testcase { const char *url; long http_code; + enum MATCH_STYLE style; const char *data; const char *errmsg; }; struct testcase tc[] = { - {.url = "https://ssb.stsci.edu/jhunk/stasis_test/test_download.txt", .http_code = 200L, .data = "It works!\n", .errmsg = NULL}, - {.url = "https://ssb.stsci.edu/jhunk/stasis_test/test_download.broken", .http_code = 404L, .data = "= 0) { STASIS_ASSERT(data != NULL, "data should not be null"); - STASIS_ASSERT(strncmp(data[0], tc[i].data, strlen(tc[i].data)) == 0, "data file does not match the expected contents"); + printf("FILE: %s\nLOCAL: '%s'\nREMOTE: '%s'\n", tc[i].url, data, tc[i].data); + if (tc[i].style == match_begins) { + STASIS_ASSERT(strncmp(data, tc[i].data, strlen(tc[i].data)) == 0, "data file does not begin with the expected contents"); + } else if (tc[i].style == match_contains) { + STASIS_ASSERT(strstr(data, tc[i].data) != NULL, "data file does not contain the expected contents"); + } else { + STASIS_ASSERT(false, "Unrecognized matching mode"); + } } else { STASIS_ASSERT(http_code == -1, "http_code should be -1 on fatal curl error"); STASIS_ASSERT(data == NULL, "data should be NULL on fatal curl error"); -- cgit From caa444b44f1ce1974bd9f46c68e45ff26d251bdf Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Wed, 30 Oct 2024 11:28:53 -0400 Subject: Implement missing SKIP conditions * The not (bang) overrides the return code. Split the command from the if-statement to obtain the real code, not just true/false * Print the last n lines of the log only when there's something to print --- tests/setup.sh | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) (limited to 'tests') diff --git a/tests/setup.sh b/tests/setup.sh index 743fafd..0875cac 100644 --- a/tests/setup.sh +++ b/tests/setup.sh @@ -103,13 +103,23 @@ run_command() { local cmd="${@}" local lines_on_error=100 /bin/echo "Testing: $cmd " - if ! $cmd &>"$logfile"; then - echo "... FAIL" - echo "#" - echo "# Last $lines_on_error line(s) follow:" - echo "#" - tail -n $lines_on_error "$logfile" - (( STASIS_TEST_RESULT_FAIL++ )) + + $cmd &>"$logfile" + code=$? + if (( code )); then + if (( code == 127 )); then + echo "... SKIP" + (( STASIS_TEST_RESULT_SKIP++ )) + else + echo "... FAIL" + if (( $(wc -c "$logfile" | cut -d ' ' -f 1) > 1 )); then + echo "#" + echo "# Last $lines_on_error line(s) follow:" + echo "#" + tail -n $lines_on_error "$logfile" + fi + (( STASIS_TEST_RESULT_FAIL++ )) + fi else echo "... PASS" (( STASIS_TEST_RESULT_PASS++ )) -- cgit From aee3c2d4089cfab235eaa66665ddf7dee76b2933 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Wed, 30 Oct 2024 11:33:38 -0400 Subject: Add RT boilerplate --- tests/_rt_boilerplate.sh | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 tests/_rt_boilerplate.sh (limited to 'tests') diff --git a/tests/_rt_boilerplate.sh b/tests/_rt_boilerplate.sh new file mode 100644 index 0000000..22a2688 --- /dev/null +++ b/tests/_rt_boilerplate.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +here="$(dirname ${BASH_SOURCE[0]})" +source $here/setup.sh + +TEST_NAME= + +#test_fails() { +# return 1 +#} +# +#test_passes() { +# return 0 +#} +# +#test_skips() { +# return 127 +#} + +setup_workspace "$TEST_NAME" + +#run_command test_fails +#run_command test_passes +#run_command test_skips +#run_command false # fails +#run_command true # passes + +teardown_workspace "$TEST_NAME" \ No newline at end of file -- cgit From dada1b364d99cdff627fba701aede00b713e61fd Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Wed, 30 Oct 2024 11:33:50 -0400 Subject: Add tests/README.md --- tests/README.md | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 tests/README.md (limited to 'tests') diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..bb1f896 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,48 @@ +# Testing STASIS + +## Unit tests + +Rules: + +* Use the boilerplate `_test_boilerplate.c` + * ```shell + cp _test_boilerplate.c test_mynewtest.c + ``` +* Test file names start with `test_` (`test_file.c`) +* Test functions start with `test_` (`void test_function()`) +* Test suites can be skipped by returning `127` from `main()` +* PASS, FAIL, and SKIP conditions are set by the assertion macros `STASIS_{ASSERT,ASSERT_FATAL,SKIP_IF}` +* Parametrized tests should implement an array of `struct testcase` + * The `testcase` structure should be local to each `test_` function + * The `testcase` variables are freeform + * Example: + ```c + void test_function() { + struct testcase { + int your; + int variables; + int here; + }; + struct testcase tc[] = { + {.your = 0, .variables = 1, .here = 2}, + {.your = 3, .variables = 4, .here = 5}, + // ... + }; + for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) { + // STASIS_ASSERT()s here + } + } + ``` + +## Regression test (RT) + +Rules: + +* Regression tests are shell scripts (BASH) +* Use the boilerplate `_rt_boilerplate.sh` + * ```shell + cp _rt_boilerplate.sh rt_mynewtest.sh + ``` +* Test file names start with `rt_` (`rt_file.sh`) +* Test function names are freeform, however they must be executed using `run_command()` (`run_command test_function`) +* Test suites can be skipped by returning `127` from the shell script \ No newline at end of file -- cgit From 3da6e513cc64990aa865613bd0bb9ba5d6624570 Mon Sep 17 00:00:00 2001 From: Joseph Hunkeler Date: Wed, 30 Oct 2024 12:11:20 -0400 Subject: Update _test_boilerplate.c --- tests/_test_boilerplate.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'tests') diff --git a/tests/_test_boilerplate.c b/tests/_test_boilerplate.c index 1d7734b..b13568d 100644 --- a/tests/_test_boilerplate.c +++ b/tests/_test_boilerplate.c @@ -1,16 +1,24 @@ #include "testing.h" +/* void test_NAME() { struct testcase { }; - STASIS_ASSERT(); + struct testcase tc[] = { + { // ... }, + } + + for (size_t i = 0; i < sizeof(tc) / sizeof(*tc); i++) { + // STASIS_ASSERT(); + } } + */ int main(int argc, char *argv[]) { STASIS_TEST_BEGIN_MAIN(); STASIS_TEST_FUNC *tests[] = { - test_NAME, + //test_NAME, }; STASIS_TEST_RUN(tests); STASIS_TEST_END_MAIN(); -- cgit