From f3268c7558b8679aef84659d98fc89eba9d82cc4 Mon Sep 17 00:00:00 2001 From: "James E.H. Turner" Date: Thu, 1 Sep 2016 10:47:13 -0300 Subject: Rename scripts consistently to reflect use of AstroConda-specific conventions. --- scripts/ac_build_iraf_pkg | 4 +- scripts/ac_check_iraf_build | 139 +++++++++++++++++++++++++++++++++++++ scripts/ac_iraf_defs | 21 ++++++ scripts/ac_update_extern_pkg | 161 +++++++++++++++++++++++++++++++++++++++++++ scripts/check_iraf_pkg_build | 139 ------------------------------------- scripts/iraf_defs | 21 ------ scripts/unmangle_interpreter | 2 +- scripts/update_extern_pkg | 161 ------------------------------------------- 8 files changed, 324 insertions(+), 324 deletions(-) create mode 100755 scripts/ac_check_iraf_build create mode 100644 scripts/ac_iraf_defs create mode 100755 scripts/ac_update_extern_pkg delete mode 100755 scripts/check_iraf_pkg_build delete mode 100644 scripts/iraf_defs delete mode 100755 scripts/update_extern_pkg diff --git a/scripts/ac_build_iraf_pkg b/scripts/ac_build_iraf_pkg index 8c8024b..767e60b 100755 --- a/scripts/ac_build_iraf_pkg +++ b/scripts/ac_build_iraf_pkg @@ -16,7 +16,7 @@ # common name definitions from the same place: script_dir=`dirname "$0"` script_dir=`cd "$script_dir" && pwd` # canonical version of path -. "$script_dir/iraf_defs" +. "$script_dir/ac_iraf_defs" # The parameters are effectively conda build variables, which must be set here: if [ -n "$1" -o -z "$PREFIX" -o -z "$RECIPE_DIR" -o -z "$PKG_NAME" ]; then @@ -75,7 +75,7 @@ for file_name in $ac_iraf_files; do done # Define the new package in the (temporary build-time) extern.pkg: -if ! "$script_dir/update_extern_pkg" "$PREFIX" "$pkg_name"; then +if ! "$script_dir/ac_update_extern_pkg" "$PREFIX" "$pkg_name"; then echo "ERROR: failed to update $extern_pkg" >&2 exit 1 fi diff --git a/scripts/ac_check_iraf_build b/scripts/ac_check_iraf_build new file mode 100755 index 0000000..aff6b92 --- /dev/null +++ b/scripts/ac_check_iraf_build @@ -0,0 +1,139 @@ +#!/bin/sh + +# Attempt to determine the success or failure of an IRAF package build (since +# mkpkg does not report a useful exit status) by checking the log for any +# unexpected patterns (assumed to be errors) and confirming whether all the +# binaries listed in an optional package manifest have been created. +# +# Status: 0 success +# 1 invocation error +# 2 probable error messages in log +# 4 build incomplete with respect to manifest +# 6 both of the above + +usage="Usage: ac_check_iraf_build PATH" + +# Source some common IRAF package defs (log_name etc.) from the same dir: +. `dirname "$0"`/ac_iraf_defs + +# Parse argument(s): +st=1 +unset pkg_path +while [ "$1" != "" ]; do + case "$1" in + -h) + st=0; pkg_path="" + break + ;; + -*) + st=1 + ;; + *) + if [ "$pkg_path" ]; then + st=1 + else + st=0; pkg_path=$1 + fi + ;; + esac + shift +done + +if [ $st != 0 -o -z "$pkg_path" ]; then + echo "$usage" >&2 + exit $st +fi + +# Add a trailing slash if needed (as per IRAF custom): +pkg_path=`echo "$pkg_path" | sed -e 's|/*$|/|'` + +# Make sure the specified path is readable: +if [ ! -d "$pkg_path" -o ! -r "$pkg_path" ]; then + echo "ERROR: cannot read dir: $pkg_path" >&2 + exit 1 +fi + +# Make sure we can find a package build log and a package-specific list of +# patterns to be ignored within it: +bld_log="${pkg_path}${log_name}" +if [ ! -r "$bld_log" ]; then + echo "ERROR: cannot read $bld_log" >&2 + st=1 +fi +pkg_patterns="${pkg_path}${pkg_patterns}" +if [ ! -r "$pkg_patterns" ]; then + echo "ERROR: cannot read $pkg_patterns" >&2 + st=1 +fi + +# Check for an optional manifest: +manifest="${pkg_path}${pkg_manifest}" +if [ ! -r "$manifest" ]; then + echo "Warning: no manifest in $pkg_path" >&2 + manifest="" +fi + +# Exit due to missing files above: +[ $st != 0 ] && exit 1 + +# The environment must already be configured for running/building IRAF: +if [ -z "$iraf" ]; then + echo "ERROR: must configure \$iraf first (eg. AstroConda"\ + "\"source activate iraf\")" >&2 + exit 1 +fi + +# Add a trailing slash if needed: +iraf=`echo $iraf | sed -e 's|/*$|/|'` + +# Check for master list of expected IRAF build log patterns (used in addition +# to the package-specific list): +iraf_patterns="$iraf/ur/mkpkg_patterns" +if [ ! -r "$iraf_patterns" ]; then + echo "ERROR: missing $iraf_patterns" >&2 + exit 1 +fi + +# Look for unexpected string patterns in the log file (NB. grep returns 1 when +# all lines are successfully filtered out): +err_log="${pkg_path}${err_log_name}" +if tr -d '\015' < "$bld_log" | egrep -v -f "$iraf_patterns" -f "$pkg_patterns" > "$err_log"; then + st=2 + echo "ERROR: unexpected build messages in $err_log" >&2 +else + rm -f "$err_log" +fi + +# Check the existence of each file listed in the manifest. Currently there is +# no support for architecture-specific manifests (as we had in Ureka), since we +# haven't actually used any to date. +if [ "$manifest" ]; then + + nmissing=0 + lmissing="" + + exec < $manifest + while read line + do + case "$line" in + '#'*|'') + continue + ;; + *) + if [ ! -r "${pkg_path}${line}" ]; then + nmissing=$((nmissing+1)) + lmissing="${lmissing} ${line}"$'\n' + fi + ;; + esac + done + + if [ $nmissing -gt 0 ]; then + echo "ERROR: $nmissing expected file(s) missing after installation:" >& 2 + echo "$lmissing" >& 2 + st=$((st+4)) + fi +fi + +exit $st + diff --git a/scripts/ac_iraf_defs b/scripts/ac_iraf_defs new file mode 100644 index 0000000..fe5aca8 --- /dev/null +++ b/scripts/ac_iraf_defs @@ -0,0 +1,21 @@ +# Some common IRAF-package name definitions for the sh scripts in this dir. +# (also see separate defs. in the python scripts for now) + +extern_dir="iraf_extern" +extern_pkg="extern.pkg" + +pkg_extpkg="ur_extern.pkg" +pkg_manifest="ur_manifest" +pkg_patterns="ur_mkpkg_patterns" +pkg_mkpkg_cmd="ur_mkpkg_cmd" +pkg_mkhelp_cmd="ur_mkhelp_cmd" +pkg_omit_list="numrec_list.txt" + +log_name="build_log" +err_log_name="build_log.errors" +help_log_name="help_log" + +# Files to copy from the recipe to the build (if present): +ac_iraf_files="$pkg_extpkg $pkg_manifest $pkg_patterns $pkg_mkpkg_cmd\ + $pkg_mkhelp_cmd $pkg_omit_list" + diff --git a/scripts/ac_update_extern_pkg b/scripts/ac_update_extern_pkg new file mode 100755 index 0000000..4243e19 --- /dev/null +++ b/scripts/ac_update_extern_pkg @@ -0,0 +1,161 @@ +#!/usr/bin/python +# +# Copyright(c) 2016 Association of Universities for Research in Astronomy, Inc. +# +# This script uses the OS Python to avoid Conda relocation/dependency problems +# (since it is run before package installation is complete). Both the LSB and +# MacOS define Python in /usr/bin as standard. + +import argparse +import os, os.path +import shutil +import re +from collections import OrderedDict + + +# AstroConda naming conventions: +extern_pkg = 'extern.pkg' +backup = 'extern.pkg.ac_bak' # avoid possible collision with user .bak +extern_dir = 'iraf_extern' +template = 'ur_extern.pkg' +path_sub = 'UR_VDIR' # a regexp + + +# Get command-line user arguments: +parser = argparse.ArgumentParser() + +parser.add_argument('path', type=str, + help='path to {0} and {1}/'.format(extern_pkg, extern_dir)) + +parser.add_argument('name', type=str, + help='name of IRAF package (and its directory)') + +args = parser.parse_args() + +# Convert to canonical path and ensure it exists, with an extern.pkg file and +# an iraf_extern/package/ur_extern.pkg (ie. files already installed): +path = os.path.abspath(args.path) # env dir with extern.pkg +extpkg_path = os.path.join(path, extern_pkg) # extern.pkg path +pkg_path = os.path.join(path, extern_dir, args.name) # path to this IRAF pkg +template_path = os.path.join(pkg_path, template) # this ur_extern.pkg + +if not os.access(extpkg_path, os.R_OK | os.W_OK): + raise ValueError("cannot access {0}".format(extpkg_path)) + +if not os.access(template_path, os.R_OK): + raise ValueError("cannot access {0}".format(template_path)) + +# Read extern.pkg: +with open(extpkg_path, 'r+') as extpkg_file: + buffer = extpkg_file.read() + +# Read the package's template ur_extern.pkg, removing any outer blank lines: +with open(template_path, 'r') as template_file: + tempbuf = template_file.read().strip() + +# Here we convert the template entry to a regex for finding and removing old, +# previously-generated entries for the same package. This mainly consists of +# escaping special characters to match literally and allowing for a new path, +# but we also allow for ignoring some trivial whitespace differences. Any other +# user-modified entries are deliberately preserved, but still get overridden +# when adding the new package entry at the end. We don't use re.escape() here +# because it's easier & more understandable to do those subs in concert with +# the others. The template ur_extern.pkg files must not contain backslashes. + +replacements = OrderedDict(( + ('special_nospace', (r'([][.^${}\|*?])', r'\\\1')), + ('special_sp_end', (r'[ \t]*([=+()])[ \t]*$', r'[ \t]*\\\1')), + ('special_space', (r'[ \t]*([=+()])[ \t]*', r'[ \t]*\\\1[ \t]*')), + ('lead_space', (r'^[ \t]+', r'')), + ('end_space', (r'[ \t]+$', r'')), + ('extra_space', (r'[ \t]+', r'[ \t]+')), + ('line_space', (r'\s*\n\s*', r'\s*\n\s*')), # see below + ('path_sub', (path_sub, r'.*')), +)) + +# Note that ^ and $ only match in conjunction with their adjoining character, +# which, unlike the \n above, can get consumed by a prior pattern. Also, +# certain other combinations of \s and other space seem to cause "catastrophic +# backtracking" when there's no match, which can take 11 minutes! + +# Concatenate individual expressions for replacement, so they can be done +# simultaneously on the original buffer (otherwise they trip each other up): +cat_re = '|'.join(['(?P<{0}>{1})'.format(key, regex) \ + for key, (regex, sub) in replacements.items()]) # py 3 way + +# Determine the replacement for a given match to the concatenated regex: +def match_fn(match): + sub_re, sub = replacements[match.lastgroup] # look up replacement tuple + return re.sub(sub_re, sub, match.group()) # do & return the substitution + +# Perform the substitutions to turn the template string into a regex itself: +# template_re = re.escape(tempbuf) # if using built-in escape for special chars +template_re = re.sub(cat_re, match_fn, tempbuf, flags=re.M) + +# Update the original template entry with our new path: +tempbuf = re.sub(path_sub, os.path.join(pkg_path, ''), tempbuf) # trailing / + +# print(template_re) # debug + +# Define expected "keep" line(s) at the end of extern.pkg: +keep_re = '(?:^[ \t]*(?:;\s*)*keep\s*)+\Z' +keep_str = ';\n\nkeep\n' + +# Temporarily remove any "keep" from the end of the buffer, before appending to +# it (and replacing with ";\n\nkeep\n", to avoid a CL parser bug): +match = re.search(keep_re, buffer, flags=re.M) +if match: + buffer = buffer[:match.start()] + +# Make sure that what goes before the "keep" ends with a new line: +buffer = re.sub('\n?\Z', '\n', buffer, flags=re.M) + +# Find the last entry in extern.pkg (if any) that matches the template: +match = None +for match in re.finditer(template_re, buffer, flags=re.M): + pass + +# Check for any other definition(s) of this package after the last one based +# on the template, requiring that our new definition be moved to the end of the +# file, to override the other defs. without actually losing the user's edits: +if match: + # The name of the path variable in our extern.pkg template entries must be + # the name of the IRAF package, otherwise we would have to parse the actual + # variable name from the template using these commented regexes and search + # for that, instead of the package name itself, to find redefinitions: + # pkgdef_re = '^[ \t]*task[ \t]+{0}[.]pkg[ \t]*=[ \t]*(.*)$'.format(name) + # pathvar_re = '["\']?(.*)[$].*[.]cl' # find path variable name in pkg def. + + # Match any non-commented instances of the package name, in the remainder + # of the buffer, that don't look like a substring of a longer name: + pkgname_re = '^[^\n#]*(?&2 - exit $st -fi - -# Add a trailing slash if needed (as per IRAF custom): -pkg_path=`echo "$pkg_path" | sed -e 's|/*$|/|'` - -# Make sure the specified path is readable: -if [ ! -d "$pkg_path" -o ! -r "$pkg_path" ]; then - echo "ERROR: cannot read dir: $pkg_path" >&2 - exit 1 -fi - -# Make sure we can find a package build log and a package-specific list of -# patterns to be ignored within it: -bld_log="${pkg_path}${log_name}" -if [ ! -r "$bld_log" ]; then - echo "ERROR: cannot read $bld_log" >&2 - st=1 -fi -pkg_patterns="${pkg_path}${pkg_patterns}" -if [ ! -r "$pkg_patterns" ]; then - echo "ERROR: cannot read $pkg_patterns" >&2 - st=1 -fi - -# Check for an optional manifest: -manifest="${pkg_path}${pkg_manifest}" -if [ ! -r "$manifest" ]; then - echo "Warning: no manifest in $pkg_path" >&2 - manifest="" -fi - -# Exit due to missing files above: -[ $st != 0 ] && exit 1 - -# The environment must already be configured for running/building IRAF: -if [ -z "$iraf" ]; then - echo "ERROR: must configure \$iraf first (eg. AstroConda"\ - "\"source activate iraf\")" >&2 - exit 1 -fi - -# Add a trailing slash if needed: -iraf=`echo $iraf | sed -e 's|/*$|/|'` - -# Check for master list of expected IRAF build log patterns (used in addition -# to the package-specific list): -iraf_patterns="$iraf/ur/mkpkg_patterns" -if [ ! -r "$iraf_patterns" ]; then - echo "ERROR: missing $iraf_patterns" >&2 - exit 1 -fi - -# Look for unexpected string patterns in the log file (NB. grep returns 1 when -# all lines are successfully filtered out): -err_log="${pkg_path}${err_log_name}" -if tr -d '\015' < "$bld_log" | egrep -v -f "$iraf_patterns" -f "$pkg_patterns" > "$err_log"; then - st=2 - echo "ERROR: unexpected build messages in $err_log" >&2 -else - rm -f "$err_log" -fi - -# Check the existence of each file listed in the manifest. Currently there is -# no support for architecture-specific manifests (as we had in Ureka), since we -# haven't actually used any to date. -if [ "$manifest" ]; then - - nmissing=0 - lmissing="" - - exec < $manifest - while read line - do - case "$line" in - '#'*|'') - continue - ;; - *) - if [ ! -r "${pkg_path}${line}" ]; then - nmissing=$((nmissing+1)) - lmissing="${lmissing} ${line}"$'\n' - fi - ;; - esac - done - - if [ $nmissing -gt 0 ]; then - echo "ERROR: $nmissing expected file(s) missing after installation:" >& 2 - echo "$lmissing" >& 2 - st=$((st+4)) - fi -fi - -exit $st - diff --git a/scripts/iraf_defs b/scripts/iraf_defs deleted file mode 100644 index fe5aca8..0000000 --- a/scripts/iraf_defs +++ /dev/null @@ -1,21 +0,0 @@ -# Some common IRAF-package name definitions for the sh scripts in this dir. -# (also see separate defs. in the python scripts for now) - -extern_dir="iraf_extern" -extern_pkg="extern.pkg" - -pkg_extpkg="ur_extern.pkg" -pkg_manifest="ur_manifest" -pkg_patterns="ur_mkpkg_patterns" -pkg_mkpkg_cmd="ur_mkpkg_cmd" -pkg_mkhelp_cmd="ur_mkhelp_cmd" -pkg_omit_list="numrec_list.txt" - -log_name="build_log" -err_log_name="build_log.errors" -help_log_name="help_log" - -# Files to copy from the recipe to the build (if present): -ac_iraf_files="$pkg_extpkg $pkg_manifest $pkg_patterns $pkg_mkpkg_cmd\ - $pkg_mkhelp_cmd $pkg_omit_list" - diff --git a/scripts/unmangle_interpreter b/scripts/unmangle_interpreter index d060ecb..bdb7128 100755 --- a/scripts/unmangle_interpreter +++ b/scripts/unmangle_interpreter @@ -1,7 +1,7 @@ #!/bin/sh # Names of the script(s) to be updated in the same directory this is run from: -scripts="update_extern_pkg" +scripts="ac_update_extern_pkg" # Work in the directory where this script and those to be updated live: cd `dirname "$0"` || exit 1 # (should never fail if this script can be executed) diff --git a/scripts/update_extern_pkg b/scripts/update_extern_pkg deleted file mode 100755 index 4243e19..0000000 --- a/scripts/update_extern_pkg +++ /dev/null @@ -1,161 +0,0 @@ -#!/usr/bin/python -# -# Copyright(c) 2016 Association of Universities for Research in Astronomy, Inc. -# -# This script uses the OS Python to avoid Conda relocation/dependency problems -# (since it is run before package installation is complete). Both the LSB and -# MacOS define Python in /usr/bin as standard. - -import argparse -import os, os.path -import shutil -import re -from collections import OrderedDict - - -# AstroConda naming conventions: -extern_pkg = 'extern.pkg' -backup = 'extern.pkg.ac_bak' # avoid possible collision with user .bak -extern_dir = 'iraf_extern' -template = 'ur_extern.pkg' -path_sub = 'UR_VDIR' # a regexp - - -# Get command-line user arguments: -parser = argparse.ArgumentParser() - -parser.add_argument('path', type=str, - help='path to {0} and {1}/'.format(extern_pkg, extern_dir)) - -parser.add_argument('name', type=str, - help='name of IRAF package (and its directory)') - -args = parser.parse_args() - -# Convert to canonical path and ensure it exists, with an extern.pkg file and -# an iraf_extern/package/ur_extern.pkg (ie. files already installed): -path = os.path.abspath(args.path) # env dir with extern.pkg -extpkg_path = os.path.join(path, extern_pkg) # extern.pkg path -pkg_path = os.path.join(path, extern_dir, args.name) # path to this IRAF pkg -template_path = os.path.join(pkg_path, template) # this ur_extern.pkg - -if not os.access(extpkg_path, os.R_OK | os.W_OK): - raise ValueError("cannot access {0}".format(extpkg_path)) - -if not os.access(template_path, os.R_OK): - raise ValueError("cannot access {0}".format(template_path)) - -# Read extern.pkg: -with open(extpkg_path, 'r+') as extpkg_file: - buffer = extpkg_file.read() - -# Read the package's template ur_extern.pkg, removing any outer blank lines: -with open(template_path, 'r') as template_file: - tempbuf = template_file.read().strip() - -# Here we convert the template entry to a regex for finding and removing old, -# previously-generated entries for the same package. This mainly consists of -# escaping special characters to match literally and allowing for a new path, -# but we also allow for ignoring some trivial whitespace differences. Any other -# user-modified entries are deliberately preserved, but still get overridden -# when adding the new package entry at the end. We don't use re.escape() here -# because it's easier & more understandable to do those subs in concert with -# the others. The template ur_extern.pkg files must not contain backslashes. - -replacements = OrderedDict(( - ('special_nospace', (r'([][.^${}\|*?])', r'\\\1')), - ('special_sp_end', (r'[ \t]*([=+()])[ \t]*$', r'[ \t]*\\\1')), - ('special_space', (r'[ \t]*([=+()])[ \t]*', r'[ \t]*\\\1[ \t]*')), - ('lead_space', (r'^[ \t]+', r'')), - ('end_space', (r'[ \t]+$', r'')), - ('extra_space', (r'[ \t]+', r'[ \t]+')), - ('line_space', (r'\s*\n\s*', r'\s*\n\s*')), # see below - ('path_sub', (path_sub, r'.*')), -)) - -# Note that ^ and $ only match in conjunction with their adjoining character, -# which, unlike the \n above, can get consumed by a prior pattern. Also, -# certain other combinations of \s and other space seem to cause "catastrophic -# backtracking" when there's no match, which can take 11 minutes! - -# Concatenate individual expressions for replacement, so they can be done -# simultaneously on the original buffer (otherwise they trip each other up): -cat_re = '|'.join(['(?P<{0}>{1})'.format(key, regex) \ - for key, (regex, sub) in replacements.items()]) # py 3 way - -# Determine the replacement for a given match to the concatenated regex: -def match_fn(match): - sub_re, sub = replacements[match.lastgroup] # look up replacement tuple - return re.sub(sub_re, sub, match.group()) # do & return the substitution - -# Perform the substitutions to turn the template string into a regex itself: -# template_re = re.escape(tempbuf) # if using built-in escape for special chars -template_re = re.sub(cat_re, match_fn, tempbuf, flags=re.M) - -# Update the original template entry with our new path: -tempbuf = re.sub(path_sub, os.path.join(pkg_path, ''), tempbuf) # trailing / - -# print(template_re) # debug - -# Define expected "keep" line(s) at the end of extern.pkg: -keep_re = '(?:^[ \t]*(?:;\s*)*keep\s*)+\Z' -keep_str = ';\n\nkeep\n' - -# Temporarily remove any "keep" from the end of the buffer, before appending to -# it (and replacing with ";\n\nkeep\n", to avoid a CL parser bug): -match = re.search(keep_re, buffer, flags=re.M) -if match: - buffer = buffer[:match.start()] - -# Make sure that what goes before the "keep" ends with a new line: -buffer = re.sub('\n?\Z', '\n', buffer, flags=re.M) - -# Find the last entry in extern.pkg (if any) that matches the template: -match = None -for match in re.finditer(template_re, buffer, flags=re.M): - pass - -# Check for any other definition(s) of this package after the last one based -# on the template, requiring that our new definition be moved to the end of the -# file, to override the other defs. without actually losing the user's edits: -if match: - # The name of the path variable in our extern.pkg template entries must be - # the name of the IRAF package, otherwise we would have to parse the actual - # variable name from the template using these commented regexes and search - # for that, instead of the package name itself, to find redefinitions: - # pkgdef_re = '^[ \t]*task[ \t]+{0}[.]pkg[ \t]*=[ \t]*(.*)$'.format(name) - # pathvar_re = '["\']?(.*)[$].*[.]cl' # find path variable name in pkg def. - - # Match any non-commented instances of the package name, in the remainder - # of the buffer, that don't look like a substring of a longer name: - pkgname_re = '^[^\n#]*(?