diff options
| -rw-r--r-- | source/app.d | 76 | ||||
| -rw-r--r-- | source/conda.d | 40 | ||||
| -rw-r--r-- | source/merge.d | 63 |
3 files changed, 132 insertions, 47 deletions
diff --git a/source/app.d b/source/app.d index 603ff5b..cbb90e3 100644 --- a/source/app.d +++ b/source/app.d @@ -3,43 +3,97 @@ import std.array; import std.format; import std.file; import std.typecons; +import std.path : buildPath, chainPath, absolutePath; import conda; import merge; int main(string[] args) { + import std.getopt; + string env_name; + string output_dir = "delivery"; + string installer_prefix = "miniconda"; + string installer_version = "4.5.12"; + bool run_tests = false; + string mergefile; + string base_spec; + + auto helpInformation = getopt( + args, + config.passThrough, + config.required, "env-name|n", "name of delivery", &env_name, + config.required, "dmfile|d", "delivery merge specification file", &mergefile, + "output-dir|o", "store delivery-related results in dir", &output_dir, + "install-prefix|p", "path to install miniconda", &installer_prefix, + "install-version|i", "version of miniconda installer", &installer_version, + "run-tests|R", "scan merged packages and execute their tests", &run_tests, + "base-spec", "conda explicit or yaml environment dump", &base_spec + ); + + if (helpInformation.helpWanted) { + defaultGetoptPrinter("Delivery merge [fill in the blanks]", + helpInformation.options); + return 0; + } + + installer_prefix = buildPath(installer_prefix).absolutePath; + output_dir = buildPath(output_dir, env_name).absolutePath; + mergefile = buildPath(mergefile).absolutePath; + + // Ingest the dump file via --base-spec or with a positional argument. + if (base_spec.empty && args.length > 1) { + base_spec = args[1]; + args.popBack(); + } + + /* + string optfmt = "env_name: %s\n" + ~ "output_dir: %s\n" + ~ "installer_prefix: %s\n" + ~ "installer_version: %s\n" + ~ "run_tests: %d\n" + ~ "mergefile: %s\n" + ~ "base_spec: %s\n" + ~ "ARGS: %s\n"; + writefln(optfmt, env_name, output_dir, installer_prefix, installer_version, + run_tests, mergefile, base_spec, args); + */ + Conda conda = new Conda(); conda.channels = [ "http://ssb.stsci.edu/astroconda", "defaults", "http://ssb.stsci.edu/astroconda-dev" ]; - conda.install_prefix = "/tmp/miniconda"; - conda.installer_version = "4.5.12"; + conda.install_prefix = installer_prefix; + conda.installer_version = installer_version; conda.installer_variant = "3"; if (!conda.installer()) { writeln("Installation failed."); return 1; } conda.initialize(); + if (!env_combine(conda, env_name, base_spec, mergefile)) { + writeln("Delivery merge failed. Adjust '*.dm' file to match constraints reported by the 'solver'."); + return 1; + } + + if (!output_dir.exists) { + output_dir.mkdirRecurse; + } + writeln(conda.dump_env_yaml(buildPath(output_dir, env_name ~ ".yml"))); + writeln(conda.dump_env_explicit(buildPath(output_dir, env_name ~ ".txt"))); + auto info = testable_packages(conda, "test.dm"); writeln(info); /* - env_combine(conda, - "delivery", - "https://raw.githubusercontent.com/astroconda/astroconda-releases/master/hstdp/2019.3/dev/hstdp-2019.3-linux-py36.02.txt", - "test.dm"); - conda.run("info"); - conda.run("list"); - */ - - /* conda.activate("base"); conda.run("info"); conda.sh("python --version"); conda.sh("python -c 'import sys; print(sys.path)'"); */ + /* conda.sh("git clone https://github.com/spacetelescope/tweakwcs"); auto project = "tweakwcs"; diff --git a/source/conda.d b/source/conda.d index 6fb7bd1..6dc66e6 100644 --- a/source/conda.d +++ b/source/conda.d @@ -147,7 +147,7 @@ class Conda { bool installer() { if (this.in_env() || this.install_prefix.exists) { - writefln("Conda is already installed: %s", this.install_prefix); + writefln("Miniconda is already installed: %s", this.install_prefix); return true; } else if (this.install_prefix.empty) { this.install_prefix = absolutePath("./miniconda"); @@ -156,22 +156,19 @@ class Conda { } if (this.have_installer()) { - writeln("Installer already exists"); + writeln("Miniconda installation script already exists"); } else { download(this.url_installer, this.installer_file()); } - auto installer = executeShell( + auto installer = this.sh( "bash " ~ this.installer_file() ~ " -b" ~ " -p " - ~ this.install_prefix, - env=this.env); + ~ this.install_prefix); - writeln(installer.output); - - if (installer.status != 0) { + if (installer != 0) { return false; } @@ -227,8 +224,8 @@ class Conda { return proc; } - auto sh_block(string command) { - auto proc = executeShell(command, env=this.env); + auto run_block(string command) { + auto proc = this.sh_block("conda " ~ command); return proc; } @@ -238,6 +235,11 @@ class Conda { return wait(proc); } + auto sh_block(string command) { + auto proc = executeShell(command, env=this.env); + return proc; + } + string multiarg(string flag, string[] arr) { return flag ~ " " ~ arr.join(" " ~ flag ~ " "); } @@ -257,4 +259,22 @@ class Conda { } return result; } + + string dump_env_yaml(string filename=null) { + string args; + if (filename !is null) { + args = "--file " ~ filename; + } + auto proc = this.run_block("env export " ~ args); + return proc.output; + } + + string dump_env_explicit(string filename=null) { + auto proc = this.run_block("list --explicit"); + if (filename !is null) { + auto file = File(filename, "w+"); + file.write(proc.output); + } + return proc.output; + } } diff --git a/source/merge.d b/source/merge.d index 4f3caea..e26ab73 100644 --- a/source/merge.d +++ b/source/merge.d @@ -44,9 +44,6 @@ string[string][] dmfile(string filename) { if (line.empty) { continue; } auto record = matchFirst(line, RE_DMFILE); - writefln("-> package: %-10s :: version: %-10s", record["name"], - !record["version"].empty ? record["version"] : "none"); - pkg["name"] = record["name"].dup; pkg["version"] = record["version"].dup; pkg["fullspec"] = record.hit.dup; @@ -65,26 +62,24 @@ bool env_combine(ref Conda conda, string name, string specfile, string mergefile int retval = 0; string[] specs; - auto merge_data = dmfile(mergefile); - foreach (record; merge_data) { - specs ~= record["fullspec"]; - } - retval = conda.run("create -n " - ~ name - ~ " --file " - ~ specfile); - if (retval) { + if(conda.run("create -n " ~ name ~ " --file " ~ specfile)) { return false; } conda.activate(name); - retval = conda.run("install " - ~ conda.multiarg("-c", conda.channels) - ~ " " - ~ safe_install(specs)); - if (retval) { + writeln("Delivery merge specification:"); + foreach (record; dmfile(mergefile)) { + writefln("-> package: %-15s :: version: %s", + record["name"], + !record["version"].empty ? record["version"] : "any"); + + specs ~= record["fullspec"]; + } + + if (conda.run("install " ~ conda.multiarg("-c", conda.channels) + ~ " " ~ safe_install(specs))) { return false; } return true; @@ -94,22 +89,33 @@ bool env_combine(ref Conda conda, string name, string specfile, string mergefile string[string][] testable_packages(ref Conda conda, string mergefile) { string[string][] results; foreach (record; dmfile(mergefile)) { + Node meta; + string pkg_d; + string pkg; + string repository; + string head; + string[] logdata; string[] found_packages = conda.scan_packages(record["name"] ~ "-" ~ record["version"] ~ "*"); - string pkg = found_packages[$-1]; - string pkg_d = chainPath(conda.install_prefix, - "pkgs", - pkg).array; + + if (found_packages.empty) { + writefln("Unable to locate package: %s", record["fullspec"]); + continue; + } else if (found_packages.length > 1) { + pkg = found_packages[$-1]; + } else { + pkg = found_packages[0]; + } + pkg_d = chainPath(conda.install_prefix, + "pkgs", + pkg).array; + string info_d = chainPath(pkg_d, "info").array; string recipe_d = chainPath(info_d, "recipe").array; string git_log = chainPath(info_d, "git").array; string recipe = chainPath(recipe_d, "meta.yaml").array; - string repository; - string head; - string[] logdata; - Node meta; if (!git_log.exists) { continue; @@ -125,7 +131,12 @@ string[string][] testable_packages(ref Conda conda, string mergefile) { head = logdata[1].split()[1]; meta = Loader.fromFile(recipe).load(); - repository = meta["source"]["git_url"].as!string; + try { + repository = meta["source"]["git_url"].as!string; + } catch (Exception e) { + writeln(e.msg); + repository = ""; + } results ~= ["repo": repository, "commit": head]; } |