diff options
| -rw-r--r-- | .gitignore | 15 | ||||
| -rw-r--r-- | dub.sdl | 6 | ||||
| -rw-r--r-- | source/app.d | 49 | ||||
| -rw-r--r-- | source/conda.d | 242 | ||||
| -rw-r--r-- | source/merge.d | 83 |
5 files changed, 395 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..04f9add --- /dev/null +++ b/.gitignore @@ -0,0 +1,15 @@ +.dub +docs.json +__dummy.html +docs/ +/dm +dm.so +dm.dylib +dm.dll +dm.a +dm.lib +dm-test-* +*.exe +*.o +*.obj +*.lst @@ -0,0 +1,6 @@ +name "dm" +description "Delivery merger" +authors "Joseph Hunkeler" +copyright "Copyright © 2019, Joseph Hunkeler" +license "BSD" +dependency "dyaml" version="~>0.7.1" diff --git a/source/app.d b/source/app.d new file mode 100644 index 0000000..94f7ac8 --- /dev/null +++ b/source/app.d @@ -0,0 +1,49 @@ +import std.stdio; +import std.array; +import std.format; +import std.file; +import std.typecons; +import conda; +import merge; + +int main(string[] args) { + Conda conda = new Conda(); + conda.channels = [ + "http://ssb.stsci.edu/astroconda", + "defaults", + "http://ssb.stsci.edu/astroconda-dev" + ]; + conda.install_prefix = "/tmp/miniconda"; + conda.installer_version = "4.5.12"; + conda.installer_variant = "3"; + if (!conda.installer()) { + writeln("Installation failed."); + return 1; + } + conda.initialize(); + env_combine(conda, + "delivery", + "https://raw.githubusercontent.com/astroconda/astroconda-releases/master/hstdp/2019.3/dev/hstdp-2019.3-linux-py36.02.txt", + "test.dm"); + conda.run("info"); + conda.run("list"); + + /* + conda.activate("base"); + conda.run("info"); + conda.sh("python --version"); + conda.sh("python -c 'import sys; print(sys.path)'"); + */ + + /* + conda.sh("git clone https://github.com/spacetelescope/tweakwcs"); + auto project = "tweakwcs"; + chdir(project); + conda.sh("pip install stsci.distutils numpy"); + conda.sh("pip install -e '.[test]'"); + conda.sh("pytest -v"); + chdir(".."); + */ + + return 0; +} diff --git a/source/conda.d b/source/conda.d new file mode 100644 index 0000000..90799c8 --- /dev/null +++ b/source/conda.d @@ -0,0 +1,242 @@ +import core.cpuid : isX86_64; +import std.file; +import std.stdio; +import std.string; +import std.system; +import std.path; +import std.process; +import std.typecons; + + +static auto getenv(string[string] base=null, string preface=null) { + const char delim = '='; + string[string] env; + string cmd = "env"; + + if (preface !is null) { + cmd = preface ~ " && " ~ cmd; + writeln("preface: " ~ cmd); + } + + auto env_sh = executeShell(cmd, env=base); + + if (env_sh.status) { + writeln(env_sh.status, env_sh.output); + throw new Exception("Unable to read shell environment"); + } + + foreach (string line; splitLines(env_sh.output)) { + if (line.empty) { + continue; + } + auto data = split(line, delim); + + // Recombine extra '=' chars + if (data.length > 2) { + data[1] = join(data[1 .. $], delim); + } + env[data[0]] = data[1]; + } + return env; +} + + +class Conda { + import std.net.curl : download; + + public bool initialized = false; + public bool override_channels = true; + public string[] channels; + public string install_prefix; + public string installer_version = "4.5.12"; + public string installer_variant = "3"; + private string[string] env; + private string[string] env_orig; + private const string url_base = "https://repo.continuum.io"; + private const string url_miniconda = join([this.url_base, "miniconda"], "/"); + private string url_installer; + + this() { + env = getenv(); + env_orig = env.dup; + this.url_installer = join([this.url_miniconda, this.installer_file()], dirSeparator); + } + + void dump_env_shell() { + foreach (pair; this.env.byKeyValue()) { + writeln(pair.key ~ " = " ~ pair.value); + } + } + + private string arch() { + if (isX86_64()) { + return "x86_64"; + } + else if (!isX86_64()) { + return "x86"; + } + throw new Exception("Unsupported CPU"); + } + + private string platform() { + import std.system : OS, os; + string report; + switch (os) { + default: + throw new Exception("Unsupported OS"); + + case OS.linux: + report = "Linux"; + break; + + case OS.osx: + report = "MacOSX"; + break; + + case OS.win32: + case OS.win64: + report = "Windows"; + break; + + } + return report; + } + + bool installed() { + if (!this.install_prefix.empty && this.install_prefix.exists) { + return true; + } + return false; + } + + bool in_env() { + string path = this.env.get("PATH", ""); + + if (path.empty || this.install_prefix.empty) { + return false; + } + + foreach (string record; split(path, pathSeparator)) { + if (record == this.install_prefix ~ pathSeparator ~ "bin") { + return true; + } + } + return false; + } + + private bool have_installer() { + if (!this.installer_file().exists) { + return false; + } + return true; + } + + private string installer_file() { + string ext = ".sh"; + version (Windows) { ext = ".exe"; } + string filename = join([ + "Miniconda" ~ this.installer_variant, + this.installer_version, + this.platform(), + this.arch() + ], "-") ~ ext; + return filename; + } + + bool installer() { + if (this.in_env() || this.install_prefix.exists) { + writefln("Conda is already installed: %s", this.install_prefix); + return true; + } else if (this.install_prefix.empty) { + this.install_prefix = absolutePath("./miniconda"); + } else { + this.install_prefix = absolutePath(this.install_prefix); + } + + if (this.have_installer()) { + writeln("Installer already exists"); + } else { + download(this.url_installer, this.installer_file()); + } + + auto installer = executeShell( + "bash " + ~ this.installer_file() + ~ " -b" + ~ " -p " + ~ this.install_prefix, + env=this.env); + + writeln(installer.output); + + if (installer.status != 0) { + return false; + } + + return true; + } + + void configure_headless() { + // YAML is cheap. + // Generate a .condarc inside the new prefix root + auto fp = File(this.install_prefix ~ dirSeparator ~ ".condarc", "w+"); + fp.write("changeps1: False\n"); + fp.write("always_yes: True\n"); + fp.write("quiet: True\n"); + fp.write("auto_update_conda: False\n"); + fp.write("rollback_enabled: False\n"); + fp.write("channels:\n"); + if (this.channels.empty) { + fp.write(" - defaults\n"); + } else { + foreach (channel; this.channels) { + fp.write(" - " ~ channel ~ "\n"); + } + } + } + + void initialize() { + if (this.initialized) { + writeln("Conda installation has already been initialized"); + return; + } + + this.env["PATH"] = join( + [this.install_prefix ~ dirSeparator ~ "bin", + this.env["PATH"]], + pathSeparator); + this.configure_headless(); + this.initialized = true; + } + + void activate(string name) { + this.env_orig = this.env.dup; + string[string] env_new = getenv(this.env, "source activate " ~ name); + this.env = env_new.dup; + } + + void deactivate() { + this.env = this.env_orig.dup; + } + + int run(string command) { + string cmd = "conda " ~ command; + auto proc = this.sh(cmd); + return proc; + } + + auto sh_block(string command) { + auto proc = executeShell(command, env=this.env); + return proc; + } + + int sh(string command) { + writeln("Running: " ~ command); + auto proc = spawnShell(command, env=this.env); + return wait(proc); + } + + string multiarg(string flag, string[] arr) { + return flag ~ " " ~ arr.join(" " ~ flag ~ " "); + } +} diff --git a/source/merge.d b/source/merge.d new file mode 100644 index 0000000..919a577 --- /dev/null +++ b/source/merge.d @@ -0,0 +1,83 @@ +module merge; +import std.stdio; +import std.string; +import std.array; +import std.format; +import std.typecons; +import std.file; +import std.regex; +import conda; + + +auto RE_COMMENT = regex(r"[;#]"); +auto RE_DMFILE = regex(r"^(?P<name>[A-z\-_l]+)(?:[=<>]+)?(?P<version>[A-z0-9. ]+)?"); +auto RE_DMFILE_INVALID_VERSION = regex(r"[ !@#$%^&\*\(\)\-_]+"); +auto RE_DELIVERY_NAME = regex(r"(?P<name>.*)[-_](?P<version>.*)[-_]py(?P<python_version>\d+)[-_.](?P<iteration>\d+)[-_.](?P<ext>.*)"); + + +string safe_spec(string s) { + return "'" ~ s ~ "'"; +} + +string safe_install(string[] specs) { + string[] result; + foreach (record; specs) { + result ~= safe_spec(record); + } + return result.join(" "); +} + +string[string][] dmfile(string filename) { + string[string][] results; + foreach (line; File(filename).byLine()) { + string[string] pkg; + line = strip(line); + auto has_comment = matchFirst(line, RE_COMMENT); + if (!has_comment.empty) { line = strip(has_comment.pre()); } + if (line.empty) { continue; } + + auto record = matchFirst(line, RE_DMFILE); + writefln("-> package: %-10s :: version: %-10s", record["name"], + !record["version"].empty ? record["version"] : "none"); + + pkg["name"] = record["name"].dup; + pkg["version"] = record["version"].dup; + pkg["fullspec"] = record.hit.dup; + results ~= pkg; + } + return results; +} + +bool env_combine(ref Conda conda, string name, string specfile, string mergefile) { + if (indexOf(specfile, "://", 0) < 0 && !specfile.exists) { + throw new Exception(specfile ~ " does not exist"); + } else if (!mergefile.exists) { + throw new Exception(mergefile ~ " does not exist"); + } + + int retval = 0; + string[] specs; + auto merge_data = dmfile(mergefile); + foreach (record; merge_data) { + specs ~= record["fullspec"]; + } + + retval = conda.run("create -n " + ~ name + ~ " --file " + ~ specfile); + if (retval) { + return false; + } + + conda.activate(name); + + retval = conda.run("install " + ~ conda.multiarg("-c", conda.channels) + ~ " " + ~ safe_install(specs)); + if (retval) { + return false; + } + return true; +} |