diff options
| -rw-r--r-- | .gitignore | 3 | ||||
| -rw-r--r-- | COPYING | 165 | ||||
| -rw-r--r-- | README | 0 | ||||
| -rw-r--r-- | scilo/__init__.py | 20 | ||||
| -rw-r--r-- | scilo/scilo.py | 178 |
5 files changed, 366 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3fc678c --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*.pyc +.project +.pydevproject @@ -0,0 +1,165 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/> + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + + This version of the GNU Lesser General Public License incorporates +the terms and conditions of version 3 of the GNU General Public +License, supplemented by the additional permissions listed below. + + 0. Additional Definitions. + + As used herein, "this License" refers to version 3 of the GNU Lesser +General Public License, and the "GNU GPL" refers to version 3 of the GNU +General Public License. + + "The Library" refers to a covered work governed by this License, +other than an Application or a Combined Work as defined below. + + An "Application" is any work that makes use of an interface provided +by the Library, but which is not otherwise based on the Library. +Defining a subclass of a class defined by the Library is deemed a mode +of using an interface provided by the Library. + + A "Combined Work" is a work produced by combining or linking an +Application with the Library. The particular version of the Library +with which the Combined Work was made is also called the "Linked +Version". + + The "Minimal Corresponding Source" for a Combined Work means the +Corresponding Source for the Combined Work, excluding any source code +for portions of the Combined Work that, considered in isolation, are +based on the Application, and not on the Linked Version. + + The "Corresponding Application Code" for a Combined Work means the +object code and/or source code for the Application, including any data +and utility programs needed for reproducing the Combined Work from the +Application, but excluding the System Libraries of the Combined Work. + + 1. Exception to Section 3 of the GNU GPL. + + You may convey a covered work under sections 3 and 4 of this License +without being bound by section 3 of the GNU GPL. + + 2. Conveying Modified Versions. + + If you modify a copy of the Library, and, in your modifications, a +facility refers to a function or data to be supplied by an Application +that uses the facility (other than as an argument passed when the +facility is invoked), then you may convey a copy of the modified +version: + + a) under this License, provided that you make a good faith effort to + ensure that, in the event an Application does not supply the + function or data, the facility still operates, and performs + whatever part of its purpose remains meaningful, or + + b) under the GNU GPL, with none of the additional permissions of + this License applicable to that copy. + + 3. Object Code Incorporating Material from Library Header Files. + + The object code form of an Application may incorporate material from +a header file that is part of the Library. You may convey such object +code under terms of your choice, provided that, if the incorporated +material is not limited to numerical parameters, data structure +layouts and accessors, or small macros, inline functions and templates +(ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the + Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the object code with a copy of the GNU GPL and this license + document. + + 4. Combined Works. + + You may convey a Combined Work under terms of your choice that, +taken together, effectively do not restrict modification of the +portions of the Library contained in the Combined Work and reverse +engineering for debugging such modifications, if you also do each of +the following: + + a) Give prominent notice with each copy of the Combined Work that + the Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the Combined Work with a copy of the GNU GPL and this license + document. + + c) For a Combined Work that displays copyright notices during + execution, include the copyright notice for the Library among + these notices, as well as a reference directing the user to the + copies of the GNU GPL and this license document. + + d) Do one of the following: + + 0) Convey the Minimal Corresponding Source under the terms of this + License, and the Corresponding Application Code in a form + suitable for, and under terms that permit, the user to + recombine or relink the Application with a modified version of + the Linked Version to produce a modified Combined Work, in the + manner specified by section 6 of the GNU GPL for conveying + Corresponding Source. + + 1) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (a) uses at run time + a copy of the Library already present on the user's computer + system, and (b) will operate properly with a modified version + of the Library that is interface-compatible with the Linked + Version. + + e) Provide Installation Information, but only if you would otherwise + be required to provide such information under section 6 of the + GNU GPL, and only to the extent that such information is + necessary to install and execute a modified version of the + Combined Work produced by recombining or relinking the + Application with a modified version of the Linked Version. (If + you use option 4d0, the Installation Information must accompany + the Minimal Corresponding Source and Corresponding Application + Code. If you use option 4d1, you must provide the Installation + Information in the manner specified by section 6 of the GNU GPL + for conveying Corresponding Source.) + + 5. Combined Libraries. + + You may place library facilities that are a work based on the +Library side by side in a single library together with other library +facilities that are not Applications and are not covered by this +License, and convey such a combined library under terms of your +choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based + on the Library, uncombined with any other library facilities, + conveyed under the terms of this License. + + b) Give prominent notice with the combined library that part of it + is a work based on the Library, and explaining where to find the + accompanying uncombined form of the same work. + + 6. Revised Versions of the GNU Lesser General Public License. + + The Free Software Foundation may publish revised and/or new versions +of the GNU Lesser General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the +Library as you received it specifies that a certain numbered version +of the GNU Lesser General Public License "or any later version" +applies to it, you have the option of following the terms and +conditions either of that published version or of any later version +published by the Free Software Foundation. If the Library as you +received it does not specify a version number of the GNU Lesser +General Public License, you may choose any version of the GNU Lesser +General Public License ever published by the Free Software Foundation. + + If the Library as you received it specifies that a proxy can decide +whether future versions of the GNU Lesser General Public License shall +apply, that proxy's public statement of acceptance of any version is +permanent authorization for you to choose that version for the +Library. diff --git a/scilo/__init__.py b/scilo/__init__.py new file mode 100644 index 0000000..deebb5b --- /dev/null +++ b/scilo/__init__.py @@ -0,0 +1,20 @@ +''' +scilo - A scientific workflow and efficiency library +Copyright (C) 2012 Joseph Hunkeler <jhunkeler@gmail.com> + +This file is part of scilo. + +scilo is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +scilo is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with scilo. If not, see <http://www.gnu.org/licenses/>. +''' +from scilo import scilo diff --git a/scilo/scilo.py b/scilo/scilo.py new file mode 100644 index 0000000..a0d9c75 --- /dev/null +++ b/scilo/scilo.py @@ -0,0 +1,178 @@ +'''
+scilo - A scientific workflow and efficiency library
+Copyright (C) 2012 Joseph Hunkeler <jhunkeler@gmail.com>
+
+This file is part of scilo.
+
+scilo is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+scilo is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with scilo. If not, see <http://www.gnu.org/licenses/>.
+'''
+import numpy as np
+import sqlite3
+import os
+import glob
+
+class scilo:
+ def __init__(self, path):
+ self.database_mtime = None
+ self.database_mtime_cursor = None
+ self.path = os.path.abspath(path)
+ #self.subdirs = ['data', 'result', 'npz']
+ self.subdirs = {
+ 'data':False,
+ 'result':False,
+ 'npy':False
+ }
+ print("Dataset '%s'..." % (path)),
+ if not os.path.exists(path):
+ print("not found")
+ os.mkdir(os.path.abspath(self.path))
+ print("Generating structure...")
+ for key in self.subdirs.iterkeys():
+ d = os.path.join(self.path, key)
+ self.subdirs[key] = d
+ print("Creating directory: '%s'" % (self.subdirs[key]))
+ os.mkdir(self.subdirs[key])
+ else:
+ print("found")
+ for key in self.subdirs.iterkeys():
+ d = os.path.join(self.path, key)
+ self.subdirs[key] = d
+
+ def __getitem__(self, key):
+ return self.subdirs[key]
+
+ def aggregate(self, globular):
+ sources = glob.glob(globular)
+ if not sources:
+ return False
+ for src in sources:
+ dest = os.path.join(self.subdirs['data'], os.path.basename(src))
+ src = os.path.abspath(src)
+ if os.rename(src, dest) == False:
+ continue
+
+ return True
+
+ def _npy_mtime_populate(self):
+ database = os.path.join(self.subdirs['npy'], 'npy_mtime.db')
+ if not os.path.exists(database):
+ print("Creating modification tracking database...")
+ connection = sqlite3.connect(database)
+ c = connection.cursor()
+ self.database_mtime = sqlite3.connect(database)
+ self.database_mtime_cursor = c
+ c.execute("CREATE TABLE npy(file, mtime)")
+ for f in glob.glob(os.path.join(self.subdirs['data'], "*.*")):
+ print("\tFile: %s\tmtime: %f" % (os.path.basename(f), os.path.getmtime(f)))
+ self._npy_mtime_insert(f, os.path.getmtime(f))
+ connection.commit()
+ else:
+ connection = sqlite3.connect(database)
+ c = connection.cursor()
+ self.database_mtime = connection
+ self.database_mtime_cursor = c
+ return
+
+ def _npy_mtime_insert(self, path, mtime):
+ connection = self.database_mtime
+ c = self.database_mtime_cursor
+ values = (path, mtime,)
+ c.execute("INSERT INTO npy VALUES (?,?)", values)
+ connection.commit()
+
+ def _npy_mtime_update(self, path, stored, current):
+ connection = self.database_mtime
+ c = self.database_mtime_cursor
+ values = (path, current, path, stored)
+ c.execute("UPDATE npy SET file=?, mtime=? WHERE file==? AND mtime==?", values)
+ connection.commit()
+ print("'%s' updated mtime: %f" % (path, current))
+
+ def _npy_mtime_delete(self, path):
+ connection = self.database_mtime
+ c = self.database_mtime_cursor
+ values = (path,)
+ c.execute("DELETE FROM npy WHERE file==?", (values))
+ connection.commit()
+ print("'%s' removed from mtime database" % path)
+ return
+
+ def _npy_mtime_check(self):
+ mtime_stored = []
+ mtime_current = []
+ c = self.database_mtime_cursor
+ c.execute("SELECT file, mtime FROM npy")
+ files = glob.glob(os.path.join(self.subdirs['data'], '*.*'))
+
+ for f in files:
+ mtime_current.append([f, os.path.getmtime(f)])
+
+ for f, mtime in c.fetchall():
+ mtime_stored.append([str(f), mtime])
+
+
+ for stored_file, stored_mtime in mtime_stored:
+ for current_file, current_mtime in mtime_current:
+ if not os.path.exists(stored_file) or not os.path.exists(current_file):
+ print("Missing data file: '%s'" % stored_file)
+ self._npy_mtime_delete(stored_file)
+ self.npy_cache_drop(stored_file)
+ break
+ if current_file == stored_file:
+ if current_mtime != stored_mtime:
+ print("'%s' differs" % current_file)
+ self._npy_mtime_update(current_file, stored_mtime, current_mtime)
+ print("Rebuilding numpy cache for '%s'" % current_file)
+ self.npy_cache_build(current_file)
+ return
+
+ def npy_cache_build(self, path):
+ ''' Generate 'path' npy file in npy directory'''
+ temp = np.loadtxt(path)
+ if np.save(os.path.join(self.subdirs['npy'], os.path.basename(path)), temp) == False:
+ return False
+ return True
+
+ def npy_cache_drop(self, path):
+ ''' Remove 'path' from npy directory '''
+ # For security reasons, you are only allowed to unlink files in the 'npy' directory
+ if os.path.dirname(path) == 'npy':
+ print("Unlinking '%s'" % (path))
+ os.unlink(path)
+
+ def npy_cache_drop_all(self):
+ ''' Remove all npy files '''
+ files = glob.glob(os.path.join(self.subdirs['npy'], '*.npy'))
+ if files:
+ [os.unlink(f) for f in files]
+
+ def npy_cache_populate(self):
+ files = glob.glob(os.path.join(self.subdirs['data'], '*.*'))
+ file_total = len(files)
+ file_current = 1
+
+ for f in files:
+ if os.path.exists(os.path.join(self.subdirs['npy'], os.path.basename(f) + '.npy')):
+ file_total -= 1
+ continue
+ print("Building cache %d of %d: '%s'..." % (file_current, file_total, os.path.basename(f))),
+ if not self.npy_cache_build(f):
+ print("failure")
+ print("success")
+ file_current += 1
+ self._npy_mtime_populate()
+ self._npy_mtime_check()
+
+if __name__ == "__main__":
+ pass
|