diff options
Diffstat (limited to 'scilo')
-rw-r--r-- | scilo/npy.py | 147 |
1 files changed, 0 insertions, 147 deletions
diff --git a/scilo/npy.py b/scilo/npy.py deleted file mode 100644 index d007e7d..0000000 --- a/scilo/npy.py +++ /dev/null @@ -1,147 +0,0 @@ -''' -scilo - A scientific workflow and efficiency library -Copyright (C) 2012 Joseph Hunkeler <jhunkeler@gmail.com> - -This file is part of scilo. - -scilo is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -scilo is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with scilo. If not, see <http://www.gnu.org/licenses/>. -''' -import database -import numpy -import glob -import os - -class settings(object): - def __init__(self, path_start): - # Define generic directory structure - self.path = path_start - self.directories = { - 'data':False, - 'result':False, - 'npy':False - } - for key in self.directories.iterkeys(): - d = os.path.join(self.path, key) - self.directories[key] = d - - -class mtimedb: - def __init__(self, **kwargs): - self.settings = kwargs - self._path_database = os.path.join(self.settings['npy'], 'npy_mtime.db') - self.db = database.s3(self._path_database) - self.populate() - - def populate(self): - #if not os.path.exists(self._path_database): - if os.path.getsize(self._path_database) == 0: - print("Creating file tracking database...") - try: - self.db.cursor.execute("CREATE TABLE npy(file, mtime)") - for f in glob.glob(os.path.join(self.settings['data'], "*.*")): - print("\tFile: %s\tmtime: %f" % (os.path.basename(f), os.path.getmtime(f))) - self.insert(f, os.path.getmtime(f)) - self.db.connection.commit() - except: - raise - return - - def insert(self, path, mtime): - values = (path, mtime,) - self.db.cursor.execute("INSERT INTO npy VALUES (?,?)", values) - self.db.connection.commit() - - def update(self, path, stored, current): - values = (path, current, path, stored) - self.db.cursor("UPDATE npy SET file=?, mtime=? WHERE file==? AND mtime==?", values) - self.db.connection.commit() - print("'%s' updated mtime: %f" % (path, current)) - - def delete(self, path): - values = (path,) - self.db.cursor.execute("DELETE FROM npy WHERE file==?", (values)) - self.db.connection.commit() - print("'%s' removed from mtime database" % path) - return - - def check(self): - mtime_stored = [] - mtime_current = [] - self.db.cursor.execute("SELECT file, mtime FROM npy") - files = glob.glob(os.path.join(self.settings['data'], '*.*')) - - for f in files: - mtime_current.append([f, os.path.getmtime(f)]) - - for f, mtime in self.db.cursor.fetchall(): - mtime_stored.append([str(f), mtime]) - - - for stored_file, stored_mtime in mtime_stored: - for current_file, current_mtime in mtime_current: - if not os.path.exists(stored_file) or not os.path.exists(current_file): - print("Missing data file: '%s'" % stored_file) - self.delete(stored_file) - self.drop(stored_file) - break - if current_file == stored_file: - if current_mtime != stored_mtime: - print("'%s' differs" % current_file) - self.update(current_file, stored_mtime, current_mtime) - print("Rebuilding numpy cache for '%s'" % current_file) - self.build(current_file) - return - -class cache: - def __init__(self, **kwargs): - self.settings = kwargs - self.files = [] - pass - - def build(self, path): - ''' Generate 'path' npy file in npy directory''' - temp = numpy.loadtxt(path) - if numpy.save(os.path.join(self.settings['npy'], os.path.basename(path)), temp) == False: - return False - return True - - def drop(self, path): - ''' Remove 'path' from npy directory ''' - # For security reasons, you are only allowed to unlink files in the 'npy' directory - if os.path.dirname(path) == 'npy': - print("Unlinking '%s'" % (path)) - os.unlink(path) - - def drop_all(self): - ''' Remove all npy files ''' - files = glob.glob(os.path.join(self.settings['npy'], '*.npy')) - if files: - [os.unlink(f) for f in files] - - def populate(self): - files = glob.glob(os.path.join(self.settings['data'], '*.*')) - file_total = len(files) - file_current = 1 - - for f in files: - exists = os.path.exists(os.path.join(self.settings['npy'], os.path.basename(f) + '.npy')) - if exists: - file_total -= 1 - continue - print("Building cache %d of %d: '%s'..." % (file_current, file_total, os.path.basename(f))), - if not self.build(f): - print("failure") - print("success") - file_current += 1 - #self.check() |