aboutsummaryrefslogtreecommitdiff
path: root/scilo/npy.py
diff options
context:
space:
mode:
authorJoseph Hunkeler <jhunkeler@gmail.com>2013-01-26 16:51:00 -0500
committerJoseph Hunkeler <jhunkeler@gmail.com>2013-01-26 16:51:00 -0500
commit718fa095677fc976d7bfff3155fb975018d6e1fc (patch)
tree8389d8e4e990cc52b45e550c212088fd3ae1eddd /scilo/npy.py
parent8234cfe8f2b0b33310d3c23799b07ced157c61b7 (diff)
downloadscilo-718fa095677fc976d7bfff3155fb975018d6e1fc.tar.gz
Remove npy.py in favor of npz.py
Diffstat (limited to 'scilo/npy.py')
-rw-r--r--scilo/npy.py147
1 files changed, 0 insertions, 147 deletions
diff --git a/scilo/npy.py b/scilo/npy.py
deleted file mode 100644
index d007e7d..0000000
--- a/scilo/npy.py
+++ /dev/null
@@ -1,147 +0,0 @@
-'''
-scilo - A scientific workflow and efficiency library
-Copyright (C) 2012 Joseph Hunkeler <jhunkeler@gmail.com>
-
-This file is part of scilo.
-
-scilo is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-scilo is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with scilo. If not, see <http://www.gnu.org/licenses/>.
-'''
-import database
-import numpy
-import glob
-import os
-
-class settings(object):
- def __init__(self, path_start):
- # Define generic directory structure
- self.path = path_start
- self.directories = {
- 'data':False,
- 'result':False,
- 'npy':False
- }
- for key in self.directories.iterkeys():
- d = os.path.join(self.path, key)
- self.directories[key] = d
-
-
-class mtimedb:
- def __init__(self, **kwargs):
- self.settings = kwargs
- self._path_database = os.path.join(self.settings['npy'], 'npy_mtime.db')
- self.db = database.s3(self._path_database)
- self.populate()
-
- def populate(self):
- #if not os.path.exists(self._path_database):
- if os.path.getsize(self._path_database) == 0:
- print("Creating file tracking database...")
- try:
- self.db.cursor.execute("CREATE TABLE npy(file, mtime)")
- for f in glob.glob(os.path.join(self.settings['data'], "*.*")):
- print("\tFile: %s\tmtime: %f" % (os.path.basename(f), os.path.getmtime(f)))
- self.insert(f, os.path.getmtime(f))
- self.db.connection.commit()
- except:
- raise
- return
-
- def insert(self, path, mtime):
- values = (path, mtime,)
- self.db.cursor.execute("INSERT INTO npy VALUES (?,?)", values)
- self.db.connection.commit()
-
- def update(self, path, stored, current):
- values = (path, current, path, stored)
- self.db.cursor("UPDATE npy SET file=?, mtime=? WHERE file==? AND mtime==?", values)
- self.db.connection.commit()
- print("'%s' updated mtime: %f" % (path, current))
-
- def delete(self, path):
- values = (path,)
- self.db.cursor.execute("DELETE FROM npy WHERE file==?", (values))
- self.db.connection.commit()
- print("'%s' removed from mtime database" % path)
- return
-
- def check(self):
- mtime_stored = []
- mtime_current = []
- self.db.cursor.execute("SELECT file, mtime FROM npy")
- files = glob.glob(os.path.join(self.settings['data'], '*.*'))
-
- for f in files:
- mtime_current.append([f, os.path.getmtime(f)])
-
- for f, mtime in self.db.cursor.fetchall():
- mtime_stored.append([str(f), mtime])
-
-
- for stored_file, stored_mtime in mtime_stored:
- for current_file, current_mtime in mtime_current:
- if not os.path.exists(stored_file) or not os.path.exists(current_file):
- print("Missing data file: '%s'" % stored_file)
- self.delete(stored_file)
- self.drop(stored_file)
- break
- if current_file == stored_file:
- if current_mtime != stored_mtime:
- print("'%s' differs" % current_file)
- self.update(current_file, stored_mtime, current_mtime)
- print("Rebuilding numpy cache for '%s'" % current_file)
- self.build(current_file)
- return
-
-class cache:
- def __init__(self, **kwargs):
- self.settings = kwargs
- self.files = []
- pass
-
- def build(self, path):
- ''' Generate 'path' npy file in npy directory'''
- temp = numpy.loadtxt(path)
- if numpy.save(os.path.join(self.settings['npy'], os.path.basename(path)), temp) == False:
- return False
- return True
-
- def drop(self, path):
- ''' Remove 'path' from npy directory '''
- # For security reasons, you are only allowed to unlink files in the 'npy' directory
- if os.path.dirname(path) == 'npy':
- print("Unlinking '%s'" % (path))
- os.unlink(path)
-
- def drop_all(self):
- ''' Remove all npy files '''
- files = glob.glob(os.path.join(self.settings['npy'], '*.npy'))
- if files:
- [os.unlink(f) for f in files]
-
- def populate(self):
- files = glob.glob(os.path.join(self.settings['data'], '*.*'))
- file_total = len(files)
- file_current = 1
-
- for f in files:
- exists = os.path.exists(os.path.join(self.settings['npy'], os.path.basename(f) + '.npy'))
- if exists:
- file_total -= 1
- continue
- print("Building cache %d of %d: '%s'..." % (file_current, file_total, os.path.basename(f))),
- if not self.build(f):
- print("failure")
- print("success")
- file_current += 1
- #self.check()