1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
|
'''
scilo - A scientific workflow and efficiency library
Copyright (C) 2012 Joseph Hunkeler <jhunkeler@gmail.com>
This file is part of scilo.
scilo is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
scilo is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with scilo. If not, see <http://www.gnu.org/licenses/>.
'''
import numpy
import os
import sys
import re
import string
import glob
import logging
import npz
class scilo:
def __init__(self, path):
self.path = os.path.abspath(path)
self.name = os.path.basename(self.path)
self.settings = npz.settings(self.path)
print("Loading dataset: %s" % (self.name))
if not os.path.exists(path):
os.mkdir(os.path.abspath(self.settings.path))
for key in self.settings.directories.iterkeys():
d = os.path.join(self.settings.path, key)
self.settings.directories[key] = d
print("\tCreating directory: '%s'" % (self.settings.directories[key]))
os.mkdir(self.settings.directories[key])
else:
for key in self.settings.directories.iterkeys():
d = os.path.join(self.path, key)
self.settings.directories[key] = d
self.mtimedb = npz.mtimedb(**self.settings.directories)
self.cache = npz.cache(**self.settings.directories)
def __getitem__(self, key):
data = {}
f = glob.glob(os.path.abspath(os.path.join(self.settings.directories['npz'], key + "*")))[0]
name = os.path.basename(string.split(f, sep='.')[0])
if name:
data[name] = numpy.load(f)
return data[key]
def add(self, path, **kwargs):
dest = os.path.join(self.settings.directories['data'], os.path.basename(path))
src = os.path.abspath(path)
if kwargs['move']:
if os.rename(src, dest) == False:
return False
else:
# It is text and the metadata barely matters.
import shutil
shutil.copy2(src, dest)
self.mtimedb = npz.mtimedb(**self.settings.directories)
self.cache.populate()
return True
def aggregate(self, *args, **kwargs):
# Horrible black magic to turn a nested list into a list
sources = [ value for item in args for value in item]
if not sources:
return False
for src in sources:
dest = os.path.join(self.settings.directories['data'], os.path.basename(src))
src = os.path.abspath(src)
if kwargs['move']:
if os.rename(src, dest) == False:
continue
else:
# It is text and the metadata barely matters.
import shutil
shutil.copy2(src, dest)
self.cache.populate()
return True
def select_available(self):
sets = [os.path.basename(string.split(f, sep='.')[0]) for f in self.cache.files]
return sets
def select_pattern(self, pattern):
found = []
available = self.select_available()
for name in available:
if re.search(pattern, name):
found.append(name)
return found
|
