convertdata.py


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

#!/usr/bin/env python3
# Used to merge old-format pickled dataset and separate log file hashes
# into the newer joined format.

import pickle
import pandas as pd

datfile = 'dataframe.dat'
hashfile = 'parsed_files.dat'

# Read dataframe
frame = pd.read_pickle(datfile)

# Read MD5 list
with open(hashfile, 'r') as f:
    hashes = f.readlines()

# Store both in a dict and pickle that dict.
data = {'file_hashes': hashes,
        'dataframe': frame}

print(data)
print('Pickling dict...')
pickle.dump(data, open('data.p', 'wb'))