-
Notifications
You must be signed in to change notification settings - Fork 0
/
gen_data.py
198 lines (162 loc) · 8.58 KB
/
gen_data.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
import numpy as np
import datetime
import h5py
import sys
import argparse
from time import time
import re
import os
parameter_help = """
Dataset Info
------------
seed : int64; default = 65536
A value to initiate the random number generator. Setting to 0 will result in random seed generation based on current time.
vector_length : int
Number of elements per vector (a.k.a "vector length"), determines number of columns in the matrix
vector_count : int
Number of vectors to be generated, equal to number of rows in the matrix
storage : ["bool", "byte", "uint32"]; default = "byte"
The format in which the values should be stored.
bool: matrices are stored as bool values, single element contains single value
byte: matrices are stored as byte/int8, single element contains 8 values (vector_length must be divisible by 8)
uint32: matrices are stored as uint32, single element contains 32 values (vector_length must be divisible by 32)
randomization : ["mutate", "full", "sparse"]; default = "mutate"
The way the vectors are generated.
mutate: the template vector is generated randomly and then a percentage of the values are randomly changed to generate each vector in the matrix
full: all vectors are completely randomly generated
sparse: the template vector is set to zeros; the vectors in the matrix are generated by activating a percentage of random bits
active_fraction : float in range (0, 1); default 0.2
This value is used if randomization is set to "mutate" or "sparse" as the fraction ("percentage") to determine how many bits should be manipulated.
random_fraction : float in range (0, 1); default 0.0
!!! not implemented yet !!!
Additional randomization for the active_fraction parameter. If above zero, the active fraction is not fixed but randomly shifted by up to <random_fraction>
amount. E.g., if active_fraction=0.5 and random_fraction=0.2, the active fraction for each vector will be randomly selected from range (0.3, 0.7).
use_fletcher : bool; default = True
"""
# generate random seed
current_dt = datetime.datetime.now()
# seed = int(current_dt.timestamp())
seed = 65536
stamp = current_dt.strftime("%Y-%m-%d_%H-%M-%S")
use_fletcher = True
compression_level = 3
datapath = "data"
storage_types = ["bool", "byte", "uint32"]
print(f"Initializing dataset generation with seed: {seed} at {stamp}.")
info = {
"alg_version": 0.2, # for forward compatibility only
"seed": seed,
"vector_length": int(2**18),
"vector_count": int(1e4),
"storage": "uint32",
"randomization": "mutate",
"active_fraction": 0.2,
"random_fraction": 0.0,
"stamp": stamp,
"use_fletcher": use_fletcher,
"compression_level": compression_level
}
parser = argparse.ArgumentParser()
parser.add_argument("n_vector", help="Number of vectors in data. Python expressions can be used, i.e. 1e3 == 1000.")
parser.add_argument("n_elements", help="Number of elements in each vector (i.e. vector length). Python expressions can be used, i.e. 2**16 == 65536")
parser.add_argument("--path", "-p", default=datapath,
help=f"Path to the data folder. (default = 'data')")
parser.add_argument("--storages", "-s", choices=storage_types, action="append", default=[],
help=f"Choose which storage types to generate. Can be used multiple times to generate multiple storage types.")
args = parser.parse_args()
datapath = args.path
storages = args.storages
info["vector_count"] = int(eval(args.n_vector))
info["vector_length"] = int(eval(args.n_elements))
# for (k, v) in info.items():
# ma, mb = re.search(r"^(\w)", k), re.search(r"_?(?<=\_)(\w)", k)
# short = "-" + ma.group(0) + (mb.group(1) if mb is not None else "")
# parser.add_argument("--" + k, short)
def activate(x, f, s):
"""Activate "f" fraction of total "s" bits in the array "x"
"""
x[rng.choice(s, int(f*s), False)] = True
return x
def mutate(x, f, s):
"""Mutate "f" fraction of total "s" bits in the array "x"
"""
x[rng.choice(s, int(f*s), False)] ^= True
return x
for storage in storages:
info["storage"] = storage
print(f"Settings:\n- number of vectors: {info['vector_count']}\n- vector length: {info['vector_length']}\n- storage type: {info['storage']}"
+ "\n- randomization method: {info['randomization']}\n- active fraction: {info['active_fraction']}\n- random fraction: {info['random_fraction']}"
+ "\n- version: {info['version']}")
rng = np.random.RandomState(info["seed"])
vector_shape = (int(info["vector_length"]), 1)
matrix_shape = (int(info["vector_count"]), int(info["vector_length"]))
rng_max_val = 2
dtype = np.bool
# old version
# if info["storage"] == "bool":
# vector_shape = (info["vector_length"], 1)
# matrix_shape = (info["vector_count"], info["vector_length"])
# rng_max_val = 2
# dtype = np.bool
# elif info["storage"] == "byte":
# if info["vector_length"] % 8 != 0:
# raise Exception("The parameter <vector_length> must be divisible by 8 when using 'byte' <storage>!")
# vector_shape = (info["vector_length"] >> 3, 1)
# matrix_shape = (info["vector_count"] >> 3, info["vector_length"])
# rng_max_val = 8
# dtype = np.bool
# elif info["storage"] == "uint32":
# if info["vector_length"] % 32 != 0:
# raise Exception("The parameter <vector_length> must be divisible by 32 when using 'uint32' <storage>!")
# vector_shape = (info["vector_length"] >> 5, 1)
# matrix_shape = (info["vector_count"] >> 5, info["vector_length"])
# rng_max_val = 32
# dtype = np.bool
# else:
# raise Exception(f"Unknown storage type: {info['storage']}!")
print("Generating data...")
if info["randomization"] == "full":
# do full randomization
vector = rng.randint(0, rng_max_val, vector_shape, dtype)
matrix = rng.randint(0, rng_max_val, matrix_shape, dtype)
elif info["randomization"] == "mutate":
vector = rng.randint(0, rng_max_val, vector_shape, dtype)
matrix = np.tile(vector.T, (matrix_shape[0], 1))
matrix = np.apply_along_axis(mutate, 0, matrix.T, info["active_fraction"], info["vector_length"]).T
elif info["randomization"] == "sparse":
vector = np.zeros(vector_shape, dtype=dtype)
matrix = np.zeros(matrix_shape, dtype=dtype)
matrix = np.apply_along_axis(activate, 0, matrix.T, 0, info["active_fraction"], info["vector_length"]).T
else:
raise Exception(f"Unknown randomization method {info['randomization']}!")
print("Data generation complete. Compressing data (if necessary).")
# compress
order = sys.byteorder
if info["storage"] == "bool":
pass # no compression
elif info["storage"] == "byte":
if info["vector_length"] % 8 != 0:
raise Exception("The parameter 'vector_length' must be divisible by 8 when using 'byte' storage!")
matrix = np.packbits(matrix, 1, bitorder="big").reshape(matrix_shape[0], vector_shape[0] >> 3)
vector = np.packbits(vector, bitorder="big")[:, np.newaxis]
elif info["storage"] == "uint32":
if info["vector_length"] % 32 != 0:
raise Exception("The parameter 'vector_length' must be divisible by 32 when using 'uint32' storage!")
if order == "little":
matrix = np.packbits(matrix.reshape(-1, 4, 8)[:, ::-1], bitorder="big").view(np.uint32).reshape(matrix_shape[0], vector_shape[0] >> 5)
vector = np.packbits(vector.reshape(-1, 4, 8)[:, ::-1], bitorder="big").view(np.uint32)[:, np.newaxis]
else:
matrix = np.packbits(matrix.reshape(-1, 4, 8), bitorder="big").view(np.uint32).reshape(matrix_shape[0], vector_shape[0] >> 5)
vector = np.packbits(vector.reshape(-1, 4, 8), bitorder="big").view(np.uint32)[:, np.newaxis]
else:
raise Exception(f"Unknown storage type: {info['storage']}!")
filename = os.path.join(datapath, f"bits_{info['vector_count']}x{info['vector_length']}_{info['storage']}_{info['randomization']}.hdf5")
print(f"Saving data to {filename}.")
with h5py.File(filename, "w") as f:
f.attrs.update(**info)
now = time()
f.create_dataset("vector", data=vector, compression="gzip", compression_opts=compression_level, fletcher32=use_fletcher)
vector_saving_runtime = time() - now
print(f"Vector saved. Saving time was {vector_saving_runtime}s. Estimated matrix saving time: {vector_saving_runtime * matrix_shape[0]}s.")
f.create_dataset("matrix", data=matrix, compression="gzip", compression_opts=compression_level, fletcher32=use_fletcher)
print("Done.")