Skip to content

Commit b392ffc

Browse files
FrancescAltedFrancescAlted
committed
Allow to run bench on disk and in memory at the same time
1 parent 01dc9d2 commit b392ffc

1 file changed

Lines changed: 76 additions & 72 deletions

File tree

‎bench/ndarray/jit-reduc-sizes.py‎

Lines changed: 76 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88

99
# Compute reductions for different array sizes, using the jit decorator
1010
# and different operands (NumPy and NDArray). Different compression
11-
# levels and codecs can be selected. For in-memory testing, set ooc to False.
11+
# levels and codecs can be selected.
1212

1313
from time import time
1414
import blosc2
@@ -21,8 +21,6 @@
2121
clevel = 1
2222
numpy = False
2323
numpy_jit = False
24-
ooc = True # for in-memory testing, set to False
25-
2624
cparams = cparams_out = None
2725

2826
# For 64 GB RAM
@@ -53,18 +51,6 @@
5351
else:
5452
raise ValueError("Invalid argument")
5553

56-
apath = bpath = None
57-
if numpy:
58-
print("Using NumPy arrays as operands")
59-
else:
60-
print("Using NDArray arrays as operands")
61-
cparams = cparams_out = blosc2.CParams(clevel=clevel, codec=blosc2.Codec[codec])
62-
# cparams_out = blosc2.CParams(clevel=clevel, codec=blosc2.Codec.LZ4)
63-
print("Using cparams: ", cparams)
64-
if ooc:
65-
apath = "a.b2nd"
66-
bpath = "b.b2nd"
67-
6854

6955
# The reductions to compute
7056
def compute_reduction_numpy(a, b, c):
@@ -75,67 +61,85 @@ def compute_reduction(a, b, c):
7561
return np.sum(((a ** 3 + np.sin(a * 2)) < c) & (b > 0), axis=1)
7662

7763

78-
create_times = []
79-
compute_times = []
80-
for n in size_list:
81-
if clevel == 0 and n not in sizes_clevel0:
82-
continue
83-
if numpy_jit and n not in sizes_numpy_jit:
84-
continue
85-
if numpy and not numpy_jit and n not in sizes_numpy:
86-
continue
87-
N = n * 1000
88-
print(f"\nN = {n}000, {dtype=}, size={N ** 2 * 2 * dtype.itemsize / 2**30 }")
89-
chunks = (100, N)
90-
blocks = (1, N)
91-
chunks, blocks = None, None # automatic chunk and block sizes
92-
# Lossy compression
93-
#filters = [blosc2.Filter.TRUNC_PREC, blosc2.Filter.SHUFFLE]
94-
#filters_meta = [8, 0] # keep 8 bits of precision in mantissa
95-
#cparams = blosc2.CParams(clevel=1, codec=blosc2.Codec.LZ4, filters=filters, filters_meta=filters_meta)
96-
97-
# Create some data operands
98-
t0 = time()
64+
# Compute for both disk or memory
65+
for disk in (True, False):
66+
print(f"\n*** Using disk={disk} ***\n")
67+
apath = bpath = None
9968
if numpy:
100-
a = np.linspace(0, 1, N * N, dtype=dtype).reshape(N, N)
101-
b = np.linspace(1, 2, N * N, dtype=dtype).reshape(N, N)
102-
#b = a + 1
103-
c = np.linspace(-10, 10, N, dtype=dtype)
69+
print("Using NumPy arrays as operands")
10470
else:
105-
a = blosc2.linspace(0, 1, N * N, dtype=dtype, shape=(N, N), cparams=cparams, urlpath=apath, mode="w")
106-
print("a.chunks, a.blocks, a.schunk.cratio: ", a.chunks, a.blocks, a.schunk.cratio)
107-
b = blosc2.linspace(1, 2, N * N, dtype=dtype, shape=(N, N), cparams=cparams, urlpath=bpath, mode="w")
108-
#b = (a + 1).compute(cparams=cparams, chunks=chunks, blocks=blocks)
109-
#print(b.chunks, b.blocks, b.schunk.cratio, b.cparams)
110-
c = blosc2.linspace(-10, 10, N, dtype=dtype, cparams=cparams) # broadcasting is supported
111-
#c = blosc2.linspace(-10, 10, N * N, dtype=dtype, shape=(N, N), cparams=cparams)
112-
t1 = time() - t0
113-
print(f"Time to create data: {t1:.4f}")
114-
create_times.append(t1)
115-
116-
if numpy:
117-
if numpy_jit:
71+
print("Using NDArray arrays as operands")
72+
cparams = cparams_out = blosc2.CParams(clevel=clevel, codec=blosc2.Codec[codec])
73+
# cparams_out = blosc2.CParams(clevel=clevel, codec=blosc2.Codec.LZ4)
74+
print("Using cparams: ", cparams)
75+
if disk:
76+
apath = "a.b2nd"
77+
bpath = "b.b2nd"
78+
79+
create_times = []
80+
compute_times = []
81+
# Iterate over different sizes
82+
for n in size_list:
83+
if clevel == 0 and n not in sizes_clevel0:
84+
continue
85+
if numpy_jit and n not in sizes_numpy_jit:
86+
continue
87+
if numpy and not numpy_jit and n not in sizes_numpy:
88+
continue
89+
N = n * 1000
90+
print(f"\nN = {n}000, {dtype=}, size={N ** 2 * 2 * dtype.itemsize / 2**30:.3f} GB")
91+
chunks = (100, N)
92+
blocks = (1, N)
93+
chunks, blocks = None, None # automatic chunk and block sizes
94+
# Lossy compression
95+
#filters = [blosc2.Filter.TRUNC_PREC, blosc2.Filter.SHUFFLE]
96+
#filters_meta = [8, 0] # keep 8 bits of precision in mantissa
97+
#cparams = blosc2.CParams(clevel=1, codec=blosc2.Codec.LZ4, filters=filters, filters_meta=filters_meta)
98+
99+
# Create some data operands
100+
t0 = time()
101+
if numpy:
102+
a = np.linspace(0, 1, N * N, dtype=dtype).reshape(N, N)
103+
b = np.linspace(1, 2, N * N, dtype=dtype).reshape(N, N)
104+
#b = a + 1
105+
c = np.linspace(-10, 10, N, dtype=dtype)
106+
else:
107+
a = blosc2.linspace(0, 1, N * N, dtype=dtype, shape=(N, N), cparams=cparams, urlpath=apath, mode="w")
108+
#print("a.chunks, a.blocks, a.schunk.cratio: ", a.chunks, a.blocks, a.schunk.cratio)
109+
print(f"{a.chunks=}, {a.blocks=}, {a.schunk.cratio=:.2f}x")
110+
111+
b = blosc2.linspace(1, 2, N * N, dtype=dtype, shape=(N, N), cparams=cparams, urlpath=bpath, mode="w")
112+
#b = (a + 1).compute(cparams=cparams, chunks=chunks, blocks=blocks)
113+
#print(b.chunks, b.blocks, b.schunk.cratio, b.cparams)
114+
c = blosc2.linspace(-10, 10, N, dtype=dtype, cparams=cparams) # broadcasting is supported
115+
#c = blosc2.linspace(-10, 10, N * N, dtype=dtype, shape=(N, N), cparams=cparams)
116+
t1 = time() - t0
117+
print(f"Time to create data: {t1:.4f}")
118+
create_times.append(t1)
119+
120+
if numpy:
121+
if numpy_jit:
122+
out = compute_reduction(a, b, c)
123+
t0 = time()
124+
for i in range(niter):
125+
out = compute_reduction(a, b, c)
126+
t1 = (time() - t0) / niter
127+
print(f"Time to compute with numpy_jit and NumPy operands: {t1:.4f}")
128+
else:
129+
t0 = time()
130+
nout = compute_reduction_numpy(a, b, c)
131+
t1 = time() - t0
132+
print(f"Time to compute with NumPy engine: {t1:.4f}")
133+
else:
118134
out = compute_reduction(a, b, c)
119135
t0 = time()
120136
for i in range(niter):
121137
out = compute_reduction(a, b, c)
122138
t1 = (time() - t0) / niter
123-
print(f"Time to compute with numpy_jit and NumPy operands: {t1:.4f}")
124-
else:
125-
t0 = time()
126-
nout = compute_reduction_numpy(a, b, c)
127-
t1 = time() - t0
128-
print(f"Time to compute with NumPy engine: {t1:.4f}")
129-
else:
130-
out = compute_reduction(a, b, c)
131-
t0 = time()
132-
for i in range(niter):
133-
out = compute_reduction(a, b, c)
134-
t1 = (time() - t0) / niter
135-
print(f"Time to compute with numpy_jit and {clevel=}: {t1:.4f}")
136-
compute_times.append(t1)
137-
del a, b, c
138-
139-
print("\nCreate times: [", ", ".join([f"{t:.4f}" for t in create_times]), "]")
140-
print("Compute times: [", ", ".join([f"{t:.4f}" for t in compute_times]), "]")
141-
print("End of run!\n\n")
139+
print(f"Time to compute with numpy_jit and {clevel=}: {t1:.4f}")
140+
compute_times.append(t1)
141+
del a, b, c
142+
143+
print("\nCreate times: [", ", ".join([f"{t:.4f}" for t in create_times]), "]")
144+
print("Compute times: [", ", ".join([f"{t:.4f}" for t in compute_times]), "]")
145+
print("End of run!\n\n")

0 commit comments

Comments
 (0)