from os.path import join

import numpy as np

def subdivide_matrix_into_files(conf, mat, file_dir, file_prefix):

def _write_submatrix_to_file(sm_bytes, row_idx, col_idx):

file_name = conf.get_submatrix_key(

file_prefix, conf.n_splits, row_idx, col_idx

)

file_path = join(file_dir, file_name)

with open(file_path, "wb") as fh:

fh.write(sm_bytes)

do_subdivide_matrix(conf, mat, _write_submatrix_to_file)

def reconstruct_matrix_from_files(conf, file_dir, file_prefix):

def _read_submatrix_from_file(row_idx, col_idx):

file_name = conf.get_submatrix_key(

file_prefix, conf.n_splits, row_idx, col_idx

)

file_path = join(file_dir, file_name)

with open(file_path, "rb") as fh:

file_bytes = fh.read()

return file_bytes

return do_reconstruct_matrix(conf, _read_submatrix_from_file)

def do_subdivide_matrix(conf, mat, write_func):

# Step through rows and columns of original matrix, appending submatrix

# bytes to the overall byte stream

sm_per_row = conf.get_submatrices_per_row(conf.n_splits)

sm_size = conf.get_submatrix_size(conf.n_splits)

for row_idx in range(0, sm_per_row):

for col_idx in range(0, sm_per_row):

# Work out the position of the top left and bottom right corner of

# the submatrix

row_start = row_idx * sm_size

col_start = col_idx * sm_size

row_end = row_start + sm_size

col_end = col_start + sm_size

# Extract the submatrix and write to bytes

sub_mat = mat[row_start:row_end, col_start:col_end]

sm_bytes = sub_mat.tobytes()

write_func(sm_bytes, row_idx, col_idx)

def do_reconstruct_matrix(conf, read_func):

result = None

sm_per_row = conf.get_submatrices_per_row(conf.n_splits)

sm_size = conf.get_submatrix_size(conf.n_splits)

# Need to read in row by row concatenate the rows

for row_idx in range(0, sm_per_row):

submatrices = []

for col_idx in range(0, sm_per_row):

sm_data = read_func(row_idx, col_idx)

this_submat = np.frombuffer(sm_data, dtype=np.float32)

this_submat = this_submat.reshape(sm_size, sm_size)

submatrices.append(this_submat)

this_row = np.concatenate(submatrices, axis=1)

if row_idx == 0:

# Initialise the result

result = this_row

else:

# Add the row to the existing result

result = np.concatenate((result, this_row), axis=0)

return result