Skip to content

Allow full parallel evaluation of a task #880

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
mfeurer wants to merge 5 commits into from
Closed

Allow full parallel evaluation of a task #880

Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
98f1a02
initial commit, introducing joblib
mfeurer Nov 12, 2019
040dee9
pep8, allow work in different processes, unit tests, fix arff not clo…
mfeurer Nov 12, 2019
1bcb3df
pep8
mfeurer Nov 12, 2019
9c1d72c
PEP8
mfeurer Nov 12, 2019
08145fc
mypy
mfeurer Nov 12, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
mypy
  • Loading branch information
@mfeurer
mfeurer committed Nov 12, 2019
commit 08145fcbe29da24eaa983ac488b37c5892a69f63
14 changes: 7 additions & 7 deletions openml/tasks/split.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,8 @@ def _from_arff_file(cls, filename: str) -> 'OpenMLSplit':
pkl_filename = filename.replace(".arff", ".pkl.py3")

if os.path.exists(pkl_filename):
with open(pkl_filename, "rb") as fh:
_ = pickle.load(fh)
with open(pkl_filename, "rb") as fh_binary:
_ = pickle.load(fh_binary)
repetitions = _["repetitions"]
name = _["name"]

Expand All @@ -88,8 +88,8 @@ def _from_arff_file(cls, filename: str) -> 'OpenMLSplit':
raise FileNotFoundError(
'Split arff %s does not exist!' % filename
)
with open(filename) as fh:
file_data = arff.load(fh, return_type=arff.DENSE_GEN)
with open(filename) as fh_text:
file_data = arff.load(fh_text, return_type=arff.DENSE_GEN)
splits = file_data['data']
name = file_data['relation']
attrnames = [attr[0] for attr in file_data['attributes']]
Expand Down Expand Up @@ -139,9 +139,9 @@ def _from_arff_file(cls, filename: str) -> 'OpenMLSplit':
np.array(repetitions[repetition][fold][sample][1],
dtype=np.int32))

with tempfile.NamedTemporaryFile(delete=False) as fh:
pickle.dump({"name": name, "repetitions": repetitions}, fh, protocol=-1)
os.rename(fh.name, pkl_filename)
with tempfile.NamedTemporaryFile(delete=False) as fh_other:
pickle.dump({"name": name, "repetitions": repetitions}, fh_other, protocol=-1)
os.rename(fh_other.name, pkl_filename)

return cls(name, '', repetitions)

Expand Down