Skip to content

Convert sparse labels to pandas series #1059

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mfeurer merged 8 commits into from
Apr 26, 2021
Merged

Convert sparse labels to pandas series #1059

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
e9a2e33
Convert sparse labels to pandas series
Neeratyoy Apr 15, 2021
1ae89f5
Handling sparse labels as Series
Neeratyoy Apr 19, 2021
244d9bb
Handling sparse targets when dataset as arrays
Neeratyoy Apr 21, 2021
2bb085a
Merge branch 'develop' into fix_1057
Neeratyoy Apr 21, 2021
fad916c
Revamping sparse dataset tests
Neeratyoy Apr 22, 2021
2170c72
Removing redundant unit test
Neeratyoy Apr 22, 2021
97fa8c7
Cleaning target column formatting
Neeratyoy Apr 23, 2021
b63eac9
Minor comment edit
Neeratyoy Apr 26, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 10 additions & 5 deletions openml/datasets/dataset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -628,7 +628,9 @@ def _encode_if_category(column):
)
elif array_format == "dataframe":
if scipy.sparse.issparse(data):
return pd.DataFrame.sparse.from_spmatrix(data, columns=attribute_names)
data = pd.DataFrame.sparse.from_spmatrix(data, columns=attribute_names)
if isinstance(data, pd.DataFrame) and data.shape[1] == 1:
Copy link
Copy Markdown
Collaborator

@mfeurer mfeurer Apr 22, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this necessary? Don't you also do this in get_data()? Also, I think it'll break the balloon datast which has only a single attribute.

data = data.squeeze() # converting single column to Pandas Series
else:
data_type = "sparse-data" if scipy.sparse.issparse(data) else "non-sparse data"
logger.warning(
Expand Down Expand Up @@ -732,6 +734,7 @@ def get_data(
else:
target = [target]
targets = np.array([True if column in target else False for column in attribute_names])
target_names = np.array([column for column in attribute_names if column in target])
if np.sum(targets) > 1:
raise NotImplementedError(
"Number of requested targets %d is not implemented." % np.sum(targets)
Expand All @@ -752,10 +755,12 @@ def get_data(
attribute_names = [att for att, k in zip(attribute_names, targets) if not k]

x = self._convert_array_format(x, dataset_format, attribute_names)
if scipy.sparse.issparse(y):
y = np.asarray(y.todense()).astype(target_dtype).flatten()
y = y.squeeze()
y = self._convert_array_format(y, dataset_format, attribute_names)
if dataset_format == "array" and scipy.sparse.issparse(y):
# scikit-learn requires dense representation of targets
y = np.asarray(y.todense()).astype(target_dtype)
if not scipy.sparse.issparse(y):
y = y.squeeze()
y = self._convert_array_format(y, dataset_format, target_names)
y = y.astype(target_dtype) if dataset_format == "array" else y
data, targets = x, y

Expand Down
6 changes: 6 additions & 0 deletions tests/test_tasks/test_task_functions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -234,3 +234,9 @@ def test_deletion_of_cache_dir(self):
self.assertTrue(os.path.exists(tid_cache_dir))
openml.utils._remove_cache_dir_for_id("tasks", tid_cache_dir)
self.assertFalse(os.path.exists(tid_cache_dir))

def test_supervised_task_target_format(self):
Comment thread
mfeurer marked this conversation as resolved.
Outdated
openml.config.server = self.production_server
task = openml.tasks.get_task(12731)
_, y = task.get_X_and_y(dataset_format="dataframe")
self.assertIsInstance(y, pd.Series)