This repository was archived by the owner on Feb 2, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 62
Df.at impl #738
Merged
Merged
Df.at impl #738
Changes from 12 commits
Commits
Show all changes
18 commits
Select commit
Hold shift + click to select a range
94a7a60
Df.at impl
cf1be41
fix check col
3f2db5c
pep
0fb45ea
fix limits+algo
20d2a35
pep
ecce368
Fix type checking
8947a62
fix attr
5f2d566
at return value (not array)
e98a1c2
add check type
928dae8
Merge branch 'master' into dfat
1e-to e39efb3
fix raise errors
a5d0662
add new example codegen
304ec98
add case of many idx
9ad570b
update example
c01a329
Add limitation
261b5b0
Merge branch 'master' into dfat
1e-to 76ffabf
pep
71be518
Merge remote-tracking branch 'origin/dfat' into dfat
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| # ***************************************************************************** | ||
| # Copyright (c) 2020, Intel Corporation All rights reserved. | ||
| # | ||
| # Redistribution and use in source and binary forms, with or without | ||
| # modification, are permitted provided that the following conditions are met: | ||
| # | ||
| # Redistributions of source code must retain the above copyright notice, | ||
| # this list of conditions and the following disclaimer. | ||
| # | ||
| # Redistributions in binary form must reproduce the above copyright notice, | ||
| # this list of conditions and the following disclaimer in the documentation | ||
| # and/or other materials provided with the distribution. | ||
| # | ||
| # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | ||
| # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | ||
| # THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | ||
| # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR | ||
| # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, | ||
| # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, | ||
| # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; | ||
| # OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | ||
| # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR | ||
| # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, | ||
| # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | ||
| # ***************************************************************************** | ||
|
|
||
|
|
||
| import pandas as pd | ||
| from numba import njit | ||
|
|
||
|
|
||
| @njit | ||
| def dataframe_at(): | ||
| df = pd.DataFrame({'A': [1.0, 2.0, 3.0, 1.0], 'B': [4, 5, 6, 7], 'C': ['a', 'b', 'c', 'd']}) | ||
|
|
||
| return df.at[1, 'C'] # ['b'] | ||
|
|
||
|
|
||
| print(dataframe_at()) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -37,7 +37,7 @@ | |
|
|
||
| from pandas.core.indexing import IndexingError | ||
|
|
||
| from numba import types | ||
| from numba import types, prange | ||
| from numba.special import literally | ||
| from numba.typed import List, Dict | ||
| from numba.errors import TypingError | ||
|
|
@@ -1876,6 +1876,37 @@ def _df_getitem_unicode_idx_impl(self, idx): | |
| ty_checker.raise_exc(idx, expected_types, 'idx') | ||
|
|
||
|
|
||
| def df_getitem_tuple_at_codegen(self, row, col): | ||
| """ | ||
| Example of generated implementation: | ||
| def _df_getitem_tuple_at_impl(self, idx): | ||
| row, _ = idx | ||
| data = self._dataframe._data[2] | ||
| res_data = pandas.Series(data, index=self._dataframe.index) | ||
| return res_data.at[row][0] | ||
| """ | ||
| func_lines = ['def _df_getitem_tuple_at_impl(self, idx):'] | ||
| for i in range(len(self.columns)): | ||
| if self.columns[i] == col: | ||
| func_lines += [ | ||
| ' row, _ = idx', | ||
| f' data = self._dataframe._data[{i}]', | ||
| ' res_data = pandas.Series(data, index=self._dataframe.index)', | ||
| ' return res_data.at[row][0]', | ||
| ] | ||
| break | ||
| else: | ||
| raise KeyError('Column is not in the DataFrame') | ||
|
|
||
| func_text = '\n'.join(func_lines) | ||
|
|
||
| global_vars = {'pandas': pandas, | ||
| 'prange': prange, | ||
| 'IndexingError': IndexingError} | ||
|
|
||
| return func_text, global_vars | ||
|
|
||
|
|
||
| def df_getitem_int_iloc_codegen(self, idx): | ||
| """ | ||
| Example of generated implementation: | ||
|
|
@@ -2010,6 +2041,15 @@ def _df_getitem_list_bool_iloc_impl(self, idx): | |
| return func_text, global_vars | ||
|
|
||
|
|
||
| def gen_df_getitem_tuple_at_impl(self, row, col): | ||
| func_text, global_vars = df_getitem_tuple_at_codegen(self, row, col) | ||
| loc_vars = {} | ||
| exec(func_text, global_vars, loc_vars) | ||
| _reduce_impl = loc_vars['_df_getitem_tuple_at_impl'] | ||
|
|
||
| return _reduce_impl | ||
|
|
||
|
|
||
| gen_df_getitem_iloc_int_impl = gen_impl_generator( | ||
| df_getitem_int_iloc_codegen, '_df_getitem_int_iloc_impl') | ||
|
|
||
|
|
@@ -2030,6 +2070,21 @@ def sdc_pandas_dataframe_accessor_getitem(self, idx): | |
|
|
||
| accessor = self.accessor.literal_value | ||
|
|
||
| if accessor == 'at': | ||
| num_idx = isinstance(idx[0], types.Number) and isinstance(self.dataframe.index, (types.Array, types.NoneType)) | ||
| str_idx = (isinstance(idx[0], (types.UnicodeType, types.StringLiteral)) | ||
| and isinstance(self.dataframe.index, StringArrayType)) | ||
| if isinstance(idx, types.Tuple) and isinstance(idx[1], types.StringLiteral): | ||
| if num_idx or str_idx: | ||
| row = idx[0] | ||
| col = idx[1].literal_value | ||
| return gen_df_getitem_tuple_at_impl(self.dataframe, row, col) | ||
|
|
||
| raise TypingError('Attribute at(). The row parameter type ({}) is different from the index type\ | ||
| ({})'.format(type(idx[0]), type(self.dataframe.index))) | ||
|
|
||
| raise TypingError('Attribute at(). The index must be a row and literal column. Given: {}'.format(idx)) | ||
|
|
||
| if accessor == 'iat': | ||
| if isinstance(idx, types.Tuple) and isinstance(idx[1], types.Literal): | ||
| col = idx[1].literal_value | ||
|
|
@@ -2181,6 +2236,57 @@ def sdc_pandas_dataframe_iat_impl(self): | |
| return sdc_pandas_dataframe_iat_impl | ||
|
|
||
|
|
||
| @sdc_overload_attribute(DataFrameType, 'at') | ||
| def sdc_pandas_dataframe_at(self): | ||
| """ | ||
| Intel Scalable Dataframe Compiler User Guide | ||
| ******************************************** | ||
|
|
||
| Limitations | ||
| ----------- | ||
| - Parameter ``column`` in ``idx`` must be a literal value. | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Please add limitation, that |
||
|
|
||
| Pandas API: pandas.DataFrame.at | ||
|
|
||
| Examples | ||
| -------- | ||
| .. literalinclude:: ../../../examples/dataframe/dataframe_at.py | ||
| :language: python | ||
| :lines: 28- | ||
| :caption: Access a single value for a row/column label pair. | ||
| :name: ex_dataframe_at | ||
|
|
||
| .. command-output:: python ./dataframe/dataframe_at.py | ||
| :cwd: ../../../examples | ||
|
|
||
| .. seealso:: | ||
|
|
||
| :ref:`DataFrame.iat <pandas.DataFrame.iat>` | ||
| Access a single value for a row/column pair by integer position. | ||
|
|
||
| :ref:`DataFrame.loc <pandas.DataFrame.loc>` | ||
| Access a group of rows and columns by label(s). | ||
|
|
||
| :ref:`Series.at <pandas.Series.at>` | ||
| Access a single value using a label. | ||
|
|
||
| Intel Scalable Dataframe Compiler Developer Guide | ||
| ************************************************* | ||
| Pandas DataFrame method :meth:`pandas.DataFrame.at` implementation. | ||
|
|
||
| .. only:: developer | ||
| Test: python -m sdc.runtests -k sdc.tests.test_dataframe.TestDataFrame.test_df_at* | ||
| """ | ||
|
|
||
| ty_checker = TypeChecker('Attribute at().') | ||
| ty_checker.check(self, DataFrameType) | ||
|
|
||
| def sdc_pandas_dataframe_at_impl(self): | ||
| return dataframe_getitem_accessor_init(self, 'at') | ||
|
|
||
| return sdc_pandas_dataframe_at_impl | ||
|
|
||
|
|
||
| @sdc_overload_method(DataFrameType, 'pct_change') | ||
| def pct_change_overload(df, periods=1, fill_method='pad', limit=None, freq=None): | ||
| """ | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -245,9 +245,14 @@ def hpat_pandas_series_loc_impl(self, idx): | |
| if isinstance(idx, (int, types.Integer, types.UnicodeType, types.StringLiteral)): | ||
| def hpat_pandas_series_at_impl(self, idx): | ||
| index = self._series.index | ||
| check = False | ||
| mask = numpy.empty(len(self._series._data), numpy.bool_) | ||
| for i in numba.prange(len(index)): | ||
| mask[i] = index[i] == idx | ||
| if mask[i] == True: # noqa | ||
| check = True | ||
| if check != True: # noqa | ||
| raise ValueError("Index is not in the DataFrame") | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Error message doesn't feel right. It is Series method, not DataFrame |
||
| return self._series._data[mask] | ||
|
|
||
| return hpat_pandas_series_at_impl | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
And what if Series returned more than one result?