Skip to content
This repository was archived by the owner on Feb 2, 2024. It is now read-only.

Re-implement dataframe boxing based on new structure #861

Merged
AlexanderKalistratov merged 7 commits into from
May 28, 2020
Merged

Re-implement dataframe boxing based on new structure #861

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
ba9ab9d
Re-implement dataframe boxing based on new structure
densmirn May 26, 2020
aa9884e
Undecorate 6 tests in test_dataframe.py
densmirn May 26, 2020
c639f88
Undecorate 10 tests in test_io.py
densmirn May 26, 2020
b4d5049
Skip check of dtype for 1 test
densmirn May 26, 2020
9d115fd
Add decref for list of array in df boxing
densmirn May 26, 2020
e059436
Minor change in 1 test on dataframe
densmirn May 26, 2020
f537e82
Change boxing of DataFrame.data
densmirn May 28, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Minor change in 1 test on dataframe
  • Loading branch information
@densmirn
densmirn committed May 28, 2020
commit e0594369324bc0f7e27dd835ae782c276c4d490a
12 changes: 10 additions & 2 deletions sdc/hiframes/boxing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,9 +228,16 @@ def box_dataframe(typ, val, c):
col_loc = typ.column_loc[cname]
type_id, col_id = col_loc.type_id, col_loc.col_id
list_type = types.List(arr_typs[i])

# dataframe.data looks like a tuple(list(array))
# e.g. ([array(int64, 1d, C), array(int64, 1d, C)], [array(float64, 1d, C)])

# extracting list from the tuple
list_val = builder.extract_value(dataframe.data, type_id)
typ_arrs = listobj.ListInstance(c.context, c.builder, list_type, list_val)
typ_arrs = listobj.ListInstance(context, builder, list_type, list_val)
# getting array from the list to box it then
arr = typ_arrs.getitem(col_id)
typ_arrs.incref_value(arr)

if dtype == string_type:
arr_obj = box_str_arr(arr_typ, arr, c)
Expand All @@ -247,7 +254,8 @@ def box_dataframe(typ, val, c):
# context.nrt.incref(builder, arr_typ, arr)
pyapi.dict_setitem(df_dict, cname_obj, arr_obj)

c.context.nrt.decref(c.builder, list_type, typ_arrs.value)
if c.context.enable_nrt:
c.context.nrt.decref(c.builder, list_type, list_val)
pyapi.decref(arr_obj)
pyapi.decref(cname_obj)

Expand Down
16 changes: 10 additions & 6 deletions sdc/tests/test_dataframe.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2333,15 +2333,19 @@ def test_append_df_diff_cols_index_ignore_false_no_unboxing(self):
def test_impl():
n1 = 11
n2 = n1 * 2
df = pd.DataFrame({'A': np.arange(n1), 'B': np.arange(n1) ** 2},
index=np.arange(n1) ** 2)
df2 = pd.DataFrame({'C': np.arange(n2), 'D': np.arange(n2) ** 2,
'E S D': np.arange(n2) + 100},
index=np.arange(n2) ** 4)
df = pd.DataFrame({
'A': np.arange(n1), 'B': np.arange(n1) ** 2
}, index=np.arange(n1) ** 2)
df2 = pd.DataFrame({
'C': np.arange(n2), 'D': np.arange(n2) ** 2,
'E S D': np.arange(n2) + 100
}, index=np.arange(n2) ** 4)
return df.append(df2, ignore_index=False)

sdc_func = self.jit(test_impl)
pd.testing.assert_frame_equal(sdc_func(), test_impl())
res_jit = sdc_func()
res_ref = test_impl()
pd.testing.assert_frame_equal(res_jit, res_ref)

@dfRefactoringNotImplemented # required re-implementing DataFrame unboxing
def test_append_df_diff_cols_index_ignore_index(self):
Expand Down