Merge new changes from upstream.

chrish42 · chrish42 · commit 48d7e1e88d8b · 2015-05-19T21:56:43.000-04:00
diff --git a/doc/changes.rst b/doc/changes.rst
@@ -20,8 +20,8 @@ v0.4.0
 v0.3.0
 ------
 
-.. image:: https://zenodo.org/badge/4175/njsmith/zs.png
-   :target: http://dx.doi.org/10.5281/zenodo.11445
+.. image:: https://zenodo.org/badge/doi/10.5281/zenodo.11444.svg
+   :target: http://dx.doi.org/10.5281/zenodo.11444
 
 |
 * New stateful transforms for computing natural and cylic cubic
diff --git a/patsy/build.py b/patsy/build.py
@@ -17,7 +17,8 @@
                                CategoricalSniffer,
                                categorical_to_int)
 from patsy.util import (atleast_2d_column_default,
-                        have_pandas, asarray_or_pandas)
+                        have_pandas, asarray_or_pandas,
+                        safe_issubdtype)
 from patsy.design_info import DesignMatrix, DesignInfo
 from patsy.redundancy import pick_contrasts_for_term
 from patsy.desc import ModelDesc
@@ -76,7 +77,7 @@ def eval(self, data, NA_action):
                                 % (self.factor.name(), self._expected_columns,
                                    result.shape[1]),
                                 self.factor)
-        if not np.issubdtype(np.asarray(result).dtype, np.number):
+        if not safe_issubdtype(np.asarray(result).dtype, np.number):
             raise PatsyError("when evaluating numeric factor %s, "
                                 "I got non-numeric data of type '%s'"
                                 % (self.factor.name(), result.dtype),
diff --git a/patsy/categorical.py b/patsy/categorical.py
@@ -38,9 +38,15 @@
 from patsy import PatsyError
 from patsy.state import stateful_transform
 from patsy.util import (SortAnythingKey,
-                        have_pandas, have_pandas_categorical,
                         safe_scalar_isnan,
-                        iterable)
+                        iterable,
+                        have_pandas, have_pandas_categorical,
+                        have_pandas_categorical_dtype,
+                        safe_is_pandas_categorical,
+                        pandas_Categorical_from_codes,
+                        pandas_Categorical_categories,
+                        pandas_Categorical_codes,
+                        safe_issubdtype)
 
 if have_pandas:
     import pandas
@@ -113,18 +119,21 @@ def test_C():
     assert c4.levels == "LEVELS"
 
 def guess_categorical(data):
-    if have_pandas_categorical and isinstance(data, pandas.Categorical):
+    if safe_is_pandas_categorical(data):
         return True
     if isinstance(data, _CategoricalBox):
         return True
     data = np.asarray(data)
-    if np.issubdtype(data.dtype, np.number):
+    if safe_issubdtype(data.dtype, np.number):
         return False
     return True
 
 def test_guess_categorical():
     if have_pandas_categorical:
-        assert guess_categorical(pandas.Categorical.from_array([1, 2, 3]))
+        c = pandas.Categorical.from_array([1, 2, 3])
+        assert guess_categorical(c)
+        if have_pandas_categorical_dtype:
+            assert guess_categorical(pandas.Series(c))
     assert guess_categorical(C([1, 2, 3]))
     assert guess_categorical([True, False])
     assert guess_categorical(["a", "b"])
@@ -168,21 +177,21 @@ def sniff(self, data):
         if hasattr(data, "contrast"):
             self._contrast = data.contrast
         # returns a bool: are we confident that we found all the levels?
-        if have_pandas_categorical and isinstance(data, pandas.Categorical):
-            # pandas.Categorical has its own NA detection, so don't try to
-            # second-guess it.
-            self._levels = tuple(data.levels)
-            return True
         if isinstance(data, _CategoricalBox):
             if data.levels is not None:
                 self._levels = tuple(data.levels)
                 return True
             else:
                 # unbox and fall through
                 data = data.data
+        if safe_is_pandas_categorical(data):
+            # pandas.Categorical has its own NA detection, so don't try to
+            # second-guess it.
+            self._levels = tuple(pandas_Categorical_categories(data))
+            return True
         # fastpath to avoid doing an item-by-item iteration over boolean
         # arrays, as requested by #44
-        if hasattr(data, "dtype") and np.issubdtype(data.dtype, np.bool_):
+        if hasattr(data, "dtype") and safe_issubdtype(data.dtype, np.bool_):
             self._level_set = set([True, False])
             return True
 
@@ -218,18 +227,26 @@ def t(NA_types, datas, exp_finish_fast, exp_levels, exp_contrast=None):
         assert sniffer.levels_contrast() == (exp_levels, exp_contrast)
     
     if have_pandas_categorical:
-        t([], [pandas.Categorical.from_array([1, 2, None])],
-          True, (1, 2))
-        # check order preservation
-        t([], [pandas.Categorical([1, 0], ["a", "b"])],
-          True, ("a", "b"))
-        t([], [pandas.Categorical([1, 0], ["b", "a"])],
-          True, ("b", "a"))
-        # check that if someone sticks a .contrast field onto a Categorical
-        # object, we pick it up:
-        c = pandas.Categorical.from_array(["a", "b"])
-        c.contrast = "CONTRAST"
-        t([], [c], True, ("a", "b"), "CONTRAST")
+        # We make sure to test with both boxed and unboxed pandas objects,
+        # because we used to have a bug where boxed pandas objects would be
+        # treated as categorical, but their levels would be lost...
+        preps = [lambda x: x,
+                 C]
+        if have_pandas_categorical_dtype:
+            preps += [pandas.Series,
+                      lambda x: C(pandas.Series(x))]
+        for prep in preps:
+            t([], [prep(pandas.Categorical.from_array([1, 2, None]))],
+              True, (1, 2))
+            # check order preservation
+            t([], [prep(pandas_Categorical_from_codes([1, 0], ["a", "b"]))],
+              True, ("a", "b"))
+            t([], [prep(pandas_Categorical_from_codes([1, 0], ["b", "a"]))],
+              True, ("b", "a"))
+            # check that if someone sticks a .contrast field onto our object
+            obj = prep(pandas.Categorical.from_array(["a", "b"]))
+            obj.contrast = "CONTRAST"
+            t([], [obj], True, ("a", "b"), "CONTRAST")
 
     t([], [C([1, 2]), C([3, 2])], False, (1, 2, 3))
     # check order preservation
@@ -286,14 +303,14 @@ def categorical_to_int(data, levels, NA_action, origin=None):
     assert isinstance(levels, tuple)
     # In this function, missing values are always mapped to -1
 
-    if have_pandas_categorical and isinstance(data, pandas.Categorical):
-        data_levels_tuple = tuple(data.levels)
+    if safe_is_pandas_categorical(data):
+        data_levels_tuple = tuple(pandas_Categorical_categories(data))
         if not data_levels_tuple == levels:
             raise PatsyError("mismatching levels: expected %r, got %r"
                              % (levels, data_levels_tuple), origin)
         # pandas.Categorical also uses -1 to indicate NA, and we don't try to
         # second-guess its NA detection, so we can just pass it back.
-        return data.labels
+        return pandas_Categorical_codes(data)
 
     if isinstance(data, _CategoricalBox):
         if data.levels is not None and tuple(data.levels) != levels:
@@ -311,7 +328,7 @@ def categorical_to_int(data, levels, NA_action, origin=None):
 
     # fastpath to avoid doing an item-by-item iteration over boolean arrays,
     # as requested by #44
-    if hasattr(data, "dtype") and np.issubdtype(data.dtype, np.bool_):
+    if hasattr(data, "dtype") and safe_issubdtype(data.dtype, np.bool_):
         if level_to_int[False] == 0 and level_to_int[True] == 1:
             return data.astype(np.int_)
     out = np.empty(len(data), dtype=int)
@@ -358,24 +375,32 @@ def test_categorical_to_int():
                       categorical_to_int,
                       pandas.DataFrame({10: s}), ("a", "b", "c"), NAAction())
     if have_pandas_categorical:
-        cat = pandas.Categorical([1, 0, -1], ("a", "b"))
-        conv = categorical_to_int(cat, ("a", "b"), NAAction())
-        assert np.all(conv == [1, 0, -1])
-        # Trust pandas NA marking
-        cat2 = pandas.Categorical([1, 0, -1], ("a", "None"))
-        conv2 = categorical_to_int(cat, ("a", "b"), NAAction(NA_types=["None"]))
-        assert np.all(conv2 == [1, 0, -1])
-        # But levels must match
-        assert_raises(PatsyError,
-                      categorical_to_int,
-                      pandas.Categorical([1, 0], ("a", "b")),
-                      ("a", "c"),
-                      NAAction())
-        assert_raises(PatsyError,
-                      categorical_to_int,
-                      pandas.Categorical([1, 0], ("a", "b")),
-                      ("b", "a"),
-                      NAAction())
+        constructors = [pandas_Categorical_from_codes]
+        if have_pandas_categorical_dtype:
+            def Series_from_codes(codes, categories):
+                c = pandas_Categorical_from_codes(codes, categories)
+                return pandas.Series(c)
+            constructors.append(Series_from_codes)
+        for con in constructors:
+            cat = con([1, 0, -1], ("a", "b"))
+            conv = categorical_to_int(cat, ("a", "b"), NAAction())
+            assert np.all(conv == [1, 0, -1])
+            # Trust pandas NA marking
+            cat2 = con([1, 0, -1], ("a", "None"))
+            conv2 = categorical_to_int(cat, ("a", "b"),
+                                       NAAction(NA_types=["None"]))
+            assert np.all(conv2 == [1, 0, -1])
+            # But levels must match
+            assert_raises(PatsyError,
+                          categorical_to_int,
+                          con([1, 0], ("a", "b")),
+                          ("a", "c"),
+                          NAAction())
+            assert_raises(PatsyError,
+                          categorical_to_int,
+                          con([1, 0], ("a", "b")),
+                          ("b", "a"),
+                          NAAction())
 
     def t(data, levels, expected, NA_action=NAAction()):
         got = categorical_to_int(data, levels, NA_action)
diff --git a/patsy/contrasts.py b/patsy/contrasts.py
@@ -15,7 +15,7 @@
 import numpy as np
 from patsy import PatsyError
 from patsy.compat import triu_indices, tril_indices, diag_indices
-from patsy.util import repr_pretty_delegate, repr_pretty_impl
+from patsy.util import repr_pretty_delegate, repr_pretty_impl, safe_issubdtype
 
 class ContrastMatrix(object):
     """A simple container for a matrix used for coding categorical factors.
@@ -567,7 +567,7 @@ def code_contrast_matrix(intercept, levels, contrast, default=None):
     if isinstance(contrast, ContrastMatrix):
         return contrast
     as_array = np.asarray(contrast)
-    if np.issubdtype(as_array.dtype, np.number):
+    if safe_issubdtype(as_array.dtype, np.number):
         return ContrastMatrix(as_array,
                               _name_levels("custom", range(as_array.shape[1])))
     if intercept:
diff --git a/patsy/design_info.py b/patsy/design_info.py
@@ -19,7 +19,7 @@
 from patsy import PatsyError
 from patsy.util import atleast_2d_column_default
 from patsy.compat import OrderedDict
-from patsy.util import repr_pretty_delegate, repr_pretty_impl
+from patsy.util import repr_pretty_delegate, repr_pretty_impl, safe_issubdtype
 from patsy.constraint import linear_constraint
 
 class DesignInfo(object):
@@ -278,7 +278,7 @@ def from_array(cls, array_like, default_column_prefix="column"):
             raise ValueError("design matrix can't have >2 dimensions")
         columns = getattr(arr, "columns", range(arr.shape[1]))
         if (hasattr(columns, "dtype")
-            and not np.issubdtype(columns.dtype, np.integer)):
+            and not safe_issubdtype(columns.dtype, np.integer)):
             column_names = [str(obj) for obj in columns]
         else:
             column_names = ["%s%s" % (default_column_prefix, i)
@@ -527,7 +527,7 @@ def __new__(cls, input_array, design_info=None,
             return input_array
         self = atleast_2d_column_default(input_array).view(cls)
         # Upcast integer to floating point
-        if np.issubdtype(self.dtype, np.integer):
+        if safe_issubdtype(self.dtype, np.integer):
             self = np.asarray(self, dtype=float).view(cls)
         if self.ndim > 2:
             raise ValueError("DesignMatrix must be 2d")
@@ -539,7 +539,7 @@ def __new__(cls, input_array, design_info=None,
                              "(got %s, wanted %s)"
                              % (len(design_info.column_names), self.shape[1]))
         self.design_info = design_info
-        if not np.issubdtype(self.dtype, np.floating):
+        if not safe_issubdtype(self.dtype, np.floating):
             raise ValueError("design matrix must be real-valued floating point")
         return self
 
diff --git a/patsy/eval.py b/patsy/eval.py
@@ -546,12 +546,6 @@ def memorize_finish(self, state, which_pass):
         for obj_name in state["pass_bins"][which_pass]:
             state["transforms"][obj_name].memorize_finish()
 
-    # XX FIXME: consider doing something cleverer with exceptions raised here,
-    # to make it clearer what's really going on. The new exception chaining
-    # stuff doesn't appear to be present in any 2.x version of Python, so we
-    # can't use that, but some other options:
-    #    http://blog.ianbicking.org/2007/09/12/re-raising-exceptions/
-    #    http://nedbatchelder.com/blog/200711/rethrowing_exceptions_in_python.html
     def eval(self, memorize_state, data):
         return self._eval(memorize_state["eval_code"], memorize_state["eval_env"],
                           memorize_state, data)
diff --git a/patsy/mgcv_cubic_splines.py b/patsy/mgcv_cubic_splines.py
@@ -815,6 +815,7 @@ def test_crs_compat():
         start_idx = stop_idx + 1
     assert tests_ran == R_crs_num_tests
 
+test_crs_compat.slow = True
 
 def test_crs_with_specific_constraint():
     from patsy.highlevel import incr_dbuilder, build_design_matrices, dmatrix
diff --git a/patsy/state.py b/patsy/state.py
@@ -27,7 +27,7 @@
 import numpy as np
 from patsy.util import (atleast_2d_column_default,
                         asarray_or_pandas, pandas_friendly_reshape,
-                        wide_dtype_for)
+                        wide_dtype_for, safe_issubdtype)
 from patsy.compat import wraps
 
 # These are made available in the patsy.* namespace
@@ -107,7 +107,7 @@ def transform(self, x):
         # heterogenous types. And in that case we're going to be munging the
         # types anyway, so copying isn't a big deal.
         x_arr = np.asarray(x)
-        if np.issubdtype(x_arr.dtype, np.integer):
+        if safe_issubdtype(x_arr.dtype, np.integer):
             dt = float
         else:
             dt = x_arr.dtype
diff --git a/patsy/test_highlevel.py b/patsy/test_highlevel.py
@@ -19,7 +19,10 @@
                          build_design_matrices,
                          DesignMatrixBuilder)
 from patsy.highlevel import *
-from patsy.util import have_pandas
+from patsy.util import (have_pandas,
+                        have_pandas_categorical,
+                        have_pandas_categorical_dtype,
+                        pandas_Categorical_from_codes)
 from patsy.origin import Origin
 
 if have_pandas:
@@ -712,3 +715,32 @@ def test_env_not_saved_in_builder():
     design_matrix2 = dmatrix(design_matrix.design_info.builder, {})
 
     assert np.allclose(design_matrix, design_matrix2)
+
+def test_C_and_pandas_categorical():
+    if not have_pandas_categorical:
+        return
+
+    objs = [pandas_Categorical_from_codes([1, 0, 1], ["b", "a"])]
+    if have_pandas_categorical_dtype:
+        objs.append(pandas.Series(objs[0]))
+    for obj in objs:
+        d = {"obj": obj}
+        assert np.allclose(dmatrix("obj", d),
+                           [[1, 1],
+                            [1, 0],
+                            [1, 1]])
+
+        assert np.allclose(dmatrix("C(obj)", d),
+                           [[1, 1],
+                            [1, 0],
+                            [1, 1]])
+
+        assert np.allclose(dmatrix("C(obj, levels=['b', 'a'])", d),
+                           [[1, 1],
+                            [1, 0],
+                            [1, 1]])
+
+        assert np.allclose(dmatrix("C(obj, levels=['a', 'b'])", d),
+                           [[1, 0],
+                            [1, 1],
+                            [1, 0]])
diff --git a/patsy/util.py b/patsy/util.py
diff --git a/setup.py b/setup.py

Original file line number	Diff line number	Diff line change
`@@ -20,8 +20,8 @@ v0.4.0`
`20`	`20`	`v0.3.0`
`21`	`21`	`------`
`22`	`22`
`23`		`-.. image:: https://zenodo.org/badge/4175/njsmith/zs.png`
`24`		`- :target: http://dx.doi.org/10.5281/zenodo.11445`
	`23`	`+.. image:: https://zenodo.org/badge/doi/10.5281/zenodo.11444.svg`
	`24`	`+ :target: http://dx.doi.org/10.5281/zenodo.11444`
`25`	`25`
`26`	`26`	`\|`
`27`	`27`	`* New stateful transforms for computing natural and cylic cubic`