checkpoint

njsmith · njsmith · commit ed653c13e0af · 2013-02-25T15:15:19.000Z
diff --git a/patsy/categorical.py b/patsy/categorical.py
@@ -17,6 +17,28 @@
 if have_pandas:
     import pandas
 
+# Conundrum:
+# - We don't want to screw around with missing value handling in Categorical;
+#   that logic is all localized inside the builder code. (And in particular,
+#   all the configurability is there as well.)
+#   - Therefore, we just pass through possible missing values, treating them
+#     like ordinary levels.
+# - All Categorical levels must be hashable.
+# - But the np.ma.masked object is non-hashable on Py3.
+# Solution: replace np.ma.masked with an equivalent, hashable object.
+class HashableMaskedConstant(object):
+    _instance = None
+
+    def __new__(cls):
+        if not cls._instance:
+            cls._instance = object.__new__(cls)
+        return cls._instance
+
+    def __str__(self):
+        return "--"
+
+hashable_masked = HashableMaskedConstant()
+
 # A simple wrapper around some categorical data. Provides basically no
 # services, but it holds data fine... eventually it'd be nice to make a custom
 # dtype for this, but doing that right will require fixes to numpy itself.
@@ -53,29 +75,20 @@ def from_pandas_categorical(cls, pandas_categorical):
                            pandas_categorical.levels)
 
     @classmethod
-    def from_sequence(cls, sequence, levels=None, **kwargs):
-        """from_sequence(sequence, levels=None, contrast=None)
+    def from_sequence(cls, sequence,
+                      levels=None, NA_policy="default", **kwargs):
+        """from_sequence(sequence, levels=None, NA_policy="default", contrast=None)
 
         Create a Categorical object given a sequence of data. Levels will be
         auto-detected if not given.
 
-        As far as this function is concerned, 'None' and 'NaN' values are not
-        possible levels; they will be treated as indicating missing
-        values. Likewise for masked elements in numpy masked arrays.
+        NA_policy is either an :class:`NAAction` object used to identify which 
         """
-        def missing_level(level):
-            # Check for np.ma.masked must come before the call to
-            # safe_scalar_isnan, because safe_scalar_isnan coerces its input
-            # to float, and float(np.ma.masked) raises a spurious warning
-            # that we want to avoid (and then returns nan).
-            return (level is None
-                    or level is np.ma.masked
-                    or safe_scalar_isnan(level))
         if levels is None:
             level_set = set()
             for level in sequence:
-                if missing_level(level):
-                    continue
+                if level is np.ma.masked:
+                    level = hashable_masked
                 try:
                     level_set.add(level)
                 except TypeError:
@@ -94,22 +107,18 @@ def missing_level(level):
         int_array = np.empty(len(sequence), dtype=int)
         for i, entry in enumerate(sequence):
             try:
-                if missing_level(entry):
-                    int_array[i] = -1
-                else:
-                    int_array[i] = level_to_int[entry]
+                int_array[i] = level_to_int[entry]
             except KeyError:
-                sorted_levels = sorted(level_to_int)
                 SHOW_LEVELS = 4
                 level_strs = []
-                if len(sorted_levels) <= SHOW_LEVELS:
-                    level_strs += [repr(level) for level in sorted_levels]
+                if len(levels) <= SHOW_LEVELS:
+                    level_strs += [repr(level) for level in levels]
                 else:
                     level_strs += [repr(level)
-                                   for level in sorted_levels[:SHOW_LEVELS//2]]
+                                   for level in levels[:SHOW_LEVELS//2]]
                     level_strs.append("...")
                     level_strs += [repr(level)
-                                   for level in sorted_levels[-SHOW_LEVELS//2:]]
+                                   for level in levels[-SHOW_LEVELS//2:]]
                 level_str = "[%s]" % (", ".join(level_strs))
                 raise PatsyError("Error converting data to categorical: "
                                  "observation with value %r does not match "
diff --git a/patsy/missing.py b/patsy/missing.py
@@ -44,7 +44,11 @@
 # These are made available in the patsy.* namespace
 __all__ = ["NAAction"]
 
-_valid_NA_types = ["NaN"]
+import threading
+current_NA_action = threading.local()
+current_NA_action.value = None
+
+_valid_NA_types = ["None", "NaN", "numpy.ma"]
 _valid_NA_responses = ["raise", "drop"]
 def _desc_options(options):
     return ", ".join([repr(opt) for opt in options])
@@ -84,18 +88,15 @@ class NAAction(object):
     class, or your own object that implements :meth:`handle_NA`, and pass that
     as the `NA_action=` argument instead.
     """
-    def __init__(self, on_NA="drop", NA_types=["NaN"]):
+    def __init__(self, on_NA="drop", NA_types=["None", "NaN", "numpy.ma"]):
         """The `NAAction` constructor takes the following arguments:
         
         :arg on_NA: How to handle missing values. The default is "drop", which
           removes all rows from all matrices which contain any missing
           values. Also available is "raise", which raises an exception when
           any missing values are encountered.
         :arg NA_types: Which values count as missing, as a list of
-          strings. Categorical missing data always count as missing. If the
-          string "NaN" is given in this argument, then not-a-number values in
-          numeric arrays also count as missing. It's anticipated that in the
-          future there will be more options here as well.
+          strings. 
         """
         self.on_NA = on_NA
         if self.on_NA not in _valid_NA_responses:
@@ -111,11 +112,11 @@ def __init__(self, on_NA="drop", NA_types=["NaN"]):
                                  "(should be one of %s)"
                                  % (NA_type, _desc_options(_valid_NA_types)))
 
-    def _where_NA(self, vector):
-        if isinstance(vector, Categorical):
-            return (vector.int_array == -1)
+    def is_NA(self, arr):
+        if isinstance(arr, Categorical):
+            return (arr.int_array == -1)
         else:
-            mask = np.zeros(vector.shape, dtype=bool)
+            mask = np.zeros(arr.shape, dtype=bool)
             if "NaN" in self.NA_types:
                 if np.issubdtype(vector.dtype, np.inexact):
                     mask |= np.isnan(vector)
@@ -162,7 +163,7 @@ def handle_NA(self, index, factor_values, origins):
 
     def _handle_NA_raise(self, index, factor_values, origins):
         for factor_value, origin in zip(factor_values, origins):
-            this_mask = self._where_NA(factor_value)
+            this_mask = self.is_NA(factor_value)
             if np.any(this_mask):
                 raise PatsyError("factor contains missing values", origin)
         return (index, factor_values)
