diff --git a/graphblas/core/formatting.py b/graphblas/core/formatting.py index 5fe9b6972..cd9380c0c 100644 --- a/graphblas/core/formatting.py +++ b/graphblas/core/formatting.py @@ -1,4 +1,11 @@ -# This file imports pandas, so it should only be imported when formatting +# The rich repr and _repr_html_ are hand-rendered here to reproduce pandas' +# DataFrame text/HTML output byte-for-byte without importing pandas. When pandas +# is installed we still read its display.* options so a user's option_context is +# honored; when it is absent we fall back to pandas' documented defaults. +import math +import re +import shutil + import numpy as np from .. import backend, config, monoid, unary @@ -14,6 +21,35 @@ except ImportError: # pragma: no cover (import) has_pandas = False +# pandas display.* defaults, used verbatim when pandas is not installed so the +# hand renderer produces the same output it would with a freshly imported pandas. +_DISPLAY_DEFAULTS = { + "max_rows": 60, + "min_rows": 10, + "max_columns": 0, + "width": 80, + "expand_frame_repr": True, + "precision": 6, + "max_colwidth": 50, + "chop_threshold": None, + "colheader_justify": "right", + "float_format": None, + "html.border": 1, + "html.use_mathjax": True, +} + + +def _display_option(name): + """Read a pandas display. option, or fall back to its default. + + Reading from pandas keeps a user's ``pd.option_context`` honored while the + rendering logic itself stays pandas-free. + """ + if has_pandas: + return pd.get_option(f"display.{name}") + return _DISPLAY_DEFAULTS[name] + + # This was written by a complete novice at CSS. # If you can help make it better, please do! CSS_STYLE = """ @@ -219,7 +255,7 @@ def _update_vector_array(arr, vector, columns, column_offset, *, mask=None): def _get_max_columns(): - max_columns = pd.options.display.max_columns + max_columns = _display_option("max_columns") if max_columns == 0: # We are probably in a terminal and pandas will automatically size the data correctly. # In this case, let's get a sufficiently large amount of data to show and defer to pandas. @@ -241,13 +277,647 @@ def _get_chunk(length, min_length, max_length): return chunk, chunk_groups +class _Column: + """One rendered column: a label, its raw cell values, and how to format them. + + ``kind`` selects the pandas array formatter this column would have used: + "object" (GenericArrayFormatter), "float" (FloatArrayFormatter, also complex), + or "int" (IntArrayFormatter). ``numeric`` mirrors pandas ``is_numeric_dtype`` + and controls whether the column header gets a leading space. + """ + + __slots__ = ("label", "values", "kind", "numeric") + + def __init__(self, label, values, kind, numeric): + self.label = label + self.values = values + self.kind = kind + self.numeric = numeric + + +class _GBFrame: + """A minimal object/typed-column table, standing in for a pandas DataFrame. + + ``col_name`` is the columns' index name (pandas ``df.columns.name``); when set + it appears in the top-left corner cell, as vector reprs rely on. + """ + + __slots__ = ("columns", "index", "col_name") + + def __init__(self, columns, index, col_name=None): + self.columns = columns + self.index = index + self.col_name = col_name + + @property + def ncols(self): + return len(self.columns) + + @property + def nrows(self): + return len(self.index) + + def slice_cols(self, idx): + return _GBFrame([self.columns[i] for i in idx], self.index, self.col_name) + + def slice_rows(self, idx): + columns = [ + _Column(c.label, [c.values[i] for i in idx], c.kind, c.numeric) for c in self.columns + ] + return _GBFrame(columns, [self.index[i] for i in idx], self.col_name) + + +def _isna_cell(x): + # Gaps in the dense grid are float NaN; present values (including displayed + # "nan"/"inf") are never a bare float NaN, so this only flags the gaps. + return x is None or (isinstance(x, float) and math.isnan(x)) + + +def _count_present(arr): + return sum(1 for x in arr.flat if not _isna_cell(x)) + + +def _dtype_kind_numeric(dtype): + kind = dtype.kind + if kind in "fc": + return "float", True + if kind in "iu": + return "int", True + if kind == "b": + # bool renders via the generic formatter, but is_numeric_dtype(bool) is True + return "object", True + return "object", False + + +def _make_dense_frame(arr, columns, index): + nrows = len(index) + out = [] + for j, label in enumerate(columns): + vals = [("" if _isna_cell(arr[i, j]) else arr[i, j]) for i in range(nrows)] + out.append(_Column(label, vals, "object", False)) + return _GBFrame(out, list(index)) + + +def _make_coo_frame(label_arrays, add_dots): + n = len(label_arrays[0][1]) + index = list(range(n)) + out = [] + for label, values in label_arrays: + values = np.asarray(values) + if add_dots: + out.append(_Column(label, [*values.tolist(), "..."], "object", False)) + else: + kind, numeric = _dtype_kind_numeric(values.dtype) + out.append(_Column(label, values.tolist(), kind, numeric)) + if add_dots: + index.append("...") + return _GBFrame(out, index) + + +# --- cell formatting (reproduces pandas array formatters for object/int/float) --- + +_NUMBER_RE = re.compile(r"^\s*[\+-]?[0-9]+\.[0-9]*$") + + +def _is_float_scalar(v): + # Matches pandas.lib.is_float: python/numpy floats, but not bool/int/complex. + return isinstance(v, (float, np.floating)) + + +def _pprint(v): + # Reproduces pandas printing.pprint_thing for our cell types (escape_chars for + # tab/cr/nl, quote_strings=False): scalars -> str, sequences recurse. + if isinstance(v, (list, tuple)): + body = ", ".join(_pprint(e) for e in v) + if isinstance(v, tuple) and len(v) == 1: + body += "," + return f"[{body}]" if isinstance(v, list) else f"({body})" + s = str(v) + return s.replace("\t", r"\t").replace("\r", r"\r").replace("\n", r"\n") + + +def _trim_zeros_single_float(s): + s = s.rstrip("0") + if s.endswith("."): + s += "0" + return s + + +def _trim_zeros_float(str_floats): + trimmed = list(str_floats) + + def is_number_with_decimal(x): + return _NUMBER_RE.match(x) is not None + + def should_trim(values): + numbers = [x for x in values if is_number_with_decimal(x)] + return len(numbers) > 0 and all(x.endswith("0") for x in numbers) + + while should_trim(trimmed): + trimmed = [x[:-1] if is_number_with_decimal(x) else x for x in trimmed] + return [x + "0" if is_number_with_decimal(x) and x.endswith(".") else x for x in trimmed] + + +def _trim_zeros_complex(str_complexes): + real_part, imag_part = [], [] + for x in str_complexes: + trimmed = re.split(r"(?{padded_length}}" + "j" + for real_pt, imag_pt in zip(padded_parts[:n], padded_parts[n:], strict=True) + ] + + +def _value_formatter(fmt_str, threshold): + def base(v): + return fmt_str.format(value=v) + + if threshold is None: + return base + + def formatter(v): + return base(v) if abs(v) > threshold else base(0.0) + + return formatter + + +def _format_reals_with_na(values, formatter, na_rep): + return [na_rep if (v != v) else formatter(v) for v in values] + + +def _format_complex_with_na(values, formatter, na_rep): + out = [] + for val in values: + re_v, im_v = val.real, val.imag + re_na, im_na = re_v != re_v, im_v != im_v + if not re_na and not im_na: + out.append(formatter(val)) + elif not re_na: + out.append(f"{formatter(re_v)}+{na_rep}j") + elif not im_na: + imag_formatted = formatter(im_v).strip() + if imag_formatted.startswith("-"): + out.append(f"{na_rep}{imag_formatted}j") + else: + out.append(f"{na_rep}+{imag_formatted}j") + else: + out.append(f"{na_rep}+{na_rep}j") + return out + + +def _format_float_column(values, digits): + # Reproduces FloatArrayFormatter for fixed_width, leading_space=True, na_rep="NaN". + arr = np.asarray(values) + is_complex = np.iscomplexobj(arr) + na_rep = "NaN" + if (float_format := _display_option("float_format")) is not None: + # A user display.float_format callable makes FloatArrayFormatter drop + # fixed_width: each value (real or complex) is just float_format(value), + # with no trailing-zero trim and no scientific switchover. Iterate the + # numpy array (not .tolist()) so the callable receives numpy scalars, as + # pandas does; e.g. a "%f"-style callable then casts complex the same way. + return [na_rep if (v != v) else float_format(v) for v in arr] + seq = arr.tolist() + threshold = _display_option("chop_threshold") + + def format_with(fmt_str): + formatter = _value_formatter(fmt_str, threshold) + if is_complex: + return _trim_zeros_complex(_format_complex_with_na(seq, formatter, na_rep)) + return _trim_zeros_float(_format_reals_with_na(seq, formatter, na_rep)) + + result = format_with(f"{{value: .{digits:d}f}}") + too_long = bool(result) and max(len(x) for x in result) > digits + 6 + abs_vals = np.abs(arr) + has_large = bool((abs_vals > 1e6).any()) + has_small = bool(((abs_vals < 10.0 ** (-digits)) & (abs_vals > 0)).any()) + if has_small or (too_long and has_large): + result = format_with(f"{{value: .{digits:d}e}}") + return list(result) + + +def _justify(strings, width, mode="right"): + if mode == "left": + return [x.ljust(width) for x in strings] + if mode == "center": + return [x.center(width) for x in strings] + return [x.rjust(width) for x in strings] + + +def _make_fixed_width(strings, justify="right", minimum=None): + if not strings: + return list(strings) + max_len = max(len(x) for x in strings) + if minimum is not None: + max_len = max(minimum, max_len) + conf_max = _display_option("max_colwidth") + if conf_max is not None and max_len > conf_max: + max_len = conf_max + + def just(x): + if conf_max is not None and conf_max > 3 and len(x) > max_len: + x = x[: max_len - 3] + "..." + return x + + return _justify([just(x) for x in strings], max_len, justify) + + +def _format_labels(labels): + # Reproduce pandas Index._format_flat(include_name=False) for our label types: + # integer labels are padded to a uniform width (left-justified, with a sign + # column when any are negative); string labels are left as-is. + if labels and all(isinstance(x, (int, np.integer)) and not isinstance(x, bool) for x in labels): + pattern = "{: d}" if any(x < 0 for x in labels) else "{:d}" + strs = [pattern.format(x) for x in labels] + width = max(len(s) for s in strs) + return [s.ljust(width) for s in strs] + return [str(x) for x in labels] + + +def _adjoin(space, lists): + # Port of pandas printing.adjoin (ascii len/ljust); glues columns with `space`. + lengths = [max(map(len, x)) + space for x in lists[:-1]] + lengths.append(max(map(len, lists[-1]))) + max_len = max(map(len, lists)) + padded = [] + for i, lst in enumerate(lists): + nl = [x.ljust(lengths[i]) for x in lst] + nl = [" " * lengths[i]] * (max_len - len(lst)) + nl + padded.append(nl) + return "\n".join("".join(parts) for parts in zip(*padded, strict=True)) + + +def _binify(cols, line_width): + adjoin_width = 1 + bins = [] + curr_width = 0 + i_last = len(cols) - 1 + for i, w in enumerate(cols): + w_adjoined = w + adjoin_width + curr_width += w_adjoined + if i_last == i: + wrap = curr_width + 1 > line_width and i > 0 + else: + wrap = curr_width + 2 > line_width and i > 0 + if wrap: + bins.append(i) + curr_width = w_adjoined + bins.append(len(cols)) + return bins + + +def _console_width(): + # pandas repr sets the wrap width from console.get_console_size(); reuse it + # when present so the wrap decision is identical. Without pandas (or if that + # private module moved) fall back to display.width. + if has_pandas: + try: + from pandas.io.formats.console import get_console_size + + return get_console_size()[0] + except Exception: # pragma: no cover (defensive across pandas versions) + pass + return _display_option("width") + + +class _TextFormatter: + """Reproduces pandas DataFrameFormatter + StringFormatter for text repr.""" + + def __init__(self, frame, max_rows, min_rows, max_cols): + self.frame = frame + self.max_rows = max_rows + self.min_rows = min_rows + self.max_cols = max_cols + self.justify = _display_option("colheader_justify") + self.tr_frame = frame + self.tr_col_num = None + self.tr_row_num = None + self.max_cols_fitted = self._calc_max_cols_fitted() + self.max_rows_fitted = self._calc_max_rows_fitted() + self.truncate() + + def _is_in_terminal(self): + return self.max_cols == 0 or self.max_rows == 0 + + def _calc_max_cols_fitted(self): + if not self._is_in_terminal(): + return self.max_cols + width = shutil.get_terminal_size()[0] + if self.max_cols == 0 and self.frame.ncols > width: + return width + return self.max_cols + + def _calc_max_rows_fitted(self): + if self._is_in_terminal() and self.max_rows == 0: + # rows available for data: terminal height minus dots + prompt + header + return shutil.get_terminal_size()[1] - 3 + max_rows = self.max_rows + if max_rows and self.frame.nrows > max_rows and self.min_rows: + max_rows = min(self.min_rows, max_rows) + return max_rows + + @property + def is_truncated_horizontally(self): + return bool(self.max_cols_fitted and self.frame.ncols > self.max_cols_fitted) + + @property + def is_truncated_vertically(self): + return bool(self.max_rows_fitted and self.frame.nrows > self.max_rows_fitted) + + @property + def is_truncated(self): + return self.is_truncated_horizontally or self.is_truncated_vertically + + def truncate(self): + if self.is_truncated_horizontally: + self._truncate_horizontally() + if self.is_truncated_vertically: + self._truncate_vertically() + + def _truncate_horizontally(self): + col_num = self.max_cols_fitted // 2 + if col_num >= 1: + _len = self.tr_frame.ncols + self.tr_frame = self.tr_frame.slice_cols( + [*range(col_num), *range(_len - col_num, _len)] + ) + else: + col_num = self.max_cols + self.tr_frame = self.tr_frame.slice_cols(list(range(col_num))) + self.tr_col_num = col_num + + def _truncate_vertically(self): + row_num = self.max_rows_fitted // 2 + if row_num >= 1: + _len = self.tr_frame.nrows + self.tr_frame = self.tr_frame.slice_rows( + [*range(row_num), *range(_len - row_num, _len)] + ) + else: + row_num = self.max_rows + self.tr_frame = self.tr_frame.slice_rows(list(range(row_num))) + self.tr_row_num = row_num + + def _format_col_raw(self, col): + if col.kind == "int": + return [f"{x: d}" for x in col.values] + if col.kind == "float": + return _format_float_column(col.values, _display_option("precision")) + precision = _display_option("precision") + float_format = _display_option("float_format") + out = [] + for v in col.values: + # A float NaN is excluded from pandas' float-format branch (it uses + # is_float(v) & notna(v)) and rendered as the na_rep "NaN" instead. + if _is_float_scalar(v) and not math.isnan(v): + if float_format is not None: + # A user display.float_format callable replaces the default + # precision render (and adds no sign-space of its own). + out.append(float_format(v)) + else: + out.append(_trim_zeros_single_float(f"{v: .{precision}f}")) + elif v is None: + out.append(" None") + elif _is_float_scalar(v): + out.append(" NaN") + else: + out.append(f" {_pprint(v)}") + return out + + def _get_body_strcols(self): + # Column labels are formatted together (integer labels padded to a uniform + # width) the way pandas Index._format_flat does, not per column. + labels = _format_labels([col.label for col in self.tr_frame.columns]) + strcols = [] + for col, label in zip(self.tr_frame.columns, labels, strict=True): + header = f" {label}" if col.numeric else label + header_colwidth = len(header) + # pandas fixes width twice: format_array right-justifies to the cell + # content width, then the body pass re-justifies with colheader_justify + # (which only matters when the header is wider, or when it is "left"). + fmt_values = _make_fixed_width(self._format_col_raw(col), "right") + fmt_values = _make_fixed_width(fmt_values, self.justify, minimum=header_colwidth) + max_len = max(max((len(x) for x in fmt_values), default=0), header_colwidth) + cheader = _justify([header], max_len, self.justify) + strcols.append(cheader + fmt_values) + return strcols + + def _get_index_strcol(self): + idx = _make_fixed_width([str(x) for x in self.tr_frame.index], justify="left") + corner = "" if self.frame.col_name is None else str(self.frame.col_name) + return [corner, *idx] + + def get_strcols(self): + strcols = self._get_body_strcols() + strcols.insert(0, self._get_index_strcol()) + return strcols + + @property + def _adjusted_tr_col_num(self): + return self.tr_col_num + 1 # index column is always shown + + def _insert_dot_separators(self, strcols): + index_length = len(self._get_index_strcol()) + if self.is_truncated_horizontally: + strcols.insert(self._adjusted_tr_col_num, [" ..."] * index_length) + if self.is_truncated_vertically: + self._insert_dots_vertical(strcols, index_length) + return strcols + + def _insert_dots_vertical(self, strcols, index_length): + n_header_rows = index_length - self.tr_frame.nrows + row_num = self.tr_row_num + for ix, col in enumerate(strcols): + cwidth = len(col[row_num]) + is_dot_col = self.is_truncated_horizontally and ix == self._adjusted_tr_col_num + dots = "..." if (cwidth > 3 or is_dot_col) else ".." + if ix == 0: + dot_mode = "left" + elif is_dot_col: + cwidth = 4 + dot_mode = "right" + else: + dot_mode = "right" + col.insert(row_num + n_header_rows, _justify([dots], cwidth, dot_mode)[0]) + + def _get_strcols(self): + strcols = self.get_strcols() + if self.is_truncated: + strcols = self._insert_dot_separators(strcols) + return strcols + + def _fit_to_terminal(self, strcols): + lines = _adjoin(1, strcols).split("\n") + max_len = max(len(x) for x in lines) + width = shutil.get_terminal_size()[0] + adj_dif = max_len - width + 1 # +1 to avoid too-wide repr (pandas GH #17023) + col_lens = [max((len(x) for x in col), default=0) for col in strcols] + n_cols = len(col_lens) + while adj_dif > 0 and n_cols > 1: + mid = round(n_cols / 2) + adj_dif -= col_lens.pop(mid) + 1 + n_cols = len(col_lens) + max_cols_fitted = max(n_cols - 1, 2) # minus index column; show at least two + self.max_cols_fitted = max_cols_fitted + self.truncate() + return _adjoin(1, self._get_strcols()) + + def _join_multiline(self, strcols, line_width): + adjoin_width = 1 + strcols = list(strcols) + idx = strcols.pop(0) + line_width -= max(len(x) for x in idx) + adjoin_width + col_widths = [max((len(x) for x in col), default=0) for col in strcols] + col_bins = _binify(col_widths, line_width) + nbins = len(col_bins) + blocks = [] + start = 0 + for i, end in enumerate(col_bins): + row = strcols[start:end] + row.insert(0, idx) + if nbins > 1: + nrows = len(row[-1]) + if end <= len(strcols) and i < nbins - 1: + row.append([" \\", *[" "] * (nrows - 1)]) + else: + row.append([" "] * nrows) + blocks.append(_adjoin(adjoin_width, row)) + start = end + return "\n\n".join(blocks) + + def to_string(self, line_width): + strcols = self._get_strcols() + if line_width is None: + return _adjoin(1, strcols) + if self.max_cols > 0: + return self._join_multiline(strcols, line_width) + return self._fit_to_terminal(strcols) + + +def _render_text(frame): + max_cols = _display_option("max_columns") + line_width = _console_width() if _display_option("expand_frame_repr") else None + fmt = _TextFormatter(frame, _display_option("max_rows"), _display_option("min_rows"), max_cols) + return fmt.to_string(line_width) + + +# The scoped style block pandas' NotebookFormatter emits ahead of the table. +_HTML_STYLE = ( + "" +) + + +def _html_escape(s): + return s.replace("&", "&").replace("<", "<").replace(">", ">") + + +class _HtmlBuilder: + """Reproduces pandas NotebookFormatter (DataFrame._repr_html_) markup.""" + + indent_delta = 2 + + def __init__(self, fmt): + self.fmt = fmt + self.tr = fmt.tr_frame + self.ncols = self.tr.ncols + self.th = fmt.is_truncated_horizontally + self.tv = fmt.is_truncated_vertically + self.row_levels = 1 # single-level index, always shown + self.elements = [] + + def write(self, s, indent=0): + self.elements.append(" " * indent + s) + + def _cell(self, s, kind, indent): + rs = _html_escape(str(s)).strip().replace(" ", "  ") + self.write(f"<{kind}>{rs}", indent) + + def write_tr(self, line, indent, header=False, align=None, nindex_levels=0): + self.write("" if align is None else f'', indent) + inner = indent + self.indent_delta + for i, s in enumerate(line): + self._cell(s, "th" if (header or i < nindex_levels) else "td", inner) + self.write("", indent) + + def _col_header(self, indent): + row = ["" if self.tr.col_name is None else str(self.tr.col_name)] + row.extend(_format_labels([col.label for col in self.tr.columns])) + if self.th: + row.insert(self.row_levels + self.fmt.tr_col_num, "...") + self.write_tr(row, indent, header=True, align=self.fmt.justify) + + def _body(self, indent): + index_labels = _format_labels(list(self.tr.index)) + col_cells = [ + _make_fixed_width(self.fmt._format_col_raw(col), "right") for col in self.tr.columns + ] + row = [] + for i in range(self.tr.nrows): + if self.tv and i == self.fmt.tr_row_num: + self.write_tr(["..."] * len(row), indent, nindex_levels=self.row_levels) + row = [index_labels[i], *(col_cells[j][i] for j in range(self.ncols))] + if self.th: + row.insert(self.fmt.tr_col_num + self.row_levels, "...") + self.write_tr(row, indent, nindex_levels=self.row_levels) + + def _table(self, indent=0): + classes = "dataframe" + if not _display_option("html.use_mathjax"): + classes = "dataframe tex2jax_ignore mathjax_ignore" + # pandas keeps the attribute for any non-None border, including 0. + border = _display_option("html.border") + border_attr = "" if border is None else f' border="{border}"' + self.write(f'', indent) + self.write("", indent + self.indent_delta) + self._col_header(indent + 2 * self.indent_delta) + self.write("", indent + self.indent_delta) + self.write("", indent + self.indent_delta) + self._body(indent + 2 * self.indent_delta) + self.write("", indent + self.indent_delta) + self.write("", indent) + + def render(self): + self.write("
") + self.write(_HTML_STYLE) + self._table(0) + self.write("
") + return "\n".join(self.elements) + + +def _render_html(frame): + fmt = _TextFormatter( + frame, + _display_option("max_rows"), + _display_option("min_rows"), + _display_option("max_columns"), + ) + return _HtmlBuilder(fmt).render() + + def _get_matrix_dataframe(matrix, max_rows, min_rows, max_columns, *, mask=None): - if not has_pandas: - return if max_rows is None: # pragma: no branch - max_rows = pd.options.display.max_rows + max_rows = _display_option("max_rows") if min_rows is None: # pragma: no branch - min_rows = pd.options.display.min_rows + min_rows = _display_option("min_rows") if max_columns is None: # pragma: no branch max_columns = _get_max_columns() rows, row_groups = _get_chunk(matrix._nrows, min_rows, max_rows) @@ -264,12 +934,12 @@ def _get_matrix_dataframe(matrix, max_rows, min_rows, max_columns, *, mask=None) column_offset, mask=mask, ) - df = pd.DataFrame(arr, columns=columns, index=rows) + present = _count_present(arr) + truncated = (len(rows), len(columns)) != matrix.shape if ( (mask is None or mask.structure) - and df.shape != matrix.shape - and min(matrix._nvals, max_rows if matrix._nvals <= max_rows else min_rows) - > 2 * df.count().sum() + and truncated + and min(matrix._nvals, max_rows if matrix._nvals <= max_rows else min_rows) > 2 * present ): # The data is sparse and it's better to show in COO format. # SS, SuiteSparse-specific: head @@ -283,47 +953,43 @@ def _get_matrix_dataframe(matrix, max_rows, min_rows, max_columns, *, mask=None) vals = np.zeros(vals.size, dtype=np.uint8) else: vals = np.ones(vals.size, dtype=np.uint8) - df = pd.DataFrame({"row": rows, "col": cols, "val": vals}) - if num_rows < matrix._nvals: - df.loc["..."] = ["..."] * 3 - return df - if mask is not None and not mask.structure and df.shape != matrix.shape: + return _make_coo_frame( + [("row", rows), ("col", cols), ("val", vals)], num_rows < matrix._nvals + ) + if mask is not None and not mask.structure and truncated: # This performs more calculation and uses more memory than I would prefer. # Perhaps we could use the efficient "constant vector or matrix" trick. nonzero = matrix.apply(unary.one["UINT8"]).new(mask=matrix.V, name="") num_rows = matrix._nvals if matrix._nvals <= max_rows else min_rows - if min(nonzero._nvals, num_rows) > 2 * df.count().sum(): + if min(nonzero._nvals, num_rows) > 2 * present: rows, cols, vals = nonzero.ss.head(num_rows, sort=True) if mask.complement: if not vals.flags.writeable: # pragma: no cover (safety) vals = vals.copy() vals[:] = 0 - df = pd.DataFrame({"row": rows, "col": cols, "val": vals}) - if num_rows < nonzero._nvals: - df.loc["..."] = ["..."] * 3 - return df - return df.where(pd.notna(df), "") + return _make_coo_frame( + [("row", rows), ("col", cols), ("val", vals)], num_rows < nonzero._nvals + ) + return _make_dense_frame(arr, columns, rows) def _get_vector_dataframe(vector, max_rows, min_rows, max_columns, *, mask=None): - if not has_pandas: - return if max_rows is None: # pragma: no branch - max_rows = pd.options.display.max_rows + max_rows = _display_option("max_rows") if min_rows is None: # pragma: no branch - min_rows = pd.options.display.min_rows + min_rows = _display_option("min_rows") if max_columns is None: # pragma: no branch max_columns = _get_max_columns() columns, column_groups = _get_chunk(vector._size, max_columns, max_columns) arr = np.full((1, len(columns)), np.nan, dtype=object) for column_group, column_offset in column_groups: _update_vector_array(arr, vector, column_group, column_offset, mask=mask) - df = pd.DataFrame(arr, columns=columns, index=[""]) + present = _count_present(arr) + truncated = len(columns) != vector._size if ( (mask is None or mask.structure) - and df.size != vector._size - and min(vector._nvals, max_rows if vector._nvals <= max_rows else min_rows) - > 2 * df.count().sum() + and truncated + and min(vector._nvals, max_rows if vector._nvals <= max_rows else min_rows) > 2 * present ): # The data is sparse and it's better to show in COO format. # SS, SuiteSparse-specific: head @@ -334,26 +1000,20 @@ def _get_vector_dataframe(vector, max_rows, min_rows, max_columns, *, mask=None) vals = np.zeros(vals.size, dtype=np.uint8) else: vals = np.ones(vals.size, dtype=np.uint8) - df = pd.DataFrame({"index": indices, "val": vals}) - if num_rows < vector._nvals: - df.loc["..."] = ["..."] * 2 - return df - if mask is not None and not mask.structure and df.size != vector._size: + return _make_coo_frame([("index", indices), ("val", vals)], num_rows < vector._nvals) + if mask is not None and not mask.structure and truncated: # This performs more calculation and uses more memory than I would prefer. # Perhaps we could use the efficient "constant vector or matrix" trick. nonzero = vector.apply(unary.one["UINT8"]).new(mask=vector.V, name="") num_rows = vector._nvals if vector._nvals <= max_rows else min_rows - if min(nonzero._nvals, num_rows) > 2 * df.count().sum(): + if min(nonzero._nvals, num_rows) > 2 * present: indices, vals = nonzero.ss.head(num_rows, sort=True) if mask.complement: if not vals.flags.writeable: # pragma: no cover (safety) vals = vals.copy() vals[:] = 0 - df = pd.DataFrame({"index": indices, "val": vals}) - if num_rows < nonzero._nvals: - df.loc["..."] = ["..."] * 2 - return df - return df.where(pd.notna(df), "") + return _make_coo_frame([("index", indices), ("val", vals)], num_rows < nonzero._nvals) + return _make_dense_frame(arr, columns, [""]) def get_format(x, is_transposed=False): @@ -436,14 +1096,9 @@ def vector_expression_header_html(matrix, expr): return create_header_html(name, keys, vals) -def _format_html(name, header, df, collapse): - if has_pandas: - state = "" if collapse else " open" - with pd.option_context("display.show_dimensions", False, "display.large_repr", "truncate"): - details = df._repr_html_() - else: - state = "" - details = "(Install pandas to see a preview of the data)" +def _format_html(name, header, frame, collapse): + state = "" if collapse else " open" + details = _render_html(frame) return ( "
" f"{CSS_STYLE}" @@ -667,17 +1322,12 @@ def format_matrix(matrix, *, max_rows=None, min_rows=None, max_columns=None, mas name, keys, vals, - lower_border=has_pandas, + lower_border=True, name=matrix.name if mask is None else mask.name, ) - if has_pandas: - df = _get_matrix_dataframe(matrix, max_rows, min_rows, max_columns, mask=mask) - if 0 not in matrix.shape: - with pd.option_context( - "display.show_dimensions", False, "display.large_repr", "truncate" - ): - df_repr = df.__repr__() - return f"{header}\n{df_repr}" + if 0 not in matrix.shape: + frame = _get_matrix_dataframe(matrix, max_rows, min_rows, max_columns, mask=mask) + return f"{header}\n{_render_text(frame)}" return header @@ -687,20 +1337,16 @@ def format_vector(vector, *, max_rows=None, min_rows=None, max_columns=None, mas name, keys, vals, - lower_border=has_pandas, + lower_border=True, name=vector.name if mask is None else mask.name, ) - if has_pandas: - df = _get_vector_dataframe(vector, max_rows, min_rows, max_columns, mask=mask) - if vector._size > 0: - if df.columns[0] != "index": - df.columns.name = "index" - df.index = ["value"] - with pd.option_context( - "display.show_dimensions", False, "display.large_repr", "truncate" - ): - df_repr = df.__repr__() - return f"{header}\n{df_repr}" + if vector._size > 0: + frame = _get_vector_dataframe(vector, max_rows, min_rows, max_columns, mask=mask) + if frame.columns[0].label != "index": + # Dense vectors label the corner "index" and the single row "value". + frame.col_name = "index" + frame.index = ["value"] + return f"{header}\n{_render_text(frame)}" return header diff --git a/graphblas/tests/test_formatting.py b/graphblas/tests/test_formatting.py index a6522dcef..264dc0502 100644 --- a/graphblas/tests/test_formatting.py +++ b/graphblas/tests/test_formatting.py @@ -146,41 +146,26 @@ def t(): def test_no_pandas_repr(A, C, v, w): - # This is a bit of a hack... + # The rich repr is hand-rendered, so it no longer depends on pandas: with + # pandas marked absent the output is byte-identical to the pandas-present + # output, data grid and all. (When pandas is genuinely absent both branches + # use the same pandas-free path, so this still exercises that code.) + objs = [A, A.T, C, C.S, ~C.V, v, v.S, ~w.V, w] + expected = [repr(x) for x in objs] has_pandas_prev = formatting.has_pandas formatting.has_pandas = False try: - repr_printer(A, "A", indent=8) - assert repr(A) == ( - '"A_1" nvals nrows ncols dtype format\n' - "gb.Matrix 3 1 5 INT64 bitmapr" - ) - repr_printer(A.T, "A.T", indent=8) - assert repr(A.T) == ( - '"A_1.T" nvals nrows ncols dtype format\n' - "gb.TransposedMatrix 3 5 1 INT64 bitmapc" - ) - repr_printer(C.S, "C.S", indent=8) - assert repr(C.S) == ( - '"C.S" nvals nrows ncols dtype format\n' - "StructuralMask\n" - "of gb.Matrix 8 70 77 INT64 hypercsr" - ) - repr_printer(v, "v", indent=8) - assert repr(v) == ( - '"v" nvals size dtype format\ngb.Vector 3 5 FP64 bitmap' - ) - repr_printer(~w.V, "~w.V", indent=8) - assert repr(~w.V) == ( - '"~w.V" nvals size dtype format\n' - "ComplementedValueMask\n" - "of gb.Vector 4 77 INT64 bitmap" - ) + actual = [repr(x) for x in objs] finally: formatting.has_pandas = has_pandas_prev + assert actual == expected + # The data grid is rendered, not just the header: the border line and the + # values are present. + lines = repr(A).split("\n") + assert lines[2].startswith("----") + assert lines[-1] == "0 0 1 2" -@pytest.mark.skipif("not pd") def test_matrix_repr_small(A, B): repr_printer(A, "A") assert repr(A) == ( @@ -212,7 +197,6 @@ def test_matrix_repr_small(A, B): ) -@pytest.mark.skipif("not pd") def test_matrix_mask_repr_small(A): repr_printer(A.S, "A.S") assert repr(A.S) == ( @@ -404,7 +388,6 @@ def test_matrix_mask_repr_large(C): ) -@pytest.mark.skipif("not pd") def test_vector_repr_small(v): repr_printer(v, "v") assert repr(v) == ( @@ -429,7 +412,6 @@ def test_vector_repr_large(w): ) -@pytest.mark.skipif("not pd") def test_vector_mask_repr_small(v): repr_printer(v.S, "v.S") assert repr(v.S) == ( @@ -517,140 +499,23 @@ def test_scalar_repr(s, t): def test_no_pandas_repr_html(A, C, v, w): - # This is a bit of a hack... + # _repr_html_ is hand-rendered too: marking pandas absent yields output that + # is byte-identical to the pandas-present output, data table included. + objs = [A, A.T, C, C.S, ~C.V, v, v.S, ~w.V, w] + expected = [repr_html(x) for x in objs] has_pandas_prev = formatting.has_pandas formatting.has_pandas = False try: - html_printer(A, "A", indent=8) - assert repr_html(A) == ( - "
" - f"{CSS_STYLE}" - '
A1
\n' - '\n' - " \n" - ' \n' - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - "
gb.Matrix
nvals
nrows
ncols
dtype
format
315INT64bitmapr
\n" - "
\n" - "(Install pandas to see a preview of the data)
" - ) - html_printer(A.T, "A.T", indent=8) - assert repr_html(A.T) == ( - "
" - f"{CSS_STYLE}" - '
A1.T
\n' - '\n' - " \n" - ' \n' - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - "
gb.TransposedMatrix
nvals
nrows
ncols
dtype
format
351INT64bitmapc
\n" - "
\n" - "(Install pandas to see a preview of the data)
" - ) - html_printer(C.S, "C.S", indent=8) - assert repr_html(C.S) == ( - "
" - f"{CSS_STYLE}" - '
C.S
\n' - '\n' - " \n" - ' \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - "
StructuralMask\n'
-            "of\n"
-            "gb.Matrix
nvals
nrows
ncols
dtype
format
87077INT64hypercsr
\n" - "
\n" - "(Install pandas to see a preview of the data)
" - ) - html_printer(v, "v", indent=8) - assert repr_html(v) == ( - "
" - f"{CSS_STYLE}" - '
v
\n' - '\n' - " \n" - ' \n' - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - "
gb.Vector
nvals
size
dtype
format
35FP64bitmap
\n" - "
\n" - "(Install pandas to see a preview of the data)
" - ) - html_printer(~w.V, "~w.V", indent=8) - assert repr_html(~w.V) == ( - "
" - f"{CSS_STYLE}" - '
~w.V
\n' - '\n' - " \n" - ' \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - "
ComplementedValueMask\n'
-            "of\n"
-            "gb.Vector
nvals
size
dtype
format
477INT64bitmap
\n" - "
\n" - "(Install pandas to see a preview of the data)
" - ) + actual = [repr_html(x) for x in objs] finally: formatting.has_pandas = has_pandas_prev + assert actual == expected + # The data table is rendered, not the "install pandas" placeholder. + html = repr_html(A) + assert "install" not in html.lower() + assert '' in html -@pytest.mark.skipif("not pd") def test_matrix_repr_html_small(A, B): html_printer(A, "A") assert repr_html(A) == ( @@ -845,7 +710,6 @@ def test_matrix_repr_html_small(A, B): ) -@pytest.mark.skipif("not pd") def test_matrix_mask_repr_html_small(A): html_printer(A.S, "A.S") assert repr_html(A.S) == ( @@ -2121,7 +1985,6 @@ def test_matrix_mask_repr_html_large(C): ) -@pytest.mark.skipif("not pd") def test_vector_repr_html_small(v): html_printer(v, "v") assert repr_html(v) == ( @@ -2267,7 +2130,6 @@ def test_vector_repr_html_large(w): ) -@pytest.mark.skipif("not pd") def test_vector_mask_repr_html_small(v): html_printer(v.S, "v.S") assert repr_html(v.S) == ( @@ -2889,7 +2751,6 @@ def test_apply_repr(v): ) -@pytest.mark.skipif("not pd") def test_apply_repr_html(v): html_printer(v.apply(unary.one), "v.apply(unary.one)") assert repr_html(v.apply(unary.one)) == ( @@ -2922,7 +2783,6 @@ def test_mxm_repr(A, B): ) -@pytest.mark.skipif("not pd") def test_mxm_repr_html(A, B): html_printer(A.mxm(B), "A.mxm(B)") assert repr_html(A.mxm(B)) == ( @@ -2957,7 +2817,6 @@ def test_mxv_repr(A, v): ) -@pytest.mark.skipif("not pd") def test_mxv_repr_html(A, v): html_printer(A.mxv(v), "A.mxv(v)") assert repr_html(A.mxv(v)) == ( @@ -2980,7 +2839,6 @@ def test_mxv_repr_html(A, v): ) -@pytest.mark.skipif("not pd") def test_matrix_reduce_columns_repr_html(A): # This is implemented using the transpose of A, so make sure we're oriented correctly! html_printer(A.reduce_columnwise(), "A.reduce_columnwise()") @@ -3014,7 +2872,6 @@ def test_matrix_reduce_repr(C, v): ) -@pytest.mark.skipif("not pd") def test_matrix_reduce_repr_html(C, v): html_printer(C.reduce_scalar(), "C.reduce_scalar()", indent=8) assert repr_html(C.reduce_scalar()) == ( @@ -3035,7 +2892,6 @@ def test_matrix_reduce_repr_html(C, v): ) -@pytest.mark.skipif("not pd") def test_matrix_huge(): M = Matrix(int, nrows=2**60, ncols=2**60, name="M") repr_printer(M, "M") @@ -3061,7 +2917,6 @@ def test_matrix_huge(): assert M.isequal(M2) -@pytest.mark.skipif("not pd") def test_matrix_huge_html(): M = Matrix(int, nrows=2**60, ncols=2**60, name="M") html_printer(M, "M") @@ -3254,7 +3109,6 @@ def test_matrix_huge_html(): ) -@pytest.mark.skipif("not pd") def test_vector_huge(): v = Vector(int, size=2**60) repr_printer(v, "v") @@ -3269,7 +3123,6 @@ def test_vector_huge(): assert v2.isequal(v) -@pytest.mark.skipif("not pd") def test_vector_huge_html(): v = Vector(int, size=2**60) html_printer(v, "v") @@ -3407,7 +3260,6 @@ def test_vector_huge_html(): ) -@pytest.mark.skipif("not pd") def test_sparse_vector_repr(): v = Vector.from_coo([100 * i for i in range(100)], [10 * i for i in range(100)], name="v") repr_printer(v, "v") @@ -3561,7 +3413,6 @@ def test_sparse_vector_repr(): ) -@pytest.mark.skipif("not pd") def test_sparse_matrix_repr(): A = Matrix.from_coo( [100 * i for i in range(100)], [10 * i for i in range(100)], list(range(100)), name="A" @@ -3735,7 +3586,6 @@ def test_sparse_matrix_repr(): ) -@pytest.mark.skipif("not pd") def test_infix_expr_repr_html(A, B, v): html_printer(v & v, "v & v") assert repr_html(v & v) == ( @@ -3934,7 +3784,6 @@ def test_infix_expr_repr_html(A, B, v): ) -@pytest.mark.skipif("not pd") def test_infix_expr_repr(A, B, v): repr_printer(v & v, "v & v") assert repr(v & v) == ( @@ -4010,7 +3859,6 @@ def test_infix_expr_repr(A, B, v): ) -@pytest.mark.skipif("not pd") def test_inner_outer_repr_html(v): html_printer(v.inner(v), "v.inner(v)") assert repr_html(v.inner(v)) == ( @@ -4052,7 +3900,6 @@ def test_inner_outer_repr_html(v): ) -@pytest.mark.skipif("not pd") def test_inner_outer_repr(v): # XXX: hmm, having `(GrB_Matrix)` here isn't so pretty repr_printer(v.inner(v), "v.inner(v)") @@ -4073,8 +3920,6 @@ def test_inner_outer_repr(v): @autocompute def test_autocompute(A, B, v): - if not pd: # pragma: no cover (import) - pytest.skip("needs pandas") repr_printer(A & A, "A & A") assert repr(A & A) == ( "gb.MatrixEwiseMultExpr nrows ncols left_dtype right_dtype\n" @@ -4155,8 +4000,6 @@ def test_autocompute(A, B, v): @autocompute def test_autocompute_html(A, B, v): - if not pd: # pragma: no cover (import) - pytest.skip("needs pandas") html_printer(A & A, "A & A") assert repr_html(A & A) == ( "
" @@ -4478,7 +4321,6 @@ def test_autocompute_html(A, B, v): ) -@pytest.mark.skipif("not pd") def test_display_nan(): v = Vector.from_coo([0, 1], [1.0, np.nan], size=3, name="v") repr_printer(v, "v") @@ -4612,7 +4454,6 @@ def test_display_nan(): ) -@pytest.mark.skipif("not pd") def test_large_iso(): A = Matrix(int, nrows=2**60, ncols=2**60) A[:, :] << 1 @@ -4841,7 +4682,6 @@ def test_index_expr_matrix_html(A): ) -@pytest.mark.skipif("not pd") def test_scalar_as_vector(): s = Scalar.from_value(5, is_cscalar=False) # pragma: is_grbscalar v = s._as_vector() @@ -4900,8 +4740,6 @@ def test_scalar_as_vector(): @autocompute def test_index_expr_autocompute(v): - if not pd: # pragma: no cover (import) - pytest.skip("needs pandas") html_printer(v[[0, 1]], "v[[0, 1]]") assert repr_html(v[[0, 1]]) == ( "
" @@ -4955,7 +4793,6 @@ def test_index_expr_autocompute(v): ) -@pytest.mark.skipif("not pd") def test_udt(): record_dtype = np.dtype([("x", np.bool_), ("y", np.int64)], align=True) udt = dtypes.register_anonymous(record_dtype, "record_dtype") @@ -5005,7 +4842,6 @@ def test_udt(): ) -@pytest.mark.skipif("not pd") def test_empty(): v = Vector(int, 0) repr_printer(v, "v") @@ -5030,7 +4866,6 @@ def test_empty(): ) -@pytest.mark.skipif("not pd") def test_vector_as_matrix(): v = Vector.from_coo([1], [2], name="v_A") A = v._as_matrix()