From fa9dac0892a53077e4cb9b8e67c127c77470267b Mon Sep 17 00:00:00 2001 From: maksii <1761348+maksii@users.noreply.github.com> Date: Sat, 28 Feb 2026 14:24:29 +0200 Subject: [PATCH 01/20] Add extended table metadata retrieval and models for columns and option sets - Implemented methods to fetch detailed metadata for tables, including columns and relationships. - Introduced `ColumnMetadata`, `OptionItem`, and `OptionSetInfo` models to represent column and option set data structures. - Updated `get` method in `TableOperations` to support optional parameters for including columns and relationships in the response. - Enhanced tests to cover new functionality and ensure backward compatibility. This update improves the SDK's ability to interact with Dataverse metadata, providing richer data for developers. --- .claude/skills/dataverse-sdk-use/SKILL.md | 57 ++++ README.md | 34 +++ .../Dataverse/common/constants.py | 27 ++ src/PowerPlatform/Dataverse/data/_odata.py | 271 +++++++++++++++++ .../Dataverse/models/metadata.py | 205 +++++++++++++ .../Dataverse/operations/tables.py | 279 +++++++++++++++++- tests/unit/models/test_metadata.py | 181 ++++++++++++ tests/unit/test_tables_operations.py | 256 ++++++++++++++++ 8 files changed, 1306 insertions(+), 4 deletions(-) create mode 100644 src/PowerPlatform/Dataverse/models/metadata.py create mode 100644 tests/unit/models/test_metadata.py diff --git a/.claude/skills/dataverse-sdk-use/SKILL.md b/.claude/skills/dataverse-sdk-use/SKILL.md index c2010a70..6dc77897 100644 --- a/.claude/skills/dataverse-sdk-use/SKILL.md +++ b/.claude/skills/dataverse-sdk-use/SKILL.md @@ -223,6 +223,63 @@ for table in tables: print(table) ``` +#### Get Extended Table Metadata +```python +# Get table with column metadata +info = client.tables.get("account", include_columns=True) +for col in info["columns"]: + print(f"{col.logical_name} ({col.attribute_type})") + +# Get table with relationship metadata +info = client.tables.get("account", include_relationships=True) + +# Get specific entity properties +info = client.tables.get("account", select=["DisplayName", "Description"]) +``` + +#### List Columns +```python +from PowerPlatform.Dataverse.models.metadata import ColumnMetadata + +columns = client.tables.get_columns("account") +for col in columns: + print(f"{col.schema_name}: {col.attribute_type} (required: {col.required_level})") + +# Filter to specific column types (OData syntax, fully-qualified enum) +picklists = client.tables.get_columns( + "account", + filter="AttributeType eq Microsoft.Dynamics.CRM.AttributeTypeCode'Picklist'", +) +``` + +#### Get Single Column +```python +col = client.tables.get_column("account", "emailaddress1") +if col: + print(f"Type: {col.attribute_type}, Required: {col.required_level}") +``` + +#### Get Column Options (Picklist/Choice Values) +```python +from PowerPlatform.Dataverse.models.metadata import OptionSetInfo + +options = client.tables.get_column_options("account", "accountcategorycode") +if options: + for opt in options.options: + print(f" Value={opt.value}, Label={opt.label}") +``` + +#### List Table Relationships +```python +# All relationships +rels = client.tables.list_relationships("account") + +# Specific type: "one_to_many" / "1:N", "many_to_one" / "N:1", "many_to_many" / "N:N" +rels = client.tables.list_relationships("account", relationship_type="one_to_many") +for rel in rels: + print(f"{rel['SchemaName']}: {rel.get('ReferencingEntity')}") +``` + #### Delete Tables ```python client.tables.delete("new_Product") diff --git a/README.md b/README.md index a83dcf9b..f70c5e4a 100644 --- a/README.md +++ b/README.md @@ -309,6 +309,40 @@ client.tables.remove_columns("new_Product", ["new_Category"]) client.tables.delete("new_Product") ``` +```python +# Get extended table metadata with columns +info = client.tables.get("account", include_columns=True) +for col in info["columns"]: + print(f"{col.logical_name} ({col.attribute_type})") + +# Get extended table metadata with relationships +info = client.tables.get("account", include_relationships=True) +for rel in info.get("one_to_many_relationships", []): + print(rel["SchemaName"]) + +# Get specific entity properties +info = client.tables.get("account", select=["DisplayName", "Description"]) + +# List all columns of a table +columns = client.tables.get_columns("account") +for col in columns: + print(f"{col.schema_name}: {col.attribute_type} (required: {col.required_level})") + +# Get a specific column's metadata +col = client.tables.get_column("account", "emailaddress1") +if col: + print(f"Type: {col.attribute_type}, Required: {col.required_level}") + +# Get picklist/choice column options +options = client.tables.get_column_options("account", "accountcategorycode") +if options: + for opt in options.options: + print(f" {opt.value}: {opt.label}") + +# List relationships for a table +rels = client.tables.list_relationships("account", relationship_type="one_to_many") +``` + > **Important**: All custom column names must include the customization prefix value (e.g., `"new_"`). > This ensures explicit, predictable naming and aligns with Dataverse metadata requirements. diff --git a/src/PowerPlatform/Dataverse/common/constants.py b/src/PowerPlatform/Dataverse/common/constants.py index c18a74f8..33724679 100644 --- a/src/PowerPlatform/Dataverse/common/constants.py +++ b/src/PowerPlatform/Dataverse/common/constants.py @@ -29,3 +29,30 @@ CASCADE_BEHAVIOR_RESTRICT = "Restrict" """Prevent the referenced table record from being deleted when referencing table records exist.""" + +# AttributeMetadata derived type OData identifiers +# Used when casting Attributes collection to a specific derived type in Web API URLs +ODATA_TYPE_PICKLIST_ATTRIBUTE = "Microsoft.Dynamics.CRM.PicklistAttributeMetadata" +ODATA_TYPE_BOOLEAN_ATTRIBUTE = "Microsoft.Dynamics.CRM.BooleanAttributeMetadata" +ODATA_TYPE_MULTISELECT_PICKLIST_ATTRIBUTE = "Microsoft.Dynamics.CRM.MultiSelectPicklistAttributeMetadata" +ODATA_TYPE_STRING_ATTRIBUTE = "Microsoft.Dynamics.CRM.StringAttributeMetadata" +ODATA_TYPE_INTEGER_ATTRIBUTE = "Microsoft.Dynamics.CRM.IntegerAttributeMetadata" +ODATA_TYPE_DECIMAL_ATTRIBUTE = "Microsoft.Dynamics.CRM.DecimalAttributeMetadata" +ODATA_TYPE_DOUBLE_ATTRIBUTE = "Microsoft.Dynamics.CRM.DoubleAttributeMetadata" +ODATA_TYPE_MONEY_ATTRIBUTE = "Microsoft.Dynamics.CRM.MoneyAttributeMetadata" +ODATA_TYPE_DATETIME_ATTRIBUTE = "Microsoft.Dynamics.CRM.DateTimeAttributeMetadata" +ODATA_TYPE_MEMO_ATTRIBUTE = "Microsoft.Dynamics.CRM.MemoAttributeMetadata" +ODATA_TYPE_FILE_ATTRIBUTE = "Microsoft.Dynamics.CRM.FileAttributeMetadata" + +# Attribute type code values returned in the AttributeType property of attribute metadata +ATTRIBUTE_TYPE_PICKLIST = "Picklist" +ATTRIBUTE_TYPE_BOOLEAN = "Boolean" +ATTRIBUTE_TYPE_STRING = "String" +ATTRIBUTE_TYPE_INTEGER = "Integer" +ATTRIBUTE_TYPE_DECIMAL = "Decimal" +ATTRIBUTE_TYPE_DOUBLE = "Double" +ATTRIBUTE_TYPE_MONEY = "Money" +ATTRIBUTE_TYPE_DATETIME = "DateTime" +ATTRIBUTE_TYPE_MEMO = "Memo" +ATTRIBUTE_TYPE_LOOKUP = "Lookup" +ATTRIBUTE_TYPE_UNIQUEIDENTIFIER = "Uniqueidentifier" diff --git a/src/PowerPlatform/Dataverse/data/_odata.py b/src/PowerPlatform/Dataverse/data/_odata.py index eb341f22..fe96308a 100644 --- a/src/PowerPlatform/Dataverse/data/_odata.py +++ b/src/PowerPlatform/Dataverse/data/_odata.py @@ -34,6 +34,11 @@ METADATA_COLUMN_NOT_FOUND, VALIDATION_UNSUPPORTED_CACHE_KIND, ) +from ..common.constants import ( + ODATA_TYPE_BOOLEAN_ATTRIBUTE, + ODATA_TYPE_MULTISELECT_PICKLIST_ATTRIBUTE, + ODATA_TYPE_PICKLIST_ATTRIBUTE, +) from .. import __version__ as _SDK_VERSION @@ -80,6 +85,15 @@ def build( class _ODataClient(_FileUploadMixin, _RelationshipOperationsMixin): """Dataverse Web API client: CRUD, SQL-over-API, and table metadata helpers.""" + _RELATIONSHIP_TYPE_MAP = { + "one_to_many": "/OneToManyRelationships", + "1:N": "/OneToManyRelationships", + "many_to_one": "/ManyToOneRelationships", + "N:1": "/ManyToOneRelationships", + "many_to_many": "/ManyToManyRelationships", + "N:N": "/ManyToManyRelationships", + } + @staticmethod def _escape_odata_quotes(value: str) -> str: """Escape single quotes for OData queries (by doubling them).""" @@ -1463,6 +1477,263 @@ def _list_tables( r = self._request("get", url, params=params) return r.json().get("value", []) + # ------------------------------------------------------------------------- + # Extended table metadata (columns, relationships, option sets) + # ------------------------------------------------------------------------- + + def _get_table_metadata( + self, + table_schema_name: str, + select: Optional[List[str]] = None, + include_attributes: bool = False, + include_one_to_many: bool = False, + include_many_to_one: bool = False, + include_many_to_many: bool = False, + ) -> Optional[Dict[str, Any]]: + """Retrieve rich table metadata using EntityDefinitions with optional $select and $expand. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param select: Optional list of PascalCase property names to project. + :type select: ``list[str]`` or ``None`` + :param include_attributes: Expand ``Attributes`` collection. + :type include_attributes: ``bool`` + :param include_one_to_many: Expand ``OneToManyRelationships``. + :type include_one_to_many: ``bool`` + :param include_many_to_one: Expand ``ManyToOneRelationships``. + :type include_many_to_one: ``bool`` + :param include_many_to_many: Expand ``ManyToManyRelationships``. + :type include_many_to_many: ``bool`` + + :return: Raw entity metadata dict, or ``None`` if not found (404). + :rtype: ``dict[str, Any]`` | ``None`` + + :raises HttpError: If the request fails (non-404). + """ + logical_lower = table_schema_name.lower() + logical_escaped = self._escape_odata_quotes(logical_lower) + url = f"{self.api}/EntityDefinitions(LogicalName='{logical_escaped}')" + + params: Dict[str, str] = {} + if select is not None and isinstance(select, str): + raise TypeError("select must be a list of property names, not a bare string") + if select: + base_fields = {"MetadataId", "LogicalName", "SchemaName", "EntitySetName"} + merged = list(base_fields | set(select)) + params["$select"] = ",".join(merged) + expand_parts: List[str] = [] + if include_attributes: + expand_parts.append("Attributes") + if include_one_to_many: + expand_parts.append("OneToManyRelationships") + if include_many_to_one: + expand_parts.append("ManyToOneRelationships") + if include_many_to_many: + expand_parts.append("ManyToManyRelationships") + if expand_parts: + params["$expand"] = ",".join(expand_parts) + + try: + r = self._request("get", url, params=params) + return r.json() + except HttpError as e: + if e.status_code == 404: + return None + raise + + def _get_table_columns( + self, + table_schema_name: str, + select: Optional[List[str]] = None, + filter: Optional[str] = None, + ) -> List[Dict[str, Any]]: + """Get all columns/attributes for a table using the Attributes collection endpoint. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param select: Optional list of PascalCase attribute property names to project. + :type select: ``list[str]`` or ``None`` + :param filter: Optional OData $filter expression for attributes. + :type filter: ``str`` or ``None`` + + :return: List of raw attribute metadata dicts. + :rtype: ``list[dict[str, Any]]`` + + :raises HttpError: If the request fails. + """ + logical_lower = table_schema_name.lower() + logical_escaped = self._escape_odata_quotes(logical_lower) + url = f"{self.api}/EntityDefinitions(LogicalName='{logical_escaped}')/Attributes" + + params: Dict[str, str] = {} + if select is not None and isinstance(select, str): + raise TypeError("select must be a list of property names, not a bare string") + if select: + params["$select"] = ",".join(select) + else: + params["$select"] = ",".join( + [ + "LogicalName", + "SchemaName", + "DisplayName", + "AttributeType", + "AttributeTypeName", + "IsCustomAttribute", + "IsPrimaryId", + "IsPrimaryName", + "RequiredLevel", + "IsValidForCreate", + "IsValidForUpdate", + "IsValidForRead", + "MetadataId", + ] + ) + if filter: + params["$filter"] = filter + + r = self._request("get", url, params=params) + return r.json().get("value", []) + + def _get_table_column( + self, + table_schema_name: str, + column_logical_name: str, + select: Optional[List[str]] = None, + ) -> Optional[Dict[str, Any]]: + """Get metadata for a single specific column using the alternate key pattern. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param column_logical_name: Logical name of the column. + :type column_logical_name: ``str`` + :param select: Optional list of PascalCase attribute property names to project. + :type select: ``list[str]`` or ``None`` + + :return: Raw attribute metadata dict, or ``None`` if not found (404). + :rtype: ``dict[str, Any]`` | ``None`` + + :raises HttpError: If the request fails (non-404). + """ + table_lower = self._escape_odata_quotes(table_schema_name.lower()) + column_lower = self._escape_odata_quotes(column_logical_name.lower()) + url = f"{self.api}/EntityDefinitions(LogicalName='{table_lower}')/Attributes(LogicalName='{column_lower}')" + + params: Dict[str, str] = {} + if select is not None and isinstance(select, str): + raise TypeError("select must be a list of property names, not a bare string") + if select: + params["$select"] = ",".join(select) + + try: + r = self._request("get", url, params=params) + return r.json() + except HttpError as e: + if e.status_code == 404: + return None + raise + + def _get_column_optionset( + self, + table_schema_name: str, + column_logical_name: str, + ) -> Optional[Dict[str, Any]]: + """Get the option set definition for a Picklist, MultiSelectPicklist, or Boolean column. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param column_logical_name: Logical name of the column. + :type column_logical_name: ``str`` + + :return: Raw option set metadata dict, or ``None`` if not found or not an option-set column. + :rtype: ``dict[str, Any]`` | ``None`` + + :raises HttpError: If the request fails (non-400/404). + """ + table_lower = self._escape_odata_quotes(table_schema_name.lower()) + column_lower = self._escape_odata_quotes(column_logical_name.lower()) + base = f"{self.api}/EntityDefinitions(LogicalName='{table_lower}')/Attributes(LogicalName='{column_lower}')" + + params = {"$select": "LogicalName", "$expand": "OptionSet,GlobalOptionSet"} + + for cast_type in [ + ODATA_TYPE_PICKLIST_ATTRIBUTE, + ODATA_TYPE_BOOLEAN_ATTRIBUTE, + ODATA_TYPE_MULTISELECT_PICKLIST_ATTRIBUTE, + ]: + url = f"{base}/{cast_type}" + try: + r = self._request("get", url, params=params) + data = r.json() + option_set = data.get("OptionSet") + if option_set is None: + option_set = data.get("GlobalOptionSet") + if option_set is not None: + return option_set + except HttpError as e: + if e.status_code not in (400, 404): + raise + + return None + + def _list_table_relationships( + self, + table_schema_name: str, + relationship_type: Optional[str] = None, + select: Optional[List[str]] = None, + ) -> List[Dict[str, Any]]: + """List relationship metadata for a table, optionally filtered by type. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param relationship_type: Optional filter (e.g. ``"one_to_many"``, ``"1:N"``, ``"N:1"``). + :type relationship_type: ``str`` or ``None`` + :param select: Optional list of PascalCase property names to project. + :type select: ``list[str]`` or ``None`` + + :return: List of raw relationship metadata dicts. + :rtype: ``list[dict[str, Any]]`` + + :raises ValueError: If ``relationship_type`` is invalid. + :raises HttpError: If the request fails. + """ + table_lower = self._escape_odata_quotes(table_schema_name.lower()) + base_url = f"{self.api}/EntityDefinitions(LogicalName='{table_lower}')" + + params: Dict[str, str] = {} + if select is not None and isinstance(select, str): + raise TypeError("select must be a list of property names, not a bare string") + if select: + params["$select"] = ",".join(select) + + if relationship_type is not None: + sub_path = self._RELATIONSHIP_TYPE_MAP.get(relationship_type) + if sub_path is None: + raise ValueError( + f"Invalid relationship_type: {relationship_type!r}. " + f"Valid values: {list(self._RELATIONSHIP_TYPE_MAP.keys())} or None for all." + ) + url = f"{base_url}{sub_path}" + r = self._request("get", url, params=params) + results = r.json().get("value", []) + type_tag = sub_path.strip("/").replace("Relationships", "") + for item in results: + item["_relationship_type"] = type_tag + return results + + all_results: List[Dict[str, Any]] = [] + for sub_path, type_tag in [ + ("/OneToManyRelationships", "OneToMany"), + ("/ManyToOneRelationships", "ManyToOne"), + ("/ManyToManyRelationships", "ManyToMany"), + ]: + url = f"{base_url}{sub_path}" + r = self._request("get", url, params=params) + items = r.json().get("value", []) + for item in items: + item["_relationship_type"] = type_tag + all_results.extend(items) + return all_results + def _delete_table(self, table_schema_name: str) -> None: """Delete a table by schema name. diff --git a/src/PowerPlatform/Dataverse/models/metadata.py b/src/PowerPlatform/Dataverse/models/metadata.py new file mode 100644 index 00000000..5289abc9 --- /dev/null +++ b/src/PowerPlatform/Dataverse/models/metadata.py @@ -0,0 +1,205 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Metadata models for table column and option set definitions.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +__all__ = ["ColumnMetadata", "OptionItem", "OptionSetInfo"] + + +@dataclass +class ColumnMetadata: + """ + Metadata for a single table column (attribute). + + :param logical_name: Logical name of the column (e.g., ``"emailaddress1"``). + :type logical_name: :class:`str` + :param schema_name: Schema name of the column (e.g., ``"EMailAddress1"``). + :type schema_name: :class:`str` + :param display_name: Localized display name, or ``None`` if not available. + :type display_name: :class:`str` or None + :param attribute_type: Attribute type (e.g., ``"String"``, ``"Picklist"``). + :type attribute_type: :class:`str` + :param attribute_type_name: Attribute type name (e.g., ``"StringType"``). + :type attribute_type_name: :class:`str` or None + :param is_custom_attribute: Whether the column is custom. + :type is_custom_attribute: :class:`bool` + :param is_primary_id: Whether this is the primary ID column. + :type is_primary_id: :class:`bool` + :param is_primary_name: Whether this is the primary name column. + :type is_primary_name: :class:`bool` + :param required_level: Required level (e.g., ``"None"``, ``"SystemRequired"``). + :type required_level: :class:`str` or None + :param is_valid_for_create: Whether valid for create operations. + :type is_valid_for_create: :class:`bool` + :param is_valid_for_update: Whether valid for update operations. + :type is_valid_for_update: :class:`bool` + :param is_valid_for_read: Whether valid for read operations. + :type is_valid_for_read: :class:`bool` + :param metadata_id: GUID of the attribute metadata. + :type metadata_id: :class:`str` or None + """ + + logical_name: str = "" + schema_name: str = "" + display_name: Optional[str] = None + attribute_type: str = "" + attribute_type_name: Optional[str] = None + is_custom_attribute: bool = False + is_primary_id: bool = False + is_primary_name: bool = False + required_level: Optional[str] = None + is_valid_for_create: bool = False + is_valid_for_update: bool = False + is_valid_for_read: bool = False + metadata_id: Optional[str] = None + + @classmethod + def from_api_response(cls, data: Dict[str, Any]) -> ColumnMetadata: + """Create a ``ColumnMetadata`` from a raw Web API attribute response. + + :param data: Raw JSON dict from the Dataverse Web API. + :type data: :class:`dict` + :return: Parsed column metadata instance. + :rtype: :class:`ColumnMetadata` + """ + display_name = None + dn = data.get("DisplayName") + if isinstance(dn, dict): + ull = dn.get("UserLocalizedLabel") + if isinstance(ull, dict): + display_name = ull.get("Label") + + attribute_type_name = None + atn = data.get("AttributeTypeName") + if isinstance(atn, dict): + attribute_type_name = atn.get("Value") + + required_level = None + rl = data.get("RequiredLevel") + if isinstance(rl, dict): + required_level = rl.get("Value") + + return cls( + logical_name=data.get("LogicalName", ""), + schema_name=data.get("SchemaName", ""), + display_name=display_name, + attribute_type=data.get("AttributeType", ""), + attribute_type_name=attribute_type_name, + is_custom_attribute=data.get("IsCustomAttribute", False), + is_primary_id=data.get("IsPrimaryId", False), + is_primary_name=data.get("IsPrimaryName", False), + required_level=required_level, + is_valid_for_create=data.get("IsValidForCreate", False), + is_valid_for_update=data.get("IsValidForUpdate", False), + is_valid_for_read=data.get("IsValidForRead", False), + metadata_id=data.get("MetadataId"), + ) + + +@dataclass +class OptionItem: + """ + A single option/choice value in an option set. + + :param value: Numeric option value. + :type value: :class:`int` + :param label: Localized display text, or ``None`` if not available. + :type label: :class:`str` or None + """ + + value: int = 0 + label: Optional[str] = None + + @classmethod + def from_api_response(cls, data: Dict[str, Any]) -> OptionItem: + """Create an ``OptionItem`` from a raw Web API option response. + + :param data: Raw JSON dict for a single option/choice value. + :type data: :class:`dict` + :return: Parsed option item. + :rtype: :class:`OptionItem` + """ + label = None + lbl = data.get("Label") + if isinstance(lbl, dict): + ull = lbl.get("UserLocalizedLabel") + if isinstance(ull, dict): + label = ull.get("Label") + return cls(value=data.get("Value", 0), label=label) + + +@dataclass +class OptionSetInfo: + """ + Option set definition including all option values. + + .. note:: + For Boolean option sets, options are ordered as + ``[FalseOption, TrueOption]``. Use :attr:`OptionItem.value` to + distinguish rather than relying on list index. + + :param name: Option set name. + :type name: :class:`str` or None + :param display_name: Localized display name. + :type display_name: :class:`str` or None + :param is_global: Whether this is a global option set. + :type is_global: :class:`bool` + :param option_set_type: Type (e.g., ``"Picklist"`` or ``"Boolean"``). + :type option_set_type: :class:`str` or None + :param options: List of option items. + :type options: :class:`list` of :class:`OptionItem` + :param metadata_id: GUID of the option set metadata. + :type metadata_id: :class:`str` or None + """ + + name: Optional[str] = None + display_name: Optional[str] = None + is_global: bool = False + option_set_type: Optional[str] = None + options: List[OptionItem] = field(default_factory=list) + metadata_id: Optional[str] = None + + @classmethod + def from_api_response(cls, data: Dict[str, Any]) -> OptionSetInfo: + """Create an ``OptionSetInfo`` from a raw Web API option set response. + + Handles both picklist-style (``Options`` array) and boolean-style + (``TrueOption``/``FalseOption``) option sets. + + :param data: Raw JSON dict from the Dataverse Web API. + :type data: :class:`dict` + :return: Parsed option set info. + :rtype: :class:`OptionSetInfo` + """ + display_name = None + dn = data.get("DisplayName") + if isinstance(dn, dict): + ull = dn.get("UserLocalizedLabel") + if isinstance(ull, dict): + display_name = ull.get("Label") + + options: List[OptionItem] = [] + raw_options = data.get("Options") + if isinstance(raw_options, list): + options = [OptionItem.from_api_response(o) for o in raw_options] + else: + false_opt = data.get("FalseOption") + true_opt = data.get("TrueOption") + if isinstance(false_opt, dict): + options.append(OptionItem.from_api_response(false_opt)) + if isinstance(true_opt, dict): + options.append(OptionItem.from_api_response(true_opt)) + + return cls( + name=data.get("Name"), + display_name=display_name, + is_global=data.get("IsGlobal", False), + option_set_type=data.get("OptionSetType"), + options=options, + metadata_id=data.get("MetadataId"), + ) diff --git a/src/PowerPlatform/Dataverse/operations/tables.py b/src/PowerPlatform/Dataverse/operations/tables.py index 6d71d929..a95f0e5c 100644 --- a/src/PowerPlatform/Dataverse/operations/tables.py +++ b/src/PowerPlatform/Dataverse/operations/tables.py @@ -15,6 +15,7 @@ RelationshipInfo, ) from ..models.labels import Label, LocalizedLabel +from ..models.metadata import ColumnMetadata, OptionSetInfo from ..common.constants import CASCADE_BEHAVIOR_REMOVE_LINK if TYPE_CHECKING: @@ -155,27 +156,297 @@ def delete(self, table: str) -> None: # -------------------------------------------------------------------- get - def get(self, table: str) -> Optional[Dict[str, Any]]: - """Get basic metadata for a table if it exists. + def get( + self, + table: str, + *, + select: Optional[List[str]] = None, + include_columns: bool = False, + include_relationships: bool = False, + ) -> Optional[Dict[str, Any]]: + """Get basic or extended metadata for a table if it exists. + + When no extra parameters are passed, returns the same lightweight + result as before (backward compatible). Use optional parameters to + request richer metadata including columns and relationships. :param table: Schema name of the table (e.g. ``"new_MyTestTable"`` or ``"account"``). :type table: :class:`str` + :param select: Optional list of PascalCase EntityDefinition property + names to include (e.g. ``["DisplayName", "Description"]``). + :type select: :class:`list` of :class:`str` or None + :param include_columns: If ``True``, expands and returns all column + metadata as :class:`~PowerPlatform.Dataverse.models.metadata.ColumnMetadata` + instances in the ``columns`` key. + :type include_columns: :class:`bool` + :param include_relationships: If ``True``, expands and returns + ``one_to_many_relationships``, ``many_to_one_relationships``, and + ``many_to_many_relationships``. + :type include_relationships: :class:`bool` :return: Dictionary containing ``table_schema_name``, ``table_logical_name``, ``entity_set_name``, and ``metadata_id``. + When extended params are used, may also include ``columns``, + ``one_to_many_relationships``, ``many_to_one_relationships``, + ``many_to_many_relationships``, and any extra selected properties. Returns None if the table is not found. :rtype: :class:`dict` or None Example:: + # Basic usage (unchanged) info = client.tables.get("new_MyTestTable") if info: print(f"Logical name: {info['table_logical_name']}") - print(f"Entity set: {info['entity_set_name']}") + + # Extended with columns + info = client.tables.get("account", include_columns=True) + for col in info.get("columns", []): + print(f"{col.logical_name} ({col.attribute_type})") + + # Extended with relationships + info = client.tables.get("account", include_relationships=True) + """ + # When no extra parameters are passed, use the original lightweight lookup. + # This ensures backward compatibility -- existing callers get identical behavior. + if not include_columns and not include_relationships and select is None: + with self._client._scoped_odata() as od: + return od._get_table_info(table) + + # Extended metadata retrieval when any extra parameter is used + with self._client._scoped_odata() as od: + raw = od._get_table_metadata( + table, + select=select, + include_attributes=include_columns, + include_one_to_many=include_relationships, + include_many_to_one=include_relationships, + include_many_to_many=include_relationships, + ) + if raw is None: + return None + + # Build result dict starting with the standard 4 fields + result: Dict[str, Any] = { + "table_schema_name": raw.get("SchemaName", table), + "table_logical_name": raw.get("LogicalName"), + "entity_set_name": raw.get("EntitySetName"), + "metadata_id": raw.get("MetadataId"), + "columns_created": [], + } + + # Include any extra selected entity properties + if select: + for prop in select: + if prop not in ("SchemaName", "LogicalName", "EntitySetName", "MetadataId"): + result[prop] = raw.get(prop) + + # Convert expanded Attributes into ColumnMetadata instances + if include_columns and "Attributes" in raw: + result["columns"] = [ColumnMetadata.from_api_response(a) for a in raw["Attributes"]] + + # Include expanded relationship collections as raw dicts + if include_relationships: + if "OneToManyRelationships" in raw: + result["one_to_many_relationships"] = raw["OneToManyRelationships"] + if "ManyToOneRelationships" in raw: + result["many_to_one_relationships"] = raw["ManyToOneRelationships"] + if "ManyToManyRelationships" in raw: + result["many_to_many_relationships"] = raw["ManyToManyRelationships"] + + return result + + # -------------------------------------------------------------- get_columns + + def get_columns( + self, + table: str, + *, + select: Optional[List[str]] = None, + filter: Optional[str] = None, + ) -> List[ColumnMetadata]: + """Get column (attribute) metadata for a table. + + Returns a list of :class:`~PowerPlatform.Dataverse.models.metadata.ColumnMetadata` + instances representing each column in the table. + + :param table: Schema name of the table (e.g. ``"account"`` + or ``"new_MyTestTable"``). + :type table: :class:`str` + :param select: Optional list of attribute metadata property names to + include (PascalCase, e.g. ``["LogicalName", "AttributeType"]``). + When ``None``, returns a default set of useful properties. + :type select: :class:`list` of :class:`str` or None + :param filter: Optional OData ``$filter`` expression. Column names in + filter expressions must use PascalCase metadata property names. + + .. note:: + Enum values in filters must use fully-qualified type names, + e.g. ``"AttributeType eq Microsoft.Dynamics.CRM.AttributeTypeCode'Picklist'"``. + + .. note:: + The ``filter`` expression is passed directly to the Web API. + If constructing filters from external input, ensure values are + properly escaped (single quotes doubled) to avoid malformed queries. + + :type filter: :class:`str` or None + + :return: List of column metadata. + :rtype: :class:`list` of :class:`~PowerPlatform.Dataverse.models.metadata.ColumnMetadata` + + :raises ~PowerPlatform.Dataverse.core.errors.HttpError: + If the Web API request fails. + + Example:: + + # Get all columns + columns = client.tables.get_columns("account") + for col in columns: + print(f"{col.logical_name}: {col.attribute_type}") + + # Filter to only picklist columns + picklists = client.tables.get_columns( + "account", + filter="AttributeType eq Microsoft.Dynamics.CRM.AttributeTypeCode'Picklist'", + ) + """ + with self._client._scoped_odata() as od: + raw_list = od._get_table_columns(table, select=select, filter=filter) + return [ColumnMetadata.from_api_response(item) for item in raw_list] + + # --------------------------------------------------------------- get_column + + def get_column( + self, + table: str, + column: str, + *, + select: Optional[List[str]] = None, + ) -> Optional[ColumnMetadata]: + """Get metadata for a single column by logical name. + + :param table: Schema name of the table (e.g. ``"account"``). + :type table: :class:`str` + :param column: Logical name of the column (e.g. ``"emailaddress1"``). + :type column: :class:`str` + :param select: Optional list of attribute metadata property names to + include (PascalCase). When ``None``, returns all properties. + :type select: :class:`list` of :class:`str` or None + + :return: Column metadata, or ``None`` if the column is not found. + :rtype: :class:`~PowerPlatform.Dataverse.models.metadata.ColumnMetadata` + or None + + :raises ~PowerPlatform.Dataverse.core.errors.HttpError: + If the Web API request fails (other than 404). + + Example:: + + col = client.tables.get_column("account", "emailaddress1") + if col: + print(f"Type: {col.attribute_type}, Required: {col.required_level}") + """ + with self._client._scoped_odata() as od: + raw = od._get_table_column(table, column, select=select) + if raw is None: + return None + return ColumnMetadata.from_api_response(raw) + + # -------------------------------------------------------- get_column_options + + def get_column_options( + self, + table: str, + column: str, + ) -> Optional[OptionSetInfo]: + """Get option set values for a Picklist, MultiSelect, or Boolean column. + + This method retrieves the available choices for a column that uses an + option set. For Picklist and MultiSelect columns, the options are the + defined choice values. For Boolean columns, the result contains the + True and False option labels. + + :param table: Schema name of the table (e.g. ``"account"``). + :type table: :class:`str` + :param column: Logical name of the column (e.g. + ``"accountcategorycode"``). + :type column: :class:`str` + + :return: Option set information with available choices, or ``None`` if + the column is not a choice/boolean type. + :rtype: :class:`~PowerPlatform.Dataverse.models.metadata.OptionSetInfo` + or None + + :raises ~PowerPlatform.Dataverse.core.errors.HttpError: + If the Web API request fails (other than expected type mismatches). + + Example:: + + options = client.tables.get_column_options("account", "accountcategorycode") + if options: + for opt in options.options: + print(f" {opt.value}: {opt.label}") """ with self._client._scoped_odata() as od: - return od._get_table_info(table) + raw = od._get_column_optionset(table, column) + if raw is None: + return None + return OptionSetInfo.from_api_response(raw) + + # -------------------------------------------------------- list_relationships + + def list_relationships( + self, + table: str, + *, + relationship_type: Optional[str] = None, + select: Optional[List[str]] = None, + ) -> List[Dict[str, Any]]: + """List relationship metadata for a table. + + Returns relationship definitions from the Web API. Each dict in the + result has a ``_relationship_type`` key added by the SDK with value + ``"OneToMany"``, ``"ManyToOne"``, or ``"ManyToMany"`` to identify the + relationship category. + + :param table: Schema name of the table (e.g. ``"account"``). + :type table: :class:`str` + :param relationship_type: Filter by relationship type. Valid values: + ``"one_to_many"`` (or ``"1:N"``), ``"many_to_one"`` + (or ``"N:1"``), ``"many_to_many"`` (or ``"N:N"``), or + ``None`` (default) to return all types. + :type relationship_type: :class:`str` or None + :param select: Optional list of relationship property names to include + (PascalCase, e.g. ``["SchemaName", "ReferencedEntity"]``). + :type select: :class:`list` of :class:`str` or None + + :return: List of relationship metadata dictionaries from the Web API. + :rtype: :class:`list` of :class:`dict` + + :raises ValueError: If ``relationship_type`` is not a valid value. + :raises ~PowerPlatform.Dataverse.core.errors.HttpError: + If the Web API request fails. + + Example:: + + # All relationships + rels = client.tables.list_relationships("account") + + # Only one-to-many + rels = client.tables.list_relationships( + "account", + relationship_type="one_to_many", + ) + for rel in rels: + print(f"{rel['SchemaName']}: {rel.get('ReferencingEntity')}") + """ + with self._client._scoped_odata() as od: + return od._list_table_relationships( + table, + relationship_type=relationship_type, + select=select, + ) # ------------------------------------------------------------------- list diff --git a/tests/unit/models/test_metadata.py b/tests/unit/models/test_metadata.py new file mode 100644 index 00000000..0b128a59 --- /dev/null +++ b/tests/unit/models/test_metadata.py @@ -0,0 +1,181 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Tests for metadata models.""" + +from PowerPlatform.Dataverse.models.metadata import ( + ColumnMetadata, + OptionItem, + OptionSetInfo, +) + + +class TestColumnMetadata: + """Tests for ColumnMetadata.""" + + def test_from_api_response_full(self): + """Test full API response maps all 13 fields correctly.""" + data = { + "@odata.type": "#Microsoft.Dynamics.CRM.StringAttributeMetadata", + "LogicalName": "emailaddress1", + "SchemaName": "EMailAddress1", + "DisplayName": { + "UserLocalizedLabel": {"Label": "Email", "LanguageCode": 1033}, + }, + "AttributeType": "String", + "AttributeTypeName": {"Value": "StringType"}, + "IsCustomAttribute": False, + "IsPrimaryId": False, + "IsPrimaryName": False, + "RequiredLevel": {"Value": "None"}, + "IsValidForCreate": True, + "IsValidForUpdate": True, + "IsValidForRead": True, + "MetadataId": "def-456", + } + col = ColumnMetadata.from_api_response(data) + assert col.logical_name == "emailaddress1" + assert col.schema_name == "EMailAddress1" + assert col.display_name == "Email" + assert col.attribute_type == "String" + assert col.attribute_type_name == "StringType" + assert col.is_custom_attribute is False + assert col.is_primary_id is False + assert col.is_primary_name is False + assert col.required_level == "None" + assert col.is_valid_for_create is True + assert col.is_valid_for_update is True + assert col.is_valid_for_read is True + assert col.metadata_id == "def-456" + + def test_from_api_response_minimal(self): + """Test minimal dict with only LogicalName and SchemaName uses defaults.""" + data = {"LogicalName": "name", "SchemaName": "Name"} + col = ColumnMetadata.from_api_response(data) + assert col.logical_name == "name" + assert col.schema_name == "Name" + assert col.display_name is None + assert col.attribute_type == "" + assert col.attribute_type_name is None + assert col.is_custom_attribute is False + assert col.is_primary_id is False + assert col.is_primary_name is False + assert col.required_level is None + assert col.is_valid_for_create is False + assert col.is_valid_for_update is False + assert col.is_valid_for_read is False + assert col.metadata_id is None + + def test_display_name_nested_none(self): + """Test DisplayName exists but UserLocalizedLabel is None.""" + data = { + "LogicalName": "col", + "SchemaName": "Col", + "DisplayName": {"UserLocalizedLabel": None}, + } + col = ColumnMetadata.from_api_response(data) + assert col.display_name is None + + def test_display_name_missing_entirely(self): + """Test dict without DisplayName key.""" + data = {"LogicalName": "col", "SchemaName": "Col"} + col = ColumnMetadata.from_api_response(data) + assert col.display_name is None + + def test_required_level_extraction(self): + """Test RequiredLevel.Value is extracted correctly.""" + data = { + "LogicalName": "col", + "SchemaName": "Col", + "RequiredLevel": {"Value": "ApplicationRequired"}, + } + col = ColumnMetadata.from_api_response(data) + assert col.required_level == "ApplicationRequired" + + +class TestOptionItem: + """Tests for OptionItem.""" + + def test_from_api_response(self): + """Test option with Value and Label.""" + data = { + "Value": 1, + "Label": { + "UserLocalizedLabel": {"Label": "Preferred Customer", "LanguageCode": 1033}, + }, + } + opt = OptionItem.from_api_response(data) + assert opt.value == 1 + assert opt.label == "Preferred Customer" + + def test_from_api_response_no_label(self): + """Test option with Value but Label.UserLocalizedLabel is None.""" + data = {"Value": 2, "Label": {"UserLocalizedLabel": None}} + opt = OptionItem.from_api_response(data) + assert opt.value == 2 + assert opt.label is None + + +class TestOptionSetInfo: + """Tests for OptionSetInfo.""" + + def test_from_api_response_picklist(self): + """Test picklist-style OptionSet with Options array.""" + data = { + "Name": "account_accountcategorycode", + "DisplayName": {"UserLocalizedLabel": {"Label": "Category", "LanguageCode": 1033}}, + "IsGlobal": False, + "OptionSetType": "Picklist", + "Options": [ + {"Value": 1, "Label": {"UserLocalizedLabel": {"Label": "Preferred Customer"}}}, + {"Value": 2, "Label": {"UserLocalizedLabel": {"Label": "Standard"}}}, + ], + "MetadataId": "meta-guid", + } + opt_set = OptionSetInfo.from_api_response(data) + assert opt_set.option_set_type == "Picklist" + assert opt_set.name == "account_accountcategorycode" + assert opt_set.display_name == "Category" + assert opt_set.is_global is False + assert len(opt_set.options) == 2 + assert opt_set.options[0].value == 1 + assert opt_set.options[0].label == "Preferred Customer" + assert opt_set.options[1].value == 2 + assert opt_set.options[1].label == "Standard" + assert opt_set.metadata_id == "meta-guid" + + def test_from_api_response_boolean(self): + """Test boolean-style OptionSet with TrueOption and FalseOption.""" + data = { + "OptionSetType": "Boolean", + "TrueOption": {"Value": 1, "Label": {"UserLocalizedLabel": {"Label": "Do Not Allow"}}}, + "FalseOption": {"Value": 0, "Label": {"UserLocalizedLabel": {"Label": "Allow"}}}, + } + opt_set = OptionSetInfo.from_api_response(data) + assert opt_set.option_set_type == "Boolean" + assert len(opt_set.options) == 2 + values = [o.value for o in opt_set.options] + assert 0 in values + assert 1 in values + labels = {o.value: o.label for o in opt_set.options} + assert labels[0] == "Allow" + assert labels[1] == "Do Not Allow" + + def test_from_api_response_empty_options(self): + """Test OptionSet with empty Options array.""" + data = {"Options": [], "OptionSetType": "Picklist"} + opt_set = OptionSetInfo.from_api_response(data) + assert opt_set.options == [] + assert opt_set.option_set_type == "Picklist" + + def test_from_api_response_global_optionset(self): + """Test OptionSet with IsGlobal True.""" + data = { + "Name": "global_options", + "IsGlobal": True, + "Options": [], + "OptionSetType": "Picklist", + } + opt_set = OptionSetInfo.from_api_response(data) + assert opt_set.is_global is True + assert opt_set.name == "global_options" diff --git a/tests/unit/test_tables_operations.py b/tests/unit/test_tables_operations.py index c2d8bede..8644ad4b 100644 --- a/tests/unit/test_tables_operations.py +++ b/tests/unit/test_tables_operations.py @@ -7,6 +7,7 @@ from azure.core.credentials import TokenCredential from PowerPlatform.Dataverse.client import DataverseClient +from PowerPlatform.Dataverse.models.metadata import ColumnMetadata, OptionSetInfo from PowerPlatform.Dataverse.models.relationship import RelationshipInfo from PowerPlatform.Dataverse.operations.tables import TableOperations @@ -89,6 +90,261 @@ def test_get_returns_none(self): self.client._odata._get_table_info.assert_called_once_with("nonexistent_Table") self.assertIsNone(result) + def test_get_basic_unchanged(self): + """get() with no extra args should use _get_table_info (backward compatibility).""" + expected_info = { + "table_schema_name": "account", + "table_logical_name": "account", + "entity_set_name": "accounts", + "metadata_id": "meta-guid-1", + } + self.client._odata._get_table_info.return_value = expected_info + + result = self.client.tables.get("account") + + self.client._odata._get_table_info.assert_called_once_with("account") + self.client._odata._get_table_metadata.assert_not_called() + self.assertEqual(result, expected_info) + + def test_get_with_include_columns(self): + """get(include_columns=True) should call _get_table_metadata and return columns.""" + raw = { + "SchemaName": "Account", + "LogicalName": "account", + "EntitySetName": "accounts", + "MetadataId": "meta-guid", + "Attributes": [ + {"LogicalName": "name", "SchemaName": "Name", "AttributeType": "String"}, + ], + } + self.client._odata._get_table_metadata.return_value = raw + + result = self.client.tables.get("account", include_columns=True) + + self.client._odata._get_table_metadata.assert_called_once_with( + "account", + select=None, + include_attributes=True, + include_one_to_many=False, + include_many_to_one=False, + include_many_to_many=False, + ) + self.assertIn("columns", result) + self.assertEqual(len(result["columns"]), 1) + self.assertIsInstance(result["columns"][0], ColumnMetadata) + self.assertEqual(result["columns"][0].logical_name, "name") + self.assertEqual(result["columns"][0].attribute_type, "String") + + def test_get_with_include_relationships(self): + """get(include_relationships=True) should return relationship arrays.""" + raw = { + "SchemaName": "Account", + "LogicalName": "account", + "EntitySetName": "accounts", + "MetadataId": "meta-guid", + "OneToManyRelationships": [{"SchemaName": "account_tasks", "ReferencingEntity": "task"}], + "ManyToOneRelationships": [], + "ManyToManyRelationships": [], + } + self.client._odata._get_table_metadata.return_value = raw + + result = self.client.tables.get("account", include_relationships=True) + + self.client._odata._get_table_metadata.assert_called_once_with( + "account", + select=None, + include_attributes=False, + include_one_to_many=True, + include_many_to_one=True, + include_many_to_many=True, + ) + self.assertIn("one_to_many_relationships", result) + self.assertEqual(len(result["one_to_many_relationships"]), 1) + self.assertEqual(result["one_to_many_relationships"][0]["SchemaName"], "account_tasks") + self.assertIn("many_to_one_relationships", result) + self.assertIn("many_to_many_relationships", result) + + def test_get_with_select(self): + """get(select=[...]) should pass select and include extra properties in result.""" + raw = { + "SchemaName": "Account", + "LogicalName": "account", + "EntitySetName": "accounts", + "MetadataId": "meta-guid", + "DisplayName": {"UserLocalizedLabel": {"Label": "Account"}}, + "Description": {"UserLocalizedLabel": {"Label": "Business account"}}, + } + self.client._odata._get_table_metadata.return_value = raw + + result = self.client.tables.get("account", select=["DisplayName", "Description"]) + + self.client._odata._get_table_metadata.assert_called_once_with( + "account", + select=["DisplayName", "Description"], + include_attributes=False, + include_one_to_many=False, + include_many_to_one=False, + include_many_to_many=False, + ) + self.assertIn("DisplayName", result) + self.assertIn("Description", result) + + def test_get_extended_returns_none(self): + """get(include_columns=True) should return None when table not found.""" + self.client._odata._get_table_metadata.return_value = None + + result = self.client.tables.get("nonexistent", include_columns=True) + + self.assertIsNone(result) + + def test_get_columns(self): + """get_columns() should return list of ColumnMetadata.""" + raw_list = [ + {"LogicalName": "name", "SchemaName": "Name", "AttributeType": "String"}, + {"LogicalName": "emailaddress1", "SchemaName": "EMailAddress1", "AttributeType": "String"}, + ] + self.client._odata._get_table_columns.return_value = raw_list + + result = self.client.tables.get_columns("account") + + self.client._odata._get_table_columns.assert_called_once_with( + "account", + select=None, + filter=None, + ) + self.assertEqual(len(result), 2) + self.assertIsInstance(result[0], ColumnMetadata) + self.assertIsInstance(result[1], ColumnMetadata) + self.assertEqual(result[0].logical_name, "name") + self.assertEqual(result[1].logical_name, "emailaddress1") + + def test_get_columns_with_filter(self): + """get_columns(filter=...) should pass filter to _get_table_columns.""" + filter_expr = "AttributeType eq Microsoft.Dynamics.CRM.AttributeTypeCode'Picklist'" + self.client._odata._get_table_columns.return_value = [] + + self.client.tables.get_columns("account", filter=filter_expr) + + self.client._odata._get_table_columns.assert_called_once_with( + "account", + select=None, + filter=filter_expr, + ) + + def test_get_column_found(self): + """get_column() should return ColumnMetadata when column exists.""" + raw = {"LogicalName": "emailaddress1", "SchemaName": "EMailAddress1", "AttributeType": "String"} + self.client._odata._get_table_column.return_value = raw + + result = self.client.tables.get_column("account", "emailaddress1") + + self.client._odata._get_table_column.assert_called_once_with( + "account", + "emailaddress1", + select=None, + ) + self.assertIsInstance(result, ColumnMetadata) + self.assertEqual(result.logical_name, "emailaddress1") + + def test_get_column_not_found(self): + """get_column() should return None when column not found.""" + self.client._odata._get_table_column.return_value = None + + result = self.client.tables.get_column("account", "nonexistent_col") + + self.assertIsNone(result) + + def test_get_column_options_picklist(self): + """get_column_options() should return OptionSetInfo for picklist column.""" + raw_optionset = { + "Name": "account_accountcategorycode", + "OptionSetType": "Picklist", + "Options": [ + {"Value": 1, "Label": {"UserLocalizedLabel": {"Label": "Preferred Customer"}}}, + {"Value": 2, "Label": {"UserLocalizedLabel": {"Label": "Standard"}}}, + ], + } + self.client._odata._get_column_optionset.return_value = raw_optionset + + result = self.client.tables.get_column_options("account", "accountcategorycode") + + self.client._odata._get_column_optionset.assert_called_once_with("account", "accountcategorycode") + self.assertIsInstance(result, OptionSetInfo) + self.assertEqual(len(result.options), 2) + self.assertEqual(result.options[0].value, 1) + self.assertEqual(result.options[0].label, "Preferred Customer") + + def test_get_column_options_not_picklist(self): + """get_column_options() should return None for non-choice column.""" + self.client._odata._get_column_optionset.return_value = None + + result = self.client.tables.get_column_options("account", "name") + + self.assertIsNone(result) + + def test_list_relationships_all(self): + """list_relationships() with no type should return all relationship types.""" + expected = [ + {"SchemaName": "account_tasks", "_relationship_type": "OneToMany"}, + {"SchemaName": "account_primarycontact", "_relationship_type": "ManyToOne"}, + ] + self.client._odata._list_table_relationships.return_value = expected + + result = self.client.tables.list_relationships("account") + + self.client._odata._list_table_relationships.assert_called_once_with( + "account", + relationship_type=None, + select=None, + ) + self.assertEqual(result, expected) + + def test_list_relationships_filtered(self): + """list_relationships(relationship_type=...) should pass type filter.""" + expected = [{"SchemaName": "account_tasks", "_relationship_type": "OneToMany"}] + self.client._odata._list_table_relationships.return_value = expected + + result = self.client.tables.list_relationships("account", relationship_type="one_to_many") + + self.client._odata._list_table_relationships.assert_called_once_with( + "account", + relationship_type="one_to_many", + select=None, + ) + self.assertEqual(result, expected) + + def test_get_select_bare_string_raises(self): + """get() with select as bare string should raise TypeError.""" + self.client._odata._get_table_metadata.side_effect = TypeError( + "select must be a list of property names, not a bare string" + ) + with self.assertRaises(TypeError): + self.client.tables.get("account", select="DisplayName") + + def test_get_columns_select_bare_string_raises(self): + """get_columns() with select as bare string should raise TypeError.""" + self.client._odata._get_table_columns.side_effect = TypeError( + "select must be a list of property names, not a bare string" + ) + with self.assertRaises(TypeError): + self.client.tables.get_columns("account", select="LogicalName") + + def test_get_column_select_bare_string_raises(self): + """get_column() with select as bare string should raise TypeError.""" + self.client._odata._get_table_column.side_effect = TypeError( + "select must be a list of property names, not a bare string" + ) + with self.assertRaises(TypeError): + self.client.tables.get_column("account", "name", select="LogicalName") + + def test_list_relationships_select_bare_string_raises(self): + """list_relationships() should raise TypeError on bare string select.""" + self.client._odata._list_table_relationships.side_effect = TypeError( + "select must be a list of property names, not a bare string" + ) + with self.assertRaises(TypeError): + self.client.tables.list_relationships("account", select="SchemaName") + # ------------------------------------------------------------------- list def test_list(self): From f3c3efb68f68b0c5f63ccff042c7a48ecfc8f523 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 28 Feb 2026 12:25:56 +0000 Subject: [PATCH 02/20] Initial plan From 92ca6bd53e2e1f0cc8608e8436b402ca030da5cf Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 28 Feb 2026 12:30:25 +0000 Subject: [PATCH 03/20] Fix get() result: remove columns_created, reduce cyclomatic complexity Co-authored-by: maksii <1761348+maksii@users.noreply.github.com> --- src/PowerPlatform/Dataverse/operations/tables.py | 14 +++++++------- tests/unit/test_tables_operations.py | 1 + 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/PowerPlatform/Dataverse/operations/tables.py b/src/PowerPlatform/Dataverse/operations/tables.py index a95f0e5c..ec2093f3 100644 --- a/src/PowerPlatform/Dataverse/operations/tables.py +++ b/src/PowerPlatform/Dataverse/operations/tables.py @@ -233,7 +233,6 @@ def get( "table_logical_name": raw.get("LogicalName"), "entity_set_name": raw.get("EntitySetName"), "metadata_id": raw.get("MetadataId"), - "columns_created": [], } # Include any extra selected entity properties @@ -248,12 +247,13 @@ def get( # Include expanded relationship collections as raw dicts if include_relationships: - if "OneToManyRelationships" in raw: - result["one_to_many_relationships"] = raw["OneToManyRelationships"] - if "ManyToOneRelationships" in raw: - result["many_to_one_relationships"] = raw["ManyToOneRelationships"] - if "ManyToManyRelationships" in raw: - result["many_to_many_relationships"] = raw["ManyToManyRelationships"] + for raw_key, result_key in ( + ("OneToManyRelationships", "one_to_many_relationships"), + ("ManyToOneRelationships", "many_to_one_relationships"), + ("ManyToManyRelationships", "many_to_many_relationships"), + ): + if raw_key in raw: + result[result_key] = raw[raw_key] return result diff --git a/tests/unit/test_tables_operations.py b/tests/unit/test_tables_operations.py index 8644ad4b..8070a438 100644 --- a/tests/unit/test_tables_operations.py +++ b/tests/unit/test_tables_operations.py @@ -130,6 +130,7 @@ def test_get_with_include_columns(self): include_many_to_many=False, ) self.assertIn("columns", result) + self.assertNotIn("columns_created", result) self.assertEqual(len(result["columns"]), 1) self.assertIsInstance(result["columns"][0], ColumnMetadata) self.assertEqual(result["columns"][0].logical_name, "name") From 53220837a54f0b5c2c4aa3d77810a882591c87c2 Mon Sep 17 00:00:00 2001 From: maksii <1761348+maksii@users.noreply.github.com> Date: Sat, 28 Feb 2026 15:26:14 +0200 Subject: [PATCH 04/20] Add test data fixtures and enhance unit tests for metadata operations - Introduced new test data fixtures in `tests/fixtures/test_data.py` for various metadata attributes, including columns, option sets, and relationships. - Updated `tests/unit/models/test_metadata.py` to utilize the new fixtures for testing `ColumnMetadata` and `OptionSetInfo` classes, improving test coverage for primary name columns, picklist columns, and status/state option sets. - Enhanced `tests/unit/test_tables_operations.py` to incorporate fixtures for table operations, ensuring accurate testing of column retrieval, relationship listing, and table metadata. - Refactored existing tests to replace hardcoded data with structured test data from the new fixtures, promoting maintainability and clarity in test cases. --- .../Dataverse/common/constants.py | 2 + src/PowerPlatform/Dataverse/data/_odata.py | 4 + tests/fixtures/__init__.py | 2 + tests/fixtures/test_data.py | 534 +++++++++++++++++- tests/unit/models/test_metadata.py | 158 ++++-- tests/unit/test_tables_operations.py | 204 +++++-- 6 files changed, 804 insertions(+), 100 deletions(-) create mode 100644 tests/fixtures/__init__.py diff --git a/src/PowerPlatform/Dataverse/common/constants.py b/src/PowerPlatform/Dataverse/common/constants.py index 33724679..552da5a7 100644 --- a/src/PowerPlatform/Dataverse/common/constants.py +++ b/src/PowerPlatform/Dataverse/common/constants.py @@ -35,6 +35,8 @@ ODATA_TYPE_PICKLIST_ATTRIBUTE = "Microsoft.Dynamics.CRM.PicklistAttributeMetadata" ODATA_TYPE_BOOLEAN_ATTRIBUTE = "Microsoft.Dynamics.CRM.BooleanAttributeMetadata" ODATA_TYPE_MULTISELECT_PICKLIST_ATTRIBUTE = "Microsoft.Dynamics.CRM.MultiSelectPicklistAttributeMetadata" +ODATA_TYPE_STATUS_ATTRIBUTE = "Microsoft.Dynamics.CRM.StatusAttributeMetadata" +ODATA_TYPE_STATE_ATTRIBUTE = "Microsoft.Dynamics.CRM.StateAttributeMetadata" ODATA_TYPE_STRING_ATTRIBUTE = "Microsoft.Dynamics.CRM.StringAttributeMetadata" ODATA_TYPE_INTEGER_ATTRIBUTE = "Microsoft.Dynamics.CRM.IntegerAttributeMetadata" ODATA_TYPE_DECIMAL_ATTRIBUTE = "Microsoft.Dynamics.CRM.DecimalAttributeMetadata" diff --git a/src/PowerPlatform/Dataverse/data/_odata.py b/src/PowerPlatform/Dataverse/data/_odata.py index fe96308a..ca1ba5ac 100644 --- a/src/PowerPlatform/Dataverse/data/_odata.py +++ b/src/PowerPlatform/Dataverse/data/_odata.py @@ -38,6 +38,8 @@ ODATA_TYPE_BOOLEAN_ATTRIBUTE, ODATA_TYPE_MULTISELECT_PICKLIST_ATTRIBUTE, ODATA_TYPE_PICKLIST_ATTRIBUTE, + ODATA_TYPE_STATE_ATTRIBUTE, + ODATA_TYPE_STATUS_ATTRIBUTE, ) from .. import __version__ as _SDK_VERSION @@ -1659,6 +1661,8 @@ def _get_column_optionset( ODATA_TYPE_PICKLIST_ATTRIBUTE, ODATA_TYPE_BOOLEAN_ATTRIBUTE, ODATA_TYPE_MULTISELECT_PICKLIST_ATTRIBUTE, + ODATA_TYPE_STATUS_ATTRIBUTE, + ODATA_TYPE_STATE_ATTRIBUTE, ]: url = f"{base}/{cast_type}" try: diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py new file mode 100644 index 00000000..9a045456 --- /dev/null +++ b/tests/fixtures/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. diff --git a/tests/fixtures/test_data.py b/tests/fixtures/test_data.py index 20e2f17a..f222d966 100644 --- a/tests/fixtures/test_data.py +++ b/tests/fixtures/test_data.py @@ -12,16 +12,80 @@ SAMPLE_ENTITY_METADATA = { "value": [ { + "MetadataId": "70816501-edb9-4740-a16c-6a5efbc05d84", "LogicalName": "account", + "SchemaName": "Account", "EntitySetName": "accounts", "PrimaryIdAttribute": "accountid", - "DisplayName": {"UserLocalizedLabel": {"Label": "Account"}}, + "DisplayName": { + "LocalizedLabels": [ + { + "Label": "Account", + "LanguageCode": 1033, + "IsManaged": True, + "MetadataId": "2a4901bf-2241-db11-898a-0007e9e17ebd", + }, + ], + "UserLocalizedLabel": { + "Label": "Account", + "LanguageCode": 1033, + "IsManaged": True, + "MetadataId": "2a4901bf-2241-db11-898a-0007e9e17ebd", + }, + }, + "Description": { + "LocalizedLabels": [ + { + "Label": "Business that represents a customer or potential customer. The company that is billed in business transactions.", + "LanguageCode": 1033, + "IsManaged": True, + "MetadataId": "294901bf-2241-db11-898a-0007e9e17ebd", + }, + ], + "UserLocalizedLabel": { + "Label": "Business that represents a customer or potential customer. The company that is billed in business transactions.", + "LanguageCode": 1033, + "IsManaged": True, + "MetadataId": "294901bf-2241-db11-898a-0007e9e17ebd", + }, + }, }, { + "MetadataId": "608861bc-50a4-4c5f-a02c-21fe1943e2cf", "LogicalName": "contact", + "SchemaName": "Contact", "EntitySetName": "contacts", "PrimaryIdAttribute": "contactid", - "DisplayName": {"UserLocalizedLabel": {"Label": "Contact"}}, + "DisplayName": { + "LocalizedLabels": [ + { + "Label": "Contact", + "LanguageCode": 1033, + "IsManaged": True, + "MetadataId": "3a4901bf-2241-db11-898a-0007e9e17ebd", + }, + ], + "UserLocalizedLabel": { + "Label": "Contact", + "LanguageCode": 1033, + "IsManaged": True, + "MetadataId": "3a4901bf-2241-db11-898a-0007e9e17ebd", + }, + }, + "Description": { + "LocalizedLabels": [ + { + "Label": "Person with whom a business unit has a relationship, such as customer, supplier, and colleague.", + "LanguageCode": 1033, + "IsManaged": True, + }, + ], + "UserLocalizedLabel": { + "Label": "Person with whom a business unit has a relationship, such as customer, supplier, and colleague.", + "LanguageCode": 1033, + "IsManaged": True, + }, + }, }, ] } @@ -52,3 +116,469 @@ # Sample SQL query results SAMPLE_SQL_RESPONSE = {"value": [{"name": "Account 1", "revenue": 1000000}, {"name": "Account 2", "revenue": 2000000}]} + + +# --------------------------------------------------------------------------- +# Column attribute metadata samples +# (Realistic responses from Dataverse Web API, used across unit tests) +# --------------------------------------------------------------------------- + +ACCOUNT_NAME_COLUMN = { + "@odata.type": "#Microsoft.Dynamics.CRM.StringAttributeMetadata", + "LogicalName": "name", + "SchemaName": "Name", + "AttributeType": "String", + "AttributeTypeName": {"Value": "StringType"}, + "IsCustomAttribute": False, + "IsPrimaryId": False, + "IsPrimaryName": True, + "IsValidForCreate": True, + "IsValidForUpdate": True, + "IsValidForRead": True, + "RequiredLevel": { + "Value": "ApplicationRequired", + "CanBeChanged": True, + "ManagedPropertyLogicalName": "canmodifyrequirementlevelsettings", + }, + "MetadataId": "a1965545-44bc-4b7b-b1ae-93074d0e3f2a", + "DisplayName": { + "LocalizedLabels": [ + { + "Label": "Account Name", + "LanguageCode": 1033, + "IsManaged": True, + "MetadataId": "ea34ed00-2341-db11-898a-0007e9e17ebd", + } + ], + "UserLocalizedLabel": { + "Label": "Account Name", + "LanguageCode": 1033, + "IsManaged": True, + "MetadataId": "ea34ed00-2341-db11-898a-0007e9e17ebd", + }, + }, + "Description": { + "LocalizedLabels": [ + { + "Label": "Type the company or business name.", + "LanguageCode": 1033, + "IsManaged": True, + } + ], + "UserLocalizedLabel": { + "Label": "Type the company or business name.", + "LanguageCode": 1033, + "IsManaged": True, + }, + }, +} + +EMAILADDRESS1_COLUMN = { + "@odata.type": "#Microsoft.Dynamics.CRM.StringAttributeMetadata", + "LogicalName": "emailaddress1", + "SchemaName": "EMailAddress1", + "AttributeType": "String", + "AttributeTypeName": {"Value": "StringType"}, + "IsCustomAttribute": False, + "IsPrimaryId": False, + "IsPrimaryName": False, + "IsValidForCreate": True, + "IsValidForUpdate": True, + "IsValidForRead": True, + "RequiredLevel": { + "Value": "None", + "CanBeChanged": True, + "ManagedPropertyLogicalName": "canmodifyrequirementlevelsettings", + }, + "MetadataId": "024a2ee3-b983-4fd8-8991-f8d548a227e0", + "DisplayName": { + "LocalizedLabels": [ + { + "Label": "Email", + "LanguageCode": 1033, + "IsManaged": True, + "MetadataId": "54c04ee3-b983-4fd8-8991-f8d548a227e0", + } + ], + "UserLocalizedLabel": { + "Label": "Email", + "LanguageCode": 1033, + "IsManaged": True, + "MetadataId": "54c04ee3-b983-4fd8-8991-f8d548a227e0", + }, + }, + "Description": { + "LocalizedLabels": [ + { + "Label": "Type the primary email address for the contact.", + "LanguageCode": 1033, + "IsManaged": True, + } + ], + "UserLocalizedLabel": { + "Label": "Type the primary email address for the contact.", + "LanguageCode": 1033, + "IsManaged": True, + }, + }, +} + +PICKLIST_COLUMN = { + "@odata.type": "#Microsoft.Dynamics.CRM.PicklistAttributeMetadata", + "LogicalName": "accountcategorycode", + "SchemaName": "AccountCategoryCode", + "AttributeType": "Picklist", + "AttributeTypeName": {"Value": "PicklistType"}, + "IsCustomAttribute": False, + "IsPrimaryId": False, + "IsPrimaryName": False, + "IsValidForCreate": True, + "IsValidForUpdate": True, + "IsValidForRead": True, + "RequiredLevel": { + "Value": "None", + "CanBeChanged": True, + "ManagedPropertyLogicalName": "canmodifyrequirementlevelsettings", + }, + "MetadataId": "118771ca-6fb9-4f60-8fd4-99b6124b63ad", + "DisplayName": { + "LocalizedLabels": [{"Label": "Category", "LanguageCode": 1033, "IsManaged": True}], + "UserLocalizedLabel": {"Label": "Category", "LanguageCode": 1033, "IsManaged": True}, + }, +} + +STATUS_COLUMN = { + "@odata.type": "#Microsoft.Dynamics.CRM.StatusAttributeMetadata", + "LogicalName": "statuscode", + "SchemaName": "StatusCode", + "AttributeType": "Status", + "AttributeTypeName": {"Value": "StatusType"}, + "IsCustomAttribute": False, + "IsPrimaryId": False, + "IsPrimaryName": False, + "IsValidForCreate": True, + "IsValidForUpdate": True, + "IsValidForRead": True, + "RequiredLevel": { + "Value": "None", + "CanBeChanged": True, + "ManagedPropertyLogicalName": "canmodifyrequirementlevelsettings", + }, + "MetadataId": "f99371c3-b1e1-4645-b2c3-c3db0f59ecf0", + "DisplayName": { + "LocalizedLabels": [{"Label": "Status Reason", "LanguageCode": 1033, "IsManaged": True}], + "UserLocalizedLabel": {"Label": "Status Reason", "LanguageCode": 1033, "IsManaged": True}, + }, +} + +STATE_COLUMN = { + "@odata.type": "#Microsoft.Dynamics.CRM.StateAttributeMetadata", + "LogicalName": "statecode", + "SchemaName": "StateCode", + "AttributeType": "State", + "AttributeTypeName": {"Value": "StateType"}, + "IsCustomAttribute": False, + "IsPrimaryId": False, + "IsPrimaryName": False, + "IsValidForCreate": False, + "IsValidForUpdate": True, + "IsValidForRead": True, + "RequiredLevel": { + "Value": "SystemRequired", + "CanBeChanged": False, + "ManagedPropertyLogicalName": "canmodifyrequirementlevelsettings", + }, + "MetadataId": "cdc3895a-7539-41e9-966b-3f9ef805aefd", + "DisplayName": { + "LocalizedLabels": [{"Label": "Status", "LanguageCode": 1033, "IsManaged": True}], + "UserLocalizedLabel": {"Label": "Status", "LanguageCode": 1033, "IsManaged": True}, + }, +} + +UNIQUEID_COLUMN = { + "LogicalName": "accountid", + "SchemaName": "AccountId", + "AttributeType": "Uniqueidentifier", + "AttributeTypeName": {"Value": "UniqueidentifierType"}, + "IsCustomAttribute": False, + "IsPrimaryId": True, + "IsPrimaryName": False, + "IsValidForCreate": True, + "IsValidForUpdate": False, + "IsValidForRead": True, + "RequiredLevel": {"Value": "SystemRequired"}, + "MetadataId": "f8cd5db9-cee8-4845-8cdd-cd4f504957e7", + "DisplayName": { + "LocalizedLabels": [{"Label": "Account", "LanguageCode": 1033, "IsManaged": True}], + "UserLocalizedLabel": {"Label": "Account", "LanguageCode": 1033, "IsManaged": True}, + }, +} + +# Reference to the first Account entity from SAMPLE_ENTITY_METADATA, +# with full DisplayName and Description (used in table-get-with-select tests). +ACCOUNT_TABLE_FULL = {k: v for k, v in SAMPLE_ENTITY_METADATA["value"][0].items() if k != "PrimaryIdAttribute"} + + +# --------------------------------------------------------------------------- +# OptionSet metadata samples +# --------------------------------------------------------------------------- + +PICKLIST_OPTIONSET = { + "MetadataId": "b994cdd8-5ce9-4ab9-bdd3-8888ebdb0407", + "HasChanged": None, + "IsCustomOptionSet": False, + "IsGlobal": False, + "IsManaged": True, + "Name": "account_accountcategorycode", + "OptionSetType": "Picklist", + "DisplayName": { + "LocalizedLabels": [ + { + "Label": "Category", + "LanguageCode": 1033, + "IsManaged": True, + "MetadataId": "d8a3356a-6d26-4f0e-b89e-8b73f25ed57b", + } + ], + "UserLocalizedLabel": { + "Label": "Category", + "LanguageCode": 1033, + "IsManaged": True, + "MetadataId": "d8a3356a-6d26-4f0e-b89e-8b73f25ed57b", + }, + }, + "Description": { + "LocalizedLabels": [ + { + "Label": "Drop-down list for selecting the category of the account.", + "LanguageCode": 1033, + "IsManaged": True, + } + ], + "UserLocalizedLabel": { + "Label": "Drop-down list for selecting the category of the account.", + "LanguageCode": 1033, + "IsManaged": True, + }, + }, + "Options": [ + { + "Value": 1, + "Color": None, + "IsManaged": True, + "ExternalValue": None, + "ParentValues": [], + "Tag": None, + "IsHidden": False, + "Label": { + "LocalizedLabels": [ + { + "Label": "Preferred Customer", + "LanguageCode": 1033, + "IsManaged": True, + "MetadataId": "0bd8a218-2341-db11-898a-0007e9e17ebd", + } + ], + "UserLocalizedLabel": { + "Label": "Preferred Customer", + "LanguageCode": 1033, + "IsManaged": True, + "MetadataId": "0bd8a218-2341-db11-898a-0007e9e17ebd", + }, + }, + "Description": {"LocalizedLabels": [], "UserLocalizedLabel": None}, + }, + { + "Value": 2, + "Color": None, + "IsManaged": True, + "ExternalValue": None, + "ParentValues": [], + "Tag": None, + "IsHidden": False, + "Label": { + "LocalizedLabels": [ + { + "Label": "Standard", + "LanguageCode": 1033, + "IsManaged": True, + "MetadataId": "0dd8a218-2341-db11-898a-0007e9e17ebd", + } + ], + "UserLocalizedLabel": { + "Label": "Standard", + "LanguageCode": 1033, + "IsManaged": True, + "MetadataId": "0dd8a218-2341-db11-898a-0007e9e17ebd", + }, + }, + "Description": {"LocalizedLabels": [], "UserLocalizedLabel": None}, + }, + ], +} + +STATUS_OPTIONSET = { + "MetadataId": "75ad977d-6f28-4c5c-ae44-7816d366ba21", + "HasChanged": None, + "IsCustomOptionSet": False, + "IsGlobal": False, + "IsManaged": True, + "Name": "account_statuscode", + "OptionSetType": "Status", + "DisplayName": { + "LocalizedLabels": [{"Label": "Status Reason", "LanguageCode": 1033, "IsManaged": True}], + "UserLocalizedLabel": {"Label": "Status Reason", "LanguageCode": 1033, "IsManaged": True}, + }, + "Options": [ + { + "@odata.type": "#Microsoft.Dynamics.CRM.StatusOptionMetadata", + "Value": 1, + "Color": None, + "IsManaged": True, + "State": 0, + "TransitionData": None, + "Label": { + "LocalizedLabels": [{"Label": "Active", "LanguageCode": 1033, "IsManaged": True}], + "UserLocalizedLabel": {"Label": "Active", "LanguageCode": 1033, "IsManaged": True}, + }, + "Description": {"LocalizedLabels": [], "UserLocalizedLabel": None}, + }, + { + "@odata.type": "#Microsoft.Dynamics.CRM.StatusOptionMetadata", + "Value": 2, + "Color": None, + "IsManaged": True, + "State": 1, + "TransitionData": None, + "Label": { + "LocalizedLabels": [{"Label": "Inactive", "LanguageCode": 1033, "IsManaged": True}], + "UserLocalizedLabel": {"Label": "Inactive", "LanguageCode": 1033, "IsManaged": True}, + }, + "Description": {"LocalizedLabels": [], "UserLocalizedLabel": None}, + }, + ], +} + +STATE_OPTIONSET = { + "MetadataId": "88fa5ad0-2a4b-4281-ac9c-b4e71fb77920", + "HasChanged": None, + "IsCustomOptionSet": False, + "IsGlobal": False, + "IsManaged": True, + "Name": "contact_statecode", + "OptionSetType": "State", + "DisplayName": { + "LocalizedLabels": [{"Label": "Status", "LanguageCode": 1033, "IsManaged": True}], + "UserLocalizedLabel": {"Label": "Status", "LanguageCode": 1033, "IsManaged": True}, + }, + "Options": [ + { + "@odata.type": "#Microsoft.Dynamics.CRM.StateOptionMetadata", + "Value": 0, + "Color": None, + "IsManaged": True, + "DefaultStatus": 1, + "InvariantName": "Active", + "Label": { + "LocalizedLabels": [{"Label": "Active", "LanguageCode": 1033, "IsManaged": True}], + "UserLocalizedLabel": {"Label": "Active", "LanguageCode": 1033, "IsManaged": True}, + }, + "Description": {"LocalizedLabels": [], "UserLocalizedLabel": None}, + }, + { + "@odata.type": "#Microsoft.Dynamics.CRM.StateOptionMetadata", + "Value": 1, + "Color": None, + "IsManaged": True, + "DefaultStatus": 2, + "InvariantName": "Inactive", + "Label": { + "LocalizedLabels": [{"Label": "Inactive", "LanguageCode": 1033, "IsManaged": True}], + "UserLocalizedLabel": {"Label": "Inactive", "LanguageCode": 1033, "IsManaged": True}, + }, + "Description": {"LocalizedLabels": [], "UserLocalizedLabel": None}, + }, + ], +} + +BOOLEAN_OPTIONSET = { + "MetadataId": "0fe276ef-76e9-4121-b570-a09edbf92ab3", + "HasChanged": None, + "IsCustomOptionSet": False, + "IsGlobal": False, + "IsManaged": True, + "Name": "contact_donotphone", + "OptionSetType": "Boolean", + "DisplayName": { + "LocalizedLabels": [{"Label": "Do not allow Phone Calls", "LanguageCode": 1033, "IsManaged": True}], + "UserLocalizedLabel": {"Label": "Do not allow Phone Calls", "LanguageCode": 1033, "IsManaged": True}, + }, + "TrueOption": { + "Value": 1, + "Color": None, + "IsManaged": True, + "ExternalValue": None, + "ParentValues": [], + "Label": { + "LocalizedLabels": [{"Label": "Do Not Allow", "LanguageCode": 1033, "IsManaged": True}], + "UserLocalizedLabel": {"Label": "Do Not Allow", "LanguageCode": 1033, "IsManaged": True}, + }, + "Description": {"LocalizedLabels": [], "UserLocalizedLabel": None}, + }, + "FalseOption": { + "Value": 0, + "Color": None, + "IsManaged": True, + "ExternalValue": None, + "ParentValues": [], + "Label": { + "LocalizedLabels": [{"Label": "Allow", "LanguageCode": 1033, "IsManaged": True}], + "UserLocalizedLabel": {"Label": "Allow", "LanguageCode": 1033, "IsManaged": True}, + }, + "Description": {"LocalizedLabels": [], "UserLocalizedLabel": None}, + }, +} + + +# --------------------------------------------------------------------------- +# Relationship metadata samples +# --------------------------------------------------------------------------- + +ACCOUNT_CHATS_RELATIONSHIP = { + "MetadataId": "4c731d0a-8713-f111-8341-7ced8d40bc10", + "SchemaName": "account_chats", + "ReferencedAttribute": "accountid", + "ReferencedEntity": "account", + "ReferencingAttribute": "regardingobjectid", + "ReferencingEntity": "chat", + "RelationshipType": "OneToManyRelationship", + "IsCustomRelationship": False, + "IsManaged": False, + "CascadeConfiguration": { + "Assign": "Cascade", + "Delete": "Cascade", + "Merge": "Cascade", + "Reparent": "Cascade", + "Share": "Cascade", + "Unshare": "Cascade", + }, +} + + +# --------------------------------------------------------------------------- +# Table list entry fixtures (commonly used in list tests) +# --------------------------------------------------------------------------- + +ACCOUNT_TABLE_ENTRY = { + "MetadataId": "70816501-edb9-4740-a16c-6a5efbc05d84", + "LogicalName": "account", + "SchemaName": "Account", + "EntitySetName": "accounts", +} + +CONTACT_TABLE_ENTRY = { + "MetadataId": "608861bc-50a4-4c5f-a02c-21fe1943e2cf", + "LogicalName": "contact", + "SchemaName": "Contact", + "EntitySetName": "contacts", +} diff --git a/tests/unit/models/test_metadata.py b/tests/unit/models/test_metadata.py index 0b128a59..ca504ca0 100644 --- a/tests/unit/models/test_metadata.py +++ b/tests/unit/models/test_metadata.py @@ -8,6 +8,18 @@ OptionItem, OptionSetInfo, ) +from tests.fixtures.test_data import ( + ACCOUNT_NAME_COLUMN, + BOOLEAN_OPTIONSET, + EMAILADDRESS1_COLUMN, + PICKLIST_COLUMN, + PICKLIST_OPTIONSET, + STATE_COLUMN, + STATE_OPTIONSET, + STATUS_COLUMN, + STATUS_OPTIONSET, + UNIQUEID_COLUMN, +) class TestColumnMetadata: @@ -15,25 +27,7 @@ class TestColumnMetadata: def test_from_api_response_full(self): """Test full API response maps all 13 fields correctly.""" - data = { - "@odata.type": "#Microsoft.Dynamics.CRM.StringAttributeMetadata", - "LogicalName": "emailaddress1", - "SchemaName": "EMailAddress1", - "DisplayName": { - "UserLocalizedLabel": {"Label": "Email", "LanguageCode": 1033}, - }, - "AttributeType": "String", - "AttributeTypeName": {"Value": "StringType"}, - "IsCustomAttribute": False, - "IsPrimaryId": False, - "IsPrimaryName": False, - "RequiredLevel": {"Value": "None"}, - "IsValidForCreate": True, - "IsValidForUpdate": True, - "IsValidForRead": True, - "MetadataId": "def-456", - } - col = ColumnMetadata.from_api_response(data) + col = ColumnMetadata.from_api_response(EMAILADDRESS1_COLUMN) assert col.logical_name == "emailaddress1" assert col.schema_name == "EMailAddress1" assert col.display_name == "Email" @@ -46,7 +40,7 @@ def test_from_api_response_full(self): assert col.is_valid_for_create is True assert col.is_valid_for_update is True assert col.is_valid_for_read is True - assert col.metadata_id == "def-456" + assert col.metadata_id == "024a2ee3-b983-4fd8-8991-f8d548a227e0" def test_from_api_response_minimal(self): """Test minimal dict with only LogicalName and SchemaName uses defaults.""" @@ -82,6 +76,57 @@ def test_display_name_missing_entirely(self): col = ColumnMetadata.from_api_response(data) assert col.display_name is None + def test_from_api_response_primary_name_column(self): + """Test primary name column (account.name) with ApplicationRequired level.""" + col = ColumnMetadata.from_api_response(ACCOUNT_NAME_COLUMN) + assert col.logical_name == "name" + assert col.schema_name == "Name" + assert col.display_name == "Account Name" + assert col.is_primary_name is True + assert col.is_primary_id is False + assert col.required_level == "ApplicationRequired" + + def test_from_api_response_picklist_column(self): + """Test picklist column (account.accountcategorycode) maps correctly.""" + col = ColumnMetadata.from_api_response(PICKLIST_COLUMN) + assert col.logical_name == "accountcategorycode" + assert col.schema_name == "AccountCategoryCode" + assert col.attribute_type == "Picklist" + assert col.attribute_type_name == "PicklistType" + assert col.display_name == "Category" + assert col.required_level == "None" + + def test_from_api_response_status_column(self): + """Test status column (account.statuscode) maps correctly.""" + col = ColumnMetadata.from_api_response(STATUS_COLUMN) + assert col.logical_name == "statuscode" + assert col.schema_name == "StatusCode" + assert col.attribute_type == "Status" + assert col.attribute_type_name == "StatusType" + assert col.display_name == "Status Reason" + + def test_from_api_response_state_column(self): + """Test state column (contact.statecode) maps correctly.""" + col = ColumnMetadata.from_api_response(STATE_COLUMN) + assert col.logical_name == "statecode" + assert col.schema_name == "StateCode" + assert col.attribute_type == "State" + assert col.attribute_type_name == "StateType" + assert col.display_name == "Status" + assert col.required_level == "SystemRequired" + assert col.is_valid_for_create is False + assert col.is_valid_for_update is True + + def test_from_api_response_uniqueidentifier_column(self): + """Test primary ID column (account.accountid) maps correctly.""" + col = ColumnMetadata.from_api_response(UNIQUEID_COLUMN) + assert col.logical_name == "accountid" + assert col.is_primary_id is True + assert col.is_primary_name is False + assert col.attribute_type == "Uniqueidentifier" + assert col.attribute_type_name == "UniqueidentifierType" + assert col.is_valid_for_update is False + def test_required_level_extraction(self): """Test RequiredLevel.Value is extracted correctly.""" data = { @@ -98,13 +143,7 @@ class TestOptionItem: def test_from_api_response(self): """Test option with Value and Label.""" - data = { - "Value": 1, - "Label": { - "UserLocalizedLabel": {"Label": "Preferred Customer", "LanguageCode": 1033}, - }, - } - opt = OptionItem.from_api_response(data) + opt = OptionItem.from_api_response(PICKLIST_OPTIONSET["Options"][0]) assert opt.value == 1 assert opt.label == "Preferred Customer" @@ -115,44 +154,42 @@ def test_from_api_response_no_label(self): assert opt.value == 2 assert opt.label is None + def test_from_api_response_status_option(self): + """Test StatusOptionMetadata with extra State and TransitionData fields.""" + opt = OptionItem.from_api_response(STATUS_OPTIONSET["Options"][0]) + assert opt.value == 1 + assert opt.label == "Active" + + def test_from_api_response_state_option(self): + """Test StateOptionMetadata with DefaultStatus and InvariantName fields.""" + opt = OptionItem.from_api_response(STATE_OPTIONSET["Options"][0]) + assert opt.value == 0 + assert opt.label == "Active" + class TestOptionSetInfo: """Tests for OptionSetInfo.""" def test_from_api_response_picklist(self): """Test picklist-style OptionSet with Options array.""" - data = { - "Name": "account_accountcategorycode", - "DisplayName": {"UserLocalizedLabel": {"Label": "Category", "LanguageCode": 1033}}, - "IsGlobal": False, - "OptionSetType": "Picklist", - "Options": [ - {"Value": 1, "Label": {"UserLocalizedLabel": {"Label": "Preferred Customer"}}}, - {"Value": 2, "Label": {"UserLocalizedLabel": {"Label": "Standard"}}}, - ], - "MetadataId": "meta-guid", - } - opt_set = OptionSetInfo.from_api_response(data) + opt_set = OptionSetInfo.from_api_response(PICKLIST_OPTIONSET) assert opt_set.option_set_type == "Picklist" assert opt_set.name == "account_accountcategorycode" assert opt_set.display_name == "Category" assert opt_set.is_global is False + assert opt_set.metadata_id == "b994cdd8-5ce9-4ab9-bdd3-8888ebdb0407" assert len(opt_set.options) == 2 assert opt_set.options[0].value == 1 assert opt_set.options[0].label == "Preferred Customer" assert opt_set.options[1].value == 2 assert opt_set.options[1].label == "Standard" - assert opt_set.metadata_id == "meta-guid" def test_from_api_response_boolean(self): """Test boolean-style OptionSet with TrueOption and FalseOption.""" - data = { - "OptionSetType": "Boolean", - "TrueOption": {"Value": 1, "Label": {"UserLocalizedLabel": {"Label": "Do Not Allow"}}}, - "FalseOption": {"Value": 0, "Label": {"UserLocalizedLabel": {"Label": "Allow"}}}, - } - opt_set = OptionSetInfo.from_api_response(data) + opt_set = OptionSetInfo.from_api_response(BOOLEAN_OPTIONSET) assert opt_set.option_set_type == "Boolean" + assert opt_set.name == "contact_donotphone" + assert opt_set.display_name == "Do not allow Phone Calls" assert len(opt_set.options) == 2 values = [o.value for o in opt_set.options] assert 0 in values @@ -161,6 +198,35 @@ def test_from_api_response_boolean(self): assert labels[0] == "Allow" assert labels[1] == "Do Not Allow" + def test_from_api_response_status_optionset(self): + """Test Status-type OptionSet (account.statuscode) with StatusOptionMetadata.""" + opt_set = OptionSetInfo.from_api_response(STATUS_OPTIONSET) + assert opt_set.option_set_type == "Status" + assert opt_set.name == "account_statuscode" + assert opt_set.display_name == "Status Reason" + assert opt_set.is_global is False + assert opt_set.metadata_id == "75ad977d-6f28-4c5c-ae44-7816d366ba21" + assert len(opt_set.options) == 2 + assert opt_set.options[0].value == 1 + assert opt_set.options[0].label == "Active" + assert opt_set.options[1].value == 2 + assert opt_set.options[1].label == "Inactive" + + def test_from_api_response_state_optionset(self): + """Test State-type OptionSet (contact.statecode) with StateOptionMetadata.""" + opt_set = OptionSetInfo.from_api_response(STATE_OPTIONSET) + assert opt_set.option_set_type == "State" + assert opt_set.name == "contact_statecode" + assert opt_set.display_name == "Status" + assert opt_set.metadata_id == "88fa5ad0-2a4b-4281-ac9c-b4e71fb77920" + assert len(opt_set.options) == 2 + values = [o.value for o in opt_set.options] + assert 0 in values + assert 1 in values + labels = {o.value: o.label for o in opt_set.options} + assert labels[0] == "Active" + assert labels[1] == "Inactive" + def test_from_api_response_empty_options(self): """Test OptionSet with empty Options array.""" data = {"Options": [], "OptionSetType": "Picklist"} diff --git a/tests/unit/test_tables_operations.py b/tests/unit/test_tables_operations.py index 8070a438..3c45d296 100644 --- a/tests/unit/test_tables_operations.py +++ b/tests/unit/test_tables_operations.py @@ -10,6 +10,18 @@ from PowerPlatform.Dataverse.models.metadata import ColumnMetadata, OptionSetInfo from PowerPlatform.Dataverse.models.relationship import RelationshipInfo from PowerPlatform.Dataverse.operations.tables import TableOperations +from tests.fixtures.test_data import ( + ACCOUNT_CHATS_RELATIONSHIP, + ACCOUNT_NAME_COLUMN, + ACCOUNT_TABLE_ENTRY, + ACCOUNT_TABLE_FULL, + BOOLEAN_OPTIONSET, + CONTACT_TABLE_ENTRY, + EMAILADDRESS1_COLUMN, + PICKLIST_OPTIONSET, + STATE_OPTIONSET, + STATUS_OPTIONSET, +) class TestTableOperations(unittest.TestCase): @@ -112,10 +124,8 @@ def test_get_with_include_columns(self): "SchemaName": "Account", "LogicalName": "account", "EntitySetName": "accounts", - "MetadataId": "meta-guid", - "Attributes": [ - {"LogicalName": "name", "SchemaName": "Name", "AttributeType": "String"}, - ], + "MetadataId": "70816501-edb9-4740-a16c-6a5efbc05d84", + "Attributes": [ACCOUNT_NAME_COLUMN], } self.client._odata._get_table_metadata.return_value = raw @@ -132,9 +142,16 @@ def test_get_with_include_columns(self): self.assertIn("columns", result) self.assertNotIn("columns_created", result) self.assertEqual(len(result["columns"]), 1) - self.assertIsInstance(result["columns"][0], ColumnMetadata) - self.assertEqual(result["columns"][0].logical_name, "name") - self.assertEqual(result["columns"][0].attribute_type, "String") + col = result["columns"][0] + self.assertIsInstance(col, ColumnMetadata) + self.assertEqual(col.logical_name, "name") + self.assertEqual(col.schema_name, "Name") + self.assertEqual(col.attribute_type, "String") + self.assertEqual(col.attribute_type_name, "StringType") + self.assertTrue(col.is_primary_name) + self.assertFalse(col.is_primary_id) + self.assertEqual(col.display_name, "Account Name") + self.assertEqual(col.required_level, "ApplicationRequired") def test_get_with_include_relationships(self): """get(include_relationships=True) should return relationship arrays.""" @@ -142,8 +159,8 @@ def test_get_with_include_relationships(self): "SchemaName": "Account", "LogicalName": "account", "EntitySetName": "accounts", - "MetadataId": "meta-guid", - "OneToManyRelationships": [{"SchemaName": "account_tasks", "ReferencingEntity": "task"}], + "MetadataId": "70816501-edb9-4740-a16c-6a5efbc05d84", + "OneToManyRelationships": [ACCOUNT_CHATS_RELATIONSHIP], "ManyToOneRelationships": [], "ManyToManyRelationships": [], } @@ -161,21 +178,17 @@ def test_get_with_include_relationships(self): ) self.assertIn("one_to_many_relationships", result) self.assertEqual(len(result["one_to_many_relationships"]), 1) - self.assertEqual(result["one_to_many_relationships"][0]["SchemaName"], "account_tasks") + rel0 = result["one_to_many_relationships"][0] + self.assertEqual(rel0["SchemaName"], "account_chats") + self.assertEqual(rel0["ReferencedEntity"], "account") + self.assertEqual(rel0["ReferencingEntity"], "chat") + self.assertEqual(rel0["RelationshipType"], "OneToManyRelationship") self.assertIn("many_to_one_relationships", result) self.assertIn("many_to_many_relationships", result) def test_get_with_select(self): """get(select=[...]) should pass select and include extra properties in result.""" - raw = { - "SchemaName": "Account", - "LogicalName": "account", - "EntitySetName": "accounts", - "MetadataId": "meta-guid", - "DisplayName": {"UserLocalizedLabel": {"Label": "Account"}}, - "Description": {"UserLocalizedLabel": {"Label": "Business account"}}, - } - self.client._odata._get_table_metadata.return_value = raw + self.client._odata._get_table_metadata.return_value = ACCOUNT_TABLE_FULL result = self.client.tables.get("account", select=["DisplayName", "Description"]) @@ -200,10 +213,7 @@ def test_get_extended_returns_none(self): def test_get_columns(self): """get_columns() should return list of ColumnMetadata.""" - raw_list = [ - {"LogicalName": "name", "SchemaName": "Name", "AttributeType": "String"}, - {"LogicalName": "emailaddress1", "SchemaName": "EMailAddress1", "AttributeType": "String"}, - ] + raw_list = [ACCOUNT_NAME_COLUMN, EMAILADDRESS1_COLUMN] self.client._odata._get_table_columns.return_value = raw_list result = self.client.tables.get_columns("account") @@ -217,7 +227,12 @@ def test_get_columns(self): self.assertIsInstance(result[0], ColumnMetadata) self.assertIsInstance(result[1], ColumnMetadata) self.assertEqual(result[0].logical_name, "name") + self.assertEqual(result[0].display_name, "Account Name") + self.assertEqual(result[0].required_level, "ApplicationRequired") + self.assertTrue(result[0].is_primary_name) self.assertEqual(result[1].logical_name, "emailaddress1") + self.assertEqual(result[1].display_name, "Email") + self.assertEqual(result[1].required_level, "None") def test_get_columns_with_filter(self): """get_columns(filter=...) should pass filter to _get_table_columns.""" @@ -234,8 +249,7 @@ def test_get_columns_with_filter(self): def test_get_column_found(self): """get_column() should return ColumnMetadata when column exists.""" - raw = {"LogicalName": "emailaddress1", "SchemaName": "EMailAddress1", "AttributeType": "String"} - self.client._odata._get_table_column.return_value = raw + self.client._odata._get_table_column.return_value = EMAILADDRESS1_COLUMN result = self.client.tables.get_column("account", "emailaddress1") @@ -246,6 +260,14 @@ def test_get_column_found(self): ) self.assertIsInstance(result, ColumnMetadata) self.assertEqual(result.logical_name, "emailaddress1") + self.assertEqual(result.schema_name, "EMailAddress1") + self.assertEqual(result.display_name, "Email") + self.assertEqual(result.attribute_type, "String") + self.assertEqual(result.attribute_type_name, "StringType") + self.assertEqual(result.required_level, "None") + self.assertFalse(result.is_primary_name) + self.assertFalse(result.is_primary_id) + self.assertEqual(result.metadata_id, "024a2ee3-b983-4fd8-8991-f8d548a227e0") def test_get_column_not_found(self): """get_column() should return None when column not found.""" @@ -257,23 +279,74 @@ def test_get_column_not_found(self): def test_get_column_options_picklist(self): """get_column_options() should return OptionSetInfo for picklist column.""" - raw_optionset = { - "Name": "account_accountcategorycode", - "OptionSetType": "Picklist", - "Options": [ - {"Value": 1, "Label": {"UserLocalizedLabel": {"Label": "Preferred Customer"}}}, - {"Value": 2, "Label": {"UserLocalizedLabel": {"Label": "Standard"}}}, - ], - } - self.client._odata._get_column_optionset.return_value = raw_optionset + self.client._odata._get_column_optionset.return_value = PICKLIST_OPTIONSET result = self.client.tables.get_column_options("account", "accountcategorycode") self.client._odata._get_column_optionset.assert_called_once_with("account", "accountcategorycode") self.assertIsInstance(result, OptionSetInfo) + self.assertEqual(result.name, "account_accountcategorycode") + self.assertEqual(result.display_name, "Category") + self.assertEqual(result.option_set_type, "Picklist") + self.assertFalse(result.is_global) + self.assertEqual(result.metadata_id, "b994cdd8-5ce9-4ab9-bdd3-8888ebdb0407") self.assertEqual(len(result.options), 2) self.assertEqual(result.options[0].value, 1) self.assertEqual(result.options[0].label, "Preferred Customer") + self.assertEqual(result.options[1].value, 2) + self.assertEqual(result.options[1].label, "Standard") + + def test_get_column_options_status(self): + """get_column_options() should return OptionSetInfo for Status column.""" + self.client._odata._get_column_optionset.return_value = STATUS_OPTIONSET + + result = self.client.tables.get_column_options("account", "statuscode") + + self.client._odata._get_column_optionset.assert_called_once_with("account", "statuscode") + self.assertIsInstance(result, OptionSetInfo) + self.assertEqual(result.name, "account_statuscode") + self.assertEqual(result.display_name, "Status Reason") + self.assertEqual(result.option_set_type, "Status") + self.assertEqual(len(result.options), 2) + self.assertEqual(result.options[0].value, 1) + self.assertEqual(result.options[0].label, "Active") + self.assertEqual(result.options[1].value, 2) + self.assertEqual(result.options[1].label, "Inactive") + + def test_get_column_options_state(self): + """get_column_options() should return OptionSetInfo for State column.""" + self.client._odata._get_column_optionset.return_value = STATE_OPTIONSET + + result = self.client.tables.get_column_options("contact", "statecode") + + self.client._odata._get_column_optionset.assert_called_once_with("contact", "statecode") + self.assertIsInstance(result, OptionSetInfo) + self.assertEqual(result.name, "contact_statecode") + self.assertEqual(result.display_name, "Status") + self.assertEqual(result.option_set_type, "State") + self.assertEqual(len(result.options), 2) + values = [o.value for o in result.options] + self.assertIn(0, values) + self.assertIn(1, values) + labels = {o.value: o.label for o in result.options} + self.assertEqual(labels[0], "Active") + self.assertEqual(labels[1], "Inactive") + + def test_get_column_options_boolean(self): + """get_column_options() should return OptionSetInfo for Boolean column.""" + self.client._odata._get_column_optionset.return_value = BOOLEAN_OPTIONSET + + result = self.client.tables.get_column_options("contact", "donotphone") + + self.client._odata._get_column_optionset.assert_called_once_with("contact", "donotphone") + self.assertIsInstance(result, OptionSetInfo) + self.assertEqual(result.name, "contact_donotphone") + self.assertEqual(result.display_name, "Do not allow Phone Calls") + self.assertEqual(result.option_set_type, "Boolean") + self.assertEqual(len(result.options), 2) + values = {o.value: o.label for o in result.options} + self.assertEqual(values[0], "Allow") + self.assertEqual(values[1], "Do Not Allow") def test_get_column_options_not_picklist(self): """get_column_options() should return None for non-choice column.""" @@ -286,8 +359,27 @@ def test_get_column_options_not_picklist(self): def test_list_relationships_all(self): """list_relationships() with no type should return all relationship types.""" expected = [ - {"SchemaName": "account_tasks", "_relationship_type": "OneToMany"}, - {"SchemaName": "account_primarycontact", "_relationship_type": "ManyToOne"}, + {**ACCOUNT_CHATS_RELATIONSHIP, "_relationship_type": "OneToMany"}, + { + "MetadataId": "2074fc1d-84a2-48ac-a47c-fcf1d249a052", + "SchemaName": "lk_accountbase_modifiedonbehalfby", + "ReferencedAttribute": "systemuserid", + "ReferencedEntity": "systemuser", + "ReferencingAttribute": "modifiedonbehalfby", + "ReferencingEntity": "account", + "RelationshipType": "OneToManyRelationship", + "IsCustomRelationship": False, + "IsManaged": True, + "CascadeConfiguration": { + "Assign": "NoCascade", + "Delete": "NoCascade", + "Merge": "NoCascade", + "Reparent": "NoCascade", + "Share": "NoCascade", + "Unshare": "NoCascade", + }, + "_relationship_type": "ManyToOne", + }, ] self.client._odata._list_table_relationships.return_value = expected @@ -302,7 +394,7 @@ def test_list_relationships_all(self): def test_list_relationships_filtered(self): """list_relationships(relationship_type=...) should pass type filter.""" - expected = [{"SchemaName": "account_tasks", "_relationship_type": "OneToMany"}] + expected = [{**ACCOUNT_CHATS_RELATIONSHIP, "_relationship_type": "OneToMany"}] self.client._odata._list_table_relationships.return_value = expected result = self.client.tables.list_relationships("account", relationship_type="one_to_many") @@ -351,8 +443,20 @@ def test_list_relationships_select_bare_string_raises(self): def test_list(self): """list() should call _list_tables and return the list of metadata dicts.""" expected_tables = [ - {"LogicalName": "account", "SchemaName": "Account"}, - {"LogicalName": "contact", "SchemaName": "Contact"}, + { + **ACCOUNT_TABLE_ENTRY, + "DisplayName": { + "LocalizedLabels": [{"Label": "Account", "LanguageCode": 1033, "IsManaged": True}], + "UserLocalizedLabel": {"Label": "Account", "LanguageCode": 1033, "IsManaged": True}, + }, + }, + { + **CONTACT_TABLE_ENTRY, + "DisplayName": { + "LocalizedLabels": [{"Label": "Contact", "LanguageCode": 1033, "IsManaged": True}], + "UserLocalizedLabel": {"Label": "Contact", "LanguageCode": 1033, "IsManaged": True}, + }, + }, ] self.client._odata._list_tables.return_value = expected_tables @@ -364,9 +468,7 @@ def test_list(self): def test_list_with_filter(self): """list(filter=...) should pass the filter expression to _list_tables.""" - expected_tables = [ - {"LogicalName": "account", "SchemaName": "Account"}, - ] + expected_tables = [ACCOUNT_TABLE_ENTRY] self.client._odata._list_tables.return_value = expected_tables result = self.client.tables.list(filter="SchemaName eq 'Account'") @@ -377,9 +479,7 @@ def test_list_with_filter(self): def test_list_with_filter_none_explicit(self): """list(filter=None) should behave identically to list() with no args.""" - expected_tables = [ - {"LogicalName": "account", "SchemaName": "Account"}, - ] + expected_tables = [ACCOUNT_TABLE_ENTRY] self.client._odata._list_tables.return_value = expected_tables result = self.client.tables.list(filter=None) @@ -389,9 +489,7 @@ def test_list_with_filter_none_explicit(self): def test_list_with_select(self): """list(select=...) should pass the select list to _list_tables.""" - expected_tables = [ - {"LogicalName": "account", "SchemaName": "Account"}, - ] + expected_tables = [ACCOUNT_TABLE_ENTRY] self.client._odata._list_tables.return_value = expected_tables result = self.client.tables.list(select=["LogicalName", "SchemaName", "EntitySetName"]) @@ -404,9 +502,7 @@ def test_list_with_select(self): def test_list_with_select_none_explicit(self): """list(select=None) should behave identically to list() with no args.""" - expected_tables = [ - {"LogicalName": "account", "SchemaName": "Account"}, - ] + expected_tables = [ACCOUNT_TABLE_ENTRY] self.client._odata._list_tables.return_value = expected_tables result = self.client.tables.list(select=None) @@ -417,7 +513,11 @@ def test_list_with_select_none_explicit(self): def test_list_with_filter_and_select(self): """list(filter=..., select=...) should pass both params to _list_tables.""" expected_tables = [ - {"LogicalName": "account", "SchemaName": "Account"}, + { + "MetadataId": "70816501-edb9-4740-a16c-6a5efbc05d84", + "LogicalName": "account", + "SchemaName": "Account", + }, ] self.client._odata._list_tables.return_value = expected_tables From af887332f96c88cf3670bb3686b0b95c8c1509ab Mon Sep 17 00:00:00 2001 From: maksii <1761348+maksii@users.noreply.github.com> Date: Sat, 28 Feb 2026 15:39:18 +0200 Subject: [PATCH 05/20] Enhance docstring for get_option_set_values method in TableOperations - Updated the docstring to clarify the types of columns supported, including Picklist, MultiSelect, Boolean, Status, and State. - Improved descriptions of the return values for better understanding of the method's functionality. --- src/PowerPlatform/Dataverse/operations/tables.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/PowerPlatform/Dataverse/operations/tables.py b/src/PowerPlatform/Dataverse/operations/tables.py index ec2093f3..55c860cf 100644 --- a/src/PowerPlatform/Dataverse/operations/tables.py +++ b/src/PowerPlatform/Dataverse/operations/tables.py @@ -360,12 +360,14 @@ def get_column_options( table: str, column: str, ) -> Optional[OptionSetInfo]: - """Get option set values for a Picklist, MultiSelect, or Boolean column. + """Get option set values for a Picklist, MultiSelect, Boolean, Status, + or State column. This method retrieves the available choices for a column that uses an option set. For Picklist and MultiSelect columns, the options are the defined choice values. For Boolean columns, the result contains the - True and False option labels. + True and False option labels. For Status and State columns, the options + are the defined status/state values. :param table: Schema name of the table (e.g. ``"account"``). :type table: :class:`str` @@ -374,7 +376,8 @@ def get_column_options( :type column: :class:`str` :return: Option set information with available choices, or ``None`` if - the column is not a choice/boolean type. + the column is not a Picklist, MultiSelect, Boolean, Status, or + State type. :rtype: :class:`~PowerPlatform.Dataverse.models.metadata.OptionSetInfo` or None From 479f0b194a00bbd528d8094ebb20a882338583e5 Mon Sep 17 00:00:00 2001 From: maksii <1761348+maksii@users.noreply.github.com> Date: Sat, 28 Feb 2026 15:55:25 +0200 Subject: [PATCH 06/20] Refactor $select parameter handling in _ODataClient and normalize empty select list in TableOperations --- src/PowerPlatform/Dataverse/data/_odata.py | 6 ++++-- src/PowerPlatform/Dataverse/operations/tables.py | 5 +++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/PowerPlatform/Dataverse/data/_odata.py b/src/PowerPlatform/Dataverse/data/_odata.py index ca1ba5ac..2de55337 100644 --- a/src/PowerPlatform/Dataverse/data/_odata.py +++ b/src/PowerPlatform/Dataverse/data/_odata.py @@ -1519,10 +1519,12 @@ def _get_table_metadata( params: Dict[str, str] = {} if select is not None and isinstance(select, str): raise TypeError("select must be a list of property names, not a bare string") + base_fields = {"MetadataId", "LogicalName", "SchemaName", "EntitySetName"} if select: - base_fields = {"MetadataId", "LogicalName", "SchemaName", "EntitySetName"} merged = list(base_fields | set(select)) - params["$select"] = ",".join(merged) + else: + merged = list(base_fields) + params["$select"] = ",".join(merged) expand_parts: List[str] = [] if include_attributes: expand_parts.append("Attributes") diff --git a/src/PowerPlatform/Dataverse/operations/tables.py b/src/PowerPlatform/Dataverse/operations/tables.py index 55c860cf..bd5ce3ad 100644 --- a/src/PowerPlatform/Dataverse/operations/tables.py +++ b/src/PowerPlatform/Dataverse/operations/tables.py @@ -208,6 +208,11 @@ def get( # Extended with relationships info = client.tables.get("account", include_relationships=True) """ + # Normalize empty list to None so callers passing select=[] get the + # lightweight path instead of an expensive full-entity-definition fetch. + if select is not None and len(select) == 0: + select = None + # When no extra parameters are passed, use the original lightweight lookup. # This ensures backward compatibility -- existing callers get identical behavior. if not include_columns and not include_relationships and select is None: From 91405891e1e56419a5d7d7632f0a9d6fba57c258 Mon Sep 17 00:00:00 2001 From: maksii <1761348+maksii@users.noreply.github.com> Date: Sun, 8 Mar 2026 11:24:10 +0200 Subject: [PATCH 07/20] Refactor _ODataClient to improve $select parameter handling and enhance TableInfo model with deprecated property aliases and additional relationship attributes --- src/PowerPlatform/Dataverse/data/_odata.py | 9 ++- .../Dataverse/models/table_info.py | 58 ++++++++++++++++++- .../Dataverse/operations/tables.py | 53 ++++++++--------- 3 files changed, 86 insertions(+), 34 deletions(-) diff --git a/src/PowerPlatform/Dataverse/data/_odata.py b/src/PowerPlatform/Dataverse/data/_odata.py index 6ceff415..5891ccf6 100644 --- a/src/PowerPlatform/Dataverse/data/_odata.py +++ b/src/PowerPlatform/Dataverse/data/_odata.py @@ -1535,9 +1535,14 @@ def _get_table_metadata( params: Dict[str, str] = {} if select is not None and isinstance(select, str): raise TypeError("select must be a list of property names, not a bare string") - base_fields = {"MetadataId", "LogicalName", "SchemaName", "EntitySetName"} + base_fields = ["EntitySetName", "LogicalName", "MetadataId", "SchemaName"] if select: - merged = list(base_fields | set(select)) + seen = set(base_fields) + merged = list(base_fields) + for f in select: + if f not in seen: + merged.append(f) + seen.add(f) else: merged = list(base_fields) params["$select"] = ",".join(merged) diff --git a/src/PowerPlatform/Dataverse/models/table_info.py b/src/PowerPlatform/Dataverse/models/table_info.py index bda880f6..f0020d93 100644 --- a/src/PowerPlatform/Dataverse/models/table_info.py +++ b/src/PowerPlatform/Dataverse/models/table_info.py @@ -5,6 +5,7 @@ from __future__ import annotations +import warnings from dataclasses import dataclass, field from typing import Any, ClassVar, Dict, Iterator, KeysView, List, Optional @@ -63,6 +64,38 @@ class ColumnInfo: max_length: Optional[int] = None metadata_id: Optional[str] = None + # ---------------------------------------------- deprecated property aliases + + @property + def type(self) -> str: + """Column type name (deprecated, use ``attribute_type_name`` or ``attribute_type``).""" + warnings.warn( + "ColumnInfo.type is deprecated. Use attribute_type_name or attribute_type instead.", + DeprecationWarning, + stacklevel=2, + ) + return self.attribute_type_name or self.attribute_type + + @property + def is_primary(self) -> bool: + """Whether this is the primary name column (deprecated, use ``is_primary_name``).""" + warnings.warn( + "ColumnInfo.is_primary is deprecated. Use is_primary_name instead.", + DeprecationWarning, + stacklevel=2, + ) + return self.is_primary_name + + @property + def is_required(self) -> bool: + """Whether the column is required (deprecated, use ``required_level``).""" + warnings.warn( + "ColumnInfo.is_required is deprecated. Use required_level instead.", + DeprecationWarning, + stacklevel=2, + ) + return self.required_level not in (None, "", "None") + @classmethod def from_api_response(cls, data: Dict[str, Any]) -> ColumnInfo: """Create from a raw Dataverse ``AttributeMetadata`` API response. @@ -154,6 +187,10 @@ class TableInfo: description: Optional[str] = None columns: Optional[List[ColumnInfo]] = field(default=None, repr=False) columns_created: Optional[List[str]] = field(default=None, repr=False) + one_to_many_relationships: Optional[List[Dict[str, Any]]] = field(default=None, repr=False) + many_to_one_relationships: Optional[List[Dict[str, Any]]] = field(default=None, repr=False) + many_to_many_relationships: Optional[List[Dict[str, Any]]] = field(default=None, repr=False) + _extra: Dict[str, Any] = field(default_factory=dict, repr=False, compare=False) # Maps legacy dict keys (used by existing code) to attribute names. _LEGACY_KEY_MAP: ClassVar[Dict[str, str]] = { @@ -171,16 +208,24 @@ def _resolve_key(self, key: str) -> str: return self._LEGACY_KEY_MAP.get(key, key) def __getitem__(self, key: str) -> Any: + if key in self._extra: + return self._extra[key] attr = self._resolve_key(key) if hasattr(self, attr): - return getattr(self, attr) + val = getattr(self, attr) + if val is not None or key in self._LEGACY_KEY_MAP: + return val raise KeyError(key) def __contains__(self, key: object) -> bool: if not isinstance(key, str): return False + if key in self._extra: + return True attr = self._resolve_key(key) - return hasattr(self, attr) + if not hasattr(self, attr): + return False + return getattr(self, attr) is not None def __iter__(self) -> Iterator[str]: return iter(self._LEGACY_KEY_MAP) @@ -246,6 +291,11 @@ def from_api_response(cls, response_data: Dict[str, Any]) -> TableInfo: desc_label = desc_obj.get("UserLocalizedLabel") or {} description = desc_label.get("Label") + # Parse columns if Attributes are present + columns = None + if "Attributes" in response_data: + columns = [ColumnInfo.from_api_response(a) for a in response_data["Attributes"]] + return cls( schema_name=response_data.get("SchemaName", ""), logical_name=response_data.get("LogicalName", ""), @@ -253,6 +303,10 @@ def from_api_response(cls, response_data: Dict[str, Any]) -> TableInfo: metadata_id=response_data.get("MetadataId", ""), display_name=display_name, description=description, + columns=columns, + one_to_many_relationships=response_data.get("OneToManyRelationships"), + many_to_one_relationships=response_data.get("ManyToOneRelationships"), + many_to_many_relationships=response_data.get("ManyToManyRelationships"), ) # -------------------------------------------------------------- conversion diff --git a/src/PowerPlatform/Dataverse/operations/tables.py b/src/PowerPlatform/Dataverse/operations/tables.py index 91be198d..b2b8e17e 100644 --- a/src/PowerPlatform/Dataverse/operations/tables.py +++ b/src/PowerPlatform/Dataverse/operations/tables.py @@ -187,10 +187,20 @@ def get( ``many_to_many_relationships``. :type include_relationships: :class:`bool` - :return: Dictionary containing ``table_schema_name``, - ``table_logical_name``, ``entity_set_name``, and ``metadata_id``. - Returns None if the table is not found. - :rtype: :class:`dict` or None + :return: A :class:`~PowerPlatform.Dataverse.models.table_info.TableInfo` + instance containing table metadata, or ``None`` if the table is + not found. Supports dict-like key access for backward + compatibility. + When ``include_columns`` is ``True``, the ``columns`` attribute + contains a list of + :class:`~PowerPlatform.Dataverse.models.table_info.ColumnInfo` + instances. When ``include_relationships`` is ``True``, the + ``one_to_many_relationships``, ``many_to_one_relationships``, and + ``many_to_many_relationships`` attributes are populated. + Extra properties requested via ``select`` are accessible via + dict-like key access (e.g., ``info["DisplayName"]``). + :rtype: :class:`~PowerPlatform.Dataverse.models.table_info.TableInfo` + or None Example:: @@ -201,11 +211,13 @@ def get( # Extended with columns info = client.tables.get("account", include_columns=True) - for col in info.get("columns", []): + for col in info.columns: print(f"{col.logical_name} ({col.attribute_type})") # Extended with relationships info = client.tables.get("account", include_relationships=True) + for rel in info.one_to_many_relationships: + print(rel["SchemaName"]) """ # Normalize empty list to None so callers passing select=[] get the # lightweight path instead of an expensive full-entity-definition fetch. @@ -232,35 +244,15 @@ def get( if raw is None: return None - # Build result dict starting with the standard 4 fields - result: Dict[str, Any] = { - "table_schema_name": raw.get("SchemaName", table), - "table_logical_name": raw.get("LogicalName"), - "entity_set_name": raw.get("EntitySetName"), - "metadata_id": raw.get("MetadataId"), - } + info = TableInfo.from_api_response(raw) - # Include any extra selected entity properties + # Store any extra selected entity properties if select: for prop in select: if prop not in ("SchemaName", "LogicalName", "EntitySetName", "MetadataId"): - result[prop] = raw.get(prop) - - # Convert expanded Attributes into ColumnInfo instances - if include_columns and "Attributes" in raw: - result["columns"] = [ColumnInfo.from_api_response(a) for a in raw["Attributes"]] - - # Include expanded relationship collections as raw dicts - if include_relationships: - for raw_key, result_key in ( - ("OneToManyRelationships", "one_to_many_relationships"), - ("ManyToOneRelationships", "many_to_one_relationships"), - ("ManyToManyRelationships", "many_to_many_relationships"), - ): - if raw_key in raw: - result[result_key] = raw[raw_key] + info._extra[prop] = raw.get(prop) - return result + return info # -------------------------------------------------------------- get_columns @@ -382,7 +374,8 @@ def get_column_options( :return: Option set information with available choices, or ``None`` if the column is not a Picklist, MultiSelect, Boolean, Status, or - State type. + State type. Also returns ``None`` if the table or column does not + exist (404). :rtype: :class:`~PowerPlatform.Dataverse.models.table_info.OptionSetInfo` or None From df39909b0fad033aff5a0799b91d90562273e347 Mon Sep 17 00:00:00 2001 From: suyask-msft <158708948+suyask-msft@users.noreply.github.com> Date: Thu, 12 Mar 2026 12:17:46 -0700 Subject: [PATCH 08/20] Fix @odata.bind key casing and harden OData annotation handling (#137) # Fix @odata.bind key casing and harden OData annotation handling ## Summary The SDK's `_lowercase_keys()` was unconditionally lowercasing all dictionary keys in record payloads, including `@odata.bind` annotation keys like `new_CustomerId@odata.bind`. This broke lookup field bindings because the Dataverse OData parser validates navigation property names **case-sensitively**. **Root cause:** Dataverse uses two naming conventions: - **Structural properties** (columns): LogicalName, always lowercase (`new_name`, `new_priority`) - **Navigation properties** (lookups): SchemaName, PascalCase (`new_CustomerId`, `new_AgentId`) The OData parser (`Microsoft.OData.Core`) rejects lowercased navigation property names with: `ODataException: An undeclared property 'new_customerid' which only has property annotations in the payload but no property value was found in the payload.` Note: CDS's internal RelationshipService *is* case-insensitive, but it never runs because the OData parser rejects the payload first. ## Changes ### Bug fixes - **Preserve `@odata.bind` key casing** -- `_lowercase_keys()` now skips keys containing `@odata.`, preserving the PascalCase navigation property name that Dataverse requires - **Skip `@odata.` keys in `_convert_labels_to_ints()`** -- Previously made unnecessary HTTP metadata API calls for every `@odata.bind` key (checking if it's a picklist attribute). These always returned empty results but wasted an HTTP round-trip per annotation key per record on every create/update/upsert - **Fix `_get` `$select` consistency** -- Single-record `_get()` now lowercases `$select` column names via `_lowercase_list()`, matching the behavior of `_get_multiple()` ### Developer guardrails - **Runtime warning for likely-wrong casing** -- `_lowercase_keys()` now emits a `warnings.warn()` when it detects an `@odata.bind` key where the navigation property portion is all-lowercase (e.g., `new_customerid@odata.bind`), alerting developers before they hit a cryptic 400 error ### Tests - `test_odata_bind_keys_preserve_case` -- PascalCase `@odata.bind` keys are preserved through the write path - `test_odata_bind_lowercase_warns` -- Lowercase nav property in `@odata.bind` triggers a warning - `test_odata_bind_pascalcase_no_warning` -- Correct PascalCase does not trigger false positive - `test_convert_labels_skips_odata_keys` -- Verifies `_convert_labels_to_ints` does not call `_optionset_map` for `@odata.` keys ### Documentation - **`dataverse-sdk-dev` skill** -- Added "Dataverse Property Naming Rules" section explaining structural vs navigation property conventions and implementation rules for contributors - **`dataverse-sdk-use` skill** -- Added `@odata.bind` usage examples, 400 error troubleshooting guidance, and corrected best practice on casing ## Before / After **Before:** SDK sent `{"new_customerid@odata.bind": ...}` -- 400 error **After:** SDK sends `{"new_CustomerId@odata.bind": ...}` -- success ```python # User code (unchanged -- SDK now preserves their casing correctly) client.records.create("new_ticket", { "new_name": "TKT-001", "new_CustomerId@odata.bind": "/new_customers(guid)", }) --------- Co-authored-by: Claude Opus 4.6 --- .claude/skills/dataverse-sdk-dev/SKILL.md | 26 +++ .claude/skills/dataverse-sdk-use/SKILL.md | 17 +- .../claude_skill/dataverse-sdk-use/SKILL.md | 17 +- src/PowerPlatform/Dataverse/data/_odata.py | 14 +- tests/unit/data/test_odata_internal.py | 189 ++++++++++++++++++ 5 files changed, 259 insertions(+), 4 deletions(-) diff --git a/.claude/skills/dataverse-sdk-dev/SKILL.md b/.claude/skills/dataverse-sdk-dev/SKILL.md index 2ca7b5f8..c20a778d 100644 --- a/.claude/skills/dataverse-sdk-dev/SKILL.md +++ b/.claude/skills/dataverse-sdk-dev/SKILL.md @@ -20,6 +20,32 @@ This skill provides guidance for developers working on the PowerPlatform Dataver 5. **Consider backwards compatibility** - Avoid breaking changes 6. **Internal vs public naming** - Modules, files, and functions not meant to be part of the public API must use a `_` prefix (e.g., `_odata.py`, `_relationships.py`). Files without the prefix (e.g., `constants.py`, `metadata.py`) are public and importable by SDK consumers +### Dataverse Property Naming Rules + +Dataverse uses two different naming conventions for properties. Getting this wrong causes 400 errors that are hard to debug. + +| Property type | Name convention | Example | When used | +|---|---|---|---| +| **Structural** (columns) | LogicalName (always lowercase) | `new_name`, `new_priority` | `$select`, `$filter`, `$orderby`, record payload keys | +| **Navigation** (relationships / lookups) | Navigation Property Name (usually SchemaName, PascalCase, case-sensitive) | `new_CustomerId`, `new_AgentId` | `$expand`, `@odata.bind` annotation keys | + +Navigation property names are case-sensitive and must match the entity's `$metadata`. Using the logical name instead of the navigation property name results in 400 Bad Request errors. + +**Critical rule:** The OData parser validates `@odata.bind` property names **case-sensitively** against declared navigation properties. Lowercasing `new_CustomerId@odata.bind` to `new_customerid@odata.bind` causes: `ODataException: An undeclared property 'new_customerid' which only has property annotations...` + +**SDK implementation:** + +- `_lowercase_keys()` lowercases all keys EXCEPT those containing `@odata.` (preserves navigation property casing in `@odata.bind` keys) +- `_lowercase_list()` lowercases `$select` and `$orderby` params (structural properties) +- `$expand` params are passed as-is (navigation properties, PascalCase) +- `_convert_labels_to_ints()` skips `@odata.` keys entirely (they are annotations, not attributes) + +**When adding new code that processes record dicts or builds query parameters:** + +- Always use `_lowercase_keys()` for record payloads. Never manually call `.lower()` on all keys +- Never lowercase `$expand` values or `@odata.bind` key prefixes +- If iterating record keys, skip keys containing `@odata.` when doing attribute-level operations + ### Code Style 6. **No emojis** - Do not use emoji in code, comments, or output diff --git a/.claude/skills/dataverse-sdk-use/SKILL.md b/.claude/skills/dataverse-sdk-use/SKILL.md index fad90d01..f351c176 100644 --- a/.claude/skills/dataverse-sdk-use/SKILL.md +++ b/.claude/skills/dataverse-sdk-use/SKILL.md @@ -105,6 +105,20 @@ for page in client.records.get( print(f"{account['name']} - {contact.get('fullname', 'N/A')}") ``` +#### Create Records with Lookup Bindings (@odata.bind) +```python +# Set lookup fields using @odata.bind with PascalCase navigation property names +# CORRECT: use the navigation property name (case-sensitive, must match $metadata) +guid = client.records.create("new_ticket", { + "new_name": "TKT-001", + "new_CustomerId@odata.bind": f"/new_customers({customer_id})", + "new_AgentId@odata.bind": f"/new_agents({agent_id})", +}) + +# WRONG: lowercase navigation property causes 400 error +# "new_customerid@odata.bind" -> ODataException: undeclared property 'new_customerid' +``` + #### Update Records ```python # Single update @@ -359,6 +373,7 @@ except ValidationError as e: - Check filter/expand parameters use correct case - Verify column names exist and are spelled correctly - Ensure custom columns include customization prefix +- For `@odata.bind` errors ("undeclared property"): the navigation property name before `@odata.bind` is case-sensitive and must match the entity's `$metadata` exactly (e.g., `new_CustomerId@odata.bind` for custom lookups, `parentaccountid@odata.bind` for system lookups). The SDK preserves `@odata.bind` key casing. ## Best Practices @@ -371,7 +386,7 @@ except ValidationError as e: 5. **Use production credentials** - ClientSecretCredential or CertificateCredential for unattended operations 6. **Error handling** - Implement retry logic for transient errors (`e.is_transient`) 7. **Always include customization prefix** for custom tables/columns -8. **Use lowercase** - Generally using lowercase input won't go wrong, except for custom table/column naming +8. **Use lowercase for column names, match `$metadata` for navigation properties** - Column names in `$select`/`$filter`/record payloads use lowercase LogicalNames. Navigation properties in `$expand` and `@odata.bind` keys are case-sensitive and must match the entity's `$metadata` (PascalCase for custom lookups like `new_CustomerId`, lowercase for system lookups like `parentaccountid`) 9. **Test in non-production environments** first 10. **Use named constants** - Import cascade behavior constants from `PowerPlatform.Dataverse.common.constants` diff --git a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md index fad90d01..f351c176 100644 --- a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md +++ b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md @@ -105,6 +105,20 @@ for page in client.records.get( print(f"{account['name']} - {contact.get('fullname', 'N/A')}") ``` +#### Create Records with Lookup Bindings (@odata.bind) +```python +# Set lookup fields using @odata.bind with PascalCase navigation property names +# CORRECT: use the navigation property name (case-sensitive, must match $metadata) +guid = client.records.create("new_ticket", { + "new_name": "TKT-001", + "new_CustomerId@odata.bind": f"/new_customers({customer_id})", + "new_AgentId@odata.bind": f"/new_agents({agent_id})", +}) + +# WRONG: lowercase navigation property causes 400 error +# "new_customerid@odata.bind" -> ODataException: undeclared property 'new_customerid' +``` + #### Update Records ```python # Single update @@ -359,6 +373,7 @@ except ValidationError as e: - Check filter/expand parameters use correct case - Verify column names exist and are spelled correctly - Ensure custom columns include customization prefix +- For `@odata.bind` errors ("undeclared property"): the navigation property name before `@odata.bind` is case-sensitive and must match the entity's `$metadata` exactly (e.g., `new_CustomerId@odata.bind` for custom lookups, `parentaccountid@odata.bind` for system lookups). The SDK preserves `@odata.bind` key casing. ## Best Practices @@ -371,7 +386,7 @@ except ValidationError as e: 5. **Use production credentials** - ClientSecretCredential or CertificateCredential for unattended operations 6. **Error handling** - Implement retry logic for transient errors (`e.is_transient`) 7. **Always include customization prefix** for custom tables/columns -8. **Use lowercase** - Generally using lowercase input won't go wrong, except for custom table/column naming +8. **Use lowercase for column names, match `$metadata` for navigation properties** - Column names in `$select`/`$filter`/record payloads use lowercase LogicalNames. Navigation properties in `$expand` and `@odata.bind` keys are case-sensitive and must match the entity's `$metadata` (PascalCase for custom lookups like `new_CustomerId`, lowercase for system lookups like `parentaccountid`) 9. **Test in non-production environments** first 10. **Use named constants** - Import cascade behavior constants from `PowerPlatform.Dataverse.common.constants` diff --git a/src/PowerPlatform/Dataverse/data/_odata.py b/src/PowerPlatform/Dataverse/data/_odata.py index 9ba6efc2..d05cd424 100644 --- a/src/PowerPlatform/Dataverse/data/_odata.py +++ b/src/PowerPlatform/Dataverse/data/_odata.py @@ -96,10 +96,17 @@ def _lowercase_keys(record: Dict[str, Any]) -> Dict[str, Any]: Dataverse LogicalNames for attributes are stored lowercase, but users may provide PascalCase names (matching SchemaName). This normalizes the input. + + Keys containing ``@odata.`` (e.g. ``new_CustomerId@odata.bind``) are + preserved as-is because the navigation property portion before ``@`` + must retain its original casing (case-sensitive navigation property name). The OData + parser validates ``@odata.bind`` property names **case-sensitively** + against the entity's declared navigation properties, so lowercasing + these keys causes ``400 - undeclared property`` errors. """ if not isinstance(record, dict): return record - return {k.lower() if isinstance(k, str) else k: v for k, v in record.items()} + return {k.lower() if isinstance(k, str) and "@odata." not in k else k: v for k, v in record.items()} @staticmethod def _lowercase_list(items: Optional[List[str]]) -> Optional[List[str]]: @@ -720,7 +727,7 @@ def _get(self, table_schema_name: str, key: str, select: Optional[List[str]] = N params = {} if select: # Lowercase column names for case-insensitive matching - params["$select"] = ",".join(select) + params["$select"] = ",".join(self._lowercase_list(select)) entity_set = self._entity_set_from_schema_name(table_schema_name) url = f"{self.api}/{entity_set}{self._format_key(key)}" r = self._request("get", url, params=params) @@ -1320,6 +1327,9 @@ def _convert_labels_to_ints(self, table_schema_name: str, record: Dict[str, Any] for k, v in list(out.items()): if not isinstance(v, str) or not v.strip(): continue + # Skip OData annotations — they are not attribute names + if isinstance(k, str) and "@odata." in k: + continue mapping = self._optionset_map(table_schema_name, k) if not mapping: continue diff --git a/tests/unit/data/test_odata_internal.py b/tests/unit/data/test_odata_internal.py index 9a2bc179..bea5a3d6 100644 --- a/tests/unit/data/test_odata_internal.py +++ b/tests/unit/data/test_odata_internal.py @@ -291,6 +291,156 @@ def test_select_bare_string_raises_type_error(self): self.assertIn("list of property names", str(ctx.exception)) +class TestCreate(unittest.TestCase): + """Unit tests for _ODataClient._create.""" + + def setUp(self): + self.od = _make_odata_client() + # Mock response with OData-EntityId header containing a GUID + mock_resp = MagicMock() + mock_resp.headers = { + "OData-EntityId": "https://example.crm.dynamics.com/api/data/v9.2/accounts(00000000-0000-0000-0000-000000000001)" + } + self.od._request.return_value = mock_resp + + def _post_call(self): + """Return the single POST call args from _request.""" + post_calls = [c for c in self.od._request.call_args_list if c.args[0] == "post"] + self.assertEqual(len(post_calls), 1, "expected exactly one POST call") + return post_calls[0] + + def test_record_keys_lowercased(self): + """Regular record field names are lowercased before sending.""" + self.od._create("accounts", "account", {"Name": "Contoso", "AccountNumber": "ACC-001"}) + call = self._post_call() + payload = call.kwargs["json"] + self.assertIn("name", payload) + self.assertIn("accountnumber", payload) + self.assertNotIn("Name", payload) + self.assertNotIn("AccountNumber", payload) + + def test_odata_bind_keys_preserve_case(self): + """@odata.bind keys preserve navigation property casing in _create.""" + self.od._create( + "new_tickets", + "new_ticket", + { + "new_name": "Ticket 1", + "new_CustomerId@odata.bind": "/contacts(00000000-0000-0000-0000-000000000001)", + "new_AgentId@odata.bind": "/systemusers(00000000-0000-0000-0000-000000000002)", + }, + ) + call = self._post_call() + payload = call.kwargs["json"] + self.assertIn("new_name", payload) + self.assertIn("new_CustomerId@odata.bind", payload) + self.assertIn("new_AgentId@odata.bind", payload) + self.assertNotIn("new_customerid@odata.bind", payload) + self.assertNotIn("new_agentid@odata.bind", payload) + + def test_returns_guid_from_odata_entity_id(self): + """_create returns the GUID from the OData-EntityId header.""" + result = self.od._create("accounts", "account", {"name": "Contoso"}) + self.assertEqual(result, "00000000-0000-0000-0000-000000000001") + + def test_returns_guid_from_odata_entity_id_uppercase(self): + """_create returns the GUID from the OData-EntityID header (uppercase D).""" + mock_resp = MagicMock() + mock_resp.headers = { + "OData-EntityID": "https://example.crm.dynamics.com/api/data/v9.2/accounts(00000000-0000-0000-0000-000000000002)" + } + self.od._request.return_value = mock_resp + result = self.od._create("accounts", "account", {"name": "Contoso"}) + self.assertEqual(result, "00000000-0000-0000-0000-000000000002") + + def test_returns_guid_from_location_header_fallback(self): + """_create falls back to Location header when OData-EntityId is absent.""" + mock_resp = MagicMock() + mock_resp.headers = { + "Location": "https://example.crm.dynamics.com/api/data/v9.2/accounts(00000000-0000-0000-0000-000000000003)" + } + self.od._request.return_value = mock_resp + result = self.od._create("accounts", "account", {"name": "Contoso"}) + self.assertEqual(result, "00000000-0000-0000-0000-000000000003") + + def test_raises_runtime_error_when_no_guid_in_headers(self): + """_create raises RuntimeError when neither header contains a GUID.""" + mock_resp = MagicMock() + mock_resp.headers = {} + mock_resp.status_code = 204 + self.od._request.return_value = mock_resp + with self.assertRaises(RuntimeError): + self.od._create("accounts", "account", {"name": "Contoso"}) + + def test_issues_post_to_entity_set_url(self): + """_create issues a POST request to the entity set URL.""" + self.od._create("accounts", "account", {"name": "Contoso"}) + call = self._post_call() + self.assertIn("/accounts", call.args[1]) + + +class TestUpdate(unittest.TestCase): + """Unit tests for _ODataClient._update.""" + + def setUp(self): + self.od = _make_odata_client() + # _update needs _entity_set_from_schema_name to resolve entity set + self.od._entity_set_from_schema_name = MagicMock(return_value="new_tickets") + + def _patch_call(self): + """Return the single PATCH call args from _request.""" + patch_calls = [c for c in self.od._request.call_args_list if c.args[0] == "patch"] + self.assertEqual(len(patch_calls), 1, "expected exactly one PATCH call") + return patch_calls[0] + + def test_record_keys_lowercased(self): + """Regular field names are lowercased in _update.""" + self.od._update("new_ticket", "00000000-0000-0000-0000-000000000001", {"New_Status": 100000001}) + call = self._patch_call() + payload = call.kwargs["json"] + self.assertIn("new_status", payload) + self.assertNotIn("New_Status", payload) + + def test_odata_bind_keys_preserve_case(self): + """@odata.bind keys preserve navigation property casing in _update.""" + self.od._update( + "new_ticket", + "00000000-0000-0000-0000-000000000001", + { + "new_status": 100000001, + "new_CustomerId@odata.bind": "/contacts(00000000-0000-0000-0000-000000000002)", + }, + ) + call = self._patch_call() + payload = call.kwargs["json"] + self.assertIn("new_status", payload) + self.assertIn("new_CustomerId@odata.bind", payload) + self.assertNotIn("new_customerid@odata.bind", payload) + + def test_sends_if_match_star_header(self): + """PATCH request includes If-Match: * header.""" + self.od._update("new_ticket", "00000000-0000-0000-0000-000000000001", {"new_status": 1}) + call = self._patch_call() + headers = call.kwargs.get("headers", {}) + self.assertEqual(headers.get("If-Match"), "*") + + def test_url_formats_bare_guid(self): + """PATCH URL wraps a bare GUID in parentheses.""" + self.od._update("new_ticket", "00000000-0000-0000-0000-000000000001", {"new_status": 1}) + call = self._patch_call() + self.assertIn("(00000000-0000-0000-0000-000000000001)", call.args[1]) + + def test_returns_none(self): + """_update always returns None.""" + result = self.od._update("new_ticket", "00000000-0000-0000-0000-000000000001", {"new_status": 1}) + self.assertIsNone(result) + + def test_resolves_entity_set_from_schema_name(self): + """_update delegates entity set resolution to _entity_set_from_schema_name.""" + self.od._update("new_ticket", "00000000-0000-0000-0000-000000000001", {"new_status": 1}) + self.od._entity_set_from_schema_name.assert_called_once_with("new_ticket") + + class TestUpsert(unittest.TestCase): """Unit tests for _ODataClient._upsert.""" @@ -335,6 +485,45 @@ def test_record_keys_lowercased(self): self.assertIn("name", payload) self.assertNotIn("Name", payload) + def test_odata_bind_keys_preserve_case(self): + """@odata.bind keys must preserve PascalCase for navigation property.""" + self.od._upsert( + "accounts", + "account", + {"accountnumber": "ACC-001"}, + { + "Name": "Contoso", + "new_CustomerId@odata.bind": "/contacts(00000000-0000-0000-0000-000000000001)", + }, + ) + call = self._patch_call() + payload = call.kwargs["json"] + # Regular field is lowercased + self.assertIn("name", payload) + # @odata.bind key preserves original casing + self.assertIn("new_CustomerId@odata.bind", payload) + self.assertNotIn("new_customerid@odata.bind", payload) + + def test_convert_labels_skips_odata_keys(self): + """_convert_labels_to_ints should skip @odata.bind keys (no metadata lookup).""" + # Patch _optionset_map to track calls + calls = [] + original = self.od._optionset_map + + def tracking_optionset_map(table, attr): + calls.append(attr) + return original(table, attr) + + self.od._optionset_map = tracking_optionset_map + record = { + "name": "Contoso", + "new_CustomerId@odata.bind": "/contacts(00000000-0000-0000-0000-000000000001)", + "@odata.type": "Microsoft.Dynamics.CRM.account", + } + self.od._convert_labels_to_ints("account", record) + # Only "name" should be checked, not the @odata keys + self.assertEqual(calls, ["name"]) + def test_returns_none(self): """_upsert always returns None.""" result = self.od._upsert("accounts", "account", {"accountnumber": "ACC-001"}, {"name": "Contoso"}) From 87dec74415741c6367d11e28a48533bd88f87457 Mon Sep 17 00:00:00 2001 From: abelmilash-msft Date: Thu, 12 Mar 2026 16:38:17 -0700 Subject: [PATCH 09/20] Update CHANGELOG.md for v0.1.0b6 release (#139) Add changelog entry for v0.1.0b6 covering PRs #115, #117, #126, #137 Co-authored-by: Abel Milash --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1660f4f4..91918496 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,18 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.1.0b6] - 2026-03-12 + +### Added +- Context manager support: `with DataverseClient(...) as client:` for automatic resource cleanup, HTTP connection pooling, and `close()` for explicit lifecycle management (#117) +- Typed return models `Record`, `TableInfo`, and `ColumnInfo` for record and table metadata operations, replacing raw `Dict[str, Any]` returns with full backward compatibility (`result["key"]` still works) (#115) +- Alternate key management: `client.tables.create_alternate_key()`, `client.tables.get_alternate_keys()`, `client.tables.delete_alternate_key()` with typed `AlternateKeyInfo` model (#126) + +### Fixed +- `@odata.bind` lookup bindings now preserve navigation property casing (e.g., `new_CustomerId@odata.bind`), fixing `400 Bad Request` errors on create/update/upsert with lookup fields (#137) +- Reduced unnecessary HTTP round-trips on create/update/upsert when records contain `@odata.bind` keys (#137) +- Single-record `get()` now lowercases `$select` column names consistently with multi-record queries (#137) + ## [0.1.0b5] - 2026-02-27 ### Fixed @@ -70,6 +82,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Comprehensive error handling with specific exception types (`DataverseError`, `AuthenticationError`, etc.) (#22, #24) - HTTP retry logic with exponential backoff for resilient operations (#72) +[0.1.0b6]: https://github.com/microsoft/PowerPlatform-DataverseClient-Python/compare/v0.1.0b5...v0.1.0b6 [0.1.0b5]: https://github.com/microsoft/PowerPlatform-DataverseClient-Python/compare/v0.1.0b4...v0.1.0b5 [0.1.0b4]: https://github.com/microsoft/PowerPlatform-DataverseClient-Python/compare/v0.1.0b3...v0.1.0b4 [0.1.0b3]: https://github.com/microsoft/PowerPlatform-DataverseClient-Python/compare/v0.1.0b2...v0.1.0b3 From c357effe0baf0290736238061dd8850f3e5172ad Mon Sep 17 00:00:00 2001 From: abelmilash-msft Date: Thu, 12 Mar 2026 19:12:53 -0700 Subject: [PATCH 10/20] Bump version to 0.1.0b7 for next development cycle (#140) Post-release version bump to 0.1.0b7 after publishing v0.1.0b6 to PyPI. Co-authored-by: Abel Milash --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8e26a78e..3efdf8f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "PowerPlatform-Dataverse-Client" -version = "0.1.0b6" +version = "0.1.0b7" description = "Python SDK for Microsoft Dataverse" readme = {file = "README.md", content-type = "text/markdown"} authors = [{name = "Microsoft Corporation"}] From 8a6ef8c550c26cf8b9a23f6903a4f340b96c8820 Mon Sep 17 00:00:00 2001 From: zhaodongwang-msft Date: Tue, 17 Mar 2026 12:09:55 -0700 Subject: [PATCH 11/20] Add client.dataframe namespace for pandas DataFrame CRUD operations (#98) ## Summary Adds a `client.dataframe` namespace with pandas DataFrame/Series wrappers for all CRUD operations, plus two advanced example scripts, and a minor SDK enhancement for table metadata. Users can now query, create, update, and delete Dataverse records using DataFrame-native inputs and outputs -- no manual dict conversion required. ## Quick Example ```python import pandas as pd from azure.identity import InteractiveBrowserCredential from PowerPlatform.Dataverse.client import DataverseClient credential = InteractiveBrowserCredential() with DataverseClient("https://yourorg.crm.dynamics.com", credential) as client: # Query records as a DataFrame (all pages consolidated automatically) df = client.dataframe.get("account", select=["name", "telephone1"], top=5) # Create records from a DataFrame (returns Series of GUIDs) new_records = pd.DataFrame([ {"name": "Acme Corp", "telephone1": "555-9000"}, {"name": "Globex Inc", "telephone1": "555-9001"}, ]) new_records["accountid"] = client.dataframe.create("account", new_records) # Update records (NaN/None skipped by default; use clear_nulls=True to clear fields) new_records["telephone1"] = ["555-1111", "555-2222"] client.dataframe.update("account", new_records[["accountid", "telephone1"]], id_column="accountid") # Delete records client.dataframe.delete("account", new_records["accountid"]) ``` ## Changes ### DataFrame CRUD (`client.dataframe` namespace) | File | Description | |------|-------------| | `src/.../operations/dataframe.py` | `DataFrameOperations` class: `get()`, `create()`, `update()`, `delete()` | | `src/.../utils/_pandas.py` | `dataframe_to_records()` helper -- normalizes NumPy, datetime, NaN/None | | `client.py` | Added `self.dataframe = DataFrameOperations(self)` | | `pyproject.toml` | Added `pandas>=2.0.0` required dependency | | `README.md` | DataFrame usage examples | | `operations/__init__.py` | Cleanup (`__all__ = []`) | ### SDK Enhancement: TableInfo primary column metadata (fixes #148) | File | Description | |------|-------------| | `src/.../data/_odata.py` | `_get_entity_by_table_schema_name()` and `_get_table_info()` now select `PrimaryNameAttribute` and `PrimaryIdAttribute` from EntityDefinitions | | `src/.../models/table_info.py` | `TableInfo` includes `primary_name_attribute` and `primary_id_attribute` fields | | `tests/unit/models/test_table_info.py` | Tests for new fields in `from_dict`, `from_api_response`, and legacy key access | ### Advanced Examples | File | Description | |------|-------------| | `examples/advanced/dataframe_operations.py` | DataFrame CRUD walkthrough | | `examples/advanced/prodev_quick_start.py` | Pro-dev: 4-table system with relationships, DataFrame CRUD, query/analyze. Uses `result.primary_name_attribute` from `tables.create()` | | `examples/advanced/datascience_risk_assessment.py` | Data science: 5-step risk pipeline with 3 LLM provider options (Azure AI Inference, OpenAI, GitHub Copilot SDK), matplotlib charts | ### Test Files | File | Tests | |------|-------| | `test_dataframe_operations.py` | 44 | | `test_client_dataframe.py` | 26 | | `test_pandas_helpers.py` | 33 | | `test_table_info.py` | +1 (primary fields) | ## API Design | Method | Input | Output | Underlying API | |--------|-------|--------|----------------| | `get(table, ...)` | OData params | `pd.DataFrame` | `records.get()` | | `get(table, record_id=...)` | GUID | 1-row `pd.DataFrame` | `records.get()` | | `create(table, df)` | `pd.DataFrame` | `pd.Series` of GUIDs | `CreateMultiple` | | `update(table, df, id_column)` | `pd.DataFrame` | `None` | `UpdateMultiple` | | `delete(table, ids)` | `pd.Series` | `Optional[str]` | `BulkDelete` | ### Design Decisions - **`clear_nulls`**: Default `False` skips NaN (field unchanged). `True` sends null to clear. - **Type normalization**: np.int64/float64/bool_/ndarray, datetime/date/np.datetime64, pd.Timestamp -- all auto-converted. - **ID validation**: Strip whitespace, report DataFrame index labels in errors. - **pandas required**: Core dependency by team decision. ## Test Results ``` 396 passed, 8 warnings (pre-existing deprecation), 4 subtests passed ``` | Check | Result | |-------|--------| | Full test suite | 396 pass, 0 fail | | mypy | 0 errors | | black / isort | Clean | | E2E prodev | PASS (4 tables, 3 relationships, 13 records, full CRUD cycle) | | E2E datascience | PASS (7 accounts, 3 cases, 8 opportunities, risk scoring, charts) | | PR review threads | 54/54 resolved | ## Issues Addressed - Fixes #148: `tables.create()` now exposes `primary_name_attribute` via Dataverse metadata - Followup #147: `QueryBuilder.to_dataframe()` tracked for future work --------- Co-authored-by: Saurabh Badenkal --- .claude/skills/dataverse-sdk-use/SKILL.md | 41 +- .gitignore | 1 + README.md | 38 + examples/advanced/dataframe_operations.py | 174 ++++ .../advanced/datascience_risk_assessment.py | 764 ++++++++++++++++++ examples/advanced/prodev_quick_start.py | 514 ++++++++++++ pyproject.toml | 1 + .../claude_skill/dataverse-sdk-use/SKILL.md | 41 +- src/PowerPlatform/Dataverse/client.py | 3 + src/PowerPlatform/Dataverse/data/_odata.py | 6 +- .../Dataverse/models/table_info.py | 8 + .../Dataverse/operations/__init__.py | 4 +- .../Dataverse/operations/dataframe.py | 358 ++++++++ src/PowerPlatform/Dataverse/utils/_pandas.py | 60 ++ tests/unit/models/test_table_info.py | 37 +- tests/unit/test_client_dataframe.py | 365 +++++++++ tests/unit/test_dataframe_operations.py | 495 ++++++++++++ tests/unit/test_pandas_helpers.py | 301 +++++++ 18 files changed, 3205 insertions(+), 6 deletions(-) create mode 100644 examples/advanced/dataframe_operations.py create mode 100644 examples/advanced/datascience_risk_assessment.py create mode 100644 examples/advanced/prodev_quick_start.py create mode 100644 src/PowerPlatform/Dataverse/operations/dataframe.py create mode 100644 src/PowerPlatform/Dataverse/utils/_pandas.py create mode 100644 tests/unit/test_client_dataframe.py create mode 100644 tests/unit/test_dataframe_operations.py create mode 100644 tests/unit/test_pandas_helpers.py diff --git a/.claude/skills/dataverse-sdk-use/SKILL.md b/.claude/skills/dataverse-sdk-use/SKILL.md index f351c176..9edb733f 100644 --- a/.claude/skills/dataverse-sdk-use/SKILL.md +++ b/.claude/skills/dataverse-sdk-use/SKILL.md @@ -30,6 +30,9 @@ The SDK supports Dataverse's native bulk operations: Pass lists to `create()`, ` - Control page size with `page_size` parameter - Use `top` parameter to limit total records returned +### DataFrame Support +- DataFrame operations are accessed via the `client.dataframe` namespace: `client.dataframe.get()`, `client.dataframe.create()`, `client.dataframe.update()`, `client.dataframe.delete()` + ## Common Operations ### Import @@ -129,7 +132,7 @@ client.records.update("account", [id1, id2, id3], {"industry": "Technology"}) ``` #### Upsert Records -Creates or updates records identified by alternate keys. Single item → PATCH; multiple items → `UpsertMultiple` bulk action. +Creates or updates records identified by alternate keys. Single item -> PATCH; multiple items -> `UpsertMultiple` bulk action. > **Prerequisite**: The table must have an alternate key configured in Dataverse for the columns used in `alternate_key`. Without it, Dataverse will reject the request with a 400 error. ```python from PowerPlatform.Dataverse.models.upsert import UpsertItem @@ -171,6 +174,42 @@ client.records.delete("account", account_id) client.records.delete("account", [id1, id2, id3], use_bulk_delete=True) ``` +### DataFrame Operations + +The SDK provides DataFrame wrappers for all CRUD operations via the `client.dataframe` namespace, using pandas DataFrames and Series as input/output. + +```python +import pandas as pd + +# Query records -- returns a single DataFrame +df = client.dataframe.get("account", filter="statecode eq 0", select=["name"]) +print(f"Got {len(df)} rows") + +# Limit results with top for large tables +df = client.dataframe.get("account", select=["name"], top=100) + +# Fetch single record as one-row DataFrame +df = client.dataframe.get("account", record_id=account_id, select=["name"]) + +# Create records from a DataFrame (returns a Series of GUIDs) +new_accounts = pd.DataFrame([ + {"name": "Contoso", "telephone1": "555-0100"}, + {"name": "Fabrikam", "telephone1": "555-0200"}, +]) +new_accounts["accountid"] = client.dataframe.create("account", new_accounts) + +# Update records from a DataFrame (id_column identifies the GUID column) +new_accounts["telephone1"] = ["555-0199", "555-0299"] +client.dataframe.update("account", new_accounts, id_column="accountid") + +# Clear a field by setting clear_nulls=True (by default, NaN/None fields are skipped) +df = pd.DataFrame([{"accountid": "guid-1", "websiteurl": None}]) +client.dataframe.update("account", df, id_column="accountid", clear_nulls=True) + +# Delete records by passing a Series of GUIDs +client.dataframe.delete("account", new_accounts["accountid"]) +``` + ### SQL Queries SQL queries are **read-only** and support limited SQL syntax. A single SELECT statement with optional WHERE, TOP (integer literal), ORDER BY (column names only), and a simple table alias after FROM is supported. But JOIN and subqueries may not be. Refer to the Dataverse documentation for the current feature set. diff --git a/.gitignore b/.gitignore index 2c664173..aa762db2 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ Thumbs.db # Claude local settings .claude/*.local.json +.claude/*.local.md diff --git a/README.md b/README.md index aaa8984a..1426f96a 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ A Python client library for Microsoft Dataverse that provides a unified interfac - [Basic CRUD operations](#basic-crud-operations) - [Bulk operations](#bulk-operations) - [Upsert operations](#upsert-operations) + - [DataFrame operations](#dataframe-operations) - [Query data](#query-data) - [Table management](#table-management) - [Relationship management](#relationship-management) @@ -39,6 +40,7 @@ A Python client library for Microsoft Dataverse that provides a unified interfac - **📊 SQL Queries**: Execute read-only SQL queries via the Dataverse Web API `?sql=` parameter - **🏗️ Table Management**: Create, inspect, and delete custom tables and columns programmatically - **🔗 Relationship Management**: Create one-to-many and many-to-many relationships between tables with full metadata control +- **🐼 DataFrame Support**: Pandas wrappers for all CRUD operations, returning DataFrames and Series - **📎 File Operations**: Upload files to Dataverse file columns with automatic chunking for large files - **🔐 Azure Identity**: Built-in authentication using Azure Identity credential providers with comprehensive support - **🛡️ Error Handling**: Structured exception hierarchy with detailed error context and retry guidance @@ -232,6 +234,42 @@ client.records.upsert("account", [ ]) ``` +### DataFrame operations + +The SDK provides pandas wrappers for all CRUD operations via the `client.dataframe` namespace, using DataFrames and Series for input and output. + +```python +import pandas as pd + +# Query records as a single DataFrame +df = client.dataframe.get("account", filter="statecode eq 0", select=["name", "telephone1"]) +print(f"Found {len(df)} accounts") + +# Limit results with top for large tables +df = client.dataframe.get("account", select=["name"], top=100) + +# Fetch a single record as a one-row DataFrame +df = client.dataframe.get("account", record_id=account_id, select=["name"]) + +# Create records from a DataFrame (returns a Series of GUIDs) +new_accounts = pd.DataFrame([ + {"name": "Contoso", "telephone1": "555-0100"}, + {"name": "Fabrikam", "telephone1": "555-0200"}, +]) +new_accounts["accountid"] = client.dataframe.create("account", new_accounts) + +# Update records from a DataFrame (id_column identifies the GUID column) +new_accounts["telephone1"] = ["555-0199", "555-0299"] +client.dataframe.update("account", new_accounts, id_column="accountid") + +# Clear a field by setting clear_nulls=True (by default, NaN/None fields are skipped) +df = pd.DataFrame([{"accountid": new_accounts["accountid"].iloc[0], "websiteurl": None}]) +client.dataframe.update("account", df, id_column="accountid", clear_nulls=True) + +# Delete records by passing a Series of GUIDs +client.dataframe.delete("account", new_accounts["accountid"]) +``` + ### Query data ```python diff --git a/examples/advanced/dataframe_operations.py b/examples/advanced/dataframe_operations.py new file mode 100644 index 00000000..7c0b6010 --- /dev/null +++ b/examples/advanced/dataframe_operations.py @@ -0,0 +1,174 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +PowerPlatform Dataverse Client - DataFrame Operations Walkthrough + +This example demonstrates how to use the pandas DataFrame extension methods +for CRUD operations with Microsoft Dataverse. + +Prerequisites: + pip install PowerPlatform-Dataverse-Client + pip install azure-identity +""" + +import sys +import uuid + +import pandas as pd +from azure.identity import InteractiveBrowserCredential + +from PowerPlatform.Dataverse.client import DataverseClient + + +def main(): + # -- Setup & Authentication ------------------------------------ + base_url = input("Enter Dataverse org URL (e.g. https://yourorg.crm.dynamics.com): ").strip() + if not base_url: + print("[ERR] No URL entered; exiting.") + sys.exit(1) + base_url = base_url.rstrip("/") + + print("[INFO] Authenticating via browser...") + credential = InteractiveBrowserCredential() + + with DataverseClient(base_url, credential) as client: + _run_walkthrough(client) + + +def _run_walkthrough(client): + table = input("Enter table schema name to use [default: account]: ").strip() or "account" + print(f"[INFO] Using table: {table}") + + # Unique tag to isolate test records from existing data + tag = uuid.uuid4().hex[:8] + test_filter = f"contains(name,'{tag}')" + print(f"[INFO] Using tag '{tag}' to identify test records") + + select_cols = ["name", "telephone1", "websiteurl", "lastonholdtime"] + + # -- 1. Create records from a DataFrame ------------------------ + print("\n" + "-" * 60) + print("1. Create records from a DataFrame") + print("-" * 60) + + new_accounts = pd.DataFrame( + [ + { + "name": f"Contoso_{tag}", + "telephone1": "555-0100", + "websiteurl": "https://contoso.com", + "lastonholdtime": pd.Timestamp("2024-06-15 10:30:00"), + }, + {"name": f"Fabrikam_{tag}", "telephone1": "555-0200", "websiteurl": None, "lastonholdtime": None}, + { + "name": f"Northwind_{tag}", + "telephone1": None, + "websiteurl": "https://northwind.com", + "lastonholdtime": pd.Timestamp("2024-12-01 08:00:00"), + }, + ] + ) + print(f" Input DataFrame:\n{new_accounts.to_string(index=False)}\n") + + # create_dataframe returns a Series of GUIDs aligned with the input rows + new_accounts["accountid"] = client.dataframe.create(table, new_accounts) + print(f"[OK] Created {len(new_accounts)} records") + print(f" IDs: {new_accounts['accountid'].tolist()}") + + # -- 2. Query records as a DataFrame ------------------------- + print("\n" + "-" * 60) + print("2. Query records as a DataFrame") + print("-" * 60) + + df_all = client.dataframe.get(table, select=select_cols, filter=test_filter) + print(f"[OK] Got {len(df_all)} records in one DataFrame") + print(f" Columns: {list(df_all.columns)}") + print(f"{df_all.to_string(index=False)}") + + # -- 3. Limit results with top ------------------------------ + print("\n" + "-" * 60) + print("3. Limit results with top") + print("-" * 60) + + df_top2 = client.dataframe.get(table, select=select_cols, filter=test_filter, top=2) + print(f"[OK] Got {len(df_top2)} records with top=2") + print(f"{df_top2.to_string(index=False)}") + + # -- 4. Fetch a single record by ID ---------------------------- + print("\n" + "-" * 60) + print("4. Fetch a single record by ID") + print("-" * 60) + + first_id = new_accounts["accountid"].iloc[0] + print(f" Fetching record {first_id}...") + single = client.dataframe.get(table, record_id=first_id, select=select_cols) + print(f"[OK] Single record DataFrame:\n{single.to_string(index=False)}") + + # -- 5. Update records from a DataFrame ------------------------ + print("\n" + "-" * 60) + print("5. Update records with different values per row") + print("-" * 60) + + new_accounts["telephone1"] = ["555-1100", "555-1200", "555-1300"] + print(f" New telephone numbers: {new_accounts['telephone1'].tolist()}") + client.dataframe.update(table, new_accounts[["accountid", "telephone1"]], id_column="accountid") + print("[OK] Updated 3 records") + + # Verify the updates + verified = client.dataframe.get(table, select=select_cols, filter=test_filter) + print(f" Verified:\n{verified.to_string(index=False)}") + + # -- 6. Broadcast update (same value to all records) ----------- + print("\n" + "-" * 60) + print("6. Broadcast update (same value to all records)") + print("-" * 60) + + broadcast_df = new_accounts[["accountid"]].copy() + broadcast_df["websiteurl"] = "https://updated.example.com" + print(f" Setting websiteurl to 'https://updated.example.com' for all {len(broadcast_df)} records") + client.dataframe.update(table, broadcast_df, id_column="accountid") + print("[OK] Broadcast update complete") + + # Verify all records have the same websiteurl + verified = client.dataframe.get(table, select=select_cols, filter=test_filter) + print(f" Verified:\n{verified.to_string(index=False)}") + + # Default: NaN/None fields are skipped (not overridden on server) + print("\n Updating with NaN values (default: clear_nulls=False, fields should stay unchanged)...") + sparse_df = pd.DataFrame( + [ + {"accountid": new_accounts["accountid"].iloc[0], "telephone1": "555-9999", "websiteurl": None}, + ] + ) + client.dataframe.update(table, sparse_df, id_column="accountid") + verified = client.dataframe.get(table, select=select_cols, filter=test_filter) + print(f" Verified (Contoso telephone1 updated, websiteurl unchanged):\n{verified.to_string(index=False)}") + + # Opt-in: clear_nulls=True sends None as null to clear the field + print("\n Clearing websiteurl for Contoso with clear_nulls=True...") + clear_df = pd.DataFrame([{"accountid": new_accounts["accountid"].iloc[0], "websiteurl": None}]) + client.dataframe.update(table, clear_df, id_column="accountid", clear_nulls=True) + verified = client.dataframe.get(table, select=select_cols, filter=test_filter) + print(f" Verified (Contoso websiteurl should be empty):\n{verified.to_string(index=False)}") + + # -- 7. Delete records by passing a Series of GUIDs ------------ + print("\n" + "-" * 60) + print("7. Delete records by passing a Series of GUIDs") + print("-" * 60) + + print(f" Deleting {len(new_accounts)} records...") + client.dataframe.delete(table, new_accounts["accountid"], use_bulk_delete=False) + print(f"[OK] Deleted {len(new_accounts)} records") + + # Verify deletions - filter for our tagged records should return 0 + remaining = client.dataframe.get(table, select=select_cols, filter=test_filter) + print(f" Verified: {len(remaining)} test records remaining (expected 0)") + + print("\n" + "=" * 60) + print("[OK] DataFrame operations walkthrough complete!") + print("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/examples/advanced/datascience_risk_assessment.py b/examples/advanced/datascience_risk_assessment.py new file mode 100644 index 00000000..338713e4 --- /dev/null +++ b/examples/advanced/datascience_risk_assessment.py @@ -0,0 +1,764 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +PowerPlatform Dataverse Client - Data Science Risk Assessment Pipeline + +End-to-end example: Extract Dataverse data into DataFrames, run statistical +analysis, generate LLM-powered risk summaries, and write results back to +Dataverse -- a realistic data analyst / data scientist workflow. + +Pipeline flow (matches the SDK architecture): + Dataverse SDK --> Pandas DataFrame --> Analysis + LLM --> Write-back & Reports + +Scenario: + A financial services company tracks customer accounts, service cases, and + revenue opportunities in Dataverse. The risk team needs to: + 1) Pull data from multiple tables into DataFrames + 2) Compute risk scores using statistical analysis (pandas/numpy) + 3) Classify and summarize risk using an LLM + 4) Write risk assessments back to Dataverse + 5) Produce a summary report + + Note: This example reads from existing Dataverse tables (account, + incident, opportunity) and does not create or delete any tables. + Step 4 (write-back) is disabled by default -- uncomment it in + run_risk_pipeline() to write risk scores back to account records. + +Prerequisites (required -- included in SDK dependencies): + pip install PowerPlatform-Dataverse-Client + pip install azure-identity + +Additional libraries (optional -- used for visualization and LLM; not part +of the SDK and must be installed separately. Pick ONE LLM provider): + pip install matplotlib # for charts / visualization + pip install azure-ai-inference # Option A: Azure AI Foundry / Azure OpenAI + pip install openai # Option B: OpenAI / Azure OpenAI + pip install github-copilot-sdk # Option C: GitHub Copilot SDK (requires Copilot CLI) +""" + +import sys +import warnings +from pathlib import Path +from textwrap import dedent + +# Suppress MSAL advisory about response_mode (third-party library, not actionable here) +warnings.filterwarnings("ignore", message="response_mode=.*form_post", category=UserWarning) + +import numpy as np +import pandas as pd +from azure.identity import InteractiveBrowserCredential + +from PowerPlatform.Dataverse.client import DataverseClient + +# -- Optional imports (graceful degradation if not installed) ------ + +try: + import matplotlib + + matplotlib.use("Agg") # non-interactive backend (no GUI required) + import matplotlib.pyplot as plt + + HAS_MATPLOTLIB = True +except ImportError: + HAS_MATPLOTLIB = False + + +# ================================================================ +# LLM Provider Configuration +# ================================================================ +# Choose one of three LLM providers. The first available one is used. +# Each provider has its own init function below. +# +# Option A: Azure AI Inference (azure-ai-inference) +# - Works with Azure AI Foundry, Azure OpenAI, and GitHub Models +# - Uses azure.core.credentials (same auth pattern as Dataverse SDK) +# - pip install azure-ai-inference +# +# Option B: OpenAI (openai) +# - Works with OpenAI API and Azure OpenAI +# - pip install openai +# +# Option C: GitHub Copilot SDK (github-copilot-sdk) +# - Uses your existing GitHub Copilot subscription (no separate API key) +# - Requires the Copilot CLI binary and async execution +# - pip install github-copilot-sdk +# ================================================================ + + +def get_llm_client(provider=None, endpoint=None, api_key=None, model="gpt-4o"): + """Create an LLM client using the specified (or first available) provider. + + Returns a callable: llm_complete(system_prompt, user_prompt) -> str + Returns None if no provider is available. + + The returned callable also has a `.log` attribute (list) that records + each call's prompt, response, timing, and provider metadata. + """ + providers = [provider] if provider else ["azure-ai-inference", "openai", "copilot-sdk"] + + for p in providers: + client = _try_init_provider(p, endpoint, api_key, model) + if client is not None: + return client + + return None + + +def _wrap_with_logging(raw_complete, provider_name, model_name): + """Wrap a raw complete function with timing and logging.""" + import time + + log = [] + + def complete(system_prompt, user_prompt): + start = time.time() + response = raw_complete(system_prompt, user_prompt) + elapsed = time.time() - start + log.append( + { + "provider": provider_name, + "model": model_name, + "system_prompt": system_prompt, + "user_prompt": user_prompt, + "response": response, + "elapsed_seconds": round(elapsed, 2), + } + ) + return response + + complete.log = log + complete.provider_name = provider_name + complete.model_name = model_name + return complete + + +def _try_init_provider(name, endpoint, api_key, model): + """Try to initialize a specific LLM provider. Returns callable or None.""" + if name == "azure-ai-inference": + return _init_azure_ai(endpoint, api_key, model) + elif name == "openai": + return _init_openai(endpoint, api_key, model) + elif name == "copilot-sdk": + return _init_copilot_sdk() + return None + + +def _init_azure_ai(endpoint, api_key, model): + """Initialize Azure AI Inference client (Azure AI Foundry / Azure OpenAI).""" + try: + from azure.ai.inference import ChatCompletionsClient + from azure.ai.inference.models import SystemMessage, UserMessage + from azure.core.credentials import AzureKeyCredential + except ImportError: + return None + + if not endpoint or not api_key: + return None + + client = ChatCompletionsClient( + endpoint=endpoint, + credential=AzureKeyCredential(api_key), + ) + + def complete(system_prompt, user_prompt): + response = client.complete( + messages=[ + SystemMessage(content=system_prompt), + UserMessage(content=user_prompt), + ], + max_tokens=150, + temperature=0.3, + ) + return response.choices[0].message.content.strip() + + print("[INFO] LLM provider: Azure AI Inference") + return _wrap_with_logging(complete, "Azure AI Inference", model) + + +def _init_openai(endpoint, api_key, model): + """Initialize OpenAI client (OpenAI API or Azure OpenAI).""" + try: + import openai + except ImportError: + return None + + if not api_key: + return None + + if endpoint: + # Azure OpenAI + client = openai.AzureOpenAI( + azure_endpoint=endpoint, + api_key=api_key, + api_version="2024-02-01", + ) + else: + # OpenAI API + client = openai.OpenAI(api_key=api_key) + + def complete(system_prompt, user_prompt): + response = client.chat.completions.create( + model=model, + messages=[ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, + ], + max_tokens=150, + temperature=0.3, + ) + return response.choices[0].message.content.strip() + + provider_name = "Azure OpenAI" if endpoint else "OpenAI" + print(f"[INFO] LLM provider: {provider_name}") + return _wrap_with_logging(complete, provider_name, model) + + +def _init_copilot_sdk(): + """Initialize GitHub Copilot SDK. + + # Uncomment and configure to use your Copilot subscription as the LLM provider. + # Requires: pip install github-copilot-sdk + # Copilot CLI must be installed. See: https://github.com/github/copilot-sdk + """ + # To enable, install the SDK and uncomment the implementation below. + # from copilot import CopilotClient + # ... (see Copilot SDK docs for session/send_and_wait usage) + return None + + +# ================================================================ +# Configuration +# ================================================================ + +# Tables used in this demo (schema names) +TABLE_ACCOUNTS = "account" +TABLE_CASES = "incident" +TABLE_OPPORTUNITIES = "opportunity" + +# Risk score thresholds +RISK_HIGH = 75 +RISK_MEDIUM = 40 + +# -- Output folder for exports and reports (relative to this script) -- +_SCRIPT_DIR = Path(__file__).resolve().parent +OUTPUT_DIR = _SCRIPT_DIR / "risk_assessment_output" + + +def main(): + """Entry point -- authenticate and run the pipeline.""" + base_url = input("Enter Dataverse org URL (e.g. https://yourorg.crm.dynamics.com): ").strip() + if not base_url: + print("[ERR] No URL entered; exiting.") + sys.exit(1) + base_url = base_url.rstrip("/") + + print("[INFO] Authenticating via browser...") + credential = InteractiveBrowserCredential() + + with DataverseClient(base_url, credential) as client: + run_risk_pipeline(client) + + +# ================================================================ +# Step 1: Extract -- Pull data from Dataverse into DataFrames +# ================================================================ + + +def step1_extract(client): + """Extract accounts, cases, and opportunities from Dataverse.""" + print("\n" + "=" * 60) + print("STEP 1: Extract data from Dataverse") + print("=" * 60) + + # Pull accounts + accounts = client.dataframe.get( + TABLE_ACCOUNTS, + select=["accountid", "name", "revenue", "numberofemployees", "industrycode"], + filter="statecode eq 0", + top=200, + ) + print(f"[OK] Extracted {len(accounts)} active accounts") + + # Pull open cases (service incidents) + cases = client.dataframe.get( + TABLE_CASES, + select=[ + "incidentid", + "_customerid_value", + "title", + "severitycode", + "prioritycode", + "createdon", + ], + filter="statecode eq 0", + top=1000, + ) + print(f"[OK] Extracted {len(cases)} open cases") + + # Pull active opportunities + opportunities = client.dataframe.get( + TABLE_OPPORTUNITIES, + select=[ + "opportunityid", + "_parentaccountid_value", + "name", + "estimatedvalue", + "closeprobability", + "estimatedclosedate", + ], + filter="statecode eq 0", + top=1000, + ) + print(f"[OK] Extracted {len(opportunities)} active opportunities") + + return accounts, cases, opportunities + + +# ================================================================ +# Step 2: Transform & Analyze -- Statistical risk scoring +# ================================================================ + + +def step2_analyze(accounts, cases, opportunities): + """Compute risk scores using pandas statistical operations.""" + print("\n" + "=" * 60) + print("STEP 2: Statistical analysis -- compute risk scores") + print("=" * 60) + + # -- Case severity analysis per account -- + if not cases.empty and "_customerid_value" in cases.columns: + case_stats = ( + cases.groupby("_customerid_value") + .agg( + total_cases=("incidentid", "count"), + high_severity_cases=("severitycode", lambda x: (x == 1).sum()), + avg_priority=("prioritycode", "mean"), + ) + .reset_index() + .rename(columns={"_customerid_value": "accountid"}) + ) + else: + case_stats = pd.DataFrame(columns=["accountid", "total_cases", "high_severity_cases", "avg_priority"]) + + # -- Opportunity pipeline analysis per account -- + if not opportunities.empty and "_parentaccountid_value" in opportunities.columns: + # Precompute weighted value to avoid closure-based aggregation + opportunities = opportunities.copy() + opportunities["_weighted_value"] = ( + pd.to_numeric(opportunities["estimatedvalue"], errors="coerce").fillna(0) + * pd.to_numeric(opportunities["closeprobability"], errors="coerce").fillna(0) + / 100 + ) + opp_stats = ( + opportunities.groupby("_parentaccountid_value") + .agg( + total_opportunities=("opportunityid", "count"), + pipeline_value=("estimatedvalue", "sum"), + avg_close_probability=("closeprobability", "mean"), + weighted_pipeline=("_weighted_value", "sum"), + ) + .reset_index() + .rename(columns={"_parentaccountid_value": "accountid"}) + ) + else: + opp_stats = pd.DataFrame( + columns=[ + "accountid", + "total_opportunities", + "pipeline_value", + "avg_close_probability", + "weighted_pipeline", + ] + ) + + # -- Join everything into a single risk DataFrame -- + risk_df = accounts.merge(case_stats, on="accountid", how="left") + risk_df = risk_df.merge(opp_stats, on="accountid", how="left") + + # Ensure source numeric columns from Dataverse are proper numeric dtypes + for col in ["revenue", "numberofemployees"]: + if col in risk_df.columns: + risk_df[col] = pd.to_numeric(risk_df[col], errors="coerce").fillna(0) + + # Fill NaN for accounts with no cases/opportunities and ensure numeric dtypes + # (Dataverse may return values as object dtype) + for col in ["total_cases", "high_severity_cases"]: + risk_df[col] = pd.to_numeric(risk_df[col], errors="coerce").fillna(0).astype(int) + for col in ["avg_priority", "pipeline_value", "avg_close_probability", "weighted_pipeline"]: + risk_df[col] = pd.to_numeric(risk_df[col], errors="coerce").fillna(0).astype(float) + risk_df["total_opportunities"] = ( + pd.to_numeric(risk_df["total_opportunities"], errors="coerce").fillna(0).astype(int) + ) + + # -- Compute composite risk score (0-100) -- + # Factors: case severity, case volume, low pipeline, low close probability + risk_df["risk_score"] = compute_risk_score(risk_df) + + # -- Classify risk tier -- + risk_df["risk_tier"] = risk_df["risk_score"].apply(classify_risk) + + print(f"[OK] Computed risk scores for {len(risk_df)} accounts") + print(f" High risk: {(risk_df['risk_tier'] == 'High').sum()}") + print(f" Medium risk: {(risk_df['risk_tier'] == 'Medium').sum()}") + print(f" Low risk: {(risk_df['risk_tier'] == 'Low').sum()}") + + # -- Summary statistics -- + print("\n Risk score distribution:") + print(f" Mean: {risk_df['risk_score'].mean():.1f}") + print(f" Median: {risk_df['risk_score'].median():.1f}") + print(f" Std: {risk_df['risk_score'].std():.1f}") + print(f" Min: {risk_df['risk_score'].min():.1f}") + print(f" Max: {risk_df['risk_score'].max():.1f}") + + return risk_df + + +def compute_risk_score(df): + """Compute a 0-100 risk score from multiple factors. + + Higher score = higher risk. Weighted formula: + - 35%: case severity (high-severity cases relative to total) + - 25%: case volume (normalized by percentile) + - 20%: pipeline weakness (inverse of weighted pipeline value) + - 20%: close probability risk (inverse of avg close probability) + """ + scores = pd.Series(0.0, index=df.index) + + # Factor 1: Case severity ratio (35%) + case_total = df["total_cases"].clip(lower=1) + severity_ratio = df["high_severity_cases"] / case_total + scores += severity_ratio * 35 + + # Factor 2: Case volume (25%) -- percentile rank + if df["total_cases"].max() > 0: + case_pctile = df["total_cases"].rank(pct=True) + scores += case_pctile * 25 + else: + scores += 12.5 # neutral if no cases exist + + # Factor 3: Pipeline weakness (20%) -- low pipeline = high risk + max_pipeline = df["weighted_pipeline"].max() + if max_pipeline > 0: + pipeline_strength = df["weighted_pipeline"] / max_pipeline + scores += (1 - pipeline_strength) * 20 + else: + scores += 10 # neutral + + # Factor 4: Close probability risk (20%) + close_risk = (100 - df["avg_close_probability"]) / 100 + scores += close_risk * 20 + + return scores.clip(0, 100).round(1) + + +def classify_risk(score): + """Classify a risk score into High / Medium / Low.""" + if score >= RISK_HIGH: + return "High" + elif score >= RISK_MEDIUM: + return "Medium" + return "Low" + + +# ================================================================ +# Step 3: LLM Summarization -- Generate risk narratives +# ================================================================ + + +def step3_summarize(risk_df, llm_complete=None): + """Generate per-account risk summaries using LLM or template fallback.""" + print("\n" + "=" * 60) + print("STEP 3: Generate risk summaries") + print("=" * 60) + + # Focus on high and medium risk accounts + flagged = risk_df[risk_df["risk_tier"].isin(["High", "Medium"])].copy() + print(f"[INFO] Generating summaries for {len(flagged)} flagged accounts") + + if llm_complete is not None: + summaries = _summarize_with_llm(flagged, llm_complete) + # Export LLM interaction log + if hasattr(llm_complete, "log") and llm_complete.log: + _export_llm_log(llm_complete) + else: + print("[INFO] No LLM provider configured -- using template-based summarization") + summaries = _summarize_with_template(flagged) + + flagged["risk_summary"] = summaries + summary_map = dict(zip(flagged["accountid"], flagged["risk_summary"])) + risk_df["risk_summary"] = risk_df["accountid"].map(summary_map).fillna("Low risk -- no action needed.") + + print(f"[OK] Generated {len(summaries)} risk summaries") + + # Show top 3 highest risk accounts + top_risk = risk_df.nlargest(3, "risk_score") + for _, row in top_risk.iterrows(): + print(f"\n Account: {row.get('name', 'Unknown')}") + print(f" Risk Score: {row['risk_score']} ({row['risk_tier']})") + print(f" Summary: {row['risk_summary'][:120]}...") + + return risk_df + + +def _summarize_with_llm(flagged_df, llm_complete): + """Generate risk narratives using the configured LLM provider.""" + system_prompt = ( + "You are a customer risk analyst at a financial services company. " + "Write exactly 2-3 sentences per account. " + "Sentence 1: State the risk level and primary driver. " + "Sentence 2: Quantify the key metric(s) behind the risk. " + "Sentence 3 (if needed): Recommend one specific action. " + "Use plain business language. Do not use bullet points or markdown." + ) + + summaries = [] + for _, row in flagged_df.iterrows(): + user_prompt = dedent(f"""\ + Summarize the risk for this account: + + Account Name: {row.get("name", "Unknown")} + Risk Score: {row["risk_score"]:.0f}/100 ({row["risk_tier"]} risk) + Open Support Cases: {row["total_cases"]} total, {row["high_severity_cases"]} high-severity + Revenue Pipeline: ${row["pipeline_value"]:,.0f} total, ${row["weighted_pipeline"]:,.0f} probability-weighted + Average Deal Close Probability: {row["avg_close_probability"]:.0f}% + """) + + summary = llm_complete(system_prompt, user_prompt) + summaries.append(summary) + + return summaries + + +def _summarize_with_template(flagged_df): + """Template-based fallback when LLM is not available.""" + summaries = [] + for _, row in flagged_df.iterrows(): + name = row.get("name", "Unknown") + parts = [] + + if row["high_severity_cases"] > 0: + parts.append(f"{row['high_severity_cases']} high-severity cases require immediate attention") + if row["total_cases"] > 5: + parts.append(f"elevated case volume ({row['total_cases']} open)") + + if row["weighted_pipeline"] < 10000: + parts.append("weak revenue pipeline") + if row["avg_close_probability"] < 30: + parts.append(f"low close probability ({row['avg_close_probability']:.0f}%)") + + if not parts: + parts.append("multiple moderate risk factors detected") + + risk_factors = "; ".join(parts) + summary = ( + f"{name} has a {row['risk_tier'].lower()} risk score of " + f"{row['risk_score']:.0f}/100. Key factors: {risk_factors}. " + f"Recommend proactive outreach and account review." + ) + summaries.append(summary) + + return summaries + + +def _export_llm_log(llm_complete, include_prompts=False): + """Export LLM interaction log (timing, provider metadata) to a text file. + + By default, prompt and response content is not included to avoid logging + sensitive data (PII, customer data). Set include_prompts=True to include + full content for debugging. + """ + log_path = OUTPUT_DIR / "llm_interactions.txt" + with open(log_path, "w", encoding="utf-8") as f: + f.write("LLM Interaction Log\n") + f.write("=" * 70 + "\n") + f.write(f"Provider: {llm_complete.provider_name}\n") + f.write(f"Model: {llm_complete.model_name}\n") + f.write(f"Total calls: {len(llm_complete.log)}\n") + total_time = sum(entry["elapsed_seconds"] for entry in llm_complete.log) + f.write(f"Total time: {total_time:.1f}s\n") + f.write("=" * 70 + "\n\n") + + for i, entry in enumerate(llm_complete.log, 1): + f.write(f"--- Call {i} ({entry['elapsed_seconds']:.2f}s) ---\n\n") + if include_prompts: + f.write(f"[System Prompt]\n{entry['system_prompt']}\n\n") + f.write(f"[User Prompt]\n{entry['user_prompt']}\n\n") + f.write(f"[Response]\n{entry['response']}\n\n") + else: + f.write(f"[Response length: {len(entry['response'])} chars]\n\n") + + print(f"[OK] LLM interaction log saved to {log_path}") + + +# ================================================================ +# Step 4: Write-back -- Store results in Dataverse +# ================================================================ + + +def step4_writeback(client, risk_df): + """Write risk scores and summaries back to Dataverse accounts.""" + print("\n" + "=" * 60) + print("STEP 4: Write risk assessments back to Dataverse") + print("=" * 60) + + # Update accounts with risk data + # Note: These columns must exist on the account table in your environment. + # In a real deployment, you would create custom columns like: + # new_riskscore (Whole Number), new_risktier (Text), new_risksummary (Multi-line Text) + update_df = risk_df[["accountid", "description"]].copy() + update_df["description"] = risk_df.apply( + lambda r: f"[Risk: {r['risk_tier']} ({r['risk_score']:.0f}/100)] {r['risk_summary']}", + axis=1, + ) + + client.dataframe.update(TABLE_ACCOUNTS, update_df, id_column="accountid") + print(f"[OK] Updated {len(update_df)} account records with risk assessments") + + +# ================================================================ +# Step 5: Report -- Produce summary output +# ================================================================ + + +def step5_report(risk_df): + """Generate a summary report with optional visualization.""" + print("\n" + "=" * 60) + print("STEP 5: Risk assessment report") + print("=" * 60) + + # -- Tabular summary -- + tier_summary = ( + risk_df.groupby("risk_tier") + .agg( + count=("accountid", "count"), + avg_score=("risk_score", "mean"), + total_cases=("total_cases", "sum"), + total_pipeline=("pipeline_value", "sum"), + ) + .round(1) + ) + print("\nRisk Tier Summary:") + print(tier_summary.to_string()) + + # -- Top 10 highest risk accounts -- + top10 = risk_df.nlargest(10, "risk_score")[ + ["name", "risk_score", "risk_tier", "total_cases", "high_severity_cases", "pipeline_value"] + ] + print("\nTop 10 Highest Risk Accounts:") + print(top10.to_string(index=False)) + + # -- Visualization (optional) -- + if HAS_MATPLOTLIB: + _generate_charts(risk_df) + else: + print("\n[INFO] Install matplotlib for risk visualization charts") + + # -- Export data to CSV -- + risk_df.to_csv(OUTPUT_DIR / "risk_scores.csv", index=False) + top10.to_csv(OUTPUT_DIR / "top10_risk.csv", index=False) + tier_summary.to_csv(OUTPUT_DIR / "tier_summary.csv") + print(f"\n[OK] Exported CSV reports to {OUTPUT_DIR}/") + + print("\n[OK] Risk assessment pipeline complete!") + + +def _generate_charts(risk_df): + """Generate risk distribution charts.""" + fig, axes = plt.subplots(1, 3, figsize=(16, 5)) + fig.suptitle("Customer Account Risk Assessment", fontsize=14, fontweight="bold") + + # Chart 1: Risk score distribution + axes[0].hist(risk_df["risk_score"], bins=20, color="#4472C4", edgecolor="white") + axes[0].axvline(RISK_HIGH, color="red", linestyle="--", label=f"High ({RISK_HIGH})") + axes[0].axvline(RISK_MEDIUM, color="orange", linestyle="--", label=f"Medium ({RISK_MEDIUM})") + axes[0].set_title("Risk Score Distribution") + axes[0].set_xlabel("Risk Score") + axes[0].set_ylabel("Number of Accounts") + axes[0].legend() + + # Chart 2: Risk tier breakdown (pie) + tier_counts = risk_df["risk_tier"].value_counts() + colors = {"High": "#FF4444", "Medium": "#FFA500", "Low": "#44BB44"} + axes[1].pie( + tier_counts.values, + labels=tier_counts.index, + colors=[colors.get(t, "#888") for t in tier_counts.index], + autopct="%1.0f%%", + startangle=90, + ) + axes[1].set_title("Risk Tier Breakdown") + + # Chart 3: Cases vs Pipeline scatter + axes[2].scatter( + risk_df["total_cases"], + risk_df["pipeline_value"], + c=risk_df["risk_score"], + cmap="RdYlGn_r", + alpha=0.7, + edgecolors="gray", + s=60, + ) + axes[2].set_title("Cases vs Pipeline (color = risk)") + axes[2].set_xlabel("Open Cases") + axes[2].set_ylabel("Pipeline Value ($)") + + plt.tight_layout() + chart_path = OUTPUT_DIR / "risk_assessment_report.png" + plt.savefig(chart_path, dpi=150, bbox_inches="tight") + print(f"[OK] Saved {chart_path}") + # plt.show() # Uncomment for interactive display + + +# ================================================================ +# Pipeline Orchestrator +# ================================================================ + + +def run_risk_pipeline(client): + """Run the full risk assessment pipeline.""" + OUTPUT_DIR.mkdir(exist_ok=True) + print(f"[INFO] Output folder: {OUTPUT_DIR.resolve()}") + + print("\n" + "#" * 60) + print(" CUSTOMER RISK ASSESSMENT PIPELINE") + print(" Dataverse SDK -> Pandas -> Analysis -> LLM -> Write-back") + print("#" * 60) + + # Step 1: Extract data from Dataverse into DataFrames + accounts, cases, opportunities = step1_extract(client) + + if accounts.empty: + print("[WARN] No accounts found -- nothing to analyze.") + return + + # Step 2: Statistical analysis and risk scoring + risk_df = step2_analyze(accounts, cases, opportunities) + + # Step 3: LLM-powered risk summarization + # Configure your LLM provider (uncomment one): + # Option A: Azure AI Inference + # llm = get_llm_client("azure-ai-inference", endpoint="https://...", api_key="...") + # Option B: OpenAI + # llm = get_llm_client("openai", api_key="sk-...") + # Option C: Azure OpenAI (via openai package) + # llm = get_llm_client("openai", endpoint="https://...", api_key="...") + # Option D: GitHub Copilot SDK (no API key -- uses your Copilot subscription) + # llm = get_llm_client("copilot-sdk") + # Auto-detect (tries all providers in order): + # llm = get_llm_client(endpoint="https://...", api_key="...") + llm = None # Set to get_llm_client(...) to enable LLM summarization + risk_df = step3_summarize(risk_df, llm_complete=llm) + + # Step 4: Write results back to Dataverse + # Uncomment the next line to write back (requires custom columns on account table) + # step4_writeback(client, risk_df) + print("\n[INFO] Step 4 (write-back) is commented out by default.") + print(" Uncomment step4_writeback() after adding custom columns to account table.") + + # Step 5: Generate summary report + charts + step5_report(risk_df) + + +if __name__ == "__main__": + main() diff --git a/examples/advanced/prodev_quick_start.py b/examples/advanced/prodev_quick_start.py new file mode 100644 index 00000000..e28d1575 --- /dev/null +++ b/examples/advanced/prodev_quick_start.py @@ -0,0 +1,514 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +PowerPlatform Dataverse Client - Pro-Dev Quick Start + +A developer-focused example that demonstrates the full SDK lifecycle: +install, authenticate, create a system with 4 related tables, populate +data, query it, and clean up -- all in a single script. + +What this example covers: + 1) SDK installation and authentication + 2) Create 4 custom tables (Customer, Project, Task, TimeEntry) + 3) Create columns and relationships between tables + 4) Populate with sample data using DataFrame CRUD + 5) Query and join data across tables + 6) Clean up (delete tables) + + Note: The last step (cleanup) automatically deletes all demo tables. + Comment out the cleanup() call in run_demo() if you want to keep the + tables in your environment for inspection. + +Why pandas DataFrames? + This example uses client.dataframe (pandas) instead of raw dict/list CRUD + because DataFrames provide significant advantages for multi-record operations: + + - Batch operations are natural: create 100 records from a DataFrame in one + call vs. looping over 100 dicts + - Column operations (broadcast a value, compute derived fields) are one-liners + instead of for-loops + - Joins and aggregations across tables use pandas merge/groupby -- far more + readable than manual dict matching + - NaN/None handling is built in (clear_nulls flag controls whether missing + values clear server fields or are skipped) + - NumPy type normalization is automatic (int64, float64, Timestamps all + serialize to JSON correctly without manual conversion) + + The SDK also supports plain dict/list CRUD via client.records for single-record + operations or when pandas is not needed. Both approaches use the same underlying + Dataverse Web API calls. + +Prerequisites: + pip install PowerPlatform-Dataverse-Client + pip install azure-identity +""" + +import sys +import uuid +import warnings +from pathlib import Path + +# Suppress MSAL advisory about response_mode (third-party library, not actionable here) +warnings.filterwarnings("ignore", message="response_mode=.*form_post", category=UserWarning) + +import pandas as pd +from azure.identity import InteractiveBrowserCredential + +from PowerPlatform.Dataverse.client import DataverseClient + +# -- Table schema names -- +# Uses the standard 'new_' publisher prefix (default Dataverse publisher). +# A unique suffix avoids collisions with existing tables. +SUFFIX = uuid.uuid4().hex[:6] +TABLE_CUSTOMER = f"new_DemoCustomer{SUFFIX}" +TABLE_PROJECT = f"new_DemoProject{SUFFIX}" +TABLE_TASK = f"new_DemoTask{SUFFIX}" +TABLE_TIMEENTRY = f"new_DemoTimeEntry{SUFFIX}" + +# -- Output folder for exported data (relative to this script) -- +_SCRIPT_DIR = Path(__file__).resolve().parent +OUTPUT_DIR = _SCRIPT_DIR / "prodev_output" + + +def main(): + """Entry point.""" + print("=" * 60) + print(" DATAVERSE PYTHON SDK -- PRO-DEV QUICK START") + print("=" * 60) + print() + print(" Step 0: pip install PowerPlatform-Dataverse-Client") + print(" (already done if you're running this script)") + print() + + base_url = input("Enter Dataverse org URL (e.g. https://yourorg.crm.dynamics.com): ").strip() + if not base_url: + print("[ERR] No URL entered; exiting.") + sys.exit(1) + base_url = base_url.rstrip("/") + + print("[INFO] Authenticating via browser (Azure Identity)...") + credential = InteractiveBrowserCredential() + + with DataverseClient(base_url, credential) as client: + try: + run_demo(client) + except Exception as e: + print(f"\n[ERR] {e}") + print("[INFO] Attempting cleanup...") + cleanup(client) + raise + + +def run_demo(client): + """Run the full pro-dev demo pipeline.""" + OUTPUT_DIR.mkdir(exist_ok=True) + print(f"[INFO] Output folder: {OUTPUT_DIR.resolve()}") + + # -- Step 1: Create 4 tables -- + primary_name_col, primary_id_col = step1_create_tables(client) + + # -- Step 2: Create relationships -- + step2_create_relationships(client) + + # -- Step 3: Populate with sample data -- + customer_ids, project_ids, task_ids = step3_populate_data(client, primary_name_col) + + # -- Step 4: Query and analyze -- + step4_query_and_analyze(client, customer_ids, primary_name_col) + + # -- Step 5: Update and delete -- + step5_update_and_delete(client, task_ids, primary_name_col, primary_id_col) + + # -- Step 6: Cleanup (optional -- prompts before deleting) -- + do_cleanup = input("\n6. Delete demo tables and cleanup? (Y/n): ").strip() or "y" + if do_cleanup.lower() in ("y", "yes"): + cleanup(client) + else: + print("[INFO] Tables kept for inspection.") + + print("\n" + "=" * 60) + print("[OK] Pro-dev quick start demo complete!") + print("=" * 60) + + +# ================================================================ +# Step 1: Create tables +# ================================================================ + + +def step1_create_tables(client): + """Create 4 custom tables.""" + print("\n" + "-" * 60) + print("STEP 1: Create 4 custom tables") + print("-" * 60) + + # Customer table + result = client.tables.create( + TABLE_CUSTOMER, + { + f"{TABLE_CUSTOMER}_Email": "string", + f"{TABLE_CUSTOMER}_Industry": "string", + f"{TABLE_CUSTOMER}_Revenue": "money", + }, + ) + # The primary column logical names are returned by tables.create() + # so we know exactly what keys to use in payloads and queries. + primary_name_col = result.primary_name_attribute + primary_id_col = result.primary_id_attribute + print(f"[OK] Created table: {TABLE_CUSTOMER} (name: {primary_name_col}, id: {primary_id_col})") + + # Project table + client.tables.create( + TABLE_PROJECT, + { + f"{TABLE_PROJECT}_Budget": "money", + f"{TABLE_PROJECT}_Status": "string", + f"{TABLE_PROJECT}_StartDate": "datetime", + }, + ) + print(f"[OK] Created table: {TABLE_PROJECT}") + + # Task table + client.tables.create( + TABLE_TASK, + { + f"{TABLE_TASK}_Priority": "integer", + f"{TABLE_TASK}_Status": "string", + f"{TABLE_TASK}_EstimatedHours": "decimal", + }, + ) + print(f"[OK] Created table: {TABLE_TASK}") + + # TimeEntry table + client.tables.create( + TABLE_TIMEENTRY, + { + f"{TABLE_TIMEENTRY}_Hours": "decimal", + f"{TABLE_TIMEENTRY}_Date": "datetime", + f"{TABLE_TIMEENTRY}_Description": "string", + }, + ) + print(f"[OK] Created table: {TABLE_TIMEENTRY}") + print(f"[OK] All 4 tables created (suffix: {SUFFIX})") + print(f"[INFO] Primary name column: '{primary_name_col}', ID column: '{primary_id_col}'") + + return primary_name_col, primary_id_col + + +# ================================================================ +# Step 2: Create relationships +# ================================================================ + + +def step2_create_relationships(client): + """Create relationships between the 4 tables using lookup fields.""" + print("\n" + "-" * 60) + print("STEP 2: Create relationships (lookup fields)") + print("-" * 60) + + # Customer 1:N Project (lookup on Project pointing to Customer) + client.tables.create_lookup_field( + referencing_table=TABLE_PROJECT.lower(), + lookup_field_name=f"{TABLE_PROJECT}_CustomerId", + referenced_table=TABLE_CUSTOMER.lower(), + display_name="Customer", + ) + print(f"[OK] {TABLE_CUSTOMER} 1:N {TABLE_PROJECT}") + + # Project 1:N Task (lookup on Task pointing to Project) + client.tables.create_lookup_field( + referencing_table=TABLE_TASK.lower(), + lookup_field_name=f"{TABLE_TASK}_ProjectId", + referenced_table=TABLE_PROJECT.lower(), + display_name="Project", + ) + print(f"[OK] {TABLE_PROJECT} 1:N {TABLE_TASK}") + + # Task 1:N TimeEntry (lookup on TimeEntry pointing to Task) + client.tables.create_lookup_field( + referencing_table=TABLE_TIMEENTRY.lower(), + lookup_field_name=f"{TABLE_TIMEENTRY}_TaskId", + referenced_table=TABLE_TASK.lower(), + display_name="Task", + ) + print(f"[OK] {TABLE_TASK} 1:N {TABLE_TIMEENTRY}") + + print("[OK] 3 lookup relationships created (Customer -> Project -> Task -> TimeEntry)") + + +# ================================================================ +# Step 3: Populate with sample data using DataFrame CRUD +# ================================================================ + + +def step3_populate_data(client, primary_name_col): + """Create sample records using client.dataframe.create(). + + Why DataFrames here instead of client.records.create()? + + With client.records (dict/list): + ids = client.records.create("Customer", [ + {"name": "Contoso", "Email": "info@contoso.com", ...}, + {"name": "Fabrikam", "Email": "contact@fabrikam.com", ...}, + ]) + # ids is a plain list -- manual index tracking needed + + With client.dataframe (pandas): + df = pd.DataFrame([{"name": "Contoso", ...}, {"name": "Fabrikam", ...}]) + df["id"] = client.dataframe.create("Customer", df) + # IDs auto-aligned to rows -- use df["id"].iloc[0] to reference later + + The DataFrame approach is more natural when you need to: + - Reference created IDs for relationship binding (as we do here) + - Compute derived columns before writing + - Join/merge data across multiple tables for analysis + """ + print("\n" + "-" * 60) + print("STEP 3: Populate with sample data (DataFrame CRUD)") + print("-" * 60) + + # -- Customers -- + # Use the primary name column returned by tables.create() + name_col = primary_name_col + customers_df = pd.DataFrame( + [ + { + name_col: "Contoso Ltd", + f"{TABLE_CUSTOMER}_Email": "info@contoso.com", + f"{TABLE_CUSTOMER}_Industry": "Technology", + f"{TABLE_CUSTOMER}_Revenue": 5000000, + }, + { + name_col: "Fabrikam Inc", + f"{TABLE_CUSTOMER}_Email": "contact@fabrikam.com", + f"{TABLE_CUSTOMER}_Industry": "Manufacturing", + f"{TABLE_CUSTOMER}_Revenue": 12000000, + }, + { + name_col: "Northwind Traders", + f"{TABLE_CUSTOMER}_Email": "sales@northwind.com", + f"{TABLE_CUSTOMER}_Industry": "Retail", + f"{TABLE_CUSTOMER}_Revenue": 3000000, + }, + ] + ) + customer_ids = client.dataframe.create(TABLE_CUSTOMER, customers_df) + customers_df["id"] = customer_ids + print(f"[OK] Created {len(customers_df)} customers") + + # -- Projects (linked to customers via lookup) -- + # @odata.bind keys use the navigation property logical name (lowercase) + # and the entity set name (also lowercase) in the value. + customer_lookup = f"{TABLE_PROJECT}_CustomerId".lower() + "@odata.bind" + customer_set = TABLE_CUSTOMER.lower() + "s" + projects_df = pd.DataFrame( + [ + { + name_col: "Cloud Migration", + f"{TABLE_PROJECT}_Budget": 250000, + f"{TABLE_PROJECT}_Status": "Active", + f"{TABLE_PROJECT}_StartDate": pd.Timestamp("2026-01-15"), + customer_lookup: f"/{customer_set}({customer_ids.iloc[0]})", + }, + { + name_col: "ERP Upgrade", + f"{TABLE_PROJECT}_Budget": 500000, + f"{TABLE_PROJECT}_Status": "Active", + f"{TABLE_PROJECT}_StartDate": pd.Timestamp("2026-02-01"), + customer_lookup: f"/{customer_set}({customer_ids.iloc[1]})", + }, + { + name_col: "POS Modernization", + f"{TABLE_PROJECT}_Budget": 150000, + f"{TABLE_PROJECT}_Status": "Planning", + f"{TABLE_PROJECT}_StartDate": pd.Timestamp("2026-03-01"), + customer_lookup: f"/{customer_set}({customer_ids.iloc[2]})", + }, + { + name_col: "Data Analytics Platform", + f"{TABLE_PROJECT}_Budget": 180000, + f"{TABLE_PROJECT}_Status": "Active", + f"{TABLE_PROJECT}_StartDate": pd.Timestamp("2026-01-20"), + customer_lookup: f"/{customer_set}({customer_ids.iloc[0]})", + }, + ] + ) + project_ids = client.dataframe.create(TABLE_PROJECT, projects_df) + projects_df["id"] = project_ids + print(f"[OK] Created {len(projects_df)} projects across 3 customers") + + # -- Tasks (linked to projects) -- + tasks_data = [] + task_names = [ + ("Infrastructure Setup", 1, "In Progress", 40), + ("Data Assessment", 2, "Not Started", 20), + ("Testing & QA", 1, "Not Started", 60), + ("Requirements Gathering", 1, "Complete", 30), + ("Development Sprint 1", 1, "In Progress", 80), + ("User Training", 3, "Not Started", 16), + ] + project_assignment = [0, 0, 0, 1, 1, 2] # which project each task belongs to + + for i, (task_name, priority, status, hours) in enumerate(task_names): + proj_idx = project_assignment[i] + project_lookup = f"{TABLE_TASK}_ProjectId".lower() + "@odata.bind" + project_set = TABLE_PROJECT.lower() + "s" + tasks_data.append( + { + name_col: task_name, + f"{TABLE_TASK}_Priority": priority, + f"{TABLE_TASK}_Status": status, + f"{TABLE_TASK}_EstimatedHours": hours, + project_lookup: f"/{project_set}({project_ids.iloc[proj_idx]})", + } + ) + + tasks_df = pd.DataFrame(tasks_data) + task_ids = client.dataframe.create(TABLE_TASK, tasks_df) + tasks_df["id"] = task_ids + print(f"[OK] Created {len(tasks_df)} tasks across 4 projects") + + print( + f"\n Total records: {len(customers_df) + len(projects_df) + len(tasks_df)} " + f"({len(customers_df)} customers, {len(projects_df)} projects, {len(tasks_df)} tasks)" + ) + + return customer_ids, project_ids, task_ids + + +# ================================================================ +# Step 4: Query and analyze data +# ================================================================ + + +def step4_query_and_analyze(client, customer_ids, primary_name_col): + """Query data and demonstrate DataFrame analysis.""" + print("\n" + "-" * 60) + print("STEP 4: Query and analyze data") + print("-" * 60) + + # Query all projects as a DataFrame + # Note: The SDK lowercases $select values automatically, so schema-name + # casing (e.g., new_DemoProject_Budget) works -- it becomes the logical name. + name_attr = primary_name_col + projects = client.dataframe.get( + TABLE_PROJECT, + select=[ + name_attr, + f"{TABLE_PROJECT}_Budget", + f"{TABLE_PROJECT}_Status", + ], + ) + print(f"\n All projects ({len(projects)} rows):") + print(f"{projects.to_string(index=False)}") + + # Query tasks and analyze + tasks = client.dataframe.get( + TABLE_TASK, + select=[ + name_attr, + f"{TABLE_TASK}_Priority", + f"{TABLE_TASK}_Status", + f"{TABLE_TASK}_EstimatedHours", + ], + ) + print(f"\n All tasks ({len(tasks)} rows):") + print(f"{tasks.to_string(index=False)}") + + # -- DataFrame analysis -- + hours_col = f"{TABLE_TASK}_EstimatedHours" + status_col = f"{TABLE_TASK}_Status" + budget_col = f"{TABLE_PROJECT}_Budget" + + if hours_col in tasks.columns: + print(f"\n Task hours summary:") + print(f" Total estimated hours: {tasks[hours_col].sum():.0f}") + print(f" Average per task: {tasks[hours_col].mean():.1f}") + print(f" Max single task: {tasks[hours_col].max():.0f}") + + if status_col in tasks.columns: + print(f"\n Tasks by status:") + status_counts = tasks[status_col].value_counts() + for status, count in status_counts.items(): + print(f" {status}: {count}") + + if budget_col in projects.columns: + print(f"\n Project budget summary:") + print(f" Total budget: ${projects[budget_col].sum():,.0f}") + print(f" Average budget: ${projects[budget_col].mean():,.0f}") + + # Fetch single record by ID + first_id = customer_ids.iloc[0] + single = client.dataframe.get(TABLE_CUSTOMER, record_id=first_id) + print(f"\n Single customer record (by ID):") + print(f"{single.to_string(index=False)}") + + # -- Export query results to CSV -- + projects.to_csv(OUTPUT_DIR / "projects.csv", index=False) + tasks.to_csv(OUTPUT_DIR / "tasks.csv", index=False) + single.to_csv(OUTPUT_DIR / "single_customer.csv", index=False) + print(f"\n[OK] Exported query results to {OUTPUT_DIR}/") + + +# ================================================================ +# Step 5: Update and delete records +# ================================================================ + + +def step5_update_and_delete(client, task_ids, primary_name_col, primary_id_col): + """Demonstrate update and delete with DataFrames.""" + print("\n" + "-" * 60) + print("STEP 5: Update and delete records") + print("-" * 60) + + status_col = f"{TABLE_TASK}_Status" + + # Update: mark first two tasks as "Complete" + # Use primary_id_col (from tables.create metadata) as the ID column name + update_df = pd.DataFrame( + { + primary_id_col: [task_ids.iloc[0], task_ids.iloc[1]], + status_col: ["Complete", "Complete"], + } + ) + client.dataframe.update(TABLE_TASK, update_df, id_column=primary_id_col) + print(f"[OK] Updated 2 tasks to 'Complete'") + + # Delete: remove the last task + delete_ids = pd.Series([task_ids.iloc[-1]]) + client.dataframe.delete(TABLE_TASK, delete_ids) + print(f"[OK] Deleted 1 task") + + # Verify + remaining = client.dataframe.get( + TABLE_TASK, + select=[primary_name_col, status_col], + ) + print(f"\n Remaining tasks ({len(remaining)}):") + print(f"{remaining.to_string(index=False)}") + + +# ================================================================ +# Cleanup +# ================================================================ + + +def cleanup(client): + """Delete all demo tables.""" + print("\n" + "-" * 60) + print("CLEANUP: Removing demo tables") + print("-" * 60) + + for table in [TABLE_TIMEENTRY, TABLE_TASK, TABLE_PROJECT, TABLE_CUSTOMER]: + try: + client.tables.delete(table) + print(f"[OK] Deleted table: {table}") + except Exception as e: + print(f"[WARN] Could not delete {table}: {e}") + + print("[OK] Cleanup complete") + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index 3efdf8f3..5c06f5ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ dependencies = [ "azure-identity>=1.17.0", "azure-core>=1.30.2", "requests>=2.32.0", + "pandas>=2.0.0", ] [project.urls] diff --git a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md index f351c176..9edb733f 100644 --- a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md +++ b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md @@ -30,6 +30,9 @@ The SDK supports Dataverse's native bulk operations: Pass lists to `create()`, ` - Control page size with `page_size` parameter - Use `top` parameter to limit total records returned +### DataFrame Support +- DataFrame operations are accessed via the `client.dataframe` namespace: `client.dataframe.get()`, `client.dataframe.create()`, `client.dataframe.update()`, `client.dataframe.delete()` + ## Common Operations ### Import @@ -129,7 +132,7 @@ client.records.update("account", [id1, id2, id3], {"industry": "Technology"}) ``` #### Upsert Records -Creates or updates records identified by alternate keys. Single item → PATCH; multiple items → `UpsertMultiple` bulk action. +Creates or updates records identified by alternate keys. Single item -> PATCH; multiple items -> `UpsertMultiple` bulk action. > **Prerequisite**: The table must have an alternate key configured in Dataverse for the columns used in `alternate_key`. Without it, Dataverse will reject the request with a 400 error. ```python from PowerPlatform.Dataverse.models.upsert import UpsertItem @@ -171,6 +174,42 @@ client.records.delete("account", account_id) client.records.delete("account", [id1, id2, id3], use_bulk_delete=True) ``` +### DataFrame Operations + +The SDK provides DataFrame wrappers for all CRUD operations via the `client.dataframe` namespace, using pandas DataFrames and Series as input/output. + +```python +import pandas as pd + +# Query records -- returns a single DataFrame +df = client.dataframe.get("account", filter="statecode eq 0", select=["name"]) +print(f"Got {len(df)} rows") + +# Limit results with top for large tables +df = client.dataframe.get("account", select=["name"], top=100) + +# Fetch single record as one-row DataFrame +df = client.dataframe.get("account", record_id=account_id, select=["name"]) + +# Create records from a DataFrame (returns a Series of GUIDs) +new_accounts = pd.DataFrame([ + {"name": "Contoso", "telephone1": "555-0100"}, + {"name": "Fabrikam", "telephone1": "555-0200"}, +]) +new_accounts["accountid"] = client.dataframe.create("account", new_accounts) + +# Update records from a DataFrame (id_column identifies the GUID column) +new_accounts["telephone1"] = ["555-0199", "555-0299"] +client.dataframe.update("account", new_accounts, id_column="accountid") + +# Clear a field by setting clear_nulls=True (by default, NaN/None fields are skipped) +df = pd.DataFrame([{"accountid": "guid-1", "websiteurl": None}]) +client.dataframe.update("account", df, id_column="accountid", clear_nulls=True) + +# Delete records by passing a Series of GUIDs +client.dataframe.delete("account", new_accounts["accountid"]) +``` + ### SQL Queries SQL queries are **read-only** and support limited SQL syntax. A single SELECT statement with optional WHERE, TOP (integer literal), ORDER BY (column names only), and a simple table alias after FROM is supported. But JOIN and subqueries may not be. Refer to the Dataverse documentation for the current feature set. diff --git a/src/PowerPlatform/Dataverse/client.py b/src/PowerPlatform/Dataverse/client.py index 38fefd39..402be881 100644 --- a/src/PowerPlatform/Dataverse/client.py +++ b/src/PowerPlatform/Dataverse/client.py @@ -14,6 +14,7 @@ from .core._auth import _AuthManager from .core.config import DataverseConfig from .data._odata import _ODataClient +from .operations.dataframe import DataFrameOperations from .operations.records import RecordOperations from .operations.query import QueryOperations from .operations.files import FileOperations @@ -60,6 +61,7 @@ class DataverseClient: - ``client.query`` -- query and search operations - ``client.tables`` -- table and column metadata management - ``client.files`` -- file upload operations + - ``client.dataframe`` -- pandas DataFrame wrappers for record CRUD The client supports Python's context manager protocol for automatic resource cleanup and HTTP connection pooling: @@ -106,6 +108,7 @@ def __init__( self.query = QueryOperations(self) self.tables = TableOperations(self) self.files = FileOperations(self) + self.dataframe = DataFrameOperations(self) def _get_odata(self) -> _ODataClient: """ diff --git a/src/PowerPlatform/Dataverse/data/_odata.py b/src/PowerPlatform/Dataverse/data/_odata.py index d05cd424..633c736f 100644 --- a/src/PowerPlatform/Dataverse/data/_odata.py +++ b/src/PowerPlatform/Dataverse/data/_odata.py @@ -960,7 +960,7 @@ def _get_entity_by_table_schema_name( logical_lower = table_schema_name.lower() logical_escaped = self._escape_odata_quotes(logical_lower) params = { - "$select": "MetadataId,LogicalName,SchemaName,EntitySetName", + "$select": "MetadataId,LogicalName,SchemaName,EntitySetName,PrimaryNameAttribute,PrimaryIdAttribute", "$filter": f"LogicalName eq '{logical_escaped}'", } r = self._request("get", url, params=params, headers=headers) @@ -1445,6 +1445,8 @@ def _get_table_info(self, table_schema_name: str) -> Optional[Dict[str, Any]]: "table_logical_name": ent.get("LogicalName"), "entity_set_name": ent.get("EntitySetName"), "metadata_id": ent.get("MetadataId"), + "primary_name_attribute": ent.get("PrimaryNameAttribute"), + "primary_id_attribute": ent.get("PrimaryIdAttribute"), "columns_created": [], } @@ -1689,6 +1691,8 @@ def _create_table( "table_logical_name": metadata.get("LogicalName"), "entity_set_name": metadata.get("EntitySetName"), "metadata_id": metadata.get("MetadataId"), + "primary_name_attribute": metadata.get("PrimaryNameAttribute"), + "primary_id_attribute": metadata.get("PrimaryIdAttribute"), "columns_created": created_cols, } diff --git a/src/PowerPlatform/Dataverse/models/table_info.py b/src/PowerPlatform/Dataverse/models/table_info.py index 34b1d383..842fe2b6 100644 --- a/src/PowerPlatform/Dataverse/models/table_info.py +++ b/src/PowerPlatform/Dataverse/models/table_info.py @@ -112,6 +112,8 @@ class TableInfo: logical_name: str = "" entity_set_name: str = "" metadata_id: str = "" + primary_name_attribute: Optional[str] = None + primary_id_attribute: Optional[str] = None display_name: Optional[str] = None description: Optional[str] = None columns: Optional[List[ColumnInfo]] = field(default=None, repr=False) @@ -123,6 +125,8 @@ class TableInfo: "table_logical_name": "logical_name", "entity_set_name": "entity_set_name", "metadata_id": "metadata_id", + "primary_name_attribute": "primary_name_attribute", + "primary_id_attribute": "primary_id_attribute", "columns_created": "columns_created", } @@ -187,6 +191,8 @@ def from_dict(cls, data: Dict[str, Any]) -> TableInfo: logical_name=data.get("table_logical_name", ""), entity_set_name=data.get("entity_set_name", ""), metadata_id=data.get("metadata_id", ""), + primary_name_attribute=data.get("primary_name_attribute"), + primary_id_attribute=data.get("primary_id_attribute"), columns_created=data.get("columns_created"), ) @@ -213,6 +219,8 @@ def from_api_response(cls, response_data: Dict[str, Any]) -> TableInfo: logical_name=response_data.get("LogicalName", ""), entity_set_name=response_data.get("EntitySetName", ""), metadata_id=response_data.get("MetadataId", ""), + primary_name_attribute=response_data.get("PrimaryNameAttribute"), + primary_id_attribute=response_data.get("PrimaryIdAttribute"), display_name=display_name, description=description, ) diff --git a/src/PowerPlatform/Dataverse/operations/__init__.py b/src/PowerPlatform/Dataverse/operations/__init__.py index bfe7386f..19c8a9e5 100644 --- a/src/PowerPlatform/Dataverse/operations/__init__.py +++ b/src/PowerPlatform/Dataverse/operations/__init__.py @@ -8,4 +8,6 @@ SDK operations into logical groups: records, query, and tables. """ -__all__ = [] +from typing import List + +__all__: List[str] = [] diff --git a/src/PowerPlatform/Dataverse/operations/dataframe.py b/src/PowerPlatform/Dataverse/operations/dataframe.py new file mode 100644 index 00000000..3aec0cc7 --- /dev/null +++ b/src/PowerPlatform/Dataverse/operations/dataframe.py @@ -0,0 +1,358 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""DataFrame CRUD operations namespace for the Dataverse SDK.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +import pandas as pd + +from ..utils._pandas import dataframe_to_records + +if TYPE_CHECKING: + from ..client import DataverseClient + + +__all__ = ["DataFrameOperations"] + + +class DataFrameOperations: + """Namespace for pandas DataFrame CRUD operations. + + Accessed via ``client.dataframe``. Provides DataFrame-oriented wrappers + around the record-level CRUD operations. + + :param client: The parent :class:`~PowerPlatform.Dataverse.client.DataverseClient` instance. + :type client: ~PowerPlatform.Dataverse.client.DataverseClient + + Example:: + + import pandas as pd + + client = DataverseClient(base_url, credential) + + # Query records as a DataFrame + df = client.dataframe.get("account", select=["name"], top=100) + + # Create records from a DataFrame + new_df = pd.DataFrame([{"name": "Contoso"}, {"name": "Fabrikam"}]) + new_df["accountid"] = client.dataframe.create("account", new_df) + + # Update records + new_df["telephone1"] = ["555-0100", "555-0200"] + client.dataframe.update("account", new_df, id_column="accountid") + + # Delete records + client.dataframe.delete("account", new_df["accountid"]) + """ + + def __init__(self, client: DataverseClient) -> None: + self._client = client + + # -------------------------------------------------------------------- get + + def get( + self, + table: str, + record_id: Optional[str] = None, + select: Optional[List[str]] = None, + filter: Optional[str] = None, + orderby: Optional[List[str]] = None, + top: Optional[int] = None, + expand: Optional[List[str]] = None, + page_size: Optional[int] = None, + ) -> pd.DataFrame: + """Fetch records and return as a single pandas DataFrame. + + When ``record_id`` is provided, returns a single-row DataFrame. + When ``record_id`` is None, internally iterates all pages and returns one + consolidated DataFrame. + + :param table: Schema name of the table (e.g. ``"account"`` or ``"new_MyTestTable"``). + :type table: :class:`str` + :param record_id: Optional GUID to fetch a specific record. If None, queries multiple records. + :type record_id: :class:`str` or None + :param select: Optional list of attribute logical names to retrieve. + :type select: :class:`list` of :class:`str` or None + :param filter: Optional OData filter string. Column names must use exact lowercase logical names. + :type filter: :class:`str` or None + :param orderby: Optional list of attributes to sort by. + :type orderby: :class:`list` of :class:`str` or None + :param top: Optional maximum number of records to return. + :type top: :class:`int` or None + :param expand: Optional list of navigation properties to expand (case-sensitive). + :type expand: :class:`list` of :class:`str` or None + :param page_size: Optional number of records per page for pagination. + :type page_size: :class:`int` or None + + :return: DataFrame containing all matching records. Returns an empty DataFrame + when no records match. + :rtype: ~pandas.DataFrame + + :raises ValueError: If ``record_id`` is not a non-empty string, or if + query parameters (``filter``, ``orderby``, ``top``, ``expand``, + ``page_size``) are provided alongside ``record_id``. + + .. tip:: + For large tables, use ``top`` or ``filter`` to limit the result set. + + Example: + Fetch a single record as a DataFrame:: + + df = client.dataframe.get("account", record_id=account_id, select=["name", "telephone1"]) + print(df) + + Query with filtering:: + + df = client.dataframe.get("account", filter="statecode eq 0", select=["name"]) + print(f"Got {len(df)} active accounts") + + Limit result size:: + + df = client.dataframe.get("account", select=["name"], top=100) + """ + if record_id is not None: + if not isinstance(record_id, str) or not record_id.strip(): + raise ValueError("record_id must be a non-empty string") + record_id = record_id.strip() + if any(p is not None for p in (filter, orderby, top, expand, page_size)): + raise ValueError( + "Cannot specify query parameters (filter, orderby, top, " + "expand, page_size) when fetching a single record by ID" + ) + result = self._client.records.get( + table, + record_id, + select=select, + ) + return pd.DataFrame([result.data]) + + rows: List[dict] = [] + for batch in self._client.records.get( + table, + select=select, + filter=filter, + orderby=orderby, + top=top, + expand=expand, + page_size=page_size, + ): + rows.extend(row.data for row in batch) + + if not rows: + return pd.DataFrame(columns=select) if select else pd.DataFrame() + return pd.DataFrame.from_records(rows) + + # ----------------------------------------------------------------- create + + def create( + self, + table: str, + records: pd.DataFrame, + ) -> pd.Series: + """Create records from a pandas DataFrame. + + :param table: Schema name of the table (e.g. ``"account"`` or ``"new_MyTestTable"``). + :type table: :class:`str` + :param records: DataFrame where each row is a record to create. + :type records: ~pandas.DataFrame + + :return: Series of created record GUIDs, aligned with the input DataFrame index. + :rtype: ~pandas.Series + + :raises TypeError: If ``records`` is not a pandas DataFrame. + :raises ValueError: If ``records`` is empty or the number of returned + IDs does not match the number of input rows. + + .. tip:: + All rows are sent in a single ``CreateMultiple`` request. For very + large DataFrames, consider splitting into smaller batches to avoid + request timeouts. + + Example: + Create records from a DataFrame:: + + import pandas as pd + + df = pd.DataFrame([ + {"name": "Contoso", "telephone1": "555-0100"}, + {"name": "Fabrikam", "telephone1": "555-0200"}, + ]) + df["accountid"] = client.dataframe.create("account", df) + """ + if not isinstance(records, pd.DataFrame): + raise TypeError("records must be a pandas DataFrame") + + if records.empty: + raise ValueError("records must be a non-empty DataFrame") + + record_list = dataframe_to_records(records) + + # Detect rows where all values were NaN/None (empty dicts after normalization) + empty_rows = [records.index[i] for i, r in enumerate(record_list) if not r] + if empty_rows: + raise ValueError( + f"Records at index(es) {empty_rows} have no non-null values. " + "All rows must contain at least one field to create." + ) + + ids = self._client.records.create(table, record_list) + + if len(ids) != len(records): + raise ValueError(f"Server returned {len(ids)} IDs for {len(records)} input rows") + + return pd.Series(ids, index=records.index) + + # ----------------------------------------------------------------- update + + def update( + self, + table: str, + changes: pd.DataFrame, + id_column: str, + clear_nulls: bool = False, + ) -> None: + """Update records from a pandas DataFrame. + + Each row in the DataFrame represents an update. The ``id_column`` specifies which + column contains the record GUIDs. + + :param table: Schema name of the table (e.g. ``"account"`` or ``"new_MyTestTable"``). + :type table: :class:`str` + :param changes: DataFrame where each row contains a record GUID and the fields to update. + :type changes: ~pandas.DataFrame + :param id_column: Name of the DataFrame column containing record GUIDs. + :type id_column: :class:`str` + :param clear_nulls: When ``False`` (default), missing values (NaN/None) are skipped + (the field is left unchanged on the server). When ``True``, missing values are sent + as ``null`` to Dataverse, clearing the field. Use ``True`` only when you intentionally + want NaN/None values to clear fields. + :type clear_nulls: :class:`bool` + + :raises TypeError: If ``changes`` is not a pandas DataFrame. + :raises ValueError: If ``changes`` is empty, ``id_column`` is not found in the + DataFrame, ``id_column`` contains invalid (non-string, empty, or whitespace-only) + values, or no updatable columns exist besides ``id_column``. + When ``clear_nulls`` is ``False`` (default), rows where all change values + are NaN/None produce empty patches and are silently skipped. If all rows + are skipped, the method returns without making an API call. When + ``clear_nulls`` is ``True``, NaN/None values become explicit nulls, so + rows are never skipped. + + .. tip:: + All rows are sent in a single ``UpdateMultiple`` request (or a + single PATCH for one row). For very large DataFrames, consider + splitting into smaller batches to avoid request timeouts. + + Example: + Update records with different values per row:: + + import pandas as pd + + df = pd.DataFrame([ + {"accountid": "guid-1", "telephone1": "555-0100"}, + {"accountid": "guid-2", "telephone1": "555-0200"}, + ]) + client.dataframe.update("account", df, id_column="accountid") + + Broadcast the same change to all records:: + + df = pd.DataFrame({"accountid": ["guid-1", "guid-2", "guid-3"]}) + df["websiteurl"] = "https://example.com" + client.dataframe.update("account", df, id_column="accountid") + + Clear a field by setting clear_nulls=True:: + + df = pd.DataFrame([{"accountid": "guid-1", "websiteurl": None}]) + client.dataframe.update("account", df, id_column="accountid", clear_nulls=True) + """ + if not isinstance(changes, pd.DataFrame): + raise TypeError("changes must be a pandas DataFrame") + if changes.empty: + raise ValueError("changes must be a non-empty DataFrame") + if id_column not in changes.columns: + raise ValueError(f"id_column '{id_column}' not found in DataFrame columns") + + raw_ids = changes[id_column].tolist() + invalid = [changes.index[i] for i, v in enumerate(raw_ids) if not isinstance(v, str) or not v.strip()] + if invalid: + raise ValueError( + f"id_column '{id_column}' contains invalid values at row index(es) {invalid}. " + "All IDs must be non-empty strings." + ) + ids = [v.strip() for v in raw_ids] + + change_columns = [column for column in changes.columns if column != id_column] + if not change_columns: + raise ValueError( + "No columns to update. The DataFrame must contain at least one column besides the id_column." + ) + change_list = dataframe_to_records(changes[change_columns], na_as_null=clear_nulls) + + # Filter out rows where all change values were NaN/None (empty dicts) + paired = [(rid, patch) for rid, patch in zip(ids, change_list) if patch] + if not paired: + return + ids_filtered: List[str] = [p[0] for p in paired] + change_filtered: List[Dict[str, Any]] = [p[1] for p in paired] + + if len(ids_filtered) == 1: + self._client.records.update(table, ids_filtered[0], change_filtered[0]) + else: + self._client.records.update(table, ids_filtered, change_filtered) + + # ----------------------------------------------------------------- delete + + def delete( + self, + table: str, + ids: pd.Series, + use_bulk_delete: bool = True, + ) -> Optional[str]: + """Delete records by passing a pandas Series of GUIDs. + + :param table: Schema name of the table (e.g. ``"account"`` or ``"new_MyTestTable"``). + :type table: :class:`str` + :param ids: Series of record GUIDs to delete. + :type ids: ~pandas.Series + :param use_bulk_delete: When ``True`` (default) and ``ids`` contains multiple values, execute the BulkDelete + action and return its async job identifier. When ``False`` each record is deleted sequentially. + :type use_bulk_delete: :class:`bool` + + :raises TypeError: If ``ids`` is not a pandas Series. + :raises ValueError: If ``ids`` contains invalid (non-string, empty, or + whitespace-only) values. + + :return: BulkDelete job ID when deleting multiple records via BulkDelete; + ``None`` when deleting a single record, using sequential deletion, or + when ``ids`` is empty. + :rtype: :class:`str` or None + + Example: + Delete records using a Series:: + + import pandas as pd + + ids = pd.Series(["guid-1", "guid-2", "guid-3"]) + client.dataframe.delete("account", ids) + """ + if not isinstance(ids, pd.Series): + raise TypeError("ids must be a pandas Series") + + raw_list = ids.tolist() + if not raw_list: + return None + + invalid = [ids.index[i] for i, v in enumerate(raw_list) if not isinstance(v, str) or not v.strip()] + if invalid: + raise ValueError( + f"ids Series contains invalid values at index(es) {invalid}. " f"All IDs must be non-empty strings." + ) + id_list = [v.strip() for v in raw_list] + + if len(id_list) == 1: + self._client.records.delete(table, id_list[0]) + return None + return self._client.records.delete(table, id_list, use_bulk_delete=use_bulk_delete) diff --git a/src/PowerPlatform/Dataverse/utils/_pandas.py b/src/PowerPlatform/Dataverse/utils/_pandas.py new file mode 100644 index 00000000..4ebd01a7 --- /dev/null +++ b/src/PowerPlatform/Dataverse/utils/_pandas.py @@ -0,0 +1,60 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Internal pandas helpers""" + +from __future__ import annotations + +import datetime +from typing import Any, Dict, List + +import numpy as np +import pandas as pd + + +def _normalize_scalar(v: Any) -> Any: + """Convert numpy scalar types to their Python native equivalents. + + :param v: A scalar value to normalize. + :return: The value converted to a JSON-serializable Python type. + """ + if isinstance(v, pd.Timestamp): + return v.isoformat() + if isinstance(v, (datetime.datetime, datetime.date)): + return v.isoformat() + if isinstance(v, np.datetime64): + return pd.Timestamp(v).isoformat() + if isinstance(v, np.integer): + return int(v) + if isinstance(v, np.floating): + return float(v) + if isinstance(v, np.bool_): + return bool(v) + return v + + +def dataframe_to_records(df: pd.DataFrame, na_as_null: bool = False) -> List[Dict[str, Any]]: + """Convert a DataFrame to a list of dicts, normalizing values for JSON serialization. + + :param df: Input DataFrame. + :param na_as_null: When False (default), missing values are omitted from each dict. + When True, missing values are included as None (sends null to Dataverse, clearing the field). + """ + records = [] + for row in df.to_dict(orient="records"): + clean = {} + for k, v in row.items(): + if pd.api.types.is_scalar(v): + if pd.notna(v): + clean[k] = _normalize_scalar(v) + elif na_as_null: + clean[k] = None + else: + # Convert np.ndarray to list for JSON serialization; + # pass through lists, dicts, etc. as-is. + if isinstance(v, np.ndarray): + clean[k] = v.tolist() + else: + clean[k] = v + records.append(clean) + return records diff --git a/tests/unit/models/test_table_info.py b/tests/unit/models/test_table_info.py index 0e0754e7..b3da07fa 100644 --- a/tests/unit/models/test_table_info.py +++ b/tests/unit/models/test_table_info.py @@ -49,11 +49,19 @@ def test_legacy_key_iteration(self): keys = list(self.info) self.assertEqual( keys, - ["table_schema_name", "table_logical_name", "entity_set_name", "metadata_id", "columns_created"], + [ + "table_schema_name", + "table_logical_name", + "entity_set_name", + "metadata_id", + "primary_name_attribute", + "primary_id_attribute", + "columns_created", + ], ) def test_len(self): - self.assertEqual(len(self.info), 5) + self.assertEqual(len(self.info), 7) def test_keys_values_items(self): self.assertEqual(list(self.info.keys()), list(self.info._LEGACY_KEY_MAP.keys())) @@ -76,6 +84,8 @@ def test_from_dict(self): "table_logical_name": "new_product", "entity_set_name": "new_products", "metadata_id": "meta-guid-1", + "primary_name_attribute": "new_name", + "primary_id_attribute": "new_productid", "columns_created": ["new_Price"], } info = TableInfo.from_dict(data) @@ -83,13 +93,32 @@ def test_from_dict(self): self.assertEqual(info.logical_name, "new_product") self.assertEqual(info.entity_set_name, "new_products") self.assertEqual(info.metadata_id, "meta-guid-1") + self.assertEqual(info.primary_name_attribute, "new_name") + self.assertEqual(info.primary_id_attribute, "new_productid") self.assertEqual(info.columns_created, ["new_Price"]) def test_from_dict_missing_keys(self): info = TableInfo.from_dict({}) self.assertEqual(info.schema_name, "") + self.assertIsNone(info.primary_name_attribute) + self.assertIsNone(info.primary_id_attribute) self.assertIsNone(info.columns_created) + def test_from_dict_legacy_access_primary_fields(self): + """Primary fields are accessible via legacy dict-key access.""" + data = { + "table_schema_name": "new_Product", + "table_logical_name": "new_product", + "entity_set_name": "new_products", + "metadata_id": "meta-guid-1", + "primary_name_attribute": "new_name", + "primary_id_attribute": "new_productid", + "columns_created": [], + } + info = TableInfo.from_dict(data) + self.assertEqual(info["primary_name_attribute"], "new_name") + self.assertEqual(info["primary_id_attribute"], "new_productid") + class TestTableInfoFromApiResponse(unittest.TestCase): """Tests for TableInfo.from_api_response factory (PascalCase keys).""" @@ -100,6 +129,8 @@ def test_from_api_response(self): "LogicalName": "account", "EntitySetName": "accounts", "MetadataId": "meta-guid-2", + "PrimaryNameAttribute": "name", + "PrimaryIdAttribute": "accountid", "DisplayName": {"UserLocalizedLabel": {"Label": "Account", "LanguageCode": 1033}}, "Description": {"UserLocalizedLabel": {"Label": "Business account", "LanguageCode": 1033}}, } @@ -108,6 +139,8 @@ def test_from_api_response(self): self.assertEqual(info.logical_name, "account") self.assertEqual(info.entity_set_name, "accounts") self.assertEqual(info.metadata_id, "meta-guid-2") + self.assertEqual(info.primary_name_attribute, "name") + self.assertEqual(info.primary_id_attribute, "accountid") self.assertEqual(info.display_name, "Account") self.assertEqual(info.description, "Business account") diff --git a/tests/unit/test_client_dataframe.py b/tests/unit/test_client_dataframe.py new file mode 100644 index 00000000..d19419ac --- /dev/null +++ b/tests/unit/test_client_dataframe.py @@ -0,0 +1,365 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import unittest +from unittest.mock import MagicMock + +import pandas as pd +from azure.core.credentials import TokenCredential + +from PowerPlatform.Dataverse.client import DataverseClient + + +class TestDataFrameGet(unittest.TestCase): + """Tests for client.dataframe.get().""" + + def setUp(self): + self.mock_credential = MagicMock(spec=TokenCredential) + self.base_url = "https://example.crm.dynamics.com" + self.client = DataverseClient(self.base_url, self.mock_credential) + self.client._odata = MagicMock() + + def test_get_single_record(self): + """Single record_id returns a one-row DataFrame.""" + expected = {"accountid": "guid-1", "name": "Contoso"} + self.client._odata._get.return_value = expected + + df = self.client.dataframe.get("account", record_id="guid-1") + + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 1) + self.assertEqual(df.iloc[0]["name"], "Contoso") + self.client._odata._get.assert_called_once_with("account", "guid-1", select=None) + + def test_get_single_record_with_select(self): + """Single record with select columns.""" + expected = {"accountid": "guid-1", "name": "Contoso"} + self.client._odata._get.return_value = expected + + df = self.client.dataframe.get("account", record_id="guid-1", select=["name"]) + + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 1) + self.client._odata._get.assert_called_once_with("account", "guid-1", select=["name"]) + + def test_get_multiple_records_single_page(self): + """Single page returns a DataFrame with all rows.""" + batch = [ + {"accountid": "guid-1", "name": "A"}, + {"accountid": "guid-2", "name": "B"}, + ] + self.client._odata._get_multiple.return_value = iter([batch]) + + df = self.client.dataframe.get("account", filter="statecode eq 0") + + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 2) + self.assertListEqual(df["name"].tolist(), ["A", "B"]) + + def test_get_multiple_records_multi_page(self): + """Multiple pages are concatenated into a single DataFrame.""" + page1 = [{"accountid": "guid-1", "name": "A"}] + page2 = [{"accountid": "guid-2", "name": "B"}] + self.client._odata._get_multiple.return_value = iter([page1, page2]) + + df = self.client.dataframe.get("account", top=100) + + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 2) + self.assertEqual(df.iloc[0]["name"], "A") + self.assertEqual(df.iloc[1]["name"], "B") + + def test_get_index_is_reset(self): + """Returned DataFrame has a clean 0-based integer index.""" + page1 = [{"accountid": "guid-1", "name": "A"}] + page2 = [{"accountid": "guid-2", "name": "B"}] + self.client._odata._get_multiple.return_value = iter([page1, page2]) + + df = self.client.dataframe.get("account", top=100) + + self.assertListEqual(list(df.index), [0, 1]) + + def test_get_empty_result(self): + """Empty result set returns an empty DataFrame.""" + self.client._odata._get_multiple.return_value = iter([]) + + df = self.client.dataframe.get("account") + + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 0) + + def test_get_passes_all_parameters(self): + """All query parameters are forwarded to the underlying get method.""" + self.client._odata._get_multiple.return_value = iter([]) + + self.client.dataframe.get( + "account", + select=["name"], + filter="statecode eq 0", + orderby=["name asc"], + top=50, + expand=["primarycontactid"], + page_size=25, + ) + + self.client._odata._get_multiple.assert_called_once_with( + "account", + select=["name"], + filter="statecode eq 0", + orderby=["name asc"], + top=50, + expand=["primarycontactid"], + page_size=25, + ) + + +class TestDataFrameCreate(unittest.TestCase): + """Tests for client.dataframe.create().""" + + def setUp(self): + self.mock_credential = MagicMock(spec=TokenCredential) + self.base_url = "https://example.crm.dynamics.com" + self.client = DataverseClient(self.base_url, self.mock_credential) + self.client._odata = MagicMock() + + def test_create_dataframe(self): + """DataFrame rows are converted to dicts and returned IDs are a Series.""" + df = pd.DataFrame( + [ + {"name": "Contoso", "telephone1": "555-0100"}, + {"name": "Fabrikam", "telephone1": "555-0200"}, + ] + ) + self.client._odata._create_multiple.return_value = ["guid-1", "guid-2"] + self.client._odata._entity_set_from_schema_name.return_value = "accounts" + + ids = self.client.dataframe.create("account", df) + + self.assertIsInstance(ids, pd.Series) + self.assertListEqual(ids.tolist(), ["guid-1", "guid-2"]) + call_args = self.client._odata._create_multiple.call_args + records_arg = call_args[0][2] + self.assertEqual(len(records_arg), 2) + self.assertEqual(records_arg[0]["name"], "Contoso") + self.assertEqual(records_arg[1]["name"], "Fabrikam") + + def test_create_assigns_to_column(self): + """Returned Series can be assigned directly as a DataFrame column.""" + df = pd.DataFrame([{"name": "Contoso"}, {"name": "Fabrikam"}]) + self.client._odata._create_multiple.return_value = ["guid-1", "guid-2"] + self.client._odata._entity_set_from_schema_name.return_value = "accounts" + + df["accountid"] = self.client.dataframe.create("account", df) + + self.assertListEqual(df["accountid"].tolist(), ["guid-1", "guid-2"]) + + def test_create_single_row_dataframe(self): + """Single-row DataFrame returns a single-element Series.""" + df = pd.DataFrame([{"name": "Contoso"}]) + self.client._odata._create_multiple.return_value = ["guid-1"] + self.client._odata._entity_set_from_schema_name.return_value = "accounts" + + ids = self.client.dataframe.create("account", df) + + self.assertIsInstance(ids, pd.Series) + self.assertEqual(ids.iloc[0], "guid-1") + + def test_create_rejects_non_dataframe(self): + """Non-DataFrame input raises TypeError.""" + with self.assertRaises(TypeError) as ctx: + self.client.dataframe.create("account", [{"name": "Contoso"}]) + self.assertIn("pandas DataFrame", str(ctx.exception)) + + def test_create_empty_dataframe_raises(self): + """Empty DataFrame raises ValueError.""" + df = pd.DataFrame(columns=["name", "telephone1"]) + + with self.assertRaises(ValueError) as ctx: + self.client.dataframe.create("account", df) + self.assertIn("non-empty DataFrame", str(ctx.exception)) + self.client._odata._create_multiple.assert_not_called() + + def test_create_length_mismatch_raises(self): + """ValueError raised when returned IDs don't match input row count.""" + df = pd.DataFrame([{"name": "Contoso"}, {"name": "Fabrikam"}]) + self.client._odata._create_multiple.return_value = ["guid-1"] + self.client._odata._entity_set_from_schema_name.return_value = "accounts" + + with self.assertRaises(ValueError) as ctx: + self.client.dataframe.create("account", df) + self.assertIn("1 IDs for 2 input rows", str(ctx.exception)) + + def test_create_drops_nan_values(self): + """NaN/None values are omitted from the create payload.""" + df = pd.DataFrame( + [ + {"name": "Contoso", "telephone1": "555-0100"}, + {"name": "Fabrikam", "telephone1": None}, + ] + ) + self.client._odata._create_multiple.return_value = ["guid-1", "guid-2"] + self.client._odata._entity_set_from_schema_name.return_value = "accounts" + + self.client.dataframe.create("account", df) + + call_args = self.client._odata._create_multiple.call_args + records_arg = call_args[0][2] + self.assertEqual(records_arg[0], {"name": "Contoso", "telephone1": "555-0100"}) + self.assertEqual(records_arg[1], {"name": "Fabrikam"}) + self.assertNotIn("telephone1", records_arg[1]) + + def test_create_converts_timestamps_to_iso(self): + """Timestamp values are converted to ISO 8601 strings.""" + ts = pd.Timestamp("2024-01-15 10:30:00") + df = pd.DataFrame([{"name": "Contoso", "createdon": ts}]) + self.client._odata._create_multiple.return_value = ["guid-1"] + self.client._odata._entity_set_from_schema_name.return_value = "accounts" + + self.client.dataframe.create("account", df) + + call_args = self.client._odata._create_multiple.call_args + records_arg = call_args[0][2] + self.assertEqual(records_arg[0]["createdon"], "2024-01-15T10:30:00") + + +class TestDataFrameUpdate(unittest.TestCase): + """Tests for client.dataframe.update().""" + + def setUp(self): + self.mock_credential = MagicMock(spec=TokenCredential) + self.base_url = "https://example.crm.dynamics.com" + self.client = DataverseClient(self.base_url, self.mock_credential) + self.client._odata = MagicMock() + + def test_update_dataframe(self): + """DataFrame rows are split into IDs and changes, then passed to update.""" + df = pd.DataFrame( + [ + {"accountid": "guid-1", "telephone1": "555-0100"}, + {"accountid": "guid-2", "telephone1": "555-0200"}, + ] + ) + + self.client.dataframe.update("account", df, id_column="accountid") + + self.client._odata._update_by_ids.assert_called_once() + call_args = self.client._odata._update_by_ids.call_args[0] + self.assertEqual(call_args[0], "account") + self.assertEqual(call_args[1], ["guid-1", "guid-2"]) + self.assertEqual(call_args[2], [{"telephone1": "555-0100"}, {"telephone1": "555-0200"}]) + + def test_update_rejects_non_dataframe(self): + """Non-DataFrame input raises TypeError.""" + with self.assertRaises(TypeError) as ctx: + self.client.dataframe.update("account", {"id": "guid-1"}, id_column="id") + self.assertIn("pandas DataFrame", str(ctx.exception)) + + def test_update_rejects_missing_id_column(self): + """Missing id_column raises ValueError.""" + df = pd.DataFrame([{"name": "Contoso"}]) + with self.assertRaises(ValueError) as ctx: + self.client.dataframe.update("account", df, id_column="accountid") + self.assertIn("accountid", str(ctx.exception)) + + def test_update_multiple_change_columns(self): + """Multiple change columns are all included in the update payload (single row uses _update).""" + df = pd.DataFrame( + [ + {"accountid": "guid-1", "name": "New Name", "telephone1": "555-0100"}, + ] + ) + + self.client.dataframe.update("account", df, id_column="accountid") + + self.client._odata._update.assert_called_once() + call_args = self.client._odata._update.call_args[0] + self.assertEqual(call_args[0], "account") + self.assertEqual(call_args[1], "guid-1") + changes = call_args[2] + self.assertIn("name", changes) + self.assertIn("telephone1", changes) + self.assertNotIn("accountid", changes) + + def test_update_skips_nan_by_default(self): + """NaN/None values are skipped by default (field left unchanged on server).""" + df = pd.DataFrame( + [ + {"accountid": "guid-1", "name": "New Name", "telephone1": None}, + {"accountid": "guid-2", "name": None, "telephone1": "555-0200"}, + ] + ) + + self.client.dataframe.update("account", df, id_column="accountid") + + call_args = self.client._odata._update_by_ids.call_args[0] + changes = call_args[2] + self.assertEqual(changes[0], {"name": "New Name"}) + self.assertEqual(changes[1], {"telephone1": "555-0200"}) + + def test_update_clear_nulls_sends_none(self): + """With clear_nulls=True, NaN/None values are sent as None to clear fields.""" + df = pd.DataFrame( + [ + {"accountid": "guid-1", "name": "New Name", "telephone1": None}, + {"accountid": "guid-2", "name": None, "telephone1": "555-0200"}, + ] + ) + + self.client.dataframe.update("account", df, id_column="accountid", clear_nulls=True) + + call_args = self.client._odata._update_by_ids.call_args[0] + changes = call_args[2] + self.assertEqual(changes[0], {"name": "New Name", "telephone1": None}) + self.assertEqual(changes[1], {"name": None, "telephone1": "555-0200"}) + + +class TestDataFrameDelete(unittest.TestCase): + """Tests for client.dataframe.delete().""" + + def setUp(self): + self.mock_credential = MagicMock(spec=TokenCredential) + self.base_url = "https://example.crm.dynamics.com" + self.client = DataverseClient(self.base_url, self.mock_credential) + self.client._odata = MagicMock() + + def test_delete_dataframe_bulk(self): + """Series of GUIDs passed to bulk delete.""" + ids = pd.Series(["guid-1", "guid-2", "guid-3"]) + self.client._odata._delete_multiple.return_value = "job-123" + + job_id = self.client.dataframe.delete("account", ids) + + self.assertEqual(job_id, "job-123") + self.client._odata._delete_multiple.assert_called_once_with("account", ["guid-1", "guid-2", "guid-3"]) + + def test_delete_from_dataframe_column(self): + """Series extracted from a DataFrame column works directly.""" + df = pd.DataFrame({"accountid": ["guid-1", "guid-2"], "name": ["A", "B"]}) + self.client._odata._delete_multiple.return_value = "job-123" + + self.client.dataframe.delete("account", df["accountid"]) + + self.client._odata._delete_multiple.assert_called_once_with("account", ["guid-1", "guid-2"]) + + def test_delete_dataframe_sequential(self): + """use_bulk_delete=False deletes records sequentially.""" + ids = pd.Series(["guid-1", "guid-2"]) + + result = self.client.dataframe.delete("account", ids, use_bulk_delete=False) + + self.assertIsNone(result) + self.assertEqual(self.client._odata._delete.call_count, 2) + + def test_delete_rejects_non_series(self): + """Non-Series input raises TypeError.""" + with self.assertRaises(TypeError) as ctx: + self.client.dataframe.delete("account", ["guid-1"]) + self.assertIn("pandas Series", str(ctx.exception)) + + def test_delete_empty_series(self): + """Empty Series returns None without calling delete.""" + ids = pd.Series([], dtype="str") + + result = self.client.dataframe.delete("account", ids) + + self.assertIsNone(result) diff --git a/tests/unit/test_dataframe_operations.py b/tests/unit/test_dataframe_operations.py new file mode 100644 index 00000000..7931c3f5 --- /dev/null +++ b/tests/unit/test_dataframe_operations.py @@ -0,0 +1,495 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Comprehensive unit tests for the DataFrameOperations namespace (client.dataframe).""" + +import unittest +from unittest.mock import MagicMock + +import numpy as np +import pandas as pd +from azure.core.credentials import TokenCredential + +from PowerPlatform.Dataverse.client import DataverseClient +from PowerPlatform.Dataverse.operations.dataframe import DataFrameOperations + + +class TestDataFrameOperationsNamespace(unittest.TestCase): + """Tests for the DataFrameOperations namespace itself.""" + + def setUp(self): + self.mock_credential = MagicMock(spec=TokenCredential) + self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) + self.client._odata = MagicMock() + + def test_namespace_exists(self): + """client.dataframe is a DataFrameOperations instance.""" + self.assertIsInstance(self.client.dataframe, DataFrameOperations) + + +class TestDataFrameGet(unittest.TestCase): + """Tests for client.dataframe.get().""" + + def setUp(self): + self.mock_credential = MagicMock(spec=TokenCredential) + self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) + self.client._odata = MagicMock() + + def test_get_single_record(self): + """record_id returns a one-row DataFrame using result.data.""" + self.client._odata._get.return_value = {"accountid": "guid-1", "name": "Contoso"} + df = self.client.dataframe.get("account", record_id="guid-1") + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 1) + self.assertEqual(df.iloc[0]["name"], "Contoso") + + def test_get_multiple_records(self): + """Without record_id, pages are iterated and consolidated into one DataFrame.""" + page1 = [{"accountid": "guid-1", "name": "A"}] + page2 = [{"accountid": "guid-2", "name": "B"}] + self.client._odata._get_multiple.return_value = iter([page1, page2]) + df = self.client.dataframe.get("account") + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 2) + + def test_get_no_results(self): + """Empty result set returns an empty DataFrame.""" + self.client._odata._get_multiple.return_value = iter([]) + df = self.client.dataframe.get("account") + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 0) + + def test_get_no_results_with_select_preserves_columns(self): + """Empty result with select returns DataFrame with expected columns.""" + self.client._odata._get_multiple.return_value = iter([]) + df = self.client.dataframe.get("account", select=["name", "telephone1"]) + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 0) + self.assertListEqual(list(df.columns), ["name", "telephone1"]) + + def test_create_all_nan_rows_raises(self): + """DataFrame where all values are NaN raises ValueError.""" + df = pd.DataFrame([{"name": None, "phone": None}]) + with self.assertRaises(ValueError) as ctx: + self.client.dataframe.create("account", df) + self.assertIn("no non-null values", str(ctx.exception)) + + def test_get_passes_all_params(self): + """All OData parameters are forwarded to the underlying API call.""" + self.client._odata._get_multiple.return_value = iter([]) + self.client.dataframe.get( + "account", + select=["name"], + filter="statecode eq 0", + orderby=["name asc"], + top=50, + expand=["primarycontactid"], + page_size=25, + ) + self.client._odata._get_multiple.assert_called_once_with( + "account", + select=["name"], + filter="statecode eq 0", + orderby=["name asc"], + top=50, + expand=["primarycontactid"], + page_size=25, + ) + + def test_get_record_id_with_query_params_raises(self): + """ValueError raised when record_id is provided with query params.""" + with self.assertRaises(ValueError) as ctx: + self.client.dataframe.get("account", record_id="guid-1", filter="name eq 'X'") + self.assertIn("Cannot specify query parameters", str(ctx.exception)) + + def test_get_record_id_with_top_raises(self): + """ValueError raised when record_id is provided with top.""" + with self.assertRaises(ValueError) as ctx: + self.client.dataframe.get("account", record_id="guid-1", top=10) + self.assertIn("Cannot specify query parameters", str(ctx.exception)) + + def test_get_empty_record_id_raises(self): + """ValueError raised when record_id is empty or whitespace.""" + with self.assertRaises(ValueError) as ctx: + self.client.dataframe.get("account", record_id=" ") + self.assertIn("non-empty string", str(ctx.exception)) + + def test_get_record_id_stripped(self): + """Leading/trailing whitespace in record_id is stripped.""" + self.client._odata._get.return_value = {"accountid": "guid-1", "name": "Contoso"} + self.client.dataframe.get("account", record_id=" guid-1 ") + self.client._odata._get.assert_called_once_with("account", "guid-1", select=None) + + +class TestDataFrameCreate(unittest.TestCase): + """Tests for client.dataframe.create().""" + + def setUp(self): + self.mock_credential = MagicMock(spec=TokenCredential) + self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) + self.client._odata = MagicMock() + self.client._odata._entity_set_from_schema_name.return_value = "accounts" + + def test_create_returns_series(self): + """Returns a Series of GUIDs aligned with the input DataFrame index.""" + df = pd.DataFrame([{"name": "Contoso"}, {"name": "Fabrikam"}]) + self.client._odata._create_multiple.return_value = ["guid-1", "guid-2"] + ids = self.client.dataframe.create("account", df) + self.assertIsInstance(ids, pd.Series) + self.assertListEqual(ids.tolist(), ["guid-1", "guid-2"]) + + def test_create_type_error(self): + """Non-DataFrame input raises TypeError.""" + with self.assertRaises(TypeError) as ctx: + self.client.dataframe.create("account", [{"name": "Contoso"}]) + self.assertIn("pandas DataFrame", str(ctx.exception)) + + def test_create_empty_dataframe_raises(self): + """Empty DataFrame raises ValueError without calling the API.""" + df = pd.DataFrame(columns=["name"]) + with self.assertRaises(ValueError) as ctx: + self.client.dataframe.create("account", df) + self.assertIn("non-empty", str(ctx.exception)) + self.client._odata._create_multiple.assert_not_called() + + def test_create_id_count_mismatch_raises(self): + """ValueError raised when returned IDs count doesn't match input row count.""" + df = pd.DataFrame([{"name": "Contoso"}, {"name": "Fabrikam"}]) + self.client._odata._create_multiple.return_value = ["guid-1"] + with self.assertRaises(ValueError) as ctx: + self.client.dataframe.create("account", df) + self.assertIn("1 IDs for 2 input rows", str(ctx.exception)) + + def test_create_normalizes_values(self): + """NumPy types and Timestamps are normalized before sending to the API.""" + ts = pd.Timestamp("2024-01-15 10:30:00") + df = pd.DataFrame([{"count": np.int64(5), "score": np.float64(9.8), "createdon": ts}]) + self.client._odata._create_multiple.return_value = ["guid-1"] + self.client.dataframe.create("account", df) + records_arg = self.client._odata._create_multiple.call_args[0][2] + rec = records_arg[0] + self.assertIsInstance(rec["count"], int) + self.assertIsInstance(rec["score"], float) + self.assertEqual(rec["createdon"], "2024-01-15T10:30:00") + + +class TestDataFrameUpdate(unittest.TestCase): + """Tests for client.dataframe.update().""" + + def setUp(self): + self.mock_credential = MagicMock(spec=TokenCredential) + self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) + self.client._odata = MagicMock() + + def test_update_single_record(self): + """Single-row DataFrame calls single-record update path.""" + df = pd.DataFrame([{"accountid": "guid-1", "name": "New Name"}]) + self.client.dataframe.update("account", df, id_column="accountid") + self.client._odata._update.assert_called_once_with("account", "guid-1", {"name": "New Name"}) + + def test_update_multiple_records(self): + """Multi-row DataFrame calls batch update path.""" + df = pd.DataFrame( + [ + {"accountid": "guid-1", "telephone1": "555-0100"}, + {"accountid": "guid-2", "telephone1": "555-0200"}, + ] + ) + self.client.dataframe.update("account", df, id_column="accountid") + self.client._odata._update_by_ids.assert_called_once_with( + "account", + ["guid-1", "guid-2"], + [{"telephone1": "555-0100"}, {"telephone1": "555-0200"}], + ) + + def test_update_type_error(self): + """Non-DataFrame input raises TypeError.""" + with self.assertRaises(TypeError) as ctx: + self.client.dataframe.update("account", {"id": "guid-1"}, id_column="id") + self.assertIn("pandas DataFrame", str(ctx.exception)) + + def test_update_missing_id_column(self): + """ValueError raised when id_column is not in DataFrame columns.""" + df = pd.DataFrame([{"name": "Contoso"}]) + with self.assertRaises(ValueError) as ctx: + self.client.dataframe.update("account", df, id_column="accountid") + self.assertIn("accountid", str(ctx.exception)) + + def test_update_invalid_id_values(self): + """ValueError raised when id_column contains NaN or non-string values.""" + df = pd.DataFrame( + [ + {"accountid": "guid-1", "name": "A"}, + {"accountid": None, "name": "B"}, + ] + ) + with self.assertRaises(ValueError) as ctx: + self.client.dataframe.update("account", df, id_column="accountid") + self.assertIn("invalid values", str(ctx.exception)) + self.assertIn("[1]", str(ctx.exception)) + + def test_update_empty_change_columns(self): + """ValueError raised when DataFrame contains only the id_column.""" + df = pd.DataFrame([{"accountid": "guid-1"}]) + with self.assertRaises(ValueError) as ctx: + self.client.dataframe.update("account", df, id_column="accountid") + self.assertIn("No columns to update", str(ctx.exception)) + + def test_update_empty_dataframe_raises(self): + """Empty DataFrame raises ValueError without calling the API.""" + df = pd.DataFrame(columns=["accountid", "name"]) + with self.assertRaises(ValueError) as ctx: + self.client.dataframe.update("account", df, id_column="accountid") + self.assertIn("non-empty", str(ctx.exception)) + self.client._odata._update.assert_not_called() + + def test_update_clear_nulls_false(self): + """NaN values are omitted from the update payload when clear_nulls=False.""" + df = pd.DataFrame([{"accountid": "guid-1", "name": "New Name", "telephone1": None}]) + self.client.dataframe.update("account", df, id_column="accountid") + call_args = self.client._odata._update.call_args[0] + changes = call_args[2] + self.assertIn("name", changes) + self.assertNotIn("telephone1", changes) + + def test_update_all_nan_rows_skipped(self): + """When all change values are NaN for every row, no API call is made.""" + df = pd.DataFrame( + [ + {"accountid": "guid-1", "telephone1": None, "websiteurl": None}, + {"accountid": "guid-2", "telephone1": None, "websiteurl": None}, + ] + ) + self.client.dataframe.update("account", df, id_column="accountid") + self.client._odata._update.assert_not_called() + self.client._odata._update_by_ids.assert_not_called() + + def test_update_partial_nan_rows_filtered(self): + """Rows where all changes are NaN are filtered; remaining rows proceed.""" + df = pd.DataFrame( + [ + {"accountid": "guid-1", "name": "Updated", "telephone1": None}, + {"accountid": "guid-2", "name": None, "telephone1": None}, + ] + ) + self.client.dataframe.update("account", df, id_column="accountid") + self.client._odata._update.assert_called_once_with("account", "guid-1", {"name": "Updated"}) + + def test_update_invalid_ids_reports_index_labels(self): + """Error message reports DataFrame index labels, not positional indices.""" + df = pd.DataFrame( + [ + {"accountid": "guid-1", "name": "A"}, + {"accountid": None, "name": "B"}, + ], + index=["row_a", "row_b"], + ) + with self.assertRaises(ValueError) as ctx: + self.client.dataframe.update("account", df, id_column="accountid") + self.assertIn("row_b", str(ctx.exception)) + + def test_update_strips_whitespace_from_ids(self): + """Leading/trailing whitespace in IDs is stripped before API call.""" + df = pd.DataFrame([{"accountid": " guid-1 ", "name": "Contoso"}]) + self.client.dataframe.update("account", df, id_column="accountid") + call_args = self.client._odata._update.call_args[0] + self.assertEqual(call_args[1], "guid-1") + + def test_update_clear_nulls_true(self): + """NaN values are sent as None in the update payload when clear_nulls=True.""" + df = pd.DataFrame([{"accountid": "guid-1", "name": "New Name", "telephone1": None}]) + self.client.dataframe.update("account", df, id_column="accountid", clear_nulls=True) + call_args = self.client._odata._update.call_args[0] + changes = call_args[2] + self.assertIn("name", changes) + self.assertIn("telephone1", changes) + self.assertIsNone(changes["telephone1"]) + + +class TestDataFrameDelete(unittest.TestCase): + """Tests for client.dataframe.delete().""" + + def setUp(self): + self.mock_credential = MagicMock(spec=TokenCredential) + self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) + self.client._odata = MagicMock() + + def test_delete_single_record(self): + """Single-element Series calls single-record delete.""" + ids = pd.Series(["guid-1"]) + self.client.dataframe.delete("account", ids) + self.client._odata._delete.assert_called_once_with("account", "guid-1") + + def test_delete_multiple_records(self): + """Multi-element Series calls bulk delete.""" + ids = pd.Series(["guid-1", "guid-2", "guid-3"]) + self.client._odata._delete_multiple.return_value = "job-123" + job_id = self.client.dataframe.delete("account", ids) + self.assertEqual(job_id, "job-123") + self.client._odata._delete_multiple.assert_called_once_with("account", ["guid-1", "guid-2", "guid-3"]) + + def test_delete_type_error(self): + """Non-Series input raises TypeError.""" + with self.assertRaises(TypeError) as ctx: + self.client.dataframe.delete("account", ["guid-1"]) + self.assertIn("pandas Series", str(ctx.exception)) + + def test_delete_empty_series(self): + """Empty Series returns None without calling delete.""" + ids = pd.Series([], dtype="str") + result = self.client.dataframe.delete("account", ids) + self.assertIsNone(result) + self.client._odata._delete.assert_not_called() + self.client._odata._delete_multiple.assert_not_called() + + def test_delete_invalid_ids(self): + """ValueError raised when Series contains NaN or non-string values.""" + ids = pd.Series(["guid-1", None, " "]) + with self.assertRaises(ValueError) as ctx: + self.client.dataframe.delete("account", ids) + self.assertIn("invalid values", str(ctx.exception)) + + def test_delete_with_bulk_delete_false(self): + """use_bulk_delete=False passes through to the underlying delete call.""" + ids = pd.Series(["guid-1", "guid-2"]) + result = self.client.dataframe.delete("account", ids, use_bulk_delete=False) + self.assertIsNone(result) + self.assertEqual(self.client._odata._delete.call_count, 2) + + def test_delete_invalid_ids_reports_index_labels(self): + """Error message reports Series index labels, not positional indices.""" + ids = pd.Series(["guid-1", None], index=["row_x", "row_y"]) + with self.assertRaises(ValueError) as ctx: + self.client.dataframe.delete("account", ids) + self.assertIn("row_y", str(ctx.exception)) + + def test_delete_strips_whitespace_from_ids(self): + """Leading/trailing whitespace in IDs is stripped before API call.""" + ids = pd.Series([" guid-1 "]) + self.client.dataframe.delete("account", ids) + self.client._odata._delete.assert_called_once_with("account", "guid-1") + + +class TestDataFrameEndToEnd(unittest.TestCase): + """End-to-end mocked flow: create -> get -> update -> delete.""" + + def setUp(self): + self.mock_credential = MagicMock(spec=TokenCredential) + self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) + self.client._odata = MagicMock() + self.client._odata._entity_set_from_schema_name.return_value = "accounts" + + def test_create_get_update_delete_flow(self): + """Full CRUD cycle works end-to-end through the dataframe namespace.""" + # Step 1: create + df = pd.DataFrame( + [{"name": "Contoso", "telephone1": "555-0100"}, {"name": "Fabrikam", "telephone1": "555-0200"}] + ) + self.client._odata._create_multiple.return_value = ["guid-1", "guid-2"] + + ids = self.client.dataframe.create("account", df) + + self.assertIsInstance(ids, pd.Series) + self.assertListEqual(ids.tolist(), ["guid-1", "guid-2"]) + + # Step 2: get + df["accountid"] = ids + self.client._odata._get_multiple.return_value = iter( + [[{"accountid": "guid-1", "name": "Contoso"}, {"accountid": "guid-2", "name": "Fabrikam"}]] + ) + + result_df = self.client.dataframe.get("account", select=["accountid", "name"]) + + self.assertIsInstance(result_df, pd.DataFrame) + self.assertEqual(len(result_df), 2) + + # Step 3: update + df["telephone1"] = ["555-9999", "555-8888"] + + self.client.dataframe.update("account", df, id_column="accountid") + + self.client._odata._update_by_ids.assert_called_once() + + # Step 4: delete + self.client._odata._delete_multiple.return_value = "job-abc" + + job_id = self.client.dataframe.delete("account", df["accountid"]) + + self.assertEqual(job_id, "job-abc") + self.client._odata._delete_multiple.assert_called_once_with("account", ["guid-1", "guid-2"]) + + def test_create_normalizes_numpy_types_before_api(self): + """NumPy types in DataFrame cells are normalized to Python types before the API call.""" + df = pd.DataFrame( + [ + { + "count": np.int64(10), + "score": np.float64(9.5), + "active": np.bool_(True), + "createdon": pd.Timestamp("2024-06-01"), + } + ] + ) + self.client._odata._create_multiple.return_value = ["guid-1"] + + self.client.dataframe.create("account", df) + + records_arg = self.client._odata._create_multiple.call_args[0][2] + rec = records_arg[0] + self.assertIsInstance(rec["count"], int) + self.assertIsInstance(rec["score"], float) + self.assertIsInstance(rec["active"], bool) + self.assertIsInstance(rec["createdon"], str) + self.assertEqual(rec["createdon"], "2024-06-01T00:00:00") + + def test_get_with_expand_includes_nested_data(self): + """get() with expand returns DataFrame including expanded navigation property data.""" + page = [ + { + "accountid": "guid-1", + "name": "Contoso", + "primarycontactid": {"contactid": "c-1", "fullname": "John"}, + } + ] + self.client._odata._get_multiple.return_value = iter([page]) + df = self.client.dataframe.get("account", expand=["primarycontactid"]) + self.assertEqual(len(df), 1) + self.assertEqual(df.iloc[0]["name"], "Contoso") + self.assertIsInstance(df.iloc[0]["primarycontactid"], dict) + self.assertEqual(df.iloc[0]["primarycontactid"]["fullname"], "John") + + def test_get_single_record_no_odata_keys(self): + """Single-record get strips @odata.* keys from the returned DataFrame.""" + self.client._odata._get.return_value = { + "@odata.context": "https://example.crm.dynamics.com/$metadata#accounts/$entity", + "@odata.etag": 'W/"123"', + "accountid": "guid-1", + "name": "Contoso", + } + df = self.client.dataframe.get("account", record_id="guid-1") + self.assertNotIn("@odata.context", df.columns) + self.assertNotIn("@odata.etag", df.columns) + self.assertIn("name", df.columns) + self.assertEqual(df.iloc[0]["name"], "Contoso") + + def test_delete_whitespace_only_ids_rejected(self): + """Series containing whitespace-only strings raises ValueError.""" + ids = pd.Series(["guid-1", " ", "guid-3"]) + with self.assertRaises(ValueError) as ctx: + self.client.dataframe.delete("account", ids) + self.assertIn("invalid values", str(ctx.exception)) + self.assertIn("[1]", str(ctx.exception)) + + def test_update_with_timezone_aware_timestamps(self): + """Update correctly normalizes timezone-aware Timestamps.""" + ts = pd.Timestamp("2024-06-15 10:30:00", tz="UTC") + df = pd.DataFrame([{"accountid": "guid-1", "lastonholdtime": ts}]) + self.client.dataframe.update("account", df, id_column="accountid") + call_args = self.client._odata._update.call_args[0] + changes = call_args[2] + self.assertIsInstance(changes["lastonholdtime"], str) + self.assertIn("2024-06-15T10:30:00", changes["lastonholdtime"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_pandas_helpers.py b/tests/unit/test_pandas_helpers.py new file mode 100644 index 00000000..f6c0c2fc --- /dev/null +++ b/tests/unit/test_pandas_helpers.py @@ -0,0 +1,301 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Unit tests for the dataframe_to_records() helper in utils/_pandas.py.""" + +import unittest + +import numpy as np +import pandas as pd + +from PowerPlatform.Dataverse.utils._pandas import _normalize_scalar, dataframe_to_records + + +class TestNormalizeScalar(unittest.TestCase): + """Unit tests for _normalize_scalar().""" + + def test_timestamp(self): + """pd.Timestamp is converted to an ISO 8601 string.""" + ts = pd.Timestamp("2024-01-15 10:30:00") + result = _normalize_scalar(ts) + self.assertEqual(result, "2024-01-15T10:30:00") + + def test_numpy_integer(self): + """np.int64 is converted to Python int.""" + result = _normalize_scalar(np.int64(42)) + self.assertIsInstance(result, int) + self.assertEqual(result, 42) + + def test_numpy_floating(self): + """np.float64 is converted to Python float.""" + result = _normalize_scalar(np.float64(3.14)) + self.assertIsInstance(result, float) + self.assertAlmostEqual(result, 3.14) + + def test_numpy_bool(self): + """np.bool_ is converted to Python bool.""" + result = _normalize_scalar(np.bool_(True)) + self.assertIsInstance(result, bool) + self.assertTrue(result) + + def test_python_str_passthrough(self): + """Python str values pass through unchanged.""" + result = _normalize_scalar("hello") + self.assertEqual(result, "hello") + + def test_python_int_passthrough(self): + """Native Python int values pass through unchanged.""" + result = _normalize_scalar(42) + self.assertIsInstance(result, int) + self.assertEqual(result, 42) + + def test_python_float_passthrough(self): + """Native Python float values pass through unchanged.""" + result = _normalize_scalar(3.14) + self.assertIsInstance(result, float) + self.assertAlmostEqual(result, 3.14) + + def test_python_bool_passthrough(self): + """Native Python bool values pass through unchanged.""" + result = _normalize_scalar(True) + self.assertIsInstance(result, bool) + self.assertTrue(result) + + def test_none_passthrough(self): + """None passes through unchanged (caller is responsible for NA handling).""" + result = _normalize_scalar(None) + self.assertIsNone(result) + + def test_timestamp_with_timezone(self): + """Timezone-aware pd.Timestamp is converted to ISO 8601 with tz offset.""" + ts = pd.Timestamp("2024-06-15 10:30:00", tz="UTC") + result = _normalize_scalar(ts) + self.assertIn("2024-06-15T10:30:00", result) + self.assertIsInstance(result, str) + + def test_numpy_int32(self): + """np.int32 is also converted to Python int.""" + result = _normalize_scalar(np.int32(7)) + self.assertIsInstance(result, int) + self.assertEqual(result, 7) + + def test_numpy_float32(self): + """np.float32 is also converted to Python float.""" + result = _normalize_scalar(np.float32(2.5)) + self.assertIsInstance(result, float) + self.assertAlmostEqual(result, 2.5, places=5) + + def test_python_datetime(self): + """datetime.datetime is converted to ISO 8601 string.""" + import datetime + + dt = datetime.datetime(2024, 6, 15, 10, 30, 0) + result = _normalize_scalar(dt) + self.assertIsInstance(result, str) + self.assertEqual(result, "2024-06-15T10:30:00") + + def test_python_date(self): + """datetime.date is converted to ISO 8601 string.""" + import datetime + + d = datetime.date(2024, 6, 15) + result = _normalize_scalar(d) + self.assertIsInstance(result, str) + self.assertEqual(result, "2024-06-15") + + def test_numpy_datetime64(self): + """np.datetime64 is converted to ISO 8601 string.""" + dt = np.datetime64("2024-06-15T10:30:00") + result = _normalize_scalar(dt) + self.assertIsInstance(result, str) + self.assertIn("2024-06-15T10:30:00", result) + + +class TestDataframeToRecords(unittest.TestCase): + """Unit tests for dataframe_to_records().""" + + def test_basic(self): + """Basic DataFrame with string values is converted correctly.""" + df = pd.DataFrame([{"name": "Contoso", "city": "Seattle"}]) + result = dataframe_to_records(df) + self.assertEqual(result, [{"name": "Contoso", "city": "Seattle"}]) + + def test_nan_dropped(self): + """NaN values are omitted from records when na_as_null=False (default).""" + df = pd.DataFrame([{"name": "Contoso", "telephone1": None}]) + result = dataframe_to_records(df) + self.assertEqual(result, [{"name": "Contoso"}]) + self.assertNotIn("telephone1", result[0]) + + def test_nan_as_null(self): + """NaN values become None when na_as_null=True.""" + df = pd.DataFrame([{"name": "Contoso", "telephone1": None}]) + result = dataframe_to_records(df, na_as_null=True) + self.assertEqual(result, [{"name": "Contoso", "telephone1": None}]) + self.assertIn("telephone1", result[0]) + self.assertIsNone(result[0]["telephone1"]) + + def test_timestamp_conversion(self): + """pd.Timestamp values are converted to ISO 8601 strings.""" + ts = pd.Timestamp("2024-01-15 10:30:00") + df = pd.DataFrame([{"name": "Contoso", "createdon": ts}]) + result = dataframe_to_records(df) + self.assertEqual(result[0]["createdon"], "2024-01-15T10:30:00") + + def test_numpy_int(self): + """np.int64 values are converted to Python int.""" + df = pd.DataFrame([{"priority": np.int64(42)}]) + result = dataframe_to_records(df) + self.assertIsInstance(result[0]["priority"], int) + self.assertEqual(result[0]["priority"], 42) + + def test_numpy_float(self): + """np.float64 values are converted to Python float.""" + df = pd.DataFrame([{"score": np.float64(3.14)}]) + result = dataframe_to_records(df) + self.assertIsInstance(result[0]["score"], float) + self.assertAlmostEqual(result[0]["score"], 3.14) + + def test_numpy_bool(self): + """np.bool_ values are converted to Python bool.""" + df = pd.DataFrame([{"active": np.bool_(True)}]) + result = dataframe_to_records(df) + self.assertIsInstance(result[0]["active"], bool) + self.assertTrue(result[0]["active"]) + + def test_list_value(self): + """Cells containing lists pass through without raising ValueError.""" + df = pd.DataFrame([{"tags": ["a", "b", "c"]}]) + result = dataframe_to_records(df) + self.assertEqual(result[0]["tags"], ["a", "b", "c"]) + + def test_dict_value(self): + """Cells containing dicts pass through without raising ValueError.""" + df = pd.DataFrame([{"metadata": {"key": "value"}}]) + result = dataframe_to_records(df) + self.assertEqual(result[0]["metadata"], {"key": "value"}) + + def test_ndarray_converted_to_list(self): + """np.ndarray values are converted to Python lists for JSON serialization.""" + arr = np.array([1, 2, 3]) + df = pd.DataFrame([{"values": arr}]) + result = dataframe_to_records(df) + self.assertIsInstance(result[0]["values"], list) + self.assertEqual(result[0]["values"], [1, 2, 3]) + + def test_empty_dataframe(self): + """Empty DataFrame returns an empty list.""" + df = pd.DataFrame(columns=["name", "telephone1"]) + result = dataframe_to_records(df) + self.assertEqual(result, []) + + def test_mixed_types(self): + """DataFrame with mixed types (str, int, float, None, Timestamp) converts correctly.""" + ts = pd.Timestamp("2024-06-01") + df = pd.DataFrame( + [ + { + "name": "Contoso", + "count": np.int64(5), + "score": np.float64(9.8), + "active": np.bool_(True), + "createdon": ts, + "notes": None, + } + ] + ) + result = dataframe_to_records(df) + self.assertEqual(len(result), 1) + rec = result[0] + self.assertEqual(rec["name"], "Contoso") + self.assertIsInstance(rec["count"], int) + self.assertEqual(rec["count"], 5) + self.assertIsInstance(rec["score"], float) + self.assertAlmostEqual(rec["score"], 9.8) + self.assertIsInstance(rec["active"], bool) + self.assertTrue(rec["active"]) + self.assertEqual(rec["createdon"], "2024-06-01T00:00:00") + self.assertNotIn("notes", rec) + + def test_timezone_aware_timestamp(self): + """Timezone-aware Timestamp in DataFrame is converted to ISO string with tz.""" + ts = pd.Timestamp("2024-06-15 10:30:00", tz="US/Eastern") + df = pd.DataFrame([{"createdon": ts}]) + result = dataframe_to_records(df) + self.assertIn("2024-06-15T10:30:00", result[0]["createdon"]) + self.assertIsInstance(result[0]["createdon"], str) + + def test_multiple_rows_some_with_nan(self): + """Multi-row DataFrame with mixed NaN positions drops correct keys per row.""" + df = pd.DataFrame( + [ + {"name": "A", "phone": "555-0100", "city": None}, + {"name": "B", "phone": None, "city": "Seattle"}, + {"name": None, "phone": "555-0300", "city": "Portland"}, + ] + ) + result = dataframe_to_records(df) + self.assertEqual(result[0], {"name": "A", "phone": "555-0100"}) + self.assertEqual(result[1], {"name": "B", "city": "Seattle"}) + self.assertEqual(result[2], {"phone": "555-0300", "city": "Portland"}) + + def test_multiple_rows_na_as_null(self): + """Multi-row DataFrame with na_as_null=True includes None for all missing values.""" + df = pd.DataFrame( + [ + {"name": "A", "phone": None}, + {"name": None, "phone": "555-0200"}, + ] + ) + result = dataframe_to_records(df, na_as_null=True) + self.assertEqual(result[0], {"name": "A", "phone": None}) + self.assertEqual(result[1], {"name": None, "phone": "555-0200"}) + + def test_empty_string_preserved(self): + """Empty string is kept in output, not treated as missing.""" + df = pd.DataFrame([{"name": ""}]) + result = dataframe_to_records(df) + self.assertIn("name", result[0]) + self.assertEqual(result[0]["name"], "") + + def test_zero_and_false_preserved(self): + """Zero and False are kept in output, not treated as missing.""" + df = pd.DataFrame([{"count": 0, "score": 0.0, "active": False}]) + result = dataframe_to_records(df) + self.assertEqual(result[0]["count"], 0) + self.assertEqual(result[0]["score"], 0.0) + self.assertIs(result[0]["active"], False) + + def test_pd_na_nullable_int(self): + """pd.NA in nullable Int64 column is dropped by default.""" + df = pd.DataFrame({"val": pd.array([1, pd.NA], dtype="Int64")}) + result = dataframe_to_records(df) + self.assertEqual(result[0]["val"], 1) + self.assertNotIn("val", result[1]) + + def test_pd_na_nullable_int_as_null(self): + """pd.NA in nullable Int64 column becomes None with na_as_null=True.""" + df = pd.DataFrame({"val": pd.array([1, pd.NA], dtype="Int64")}) + result = dataframe_to_records(df, na_as_null=True) + self.assertEqual(result[0]["val"], 1) + self.assertIsNone(result[1]["val"]) + + def test_datetime_in_dataframe(self): + """datetime.datetime values in a DataFrame are converted to ISO strings.""" + import datetime + + dt = datetime.datetime(2024, 6, 15, 10, 30) + df = pd.DataFrame([{"createdon": dt}]) + result = dataframe_to_records(df) + self.assertIsInstance(result[0]["createdon"], str) + self.assertIn("2024-06-15", result[0]["createdon"]) + + def test_literal_nan_string(self): + """Literal string 'NaN' is preserved, not treated as missing.""" + df = pd.DataFrame([{"name": "NaN"}]) + result = dataframe_to_records(df) + self.assertEqual(result[0]["name"], "NaN") + + +if __name__ == "__main__": + unittest.main() From ddab5f854a7e76f3ecf22023ddee49002d2325ab Mon Sep 17 00:00:00 2001 From: abelmilash-msft Date: Tue, 17 Mar 2026 14:44:29 -0700 Subject: [PATCH 12/20] Update CHANGELOG.md for v0.1.0b7 release (#150) Updates CHANGELOG.md for v0.1.0b7 release Co-authored-by: Abel Milash --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91918496..040184a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.1.0b7] - 2026-03-17 + +### Added +- DataFrame namespace: `client.dataframe.get()`, `.create()`, `.update()`, `.delete()` for working with Dataverse records as pandas DataFrames and Series — no manual dict conversion required (#98) +- Table metadata now includes `primary_name_attribute` and `primary_id_attribute` from `tables.create()` and `tables.get_info()` (#148) + +### Changed +- `pandas>=2.0.0` is now a required dependency (#98) + ## [0.1.0b6] - 2026-03-12 ### Added @@ -82,6 +91,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Comprehensive error handling with specific exception types (`DataverseError`, `AuthenticationError`, etc.) (#22, #24) - HTTP retry logic with exponential backoff for resilient operations (#72) +[0.1.0b7]: https://github.com/microsoft/PowerPlatform-DataverseClient-Python/compare/v0.1.0b6...v0.1.0b7 [0.1.0b6]: https://github.com/microsoft/PowerPlatform-DataverseClient-Python/compare/v0.1.0b5...v0.1.0b6 [0.1.0b5]: https://github.com/microsoft/PowerPlatform-DataverseClient-Python/compare/v0.1.0b4...v0.1.0b5 [0.1.0b4]: https://github.com/microsoft/PowerPlatform-DataverseClient-Python/compare/v0.1.0b3...v0.1.0b4 From eebee60967f5f379578c544aa6b86bea39af4235 Mon Sep 17 00:00:00 2001 From: abelmilash-msft Date: Tue, 17 Mar 2026 17:11:26 -0700 Subject: [PATCH 13/20] Bump version to 0.1.0b8 for next development cycle (#151) Bumps version to 0.1.0b8 for next development cycle Co-authored-by: Abel Milash --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5c06f5ad..1206ad5d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "PowerPlatform-Dataverse-Client" -version = "0.1.0b7" +version = "0.1.0b8" description = "Python SDK for Microsoft Dataverse" readme = {file = "README.md", content-type = "text/markdown"} authors = [{name = "Microsoft Corporation"}] From 19e11c5ebba100a95c384f4b0421e45313e94f35 Mon Sep 17 00:00:00 2001 From: Saurabh Ravindra Badenkal <32964911+saurabhrb@users.noreply.github.com> Date: Wed, 18 Mar 2026 14:24:20 -0700 Subject: [PATCH 14/20] Fix docstring type annotations for Microsoft Learn compatibility (#153) ## Summary Fix broken cross-references (``, ``, ``) in the Microsoft Learn API reference docs caused by Sphinx-style `:type:` and `:rtype:` directives. ## Problem The Learn doc pipeline processes `:type:` and `:rtype:` Sphinx directives differently from standard Sphinx. Every word between `:class:` back-tick references is treated as a separate cross-reference. For example: ``` :rtype: :class:`list` of :class:`str` ``` Becomes: ```yaml types: - ``` There is no type `of` in the Learn cross-reference database, so it renders as a broken link on the published page. ## Root Cause This was introduced in commit f0e8987 ("sphinx doc string", 2025-11-17) which converted the original bracket-notation docstrings (`list[str]`, `dict or list[dict]`) to Sphinx-style `:class:` syntax. Later commits that added new APIs (operation namespaces, dataframe, etc.) perpetuated the same broken pattern. ## Fix Replaced all 42 occurrences across 6 source files with Python bracket notation that the Learn pipeline handles correctly: | Before (broken) | After (correct) | |---|---| | `:class:\`list\` of :class:\`str\`` | `list[str]` | | `:class:\`dict\` or :class:\`list\` of :class:\`dict\`` | `dict or list[dict]` | | `:class:\`collections.abc.Iterable\` of :class:\`list\` of :class:\`dict\`` | `collections.abc.Iterable[list[dict]]` | | `:class:\`dict\` mapping :class:\`str\` to :class:\`typing.Any\`` | `dict[str, typing.Any]` | ### Files changed **Docstring fixes:** - `src/PowerPlatform/Dataverse/client.py` (14 occurrences) - `src/PowerPlatform/Dataverse/operations/records.py` (14 occurrences) - `src/PowerPlatform/Dataverse/operations/tables.py` (7 occurrences) - `src/PowerPlatform/Dataverse/operations/dataframe.py` (3 occurrences) - `src/PowerPlatform/Dataverse/operations/query.py` (1 occurrence) - `src/PowerPlatform/Dataverse/models/table_info.py` (3 occurrences) **Prevention guidelines:** - `.claude/skills/dataverse-sdk-dev/SKILL.md` -- added "Docstring Type Annotations (Microsoft Learn Compatibility)" section - `src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-dev/SKILL.md` -- same section (kept both copies in sync) ## Testing - All 398 unit tests pass - Verified zero remaining occurrences of the broken pattern via regex scan --------- Co-authored-by: Saurabh Badenkal --- .claude/skills/dataverse-sdk-dev/SKILL.md | 32 ++++++++++++++++ CONTRIBUTING.md | 37 +++++++++++++++++++ .../claude_skill/dataverse-sdk-dev/SKILL.md | 32 ++++++++++++++++ src/PowerPlatform/Dataverse/client.py | 32 ++++++++-------- .../Dataverse/models/table_info.py | 6 +-- .../Dataverse/operations/dataframe.py | 6 +-- .../Dataverse/operations/query.py | 3 +- .../Dataverse/operations/records.py | 30 +++++++-------- .../Dataverse/operations/tables.py | 14 +++---- 9 files changed, 145 insertions(+), 47 deletions(-) diff --git a/.claude/skills/dataverse-sdk-dev/SKILL.md b/.claude/skills/dataverse-sdk-dev/SKILL.md index c20a778d..63af04dc 100644 --- a/.claude/skills/dataverse-sdk-dev/SKILL.md +++ b/.claude/skills/dataverse-sdk-dev/SKILL.md @@ -54,3 +54,35 @@ Navigation property names are case-sensitive and must match the entity's `$metad 9. **Document public APIs** - Add Sphinx-style docstrings with examples for public methods 10. **Define __all__ in module files** - Each module declares its own exports via `__all__` (e.g., `errors.py` defines `__all__ = ["HttpError", ...]`). Package `__init__.py` files should not re-export or redefine another module's `__all__`; they use `__all__ = []` to indicate no star-import exports. 11. **Run black before committing** - Always run `python -m black ` before committing. CI will reject unformatted code. Config is in `pyproject.toml` under `[tool.black]`. + +### Docstring Type Annotations (Microsoft Learn Compatibility) + +This SDK's API reference is published on Microsoft Learn. The Learn doc pipeline parses `:type:` and `:rtype:` directives differently from standard Sphinx -- every word between `:class:` references is treated as a separate cross-reference (``). Using Sphinx-style `:class:\`list\` of :class:\`str\`` produces broken `` links on Learn. + +**Rules for `:type:` and `:rtype:` directives:** + +- Use Python bracket notation for generic types: `list[str]`, `dict[str, typing.Any]`, `list[dict]` +- Use `or` (without `:class:`) for union types: `str or None`, `dict or list[dict]` +- Use bracket nesting for complex types: `collections.abc.Iterable[list[dict]]` +- Use `~` prefix for SDK types to show short name: `list[~PowerPlatform.Dataverse.models.record.Record]` +- `:class:` is fine for single standalone types: `:class:\`str\``, `:class:\`bool\`` + +**Never** use `:class:\`X\` of :class:\`Y\`` or `:class:\`X\` mapping :class:\`Y\` to :class:\`Z\`` -- the words `of`, `mapping`, `to` become broken `` links. + +**Correct examples:** + +```rst +:type data: dict or list[dict] +:rtype: list[str] +:rtype: collections.abc.Iterable[list[~PowerPlatform.Dataverse.models.record.Record]] +:type select: list[str] or None +:type columns: dict[str, typing.Any] +``` + +**Wrong examples (NEVER use):** + +```rst +:type data: :class:`dict` or :class:`list` of :class:`dict` +:rtype: :class:`list` of :class:`str` +:type columns: :class:`dict` mapping :class:`str` to :class:`typing.Any` +``` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d09ba322..ecb8ccfe 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -121,4 +121,41 @@ published release: # After publishing v0.1.0b4, bump to v0.1.0b5 on main # Update version in pyproject.toml # Commit directly to main: "Bump version to 0.1.0b5 for next development cycle" +``` + +### Docstring Type Annotations (Microsoft Learn Compatibility) + +This SDK's API reference is published on [Microsoft Learn](https://learn.microsoft.com). The Learn doc pipeline processes `:type:` and `:rtype:` Sphinx directives differently from standard Sphinx -- every word between `:class:` back-tick references is treated as a separate cross-reference (``). For example: + +``` +:rtype: :class:`list` of :class:`str` +``` + +This produces a broken `` link because `of` is not a valid type. + +**Rules for `:type:` and `:rtype:` directives:** + +- Use **Python bracket notation** for generic types: `list[str]`, `dict[str, typing.Any]`, `list[dict]` +- Use **`or`** (without `:class:`) for union types: `str or None`, `dict or list[dict]` +- Use **bracket nesting** for complex types: `collections.abc.Iterable[list[dict]]` +- `:class:` is fine for **single standalone types**: `` :class:`str` ``, `` :class:`bool` `` + +**NEVER** use the following patterns -- the connector words (`of`, `mapping`, `to`) become broken `` links on Learn: + +``` +:class:`X` of :class:`Y` +:class:`X` mapping :class:`Y` to :class:`Z` +``` + +Correct: +``` +:type data: dict or list[dict] +:rtype: list[str] +:type select: list[str] or None +``` + +Wrong: +``` +:type data: :class:`dict` or :class:`list` of :class:`dict` +:rtype: :class:`list` of :class:`str` ``` \ No newline at end of file diff --git a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-dev/SKILL.md b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-dev/SKILL.md index 2ca7b5f8..f26efbb7 100644 --- a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-dev/SKILL.md +++ b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-dev/SKILL.md @@ -28,3 +28,35 @@ This skill provides guidance for developers working on the PowerPlatform Dataver 9. **Document public APIs** - Add Sphinx-style docstrings with examples for public methods 10. **Define __all__ in module files** - Each module declares its own exports via `__all__` (e.g., `errors.py` defines `__all__ = ["HttpError", ...]`). Package `__init__.py` files should not re-export or redefine another module's `__all__`; they use `__all__ = []` to indicate no star-import exports. 11. **Run black before committing** - Always run `python -m black ` before committing. CI will reject unformatted code. Config is in `pyproject.toml` under `[tool.black]`. + +### Docstring Type Annotations (Microsoft Learn Compatibility) + +This SDK's API reference is published on Microsoft Learn. The Learn doc pipeline parses `:type:` and `:rtype:` directives differently from standard Sphinx -- every word between `:class:` references is treated as a separate cross-reference (``). Using Sphinx-style `:class:\`list\` of :class:\`str\`` produces broken `` links on Learn. + +**Rules for `:type:` and `:rtype:` directives:** + +- Use Python bracket notation for generic types: `list[str]`, `dict[str, typing.Any]`, `list[dict]` +- Use `or` (without `:class:`) for union types: `str or None`, `dict or list[dict]` +- Use bracket nesting for complex types: `collections.abc.Iterable[list[dict]]` +- Use `~` prefix for SDK types to show short name: `list[~PowerPlatform.Dataverse.models.record.Record]` +- `:class:` is fine for single standalone types: `:class:\`str\``, `:class:\`bool\`` + +**Never** use `:class:\`X\` of :class:\`Y\`` or `:class:\`X\` mapping :class:\`Y\` to :class:\`Z\`` -- the words `of`, `mapping`, `to` become broken `` links. + +**Correct examples:** + +```rst +:type data: dict or list[dict] +:rtype: list[str] +:rtype: collections.abc.Iterable[list[~PowerPlatform.Dataverse.models.record.Record]] +:type select: list[str] or None +:type columns: dict[str, typing.Any] +``` + +**Wrong examples (NEVER use):** + +```rst +:type data: :class:`dict` or :class:`list` of :class:`dict` +:rtype: :class:`list` of :class:`str` +:type columns: :class:`dict` mapping :class:`str` to :class:`typing.Any` +``` diff --git a/src/PowerPlatform/Dataverse/client.py b/src/PowerPlatform/Dataverse/client.py index 402be881..e0f9c6e9 100644 --- a/src/PowerPlatform/Dataverse/client.py +++ b/src/PowerPlatform/Dataverse/client.py @@ -208,10 +208,10 @@ def create(self, table_schema_name: str, records: Union[Dict[str, Any], List[Dic :type table_schema_name: :class:`str` :param records: A single record dictionary or a list of record dictionaries. Each dictionary should contain column schema names as keys. - :type records: :class:`dict` or :class:`list` of :class:`dict` + :type records: dict or list[dict] :return: List of created record GUIDs. Returns a single-element list for a single input. - :rtype: :class:`list` of :class:`str` + :rtype: list[str] :raises TypeError: If ``records`` is not a dict or list[dict], or if the internal client returns an unexpected type. @@ -260,12 +260,12 @@ def update( :param table_schema_name: Schema name of the table (e.g. ``"account"`` or ``"new_MyTestTable"``). :type table_schema_name: :class:`str` :param ids: Single GUID string or list of GUID strings to update. - :type ids: :class:`str` or :class:`list` of :class:`str` + :type ids: str or list[str] :param changes: Dictionary of changes for single/broadcast mode, or list of dictionaries for paired mode. When ``ids`` is a list and ``changes`` is a single dict, the same changes are broadcast to all records. When both are lists, they must have equal length for one-to-one mapping. - :type changes: :class:`dict` or :class:`list` of :class:`dict` + :type changes: dict or list[dict] :raises TypeError: If ``ids`` is not str or list[str], or if ``changes`` type doesn't match usage pattern. @@ -312,7 +312,7 @@ def delete( :param table_schema_name: Schema name of the table (e.g. ``"account"`` or ``"new_MyTestTable"``). :type table_schema_name: :class:`str` :param ids: Single GUID string or list of GUID strings to delete. - :type ids: :class:`str` or :class:`list` of :class:`str` + :type ids: str or list[str] :param use_bulk_delete: When ``True`` (default) and ``ids`` is a list, execute the BulkDelete action and return its async job identifier. When ``False`` each record is deleted sequentially. :type use_bulk_delete: :class:`bool` @@ -367,21 +367,21 @@ def get( :param record_id: Optional GUID to fetch a specific record. If None, queries multiple records. :type record_id: :class:`str` or None :param select: Optional list of attribute logical names to retrieve. Column names are case-insensitive and automatically lowercased (e.g. ``["new_Title", "new_Amount"]`` becomes ``"new_title,new_amount"``). - :type select: :class:`list` of :class:`str` or None + :type select: list[str] or None :param filter: Optional OData filter string, e.g. ``"name eq 'Contoso'"`` or ``"new_quantity gt 5"``. Column names in filter expressions must use exact lowercase logical names (e.g. ``"new_quantity"``, not ``"new_Quantity"``). The filter string is passed directly to the Dataverse Web API without transformation. :type filter: :class:`str` or None :param orderby: Optional list of attributes to sort by, e.g. ``["name asc", "createdon desc"]``. Column names are automatically lowercased. - :type orderby: :class:`list` of :class:`str` or None + :type orderby: list[str] or None :param top: Optional maximum number of records to return. :type top: :class:`int` or None :param expand: Optional list of navigation properties to expand, e.g. ``["primarycontactid"]``. Navigation property names are case-sensitive and must match the server-defined names exactly. These are NOT automatically transformed. Consult entity metadata for correct casing. - :type expand: :class:`list` of :class:`str` or None + :type expand: list[str] or None :param page_size: Optional number of records per page for pagination. :type page_size: :class:`int` or None :return: Single record dict if ``record_id`` is provided, otherwise a generator yielding lists of record dictionaries (one list per page). - :rtype: :class:`dict` or :class:`collections.abc.Iterable` of :class:`list` of :class:`dict` + :rtype: dict or collections.abc.Iterable[list[dict]] :raises TypeError: If ``record_id`` is provided but not a string. @@ -456,7 +456,7 @@ def query_sql(self, sql: str) -> List[Dict[str, Any]]: :type sql: :class:`str` :return: List of result row dictionaries. Returns an empty list if no rows match. - :rtype: :class:`list` of :class:`dict` + :rtype: list[dict] :raises ~PowerPlatform.Dataverse.core.errors.SQLParseError: If the SQL query uses unsupported syntax. :raises ~PowerPlatform.Dataverse.core.errors.HttpError: If the Web API returns an error. @@ -545,7 +545,7 @@ class ItemStatus(IntEnum): 1036: {"Active": "Actif", "Inactive": "Inactif"} } - :type columns: :class:`dict` mapping :class:`str` to :class:`typing.Any` + :type columns: dict[str, typing.Any] :param solution_unique_name: Optional solution unique name that should own the new table. When omitted the table is created in the default solution. :type solution_unique_name: :class:`str` or None :param primary_column_schema_name: Optional primary name column schema name with customization prefix value (e.g. ``"new_MyTestTable"``). If not provided, defaults to ``"{customization prefix value}_Name"``. @@ -634,7 +634,7 @@ def list_tables(self) -> list[dict[str, Any]]: List all non-private tables in the Dataverse environment. :return: List of EntityDefinition metadata dictionaries. - :rtype: :class:`list` of :class:`dict` + :rtype: list[dict] Example: List all non-private tables and print their logical names:: @@ -666,9 +666,9 @@ def create_columns( :param columns: Mapping of column schema names (with customization prefix value) to supported types. All custom column names must include the customization prefix value** (e.g. ``"new_Notes"``). Primitive types include ``"string"`` (alias: ``"text"``), ``"int"`` (alias: ``"integer"``), ``"decimal"`` (alias: ``"money"``), ``"float"`` (alias: ``"double"``), ``"datetime"`` (alias: ``"date"``), ``"bool"`` (alias: ``"boolean"``), and ``"file"``. Enum subclasses (IntEnum preferred) generate a local option set and can specify localized labels via ``__labels__``. - :type columns: :class:`dict` mapping :class:`str` to :class:`typing.Any` + :type columns: dict[str, typing.Any] :returns: Schema names for the columns that were created. - :rtype: :class:`list` of :class:`str` + :rtype: list[str] Example: Create multiple columns on the custom table:: @@ -703,9 +703,9 @@ def delete_columns( :param table_schema_name: Schema name of the table (e.g. ``"new_MyTestTable"``). :type table_schema_name: :class:`str` :param columns: Column name or list of column names to remove. Must include customization prefix value (e.g. ``"new_TestColumn"``). - :type columns: :class:`str` or :class:`list` of :class:`str` + :type columns: str or list[str] :returns: Schema names for the columns that were removed. - :rtype: :class:`list` of :class:`str` + :rtype: list[str] Example: Remove two custom columns by schema name: diff --git a/src/PowerPlatform/Dataverse/models/table_info.py b/src/PowerPlatform/Dataverse/models/table_info.py index 842fe2b6..a4813044 100644 --- a/src/PowerPlatform/Dataverse/models/table_info.py +++ b/src/PowerPlatform/Dataverse/models/table_info.py @@ -97,9 +97,9 @@ class TableInfo: :param description: Table description. :type description: :class:`str` or None :param columns: Column metadata (when retrieved). - :type columns: :class:`list` of :class:`ColumnInfo` or None + :type columns: list[ColumnInfo] or None :param columns_created: Column schema names created with the table. - :type columns_created: :class:`list` of :class:`str` or None + :type columns_created: list[str] or None Example:: @@ -241,7 +241,7 @@ class AlternateKeyInfo: :param schema_name: Key schema name. :type schema_name: :class:`str` :param key_attributes: List of column logical names that compose the key. - :type key_attributes: :class:`list` of :class:`str` + :type key_attributes: list[str] :param status: Index creation status (``"Active"``, ``"Pending"``, ``"InProgress"``, ``"Failed"``). :type status: :class:`str` """ diff --git a/src/PowerPlatform/Dataverse/operations/dataframe.py b/src/PowerPlatform/Dataverse/operations/dataframe.py index 3aec0cc7..44843b54 100644 --- a/src/PowerPlatform/Dataverse/operations/dataframe.py +++ b/src/PowerPlatform/Dataverse/operations/dataframe.py @@ -75,15 +75,15 @@ def get( :param record_id: Optional GUID to fetch a specific record. If None, queries multiple records. :type record_id: :class:`str` or None :param select: Optional list of attribute logical names to retrieve. - :type select: :class:`list` of :class:`str` or None + :type select: list[str] or None :param filter: Optional OData filter string. Column names must use exact lowercase logical names. :type filter: :class:`str` or None :param orderby: Optional list of attributes to sort by. - :type orderby: :class:`list` of :class:`str` or None + :type orderby: list[str] or None :param top: Optional maximum number of records to return. :type top: :class:`int` or None :param expand: Optional list of navigation properties to expand (case-sensitive). - :type expand: :class:`list` of :class:`str` or None + :type expand: list[str] or None :param page_size: Optional number of records per page for pagination. :type page_size: :class:`int` or None diff --git a/src/PowerPlatform/Dataverse/operations/query.py b/src/PowerPlatform/Dataverse/operations/query.py index 73f8f540..158ff229 100644 --- a/src/PowerPlatform/Dataverse/operations/query.py +++ b/src/PowerPlatform/Dataverse/operations/query.py @@ -51,8 +51,7 @@ def sql(self, sql: str) -> List[Record]: :return: List of :class:`~PowerPlatform.Dataverse.models.record.Record` objects. Returns an empty list when no rows match. - :rtype: :class:`list` of - :class:`~PowerPlatform.Dataverse.models.record.Record` + :rtype: list[~PowerPlatform.Dataverse.models.record.Record] :raises ~PowerPlatform.Dataverse.core.errors.ValidationError: If ``sql`` is not a string or is empty. diff --git a/src/PowerPlatform/Dataverse/operations/records.py b/src/PowerPlatform/Dataverse/operations/records.py index 03363de3..13ad753c 100644 --- a/src/PowerPlatform/Dataverse/operations/records.py +++ b/src/PowerPlatform/Dataverse/operations/records.py @@ -69,11 +69,11 @@ def create( :type table: :class:`str` :param data: A single record dictionary or a list of record dictionaries. Each dictionary maps column schema names to values. - :type data: :class:`dict` or :class:`list` of :class:`dict` + :type data: dict or list[dict] :return: A single GUID string for a single record, or a list of GUID strings for bulk creation. - :rtype: :class:`str` or :class:`list` of :class:`str` + :rtype: str or list[str] :raises TypeError: If ``data`` is not a dict or list[dict]. @@ -127,10 +127,10 @@ def update( :param table: Schema name of the table (e.g. ``"account"``). :type table: :class:`str` :param ids: A single GUID string, or a list of GUID strings. - :type ids: :class:`str` or :class:`list` of :class:`str` + :type ids: str or list[str] :param changes: A dictionary of field changes (single/broadcast), or a list of dictionaries (paired, one per ID). - :type changes: :class:`dict` or :class:`list` of :class:`dict` + :type changes: dict or list[dict] :raises TypeError: If ``ids`` is not str or list[str], or if ``changes`` does not match the expected pattern. @@ -187,7 +187,7 @@ def delete( :param table: Schema name of the table (e.g. ``"account"``). :type table: :class:`str` :param ids: A single GUID string, or a list of GUID strings. - :type ids: :class:`str` or :class:`list` of :class:`str` + :type ids: str or list[str] :param use_bulk_delete: When True (default) and ``ids`` is a list, use the BulkDelete action and return its async job ID. When False, delete records one at a time. @@ -241,7 +241,7 @@ def get( :type record_id: :class:`str` :param select: Optional list of column logical names to include in the response. - :type select: :class:`list` of :class:`str` or None + :type select: list[str] or None :return: Typed record with dict-like access for backward compatibility. :rtype: :class:`~PowerPlatform.Dataverse.models.record.Record` @@ -282,7 +282,7 @@ def get( :type table: :class:`str` :param select: Optional list of column logical names to include. Column names are automatically lowercased. - :type select: :class:`list` of :class:`str` or None + :type select: list[str] or None :param filter: Optional OData ``$filter`` expression (e.g. ``"name eq 'Contoso'"``). Column names in filter expressions must use exact lowercase logical names. @@ -290,21 +290,20 @@ def get( :param orderby: Optional list of sort expressions (e.g. ``["name asc", "createdon desc"]``). Column names are automatically lowercased. - :type orderby: :class:`list` of :class:`str` or None + :type orderby: list[str] or None :param top: Optional maximum total number of records to return. :type top: :class:`int` or None :param expand: Optional list of navigation properties to expand (e.g. ``["primarycontactid"]``). Case-sensitive; must match server-defined names exactly. - :type expand: :class:`list` of :class:`str` or None + :type expand: list[str] or None :param page_size: Optional per-page size hint sent via ``Prefer: odata.maxpagesize``. :type page_size: :class:`int` or None :return: Generator yielding pages, where each page is a list of :class:`~PowerPlatform.Dataverse.models.record.Record` objects. - :rtype: :class:`collections.abc.Iterable` of :class:`list` of - :class:`~PowerPlatform.Dataverse.models.record.Record` + :rtype: collections.abc.Iterable[list[~PowerPlatform.Dataverse.models.record.Record]] Example: Fetch with filtering and pagination:: @@ -356,7 +355,7 @@ def get( :type record_id: :class:`str` or None :param select: Optional list of column logical names to include. Column names are automatically lowercased. - :type select: :class:`list` of :class:`str` or None + :type select: list[str] or None :param filter: Optional OData ``$filter`` expression (e.g. ``"name eq 'Contoso'"``). Column names in filter expressions must use exact lowercase logical names. Only used for multi-record @@ -365,14 +364,14 @@ def get( :param orderby: Optional list of sort expressions (e.g. ``["name asc", "createdon desc"]``). Column names are automatically lowercased. Only used for multi-record queries. - :type orderby: :class:`list` of :class:`str` or None + :type orderby: list[str] or None :param top: Optional maximum total number of records to return. Only used for multi-record queries. :type top: :class:`int` or None :param expand: Optional list of navigation properties to expand (e.g. ``["primarycontactid"]``). Case-sensitive; must match server-defined names exactly. Only used for multi-record queries. - :type expand: :class:`list` of :class:`str` or None + :type expand: list[str] or None :param page_size: Optional per-page size hint sent via ``Prefer: odata.maxpagesize``. Only used for multi-record queries. :type page_size: :class:`int` or None @@ -380,8 +379,7 @@ def get( :return: A single record dict when ``record_id`` is provided, or a generator yielding pages (lists of record dicts) when fetching multiple records. - :rtype: :class:`dict` or :class:`collections.abc.Iterable` of - :class:`list` of :class:`dict` + :rtype: dict or collections.abc.Iterable[list[dict]] :raises TypeError: If ``record_id`` is provided but not a string. :raises ValueError: If query parameters are provided alongside diff --git a/src/PowerPlatform/Dataverse/operations/tables.py b/src/PowerPlatform/Dataverse/operations/tables.py index 42c4dd8b..729e0eba 100644 --- a/src/PowerPlatform/Dataverse/operations/tables.py +++ b/src/PowerPlatform/Dataverse/operations/tables.py @@ -213,10 +213,10 @@ def list( ``["LogicalName", "SchemaName", "DisplayName"]``). When ``None`` (the default) or an empty list, all properties are returned. - :type select: :class:`list` of :class:`str` or None + :type select: list[str] or None :return: List of EntityDefinition metadata dictionaries. - :rtype: :class:`list` of :class:`dict` + :rtype: list[dict] Example:: @@ -255,7 +255,7 @@ def add_columns( :type columns: :class:`dict` :return: Schema names of the columns that were created. - :rtype: :class:`list` of :class:`str` + :rtype: list[str] :raises ~PowerPlatform.Dataverse.core.errors.MetadataError: If the table does not exist. @@ -285,10 +285,10 @@ def remove_columns( :param columns: Column schema name or list of column schema names to remove. Must include the customization prefix (e.g. ``"new_TestColumn"``). - :type columns: :class:`str` or :class:`list` of :class:`str` + :type columns: str or list[str] :return: Schema names of the columns that were removed. - :rtype: :class:`list` of :class:`str` + :rtype: list[str] :raises ~PowerPlatform.Dataverse.core.errors.MetadataError: If the table or a specified column does not exist. @@ -612,7 +612,7 @@ def create_alternate_key( :type key_name: :class:`str` :param columns: List of column logical names that compose the key (e.g. ``["new_productcode"]``). - :type columns: :class:`list` of :class:`str` + :type columns: list[str] :param display_name: Display name for the key. Defaults to ``key_name`` if not provided. :type display_name: :class:`str` or None @@ -660,7 +660,7 @@ def get_alternate_keys(self, table: str) -> List[AlternateKeyInfo]: :return: List of alternate key metadata objects. May be empty if no alternate keys are defined. - :rtype: :class:`list` of :class:`~PowerPlatform.Dataverse.models.table_info.AlternateKeyInfo` + :rtype: list[~PowerPlatform.Dataverse.models.table_info.AlternateKeyInfo] :raises ~PowerPlatform.Dataverse.core.errors.MetadataError: If the table does not exist. From 9788cbb18f84a53cc32aa88d13732c4274784f27 Mon Sep 17 00:00:00 2001 From: Saurabh Ravindra Badenkal <32964911+saurabhrb@users.noreply.github.com> Date: Wed, 18 Mar 2026 16:32:59 -0700 Subject: [PATCH 15/20] Add e2e relationship tests for pre-GA validation (#152) ## Summary Add end-to-end relationship tests that validate the full relationship API lifecycle against a live Dataverse environment. This is the primary pre-GA validation for Tim's relationship PRs (#88, #105, #114). ## Changes ### New: `tests/e2e/test_relationships_e2e.py` 11 curated e2e tests covering: | Test Class | Tests | Coverage | |---|---|---| | `TestOneToManyCore` | 1 | Full 1:N lifecycle: create, get, delete, field assertions | | `TestLookupField` | 1 | Convenience `create_lookup_field` to system table | | `TestManyToMany` | 2 | N:N lifecycle + nonexistent returns None | | `TestDataThroughRelationships` | 4 | `@odata.bind`, `$expand`, `$filter` on lookup, update binding | | `TestCascadeBehaviors` | 2 | Restrict blocks delete; Cascade deletes children | | `TestTypeDetection` | 1 | `get_relationship` distinguishes 1:N vs N:N | ### Updated: `examples/basic/functional_testing.py` - Added relationship testing section covering 1:N core API, convenience API, N:N, get, and delete - Added relationship imports and retry helpers ### Updated: `pyproject.toml` - Added `[tool.pytest.ini_options]` with `testpaths = ["tests/unit"]` - Default `pytest` runs only unit tests; e2e tests require explicit invocation ## How to run e2e tests ```bash # Set your Dataverse org URL export DATAVERSE_URL=https://yourorg.crm.dynamics.com # Run relationship e2e tests pytest tests/e2e/ -v -s ``` The tests authenticate via `InteractiveBrowserCredential` and create/delete temporary tables (prefixed `test_E2E*`). ## E2E Test Results (from `.scratch/` comprehensive suite) Ran 30 tests against `https://aurorabapenv71aff.crm10.dynamics.com`: - 25/30 passed on first run - 5 failures were test bugs (not SDK bugs), all fixed: - Metadata propagation timing (increased retries) - Navigation property name casing (`$expand` needs server-assigned nav prop) - `IsValidForAdvancedFind` requires `BooleanManagedProperty` complex type ## Finding: SDK inconsistency to address before GA `create_one_to_many_relationship()` returns `lookup_schema_name` as the user-provided SchemaName, but `$expand` requires the server-assigned `ReferencingEntityNavigationPropertyName` (which may differ in casing). The e2e tests work around this by calling `get_relationship()` after create to get the correct nav prop name. This should be harmonized before GA. ## Checklist - [x] 398 unit tests pass - [x] 11 e2e tests collected by pytest - [x] Default `pytest` excludes e2e (runs unit only) - [x] Code formatted with black - [x] Branch rebased on origin/main --------- Co-authored-by: Saurabh Badenkal --- examples/basic/functional_testing.py | 284 ++++++++++ pyproject.toml | 8 + tests/e2e/__init__.py | 2 + tests/e2e/test_relationships_e2e.py | 761 +++++++++++++++++++++++++++ 4 files changed, 1055 insertions(+) create mode 100644 tests/e2e/__init__.py create mode 100644 tests/e2e/test_relationships_e2e.py diff --git a/examples/basic/functional_testing.py b/examples/basic/functional_testing.py index d8895a62..63f65aa6 100644 --- a/examples/basic/functional_testing.py +++ b/examples/basic/functional_testing.py @@ -32,6 +32,17 @@ # Import SDK components (assumes installation is already validated) from PowerPlatform.Dataverse.client import DataverseClient from PowerPlatform.Dataverse.core.errors import HttpError, MetadataError +from PowerPlatform.Dataverse.models.relationship import ( + LookupAttributeMetadata, + OneToManyRelationshipMetadata, + ManyToManyRelationshipMetadata, + CascadeConfiguration, +) +from PowerPlatform.Dataverse.models.labels import Label, LocalizedLabel +from PowerPlatform.Dataverse.common.constants import ( + CASCADE_BEHAVIOR_NO_CASCADE, + CASCADE_BEHAVIOR_REMOVE_LINK, +) from azure.identity import InteractiveBrowserCredential @@ -380,6 +391,274 @@ def cleanup_test_data(client: DataverseClient, table_info: Dict[str, Any], recor print("Test table kept for future testing") +def backoff(op, *, delays=(0, 2, 5, 10, 20, 20)): + """Retry helper with exponential backoff for metadata propagation delays.""" + last = None + total_delay = 0 + attempts = 0 + for d in delays: + if d: + time.sleep(d) + total_delay += d + attempts += 1 + try: + result = op() + if attempts > 1: + print(f" * Backoff succeeded after {attempts - 1} retry(s); waited {total_delay}s total.") + return result + except Exception as ex: + last = ex + continue + if last: + if attempts: + print(f" [WARN] Backoff exhausted after {max(attempts - 1, 0)} retry(s); waited {total_delay}s total.") + raise last + + +def test_relationships(client: DataverseClient) -> None: + """Test relationship lifecycle: create tables, 1:N, N:N, query, delete.""" + print("\n-> Relationship Tests") + print("=" * 50) + + rel_parent_schema = "test_RelParent" + rel_child_schema = "test_RelChild" + rel_m2m_schema = "test_RelProject" + + # Track IDs for cleanup + rel_id_1n = None + rel_id_lookup = None + rel_id_nn = None + created_tables = [] + + try: + # --- Cleanup any leftover resources from previous run --- + print("Checking for leftover relationship test resources...") + found_leftovers = False + for rel_name in [ + "test_RelParent_RelChild", + "contact_test_relchild_test_ManagerId", + "test_relchild_relproject", + ]: + try: + rel = client.tables.get_relationship(rel_name) + if rel: + found_leftovers = True + break + except Exception: + pass + + if not found_leftovers: + for tbl in [rel_child_schema, rel_parent_schema, rel_m2m_schema]: + try: + if client.tables.get(tbl): + found_leftovers = True + break + except Exception: + pass + + if found_leftovers: + cleanup_ok = input("Found leftover test resources. Clean up? (y/N): ").strip().lower() in ["y", "yes"] + if cleanup_ok: + for rel_name in [ + "test_RelParent_RelChild", + "contact_test_relchild_test_ManagerId", + "test_relchild_relproject", + ]: + try: + rel = client.tables.get_relationship(rel_name) + if rel: + client.tables.delete_relationship(rel.relationship_id) + print(f" (Cleaned up relationship: {rel_name})") + except Exception: + pass + + for tbl in [rel_child_schema, rel_parent_schema, rel_m2m_schema]: + try: + if client.tables.get(tbl): + client.tables.delete(tbl) + print(f" (Cleaned up table: {tbl})") + except Exception: + pass + else: + print("Skipping cleanup -- resources may conflict with new test run.") + + # --- Create parent and child tables --- + print("\nCreating relationship test tables...") + + parent_info = backoff( + lambda: client.tables.create( + rel_parent_schema, + {"test_Code": "string"}, + ) + ) + created_tables.append(rel_parent_schema) + print(f"[OK] Created parent table: {parent_info['table_schema_name']}") + + child_info = backoff( + lambda: client.tables.create( + rel_child_schema, + {"test_Number": "string"}, + ) + ) + created_tables.append(rel_child_schema) + print(f"[OK] Created child table: {child_info['table_schema_name']}") + + proj_info = backoff( + lambda: client.tables.create( + rel_m2m_schema, + {"test_ProjectCode": "string"}, + ) + ) + created_tables.append(rel_m2m_schema) + print(f"[OK] Created M:N table: {proj_info['table_schema_name']}") + + # --- Wait for table metadata to propagate --- + wait_for_table_metadata(client, rel_parent_schema) + wait_for_table_metadata(client, rel_child_schema) + wait_for_table_metadata(client, rel_m2m_schema) + + # --- Test 1: Create 1:N relationship (core API) --- + print("\n Test 1: Create 1:N relationship (core API)") + print(" " + "-" * 45) + + lookup = LookupAttributeMetadata( + schema_name="test_ParentId", + display_name=Label(localized_labels=[LocalizedLabel(label="Parent", language_code=1033)]), + required_level="None", + ) + + relationship = OneToManyRelationshipMetadata( + schema_name="test_RelParent_RelChild", + referenced_entity=parent_info["table_logical_name"], + referencing_entity=child_info["table_logical_name"], + referenced_attribute=f"{parent_info['table_logical_name']}id", + cascade_configuration=CascadeConfiguration( + delete=CASCADE_BEHAVIOR_REMOVE_LINK, + assign=CASCADE_BEHAVIOR_NO_CASCADE, + merge=CASCADE_BEHAVIOR_NO_CASCADE, + ), + ) + + result_1n = backoff( + lambda: client.tables.create_one_to_many_relationship( + lookup=lookup, + relationship=relationship, + ) + ) + + assert result_1n.relationship_schema_name == "test_RelParent_RelChild" + assert result_1n.relationship_type == "one_to_many" + assert result_1n.lookup_schema_name is not None + rel_id_1n = result_1n.relationship_id + print(f" [OK] Created 1:N relationship: {result_1n.relationship_schema_name}") + print(f" Lookup: {result_1n.lookup_schema_name}") + print(f" ID: {rel_id_1n}") + + # --- Test 2: Create lookup field (convenience API) --- + print("\n Test 2: Create lookup field (convenience API)") + print(" " + "-" * 45) + + result_lookup = backoff( + lambda: client.tables.create_lookup_field( + referencing_table=child_info["table_logical_name"], + lookup_field_name="test_ManagerId", + referenced_table="contact", + display_name="Manager", + description="The record's manager contact", + required=False, + cascade_delete=CASCADE_BEHAVIOR_REMOVE_LINK, + ) + ) + + assert result_lookup.relationship_type == "one_to_many" + assert result_lookup.lookup_schema_name is not None + rel_id_lookup = result_lookup.relationship_id + print(f" [OK] Created lookup: {result_lookup.lookup_schema_name}") + print(f" Relationship: {result_lookup.relationship_schema_name}") + + # --- Test 3: Create N:N relationship --- + print("\n Test 3: Create N:N relationship") + print(" " + "-" * 45) + + m2m = ManyToManyRelationshipMetadata( + schema_name="test_relchild_relproject", + entity1_logical_name=child_info["table_logical_name"], + entity2_logical_name=proj_info["table_logical_name"], + ) + + result_nn = backoff(lambda: client.tables.create_many_to_many_relationship(relationship=m2m)) + + assert result_nn.relationship_schema_name == "test_relchild_relproject" + assert result_nn.relationship_type == "many_to_many" + rel_id_nn = result_nn.relationship_id + print(f" [OK] Created N:N relationship: {result_nn.relationship_schema_name}") + print(f" ID: {rel_id_nn}") + + # --- Test 4: Get relationship metadata --- + print("\n Test 4: Query relationship metadata") + print(" " + "-" * 45) + + fetched_1n = client.tables.get_relationship("test_RelParent_RelChild") + assert fetched_1n is not None + assert fetched_1n.relationship_type == "one_to_many" + assert fetched_1n.relationship_id == rel_id_1n + print(f" [OK] Retrieved 1:N: {fetched_1n.relationship_schema_name}") + print(f" Referenced: {fetched_1n.referenced_entity}") + print(f" Referencing: {fetched_1n.referencing_entity}") + + fetched_nn = client.tables.get_relationship("test_relchild_relproject") + assert fetched_nn is not None + assert fetched_nn.relationship_type == "many_to_many" + assert fetched_nn.relationship_id == rel_id_nn + print(f" [OK] Retrieved N:N: {fetched_nn.relationship_schema_name}") + print(f" Entity1: {fetched_nn.entity1_logical_name}") + print(f" Entity2: {fetched_nn.entity2_logical_name}") + + # Non-existent relationship should return None + missing = client.tables.get_relationship("nonexistent_relationship_xyz") + assert missing is None + print(" [OK] Non-existent relationship returns None") + + # --- Test 5: Delete relationships --- + print("\n Test 5: Delete relationships") + print(" " + "-" * 45) + + backoff(lambda: client.tables.delete_relationship(rel_id_1n)) + rel_id_1n = None + print(" [OK] Deleted 1:N relationship") + + backoff(lambda: client.tables.delete_relationship(rel_id_lookup)) + rel_id_lookup = None + print(" [OK] Deleted lookup relationship") + + backoff(lambda: client.tables.delete_relationship(rel_id_nn)) + rel_id_nn = None + print(" [OK] Deleted N:N relationship") + + # Verify deletion + verify = client.tables.get_relationship("test_RelParent_RelChild") + assert verify is None + print(" [OK] Verified 1:N deletion (get returns None)") + + print("\n[OK] All relationship tests passed!") + + finally: + # Cleanup: delete any remaining relationships then tables + for rid in [rel_id_1n, rel_id_lookup, rel_id_nn]: + if rid: + try: + client.tables.delete_relationship(rid) + except Exception: + pass + + for tbl in reversed(created_tables): + try: + backoff(lambda name=tbl: client.tables.delete(name)) + print(f" (Cleaned up table: {tbl})") + except Exception as e: + print(f" [WARN] Could not delete {tbl}: {e}") + + def _table_still_exists(client: DataverseClient, table_schema_name: Optional[str]) -> bool: if not table_schema_name: return False @@ -403,6 +682,7 @@ def main(): print(" - Table Creation & Metadata Operations") print(" - Record CRUD Operations") print(" - Query Functionality") + print(" - Relationship Operations (1:N, N:N, lookup, get, delete)") print(" - Interactive Cleanup") print("=" * 70) print("For installation validation, run examples/basic/installation_example.py first") @@ -422,6 +702,9 @@ def main(): # Test querying test_query_records(client, table_info) + # Test relationships + test_relationships(client) + # Success summary print("\nFunctional Test Summary") print("=" * 50) @@ -430,6 +713,7 @@ def main(): print("[OK] Record Creation: Success") print("[OK] Record Reading: Success") print("[OK] Record Querying: Success") + print("[OK] Relationship Operations: Success") print("\nYour PowerPlatform Dataverse Client SDK is fully functional!") # Cleanup diff --git a/pyproject.toml b/pyproject.toml index 1206ad5d..3df59c9d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -90,3 +90,11 @@ select = [ "UP", # pyupgrade "B", # flake8-bugbear ] + +[tool.pytest.ini_options] +testpaths = ["tests/unit"] +markers = [ + "e2e: end-to-end tests requiring a live Dataverse environment (DATAVERSE_URL)", +] +# e2e tests require a live Dataverse environment: +# DATAVERSE_URL=https://yourorg.crm.dynamics.com pytest tests/e2e/ -v -s diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py new file mode 100644 index 00000000..9a045456 --- /dev/null +++ b/tests/e2e/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. diff --git a/tests/e2e/test_relationships_e2e.py b/tests/e2e/test_relationships_e2e.py new file mode 100644 index 00000000..46aa46d8 --- /dev/null +++ b/tests/e2e/test_relationships_e2e.py @@ -0,0 +1,761 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +End-to-end relationship tests for the Dataverse SDK. + +These tests run against a LIVE Dataverse environment and validate +the full relationship API lifecycle: +- 1:N (one-to-many) relationship CRUD +- N:N (many-to-many) relationship CRUD +- Convenience create_lookup_field API +- Data operations through relationships (@odata.bind, $expand, $filter) +- Cascade behavior verification (RemoveLink, Restrict, Cascade) + +Requirements: +- Set DATAVERSE_URL environment variable (e.g. https://yourorg.crm.dynamics.com) +- Azure identity with permissions to create/delete tables and relationships +- Run with: pytest tests/e2e/test_relationships_e2e.py -v -s + +These tests are NOT run in CI (they require a live environment). +Mark: @pytest.mark.e2e +""" + +import os +import time + +import pytest +from azure.identity import InteractiveBrowserCredential + +from PowerPlatform.Dataverse.client import DataverseClient +from PowerPlatform.Dataverse.core.errors import HttpError +from PowerPlatform.Dataverse.models.relationship import ( + LookupAttributeMetadata, + OneToManyRelationshipMetadata, + ManyToManyRelationshipMetadata, + CascadeConfiguration, +) +from PowerPlatform.Dataverse.models.labels import Label, LocalizedLabel +from PowerPlatform.Dataverse.common.constants import ( + CASCADE_BEHAVIOR_CASCADE, + CASCADE_BEHAVIOR_NO_CASCADE, + CASCADE_BEHAVIOR_REMOVE_LINK, + CASCADE_BEHAVIOR_RESTRICT, +) + +pytestmark = pytest.mark.e2e + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +def _get_org_url(): + """Read DATAVERSE_URL at call time so late-set env vars are picked up.""" + return os.environ.get("DATAVERSE_URL", "") + + +def _skip_if_no_url(): + if not _get_org_url(): + pytest.skip("DATAVERSE_URL not set -- skipping e2e tests") + + +@pytest.fixture(scope="module") +def client(): + """Authenticated DataverseClient for the test module.""" + _skip_if_no_url() + cred = InteractiveBrowserCredential(timeout=600) + c = DataverseClient(_get_org_url(), cred) + try: + yield c + finally: + c.close() + + +def _backoff(op, *, delays=(0, 2, 5, 10, 20, 20)): + """Retry with exponential backoff for metadata propagation.""" + last = None + for d in delays: + if d: + time.sleep(d) + try: + return op() + except Exception as ex: + last = ex + raise last + + +def _wait_for_table(client, schema_name, retries=15, delay=3): + """Poll until table metadata is available.""" + last_exc = None + for attempt in range(1, retries + 1): + try: + info = client.tables.get(schema_name) + if info and info.get("entity_set_name"): + odata = client._get_odata() + odata._entity_set_from_schema_name(schema_name) + return info + except Exception as exc: + last_exc = exc + if attempt < retries: + time.sleep(delay) + msg = f"Table {schema_name} metadata not available after {retries} attempts" + if last_exc: + raise RuntimeError(msg) from last_exc + raise RuntimeError(msg) + + +def _wait_for_relationship(client, schema_name, retries=15, delay=3): + """Poll until get_relationship returns a non-None result.""" + for attempt in range(1, retries + 1): + result = client.tables.get_relationship(schema_name) + if result is not None: + return result + if attempt < retries: + time.sleep(delay) + raise RuntimeError(f"Relationship {schema_name} not queryable after {retries} attempts") + + +def _wait_for_lookup_ready(client, table_schema, lookup_logical, retries=15, delay=3): + """Poll until a lookup column is queryable on the table.""" + for attempt in range(1, retries + 1): + try: + for page in client.records.get(table_schema, select=[lookup_logical], top=1): + pass # If we get here without error, the column is ready + return + except HttpError: + pass + except Exception: + pass + if attempt < retries: + time.sleep(delay) + raise RuntimeError(f"Lookup column {lookup_logical} not ready on {table_schema} after {retries} attempts") + + +def _safe_delete_relationship(client, schema_name): + try: + rel = client.tables.get_relationship(schema_name) + if rel and rel.relationship_id: + client.tables.delete_relationship(rel.relationship_id) + except Exception: + pass + + +def _safe_delete_table(client, schema_name): + try: + if client.tables.get(schema_name): + _backoff(lambda: client.tables.delete(schema_name)) + except Exception: + pass + + +def _create_table(client, schema_name, columns=None): + """Create a table and wait for metadata.""" + cols = columns or {"new_Name": "string"} + info = _backoff(lambda: client.tables.create(schema_name, cols)) + _wait_for_table(client, schema_name) + return info + + +# --------------------------------------------------------------------------- +# Test 1: 1:N Core API -- create, get, delete +# --------------------------------------------------------------------------- + + +class TestOneToManyCore: + """One-to-many relationship lifecycle via core API.""" + + PARENT = "new_E2E1NPar" + CHILD = "new_E2E1NChi" + REL_NAME = "new_E2E1NPar_1NChi" + + @pytest.fixture(autouse=True) + def setup_and_teardown(self, client): + _safe_delete_relationship(client, self.REL_NAME) + _safe_delete_table(client, self.CHILD) + _safe_delete_table(client, self.PARENT) + yield + _safe_delete_relationship(client, self.REL_NAME) + _safe_delete_table(client, self.CHILD) + _safe_delete_table(client, self.PARENT) + + def test_create_get_delete_1n(self, client): + """Full 1:N lifecycle: create tables, create relationship, get, delete.""" + parent = _create_table(client, self.PARENT, {"new_Code": "string"}) + child = _create_table(client, self.CHILD, {"new_Num": "string"}) + + lookup = LookupAttributeMetadata( + schema_name="new_ParRef", + display_name=Label(localized_labels=[LocalizedLabel(label="Parent Ref", language_code=1033)]), + required_level="None", + ) + relationship = OneToManyRelationshipMetadata( + schema_name=self.REL_NAME, + referenced_entity=parent["table_logical_name"], + referencing_entity=child["table_logical_name"], + referenced_attribute=f"{parent['table_logical_name']}id", + cascade_configuration=CascadeConfiguration( + delete=CASCADE_BEHAVIOR_REMOVE_LINK, + assign=CASCADE_BEHAVIOR_NO_CASCADE, + ), + ) + + result = _backoff( + lambda: client.tables.create_one_to_many_relationship(lookup=lookup, relationship=relationship) + ) + + # Verify create result + assert result.relationship_type == "one_to_many" + assert result.relationship_schema_name == self.REL_NAME + assert result.relationship_id is not None + assert result.lookup_schema_name is not None + assert result.referenced_entity == parent["table_logical_name"] + assert result.referencing_entity == child["table_logical_name"] + assert result.entity1_logical_name is None # N:N only + + # Verify get_relationship + fetched = _wait_for_relationship(client, self.REL_NAME) + assert fetched.relationship_type == "one_to_many" + assert fetched.relationship_id == result.relationship_id + assert fetched.referenced_entity == parent["table_logical_name"] + + # Verify delete + client.tables.delete_relationship(result.relationship_id) + post_delete = client.tables.get_relationship(self.REL_NAME) + assert post_delete is None + + +# --------------------------------------------------------------------------- +# Test 2: 1:N Convenience API -- create_lookup_field +# --------------------------------------------------------------------------- + + +class TestLookupField: + """Convenience create_lookup_field API.""" + + CHILD = "new_E2ELkpChi" + + @pytest.fixture(autouse=True) + def setup_and_teardown(self, client): + _safe_delete_table(client, self.CHILD) + self._rel_id = None + yield + if self._rel_id: + try: + client.tables.delete_relationship(self._rel_id) + except Exception: + pass + _safe_delete_table(client, self.CHILD) + + def test_lookup_to_system_table(self, client): + """Create lookup from custom table to system 'account'.""" + child = _create_table(client, self.CHILD, {"new_Info": "string"}) + + result = _backoff( + lambda: client.tables.create_lookup_field( + referencing_table=child["table_logical_name"], + lookup_field_name="new_AcctLkp", + referenced_table="account", + display_name="Account", + required=False, + cascade_delete=CASCADE_BEHAVIOR_REMOVE_LINK, + ) + ) + self._rel_id = result.relationship_id + + assert result.relationship_type == "one_to_many" + assert result.lookup_schema_name is not None + assert result.referenced_entity == "account" + + # Verify via get_relationship + fetched = _wait_for_relationship(client, result.relationship_schema_name) + assert fetched.referenced_entity == "account" + + +# --------------------------------------------------------------------------- +# Test 3: N:N -- create, get, delete +# --------------------------------------------------------------------------- + + +class TestManyToMany: + """Many-to-many relationship lifecycle.""" + + TBL1 = "new_E2ENNTbl1" + TBL2 = "new_E2ENNTbl2" + REL_NAME = "new_e2enntbl1_nntbl2" + + @pytest.fixture(autouse=True) + def setup_and_teardown(self, client): + _safe_delete_relationship(client, self.REL_NAME) + _safe_delete_table(client, self.TBL1) + _safe_delete_table(client, self.TBL2) + yield + _safe_delete_relationship(client, self.REL_NAME) + _safe_delete_table(client, self.TBL1) + _safe_delete_table(client, self.TBL2) + + def test_create_get_delete_nn(self, client): + """Full N:N lifecycle: create, get, delete.""" + tbl1 = _create_table(client, self.TBL1, {"new_C1": "string"}) + tbl2 = _create_table(client, self.TBL2, {"new_C2": "string"}) + + m2m = ManyToManyRelationshipMetadata( + schema_name=self.REL_NAME, + entity1_logical_name=tbl1["table_logical_name"], + entity2_logical_name=tbl2["table_logical_name"], + ) + + result = _backoff(lambda: client.tables.create_many_to_many_relationship(relationship=m2m)) + + assert result.relationship_type == "many_to_many" + assert result.relationship_schema_name == self.REL_NAME + assert result.entity1_logical_name == tbl1["table_logical_name"] + assert result.entity2_logical_name == tbl2["table_logical_name"] + assert result.lookup_schema_name is None # 1:N only + + # Verify get + fetched = _wait_for_relationship(client, self.REL_NAME) + assert fetched.relationship_type == "many_to_many" + assert fetched.relationship_id == result.relationship_id + + # Verify delete + client.tables.delete_relationship(result.relationship_id) + post_delete = client.tables.get_relationship(self.REL_NAME) + assert post_delete is None + + def test_get_nonexistent_returns_none(self, client): + """get_relationship returns None for nonexistent relationships.""" + result = client.tables.get_relationship("nonexistent_xyz_relationship_99") + assert result is None + + +# --------------------------------------------------------------------------- +# Test 4: Data through relationships -- @odata.bind, $expand, $filter +# --------------------------------------------------------------------------- + + +class TestDataThroughRelationships: + """Verify relationships work with actual record operations.""" + + PARENT = "new_E2EDataPar" + CHILD = "new_E2EDataChi" + REL_NAME = "new_E2EDataPar_DataChi" + + @pytest.fixture(autouse=True) + def setup_and_teardown(self, client): + _safe_delete_relationship(client, self.REL_NAME) + _safe_delete_table(client, self.CHILD) + _safe_delete_table(client, self.PARENT) + + # Create tables + relationship + data + self.parent_info = _create_table( + client, + self.PARENT, + { + "new_ParName": "string", + }, + ) + self.child_info = _create_table( + client, + self.CHILD, + { + "new_ChiName": "string", + "new_ChiVal": "int", + }, + ) + + lookup = LookupAttributeMetadata( + schema_name="new_DataParLkp", + display_name=Label(localized_labels=[LocalizedLabel(label="Data Parent", language_code=1033)]), + ) + rel = OneToManyRelationshipMetadata( + schema_name=self.REL_NAME, + referenced_entity=self.parent_info["table_logical_name"], + referencing_entity=self.child_info["table_logical_name"], + referenced_attribute=f"{self.parent_info['table_logical_name']}id", + cascade_configuration=CascadeConfiguration( + delete=CASCADE_BEHAVIOR_REMOVE_LINK, + ), + ) + + self.rel_result = _backoff( + lambda: client.tables.create_one_to_many_relationship(lookup=lookup, relationship=rel) + ) + + # Get server-assigned navigation property name + rel_info = _wait_for_relationship(client, self.REL_NAME) + self.server_nav_prop = rel_info.lookup_schema_name + self.lookup_value_key = f"_{self.server_nav_prop.lower()}_value" + + # Wait for lookup column to become queryable (instead of hard-coded sleep) + _wait_for_lookup_ready(client, self.CHILD, self.lookup_value_key) + + # Get entity set for @odata.bind + parent_full = client.tables.get(self.PARENT) + self.entity_set = parent_full["entity_set_name"] + + # Create parent records + self.p1_id = _backoff(lambda: client.records.create(self.PARENT, {"new_parname": "Alpha Corp"})) + self.p2_id = _backoff(lambda: client.records.create(self.PARENT, {"new_parname": "Beta Inc"})) + + # Create child records -- use server_nav_prop for @odata.bind + # (server-assigned nav prop is authoritative for case-sensitive OData operations) + nav = self.server_nav_prop + es = self.entity_set + p1 = self.p1_id + p2 = self.p2_id + + self.c1_id = _backoff( + lambda: client.records.create( + self.CHILD, + { + "new_chiname": "Child A1", + "new_chival": 100, + f"{nav}@odata.bind": f"/{es}({p1})", + }, + ) + ) + self.c2_id = _backoff( + lambda: client.records.create( + self.CHILD, + { + "new_chiname": "Child A2", + "new_chival": 200, + f"{nav}@odata.bind": f"/{es}({p1})", + }, + ) + ) + self.c3_id = _backoff( + lambda: client.records.create( + self.CHILD, + { + "new_chiname": "Child B1", + "new_chival": 300, + f"{nav}@odata.bind": f"/{es}({p2})", + }, + ) + ) + self.c4_id = _backoff( + lambda: client.records.create( + self.CHILD, + { + "new_chiname": "Orphan", + "new_chival": 0, + }, + ) + ) + + yield + + # Cleanup + for cid in [self.c1_id, self.c2_id, self.c3_id, self.c4_id]: + try: + client.records.delete(self.CHILD, cid) + except Exception: + pass + for pid in [self.p1_id, self.p2_id]: + try: + client.records.delete(self.PARENT, pid) + except Exception: + pass + try: + client.tables.delete_relationship(self.rel_result.relationship_id) + except Exception: + pass + _safe_delete_table(client, self.CHILD) + _safe_delete_table(client, self.PARENT) + + def test_odata_bind_creates_lookup(self, client): + """Records created with @odata.bind have correct lookup values.""" + c1 = client.records.get(self.CHILD, self.c1_id) + assert c1.get(self.lookup_value_key) is not None + assert c1[self.lookup_value_key].lower() == self.p1_id.lower() + + c4 = client.records.get(self.CHILD, self.c4_id) + assert c4.get(self.lookup_value_key) is None + + def test_expand_returns_parent_data(self, client): + """$expand on navigation property returns parent fields.""" + all_recs = [] + for page in client.records.get( + self.CHILD, + select=["new_chiname"], + expand=[self.server_nav_prop], + top=10, + ): + all_recs.extend(page) + + assert len(all_recs) >= 4 + bound = [r for r in all_recs if r.get(self.server_nav_prop) is not None] + assert len(bound) >= 3 + for rec in bound: + assert rec[self.server_nav_prop].get("new_parname") is not None + + def test_filter_on_lookup_value(self, client): + """$filter on lookup _value returns correct children.""" + filtered = [] + for page in client.records.get( + self.CHILD, + select=["new_chiname"], + filter=f"{self.lookup_value_key} eq {self.p1_id}", + top=10, + ): + filtered.extend(page) + + assert len(filtered) == 2 + names = {r["new_chiname"] for r in filtered} + assert names == {"Child A1", "Child A2"} + + def test_update_lookup_binding(self, client): + """Updating @odata.bind changes lookup value.""" + client.records.update( + self.CHILD, + self.c1_id, + { + f"{self.server_nav_prop}@odata.bind": f"/{self.entity_set}({self.p2_id})", + }, + ) + updated = client.records.get(self.CHILD, self.c1_id) + assert updated[self.lookup_value_key].lower() == self.p2_id.lower() + + # Restore + client.records.update( + self.CHILD, + self.c1_id, + { + f"{self.server_nav_prop}@odata.bind": f"/{self.entity_set}({self.p1_id})", + }, + ) + + +# --------------------------------------------------------------------------- +# Test 5: Cascade behaviors +# --------------------------------------------------------------------------- + + +class TestCascadeBehaviors: + """Verify cascade configuration affects data operations.""" + + def _create_cascade_setup(self, client, parent_schema, child_schema, rel_name, lookup_name, cascade_delete): + """Create parent/child with specified cascade and return IDs.""" + _safe_delete_relationship(client, rel_name) + _safe_delete_table(client, child_schema) + _safe_delete_table(client, parent_schema) + + parent = _create_table(client, parent_schema, {"new_Name": "string"}) + child = _create_table(client, child_schema, {"new_Info": "string"}) + + lookup = LookupAttributeMetadata( + schema_name=lookup_name, + display_name=Label(localized_labels=[LocalizedLabel(label="Cascade", language_code=1033)]), + ) + relationship = OneToManyRelationshipMetadata( + schema_name=rel_name, + referenced_entity=parent["table_logical_name"], + referencing_entity=child["table_logical_name"], + referenced_attribute=f"{parent['table_logical_name']}id", + cascade_configuration=CascadeConfiguration( + delete=cascade_delete, + assign=CASCADE_BEHAVIOR_NO_CASCADE, + ), + ) + + rel_result = _backoff( + lambda: client.tables.create_one_to_many_relationship(lookup=lookup, relationship=relationship) + ) + rel_info = _wait_for_relationship(client, rel_name) + + parent_full = client.tables.get(parent_schema) + entity_set = parent_full["entity_set_name"] + nav_prop = rel_info.lookup_schema_name + + # Wait for lookup column to become queryable + lookup_value_key = f"_{nav_prop.lower()}_value" + _wait_for_lookup_ready(client, child_schema, lookup_value_key) + + return rel_result, entity_set, nav_prop + + def test_restrict_prevents_parent_delete(self, client): + """Restrict cascade: deleting parent with children fails.""" + ps, cs, rn = "new_E2ERestrPar", "new_E2ERestrChi", "new_E2ERestrPar_RestrChi" + rel_result = None + p_id = c_id = None + try: + rel_result, entity_set, nav_prop = self._create_cascade_setup( + client, ps, cs, rn, "new_RestrRef", CASCADE_BEHAVIOR_RESTRICT + ) + + p_id = _backoff(lambda: client.records.create(ps, {"new_name": "Restrict Parent"})) + c_id = _backoff( + lambda: client.records.create( + cs, + { + "new_info": "Restrict Child", + f"{nav_prop}@odata.bind": f"/{entity_set}({p_id})", + }, + ) + ) + + # Delete parent should fail + with pytest.raises(HttpError): + client.records.delete(ps, p_id) + + # Both records still exist + assert client.records.get(ps, p_id) is not None + assert client.records.get(cs, c_id) is not None + + # Remove child, then parent delete succeeds + client.records.delete(cs, c_id) + c_id = None + client.records.delete(ps, p_id) + p_id = None + + finally: + if c_id: + try: + client.records.delete(cs, c_id) + except Exception: + pass + if p_id: + try: + client.records.delete(ps, p_id) + except Exception: + pass + if rel_result: + try: + client.tables.delete_relationship(rel_result.relationship_id) + except Exception: + pass + _safe_delete_table(client, cs) + _safe_delete_table(client, ps) + + def test_cascade_deletes_children(self, client): + """Cascade delete: deleting parent also deletes children.""" + ps, cs, rn = "new_E2ECascPar", "new_E2ECascChi", "new_E2ECascPar_CascChi" + rel_result = None + try: + rel_result, entity_set, nav_prop = self._create_cascade_setup( + client, ps, cs, rn, "new_CascRef", CASCADE_BEHAVIOR_CASCADE + ) + + p_id = _backoff(lambda: client.records.create(ps, {"new_name": "Cascade Parent"})) + c1_id = _backoff( + lambda: client.records.create( + cs, + { + "new_info": "Cascade Child 1", + f"{nav_prop}@odata.bind": f"/{entity_set}({p_id})", + }, + ) + ) + c2_id = _backoff( + lambda: client.records.create( + cs, + { + "new_info": "Cascade Child 2", + f"{nav_prop}@odata.bind": f"/{entity_set}({p_id})", + }, + ) + ) + + # Delete parent -- children should be cascade-deleted + client.records.delete(ps, p_id) + + # Poll until children return 404 (cascade may be async) + for cid in [c1_id, c2_id]: + for attempt in range(1, 11): + try: + client.records.get(cs, cid) + except HttpError as e: + if e.status_code == 404: + break + raise + if attempt < 10: + time.sleep(2) + else: + pytest.fail(f"Child {cid} still exists after cascade delete") + + finally: + if rel_result: + try: + client.tables.delete_relationship(rel_result.relationship_id) + except Exception: + pass + _safe_delete_table(client, cs) + _safe_delete_table(client, ps) + + +# --------------------------------------------------------------------------- +# Test 6: Type detection -- get_relationship distinguishes 1:N vs N:N +# --------------------------------------------------------------------------- + + +class TestTypeDetection: + """get_relationship returns correct type for both relationship kinds.""" + + TBL1 = "new_E2ETypeTbl1" + TBL2 = "new_E2ETypeTbl2" + REL_1N = "new_E2ETypeTbl1_TypeTbl2" + REL_NN = "new_e2etypetbl1_typetbl2_nn" + + @pytest.fixture(autouse=True) + def setup_and_teardown(self, client): + for r in [self.REL_1N, self.REL_NN]: + _safe_delete_relationship(client, r) + for t in [self.TBL1, self.TBL2]: + _safe_delete_table(client, t) + self._rel_ids = [] + yield + for rid in self._rel_ids: + try: + client.tables.delete_relationship(rid) + except Exception: + pass + for t in [self.TBL1, self.TBL2]: + _safe_delete_table(client, t) + + def test_type_detection(self, client): + """get_relationship correctly detects 1:N vs N:N.""" + tbl1 = _create_table(client, self.TBL1, {"new_TC1": "string"}) + tbl2 = _create_table(client, self.TBL2, {"new_TC2": "string"}) + + # Create 1:N + lookup = LookupAttributeMetadata( + schema_name="new_TypeRef", + display_name=Label(localized_labels=[LocalizedLabel(label="Type Test", language_code=1033)]), + ) + result_1n = _backoff( + lambda: client.tables.create_one_to_many_relationship( + lookup=lookup, + relationship=OneToManyRelationshipMetadata( + schema_name=self.REL_1N, + referenced_entity=tbl1["table_logical_name"], + referencing_entity=tbl2["table_logical_name"], + referenced_attribute=f"{tbl1['table_logical_name']}id", + ), + ) + ) + self._rel_ids.append(result_1n.relationship_id) + + # Create N:N + result_nn = _backoff( + lambda: client.tables.create_many_to_many_relationship( + relationship=ManyToManyRelationshipMetadata( + schema_name=self.REL_NN, + entity1_logical_name=tbl1["table_logical_name"], + entity2_logical_name=tbl2["table_logical_name"], + ), + ) + ) + self._rel_ids.append(result_nn.relationship_id) + + # Verify type detection + fetched_1n = _wait_for_relationship(client, self.REL_1N) + assert fetched_1n.relationship_type == "one_to_many" + assert fetched_1n.referenced_entity is not None + assert fetched_1n.entity1_logical_name is None + + fetched_nn = _wait_for_relationship(client, self.REL_NN) + assert fetched_nn.relationship_type == "many_to_many" + assert fetched_nn.entity1_logical_name is not None + assert fetched_nn.lookup_schema_name is None From 5a395ec9db247cdb17eb24273265c3f26bf1c588 Mon Sep 17 00:00:00 2001 From: tpellissier-msft Date: Thu, 19 Mar 2026 12:06:14 -0700 Subject: [PATCH 16/20] Add QueryBuilder with fluent API and composable filter expressions (#118) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Implements the QueryBuilder feature from the SDK redesign design doc (ADO PR 1504429): - **Fluent query builder** via `client.query.builder("table")` with 20 chainable methods including `select`, `filter_eq/ne/gt/ge/lt/le`, `filter_contains/startswith/endswith`, `filter_in`, `filter_between`, `filter_null/not_null`, `filter_raw`, `where`, `order_by`, `top`, `page_size`, `expand`, and `execute` - **Composable filter expression tree** (`models/filters.py`) with Python operator overloads (`&`, `|`, `~`) for AND, OR, NOT composition - **Value auto-formatting** for `str`, `int`, `float`, `bool`, `None`, `datetime`, `date`, `uuid.UUID` - 126 new unit tests (57 filters + 69 query builder), 309 total passing ### Usage examples ```python # Fluent builder for page in (client.query.builder("account") .select("name", "revenue") .filter_eq("statecode", 0) .filter_gt("revenue", 1000000) .order_by("revenue", descending=True) .top(100) .page_size(50) .execute()): for record in page: print(record["name"]) # Composable expression tree with where() from PowerPlatform.Dataverse.models.filters import eq, gt, filter_in for page in (client.query.builder("account") .where((eq("statecode", 0) | eq("statecode", 1)) & gt("revenue", 100000)) .execute()): for record in page: print(record["name"]) ``` ### Design decisions - **Regular class, not dataclass** — prevents leaking internal state as constructor params - **Unified `_filter_parts` list** — preserves call order when mixing `filter_*()` and `where()` - **`execute()` calls `build()` internally** — single source of truth for filter compilation - **No public `get()` on QueryOperations** — only `builder()` added; paginated queries remain on `records.get()` - **Parenthesized `filter_between`** — `(col ge low and col le high)` for correct precedence ### Files changed | File | Description | |------|-------------| | `src/.../models/filters.py` | **NEW** — Composable expression tree | | `src/.../models/query_builder.py` | **NEW** — Fluent QueryBuilder class | | `src/.../operations/query.py` | Add `builder()` to QueryOperations | | `src/.../models/__init__.py` | Updated docstring | | `tests/.../models/test_filters.py` | **NEW** — 57 filter tests | | `tests/.../models/test_query_builder.py` | **NEW** — 69 builder tests | | `tests/.../test_query_operations.py` | 6 new integration tests | ### Merge conflict note `operations/query.py` may conflict with PR #115 (typed return models) — resolution is straightforward since we only add a `builder()` method. ## Test plan - [x] `pytest tests/unit/models/test_filters.py` — 57 passed - [x] `pytest tests/unit/models/test_query_builder.py` — 69 passed - [x] `pytest tests/unit/test_query_operations.py` — 9 passed - [x] `pytest tests/` — 309 passed, 0 failed 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: tpellissier Co-authored-by: Claude Opus 4.6 (1M context) Co-authored-by: Abel Milash Co-authored-by: Saurabh Badenkal Co-authored-by: Saurabh Ravindra Badenkal <32964911+saurabhrb@users.noreply.github.com> --- README.md | 136 ++- examples/advanced/walkthrough.py | 184 +++- src/PowerPlatform/Dataverse/data/_odata.py | 15 +- .../Dataverse/models/__init__.py | 15 +- src/PowerPlatform/Dataverse/models/filters.py | 465 +++++++++ .../Dataverse/models/query_builder.py | 792 ++++++++++++++ .../Dataverse/operations/dataframe.py | 11 + .../Dataverse/operations/query.py | 53 + .../Dataverse/operations/records.py | 26 +- tests/unit/models/test_filters.py | 522 ++++++++++ tests/unit/models/test_query_builder.py | 977 ++++++++++++++++++ tests/unit/test_client.py | 2 + tests/unit/test_client_dataframe.py | 2 + tests/unit/test_client_deprecations.py | 2 + tests/unit/test_dataframe_operations.py | 2 + tests/unit/test_query_operations.py | 221 ++++ tests/unit/test_records_operations.py | 2 + 17 files changed, 3388 insertions(+), 39 deletions(-) create mode 100644 src/PowerPlatform/Dataverse/models/filters.py create mode 100644 src/PowerPlatform/Dataverse/models/query_builder.py create mode 100644 tests/unit/models/test_filters.py create mode 100644 tests/unit/models/test_query_builder.py diff --git a/README.md b/README.md index 1426f96a..3b892644 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ A Python client library for Microsoft Dataverse that provides a unified interfac - [Bulk operations](#bulk-operations) - [Upsert operations](#upsert-operations) - [DataFrame operations](#dataframe-operations) - - [Query data](#query-data) + - [Query data](#query-data) *(QueryBuilder, SQL, raw OData)* - [Table management](#table-management) - [Relationship management](#relationship-management) - [File operations](#file-operations) @@ -37,7 +37,8 @@ A Python client library for Microsoft Dataverse that provides a unified interfac - **🔄 CRUD Operations**: Create, read, update, and delete records with support for bulk operations and automatic retry - **⚡ True Bulk Operations**: Automatically uses Dataverse's native `CreateMultiple`, `UpdateMultiple`, `UpsertMultiple`, and `BulkDelete` Web API operations for maximum performance and transactional integrity -- **📊 SQL Queries**: Execute read-only SQL queries via the Dataverse Web API `?sql=` parameter +- **🔍 Fluent QueryBuilder**: Type-safe query construction with method chaining, composable filter expressions, and automatic OData generation +- **📊 SQL Queries**: Execute read-only SQL queries via the Dataverse Web API `?sql=` parameter - **🏗️ Table Management**: Create, inspect, and delete custom tables and columns programmatically - **🔗 Relationship Management**: Create one-to-many and many-to-many relationships between tables with full metadata control - **🐼 DataFrame Support**: Pandas wrappers for all CRUD operations, returning DataFrames and Series @@ -116,7 +117,7 @@ The SDK provides a simple, pythonic interface for Dataverse operations: |---------|-------------| | **DataverseClient** | Main entry point; provides `records`, `query`, `tables`, and `files` namespaces | | **Context Manager** | Use `with DataverseClient(...) as client:` for automatic cleanup and HTTP connection pooling | -| **Namespaces** | Operations are organized into `client.records` (CRUD & OData queries), `client.query` (query & search), `client.tables` (metadata), and `client.files` (file uploads) | +| **Namespaces** | Operations are organized into `client.records` (CRUD & OData queries), `client.query` (QueryBuilder & SQL), `client.tables` (metadata), and `client.files` (file uploads) | | **Records** | Dataverse records represented as Python dictionaries with column schema names | | **Schema names** | Use table schema names (`"account"`, `"new_MyTestTable"`) and column schema names (`"name"`, `"new_MyTestColumn"`). See: [Table definitions in Microsoft Dataverse](https://learn.microsoft.com/en-us/power-apps/developer/data-platform/entity-metadata) | | **Bulk Operations** | Efficient bulk processing for multiple records with automatic optimization | @@ -272,42 +273,129 @@ client.dataframe.delete("account", new_accounts["accountid"]) ### Query data +The **QueryBuilder** is the recommended way to query records. It provides a fluent, type-safe interface that generates correct OData queries automatically — no need to remember OData filter syntax. + +```python +# Fluent query builder (recommended) +for record in (client.query.builder("account") + .select("name", "revenue") + .filter_eq("statecode", 0) + .filter_gt("revenue", 1000000) + .order_by("revenue", descending=True) + .top(100) + .page_size(50) + .execute()): + print(f"{record['name']}: {record['revenue']}") +``` + +The QueryBuilder handles value formatting, column name casing, and OData syntax automatically. All filter methods are discoverable via IDE autocomplete: + +```python +# Get results as a pandas DataFrame (consolidates all pages) +df = (client.query.builder("account") + .select("name", "telephone1") + .filter_eq("statecode", 0) + .top(100) + .to_dataframe()) +print(f"Got {len(df)} accounts") +``` + +```python +# Comparison filters +query = (client.query.builder("contact") + .filter_eq("statecode", 0) # statecode eq 0 + .filter_gt("revenue", 1000000) # revenue gt 1000000 + .filter_contains("name", "Corp") # contains(name, 'Corp') + .filter_in("statecode", [0, 1]) # Microsoft.Dynamics.CRM.In(...) + .filter_between("revenue", 100000, 500000) # (revenue ge 100000 and revenue le 500000) + .filter_null("telephone1") # telephone1 eq null + ) +``` + +For complex logic (OR, NOT, grouping), use the composable expression tree with `where()`: + +```python +from PowerPlatform.Dataverse.models.filters import eq, gt, filter_in, between + +# OR conditions: (statecode = 0 OR statecode = 1) AND revenue > 100k +for record in (client.query.builder("account") + .select("name", "revenue") + .where((eq("statecode", 0) | eq("statecode", 1)) + & gt("revenue", 100000)) + .execute()): + print(record["name"]) + +# NOT, between, and in operators +for record in (client.query.builder("account") + .where(~eq("statecode", 2)) # NOT inactive + .where(between("revenue", 100000, 500000)) # revenue in range + .execute()): + print(record["name"]) +``` + +**Formatted values and annotations** -- request localized labels, currency symbols, and display names: + +```python +# Get formatted values (choice labels, currency, lookup names) +for record in (client.query.builder("account") + .select("name", "statecode", "revenue") + .include_formatted_values() + .execute()): + status = record["statecode@OData.Community.Display.V1.FormattedValue"] + print(f"{record['name']}: {status}") +``` + +**Nested expand with options** -- expand navigation properties with `$select`, `$filter`, `$orderby`, and `$top`: + +```python +from PowerPlatform.Dataverse.models.query_builder import ExpandOption + +# Expand related tasks with filtering and sorting +for record in (client.query.builder("account") + .select("name") + .expand(ExpandOption("Account_Tasks") + .select("subject", "createdon") + .filter("contains(subject,'Task')") + .order_by("createdon", descending=True) + .top(5)) + .execute()): + print(record["name"], record.get("Account_Tasks")) +``` + +**Record count** -- include `$count=true` in the request: + +```python +# Request count alongside results +results = (client.query.builder("account") + .filter_eq("statecode", 0) + .count() + .execute()) +``` + +**SQL queries** provide an alternative read-only query syntax: + ```python -# SQL query (read-only) results = client.query.sql( "SELECT TOP 10 accountid, name FROM account WHERE statecode = 0" ) for record in results: print(record["name"]) +``` + +**Raw OData queries** are available via `records.get()` for cases where you need direct control over the OData filter string: -# OData query with paging -# Note: filter and expand parameters are case sensitive +```python for page in client.records.get( "account", - select=["accountid", "name"], # select is case-insensitive (automatically lowercased) - filter="statecode eq 0", # filter must use lowercase logical names (not transformed) + select=["name"], + filter="statecode eq 0", # Raw OData: column names must be lowercase + expand=["primarycontactid"], # Navigation properties are case-sensitive top=100, ): for record in page: print(record["name"]) - -# Query with navigation property expansion (case-sensitive!) -for page in client.records.get( - "account", - select=["name"], - expand=["primarycontactid"], # Navigation property names are case-sensitive - filter="statecode eq 0", # Column names must be lowercase logical names -): - for account in page: - contact = account.get("primarycontactid", {}) - print(f"{account['name']} - Contact: {contact.get('fullname', 'N/A')}") ``` -> **Important**: When using `filter` and `expand` parameters: -> - **`filter`**: Column names must use exact lowercase logical names (e.g., `"statecode eq 0"`, not `"StateCode eq 0"`) -> - **`expand`**: Navigation property names are case-sensitive and must match the exact server names -> - **`select`** and **`orderby`**: Case-insensitive; automatically converted to lowercase - ### Table management ```python diff --git a/examples/advanced/walkthrough.py b/examples/advanced/walkthrough.py index 2b0fa849..ef633d00 100644 --- a/examples/advanced/walkthrough.py +++ b/examples/advanced/walkthrough.py @@ -7,7 +7,8 @@ This example shows: - Table creation with various column types including enums - Single and multiple record CRUD operations -- Querying with filtering, paging, and SQL +- Querying with filtering, paging, QueryBuilder, and SQL +- Expand (navigation properties) with QueryBuilder - Picklist label-to-value conversion - Column management - Cleanup @@ -24,6 +25,8 @@ from azure.identity import InteractiveBrowserCredential from PowerPlatform.Dataverse.client import DataverseClient from PowerPlatform.Dataverse.core.errors import MetadataError +from PowerPlatform.Dataverse.models.filters import eq, gt, between +from PowerPlatform.Dataverse.models.query_builder import ExpandOption import requests @@ -254,10 +257,160 @@ def _run_walkthrough(client): print(f" Page {page_num}: {len(page)} records - IDs: {record_ids}") # ============================================================================ - # 7. SQL QUERY + # 7. QUERYBUILDER - FLUENT QUERIES # ============================================================================ print("\n" + "=" * 80) - print("7. SQL Query") + print("7. QueryBuilder - Fluent Queries") + print("=" * 80) + + # Basic fluent query: active records sorted by amount (flat iteration) + log_call("client.query.builder(...).select().filter_eq().order_by().execute()") + print("Querying incomplete records ordered by amount (fluent builder)...") + qb_records = list( + backoff( + lambda: client.query.builder(table_name) + .select("new_Title", "new_Amount", "new_Priority") + .filter_eq("new_Completed", False) + .order_by("new_Amount", descending=True) + .top(10) + .execute() + ) + ) + print(f"[OK] QueryBuilder found {len(qb_records)} incomplete records:") + for rec in qb_records[:5]: + print(f" - '{rec.get('new_title')}' Amount={rec.get('new_amount')}") + + # filter_in: records with specific priorities + log_call("client.query.builder(...).filter_in('new_Priority', [HIGH, LOW]).execute()") + print("Querying records with HIGH or LOW priority (filter_in)...") + priority_records = list( + backoff( + lambda: client.query.builder(table_name) + .select("new_Title", "new_Priority") + .filter_in("new_Priority", [Priority.HIGH, Priority.LOW]) + .execute() + ) + ) + print(f"[OK] Found {len(priority_records)} records with HIGH or LOW priority") + for rec in priority_records[:5]: + print(f" - '{rec.get('new_title')}' Priority={rec.get('new_priority')}") + + # filter_between: amount in a range + log_call("client.query.builder(...).filter_between('new_Amount', 500, 1500).execute()") + print("Querying records with amount between 500 and 1500 (filter_between)...") + range_records = list( + backoff( + lambda: client.query.builder(table_name) + .select("new_Title", "new_Amount") + .filter_between("new_Amount", 500, 1500) + .execute() + ) + ) + print(f"[OK] Found {len(range_records)} records with amount in [500, 1500]") + for rec in range_records: + print(f" - '{rec.get('new_title')}' Amount={rec.get('new_amount')}") + + # Composable expression tree with where() + log_call("client.query.builder(...).where((eq(...) | eq(...)) & gt(...)).execute()") + print("Querying with composable expression tree (where)...") + expr_records = list( + backoff( + lambda: client.query.builder(table_name) + .select("new_Title", "new_Amount", "new_Quantity") + .where((eq("new_Completed", False) & gt("new_Amount", 100))) + .order_by("new_Amount", descending=True) + .top(5) + .execute() + ) + ) + print(f"[OK] Expression tree query found {len(expr_records)} records:") + for rec in expr_records: + print(f" - '{rec.get('new_title')}' Amount={rec.get('new_amount')} Qty={rec.get('new_quantity')}") + + # Combined: fluent filters + expression tree + paging (by_page=True) + log_call("client.query.builder(...).filter_eq().where(between()).page_size().execute(by_page=True)") + print("Querying with combined fluent + expression filters and paging...") + combined_page_count = 0 + combined_record_count = 0 + for page in backoff( + lambda: client.query.builder(table_name) + .select("new_Title", "new_Quantity") + .filter_eq("new_Completed", False) + .where(between("new_Quantity", 1, 15)) + .order_by("new_Quantity") + .page_size(3) + .execute(by_page=True) + ): + combined_page_count += 1 + combined_record_count += len(page) + titles = [r.get("new_title", "?") for r in page] + print(f" Page {combined_page_count}: {len(page)} records - {titles}") + print(f"[OK] Combined query: {combined_record_count} records across {combined_page_count} page(s)") + + # to_dataframe: get results as a pandas DataFrame + log_call(f"client.query.builder('{table_name}').select(...).filter_eq(...).to_dataframe()") + print("Querying completed records as a pandas DataFrame (to_dataframe)...") + df = backoff( + lambda: ( + client.query.builder(table_name) + .select("new_title", "new_quantity") + .filter_eq("new_completed", True) + .to_dataframe() + ) + ) + print(f"[OK] to_dataframe() returned {len(df)} rows, columns: {list(df.columns)}") + if not df.empty: + print(f" First row: new_title='{df.iloc[0].get('new_title')}', new_quantity={df.iloc[0].get('new_quantity')}") + print(f" Sum of new_quantity: {df['new_quantity'].sum()}") + else: + print(" (empty DataFrame)") + + # ============================================================================ + # 8. EXPAND (NAVIGATION PROPERTIES) + # ============================================================================ + print("\n" + "=" * 80) + print("8. Expand (Navigation Properties)") + print("=" * 80) + + # Simple expand: fetch accounts with their primary contact in one request + log_call("client.query.builder('account').select('name').expand('primarycontactid').top(3).execute()") + print("Querying accounts with primary contact expanded...") + try: + expanded_records = list( + backoff(lambda: client.query.builder("account").select("name").expand("primarycontactid").top(3).execute()) + ) + print(f"[OK] Found {len(expanded_records)} accounts with expanded contact:") + for rec in expanded_records: + contact = rec.get("primarycontactid") + contact_name = contact.get("fullname", "(none)") if contact else "(no contact)" + print(f" - '{rec.get('name')}' -> Contact: {contact_name}") + except Exception as e: # noqa: BLE001 + print(f"[SKIP] Expand demo skipped (no accounts in org): {e}") + + # ExpandOption with nested $select, $filter, $orderby, $top + log_call("ExpandOption('Account_Tasks').select('subject').order_by('createdon', descending=True).top(3)") + print("Querying accounts with nested expand options on tasks...") + try: + tasks_opt = ( + ExpandOption("Account_Tasks").select("subject", "createdon").order_by("createdon", descending=True).top(3) + ) + nested_records = list( + backoff(lambda: client.query.builder("account").select("name").expand(tasks_opt).top(3).execute()) + ) + print(f"[OK] Found {len(nested_records)} accounts with nested task expansion:") + for rec in nested_records: + tasks = rec.get("Account_Tasks", []) + print(f" - '{rec.get('name')}' has {len(tasks)} task(s)") + for task in tasks: + print(f" - {task.get('subject')}") + except Exception as e: # noqa: BLE001 + print(f"[SKIP] Nested expand demo skipped: {e}") + + # ============================================================================ + # 9. SQL QUERY + # ============================================================================ + print("\n" + "=" * 80) + print("9. SQL Query") print("=" * 80) log_call(f"client.query.sql('SELECT new_title, new_quantity FROM {table_name} WHERE new_completed = 1')") @@ -271,10 +424,10 @@ def _run_walkthrough(client): print(f"[WARN] SQL query failed (known server-side bug): {str(e)}") # ============================================================================ - # 8. PICKLIST LABEL CONVERSION + # 10. PICKLIST LABEL CONVERSION # ============================================================================ print("\n" + "=" * 80) - print("8. Picklist Label Conversion") + print("10. Picklist Label Conversion") print("=" * 80) log_call(f"client.records.create('{table_name}', {{'new_Priority': 'High'}})") @@ -292,10 +445,10 @@ def _run_walkthrough(client): print(f" new_Priority@FormattedValue: {retrieved.get('new_priority@OData.Community.Display.V1.FormattedValue')}") # ============================================================================ - # 9. COLUMN MANAGEMENT + # 11. COLUMN MANAGEMENT # ============================================================================ print("\n" + "=" * 80) - print("9. Column Management") + print("11. Column Management") print("=" * 80) log_call(f"client.tables.add_columns('{table_name}', {{'new_Notes': 'string'}})") @@ -308,10 +461,10 @@ def _run_walkthrough(client): print(f"[OK] Deleted column: new_Notes") # ============================================================================ - # 10. DELETE OPERATIONS + # 12. DELETE OPERATIONS # ============================================================================ print("\n" + "=" * 80) - print("10. Delete Operations") + print("12. Delete Operations") print("=" * 80) # Single delete @@ -326,19 +479,24 @@ def _run_walkthrough(client): print(f" (Deleting {len(paging_ids)} paging demo records)") # ============================================================================ - # 11. CLEANUP + # 13. CLEANUP # ============================================================================ print("\n" + "=" * 80) - print("11. Cleanup") + print("13. Cleanup") print("=" * 80) log_call(f"client.tables.delete('{table_name}')") try: backoff(lambda: client.tables.delete(table_name)) print(f"[OK] Deleted table: {table_name}") + except MetadataError as ex: + if "not found" in str(ex).lower(): + print(f"[OK] Table already removed: {table_name}") + else: + raise except Exception as ex: # noqa: BLE001 code = getattr(getattr(ex, "response", None), "status_code", None) - if isinstance(ex, (requests.exceptions.HTTPError, MetadataError)) and code == 404: + if isinstance(ex, requests.exceptions.HTTPError) and code == 404: print(f"[OK] Table removed: {table_name}") else: raise @@ -355,6 +513,8 @@ def _run_walkthrough(client): print(" [OK] Reading records by ID and with filters") print(" [OK] Single and multiple record updates") print(" [OK] Paging through large result sets") + print(" [OK] QueryBuilder fluent queries (filter_eq, filter_in, filter_between, where, to_dataframe)") + print(" [OK] Expand navigation properties (simple + nested ExpandOption)") print(" [OK] SQL queries") print(" [OK] Picklist label-to-value conversion") print(" [OK] Column management") diff --git a/src/PowerPlatform/Dataverse/data/_odata.py b/src/PowerPlatform/Dataverse/data/_odata.py index 633c736f..78ce4091 100644 --- a/src/PowerPlatform/Dataverse/data/_odata.py +++ b/src/PowerPlatform/Dataverse/data/_odata.py @@ -742,6 +742,8 @@ def _get_multiple( top: Optional[int] = None, expand: Optional[List[str]] = None, page_size: Optional[int] = None, + count: bool = False, + include_annotations: Optional[str] = None, ) -> Iterable[List[Dict[str, Any]]]: """Iterate records from an entity set, yielding one page (list of dicts) at a time. @@ -759,16 +761,25 @@ def _get_multiple( :type expand: ``list[str]`` | ``None`` :param page_size: Per-page size hint via ``Prefer: odata.maxpagesize``. :type page_size: ``int`` | ``None`` + :param count: If ``True``, adds ``$count=true`` to include a total record count in the response. + :type count: ``bool`` + :param include_annotations: OData annotation pattern for the ``Prefer: odata.include-annotations`` header (e.g. ``"*"`` or ``"OData.Community.Display.V1.FormattedValue"``), or ``None``. + :type include_annotations: ``str`` | ``None`` :return: Iterator yielding pages (each page is a ``list`` of record dicts). :rtype: ``Iterable[list[dict[str, Any]]]`` """ extra_headers: Dict[str, str] = {} + prefer_parts: List[str] = [] if page_size is not None: ps = int(page_size) if ps > 0: - extra_headers["Prefer"] = f"odata.maxpagesize={ps}" + prefer_parts.append(f"odata.maxpagesize={ps}") + if include_annotations: + prefer_parts.append(f'odata.include-annotations="{include_annotations}"') + if prefer_parts: + extra_headers["Prefer"] = ",".join(prefer_parts) def _do_request(url: str, *, params: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: headers = extra_headers if extra_headers else None @@ -795,6 +806,8 @@ def _do_request(url: str, *, params: Optional[Dict[str, Any]] = None) -> Dict[st params["$expand"] = ",".join(expand) if top is not None: params["$top"] = int(top) + if count: + params["$count"] = "true" data = _do_request(base_url, params=params) items = data.get("value") if isinstance(data, dict) else None diff --git a/src/PowerPlatform/Dataverse/models/__init__.py b/src/PowerPlatform/Dataverse/models/__init__.py index d37d0849..dc10a4c0 100644 --- a/src/PowerPlatform/Dataverse/models/__init__.py +++ b/src/PowerPlatform/Dataverse/models/__init__.py @@ -1,6 +1,19 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -"""Data models for Dataverse metadata types.""" +""" +Data models and type definitions for the Dataverse SDK. + +Provides dataclasses and helpers for Dataverse entities: + +- :class:`~PowerPlatform.Dataverse.models.query_builder.QueryBuilder`: Fluent query builder. +- :mod:`~PowerPlatform.Dataverse.models.filters`: Composable OData filter expressions. +- :class:`~PowerPlatform.Dataverse.models.upsert.UpsertItem`: Upsert operation item. + +Import directly from the specific module, e.g.:: + + from PowerPlatform.Dataverse.models.query_builder import QueryBuilder + from PowerPlatform.Dataverse.models.filters import eq, gt +""" __all__ = [] diff --git a/src/PowerPlatform/Dataverse/models/filters.py b/src/PowerPlatform/Dataverse/models/filters.py new file mode 100644 index 00000000..c5de258c --- /dev/null +++ b/src/PowerPlatform/Dataverse/models/filters.py @@ -0,0 +1,465 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Composable OData filter expressions for the Dataverse SDK. + +Provides an expression tree that compiles to OData ``$filter`` strings, +with Python operator overloads (``&``, ``|``, ``~``) for composing +complex filter conditions. + +Example:: + + from PowerPlatform.Dataverse.models.filters import eq, gt, filter_in + + # Simple comparison + expr = eq("statecode", 0) + print(expr.to_odata()) # statecode eq 0 + + # Complex composition with OR and AND + expr = (eq("statecode", 0) | eq("statecode", 1)) & gt("revenue", 100000) + print(expr.to_odata()) + # ((statecode eq 0 or statecode eq 1) and revenue gt 100000) + + # In operator (Dataverse function) + expr = filter_in("statecode", [0, 1, 2]) + print(expr.to_odata()) + # Microsoft.Dynamics.CRM.In(PropertyName='statecode',PropertyValues=["0","1","2"]) + + # Negation + expr = ~eq("statecode", 1) + print(expr.to_odata()) # not (statecode eq 1) +""" + +from __future__ import annotations + +import enum +import uuid +from datetime import date, datetime, timezone +from typing import Any, Collection, Sequence + +__all__ = [ + "FilterExpression", + "eq", + "ne", + "gt", + "ge", + "lt", + "le", + "contains", + "startswith", + "endswith", + "between", + "is_null", + "is_not_null", + "filter_in", + "not_in", + "not_between", + "raw", +] + + +# --------------------------------------------------------------------------- +# Value formatting +# --------------------------------------------------------------------------- + + +def _format_value(value: Any) -> str: + """Format a Python value for OData query syntax. + + Handles: ``None``, ``bool``, ``int``, ``float``, ``str``, + ``datetime``, ``date``, ``uuid.UUID``. + + .. note:: + ``bool`` is checked before ``int`` because ``bool`` is a subclass + of ``int`` in Python. Without this ordering ``True`` would format + as ``1`` instead of ``true``. + """ + if value is None: + return "null" + # bool MUST be checked before int (bool is a subclass of int) + if isinstance(value, bool): + return "true" if value else "false" + # Enum/IntEnum MUST be checked before int (IntEnum is a subclass of int) + if isinstance(value, enum.Enum): + return _format_value(value.value) + if isinstance(value, int): + return str(value) + if isinstance(value, float): + return str(value) + if isinstance(value, str): + escaped = value.replace("'", "''") + return f"'{escaped}'" + if isinstance(value, datetime): + # Convert timezone-aware datetimes to UTC; assume naive datetimes are UTC + if value.tzinfo is not None: + value = value.astimezone(timezone.utc) + if value.microsecond: + return value.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + return value.strftime("%Y-%m-%dT%H:%M:%SZ") + if isinstance(value, date): + return value.strftime("%Y-%m-%d") + if isinstance(value, uuid.UUID): + return str(value) + # Fallback + return str(value) + + +# --------------------------------------------------------------------------- +# Base class +# --------------------------------------------------------------------------- + + +class FilterExpression: + """Base class for composable OData filter expressions. + + Supports Python operator overloads for logical composition: + + - ``expr1 & expr2`` produces ``(expr1 and expr2)`` + - ``expr1 | expr2`` produces ``(expr1 or expr2)`` + - ``~expr`` produces ``not (expr)`` + """ + + def to_odata(self) -> str: + """Compile this expression to an OData ``$filter`` string.""" + raise NotImplementedError + + def __and__(self, other: FilterExpression) -> FilterExpression: + if not isinstance(other, FilterExpression): + return NotImplemented + return _AndFilter(self, other) + + def __or__(self, other: FilterExpression) -> FilterExpression: + if not isinstance(other, FilterExpression): + return NotImplemented + return _OrFilter(self, other) + + def __invert__(self) -> FilterExpression: + return _NotFilter(self) + + def __str__(self) -> str: + return self.to_odata() + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.to_odata()!r})" + + +# --------------------------------------------------------------------------- +# Internal expression classes +# --------------------------------------------------------------------------- + + +class _ComparisonFilter(FilterExpression): + """Comparison filter: ``column op value``.""" + + __slots__ = ("column", "op", "value") + + def __init__(self, column: str, op: str, value: Any) -> None: + self.column = column.lower() + self.op = op + self.value = value + + def to_odata(self) -> str: + return f"{self.column} {self.op} {_format_value(self.value)}" + + +class _FunctionFilter(FilterExpression): + """Function filter: ``func(column, value)``.""" + + __slots__ = ("func_name", "column", "value") + + def __init__(self, func_name: str, column: str, value: Any) -> None: + self.func_name = func_name + self.column = column.lower() + self.value = value + + def to_odata(self) -> str: + return f"{self.func_name}({self.column}, {_format_value(self.value)})" + + +class _AndFilter(FilterExpression): + """Logical AND: ``(left and right)``.""" + + __slots__ = ("left", "right") + + def __init__(self, left: FilterExpression, right: FilterExpression) -> None: + self.left = left + self.right = right + + def to_odata(self) -> str: + return f"({self.left.to_odata()} and {self.right.to_odata()})" + + +class _OrFilter(FilterExpression): + """Logical OR: ``(left or right)``.""" + + __slots__ = ("left", "right") + + def __init__(self, left: FilterExpression, right: FilterExpression) -> None: + self.left = left + self.right = right + + def to_odata(self) -> str: + return f"({self.left.to_odata()} or {self.right.to_odata()})" + + +class _NotFilter(FilterExpression): + """Logical NOT: ``not (expr)``.""" + + __slots__ = ("expr",) + + def __init__(self, expr: FilterExpression) -> None: + self.expr = expr + + def to_odata(self) -> str: + return f"not ({self.expr.to_odata()})" + + +class _InFilter(FilterExpression): + """In filter using ``Microsoft.Dynamics.CRM.In``.""" + + __slots__ = ("column", "values") + + def __init__(self, column: str, values: Collection[Any]) -> None: + if not values: + raise ValueError("filter_in requires at least one value") + self.column = column.lower() + self.values = list(values) + + def to_odata(self) -> str: + # PropertyValues is Collection(Edm.String) + parts = [f'"{_format_value(v).strip("'")}"' for v in self.values] + formatted = ",".join(parts) + return f"Microsoft.Dynamics.CRM.In" f"(PropertyName='{self.column}',PropertyValues=[{formatted}])" + + +class _NotInFilter(FilterExpression): + """Not-in filter using ``Microsoft.Dynamics.CRM.NotIn``.""" + + __slots__ = ("column", "values") + + def __init__(self, column: str, values: Collection[Any]) -> None: + if not values: + raise ValueError("not_in requires at least one value") + self.column = column.lower() + self.values = list(values) + + def to_odata(self) -> str: + # Same Collection(Edm.String) rules as _InFilter. + parts = [f'"{_format_value(v).strip("'")}"' for v in self.values] + formatted = ",".join(parts) + return f"Microsoft.Dynamics.CRM.NotIn" f"(PropertyName='{self.column}',PropertyValues=[{formatted}])" + + +class _RawFilter(FilterExpression): + """Raw verbatim OData filter expression.""" + + __slots__ = ("filter_string",) + + def __init__(self, filter_string: str) -> None: + self.filter_string = filter_string + + def to_odata(self) -> str: + return self.filter_string + + +# --------------------------------------------------------------------------- +# Public factory functions +# --------------------------------------------------------------------------- + + +def eq(column: str, value: Any) -> FilterExpression: + """Equality filter: ``column eq value``. + + :param column: Column name (will be lowercased). + :param value: Value to compare against. + :return: A filter expression. + + Example:: + + eq("statecode", 0).to_odata() # "statecode eq 0" + """ + return _ComparisonFilter(column, "eq", value) + + +def ne(column: str, value: Any) -> FilterExpression: + """Not-equal filter: ``column ne value``. + + :param column: Column name (will be lowercased). + :param value: Value to compare against. + :return: A filter expression. + """ + return _ComparisonFilter(column, "ne", value) + + +def gt(column: str, value: Any) -> FilterExpression: + """Greater-than filter: ``column gt value``. + + :param column: Column name (will be lowercased). + :param value: Value to compare against. + :return: A filter expression. + """ + return _ComparisonFilter(column, "gt", value) + + +def ge(column: str, value: Any) -> FilterExpression: + """Greater-than-or-equal filter: ``column ge value``. + + :param column: Column name (will be lowercased). + :param value: Value to compare against. + :return: A filter expression. + """ + return _ComparisonFilter(column, "ge", value) + + +def lt(column: str, value: Any) -> FilterExpression: + """Less-than filter: ``column lt value``. + + :param column: Column name (will be lowercased). + :param value: Value to compare against. + :return: A filter expression. + """ + return _ComparisonFilter(column, "lt", value) + + +def le(column: str, value: Any) -> FilterExpression: + """Less-than-or-equal filter: ``column le value``. + + :param column: Column name (will be lowercased). + :param value: Value to compare against. + :return: A filter expression. + """ + return _ComparisonFilter(column, "le", value) + + +def contains(column: str, value: str) -> FilterExpression: + """Contains filter: ``contains(column, value)``. + + :param column: Column name (will be lowercased). + :param value: Substring to search for. + :return: A filter expression. + """ + return _FunctionFilter("contains", column, value) + + +def startswith(column: str, value: str) -> FilterExpression: + """Startswith filter: ``startswith(column, value)``. + + :param column: Column name (will be lowercased). + :param value: Prefix to match. + :return: A filter expression. + """ + return _FunctionFilter("startswith", column, value) + + +def endswith(column: str, value: str) -> FilterExpression: + """Endswith filter: ``endswith(column, value)``. + + :param column: Column name (will be lowercased). + :param value: Suffix to match. + :return: A filter expression. + """ + return _FunctionFilter("endswith", column, value) + + +def between(column: str, low: Any, high: Any) -> FilterExpression: + """Between filter: ``(column ge low and column le high)``. + + Syntactic sugar that composes :func:`ge` and :func:`le` with ``&``. + + :param column: Column name (will be lowercased). + :param low: Lower bound (inclusive). + :param high: Upper bound (inclusive). + :return: A composed filter expression. + + Example:: + + between("revenue", 100000, 500000).to_odata() + # "(revenue ge 100000 and revenue le 500000)" + """ + return ge(column, low) & le(column, high) + + +def is_null(column: str) -> FilterExpression: + """Null check: ``column eq null``. + + :param column: Column name (will be lowercased). + :return: A filter expression. + """ + return _ComparisonFilter(column, "eq", None) + + +def is_not_null(column: str) -> FilterExpression: + """Not-null check: ``column ne null``. + + :param column: Column name (will be lowercased). + :return: A filter expression. + """ + return _ComparisonFilter(column, "ne", None) + + +def filter_in(column: str, values: Collection[Any]) -> FilterExpression: + """In filter using ``Microsoft.Dynamics.CRM.In``. + + Named ``filter_in`` because ``in`` is a Python keyword. + + :param column: Column name (will be lowercased). + :param values: Non-empty sequence of values. + :return: A filter expression. + :raises ValueError: If ``values`` is empty. + + Example:: + + filter_in("statecode", [0, 1, 2]).to_odata() + # "Microsoft.Dynamics.CRM.In(PropertyName='statecode',PropertyValues=["0","1","2"])" + """ + return _InFilter(column, values) + + +def not_in(column: str, values: Collection[Any]) -> FilterExpression: + """Not-in filter using ``Microsoft.Dynamics.CRM.NotIn``. + + Named ``not_in`` to parallel :func:`filter_in`. + + :param column: Column name (will be lowercased). + :param values: Non-empty sequence of values. + :return: A filter expression. + :raises ValueError: If ``values`` is empty. + + Example:: + + not_in("statecode", [0, 1]).to_odata() + # "Microsoft.Dynamics.CRM.NotIn(PropertyName='statecode',PropertyValues=[\"0\",\"1\"])" + """ + return _NotInFilter(column, values) + + +def not_between(column: str, low: Any, high: Any) -> FilterExpression: + """Not-between filter: ``not (column ge low and column le high)``. + + Syntactic sugar that negates :func:`between` with ``~``. + + :param column: Column name (will be lowercased). + :param low: Lower bound (inclusive, will be excluded). + :param high: Upper bound (inclusive, will be excluded). + :return: A composed filter expression. + + Example:: + + not_between("revenue", 100000, 500000).to_odata() + # "not ((revenue ge 100000 and revenue le 500000))" + """ + return ~between(column, low, high) + + +def raw(filter_string: str) -> FilterExpression: + """Verbatim OData filter expression (passed through unchanged). + + :param filter_string: Raw OData filter string. + :return: A filter expression. + + Example:: + + raw("Microsoft.Dynamics.CRM.Today(PropertyName='createdon')") + """ + return _RawFilter(filter_string) diff --git a/src/PowerPlatform/Dataverse/models/query_builder.py b/src/PowerPlatform/Dataverse/models/query_builder.py new file mode 100644 index 00000000..dbd79b36 --- /dev/null +++ b/src/PowerPlatform/Dataverse/models/query_builder.py @@ -0,0 +1,792 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Fluent query builder for constructing OData queries. + +Provides a type-safe, discoverable interface for building complex queries +against Dataverse tables with method chaining. + +Example:: + + # Via client (recommended) -- flat iteration over records + for record in (client.query.builder("account") + .select("name", "revenue") + .filter_eq("statecode", 0) + .filter_gt("revenue", 1000000) + .order_by("revenue", descending=True) + .top(100) + .execute()): + print(record["name"]) + + # With composable expression tree + from PowerPlatform.Dataverse.models.filters import eq, gt + + for record in (client.query.builder("account") + .select("name", "revenue") + .where((eq("statecode", 0) | eq("statecode", 1)) + & gt("revenue", 100000)) + .top(100) + .execute()): + print(record["name"]) + + # Opt-in paged iteration (for batch processing) + for page in (client.query.builder("account") + .select("name") + .execute(by_page=True)): + process_batch(page) + + # Get results as a pandas DataFrame + df = (client.query.builder("account") + .select("name", "telephone1") + .filter_eq("statecode", 0) + .top(100) + .to_dataframe()) +""" + +from __future__ import annotations + +from typing import Any, Collection, Dict, Iterable, List, Optional, Sequence, TypedDict, Union + +import pandas as pd + +from . import filters +from .record import Record + +__all__ = ["QueryBuilder", "QueryParams", "ExpandOption"] + + +class QueryParams(TypedDict, total=False): + """Typed dictionary returned by :meth:`QueryBuilder.build`. + + Provides IDE autocomplete when passing build results to + ``client.records.get()`` manually. + """ + + table: str + select: List[str] + filter: str + orderby: List[str] + expand: List[str] + top: int + page_size: int + count: bool + include_annotations: str + + +class ExpandOption: + """Structured options for an ``$expand`` navigation property. + + Allows specifying nested ``$select``, ``$filter``, ``$orderby``, and + ``$top`` options for a single navigation property expansion, following + the OData ``$expand`` syntax. + + :param relation: Navigation property name (case-sensitive). + :type relation: str + + Example:: + + # Expand Account_Tasks with nested options + opt = (ExpandOption("Account_Tasks") + .select("subject", "createdon") + .filter("contains(subject,'Task')") + .order_by("createdon", descending=True) + .top(5)) + + query = (client.query.builder("account") + .select("name") + .expand(opt) + .execute()) + """ + + def __init__(self, relation: str) -> None: + self.relation = relation + self._select: List[str] = [] + self._filter: Optional[str] = None + self._orderby: List[str] = [] + self._top: Optional[int] = None + + def select(self, *columns: str) -> ExpandOption: + """Select specific columns from the expanded entity. + + :param columns: Column names to select. + :return: Self for method chaining. + """ + self._select.extend(columns) + return self + + def filter(self, filter_str: str) -> ExpandOption: + """Filter the expanded collection. + + :param filter_str: OData ``$filter`` expression. + :return: Self for method chaining. + """ + self._filter = filter_str + return self + + def order_by(self, column: str, descending: bool = False) -> ExpandOption: + """Sort the expanded collection. + + :param column: Column name to sort by. + :param descending: Sort descending if ``True``. + :return: Self for method chaining. + """ + order = f"{column} desc" if descending else column + self._orderby.append(order) + return self + + def top(self, count: int) -> ExpandOption: + """Limit expanded results. + + :param count: Maximum number of expanded records. + :return: Self for method chaining. + """ + self._top = count + return self + + def to_odata(self) -> str: + """Compile to OData ``$expand`` syntax. + + :return: OData expand string like ``"Nav($select=col1,col2;$filter=...)"`` + :rtype: str + """ + options: List[str] = [] + if self._select: + options.append(f"$select={','.join(self._select)}") + if self._filter: + options.append(f"$filter={self._filter}") + if self._orderby: + options.append(f"$orderby={','.join(self._orderby)}") + if self._top is not None: + options.append(f"$top={self._top}") + if options: + return f"{self.relation}({';'.join(options)})" + return self.relation + + +class QueryBuilder: + """Fluent interface for building OData queries. + + Provides method chaining for constructing complex queries with + type-safe filter operations. Can be used standalone (via :meth:`build`) + or bound to a client (via :meth:`execute`). + + :param table: Table schema name to query. + :type table: str + :raises ValueError: If ``table`` is empty. + + Example: + Standalone query construction:: + + query = (QueryBuilder("account") + .select("name") + .filter_eq("statecode", 0) + .top(10)) + params = query.build() + # {"table": "account", "select": ["name"], + # "filter": "statecode eq 0", "top": 10} + """ + + def __init__(self, table: str) -> None: + table = table.strip() if table else "" + if not table: + raise ValueError("table name is required") + self.table = table + self._select: List[str] = [] + self._filter_parts: List[Union[str, filters.FilterExpression]] = [] + self._orderby: List[str] = [] + self._expand: List[str] = [] + self._top: Optional[int] = None + self._page_size: Optional[int] = None + self._count: bool = False + self._include_annotations: Optional[str] = None + self._query_ops: Optional[Any] = None # Set by QueryOperations.builder() + + # ----------------------------------------------------------------- select + + def select(self, *columns: str) -> QueryBuilder: + """Select specific columns to retrieve. + + Column names are passed as-is; the OData layer lowercases them + automatically. Can be called multiple times (additive). + + :param columns: Column names to select. + :return: Self for method chaining. + + Example:: + + query = QueryBuilder("account").select("name", "telephone1", "revenue") + """ + self._select.extend(columns) + return self + + # ----------------------------------------------------------- filter: comparison + + def filter_eq(self, column: str, value: Any) -> QueryBuilder: + """Add equality filter: ``column eq value``. + + :param column: Column name (will be lowercased). + :param value: Value to compare against. + :return: Self for method chaining. + """ + self._filter_parts.append(filters.eq(column, value)) + return self + + def filter_ne(self, column: str, value: Any) -> QueryBuilder: + """Add not-equal filter: ``column ne value``. + + :param column: Column name (will be lowercased). + :param value: Value to compare against. + :return: Self for method chaining. + """ + self._filter_parts.append(filters.ne(column, value)) + return self + + def filter_gt(self, column: str, value: Any) -> QueryBuilder: + """Add greater-than filter: ``column gt value``. + + :param column: Column name (will be lowercased). + :param value: Value to compare against. + :return: Self for method chaining. + """ + self._filter_parts.append(filters.gt(column, value)) + return self + + def filter_ge(self, column: str, value: Any) -> QueryBuilder: + """Add greater-than-or-equal filter: ``column ge value``. + + :param column: Column name (will be lowercased). + :param value: Value to compare against. + :return: Self for method chaining. + """ + self._filter_parts.append(filters.ge(column, value)) + return self + + def filter_lt(self, column: str, value: Any) -> QueryBuilder: + """Add less-than filter: ``column lt value``. + + :param column: Column name (will be lowercased). + :param value: Value to compare against. + :return: Self for method chaining. + """ + self._filter_parts.append(filters.lt(column, value)) + return self + + def filter_le(self, column: str, value: Any) -> QueryBuilder: + """Add less-than-or-equal filter: ``column le value``. + + :param column: Column name (will be lowercased). + :param value: Value to compare against. + :return: Self for method chaining. + """ + self._filter_parts.append(filters.le(column, value)) + return self + + # --------------------------------------------------------- filter: string functions + + def filter_contains(self, column: str, value: str) -> QueryBuilder: + """Add contains filter: ``contains(column, value)``. + + :param column: Column name (will be lowercased). + :param value: Substring to search for. + :return: Self for method chaining. + """ + self._filter_parts.append(filters.contains(column, value)) + return self + + def filter_startswith(self, column: str, value: str) -> QueryBuilder: + """Add startswith filter: ``startswith(column, value)``. + + :param column: Column name (will be lowercased). + :param value: Prefix to match. + :return: Self for method chaining. + """ + self._filter_parts.append(filters.startswith(column, value)) + return self + + def filter_endswith(self, column: str, value: str) -> QueryBuilder: + """Add endswith filter: ``endswith(column, value)``. + + :param column: Column name (will be lowercased). + :param value: Suffix to match. + :return: Self for method chaining. + """ + self._filter_parts.append(filters.endswith(column, value)) + return self + + # --------------------------------------------------------- filter: null checks + + def filter_null(self, column: str) -> QueryBuilder: + """Add null check: ``column eq null``. + + :param column: Column name (will be lowercased). + :return: Self for method chaining. + """ + self._filter_parts.append(filters.is_null(column)) + return self + + def filter_not_null(self, column: str) -> QueryBuilder: + """Add not-null check: ``column ne null``. + + :param column: Column name (will be lowercased). + :return: Self for method chaining. + """ + self._filter_parts.append(filters.is_not_null(column)) + return self + + # --------------------------------------------------------- filter: special + + def filter_in(self, column: str, values: Collection[Any]) -> QueryBuilder: + """Add an ``in`` filter using ``Microsoft.Dynamics.CRM.In``. + + :param column: Column name (will be lowercased). + :param values: Non-empty list of values for the ``in`` clause. + :return: Self for method chaining. + :raises ValueError: If ``values`` is empty. + + Example:: + + query = QueryBuilder("account").filter_in("statecode", [0, 1, 2]) + # Produces: Microsoft.Dynamics.CRM.In( + # PropertyName='statecode',PropertyValues=["0","1","2"]) + """ + self._filter_parts.append(filters.filter_in(column, values)) + return self + + def filter_not_in(self, column: str, values: Collection[Any]) -> QueryBuilder: + """Add a ``not in`` filter using ``Microsoft.Dynamics.CRM.NotIn``. + + :param column: Column name (will be lowercased). + :param values: Non-empty list of values to exclude. + :return: Self for method chaining. + :raises ValueError: If ``values`` is empty. + + Example:: + + query = QueryBuilder("account").filter_not_in("statecode", [2, 3]) + # Produces: Microsoft.Dynamics.CRM.NotIn( + # PropertyName='statecode',PropertyValues=["2","3"]) + """ + self._filter_parts.append(filters.not_in(column, values)) + return self + + def filter_between(self, column: str, low: Any, high: Any) -> QueryBuilder: + """Add a between filter: ``(column ge low and column le high)``. + + :param column: Column name (will be lowercased). + :param low: Lower bound (inclusive). + :param high: Upper bound (inclusive). + :return: Self for method chaining. + + Example:: + + query = QueryBuilder("account").filter_between("revenue", 100000, 500000) + # Produces: (revenue ge 100000 and revenue le 500000) + """ + self._filter_parts.append(filters.between(column, low, high)) + return self + + def filter_not_between(self, column: str, low: Any, high: Any) -> QueryBuilder: + """Add a not-between filter: ``not (column ge low and column le high)``. + + :param column: Column name (will be lowercased). + :param low: Lower bound (inclusive, will be excluded). + :param high: Upper bound (inclusive, will be excluded). + :return: Self for method chaining. + + Example:: + + query = QueryBuilder("account").filter_not_between("revenue", 100000, 500000) + # Produces: not ((revenue ge 100000 and revenue le 500000)) + """ + self._filter_parts.append(filters.not_between(column, low, high)) + return self + + def filter_raw(self, filter_string: str) -> QueryBuilder: + """Add a raw OData filter string. + + Use this for complex filters not covered by other methods. + Column names in the filter string should be lowercase. + + .. warning:: + The filter string is passed directly to Dataverse without validation. + Ensure it follows OData filter syntax; a malformed expression will result + in a ``400 Bad Request`` error from the server. + + :param filter_string: Raw OData filter expression. + :return: Self for method chaining. + + Example:: + + query = QueryBuilder("account").filter_raw( + "(statecode eq 0 or statecode eq 1)" + ) + """ + self._filter_parts.append(filters.raw(filter_string)) + return self + + # ------------------------------------------------------ filter: expression tree + + def where(self, expression: filters.FilterExpression) -> QueryBuilder: + """Add a composable filter expression. + + Accepts a :class:`~PowerPlatform.Dataverse.models.filters.FilterExpression` + built using the convenience functions from + :mod:`~PowerPlatform.Dataverse.models.filters`. + + Multiple ``where()`` calls and ``filter_*()`` calls are all + AND-joined together in the order they were called. + + :param expression: A composable filter expression. + :type expression: FilterExpression + :return: Self for method chaining. + :raises TypeError: If ``expression`` is not a FilterExpression. + + Example:: + + from PowerPlatform.Dataverse.models.filters import eq, gt + + query = (QueryBuilder("account") + .where((eq("statecode", 0) | eq("statecode", 1)) + & gt("revenue", 100000))) + """ + if not isinstance(expression, filters.FilterExpression): + raise TypeError(f"where() requires a FilterExpression, got {type(expression).__name__}") + self._filter_parts.append(expression) + return self + + # --------------------------------------------------------------- ordering + + def order_by(self, column: str, descending: bool = False) -> QueryBuilder: + """Add sorting order. + + Can be called multiple times for multi-column sorting. + + :param column: Column name to sort by (will be lowercased). + :param descending: Sort in descending order. + :return: Self for method chaining. + """ + order = f"{column.lower()} desc" if descending else column.lower() + self._orderby.append(order) + return self + + # --------------------------------------------------------------- pagination + + def top(self, count: int) -> QueryBuilder: + """Limit the total number of results. + + :param count: Maximum number of records to return (must be >= 1). + :return: Self for method chaining. + :raises ValueError: If ``count`` is less than 1. + """ + if count < 1: + raise ValueError("top count must be at least 1") + self._top = count + return self + + def page_size(self, size: int) -> QueryBuilder: + """Set the number of records per page. + + Controls how many records are returned in each page/batch + via the ``Prefer: odata.maxpagesize`` header. + + :param size: Number of records per page (must be >= 1). + :return: Self for method chaining. + :raises ValueError: If ``size`` is less than 1. + """ + if size < 1: + raise ValueError("page_size must be at least 1") + self._page_size = size + return self + + def count(self) -> QueryBuilder: + """Request a count of matching records in the response. + + Adds ``$count=true`` to the query, causing the server to include + an ``@odata.count`` annotation in the response with the total + number of matching records (up to 5,000). + + :return: Self for method chaining. + + Example:: + + results = (client.query.builder("account") + .filter_eq("statecode", 0) + .count() + .execute()) + """ + self._count = True + return self + + def include_formatted_values(self) -> QueryBuilder: + """Request formatted values in the response. + + Adds ``Prefer: odata.include-annotations="OData.Community.Display.V1.FormattedValue"`` + to the request, causing the server to return formatted string + representations alongside raw values. This includes: + + - Localized labels for choice, yes/no, status, and status reason columns + - Primary name values for lookup and owner properties + - Currency values with currency symbols + - Formatted dates in the user's time zone + + Access formatted values in the response via the annotation key:: + + record["statecode@OData.Community.Display.V1.FormattedValue"] + + :return: Self for method chaining. + + Example:: + + for record in (client.query.builder("account") + .select("name", "statecode") + .include_formatted_values() + .execute()): + label = record["statecode@OData.Community.Display.V1.FormattedValue"] + print(f"{record['name']}: {label}") + """ + self._include_annotations = "OData.Community.Display.V1.FormattedValue" + return self + + def include_annotations(self, annotation: str = "*") -> QueryBuilder: + """Request specific OData annotations in the response. + + Sets the ``Prefer: odata.include-annotations`` header. Use ``"*"`` + to request all annotations, or specify a particular annotation + pattern. + + :param annotation: Annotation pattern to request. Defaults to + ``"*"`` (all annotations). + :return: Self for method chaining. + + Example:: + + # Request all annotations + builder = (client.query.builder("account") + .select("name", "_ownerid_value") + .include_annotations("*")) + + # Request only lookup metadata + builder = (client.query.builder("account") + .include_annotations( + "Microsoft.Dynamics.CRM.lookuplogicalname")) + """ + self._include_annotations = annotation + return self + + # --------------------------------------------------------------- expand + + def expand(self, *relations: Union[str, ExpandOption]) -> QueryBuilder: + """Expand navigation properties. + + Accepts plain navigation property names (case-sensitive, passed + as-is) or :class:`ExpandOption` objects for nested options like + ``$select``, ``$filter``, ``$orderby``, and ``$top``. + + :param relations: Navigation property names or + :class:`ExpandOption` objects. + :return: Self for method chaining. + + Example:: + + # Simple expand + query = QueryBuilder("account").expand("primarycontactid") + + # Nested expand with options + query = (QueryBuilder("account") + .expand(ExpandOption("Account_Tasks") + .select("subject") + .filter("contains(subject,'Task')") + .top(5))) + """ + for rel in relations: + if isinstance(rel, ExpandOption): + self._expand.append(rel.to_odata()) + else: + self._expand.append(rel) + return self + + # --------------------------------------------------------------- build + + def build(self) -> QueryParams: + """Build query parameters dictionary. + + Returns a :class:`QueryParams` dictionary suitable for passing to + the OData layer. All ``filter_*()`` and ``where()`` clauses are + AND-joined into a single ``filter`` string in call order. + + :return: Dictionary with ``table`` and optionally ``select``, + ``filter``, ``orderby``, ``expand``, ``top``, ``page_size``, + ``count``, ``include_annotations``. + :rtype: QueryParams + """ + params: QueryParams = {"table": self.table} + if self._select: + params["select"] = list(self._select) + if self._filter_parts: + parts: List[str] = [] + for part in self._filter_parts: + if isinstance(part, filters.FilterExpression): + parts.append(part.to_odata()) + else: + parts.append(part) + params["filter"] = " and ".join(parts) + if self._orderby: + params["orderby"] = list(self._orderby) + if self._expand: + params["expand"] = list(self._expand) + if self._top is not None: + params["top"] = self._top + if self._page_size is not None: + params["page_size"] = self._page_size + if self._count: + params["count"] = True + if self._include_annotations is not None: + params["include_annotations"] = self._include_annotations + return params + + # --------------------------------------------------------------- guards + + def _validate_constraints(self) -> None: + """Raise if the query has no limiting constraints. + + At least one of ``select``, ``filter``, or ``top`` must be set + before executing a query to prevent accidental full-table scans. + + :raises ValueError: If none of ``select()``, ``filter_*()``, + ``where()``, or ``top()`` has been called. + """ + if not (self._select or self._filter_parts or self._top is not None): + raise ValueError( + "Unbounded query: set at least one of select(), filter_*(), " + "where(), or top() before calling execute() or to_dataframe()." + ) + + # --------------------------------------------------------------- execute + + def execute(self, *, by_page: bool = False) -> Union[Iterable[Record], Iterable[List[Record]]]: + """Execute the query and return results. + + By default, returns a flat iterator over individual records, + abstracting away OData paging. Pass ``by_page=True`` to get + page-level iteration instead (useful for batch processing). + + This method is only available when the QueryBuilder was created + via ``client.query.builder(table)``. Standalone ``QueryBuilder`` + instances should use :meth:`build` to get parameters and pass them + to ``client.records.get()`` manually. + + At least one of ``select()``, ``filter_*()``, ``where()``, or + ``top()`` must be called before ``execute()``; otherwise a + :class:`ValueError` is raised to prevent accidental full-table + scans. + + :param by_page: If ``True``, yield pages (lists of + :class:`~PowerPlatform.Dataverse.models.record.Record` objects) + instead of individual records. Defaults to ``False``. + :type by_page: bool + :return: Generator yielding individual + :class:`~PowerPlatform.Dataverse.models.record.Record` objects + (default) or pages of records (when ``by_page=True``). + :rtype: Iterable[Record] or Iterable[List[Record]] + :raises ValueError: If no ``select``, ``filter``, or ``top`` + constraint has been set. + :raises RuntimeError: If the query was not created via + ``client.query.builder()``. + + Example: + Flat iteration (default):: + + for record in (client.query.builder("account") + .select("name") + .filter_eq("statecode", 0) + .execute()): + print(record["name"]) + + Paged iteration:: + + for page in (client.query.builder("account") + .select("name") + .execute(by_page=True)): + process_batch(page) + """ + if self._query_ops is None: + raise RuntimeError( + "Cannot execute: query was not created via client.query.builder(). " + "Use build() and pass parameters to client.records.get() instead." + ) + self._validate_constraints() + params = self.build() + client = self._query_ops._client + + pages = client.records.get( + params["table"], + select=params.get("select"), + filter=params.get("filter"), + orderby=params.get("orderby"), + top=params.get("top"), + expand=params.get("expand"), + page_size=params.get("page_size"), + count=params.get("count", False), + include_annotations=params.get("include_annotations"), + ) + + if by_page: + return pages + + def _flat() -> Iterable[Record]: + for page in pages: + yield from page + + return _flat() + + # ----------------------------------------------------------- to_dataframe + + def to_dataframe(self) -> pd.DataFrame: + """Execute the query and return results as a pandas DataFrame. + + All pages are consolidated into a single DataFrame, matching + the behavior of ``client.dataframe.get()``. + + This method is only available when the QueryBuilder was created + via ``client.query.builder(table)``. + + At least one of ``select()``, ``filter_*()``, ``where()``, or + ``top()`` must be called before ``to_dataframe()``; otherwise a + :class:`ValueError` is raised to prevent accidental full-table + scans. + + :return: DataFrame containing all matching records. Returns an empty + DataFrame when no records match. + :rtype: ~pandas.DataFrame + :raises ValueError: If no ``select``, ``filter``, or ``top`` + constraint has been set. + :raises RuntimeError: If the query was not created via + ``client.query.builder()``. + + Example:: + + df = (client.query.builder("account") + .select("name", "telephone1") + .filter_eq("statecode", 0) + .top(100) + .to_dataframe()) + """ + if self._query_ops is None: + raise RuntimeError( + "Cannot execute: query was not created via client.query.builder(). " + "Use build() and pass parameters to client.dataframe.get() instead." + ) + self._validate_constraints() + params = self.build() + return self._query_ops._client.dataframe.get( + params["table"], + select=params.get("select"), + filter=params.get("filter"), + orderby=params.get("orderby"), + top=params.get("top"), + expand=params.get("expand"), + page_size=params.get("page_size"), + count=params.get("count", False), + include_annotations=params.get("include_annotations"), + ) diff --git a/src/PowerPlatform/Dataverse/operations/dataframe.py b/src/PowerPlatform/Dataverse/operations/dataframe.py index 44843b54..89e31e7f 100644 --- a/src/PowerPlatform/Dataverse/operations/dataframe.py +++ b/src/PowerPlatform/Dataverse/operations/dataframe.py @@ -63,6 +63,8 @@ def get( top: Optional[int] = None, expand: Optional[List[str]] = None, page_size: Optional[int] = None, + count: bool = False, + include_annotations: Optional[str] = None, ) -> pd.DataFrame: """Fetch records and return as a single pandas DataFrame. @@ -86,6 +88,13 @@ def get( :type expand: list[str] or None :param page_size: Optional number of records per page for pagination. :type page_size: :class:`int` or None + :param count: If ``True``, adds ``$count=true`` to include a total + record count in the response. + :type count: :class:`bool` + :param include_annotations: OData annotation pattern for the + ``Prefer: odata.include-annotations`` header (e.g. ``"*"`` or + ``"OData.Community.Display.V1.FormattedValue"``), or ``None``. + :type include_annotations: :class:`str` or None :return: DataFrame containing all matching records. Returns an empty DataFrame when no records match. @@ -138,6 +147,8 @@ def get( top=top, expand=expand, page_size=page_size, + count=count, + include_annotations=include_annotations, ): rows.extend(row.data for row in batch) diff --git a/src/PowerPlatform/Dataverse/operations/query.py b/src/PowerPlatform/Dataverse/operations/query.py index 158ff229..e0dc6972 100644 --- a/src/PowerPlatform/Dataverse/operations/query.py +++ b/src/PowerPlatform/Dataverse/operations/query.py @@ -9,6 +9,8 @@ from ..models.record import Record +from ..models.query_builder import QueryBuilder + if TYPE_CHECKING: from ..client import DataverseClient @@ -29,6 +31,16 @@ class QueryOperations: client = DataverseClient(base_url, credential) + # Fluent query builder (recommended) + for record in (client.query.builder("account") + .select("name", "revenue") + .filter_eq("statecode", 0) + .order_by("revenue", descending=True) + .top(100) + .execute()): + print(record["name"]) + + # SQL query rows = client.query.sql("SELECT TOP 10 name FROM account ORDER BY name") for row in rows: print(row["name"]) @@ -37,6 +49,47 @@ class QueryOperations: def __init__(self, client: DataverseClient) -> None: self._client = client + # ----------------------------------------------------------------- builder + + def builder(self, table: str) -> QueryBuilder: + """Create a fluent query builder for the specified table. + + Returns a :class:`~PowerPlatform.Dataverse.models.query_builder.QueryBuilder` + that can be chained with filter, select, and order methods, then + executed directly via ``.execute()``. + + :param table: Table schema name (e.g. ``"account"``). + :type table: :class:`str` + :return: A QueryBuilder instance bound to this client. + :rtype: ~PowerPlatform.Dataverse.models.query_builder.QueryBuilder + + Example: + Build and execute a query fluently:: + + for record in (client.query.builder("account") + .select("name", "revenue") + .filter_eq("statecode", 0) + .filter_gt("revenue", 1000000) + .order_by("revenue", descending=True) + .top(100) + .page_size(50) + .execute()): + print(record["name"]) + + With composable expression tree:: + + from PowerPlatform.Dataverse.models.filters import eq, gt + + for record in (client.query.builder("account") + .where((eq("statecode", 0) | eq("statecode", 1)) + & gt("revenue", 100000)) + .execute()): + print(record["name"]) + """ + qb = QueryBuilder(table) + qb._query_ops = self + return qb + # -------------------------------------------------------------------- sql def sql(self, sql: str) -> List[Record]: diff --git a/src/PowerPlatform/Dataverse/operations/records.py b/src/PowerPlatform/Dataverse/operations/records.py index 13ad753c..7888a52a 100644 --- a/src/PowerPlatform/Dataverse/operations/records.py +++ b/src/PowerPlatform/Dataverse/operations/records.py @@ -270,6 +270,8 @@ def get( top: Optional[int] = None, expand: Optional[List[str]] = None, page_size: Optional[int] = None, + count: bool = False, + include_annotations: Optional[str] = None, ) -> Iterable[List[Record]]: """Fetch multiple records from a Dataverse table with pagination. @@ -300,6 +302,13 @@ def get( :param page_size: Optional per-page size hint sent via ``Prefer: odata.maxpagesize``. :type page_size: :class:`int` or None + :param count: If ``True``, adds ``$count=true`` to include a total + record count in the response. + :type count: :class:`bool` + :param include_annotations: OData annotation pattern for the + ``Prefer: odata.include-annotations`` header (e.g. ``"*"`` or + ``"OData.Community.Display.V1.FormattedValue"``), or ``None``. + :type include_annotations: :class:`str` or None :return: Generator yielding pages, where each page is a list of :class:`~PowerPlatform.Dataverse.models.record.Record` objects. @@ -330,6 +339,8 @@ def get( top: Optional[int] = None, expand: Optional[List[str]] = None, page_size: Optional[int] = None, + count: bool = False, + include_annotations: Optional[str] = None, ) -> Union[Record, Iterable[List[Record]]]: """Fetch a single record by ID, or fetch multiple records with pagination. @@ -375,6 +386,14 @@ def get( :param page_size: Optional per-page size hint sent via ``Prefer: odata.maxpagesize``. Only used for multi-record queries. :type page_size: :class:`int` or None + :param count: If ``True``, adds ``$count=true`` to include a total + record count in the response. Only used for multi-record queries. + :type count: :class:`bool` + :param include_annotations: OData annotation pattern for the + ``Prefer: odata.include-annotations`` header (e.g. ``"*"`` or + ``"OData.Community.Display.V1.FormattedValue"``), or ``None``. + Only used for multi-record queries. + :type include_annotations: :class:`str` or None :return: A single record dict when ``record_id`` is provided, or a generator yielding pages (lists of record dicts) when fetching @@ -413,10 +432,13 @@ def get( or top is not None or expand is not None or page_size is not None + or count is not False + or include_annotations is not None ): raise ValueError( "Cannot specify query parameters (filter, orderby, top, " - "expand, page_size) when fetching a single record by ID" + "expand, page_size, count, include_annotations) when " + "fetching a single record by ID" ) with self._client._scoped_odata() as od: raw = od._get(table, record_id, select=select) @@ -432,6 +454,8 @@ def _paged() -> Iterable[List[Record]]: top=top, expand=expand, page_size=page_size, + count=count, + include_annotations=include_annotations, ): yield [Record.from_api_response(table, row) for row in page] diff --git a/tests/unit/models/test_filters.py b/tests/unit/models/test_filters.py new file mode 100644 index 00000000..f0ca33a3 --- /dev/null +++ b/tests/unit/models/test_filters.py @@ -0,0 +1,522 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Unit tests for composable OData filter expressions.""" + +import unittest +import uuid +from datetime import date, datetime, timezone, timedelta + +from PowerPlatform.Dataverse.models.filters import ( + FilterExpression, + _format_value, + eq, + ne, + gt, + ge, + lt, + le, + contains, + startswith, + endswith, + between, + is_null, + is_not_null, + filter_in, + not_in, + not_between, + raw, +) + + +class TestFormatValue(unittest.TestCase): + """Tests for _format_value().""" + + def test_none(self): + self.assertEqual(_format_value(None), "null") + + def test_bool_true(self): + self.assertEqual(_format_value(True), "true") + + def test_bool_false(self): + self.assertEqual(_format_value(False), "false") + + def test_bool_before_int(self): + """bool is a subclass of int; must format as true/false, not 1/0.""" + self.assertEqual(_format_value(True), "true") + self.assertNotEqual(_format_value(True), "1") + + def test_int(self): + self.assertEqual(_format_value(42), "42") + + def test_int_zero(self): + self.assertEqual(_format_value(0), "0") + + def test_int_negative(self): + self.assertEqual(_format_value(-5), "-5") + + def test_float(self): + self.assertEqual(_format_value(3.14), "3.14") + + def test_float_integer_value(self): + self.assertEqual(_format_value(1000000.0), "1000000.0") + + def test_int_enum(self): + from enum import IntEnum + + class Priority(IntEnum): + LOW = 1 + HIGH = 2 + + self.assertEqual(_format_value(Priority.HIGH), "2") + self.assertEqual(_format_value(Priority.LOW), "1") + + def test_str_enum(self): + from enum import Enum + + class Color(Enum): + RED = "red" + + self.assertEqual(_format_value(Color.RED), "'red'") + + def test_string(self): + self.assertEqual(_format_value("hello"), "'hello'") + + def test_string_with_single_quotes(self): + self.assertEqual(_format_value("O'Brien"), "'O''Brien'") + + def test_string_with_multiple_quotes(self): + self.assertEqual(_format_value("O'Brien's Corp"), "'O''Brien''s Corp'") + + def test_string_empty(self): + self.assertEqual(_format_value(""), "''") + + def test_datetime_naive(self): + """Naive datetimes are assumed UTC.""" + dt = datetime(2024, 1, 15, 10, 30, 0) + self.assertEqual(_format_value(dt), "2024-01-15T10:30:00Z") + + def test_datetime_utc(self): + dt = datetime(2024, 1, 15, 10, 30, 0, tzinfo=timezone.utc) + self.assertEqual(_format_value(dt), "2024-01-15T10:30:00Z") + + def test_datetime_with_microseconds(self): + """Microseconds should be preserved when non-zero.""" + dt = datetime(2024, 1, 15, 10, 30, 0, 123456) + self.assertEqual(_format_value(dt), "2024-01-15T10:30:00.123456Z") + + def test_datetime_non_utc_converted(self): + """Timezone-aware non-UTC datetimes should be converted to UTC.""" + eastern = timezone(timedelta(hours=-5)) + dt = datetime(2024, 1, 15, 10, 30, 0, tzinfo=eastern) + # 10:30 EST = 15:30 UTC + self.assertEqual(_format_value(dt), "2024-01-15T15:30:00Z") + + def test_date(self): + d = date(2024, 1, 15) + self.assertEqual(_format_value(d), "2024-01-15") + + def test_uuid(self): + uid = uuid.UUID("12345678-1234-1234-1234-123456789abc") + self.assertEqual(_format_value(uid), "12345678-1234-1234-1234-123456789abc") + + def test_fallback_converts_to_string(self): + """Unknown types fall back to str().""" + + class Custom: + def __str__(self): + return "custom-value" + + self.assertEqual(_format_value(Custom()), "custom-value") + + +class TestComparisonFilters(unittest.TestCase): + """Tests for comparison filter factory functions.""" + + def test_eq_string(self): + self.assertEqual(eq("name", "Contoso").to_odata(), "name eq 'Contoso'") + + def test_eq_int(self): + self.assertEqual(eq("statecode", 0).to_odata(), "statecode eq 0") + + def test_eq_none(self): + self.assertEqual(eq("phone", None).to_odata(), "phone eq null") + + def test_eq_bool(self): + self.assertEqual(eq("active", True).to_odata(), "active eq true") + + def test_ne(self): + self.assertEqual(ne("statecode", 1).to_odata(), "statecode ne 1") + + def test_gt(self): + self.assertEqual(gt("revenue", 1000000).to_odata(), "revenue gt 1000000") + + def test_ge(self): + self.assertEqual(ge("revenue", 1000000).to_odata(), "revenue ge 1000000") + + def test_lt(self): + self.assertEqual(lt("revenue", 500000).to_odata(), "revenue lt 500000") + + def test_le(self): + self.assertEqual(le("revenue", 500000).to_odata(), "revenue le 500000") + + def test_column_name_lowercased(self): + self.assertEqual(eq("StateCode", 0).to_odata(), "statecode eq 0") + + def test_eq_float(self): + self.assertEqual(eq("revenue", 1000000.5).to_odata(), "revenue eq 1000000.5") + + def test_eq_uuid(self): + uid = uuid.UUID("12345678-1234-1234-1234-123456789abc") + self.assertEqual( + eq("accountid", uid).to_odata(), + "accountid eq 12345678-1234-1234-1234-123456789abc", + ) + + def test_eq_datetime(self): + dt = datetime(2024, 1, 15, 10, 30, 0, tzinfo=timezone.utc) + self.assertEqual( + eq("createdon", dt).to_odata(), + "createdon eq 2024-01-15T10:30:00Z", + ) + + def test_eq_int_enum(self): + from enum import IntEnum + + class Priority(IntEnum): + LOW = 1 + HIGH = 2 + + self.assertEqual(eq("priority", Priority.HIGH).to_odata(), "priority eq 2") + + def test_ne_string(self): + self.assertEqual(ne("name", "Contoso").to_odata(), "name ne 'Contoso'") + + def test_gt_negative(self): + self.assertEqual(gt("temperature", -10).to_odata(), "temperature gt -10") + + def test_gt_float(self): + self.assertEqual(gt("revenue", 99.5).to_odata(), "revenue gt 99.5") + + +class TestFunctionFilters(unittest.TestCase): + """Tests for string function filter factory functions.""" + + def test_contains(self): + self.assertEqual(contains("name", "Corp").to_odata(), "contains(name, 'Corp')") + + def test_startswith(self): + self.assertEqual(startswith("name", "Con").to_odata(), "startswith(name, 'Con')") + + def test_endswith(self): + self.assertEqual(endswith("name", "Ltd").to_odata(), "endswith(name, 'Ltd')") + + def test_function_column_lowercased(self): + self.assertEqual(contains("Name", "Corp").to_odata(), "contains(name, 'Corp')") + + def test_contains_single_quotes(self): + self.assertEqual( + contains("name", "O'Brien").to_odata(), + "contains(name, 'O''Brien')", + ) + + +class TestBetween(unittest.TestCase): + """Tests for the between factory function.""" + + def test_between_ints(self): + self.assertEqual( + between("revenue", 100000, 500000).to_odata(), + "(revenue ge 100000 and revenue le 500000)", + ) + + def test_between_dates(self): + result = between("createdon", date(2024, 1, 1), date(2024, 12, 31)).to_odata() + self.assertEqual(result, "(createdon ge 2024-01-01 and createdon le 2024-12-31)") + + def test_between_floats(self): + self.assertEqual( + between("revenue", 100.5, 999.9).to_odata(), + "(revenue ge 100.5 and revenue le 999.9)", + ) + + def test_between_datetimes(self): + start = datetime(2024, 1, 1, 0, 0, 0, tzinfo=timezone.utc) + end = datetime(2024, 12, 31, 23, 59, 59, tzinfo=timezone.utc) + self.assertEqual( + between("createdon", start, end).to_odata(), + "(createdon ge 2024-01-01T00:00:00Z and createdon le 2024-12-31T23:59:59Z)", + ) + + +class TestNullChecks(unittest.TestCase): + """Tests for is_null and is_not_null.""" + + def test_is_null(self): + self.assertEqual(is_null("phone").to_odata(), "phone eq null") + + def test_is_not_null(self): + self.assertEqual(is_not_null("phone").to_odata(), "phone ne null") + + +class TestRawFilter(unittest.TestCase): + """Tests for the raw filter function.""" + + def test_raw(self): + expr = raw("Microsoft.Dynamics.CRM.Today(PropertyName='createdon')") + self.assertEqual( + expr.to_odata(), + "Microsoft.Dynamics.CRM.Today(PropertyName='createdon')", + ) + + def test_raw_passthrough(self): + """Raw filter should pass through exactly as given.""" + text = "(statecode eq 0 or statecode eq 1)" + self.assertEqual(raw(text).to_odata(), text) + + +class TestInFilter(unittest.TestCase): + """Tests for the filter_in factory function.""" + + def test_filter_in_ints(self): + self.assertEqual( + filter_in("statecode", [0, 1, 2]).to_odata(), + 'Microsoft.Dynamics.CRM.In(PropertyName=\'statecode\',PropertyValues=["0","1","2"])', + ) + + def test_filter_in_strings(self): + self.assertEqual( + filter_in("name", ["Contoso", "Fabrikam"]).to_odata(), + 'Microsoft.Dynamics.CRM.In(PropertyName=\'name\',PropertyValues=["Contoso","Fabrikam"])', + ) + + def test_filter_in_single_value(self): + self.assertEqual( + filter_in("statecode", [0]).to_odata(), + "Microsoft.Dynamics.CRM.In(PropertyName='statecode',PropertyValues=[\"0\"])", + ) + + def test_filter_in_column_lowercased(self): + self.assertEqual( + filter_in("StateCode", [0, 1]).to_odata(), + 'Microsoft.Dynamics.CRM.In(PropertyName=\'statecode\',PropertyValues=["0","1"])', + ) + + def test_filter_in_empty_raises(self): + with self.assertRaises(ValueError): + filter_in("statecode", []) + + def test_filter_in_int_enum(self): + from enum import IntEnum + + class Priority(IntEnum): + LOW = 1 + MEDIUM = 2 + HIGH = 3 + + self.assertEqual( + filter_in("priority", [Priority.LOW, Priority.HIGH]).to_odata(), + 'Microsoft.Dynamics.CRM.In(PropertyName=\'priority\',PropertyValues=["1","3"])', + ) + + def test_filter_in_uuids(self): + uid1 = uuid.UUID("12345678-1234-1234-1234-123456789abc") + uid2 = uuid.UUID("87654321-4321-4321-4321-cba987654321") + self.assertEqual( + filter_in("accountid", [uid1, uid2]).to_odata(), + 'Microsoft.Dynamics.CRM.In(PropertyName=\'accountid\',PropertyValues=["12345678-1234-1234-1234-123456789abc","87654321-4321-4321-4321-cba987654321"])', + ) + + def test_filter_in_bools(self): + self.assertEqual( + filter_in("completed", [True, False]).to_odata(), + 'Microsoft.Dynamics.CRM.In(PropertyName=\'completed\',PropertyValues=["true","false"])', + ) + + def test_filter_in_floats(self): + self.assertEqual( + filter_in("amount", [10.5, 20.0]).to_odata(), + 'Microsoft.Dynamics.CRM.In(PropertyName=\'amount\',PropertyValues=["10.5","20.0"])', + ) + + def test_filter_in_datetimes(self): + from datetime import datetime, timezone + + dt1 = datetime(2024, 1, 15, 10, 30, 0, tzinfo=timezone.utc) + dt2 = datetime(2024, 6, 1, 0, 0, 0, tzinfo=timezone.utc) + self.assertEqual( + filter_in("createdon", [dt1, dt2]).to_odata(), + 'Microsoft.Dynamics.CRM.In(PropertyName=\'createdon\',PropertyValues=["2024-01-15T10:30:00Z","2024-06-01T00:00:00Z"])', + ) + + def test_filter_in_none(self): + self.assertEqual( + filter_in("status", [None, 1]).to_odata(), + 'Microsoft.Dynamics.CRM.In(PropertyName=\'status\',PropertyValues=["null","1"])', + ) + + def test_filter_in_mixed_types(self): + """Ints, bools, and strings together.""" + result = filter_in("field", [1, True, "hello"]).to_odata() + self.assertIn('"1"', result) + self.assertIn('"true"', result) + self.assertIn('"hello"', result) + + +class TestNotInFilter(unittest.TestCase): + """Tests for the not_in factory function.""" + + def test_not_in_ints(self): + self.assertEqual( + not_in("statecode", [2, 3]).to_odata(), + 'Microsoft.Dynamics.CRM.NotIn(PropertyName=\'statecode\',PropertyValues=["2","3"])', + ) + + def test_not_in_strings(self): + self.assertEqual( + not_in("name", ["Contoso", "Fabrikam"]).to_odata(), + 'Microsoft.Dynamics.CRM.NotIn(PropertyName=\'name\',PropertyValues=["Contoso","Fabrikam"])', + ) + + def test_not_in_single_value(self): + self.assertEqual( + not_in("statecode", [0]).to_odata(), + "Microsoft.Dynamics.CRM.NotIn(PropertyName='statecode',PropertyValues=[\"0\"])", + ) + + def test_not_in_column_lowercased(self): + self.assertEqual( + not_in("StateCode", [0, 1]).to_odata(), + 'Microsoft.Dynamics.CRM.NotIn(PropertyName=\'statecode\',PropertyValues=["0","1"])', + ) + + def test_not_in_empty_raises(self): + with self.assertRaises(ValueError): + not_in("statecode", []) + + def test_not_in_int_enum(self): + from enum import IntEnum + + class Priority(IntEnum): + LOW = 1 + HIGH = 3 + + self.assertEqual( + not_in("priority", [Priority.LOW, Priority.HIGH]).to_odata(), + 'Microsoft.Dynamics.CRM.NotIn(PropertyName=\'priority\',PropertyValues=["1","3"])', + ) + + +class TestNotBetween(unittest.TestCase): + """Tests for the not_between factory function.""" + + def test_not_between_ints(self): + self.assertEqual( + not_between("revenue", 100000, 500000).to_odata(), + "not ((revenue ge 100000 and revenue le 500000))", + ) + + def test_not_between_floats(self): + self.assertEqual( + not_between("amount", 10.5, 99.9).to_odata(), + "not ((amount ge 10.5 and amount le 99.9))", + ) + + def test_not_between_datetimes(self): + from datetime import datetime, timezone + + start = datetime(2024, 1, 1, 0, 0, 0, tzinfo=timezone.utc) + end = datetime(2024, 12, 31, 23, 59, 59, tzinfo=timezone.utc) + self.assertEqual( + not_between("createdon", start, end).to_odata(), + "not ((createdon ge 2024-01-01T00:00:00Z and createdon le 2024-12-31T23:59:59Z))", + ) + + def test_not_between_column_lowercased(self): + self.assertEqual( + not_between("Revenue", 100, 500).to_odata(), + "not ((revenue ge 100 and revenue le 500))", + ) + + +class TestLogicalOperators(unittest.TestCase): + """Tests for &, |, ~ operator overloads.""" + + def test_and_operator(self): + self.assertEqual( + (eq("a", 1) & eq("b", 2)).to_odata(), + "(a eq 1 and b eq 2)", + ) + + def test_or_operator(self): + self.assertEqual( + (eq("a", 1) | eq("b", 2)).to_odata(), + "(a eq 1 or b eq 2)", + ) + + def test_not_operator(self): + self.assertEqual( + (~eq("a", 1)).to_odata(), + "not (a eq 1)", + ) + + def test_complex_composition(self): + """(statecode in {0,1}) AND (revenue > 100k)""" + expr = (eq("statecode", 0) | eq("statecode", 1)) & gt("revenue", 100000) + self.assertEqual( + expr.to_odata(), + "((statecode eq 0 or statecode eq 1) and revenue gt 100000)", + ) + + def test_triple_and(self): + expr = eq("a", 1) & eq("b", 2) & eq("c", 3) + self.assertEqual( + expr.to_odata(), + "((a eq 1 and b eq 2) and c eq 3)", + ) + + def test_not_or(self): + expr = ~(eq("a", 1) | eq("b", 2)) + self.assertEqual( + expr.to_odata(), + "not ((a eq 1 or b eq 2))", + ) + + def test_and_with_non_expression_returns_not_implemented(self): + result = eq("a", 1).__and__("not an expression") + self.assertIs(result, NotImplemented) + + def test_or_with_non_expression_returns_not_implemented(self): + result = eq("a", 1).__or__("not an expression") + self.assertIs(result, NotImplemented) + + def test_and_with_filter_in(self): + expr = filter_in("statecode", [0, 1]) & gt("revenue", 100000) + self.assertEqual( + expr.to_odata(), + '(Microsoft.Dynamics.CRM.In(PropertyName=\'statecode\',PropertyValues=["0","1"]) and revenue gt 100000)', + ) + + +class TestStrAndRepr(unittest.TestCase): + """Tests for __str__ and __repr__.""" + + def test_str_delegates_to_odata(self): + self.assertEqual(str(eq("a", 1)), "a eq 1") + + def test_repr_includes_class_name(self): + r = repr(eq("a", 1)) + self.assertIn("_ComparisonFilter", r) + self.assertIn("a eq 1", r) + + +class TestFilterExpressionBase(unittest.TestCase): + """Tests for the FilterExpression base class.""" + + def test_base_to_odata_raises(self): + with self.assertRaises(NotImplementedError): + FilterExpression().to_odata() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/models/test_query_builder.py b/tests/unit/models/test_query_builder.py new file mode 100644 index 00000000..47bb28fc --- /dev/null +++ b/tests/unit/models/test_query_builder.py @@ -0,0 +1,977 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Unit tests for QueryBuilder class.""" + +import unittest +from unittest.mock import MagicMock + +from PowerPlatform.Dataverse.models.query_builder import QueryBuilder + + +class TestQueryBuilderConstruction(unittest.TestCase): + """Tests for QueryBuilder construction and validation.""" + + def test_basic_construction(self): + qb = QueryBuilder("account") + self.assertEqual(qb.table, "account") + self.assertEqual(qb.build(), {"table": "account"}) + + def test_empty_table_raises(self): + with self.assertRaises(ValueError): + QueryBuilder("") + + def test_whitespace_table_raises(self): + with self.assertRaises(ValueError): + QueryBuilder(" ") + + def test_internal_state_not_exposed_as_constructor_params(self): + """Unlike a dataclass, internal state should not be settable via constructor.""" + with self.assertRaises(TypeError): + QueryBuilder("account", _select=["name"]) # type: ignore + + +class TestSelect(unittest.TestCase): + """Tests for the select() method.""" + + def test_select_single(self): + qb = QueryBuilder("account").select("name") + self.assertEqual(qb.build()["select"], ["name"]) + + def test_select_multiple(self): + qb = QueryBuilder("account").select("name", "revenue", "telephone1") + self.assertEqual(qb.build()["select"], ["name", "revenue", "telephone1"]) + + def test_select_chained(self): + qb = QueryBuilder("account").select("name").select("revenue") + self.assertEqual(qb.build()["select"], ["name", "revenue"]) + + def test_select_returns_self(self): + qb = QueryBuilder("account") + self.assertIs(qb.select("name"), qb) + + +class TestComparisonFilters(unittest.TestCase): + """Tests for comparison filter methods.""" + + def test_filter_eq_string(self): + qb = QueryBuilder("account").filter_eq("name", "Contoso") + self.assertEqual(qb.build()["filter"], "name eq 'Contoso'") + + def test_filter_eq_integer(self): + qb = QueryBuilder("account").filter_eq("statecode", 0) + self.assertEqual(qb.build()["filter"], "statecode eq 0") + + def test_filter_eq_boolean_true(self): + qb = QueryBuilder("account").filter_eq("active", True) + self.assertEqual(qb.build()["filter"], "active eq true") + + def test_filter_eq_boolean_false(self): + qb = QueryBuilder("account").filter_eq("active", False) + self.assertEqual(qb.build()["filter"], "active eq false") + + def test_filter_eq_none(self): + qb = QueryBuilder("account").filter_eq("telephone1", None) + self.assertEqual(qb.build()["filter"], "telephone1 eq null") + + def test_filter_eq_float(self): + qb = QueryBuilder("account").filter_eq("revenue", 1000000.5) + self.assertEqual(qb.build()["filter"], "revenue eq 1000000.5") + + def test_filter_eq_datetime(self): + from datetime import datetime, timezone + + dt = datetime(2024, 1, 15, 10, 30, 0, tzinfo=timezone.utc) + qb = QueryBuilder("account").filter_eq("createdon", dt) + self.assertEqual(qb.build()["filter"], "createdon eq 2024-01-15T10:30:00Z") + + +class TestFilterIn(unittest.TestCase): + """Tests for the filter_in() method.""" + + def test_filter_in_integers(self): + qb = QueryBuilder("account").filter_in("statecode", [0, 1, 2]) + self.assertEqual( + qb.build()["filter"], + 'Microsoft.Dynamics.CRM.In(PropertyName=\'statecode\',PropertyValues=["0","1","2"])', + ) + + def test_filter_in_strings(self): + qb = QueryBuilder("account").filter_in("name", ["Contoso", "Fabrikam"]) + self.assertEqual( + qb.build()["filter"], + 'Microsoft.Dynamics.CRM.In(PropertyName=\'name\',PropertyValues=["Contoso","Fabrikam"])', + ) + + def test_filter_in_single_value(self): + qb = QueryBuilder("account").filter_in("statecode", [0]) + self.assertEqual( + qb.build()["filter"], + "Microsoft.Dynamics.CRM.In(PropertyName='statecode',PropertyValues=[\"0\"])", + ) + + def test_filter_in_column_lowercased(self): + qb = QueryBuilder("account").filter_in("StateCode", [0, 1]) + self.assertEqual( + qb.build()["filter"], + 'Microsoft.Dynamics.CRM.In(PropertyName=\'statecode\',PropertyValues=["0","1"])', + ) + + def test_filter_in_empty_raises(self): + with self.assertRaises(ValueError): + QueryBuilder("account").filter_in("statecode", []) + + def test_filter_in_returns_self(self): + qb = QueryBuilder("account") + self.assertIs(qb.filter_in("statecode", [0, 1]), qb) + + def test_filter_in_with_set(self): + qb = QueryBuilder("account").filter_in("statecode", {0, 1}) + result = qb.build()["filter"] + self.assertIn("Microsoft.Dynamics.CRM.In", result) + self.assertIn("statecode", result) + + def test_filter_in_with_tuple(self): + qb = QueryBuilder("account").filter_in("statecode", (0, 1, 2)) + self.assertEqual( + qb.build()["filter"], + 'Microsoft.Dynamics.CRM.In(PropertyName=\'statecode\',PropertyValues=["0","1","2"])', + ) + + def test_filter_in_int_enum(self): + from enum import IntEnum + + class Priority(IntEnum): + LOW = 1 + HIGH = 3 + + qb = QueryBuilder("account").filter_in("priority", [Priority.LOW, Priority.HIGH]) + self.assertEqual( + qb.build()["filter"], + 'Microsoft.Dynamics.CRM.In(PropertyName=\'priority\',PropertyValues=["1","3"])', + ) + + def test_filter_in_combined_with_other_filters(self): + qb = QueryBuilder("account").filter_eq("statecode", 0).filter_in("priority", [1, 2, 3]) + self.assertEqual( + qb.build()["filter"], + 'statecode eq 0 and Microsoft.Dynamics.CRM.In(PropertyName=\'priority\',PropertyValues=["1","2","3"])', + ) + + def test_filter_ne(self): + qb = QueryBuilder("account").filter_ne("statecode", 1) + self.assertEqual(qb.build()["filter"], "statecode ne 1") + + def test_filter_gt(self): + qb = QueryBuilder("account").filter_gt("revenue", 1000000) + self.assertEqual(qb.build()["filter"], "revenue gt 1000000") + + def test_filter_ge(self): + qb = QueryBuilder("account").filter_ge("revenue", 1000000) + self.assertEqual(qb.build()["filter"], "revenue ge 1000000") + + def test_filter_lt(self): + qb = QueryBuilder("account").filter_lt("revenue", 500000) + self.assertEqual(qb.build()["filter"], "revenue lt 500000") + + def test_filter_le(self): + qb = QueryBuilder("account").filter_le("revenue", 500000) + self.assertEqual(qb.build()["filter"], "revenue le 500000") + + def test_column_names_lowercased(self): + qb = QueryBuilder("account").filter_eq("StateCode", 0).order_by("Revenue") + params = qb.build() + self.assertEqual(params["filter"], "statecode eq 0") + self.assertEqual(params["orderby"], ["revenue"]) + + def test_string_with_quotes_escaped(self): + qb = QueryBuilder("account").filter_eq("name", "O'Brien's Corp") + self.assertEqual(qb.build()["filter"], "name eq 'O''Brien''s Corp'") + + def test_multiple_filters_and_joined(self): + qb = QueryBuilder("account").filter_eq("statecode", 0).filter_gt("revenue", 1000000) + self.assertEqual(qb.build()["filter"], "statecode eq 0 and revenue gt 1000000") + + +class TestStringFunctionFilters(unittest.TestCase): + """Tests for string function filter methods.""" + + def test_filter_contains(self): + qb = QueryBuilder("account").filter_contains("name", "Corp") + self.assertEqual(qb.build()["filter"], "contains(name, 'Corp')") + + def test_filter_startswith(self): + qb = QueryBuilder("account").filter_startswith("name", "Con") + self.assertEqual(qb.build()["filter"], "startswith(name, 'Con')") + + def test_filter_endswith(self): + qb = QueryBuilder("account").filter_endswith("name", "Ltd") + self.assertEqual(qb.build()["filter"], "endswith(name, 'Ltd')") + + def test_filter_contains_single_quotes(self): + qb = QueryBuilder("account").filter_contains("name", "O'Brien") + self.assertEqual(qb.build()["filter"], "contains(name, 'O''Brien')") + + +class TestNullFilters(unittest.TestCase): + """Tests for null/not-null filter methods.""" + + def test_filter_null(self): + qb = QueryBuilder("account").filter_null("telephone1") + self.assertEqual(qb.build()["filter"], "telephone1 eq null") + + def test_filter_not_null(self): + qb = QueryBuilder("account").filter_not_null("telephone1") + self.assertEqual(qb.build()["filter"], "telephone1 ne null") + + +class TestFilterBetween(unittest.TestCase): + """Tests for the filter_between() method.""" + + def test_filter_between_parenthesized(self): + qb = QueryBuilder("account").filter_between("revenue", 100000, 500000) + self.assertEqual( + qb.build()["filter"], + "(revenue ge 100000 and revenue le 500000)", + ) + + def test_filter_between_column_lowercased(self): + qb = QueryBuilder("account").filter_between("Revenue", 100, 500) + self.assertEqual( + qb.build()["filter"], + "(revenue ge 100 and revenue le 500)", + ) + + def test_filter_between_returns_self(self): + qb = QueryBuilder("account") + self.assertIs(qb.filter_between("revenue", 100, 500), qb) + + def test_filter_between_combined_with_other_filters(self): + qb = QueryBuilder("account").filter_eq("statecode", 0).filter_between("revenue", 100000, 500000) + self.assertEqual( + qb.build()["filter"], + "statecode eq 0 and (revenue ge 100000 and revenue le 500000)", + ) + + def test_filter_between_datetimes(self): + from datetime import datetime, timezone + + start = datetime(2024, 1, 1, 0, 0, 0, tzinfo=timezone.utc) + end = datetime(2024, 12, 31, 23, 59, 59, tzinfo=timezone.utc) + qb = QueryBuilder("account").filter_between("createdon", start, end) + self.assertEqual( + qb.build()["filter"], + "(createdon ge 2024-01-01T00:00:00Z and createdon le 2024-12-31T23:59:59Z)", + ) + + +class TestFilterNotIn(unittest.TestCase): + """Tests for the filter_not_in() method.""" + + def test_filter_not_in_ints(self): + qb = QueryBuilder("account").filter_not_in("statecode", [2, 3]) + self.assertEqual( + qb.build()["filter"], + 'Microsoft.Dynamics.CRM.NotIn(PropertyName=\'statecode\',PropertyValues=["2","3"])', + ) + + def test_filter_not_in_strings(self): + qb = QueryBuilder("account").filter_not_in("name", ["Contoso", "Fabrikam"]) + self.assertEqual( + qb.build()["filter"], + 'Microsoft.Dynamics.CRM.NotIn(PropertyName=\'name\',PropertyValues=["Contoso","Fabrikam"])', + ) + + def test_filter_not_in_empty_raises(self): + with self.assertRaises(ValueError): + QueryBuilder("account").filter_not_in("statecode", []) + + def test_filter_not_in_returns_self(self): + qb = QueryBuilder("account") + self.assertIs(qb.filter_not_in("statecode", [0, 1]), qb) + + def test_filter_not_in_combined_with_other_filters(self): + qb = QueryBuilder("account").filter_eq("statecode", 0).filter_not_in("priority", [1, 2]) + self.assertEqual( + qb.build()["filter"], + 'statecode eq 0 and Microsoft.Dynamics.CRM.NotIn(PropertyName=\'priority\',PropertyValues=["1","2"])', + ) + + def test_filter_not_in_with_set(self): + qb = QueryBuilder("account").filter_not_in("statecode", {2, 3}) + result = qb.build()["filter"] + self.assertIn("Microsoft.Dynamics.CRM.NotIn", result) + self.assertIn("statecode", result) + + def test_filter_not_in_with_tuple(self): + qb = QueryBuilder("account").filter_not_in("statecode", (2, 3)) + self.assertEqual( + qb.build()["filter"], + 'Microsoft.Dynamics.CRM.NotIn(PropertyName=\'statecode\',PropertyValues=["2","3"])', + ) + + +class TestFilterNotBetween(unittest.TestCase): + """Tests for the filter_not_between() method.""" + + def test_filter_not_between_ints(self): + qb = QueryBuilder("account").filter_not_between("revenue", 100000, 500000) + self.assertEqual( + qb.build()["filter"], + "not ((revenue ge 100000 and revenue le 500000))", + ) + + def test_filter_not_between_returns_self(self): + qb = QueryBuilder("account") + self.assertIs(qb.filter_not_between("revenue", 100, 500), qb) + + def test_filter_not_between_combined_with_other_filters(self): + qb = QueryBuilder("account").filter_eq("statecode", 0).filter_not_between("revenue", 100000, 500000) + self.assertEqual( + qb.build()["filter"], + "statecode eq 0 and not ((revenue ge 100000 and revenue le 500000))", + ) + + +class TestFilterRaw(unittest.TestCase): + """Tests for the filter_raw() method.""" + + def test_filter_raw(self): + qb = QueryBuilder("account").filter_raw("(statecode eq 0 or statecode eq 1)") + self.assertEqual(qb.build()["filter"], "(statecode eq 0 or statecode eq 1)") + + def test_filter_raw_returns_self(self): + qb = QueryBuilder("account") + self.assertIs(qb.filter_raw("a eq 1"), qb) + + +class TestWhere(unittest.TestCase): + """Tests for the where() method with composable expressions.""" + + def test_where_simple(self): + from PowerPlatform.Dataverse.models.filters import eq + + qb = QueryBuilder("account").where(eq("statecode", 0)) + self.assertEqual(qb.build()["filter"], "statecode eq 0") + + def test_where_complex(self): + from PowerPlatform.Dataverse.models.filters import eq, gt + + expr = (eq("statecode", 0) | eq("statecode", 1)) & gt("revenue", 100000) + qb = QueryBuilder("account").where(expr) + self.assertEqual( + qb.build()["filter"], + "((statecode eq 0 or statecode eq 1) and revenue gt 100000)", + ) + + def test_where_combined_with_filter_methods(self): + from PowerPlatform.Dataverse.models.filters import gt + + qb = QueryBuilder("account").filter_eq("statecode", 0).where(gt("revenue", 100000)) + self.assertEqual(qb.build()["filter"], "statecode eq 0 and revenue gt 100000") + + def test_where_multiple_calls(self): + from PowerPlatform.Dataverse.models.filters import eq, gt + + qb = QueryBuilder("account").where(eq("statecode", 0)).where(gt("revenue", 100000)) + self.assertEqual(qb.build()["filter"], "statecode eq 0 and revenue gt 100000") + + def test_where_preserves_call_order(self): + """Interleaved filter_*() and where() should preserve call order.""" + from PowerPlatform.Dataverse.models.filters import eq, gt + + qb = QueryBuilder("account").where(eq("a", 1)).filter_eq("b", 2).where(gt("c", 3)) + self.assertEqual(qb.build()["filter"], "a eq 1 and b eq 2 and c gt 3") + + def test_where_returns_self(self): + from PowerPlatform.Dataverse.models.filters import eq + + qb = QueryBuilder("account") + self.assertIs(qb.where(eq("statecode", 0)), qb) + + def test_where_non_expression_raises(self): + qb = QueryBuilder("account") + with self.assertRaises(TypeError): + qb.where("statecode eq 0") # type: ignore + + def test_where_with_not(self): + from PowerPlatform.Dataverse.models.filters import eq + + qb = QueryBuilder("account").where(~eq("statecode", 1)) + self.assertEqual(qb.build()["filter"], "not (statecode eq 1)") + + def test_where_with_filter_in(self): + from PowerPlatform.Dataverse.models.filters import filter_in, gt + + expr = filter_in("statecode", [0, 1]) & gt("revenue", 100000) + qb = QueryBuilder("account").where(expr) + self.assertEqual( + qb.build()["filter"], + '(Microsoft.Dynamics.CRM.In(PropertyName=\'statecode\',PropertyValues=["0","1"]) and revenue gt 100000)', + ) + + +class TestOrderBy(unittest.TestCase): + """Tests for the order_by() method.""" + + def test_ascending(self): + qb = QueryBuilder("account").order_by("name") + self.assertEqual(qb.build()["orderby"], ["name"]) + + def test_descending(self): + qb = QueryBuilder("account").order_by("revenue", descending=True) + self.assertEqual(qb.build()["orderby"], ["revenue desc"]) + + def test_multiple(self): + qb = QueryBuilder("account").order_by("revenue", descending=True).order_by("name") + self.assertEqual(qb.build()["orderby"], ["revenue desc", "name"]) + + def test_returns_self(self): + qb = QueryBuilder("account") + self.assertIs(qb.order_by("name"), qb) + + +class TestTopAndPageSize(unittest.TestCase): + """Tests for top() and page_size() methods.""" + + def test_top(self): + qb = QueryBuilder("account").top(10) + self.assertEqual(qb.build()["top"], 10) + + def test_top_invalid_raises(self): + with self.assertRaises(ValueError): + QueryBuilder("account").top(0) + with self.assertRaises(ValueError): + QueryBuilder("account").top(-1) + + def test_top_returns_self(self): + qb = QueryBuilder("account") + self.assertIs(qb.top(10), qb) + + def test_page_size(self): + qb = QueryBuilder("account").page_size(50) + self.assertEqual(qb.build()["page_size"], 50) + + def test_page_size_invalid_raises(self): + with self.assertRaises(ValueError): + QueryBuilder("account").page_size(0) + with self.assertRaises(ValueError): + QueryBuilder("account").page_size(-1) + + def test_page_size_returns_self(self): + qb = QueryBuilder("account") + self.assertIs(qb.page_size(50), qb) + + +class TestExpand(unittest.TestCase): + """Tests for the expand() method.""" + + def test_expand_single(self): + qb = QueryBuilder("account").expand("primarycontactid") + self.assertEqual(qb.build()["expand"], ["primarycontactid"]) + + def test_expand_multiple(self): + qb = QueryBuilder("account").expand("primarycontactid", "ownerid") + self.assertEqual(qb.build()["expand"], ["primarycontactid", "ownerid"]) + + def test_expand_returns_self(self): + qb = QueryBuilder("account") + self.assertIs(qb.expand("primarycontactid"), qb) + + def test_expand_with_expand_option(self): + from PowerPlatform.Dataverse.models.query_builder import ExpandOption + + opt = ExpandOption("Account_Tasks").select("subject", "createdon").top(5) + qb = QueryBuilder("account").expand(opt) + self.assertEqual( + qb.build()["expand"], + ["Account_Tasks($select=subject,createdon;$top=5)"], + ) + + def test_expand_option_with_filter_and_orderby(self): + from PowerPlatform.Dataverse.models.query_builder import ExpandOption + + opt = ( + ExpandOption("Account_Tasks") + .select("subject") + .filter("contains(subject,'Task')") + .order_by("createdon", descending=True) + .top(10) + ) + self.assertEqual( + opt.to_odata(), + "Account_Tasks($select=subject;$filter=contains(subject,'Task');$orderby=createdon desc;$top=10)", + ) + + def test_expand_option_no_options_returns_plain_name(self): + from PowerPlatform.Dataverse.models.query_builder import ExpandOption + + opt = ExpandOption("primarycontactid") + self.assertEqual(opt.to_odata(), "primarycontactid") + + def test_expand_mixed_strings_and_options(self): + from PowerPlatform.Dataverse.models.query_builder import ExpandOption + + opt = ExpandOption("Account_Tasks").select("subject") + qb = QueryBuilder("account").expand("primarycontactid", opt) + self.assertEqual( + qb.build()["expand"], + ["primarycontactid", "Account_Tasks($select=subject)"], + ) + + def test_expand_option_chained_select_accumulates(self): + """Calling select() multiple times should accumulate columns.""" + from PowerPlatform.Dataverse.models.query_builder import ExpandOption + + opt = ExpandOption("Account_Tasks").select("subject").select("createdon") + self.assertEqual( + opt.to_odata(), + "Account_Tasks($select=subject,createdon)", + ) + + def test_expand_option_multiple_order_by(self): + """Calling order_by() multiple times should accumulate sort clauses.""" + from PowerPlatform.Dataverse.models.query_builder import ExpandOption + + opt = ( + ExpandOption("Account_Tasks").select("subject").order_by("priority", descending=True).order_by("createdon") + ) + self.assertEqual( + opt.to_odata(), + "Account_Tasks($select=subject;$orderby=priority desc,createdon)", + ) + + def test_expand_option_filter_last_wins(self): + """Calling filter() multiple times should use the last value.""" + from PowerPlatform.Dataverse.models.query_builder import ExpandOption + + opt = ExpandOption("Account_Tasks").filter("statecode eq 0").filter("contains(subject,'Task')") + self.assertEqual( + opt.to_odata(), + "Account_Tasks($filter=contains(subject,'Task'))", + ) + + +class TestCount(unittest.TestCase): + """Tests for the count() method.""" + + def test_count_sets_flag(self): + qb = QueryBuilder("account").count() + self.assertTrue(qb.build()["count"]) + + def test_count_not_in_build_by_default(self): + params = QueryBuilder("account").build() + self.assertNotIn("count", params) + + def test_count_returns_self(self): + qb = QueryBuilder("account") + self.assertIs(qb.count(), qb) + + +class TestIncludeAnnotations(unittest.TestCase): + """Tests for include_formatted_values() and include_annotations().""" + + def test_include_formatted_values(self): + qb = QueryBuilder("account").include_formatted_values() + self.assertEqual( + qb.build()["include_annotations"], + "OData.Community.Display.V1.FormattedValue", + ) + + def test_include_annotations_default_wildcard(self): + qb = QueryBuilder("account").include_annotations() + self.assertEqual(qb.build()["include_annotations"], "*") + + def test_include_annotations_custom(self): + qb = QueryBuilder("account").include_annotations("Microsoft.Dynamics.CRM.lookuplogicalname") + self.assertEqual( + qb.build()["include_annotations"], + "Microsoft.Dynamics.CRM.lookuplogicalname", + ) + + def test_annotations_not_in_build_by_default(self): + params = QueryBuilder("account").build() + self.assertNotIn("include_annotations", params) + + def test_include_formatted_values_returns_self(self): + qb = QueryBuilder("account") + self.assertIs(qb.include_formatted_values(), qb) + + def test_include_annotations_returns_self(self): + qb = QueryBuilder("account") + self.assertIs(qb.include_annotations(), qb) + + def test_include_annotations_overrides_formatted_values(self): + """Last annotation call should win.""" + qb = QueryBuilder("account").include_formatted_values().include_annotations("*") + self.assertEqual(qb.build()["include_annotations"], "*") + + def test_include_formatted_values_overrides_annotations(self): + """Last annotation call should win (reverse order).""" + qb = QueryBuilder("account").include_annotations("*").include_formatted_values() + self.assertEqual( + qb.build()["include_annotations"], + "OData.Community.Display.V1.FormattedValue", + ) + + +class TestBuild(unittest.TestCase): + """Tests for the build() method.""" + + def test_empty_builder_only_has_table(self): + params = QueryBuilder("account").build() + self.assertEqual(params, {"table": "account"}) + self.assertNotIn("select", params) + self.assertNotIn("filter", params) + self.assertNotIn("orderby", params) + self.assertNotIn("expand", params) + self.assertNotIn("top", params) + self.assertNotIn("page_size", params) + + def test_full_query_build(self): + qb = ( + QueryBuilder("account") + .select("name", "revenue", "telephone1") + .filter_eq("statecode", 0) + .filter_gt("revenue", 1000000) + .order_by("revenue", descending=True) + .order_by("name") + .expand("primarycontactid") + .top(50) + .page_size(25) + ) + params = qb.build() + self.assertEqual(params["table"], "account") + self.assertEqual(params["select"], ["name", "revenue", "telephone1"]) + self.assertEqual(params["filter"], "statecode eq 0 and revenue gt 1000000") + self.assertEqual(params["orderby"], ["revenue desc", "name"]) + self.assertEqual(params["expand"], ["primarycontactid"]) + self.assertEqual(params["top"], 50) + self.assertEqual(params["page_size"], 25) + + def test_build_returns_fresh_lists(self): + """build() should return copies of internal lists.""" + qb = QueryBuilder("account").select("name") + params1 = qb.build() + params2 = qb.build() + self.assertEqual(params1["select"], params2["select"]) + self.assertIsNot(params1["select"], params2["select"]) + + +class TestMethodChainingReturnsSelf(unittest.TestCase): + """Verify all methods return self for chaining.""" + + def test_all_methods_return_self(self): + from PowerPlatform.Dataverse.models.filters import eq + + qb = QueryBuilder("account") + + self.assertIs(qb.select("name"), qb) + self.assertIs(qb.filter_eq("a", 1), qb) + self.assertIs(qb.filter_ne("b", 2), qb) + self.assertIs(qb.filter_gt("c", 3), qb) + self.assertIs(qb.filter_ge("d", 4), qb) + self.assertIs(qb.filter_lt("e", 5), qb) + self.assertIs(qb.filter_le("f", 6), qb) + self.assertIs(qb.filter_contains("g", "x"), qb) + self.assertIs(qb.filter_startswith("h", "y"), qb) + self.assertIs(qb.filter_endswith("i", "z"), qb) + self.assertIs(qb.filter_null("j"), qb) + self.assertIs(qb.filter_not_null("k"), qb) + self.assertIs(qb.filter_raw("l eq 1"), qb) + self.assertIs(qb.filter_in("m", [1, 2]), qb) + self.assertIs(qb.filter_between("n", 1, 10), qb) + self.assertIs(qb.where(eq("o", 1)), qb) + self.assertIs(qb.order_by("p"), qb) + self.assertIs(qb.expand("q"), qb) + self.assertIs(qb.top(10), qb) + self.assertIs(qb.page_size(5), qb) + self.assertIs(qb.count(), qb) + self.assertIs(qb.include_formatted_values(), qb) + self.assertIs(qb.include_annotations(), qb) + + +class TestExecute(unittest.TestCase): + """Tests for the execute() terminal method.""" + + def test_execute_without_query_ops_raises(self): + qb = QueryBuilder("account").filter_eq("statecode", 0) + with self.assertRaises(RuntimeError) as ctx: + qb.execute() + self.assertIn("client.query.builder()", str(ctx.exception)) + + def test_execute_calls_records_get(self): + """execute() should delegate to client.records.get() with built params.""" + mock_query_ops = MagicMock() + mock_client = mock_query_ops._client + mock_client.records.get.return_value = iter([[{"name": "Test"}]]) + + qb = QueryBuilder("account") + qb._query_ops = mock_query_ops + qb.select("name", "revenue").filter_eq("statecode", 0).order_by("revenue", descending=True).top(100).page_size( + 50 + ).expand("primarycontactid") + + list(qb.execute()) + + mock_client.records.get.assert_called_once_with( + "account", + select=["name", "revenue"], + filter="statecode eq 0", + orderby=["revenue desc"], + top=100, + expand=["primarycontactid"], + page_size=50, + count=False, + include_annotations=None, + ) + + def test_execute_returns_flat_records_by_default(self): + mock_query_ops = MagicMock() + mock_client = mock_query_ops._client + mock_client.records.get.return_value = iter([[{"name": "A"}, {"name": "B"}], [{"name": "C"}]]) + + qb = QueryBuilder("account") + qb._query_ops = mock_query_ops + qb.select("name") + records = list(qb.execute()) + + self.assertEqual(len(records), 3) + self.assertEqual(records[0]["name"], "A") + self.assertEqual(records[1]["name"], "B") + self.assertEqual(records[2]["name"], "C") + + def test_execute_by_page_returns_pages(self): + mock_query_ops = MagicMock() + mock_client = mock_query_ops._client + + page1 = [{"name": "A"}, {"name": "B"}] + page2 = [{"name": "C"}] + mock_client.records.get.return_value = iter([page1, page2]) + + qb = QueryBuilder("account") + qb._query_ops = mock_query_ops + qb.select("name") + pages = list(qb.execute(by_page=True)) + + self.assertEqual(len(pages), 2) + self.assertEqual(pages[0], page1) + self.assertEqual(pages[1], page2) + + def test_execute_unbounded_raises(self): + """execute() with no select/filter/top should raise ValueError.""" + mock_query_ops = MagicMock() + qb = QueryBuilder("account") + qb._query_ops = mock_query_ops + with self.assertRaises(ValueError) as ctx: + qb.execute() + self.assertIn("Unbounded query", str(ctx.exception)) + + def test_execute_with_only_select_succeeds(self): + """execute() with select only should not raise.""" + mock_query_ops = MagicMock() + mock_client = mock_query_ops._client + mock_client.records.get.return_value = iter([]) + + qb = QueryBuilder("account") + qb._query_ops = mock_query_ops + qb.select("name") + list(qb.execute()) # should not raise + mock_client.records.get.assert_called_once() + + def test_execute_with_only_filter_succeeds(self): + """execute() with filter only should not raise.""" + mock_query_ops = MagicMock() + mock_client = mock_query_ops._client + mock_client.records.get.return_value = iter([]) + + qb = QueryBuilder("account") + qb._query_ops = mock_query_ops + qb.filter_eq("statecode", 0) + list(qb.execute()) # should not raise + mock_client.records.get.assert_called_once() + + def test_execute_with_only_top_succeeds(self): + """execute() with top only should not raise.""" + mock_query_ops = MagicMock() + mock_client = mock_query_ops._client + mock_client.records.get.return_value = iter([]) + + qb = QueryBuilder("account") + qb._query_ops = mock_query_ops + qb.top(10) + list(qb.execute()) # should not raise + mock_client.records.get.assert_called_once() + + def test_execute_with_only_expand_raises(self): + """expand alone is not a sufficient constraint.""" + mock_query_ops = MagicMock() + qb = QueryBuilder("account") + qb._query_ops = mock_query_ops + qb.expand("primarycontactid") + with self.assertRaises(ValueError): + qb.execute() + + def test_execute_with_only_count_raises(self): + """count alone is not a sufficient constraint.""" + mock_query_ops = MagicMock() + qb = QueryBuilder("account") + qb._query_ops = mock_query_ops + qb.count() + with self.assertRaises(ValueError): + qb.execute() + + def test_execute_with_where_expressions(self): + from PowerPlatform.Dataverse.models.filters import eq, gt + + mock_query_ops = MagicMock() + mock_client = mock_query_ops._client + mock_client.records.get.return_value = iter([]) + + qb = QueryBuilder("account") + qb._query_ops = mock_query_ops + qb.where((eq("statecode", 0) | eq("statecode", 1)) & gt("revenue", 100000)) + list(qb.execute()) + + call_args = mock_client.records.get.call_args + self.assertEqual( + call_args.kwargs["filter"], + "((statecode eq 0 or statecode eq 1) and revenue gt 100000)", + ) + + def test_execute_with_filter_in(self): + mock_query_ops = MagicMock() + mock_client = mock_query_ops._client + mock_client.records.get.return_value = iter([]) + + qb = QueryBuilder("account") + qb._query_ops = mock_query_ops + qb.filter_in("statecode", [0, 1, 2]) + list(qb.execute()) + + call_args = mock_client.records.get.call_args + self.assertEqual( + call_args.kwargs["filter"], + 'Microsoft.Dynamics.CRM.In(PropertyName=\'statecode\',PropertyValues=["0","1","2"])', + ) + + def test_execute_passes_count_and_annotations(self): + """execute() should forward count and include_annotations when set.""" + mock_query_ops = MagicMock() + mock_client = mock_query_ops._client + mock_client.records.get.return_value = iter([]) + + qb = QueryBuilder("account") + qb._query_ops = mock_query_ops + qb.select("name").count().include_formatted_values() + list(qb.execute()) + + mock_client.records.get.assert_called_once_with( + "account", + select=["name"], + filter=None, + orderby=None, + top=None, + expand=None, + page_size=None, + count=True, + include_annotations="OData.Community.Display.V1.FormattedValue", + ) + + +class TestToDataframe(unittest.TestCase): + """Tests for the to_dataframe() terminal method.""" + + def test_to_dataframe_without_query_ops_raises(self): + qb = QueryBuilder("account").filter_eq("statecode", 0) + with self.assertRaises(RuntimeError) as ctx: + qb.to_dataframe() + self.assertIn("client.query.builder()", str(ctx.exception)) + + def test_to_dataframe_delegates_to_dataframe_get(self): + """to_dataframe() should delegate to client.dataframe.get() with built params.""" + import pandas as pd + + mock_query_ops = MagicMock() + mock_client = mock_query_ops._client + expected_df = pd.DataFrame([{"name": "Contoso", "revenue": 1000}]) + mock_client.dataframe.get.return_value = expected_df + + qb = QueryBuilder("account") + qb._query_ops = mock_query_ops + qb.select("name", "revenue").filter_eq("statecode", 0).order_by("revenue", descending=True).top(100).page_size( + 50 + ).expand("primarycontactid") + + result = qb.to_dataframe() + + mock_client.dataframe.get.assert_called_once_with( + "account", + select=["name", "revenue"], + filter="statecode eq 0", + orderby=["revenue desc"], + top=100, + expand=["primarycontactid"], + page_size=50, + count=False, + include_annotations=None, + ) + pd.testing.assert_frame_equal(result, expected_df) + + def test_to_dataframe_unbounded_raises(self): + """to_dataframe() with no select/filter/top should raise ValueError.""" + mock_query_ops = MagicMock() + qb = QueryBuilder("account") + qb._query_ops = mock_query_ops + with self.assertRaises(ValueError) as ctx: + qb.to_dataframe() + self.assertIn("Unbounded query", str(ctx.exception)) + + def test_to_dataframe_returns_dataframe(self): + """to_dataframe() should return a pandas DataFrame.""" + import pandas as pd + + mock_query_ops = MagicMock() + mock_client = mock_query_ops._client + mock_client.dataframe.get.return_value = pd.DataFrame( + [{"name": "A", "revenue": 100}, {"name": "B", "revenue": 200}] + ) + + qb = QueryBuilder("account") + qb._query_ops = mock_query_ops + qb.select("name", "revenue") + + result = qb.to_dataframe() + + self.assertIsInstance(result, pd.DataFrame) + self.assertEqual(len(result), 2) + self.assertListEqual(list(result.columns), ["name", "revenue"]) + + def test_to_dataframe_forwards_count_and_annotations(self): + """to_dataframe() should forward count and include_annotations when set.""" + import pandas as pd + + mock_query_ops = MagicMock() + mock_client = mock_query_ops._client + mock_client.dataframe.get.return_value = pd.DataFrame() + + qb = QueryBuilder("account") + qb._query_ops = mock_query_ops + qb.select("name").count().include_formatted_values() + qb.to_dataframe() + + mock_client.dataframe.get.assert_called_once_with( + "account", + select=["name"], + filter=None, + orderby=None, + top=None, + expand=None, + page_size=None, + count=True, + include_annotations="OData.Community.Display.V1.FormattedValue", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 8eb07ca1..cfad101e 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -126,6 +126,8 @@ def test_get_multiple(self): top=10, expand=None, page_size=None, + count=False, + include_annotations=None, ) self.assertEqual(len(results), 1) self.assertEqual(len(results[0]), 2) diff --git a/tests/unit/test_client_dataframe.py b/tests/unit/test_client_dataframe.py index d19419ac..72c30353 100644 --- a/tests/unit/test_client_dataframe.py +++ b/tests/unit/test_client_dataframe.py @@ -110,6 +110,8 @@ def test_get_passes_all_parameters(self): top=50, expand=["primarycontactid"], page_size=25, + count=False, + include_annotations=None, ) diff --git a/tests/unit/test_client_deprecations.py b/tests/unit/test_client_deprecations.py index 7c997079..54ece8bc 100644 --- a/tests/unit/test_client_deprecations.py +++ b/tests/unit/test_client_deprecations.py @@ -130,6 +130,8 @@ def test_get_multiple_warns(self): top=10, expand=None, page_size=None, + count=False, + include_annotations=None, ) # ------------------------------------------------------------- query_sql diff --git a/tests/unit/test_dataframe_operations.py b/tests/unit/test_dataframe_operations.py index 7931c3f5..c83ae05b 100644 --- a/tests/unit/test_dataframe_operations.py +++ b/tests/unit/test_dataframe_operations.py @@ -94,6 +94,8 @@ def test_get_passes_all_params(self): top=50, expand=["primarycontactid"], page_size=25, + count=False, + include_annotations=None, ) def test_get_record_id_with_query_params_raises(self): diff --git a/tests/unit/test_query_operations.py b/tests/unit/test_query_operations.py index b2abf97b..2d025b52 100644 --- a/tests/unit/test_query_operations.py +++ b/tests/unit/test_query_operations.py @@ -55,6 +55,227 @@ def test_sql_empty_result(self): self.assertIsInstance(result, list) self.assertEqual(result, []) + # ----------------------------------------------------------------- builder + + def test_builder_returns_query_builder(self): + """builder() should return a QueryBuilder with _query_ops set.""" + from PowerPlatform.Dataverse.models.query_builder import QueryBuilder + + qb = self.client.query.builder("account") + + self.assertIsInstance(qb, QueryBuilder) + self.assertEqual(qb.table, "account") + self.assertIs(qb._query_ops, self.client.query) + + def test_builder_execute_flat_default(self): + """builder().execute() should return flat records by default.""" + self.client._odata._get_multiple.return_value = iter([[{"accountid": "1", "name": "Test"}]]) + + records = list(self.client.query.builder("account").select("name").filter_eq("statecode", 0).top(10).execute()) + + self.client._odata._get_multiple.assert_called_once_with( + "account", + select=["name"], + filter="statecode eq 0", + orderby=None, + top=10, + expand=None, + page_size=None, + count=False, + include_annotations=None, + ) + self.assertEqual(len(records), 1) + self.assertEqual(records[0]["name"], "Test") + + def test_builder_execute_flat_multiple_pages(self): + """execute() should flatten records from multiple pages.""" + self.client._odata._get_multiple.return_value = iter([[{"accountid": "1"}], [{"accountid": "2"}]]) + + records = list(self.client.query.builder("account").select("name").execute()) + + self.assertEqual(len(records), 2) + self.assertEqual(records[0]["accountid"], "1") + self.assertEqual(records[1]["accountid"], "2") + + def test_builder_execute_by_page(self): + """execute(by_page=True) should yield pages.""" + self.client._odata._get_multiple.return_value = iter([[{"accountid": "1"}], [{"accountid": "2"}]]) + + pages = list(self.client.query.builder("account").select("name").execute(by_page=True)) + + self.assertEqual(len(pages), 2) + self.assertEqual(len(pages[0]), 1) + self.assertEqual(pages[0][0]["accountid"], "1") + self.assertEqual(pages[1][0]["accountid"], "2") + + def test_builder_execute_all_params(self): + """builder().execute() should forward all parameters.""" + self.client._odata._get_multiple.return_value = iter([[{"name": "Test"}]]) + + list( + self.client.query.builder("account") + .select("name", "revenue") + .filter_eq("statecode", 0) + .filter_gt("revenue", 1000000) + .order_by("revenue", descending=True) + .expand("primarycontactid") + .top(50) + .page_size(25) + .execute() + ) + + self.client._odata._get_multiple.assert_called_once_with( + "account", + select=["name", "revenue"], + filter="statecode eq 0 and revenue gt 1000000", + orderby=["revenue desc"], + top=50, + expand=["primarycontactid"], + page_size=25, + count=False, + include_annotations=None, + ) + + def test_builder_execute_with_where(self): + """builder().where().execute() should compile expression to filter.""" + from PowerPlatform.Dataverse.models.filters import eq, gt + + self.client._odata._get_multiple.return_value = iter([[{"name": "Test"}]]) + + list( + self.client.query.builder("account") + .where((eq("statecode", 0) | eq("statecode", 1)) & gt("revenue", 100000)) + .execute() + ) + + call_kwargs = self.client._odata._get_multiple.call_args + self.assertEqual( + call_kwargs.kwargs["filter"], + "((statecode eq 0 or statecode eq 1) and revenue gt 100000)", + ) + + def test_builder_execute_with_filter_in(self): + """builder().filter_in().execute() should forward CRM.In filter to _get_multiple.""" + self.client._odata._get_multiple.return_value = iter([[{"accountid": "1"}]]) + + list(self.client.query.builder("account").select("name").filter_in("statecode", [0, 1, 2]).execute()) + + call_kwargs = self.client._odata._get_multiple.call_args + self.assertEqual( + call_kwargs.kwargs["filter"], + 'Microsoft.Dynamics.CRM.In(PropertyName=\'statecode\',PropertyValues=["0","1","2"])', + ) + + def test_builder_execute_with_where_filter_in(self): + """builder().where(filter_in(...) & ...).execute() should compile composed expression.""" + from PowerPlatform.Dataverse.models.filters import filter_in, gt + + self.client._odata._get_multiple.return_value = iter([[{"accountid": "1"}]]) + + list( + self.client.query.builder("account").where(filter_in("statecode", [0, 1]) & gt("revenue", 100000)).execute() + ) + + call_kwargs = self.client._odata._get_multiple.call_args + self.assertEqual( + call_kwargs.kwargs["filter"], + '(Microsoft.Dynamics.CRM.In(PropertyName=\'statecode\',PropertyValues=["0","1"]) and revenue gt 100000)', + ) + + def test_builder_execute_with_filter_between_datetimes(self): + """builder().filter_between() with datetimes should forward correct OData.""" + from datetime import datetime, timezone + + self.client._odata._get_multiple.return_value = iter([[{"accountid": "1"}]]) + + start = datetime(2024, 1, 1, 0, 0, 0, tzinfo=timezone.utc) + end = datetime(2024, 12, 31, 23, 59, 59, tzinfo=timezone.utc) + list(self.client.query.builder("account").filter_between("createdon", start, end).execute()) + + call_kwargs = self.client._odata._get_multiple.call_args + self.assertEqual( + call_kwargs.kwargs["filter"], + "(createdon ge 2024-01-01T00:00:00Z and createdon le 2024-12-31T23:59:59Z)", + ) + + def test_builder_execute_with_filter_not_in(self): + """builder().filter_not_in().execute() should forward CRM.NotIn filter.""" + self.client._odata._get_multiple.return_value = iter([[{"accountid": "1"}]]) + + list(self.client.query.builder("account").select("name").filter_not_in("statecode", [2, 3]).execute()) + + call_kwargs = self.client._odata._get_multiple.call_args + self.assertEqual( + call_kwargs.kwargs["filter"], + 'Microsoft.Dynamics.CRM.NotIn(PropertyName=\'statecode\',PropertyValues=["2","3"])', + ) + + def test_builder_execute_with_filter_not_between(self): + """builder().filter_not_between().execute() should forward negated between filter.""" + self.client._odata._get_multiple.return_value = iter([[{"accountid": "1"}]]) + + list(self.client.query.builder("account").filter_not_between("revenue", 100000, 500000).execute()) + + call_kwargs = self.client._odata._get_multiple.call_args + self.assertEqual( + call_kwargs.kwargs["filter"], + "not ((revenue ge 100000 and revenue le 500000))", + ) + + def test_builder_full_fluent_workflow(self): + """End-to-end test of the fluent query workflow.""" + expected_records = [ + {"accountid": "1", "name": "Big Corp", "revenue": 5000000}, + {"accountid": "2", "name": "Mega Inc", "revenue": 4000000}, + ] + self.client._odata._get_multiple.return_value = iter([expected_records]) + + records = list( + self.client.query.builder("account") + .select("name", "revenue") + .filter_eq("statecode", 0) + .filter_gt("revenue", 1000000) + .order_by("revenue", descending=True) + .expand("primarycontactid") + .top(10) + .page_size(5) + .execute() + ) + + self.assertEqual(len(records), 2) + self.assertEqual(records[0]["name"], "Big Corp") + self.assertEqual(records[1]["name"], "Mega Inc") + + def test_builder_to_dataframe(self): + """builder().to_dataframe() should delegate to client.dataframe.get().""" + import pandas as pd + + expected_df = pd.DataFrame([{"name": "Contoso", "revenue": 1000}]) + self.client.dataframe = MagicMock() + self.client.dataframe.get.return_value = expected_df + + result = ( + self.client.query.builder("account") + .select("name", "revenue") + .filter_eq("statecode", 0) + .order_by("name") + .top(50) + .to_dataframe() + ) + + self.client.dataframe.get.assert_called_once_with( + "account", + select=["name", "revenue"], + filter="statecode eq 0", + orderby=["name"], + top=50, + expand=None, + page_size=None, + count=False, + include_annotations=None, + ) + pd.testing.assert_frame_equal(result, expected_df) + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_records_operations.py b/tests/unit/test_records_operations.py index 8387db4d..97da5a40 100644 --- a/tests/unit/test_records_operations.py +++ b/tests/unit/test_records_operations.py @@ -197,6 +197,8 @@ def test_get_paginated_with_all_params(self): top=50, expand=["primarycontactid"], page_size=25, + count=False, + include_annotations=None, ) # ------------------------------------------------------------------ upsert From 5cd086c12d08d4b8a35ea6067fc82951c1ed3de2 Mon Sep 17 00:00:00 2001 From: sagebree Date: Tue, 7 Apr 2026 15:48:11 -0700 Subject: [PATCH 17/20] Implement batch API with changeset, upsert, and DataFrame integration (#129) ## Summary - Adds `client.batch` namespace -- a deferred-execution batch API that packs multiple Dataverse Web API operations into a single `POST $batch` HTTP request - Adds `client.batch.dataframe` namespace -- pandas DataFrame wrappers for batch operations - Adds `client.records.upsert()` and `client.batch.records.upsert()` backed by the `UpsertMultiple` bound action with alternate-key support - Fixes a bug where alternate key fields were merged into the UpsertMultiple request body, causing `400 Bad Request` on the create path ## Batch API Design Implements the [Batch API Design](https://github.com/microsoft/PowerPlatform-DataverseClient-Python/pull/129#issuecomment-3974570865) spec from @sagebree: | Capability | How to use | Status | |---|---|---| | Record CRUD (create / update / delete / get) | `batch.records.*` | Done | | Upsert by alternate key | `batch.records.upsert(...)` | Done | | Table metadata (create / delete / columns / relationships) | `batch.tables.*` | Done | | SQL queries | `batch.query.sql(...)` | Done | | Atomic write groups | `batch.changeset()` | Done | | Continue past failures | `batch.execute(continue_on_error=True)` | Done | | DataFrame integration | `batch.dataframe.create/update/delete` | Done (new) | **Design constraints enforced:** - Maximum 1000 operations per batch (validated before sending) - `records.get` paginated overload not supported -- single-record only - GET operations cannot be placed inside a changeset (enforced by API design) - Content-ID references are only valid within the same changeset - File upload operations not batchable - `tables.create` returns no table metadata on success (HTTP 204) - `tables.add_columns` / `tables.remove_columns` do not flush the picklist cache - `client.flush_cache()` not supported in batch (client-side operation) ## What's included ### New: `client.batch` API - `batch.records.create / get / update / delete / upsert` - `batch.tables.create / get / list / add_columns / remove_columns / delete` - `batch.tables.list(filter=..., select=...)` -- parity with `client.tables.list()` from #112 - `batch.tables.create_one_to_many_relationship / create_many_to_many_relationship / delete_relationship / get_relationship / create_lookup_field` - `batch.query.sql` - `batch.changeset()` context manager for transactional (all-or-nothing) operations - Content-ID reference chaining inside changesets (globally unique across all changesets via shared counter) - `execute(continue_on_error=True)` for mixed success/failure batches - `BatchResult` with `.responses`, `.succeeded`, `.failed`, `.created_ids`, `.has_errors` ### New: `client.batch.dataframe` API - `batch.dataframe.create(table, df)` -- DataFrame rows to CreateMultiple batch item - `batch.dataframe.update(table, df, id_column)` -- DataFrame rows to update batch items - `batch.dataframe.delete(table, ids_series)` -- pandas Series to delete batch items ### Existing: Refactored existing APIs - Payload generation shared between batch and direct API via `_build_*` / `_RawRequest` pattern - Execution of batch operations deferred to `execute()` ### OData $batch spec compliance - Audited against [Microsoft Learn docs](https://learn.microsoft.com/en-us/power-apps/developer/data-platform/webapi/execute-batch-operations-using-web-api) - `Content-Transfer-Encoding: binary` per part - `Content-Type: application/http` per part - `Content-Type: application/json; type=entry` for POST/PATCH bodies - CRLF line endings throughout - Absolute URLs in batch parts - Empty changesets silently skipped (prevents invalid multipart) - Top-level batch error handling (non-multipart 4xx/5xx raises `HttpError` with parsed Dataverse error details) - Accepts `200`, `202 Accepted`, `207 Multi-Status`, and `400` batch response codes ### Review comment fixes - Fixed `expected` status codes to include `202`/`207` for all Dataverse environments - Fixed `_split_multipart` / `_parse_mime_part` return type annotations: `List[Tuple[Dict[str, str], str]]` - Fixed OptionSet string check regression: now uses dict key lookup instead of JSON string search - Fixed `_build_get` to lowercase select column names (consistency with `_get_multiple`) - Added RFC 3986 `%20` encoding documentation in `_build_sql` docstring - Fixed content-id response parsing for non-changeset parts - Fixed test assertions after merge: `data` bytes instead of `json` kwarg - Exception type parity: `batch.records.upsert()` raises `TypeError` (matching `client.records.upsert()`) ### Testing **Unit tests -- 579 tests passing:** - `test_batch_operations.py` -- BatchRequest, BatchRecordOperations, BatchTableOperations, BatchQueryOperations, ChangeSet, BatchItemResponse, BatchResult - `test_batch_serialization.py` -- multipart serialization, response parsing, intent resolution, upsert dispatch, batch size limit, content-ID uniqueness, top-level error handling - `test_batch_edge_cases.py` -- 40 edge case tests: empty changeset, changeset rollback, content-ID in standalone parts, mixed batch, multiple changesets, batch size limits, top-level errors, continue-on-error, serialization compliance, multipart parsing, content-ID references, intent validation - `test_batch_dataframe.py` -- 18 tests: DataFrame create/update/delete, validation, NaN handling, empty series, bulk delete - `test_odata_internal.py` -- `_build_upsert_multiple` body exclusion, conflict detection, URL/method correctness **E2E tests -- 14 tests passing against live Dataverse (`crm10.dynamics.com`):** 1. Basic batch CRUD (single create + CreateMultiple, update, get, delete) 2. Changeset happy path (create + update via `$ref` content-ID) 3. Changeset rollback (failing op rolls back entire changeset) 4. Multiple changesets (globally unique content-IDs) 5. Continue-on-error (mixed success/failure) 6. Batch SQL query 7. Batch tables.get + tables.list 8. DataFrame batch create 9. DataFrame batch update 10. DataFrame batch delete 11. Mixed batch (changeset + standalone GET) 12. Empty changeset (silently skipped) 13. Content-ID chaining (2 creates + 2 updates via `$ref`) 14. Table setup/teardown ### Examples & docs - `examples/advanced/batch.py` -- reference examples for all batch operation types - `examples/advanced/walkthrough.py` -- batch section added (section 11) - `examples/basic/functional_testing.py` -- `test_batch_all_operations()` covering all operation categories against a live environment --------- Co-authored-by: Samson Gebre Co-authored-by: Saurabh Badenkal --- .claude/skills/dataverse-sdk-dev/SKILL.md | 2 +- .claude/skills/dataverse-sdk-use/SKILL.md | 45 + README.md | 91 +- examples/advanced/batch.py | 260 ++++ examples/advanced/walkthrough.py | 84 +- examples/basic/functional_testing.py | 478 ++++++++ .../claude_skill/dataverse-sdk-dev/SKILL.md | 2 +- .../claude_skill/dataverse-sdk-use/SKILL.md | 45 + src/PowerPlatform/Dataverse/client.py | 3 + src/PowerPlatform/Dataverse/data/_batch.py | 695 +++++++++++ src/PowerPlatform/Dataverse/data/_odata.py | 666 +++++++--- .../Dataverse/data/_raw_request.py | 33 + src/PowerPlatform/Dataverse/models/batch.py | 106 ++ .../Dataverse/operations/batch.py | 890 ++++++++++++++ .../Dataverse/operations/records.py | 2 +- .../Dataverse/operations/tables.py | 34 +- tests/unit/data/test_batch_edge_cases.py | 1069 +++++++++++++++++ tests/unit/data/test_batch_serialization.py | 632 ++++++++++ tests/unit/data/test_format_key.py | 58 + tests/unit/data/test_odata_internal.py | 148 ++- tests/unit/data/test_sql_parse.py | 60 + tests/unit/test_batch_dataframe.py | 197 +++ tests/unit/test_batch_operations.py | 463 +++++++ tests/unit/test_batch_scenarios.py | 535 +++++++++ 24 files changed, 6381 insertions(+), 217 deletions(-) create mode 100644 examples/advanced/batch.py create mode 100644 src/PowerPlatform/Dataverse/data/_batch.py create mode 100644 src/PowerPlatform/Dataverse/data/_raw_request.py create mode 100644 src/PowerPlatform/Dataverse/models/batch.py create mode 100644 src/PowerPlatform/Dataverse/operations/batch.py create mode 100644 tests/unit/data/test_batch_edge_cases.py create mode 100644 tests/unit/data/test_batch_serialization.py create mode 100644 tests/unit/data/test_format_key.py create mode 100644 tests/unit/test_batch_dataframe.py create mode 100644 tests/unit/test_batch_operations.py create mode 100644 tests/unit/test_batch_scenarios.py diff --git a/.claude/skills/dataverse-sdk-dev/SKILL.md b/.claude/skills/dataverse-sdk-dev/SKILL.md index 63af04dc..c3af44c5 100644 --- a/.claude/skills/dataverse-sdk-dev/SKILL.md +++ b/.claude/skills/dataverse-sdk-dev/SKILL.md @@ -13,7 +13,7 @@ This skill provides guidance for developers working on the PowerPlatform Dataver ### API Design -1. **Public methods in operation namespaces** - New public methods go in the appropriate namespace module under `src/PowerPlatform/Dataverse/operations/` (`records.py`, `query.py`, `tables.py`). The `client.py` file exposes these via namespace properties (`client.records`, `client.query`, `client.tables`). Public types and constants live in their own modules (e.g., `models/metadata.py`, `common/constants.py`) +1. **Public methods in operation namespaces** - New public methods go in the appropriate namespace module under `src/PowerPlatform/Dataverse/operations/` (`records.py`, `query.py`, `tables.py`, `batch.py`). The `client.py` file exposes these via namespace properties (`client.records`, `client.query`, `client.tables`, `client.batch`). Public types and constants live in their own modules (e.g., `models/metadata.py`, `models/batch.py`, `common/constants.py`) 2. **Every public method needs README example** - Public API methods must have examples in README.md 3. **Reuse existing APIs** - Always check if an existing method can be used before making direct Web API calls 4. **Update documentation** when adding features - Keep README and SKILL files (both copies) in sync diff --git a/.claude/skills/dataverse-sdk-use/SKILL.md b/.claude/skills/dataverse-sdk-use/SKILL.md index 9edb733f..569caec0 100644 --- a/.claude/skills/dataverse-sdk-use/SKILL.md +++ b/.claude/skills/dataverse-sdk-use/SKILL.md @@ -22,6 +22,7 @@ Use the PowerPlatform Dataverse Client Python SDK to interact with Microsoft Dat - `client.query` -- query and search operations - `client.tables` -- table metadata, columns, and relationships - `client.files` -- file upload operations +- `client.batch` -- batch multiple operations into a single HTTP request ### Bulk Operations The SDK supports Dataverse's native bulk operations: Pass lists to `create()`, `update()` for automatic bulk processing, for `delete()`, set `use_bulk_delete` when passing lists to use bulk operation @@ -369,6 +370,50 @@ client.files.upload( ) ``` +### Batch Operations + +Use `client.batch` to send multiple operations in one HTTP request. All batch methods return `None`; results arrive via `BatchResult` after `execute()`. + +```python +# Build a batch request +batch = client.batch.new() +batch.records.create("account", {"name": "Contoso"}) +batch.records.update("account", account_id, {"telephone1": "555-0100"}) +batch.records.get("account", account_id, select=["name"]) +batch.query.sql("SELECT TOP 5 name FROM account") + +result = batch.execute() +for item in result.responses: + if item.is_success: + print(f"[OK] {item.status_code} entity_id={item.entity_id}") + if item.data: + # GET responses populate item.data with the parsed JSON record + print(item.data.get("name")) + else: + print(f"[ERR] {item.status_code}: {item.error_message}") + +# Transactional changeset (all succeed or roll back) +with batch.changeset() as cs: + ref = cs.records.create("contact", {"firstname": "Alice"}) + cs.records.update("account", account_id, {"primarycontactid@odata.bind": ref}) + +# Continue on error +result = batch.execute(continue_on_error=True) +print(f"Succeeded: {len(result.succeeded)}, Failed: {len(result.failed)}") +``` + +**BatchResult properties:** +- `result.responses` -- list of `BatchItemResponse` in submission order +- `result.succeeded` -- responses with 2xx status codes +- `result.failed` -- responses with non-2xx status codes +- `result.has_errors` -- True if any response failed +- `result.entity_ids` -- GUIDs from OData-EntityId headers (creates and updates) + +**Batch limitations:** +- Maximum 1000 operations per batch +- Paginated `records.get()` (without `record_id`) is not supported in batch +- `flush_cache()` is not supported in batch + ## Error Handling The SDK provides structured exceptions with detailed error information: diff --git a/README.md b/README.md index 3b892644..d6c6403d 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ A Python client library for Microsoft Dataverse that provides a unified interfac - [Table management](#table-management) - [Relationship management](#relationship-management) - [File operations](#file-operations) + - [Batch operations](#batch-operations) - [Next steps](#next-steps) - [Troubleshooting](#troubleshooting) - [Contributing](#contributing) @@ -43,6 +44,7 @@ A Python client library for Microsoft Dataverse that provides a unified interfac - **🔗 Relationship Management**: Create one-to-many and many-to-many relationships between tables with full metadata control - **🐼 DataFrame Support**: Pandas wrappers for all CRUD operations, returning DataFrames and Series - **📎 File Operations**: Upload files to Dataverse file columns with automatic chunking for large files +- **📦 Batch Operations**: Send multiple CRUD, table metadata, and SQL query operations in a single HTTP request with optional transactional changesets - **🔐 Azure Identity**: Built-in authentication using Azure Identity credential providers with comprehensive support - **🛡️ Error Handling**: Structured exception hierarchy with detailed error context and retry guidance @@ -115,9 +117,9 @@ The SDK provides a simple, pythonic interface for Dataverse operations: | Concept | Description | |---------|-------------| -| **DataverseClient** | Main entry point; provides `records`, `query`, `tables`, and `files` namespaces | +| **DataverseClient** | Main entry point; provides `records`, `query`, `tables`, `files`, and `batch` namespaces | | **Context Manager** | Use `with DataverseClient(...) as client:` for automatic cleanup and HTTP connection pooling | -| **Namespaces** | Operations are organized into `client.records` (CRUD & OData queries), `client.query` (QueryBuilder & SQL), `client.tables` (metadata), and `client.files` (file uploads) | +| **Namespaces** | Operations are organized into `client.records` (CRUD & OData queries), `client.query` (QueryBuilder & SQL), `client.tables` (metadata), `client.files` (file uploads), and `client.batch` (batch requests) | | **Records** | Dataverse records represented as Python dictionaries with column schema names | | **Schema names** | Use table schema names (`"account"`, `"new_MyTestTable"`) and column schema names (`"name"`, `"new_MyTestColumn"`). See: [Table definitions in Microsoft Dataverse](https://learn.microsoft.com/en-us/power-apps/developer/data-platform/entity-metadata) | | **Bulk Operations** | Efficient bulk processing for multiple records with automatic optimization | @@ -513,6 +515,90 @@ client.files.upload( ) ``` +### Batch operations + +Use `client.batch` to send multiple operations in one HTTP request. The batch namespace mirrors `client.records`, `client.tables`, and `client.query`. + +```python +# Build a batch request and add operations +batch = client.batch.new() +batch.records.create("account", {"name": "Contoso"}) +batch.records.create("account", [{"name": "Fabrikam"}, {"name": "Woodgrove"}]) +batch.records.update("account", account_id, {"telephone1": "555-0100"}) +batch.records.delete("account", old_id) +batch.records.get("account", account_id, select=["name"]) + +result = batch.execute() +for item in result.responses: + if item.is_success: + print(f"[OK] {item.status_code} entity_id={item.entity_id}") + else: + print(f"[ERR] {item.status_code}: {item.error_message}") +``` + +**Transactional changeset** — all operations in a changeset succeed or roll back together: + +```python +batch = client.batch.new() +with batch.changeset() as cs: + lead_ref = cs.records.create("lead", {"firstname": "Ada"}) + contact_ref = cs.records.create("contact", {"firstname": "Ada"}) + cs.records.create("account", { + "name": "Babbage & Co.", + "originatingleadid@odata.bind": lead_ref, + "primarycontactid@odata.bind": contact_ref, + }) +result = batch.execute() +print(f"Created {len(result.entity_ids)} records atomically") +``` + +**Table metadata and SQL queries in a batch:** + +```python +batch = client.batch.new() +batch.tables.create("new_Product", {"new_Price": "decimal", "new_InStock": "bool"}) +batch.tables.add_columns("new_Product", {"new_Rating": "int"}) +batch.tables.get("new_Product") +batch.query.sql("SELECT TOP 5 name FROM account") + +result = batch.execute() +``` + +**Continue on error** — attempt all operations even when one fails: + +```python +result = batch.execute(continue_on_error=True) +print(f"Succeeded: {len(result.succeeded)}, Failed: {len(result.failed)}") +for item in result.failed: + print(f"[ERR] {item.status_code}: {item.error_message}") +``` + +**DataFrame integration** -- feed pandas DataFrames directly into a batch: + +```python +import pandas as pd + +batch = client.batch.new() + +# Create records from a DataFrame +df = pd.DataFrame([{"name": "Contoso"}, {"name": "Fabrikam"}]) +batch.dataframe.create("account", df) + +# Update records from a DataFrame +updates = pd.DataFrame([ + {"accountid": id1, "telephone1": "555-0100"}, + {"accountid": id2, "telephone1": "555-0200"}, +]) +batch.dataframe.update("account", updates, id_column="accountid") + +# Delete records from a Series +batch.dataframe.delete("account", pd.Series([id1, id2])) + +result = batch.execute() +``` + +For a complete example see [examples/advanced/batch.py](https://github.com/microsoft/PowerPlatform-DataverseClient-Python/blob/main/examples/advanced/batch.py). + ## Next steps ### More sample code @@ -527,6 +613,7 @@ Explore our comprehensive examples in the [`examples/`](https://github.com/micro - **[Complete Walkthrough](https://github.com/microsoft/PowerPlatform-DataverseClient-Python/blob/main/examples/advanced/walkthrough.py)** - Full feature demonstration with production patterns - **[Relationship Management](https://github.com/microsoft/PowerPlatform-DataverseClient-Python/blob/main/examples/advanced/relationships.py)** - Create and manage table relationships - **[File Upload](https://github.com/microsoft/PowerPlatform-DataverseClient-Python/blob/main/examples/advanced/file_upload.py)** - Upload files to Dataverse file columns +- **[Batch Operations](https://github.com/microsoft/PowerPlatform-DataverseClient-Python/blob/main/examples/advanced/batch.py)** - Send multiple operations in a single request with changesets 📖 See the [examples README](https://github.com/microsoft/PowerPlatform-DataverseClient-Python/blob/main/examples/README.md) for detailed guidance and learning progression. diff --git a/examples/advanced/batch.py b/examples/advanced/batch.py new file mode 100644 index 00000000..a95aa303 --- /dev/null +++ b/examples/advanced/batch.py @@ -0,0 +1,260 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Batch operations example for the Dataverse Python SDK. + +Demonstrates how to use client.batch to send multiple operations in a single +HTTP request to the Dataverse Web API. + +Requirements: + pip install PowerPlatform-Dataverse-Client azure-identity +""" + +from __future__ import annotations + +# --------------------------------------------------------------------------- +# Setup — replace with your environment URL and credential +# --------------------------------------------------------------------------- + +from azure.identity import InteractiveBrowserCredential +from PowerPlatform.Dataverse.client import DataverseClient + +credential = InteractiveBrowserCredential() + +with DataverseClient("https://org.crm.dynamics.com", credential) as client: + + # --------------------------------------------------------------------------- + # Example 1: Record CRUD in a single batch + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 1: Record CRUD in a single batch") + + batch = client.batch.new() + + # Create a single record + batch.records.create("account", {"name": "Contoso Ltd", "telephone1": "555-0100"}) + + # Create multiple records via CreateMultiple (one batch item) + batch.records.create( + "contact", + [ + {"firstname": "Alice", "lastname": "Smith"}, + {"firstname": "Bob", "lastname": "Jones"}, + ], + ) + + # Assume we have an existing account_id from a prior operation + # batch.records.update("account", account_id, {"telephone1": "555-9999"}) + # batch.records.delete("account", old_id) + + result = batch.execute() + + print(f"[OK] Total: {len(result.responses)}, Succeeded: {len(result.succeeded)}, Failed: {len(result.failed)}") + for guid in result.entity_ids: + print(f"[OK] Created: {guid}") + for item in result.failed: + print(f"[ERR] {item.status_code}: {item.error_message}") + + # --------------------------------------------------------------------------- + # Example 2: Transactional changeset with content-ID chaining + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 2: Transactional changeset") + + batch = client.batch.new() + + with batch.changeset() as cs: + # Each create() returns a "$n" reference usable in subsequent operations + lead_ref = cs.records.create( + "lead", + {"firstname": "Ada", "lastname": "Lovelace"}, + ) + contact_ref = cs.records.create("contact", {"firstname": "Ada"}) + + # Reference the newly created lead and contact in the account + cs.records.create( + "account", + { + "name": "Babbage & Co.", + "originatingleadid@odata.bind": lead_ref, + "primarycontactid@odata.bind": contact_ref, + }, + ) + + # Update using a content-ID reference as the record_id + cs.records.update("contact", contact_ref, {"lastname": "Lovelace"}) + + result = batch.execute() + + if result.has_errors: + print("[ERR] Changeset rolled back") + for item in result.failed: + print(f" {item.status_code}: {item.error_message}") + else: + print(f"[OK] {len(result.entity_ids)} records created atomically") + + # --------------------------------------------------------------------------- + # Example 3: Table metadata operations in a batch + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 3: Table metadata operations") + + batch = client.batch.new() + + # Create a new custom table + batch.tables.create( + "new_Product", + {"new_Price": "decimal", "new_InStock": "bool"}, + solution="MySolution", + ) + + # Read table metadata + batch.tables.get("new_Product") + + # List all non-private tables + batch.tables.list() + + result = batch.execute() + print(f"[OK] Table ops: {[(r.status_code, r.is_success) for r in result.responses]}") + + # --------------------------------------------------------------------------- + # Example 4: SQL query in a batch + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 4: SQL query in batch") + + batch = client.batch.new() + batch.query.sql("SELECT TOP 5 accountid, name FROM account ORDER BY name") + + result = batch.execute() + if result.responses and result.responses[0].is_success and result.responses[0].data: + rows = result.responses[0].data.get("value", []) + print(f"[OK] Retrieved {len(rows)} accounts") + for row in rows: + print(f" {row.get('name')}") + + # --------------------------------------------------------------------------- + # Example 5: Mixed batch — changeset writes + standalone GETs + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 5: Mixed batch") + + # NOTE: Commented out because it requires a pre-existing account_id. + # Uncomment and set account_id to run this example. + # batch = client.batch.new() + # + # with batch.changeset() as cs: + # cs.records.update("account", account_id, {"statecode": 0}) + # + # batch.records.get("account", account_id, select=["name", "statecode"]) + # + # result = batch.execute() + # update_response = result.responses[0] + # account_data = result.responses[1] + # if account_data.is_success and account_data.data: + # print(f"Account: {account_data.data.get('name')}") + + # --------------------------------------------------------------------------- + # Example 6: Continue on error + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 6: Continue on error") + + batch = client.batch.new() + batch.records.get("account", "00000000-0000-0000-0000-000000000000") + batch.query.sql("SELECT TOP 1 name FROM account") + + result = batch.execute(continue_on_error=True) + print(f"[OK] Succeeded: {len(result.succeeded)}, Failed: {len(result.failed)}") + for item in result.failed: + print(f"[ERR] {item.status_code}: {item.error_message}") + + # --------------------------------------------------------------------------- + # Example 7: DataFrame integration + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 7: DataFrame batch operations") + + import pandas as pd + + # Create records from a DataFrame + df = pd.DataFrame( + [ + {"name": "DF-Batch-A", "telephone1": "555-0100"}, + {"name": "DF-Batch-B", "telephone1": "555-0200"}, + ] + ) + batch = client.batch.new() + batch.dataframe.create("account", df) + result = batch.execute() + print(f"[OK] DataFrame create: {len(result.succeeded)} succeeded") + created_ids = list(result.entity_ids) + + # Update records from a DataFrame + if len(created_ids) >= 2: + update_df = pd.DataFrame( + [ + {"accountid": created_ids[0], "telephone1": "555-9990"}, + {"accountid": created_ids[1], "telephone1": "555-9991"}, + ] + ) + batch = client.batch.new() + batch.dataframe.update("account", update_df, id_column="accountid") + result = batch.execute() + print(f"[OK] DataFrame update: {len(result.succeeded)} succeeded") + + # Delete records from a Series + if created_ids: + batch = client.batch.new() + batch.dataframe.delete("account", pd.Series(created_ids), use_bulk_delete=False) + result = batch.execute() + print(f"[OK] DataFrame delete: {len(result.succeeded)} succeeded") + + # --------------------------------------------------------------------------- + # Example 8: Understanding response data patterns + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 8: Response data patterns") + + # Every batch result maps 1:1 with the operations you added. + # Different operations return different response shapes: + + batch = client.batch.new() + # Op 0: single create -> 204 No Content, entity_id in OData-EntityId header + batch.records.create("account", {"name": "Pattern-Demo"}) + # Op 1: bulk create -> 200 OK, IDs in body as {"Ids": [...]} + batch.records.create("account", [{"name": "Bulk-A"}, {"name": "Bulk-B"}]) + # Op 2: SQL query -> 200 OK, rows in body as {"value": [...]} + batch.query.sql("SELECT TOP 3 name FROM account") + + result = batch.execute() + + for i, resp in enumerate(result.responses): + if not resp.is_success: + print(f" Op {i}: [FAIL] {resp.status_code}: {resp.error_message}") + continue + + # Single create: entity_id from OData-EntityId header + if resp.entity_id: + print(f" Op {i}: [CREATE] entity_id={resp.entity_id}") + + # Bulk action (CreateMultiple/UpsertMultiple): IDs in body + elif resp.data and "Ids" in resp.data: + print(f" Op {i}: [BULK] {len(resp.data['Ids'])} IDs: {resp.data['Ids']}") + + # Query: rows in body + elif resp.data and "value" in resp.data: + print(f" Op {i}: [QUERY] {len(resp.data['value'])} rows") + + # Delete or metadata operation: 204, no data + else: + print(f" Op {i}: [OK] {resp.status_code}") + + # Clean up demo records + for rid in result.entity_ids: + client.records.delete("account", rid) + for resp in result.succeeded: + if resp.data and "Ids" in resp.data: + for rid in resp.data["Ids"]: + client.records.delete("account", rid) diff --git a/examples/advanced/walkthrough.py b/examples/advanced/walkthrough.py index ef633d00..52c4d7a3 100644 --- a/examples/advanced/walkthrough.py +++ b/examples/advanced/walkthrough.py @@ -11,6 +11,7 @@ - Expand (navigation properties) with QueryBuilder - Picklist label-to-value conversion - Column management +- Batch operations (create, read, update, changeset, delete in one HTTP request) - Cleanup Prerequisites: @@ -479,10 +480,88 @@ def _run_walkthrough(client): print(f" (Deleting {len(paging_ids)} paging demo records)") # ============================================================================ - # 13. CLEANUP + # 13. BATCH OPERATIONS # ============================================================================ print("\n" + "=" * 80) - print("13. Cleanup") + print("13. Batch Operations") + print("=" * 80) + + # Batch create: send 2 creates in a single POST $batch + log_call("client.batch.new() + batch.records.create(...) x2 + batch.execute()") + batch = client.batch.new() + batch.records.create( + table_name, + { + "new_Title": "Batch task alpha", + "new_Quantity": 1, + "new_Amount": 25.0, + "new_Completed": False, + "new_Priority": Priority.LOW, + }, + ) + batch.records.create( + table_name, + { + "new_Title": "Batch task beta", + "new_Quantity": 2, + "new_Amount": 50.0, + "new_Completed": False, + "new_Priority": Priority.MEDIUM, + }, + ) + result = batch.execute() + batch_ids = list(result.entity_ids) + print( + f"[OK] Batch create: {len(result.succeeded)} operations in one HTTP request, {len(batch_ids)} records created" + ) + + # Batch get: read both records in a single request + log_call("client.batch.new() + batch.records.get(...) x2 + batch.execute()") + batch = client.batch.new() + for bid in batch_ids: + batch.records.get(table_name, bid, select=["new_title", "new_quantity"]) + result = batch.execute() + print(f"[OK] Batch get: {len(result.succeeded)} reads in one HTTP request") + for resp in result.succeeded: + if resp.data: + print(f" new_title='{resp.data.get('new_title')}', new_quantity={resp.data.get('new_quantity')}") + + # Changeset: create + update atomically (all-or-nothing) + log_call("with batch.changeset() as cs: cs.records.create(...); cs.records.update(cs_ref, ...)") + batch = client.batch.new() + with batch.changeset() as cs: + cs_ref = cs.records.create( + table_name, + { + "new_Title": "Changeset task", + "new_Quantity": 5, + "new_Amount": 100.0, + "new_Completed": False, + "new_Priority": Priority.HIGH, + }, + ) + cs.records.update(table_name, cs_ref, {"new_Completed": True}) + result = batch.execute() + if not result.has_errors: + batch_ids.extend(result.entity_ids) + print(f"[OK] Changeset: {len(result.succeeded)} operations committed atomically") + else: + for item in result.failed: + print(f"[WARN] Changeset error {item.status_code}: {item.error_message}") + + # Batch delete: clean up all batch-created records in one request + log_call(f"client.batch.new() + batch.records.delete(...) x{len(batch_ids)} + batch.execute()") + batch = client.batch.new() + for bid in batch_ids: + batch.records.delete(table_name, bid) + result = batch.execute(continue_on_error=True) + print(f"[OK] Batch delete: {len(result.succeeded)} records deleted in one HTTP request") + + # ============================================================================ + # 14. CLEANUP + # ============================================================================ + print("\n" + "=" * 80) + print("14. Cleanup") print("=" * 80) log_call(f"client.tables.delete('{table_name}')") @@ -519,6 +598,7 @@ def _run_walkthrough(client): print(" [OK] Picklist label-to-value conversion") print(" [OK] Column management") print(" [OK] Single and bulk delete operations") + print(" [OK] Batch operations (create, read, changeset, delete)") print(" [OK] Table cleanup") print("=" * 80) diff --git a/examples/basic/functional_testing.py b/examples/basic/functional_testing.py index 63f65aa6..1ea0d5f0 100644 --- a/examples/basic/functional_testing.py +++ b/examples/basic/functional_testing.py @@ -10,6 +10,7 @@ - Table creation and metadata operations - Full CRUD operations testing - Query functionality validation +- Batch operations (create, read, update, changeset, delete) - Interactive cleanup options Prerequisites: @@ -43,6 +44,7 @@ CASCADE_BEHAVIOR_NO_CASCADE, CASCADE_BEHAVIOR_REMOVE_LINK, ) +from PowerPlatform.Dataverse.models.upsert import UpsertItem from azure.identity import InteractiveBrowserCredential @@ -320,6 +322,473 @@ def test_query_records(client: DataverseClient, table_info: Dict[str, Any]) -> N print(" This might be expected if the table is very new.") +def test_sql_encoding( + client: DataverseClient, + table_info: Dict[str, Any], + retrieved_record: Dict[str, Any], +) -> None: + """Verify SQL encoding parity between client.query.sql() and batch.query.sql(). + + The direct path (client.query.sql) delegates to _build_sql which encodes the + SQL via urllib.parse.quote(safe=''), producing %20 for spaces. The batch path + uses the same _build_sql method, so both should behave identically. + + Specifically tests SQL containing: + - Spaces in a WHERE string literal (requires %20 encoding) + - Colons in a WHERE string literal (the HH:MM:SS timestamp in the name) + + Both paths are run against the same SQL and their results are compared + to confirm the encoding produces matching Dataverse responses. + """ + print("\n-> SQL Encoding Verification Test") + print("=" * 50) + + table_schema_name = table_info.get("table_schema_name") + logical_name = table_info.get("table_logical_name", table_schema_name.lower()) + attr_prefix = table_schema_name.split("_", 1)[0] if "_" in table_schema_name else table_schema_name + name_col = f"{attr_prefix}_name" + known_name = retrieved_record.get(name_col, "") + + try: + # ------------------------------------------------------------------ + # Case 1: Basic SELECT — no special characters in WHERE clause. + # Baseline: confirms the path works before adding complexity. + # ------------------------------------------------------------------ + basic_sql = f"SELECT TOP 5 {name_col} FROM {logical_name}" + print(f" [1/3] Basic SELECT (no special chars): {basic_sql}") + + direct_rows = client.query.sql(basic_sql) + direct_count = len(direct_rows) + + batch = client.batch.new() + batch.query.sql(basic_sql) + result = batch.execute() + batch_count = ( + len(result.responses[0].data.get("value", [])) + if result.responses and result.responses[0].is_success and result.responses[0].data + else 0 + ) + + assert direct_count == batch_count, f"Row count mismatch: client={direct_count}, batch={batch_count}" + print(f" [OK] Both paths returned {direct_count} rows") + + # ------------------------------------------------------------------ + # Case 2: WHERE clause with spaces and colons in the string literal. + # This is the critical case: the record name is of the form + # "Test Record HH:MM:SS" which contains spaces (-> %20) and + # colons. If encoding differs between direct and batch, only + # one path would find the record. + # ------------------------------------------------------------------ + if known_name: + escaped_name = known_name.replace("'", "''") + where_sql = f"SELECT TOP 1 {name_col} FROM {logical_name} WHERE {name_col} = '{escaped_name}'" + print(f" [2/3] WHERE with spaces/colons: ...WHERE {name_col} = '{escaped_name}'") + + direct_rows_where = client.query.sql(where_sql) + direct_where_count = len(direct_rows_where) + + batch2 = client.batch.new() + batch2.query.sql(where_sql) + result2 = batch2.execute() + batch_where_count = ( + len(result2.responses[0].data.get("value", [])) + if result2.responses and result2.responses[0].is_success and result2.responses[0].data + else 0 + ) + + assert ( + direct_where_count == batch_where_count + ), f"Row count mismatch on WHERE query: client={direct_where_count}, batch={batch_where_count}" + assert direct_where_count == 1, f"Expected exactly 1 row for known record name, got {direct_where_count}" + direct_name = direct_rows_where[0].get(name_col) + assert direct_name == known_name, f"Returned name '{direct_name}' does not match expected '{known_name}'" + print(f" [OK] Both paths found the record: '{direct_name}'") + else: + print(" [2/3] Skipped WHERE test — record name not available in retrieved_record") + + # ------------------------------------------------------------------ + # Case 3: WHERE clause with an equals sign inside the string literal. + # Creates a temporary record whose name contains '=' (which + # must be percent-encoded as %3D in the query string), queries + # it via both paths, then deletes it. + # ------------------------------------------------------------------ + print(" [3/3] WHERE with '=' in string literal (tests %3D encoding)") + equals_name = f"SQL=Test {datetime.now().strftime('%H:%M:%S')}" + eq_id = client.records.create(table_schema_name, {name_col: equals_name}) + try: + escaped_eq = equals_name.replace("'", "''") + eq_sql = f"SELECT TOP 1 {name_col} FROM {logical_name} WHERE {name_col} = '{escaped_eq}'" + + direct_eq_rows = client.query.sql(eq_sql) + direct_eq_count = len(direct_eq_rows) + + batch3 = client.batch.new() + batch3.query.sql(eq_sql) + result3 = batch3.execute() + batch_eq_count = ( + len(result3.responses[0].data.get("value", [])) + if result3.responses and result3.responses[0].is_success and result3.responses[0].data + else 0 + ) + + assert ( + direct_eq_count == batch_eq_count + ), f"Row count mismatch on '=' query: client={direct_eq_count}, batch={batch_eq_count}" + assert direct_eq_count == 1, f"Expected 1 row for '=' record, got {direct_eq_count}" + print(f" [OK] Both paths found record with '=' in name: '{direct_eq_rows[0].get(name_col)}'") + finally: + client.records.delete(table_schema_name, eq_id) + + print("[OK] SQL encoding verification passed — %20/%3D encoding is consistent across both paths") + + except AssertionError as e: + print(f"[ERR] Encoding parity assertion failed: {e}") + raise + except Exception as e: + print(f"[WARN] SQL encoding test encountered an issue: {e}") + print(" Check that the test table exists and has at least one record.") + + +def test_batch_all_operations(client: DataverseClient, table_info: Dict[str, Any]) -> None: + """Test every available batch operation type in a structured sequence. + + Operations covered: + records.create (single + CreateMultiple) + records.get (single by ID) + records.update (single PATCH + UpdateMultiple) + records.delete (multi, use_bulk_delete=False) + records.upsert (graceful — requires configured alternate key) + tables.get, tables.list + tables.add_columns + tables.remove_columns (two requests, each adding + one column, verified then removed in a second batch) + query.sql + changeset happy path (create + update via content-ID ref + delete) + changeset rollback (failing op rolls back entire changeset) + two changesets in one batch (Content-IDs are globally unique across + the batch via a shared counter) + content-ID reference chaining ($n refs) across multiple creates in one + changeset — regression guard for the shared counter fix + execute(continue_on_error=True) — mixed success/failure + """ + print("\n-> Batch Operations Test (All Operations)") + print("=" * 50) + + table_schema_name = table_info.get("table_schema_name") + logical_name = table_info.get("table_logical_name", table_schema_name.lower()) + attr_prefix = table_schema_name.split("_", 1)[0] if "_" in table_schema_name else table_schema_name + all_ids: list = [] + + try: + # ------------------------------------------------------------------- + # [1/11] CREATE — single record + CreateMultiple (list) in one batch + # ------------------------------------------------------------------- + print("\n[1/11] Create — single + CreateMultiple (2 ops, 1 POST $batch)") + batch = client.batch.new() + batch.records.create( + table_schema_name, + { + f"{attr_prefix}_name": f"Batch-A {datetime.now().strftime('%H:%M:%S')}", + f"{attr_prefix}_count": 1, + f"{attr_prefix}_is_active": True, + }, + ) + batch.records.create( + table_schema_name, + [ + { + f"{attr_prefix}_name": f"Batch-B {datetime.now().strftime('%H:%M:%S')}", + f"{attr_prefix}_count": 2, + f"{attr_prefix}_is_active": True, + }, + { + f"{attr_prefix}_name": f"Batch-C {datetime.now().strftime('%H:%M:%S')}", + f"{attr_prefix}_count": 3, + f"{attr_prefix}_is_active": True, + }, + ], + ) + result = batch.execute() + all_ids = list(result.entity_ids) + if result.has_errors: + for item in result.failed: + print(f"[WARN] {item.status_code}: {item.error_message}") + else: + print(f"[OK] {len(result.succeeded)} ops → {len(all_ids)} records created: {all_ids}") + + # ------------------------------------------------------------------- + # [2/11] READ — get by ID + tables.get + tables.list + query.sql + # All 4 reads in one batch request + # ------------------------------------------------------------------- + if all_ids: + print("\n[2/11] Read — records.get + tables.get + tables.list + query.sql (4 ops, 1 POST $batch)") + batch = client.batch.new() + batch.records.get( + table_schema_name, + all_ids[0], + select=[f"{attr_prefix}_name", f"{attr_prefix}_count"], + ) + batch.tables.get(table_schema_name) + batch.tables.list() + batch.query.sql(f"SELECT TOP 3 {attr_prefix}_name FROM {logical_name}") + result = batch.execute() + print(f"[OK] {len(result.succeeded)} succeeded, {len(result.failed)} failed") + for i, resp in enumerate(result.responses): + if not resp.is_success: + print(f" [{i}] FAILED {resp.status_code}: {resp.error_message}") + continue + if i == 0 and resp.data: + print( + f" records.get → name='{resp.data.get(f'{attr_prefix}_name')}', count={resp.data.get(f'{attr_prefix}_count')}" + ) + elif i == 1 and resp.data: + print( + f" tables.get → LogicalName='{resp.data.get('LogicalName')}', EntitySet='{resp.data.get('EntitySetName')}'" + ) + elif i == 2 and resp.data: + print(f" tables.list → {len(resp.data.get('value', []))} tables returned") + elif i == 3 and resp.data: + print(f" query.sql → {len(resp.data.get('value', []))} rows returned") + + # ------------------------------------------------------------------- + # [3/11] UPDATE — single PATCH + UpdateMultiple (broadcast) in one batch + # ------------------------------------------------------------------- + if len(all_ids) >= 3: + print(f"\n[3/11] Update — single PATCH + UpdateMultiple ({len(all_ids)} records, 2 ops, 1 POST $batch)") + batch = client.batch.new() + batch.records.update(table_schema_name, all_ids[0], {f"{attr_prefix}_count": 10}) + batch.records.update(table_schema_name, all_ids[1:], {f"{attr_prefix}_count": 20}) + result = batch.execute() + print(f"[OK] {len(result.succeeded)} updates succeeded, {len(result.failed)} failed") + + # ------------------------------------------------------------------- + # [4/11] CHANGESET (happy path) — create + update via content-ID + delete + # All three changeset operation types committed atomically + # ------------------------------------------------------------------- + if len(all_ids) >= 1: + print("\n[4/11] Changeset (happy path) — cs.create + cs.update(ref) + cs.delete (1 transaction)") + batch = client.batch.new() + with batch.changeset() as cs: + ref = cs.records.create( + table_schema_name, + { + f"{attr_prefix}_name": f"Batch-D {datetime.now().strftime('%H:%M:%S')}", + f"{attr_prefix}_count": 4, + f"{attr_prefix}_is_active": False, + }, + ) + cs.records.update(table_schema_name, ref, {f"{attr_prefix}_is_active": True}) + cs.records.delete(table_schema_name, all_ids[-1]) + result = batch.execute() + if result.has_errors: + for item in result.failed: + print(f"[WARN] Changeset error {item.status_code}: {item.error_message}") + else: + new_id = next(iter(result.entity_ids), None) + if new_id: + all_ids[-1] = new_id # replace deleted id with the new one + print(f"[OK] {len(result.succeeded)} ops committed atomically (create + update + delete)") + + # ------------------------------------------------------------------- + # [5/11] CHANGESET (rollback) — failing update rolls back the create + # ------------------------------------------------------------------- + print("\n[5/11] Changeset (rollback) — cs.create + cs.update(nonexistent) → full rollback") + nonexistent_id = "00000000-0000-0000-0000-000000000001" + batch = client.batch.new() + with batch.changeset() as cs: + cs.records.create( + table_schema_name, + { + f"{attr_prefix}_name": f"Rollback-test {datetime.now().strftime('%H:%M:%S')}", + f"{attr_prefix}_count": 0, + f"{attr_prefix}_is_active": False, + }, + ) + cs.records.update(table_schema_name, nonexistent_id, {f"{attr_prefix}_count": 999}) + # continue_on_error=True ensures Dataverse returns a 200 multipart response + # with the changeset failure embedded, rather than propagating the inner + # 404 to the outer batch HTTP status (which some environments do). + result = batch.execute(continue_on_error=True) + if result.has_errors: + leaked = list(result.entity_ids) + if not leaked: + print("[OK] Changeset rollback verified: changeset failed, no records created") + else: + print(f"[WARN] Changeset failed but {len(leaked)} IDs leaked — queuing for cleanup") + all_ids.extend(leaked) + else: + print("[WARN] Expected rollback but changeset succeeded (unexpected)") + all_ids.extend(result.entity_ids) + + # ------------------------------------------------------------------- + # [6/11] TWO CHANGESETS — Content-IDs are unique across the entire batch + # (shared counter). Verifies both changesets commit atomically. + # ------------------------------------------------------------------- + print("\n[6/11] Two changesets in one batch — globally unique Content-IDs across changesets") + batch = client.batch.new() + with batch.changeset() as cs1: + ref1 = cs1.records.create( + table_schema_name, + { + f"{attr_prefix}_name": f"CS1-E {datetime.now().strftime('%H:%M:%S')}", + f"{attr_prefix}_count": 10, + f"{attr_prefix}_is_active": False, + }, + ) + cs1.records.update(table_schema_name, ref1, {f"{attr_prefix}_is_active": True}) + with batch.changeset() as cs2: + ref2 = cs2.records.create( + table_schema_name, + { + f"{attr_prefix}_name": f"CS2-F {datetime.now().strftime('%H:%M:%S')}", + f"{attr_prefix}_count": 20, + f"{attr_prefix}_is_active": False, + }, + ) + cs2.records.update(table_schema_name, ref2, {f"{attr_prefix}_is_active": True}) + result = batch.execute() + if result.has_errors: + for item in result.failed: + print(f"[WARN] Two-changeset error {item.status_code}: {item.error_message}") + else: + cs_ids = list(result.entity_ids) + all_ids.extend(cs_ids) + print( + f"[OK] Both changesets committed — {len(cs_ids)} records created " + f"with globally unique Content-IDs across changesets: {cs_ids}" + ) + + # ------------------------------------------------------------------- + # [7/11] CONTENT-ID REFERENCE CHAINING — two creates in one changeset, + # each update references its own $n — regression guard for the + # shared-counter fix (ensures references stay self-consistent). + # ------------------------------------------------------------------- + print("\n[7/11] Content-ID reference chaining — two creates + two updates via $n refs") + batch = client.batch.new() + with batch.changeset() as cs: + ref_a = cs.records.create( + table_schema_name, + { + f"{attr_prefix}_name": f"Chain-A {datetime.now().strftime('%H:%M:%S')}", + f"{attr_prefix}_count": 0, + f"{attr_prefix}_is_active": False, + }, + ) + ref_b = cs.records.create( + table_schema_name, + { + f"{attr_prefix}_name": f"Chain-B {datetime.now().strftime('%H:%M:%S')}", + f"{attr_prefix}_count": 0, + f"{attr_prefix}_is_active": False, + }, + ) + # Update both records via their content-ID references + cs.records.update(table_schema_name, ref_a, {f"{attr_prefix}_count": 100}) + cs.records.update(table_schema_name, ref_b, {f"{attr_prefix}_count": 200}) + result = batch.execute() + if result.has_errors: + for item in result.failed: + print(f"[WARN] Chaining error {item.status_code}: {item.error_message}") + else: + chain_ids = list(result.entity_ids) + all_ids.extend(chain_ids) + print(f"[OK] Both records created and updated via content-ID refs " f"{ref_a} and {ref_b}: {chain_ids}") + + # ------------------------------------------------------------------- + # [8/11] BATCH TABLES ADD COLUMNS — two batch.tables.add_columns() + # requests in one batch, each adding one column. Verifies + # that metadata write operations work inside a $batch request. + # The two columns are removed via a follow-up batch after the + # assertion so they do not accumulate on the test table. + # ------------------------------------------------------------------- + col_a = f"{attr_prefix}_batch_extra_a" + col_b = f"{attr_prefix}_batch_extra_b" + print(f"\n[8/11] Batch tables.add_columns — two add-column requests in one batch") + batch = client.batch.new() + batch.tables.add_columns(table_schema_name, {col_a: "string"}) + batch.tables.add_columns(table_schema_name, {col_b: "int"}) + result = batch.execute() + if result.has_errors: + for item in result.failed: + print(f"[WARN] add_columns error {item.status_code}: {item.error_message}") + else: + print(f"[OK] {len(result.succeeded)} column(s) added via batch: {col_a}, {col_b}") + # Remove the two test columns so the table stays clean + batch_rm = client.batch.new() + batch_rm.tables.remove_columns(table_schema_name, [col_a, col_b]) + rm_result = batch_rm.execute(continue_on_error=True) + print(f"[OK] Removed {len(rm_result.succeeded)} batch-added column(s) via batch.tables.remove_columns") + + # ------------------------------------------------------------------- + # [9/11] UPSERT — requires an alternate key configured on the table. + # The test table has none, so this is expected to fail (graceful). + # ------------------------------------------------------------------- + print(f"\n[9/11] Upsert — UpsertItem with alternate key (expected to fail: no alt key on test table)") + try: + batch = client.batch.new() + batch.records.upsert( + table_schema_name, + [ + UpsertItem( + alternate_key={f"{attr_prefix}_name": f"Upsert-E {datetime.now().strftime('%H:%M:%S')}"}, + record={f"{attr_prefix}_count": 5, f"{attr_prefix}_is_active": True}, + ) + ], + ) + result = batch.execute() + if result.has_errors: + print(f"[WARN] Upsert failed as expected (no alternate key configured): {result.failed[0].status_code}") + else: + upsert_ids = list(result.entity_ids) + all_ids.extend(upsert_ids) + print(f"[OK] Upsert succeeded: {len(upsert_ids)} record(s) — alternate key was accepted") + except Exception as e: + print(f"[WARN] Upsert skipped due to exception: {e}") + + # ------------------------------------------------------------------- + # [10/11] MIXED BATCH with continue_on_error + # One intentional 404 alongside a valid get — both attempted + # ------------------------------------------------------------------- + if all_ids: + print(f"\n[10/11] Mixed batch (continue_on_error=True) — 1 bad get + 1 good get") + batch = client.batch.new() + batch.records.get( + table_schema_name, + "00000000-0000-0000-0000-000000000002", + select=[f"{attr_prefix}_name"], + ) + batch.records.get( + table_schema_name, + all_ids[0], + select=[f"{attr_prefix}_name"], + ) + result = batch.execute(continue_on_error=True) + print(f"[OK] Succeeded: {len(result.succeeded)}, Failed: {len(result.failed)}") + for item in result.failed: + print(f" Expected failure: {item.status_code} {item.error_message}") + + # ------------------------------------------------------------------- + # [11/11] DELETE — multi-delete (use_bulk_delete=False → individual DELETEs) + # ------------------------------------------------------------------- + if all_ids: + print(f"\n[11/11] Delete — {len(all_ids)} records via multi-delete (use_bulk_delete=False, 1 POST $batch)") + batch = client.batch.new() + batch.records.delete(table_schema_name, all_ids, use_bulk_delete=False) + result = batch.execute(continue_on_error=True) + print(f"[OK] Deleted {len(result.succeeded)}, failed {len(result.failed)}") + + print("\n[OK] Batch all-operations test completed!") + + except Exception as e: + print(f"[WARN] Batch all-operations test encountered an issue: {e}") + if all_ids: + try: + batch = client.batch.new() + batch.records.delete(table_schema_name, all_ids, use_bulk_delete=False) + batch.execute(continue_on_error=True) + except Exception: + pass + + def cleanup_test_data(client: DataverseClient, table_info: Dict[str, Any], record_id: str) -> None: """Clean up test data.""" print("\n-> Cleanup") @@ -683,6 +1152,7 @@ def main(): print(" - Record CRUD Operations") print(" - Query Functionality") print(" - Relationship Operations (1:N, N:N, lookup, get, delete)") + print(" - Batch Operations (create, read, update, changeset, delete)") print(" - Interactive Cleanup") print("=" * 70) print("For installation validation, run examples/basic/installation_example.py first") @@ -705,6 +1175,12 @@ def main(): # Test relationships test_relationships(client) + # Verify SQL encoding parity between direct and batch paths + test_sql_encoding(client, table_info, retrieved_record) + + # Test batch operations (all operation types) + test_batch_all_operations(client, table_info) + # Success summary print("\nFunctional Test Summary") print("=" * 50) @@ -714,6 +1190,8 @@ def main(): print("[OK] Record Reading: Success") print("[OK] Record Querying: Success") print("[OK] Relationship Operations: Success") + print("[OK] SQL Encoding: Success") + print("[OK] Batch Operations: Success") print("\nYour PowerPlatform Dataverse Client SDK is fully functional!") # Cleanup diff --git a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-dev/SKILL.md b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-dev/SKILL.md index f26efbb7..79d4e342 100644 --- a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-dev/SKILL.md +++ b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-dev/SKILL.md @@ -13,7 +13,7 @@ This skill provides guidance for developers working on the PowerPlatform Dataver ### API Design -1. **Public methods in operation namespaces** - New public methods go in the appropriate namespace module under `src/PowerPlatform/Dataverse/operations/` (`records.py`, `query.py`, `tables.py`). The `client.py` file exposes these via namespace properties (`client.records`, `client.query`, `client.tables`). Public types and constants live in their own modules (e.g., `models/metadata.py`, `common/constants.py`) +1. **Public methods in operation namespaces** - New public methods go in the appropriate namespace module under `src/PowerPlatform/Dataverse/operations/` (`records.py`, `query.py`, `tables.py`, `batch.py`). The `client.py` file exposes these via namespace properties (`client.records`, `client.query`, `client.tables`, `client.batch`). Public types and constants live in their own modules (e.g., `models/metadata.py`, `models/batch.py`, `common/constants.py`) 2. **Every public method needs README example** - Public API methods must have examples in README.md 3. **Reuse existing APIs** - Always check if an existing method can be used before making direct Web API calls 4. **Update documentation** when adding features - Keep README and SKILL files (both copies) in sync diff --git a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md index 9edb733f..569caec0 100644 --- a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md +++ b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md @@ -22,6 +22,7 @@ Use the PowerPlatform Dataverse Client Python SDK to interact with Microsoft Dat - `client.query` -- query and search operations - `client.tables` -- table metadata, columns, and relationships - `client.files` -- file upload operations +- `client.batch` -- batch multiple operations into a single HTTP request ### Bulk Operations The SDK supports Dataverse's native bulk operations: Pass lists to `create()`, `update()` for automatic bulk processing, for `delete()`, set `use_bulk_delete` when passing lists to use bulk operation @@ -369,6 +370,50 @@ client.files.upload( ) ``` +### Batch Operations + +Use `client.batch` to send multiple operations in one HTTP request. All batch methods return `None`; results arrive via `BatchResult` after `execute()`. + +```python +# Build a batch request +batch = client.batch.new() +batch.records.create("account", {"name": "Contoso"}) +batch.records.update("account", account_id, {"telephone1": "555-0100"}) +batch.records.get("account", account_id, select=["name"]) +batch.query.sql("SELECT TOP 5 name FROM account") + +result = batch.execute() +for item in result.responses: + if item.is_success: + print(f"[OK] {item.status_code} entity_id={item.entity_id}") + if item.data: + # GET responses populate item.data with the parsed JSON record + print(item.data.get("name")) + else: + print(f"[ERR] {item.status_code}: {item.error_message}") + +# Transactional changeset (all succeed or roll back) +with batch.changeset() as cs: + ref = cs.records.create("contact", {"firstname": "Alice"}) + cs.records.update("account", account_id, {"primarycontactid@odata.bind": ref}) + +# Continue on error +result = batch.execute(continue_on_error=True) +print(f"Succeeded: {len(result.succeeded)}, Failed: {len(result.failed)}") +``` + +**BatchResult properties:** +- `result.responses` -- list of `BatchItemResponse` in submission order +- `result.succeeded` -- responses with 2xx status codes +- `result.failed` -- responses with non-2xx status codes +- `result.has_errors` -- True if any response failed +- `result.entity_ids` -- GUIDs from OData-EntityId headers (creates and updates) + +**Batch limitations:** +- Maximum 1000 operations per batch +- Paginated `records.get()` (without `record_id`) is not supported in batch +- `flush_cache()` is not supported in batch + ## Error Handling The SDK provides structured exceptions with detailed error information: diff --git a/src/PowerPlatform/Dataverse/client.py b/src/PowerPlatform/Dataverse/client.py index e0f9c6e9..ea9dd6b8 100644 --- a/src/PowerPlatform/Dataverse/client.py +++ b/src/PowerPlatform/Dataverse/client.py @@ -19,6 +19,7 @@ from .operations.query import QueryOperations from .operations.files import FileOperations from .operations.tables import TableOperations +from .operations.batch import BatchOperations class DataverseClient: @@ -62,6 +63,7 @@ class DataverseClient: - ``client.tables`` -- table and column metadata management - ``client.files`` -- file upload operations - ``client.dataframe`` -- pandas DataFrame wrappers for record CRUD + - ``client.batch`` -- batch multiple operations into a single HTTP request The client supports Python's context manager protocol for automatic resource cleanup and HTTP connection pooling: @@ -109,6 +111,7 @@ def __init__( self.tables = TableOperations(self) self.files = FileOperations(self) self.dataframe = DataFrameOperations(self) + self.batch = BatchOperations(self) def _get_odata(self) -> _ODataClient: """ diff --git a/src/PowerPlatform/Dataverse/data/_batch.py b/src/PowerPlatform/Dataverse/data/_batch.py new file mode 100644 index 00000000..bd110da5 --- /dev/null +++ b/src/PowerPlatform/Dataverse/data/_batch.py @@ -0,0 +1,695 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Internal batch intent dataclasses, raw-request builder, and multipart serializer.""" + +from __future__ import annotations + +import json +import re +import uuid +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union + +from ..core.errors import HttpError, MetadataError, ValidationError +from ..core._error_codes import METADATA_TABLE_NOT_FOUND, METADATA_COLUMN_NOT_FOUND, _http_subcode +from ..models.batch import BatchItemResponse, BatchResult +from ..models.relationship import ( + LookupAttributeMetadata, + OneToManyRelationshipMetadata, + ManyToManyRelationshipMetadata, +) +from ..models.upsert import UpsertItem +from ..common.constants import CASCADE_BEHAVIOR_REMOVE_LINK +from ._raw_request import _RawRequest +from ._odata import _GUID_RE + +if TYPE_CHECKING: + from ._odata import _ODataClient + +__all__ = [] + +_CRLF = "\r\n" +_MAX_BATCH_SIZE = 1000 + + +# --------------------------------------------------------------------------- +# Intent dataclasses — one per supported operation type +# (stored at batch-build time; resolved to _RawRequest at execute() time) +# --------------------------------------------------------------------------- + +# --- Record intent types --- + + +@dataclass +class _RecordCreate: + table: str + data: Union[Dict[str, Any], List[Dict[str, Any]]] + content_id: Optional[int] = None # set only for changeset items + + +@dataclass +class _RecordUpdate: + table: str + ids: Union[str, List[str]] + changes: Union[Dict[str, Any], List[Dict[str, Any]]] + content_id: Optional[int] = None # set only for changeset single-record updates + + +@dataclass +class _RecordDelete: + table: str + ids: Union[str, List[str]] + use_bulk_delete: bool = True + content_id: Optional[int] = None # set only for changeset single-record deletes + + +@dataclass +class _RecordGet: + table: str + record_id: str + select: Optional[List[str]] = None + + +@dataclass +class _RecordUpsert: + table: str + items: List[UpsertItem] # always non-empty; normalised by BatchRecordOperations + + +# --- Table intent types --- + + +@dataclass +class _TableCreate: + table: str + columns: Dict[str, Any] + solution: Optional[str] = None + primary_column: Optional[str] = None + + +@dataclass +class _TableDelete: + table: str + + +@dataclass +class _TableGet: + table: str + + +@dataclass +class _TableList: + filter: Optional[str] = None + select: Optional[List[str]] = None + + +@dataclass +class _TableAddColumns: + table: str + columns: Dict[str, Any] + + +@dataclass +class _TableRemoveColumns: + table: str + columns: Union[str, List[str]] + + +@dataclass +class _TableCreateOneToMany: + lookup: LookupAttributeMetadata + relationship: OneToManyRelationshipMetadata + solution: Optional[str] = None + + +@dataclass +class _TableCreateManyToMany: + relationship: ManyToManyRelationshipMetadata + solution: Optional[str] = None + + +@dataclass +class _TableDeleteRelationship: + relationship_id: str + + +@dataclass +class _TableGetRelationship: + schema_name: str + + +@dataclass +class _TableCreateLookupField: + referencing_table: str + lookup_field_name: str + referenced_table: str + display_name: Optional[str] = None + description: Optional[str] = None + required: bool = False + cascade_delete: str = CASCADE_BEHAVIOR_REMOVE_LINK + solution: Optional[str] = None + language_code: int = 1033 + + +# --- Query intent types --- + + +@dataclass +class _QuerySql: + sql: str + + +# --------------------------------------------------------------------------- +# Changeset container +# --------------------------------------------------------------------------- + + +@dataclass +class _ChangeSet: + """Ordered group of single-record write operations that execute atomically. + + Content-IDs are allocated from ``_counter``, a single-element ``List[int]`` + that is shared across all changesets in the same batch. Passing the same + list object to every ``_ChangeSet`` created by a :class:`BatchRequest` + ensures Content-ID values are unique within the entire batch request, not + just within an individual changeset, as required by the OData spec. + + When constructed in isolation (e.g. in unit tests), ``_counter`` defaults + to a fresh ``[1]`` so the class remains self-contained. + """ + + operations: List[Union[_RecordCreate, _RecordUpdate, _RecordDelete]] = field(default_factory=list) + _counter: List[int] = field(default_factory=lambda: [1], repr=False) + + def add_create(self, table: str, data: Dict[str, Any]) -> str: + """Add a single-record create; return its content-ID reference string.""" + cid = self._counter[0] + self._counter[0] += 1 + self.operations.append(_RecordCreate(table=table, data=data, content_id=cid)) + return f"${cid}" + + def add_update(self, table: str, record_id: str, changes: Dict[str, Any]) -> None: + """Add a single-record update (record_id may be a '$n' reference).""" + cid = self._counter[0] + self._counter[0] += 1 + self.operations.append(_RecordUpdate(table=table, ids=record_id, changes=changes, content_id=cid)) + + def add_delete(self, table: str, record_id: str) -> None: + """Add a single-record delete (record_id may be a '$n' reference).""" + cid = self._counter[0] + self._counter[0] += 1 + self.operations.append(_RecordDelete(table=table, ids=record_id, content_id=cid)) + + +# --------------------------------------------------------------------------- +# Changeset batch item +# (_RawRequest is imported from ._raw_request — defined there so _odata.py +# can also import it without a circular dependency) +# --------------------------------------------------------------------------- + + +@dataclass +class _ChangeSetBatchItem: + """A resolved changeset — serialised as a nested multipart/mixed part.""" + + requests: List[_RawRequest] + + +# --------------------------------------------------------------------------- +# Batch client: resolves intents → raw requests → multipart body → HTTP → result +# --------------------------------------------------------------------------- + + +class _BatchClient: + """ + Serialises a list of intent objects into an OData ``$batch`` multipart/mixed + request, dispatches it, and parses the response. + + :param od: The active OData client (provides helpers and HTTP transport). + """ + + def __init__(self, od: "_ODataClient") -> None: + self._od = od + + # ------------------------------------------------------------------ + # Public entry point + # ------------------------------------------------------------------ + + def execute( + self, + items: List[Any], + continue_on_error: bool = False, + ) -> BatchResult: + """ + Resolve all intent objects, build the batch body, send it, and return results. + + Metadata pre-resolution (GET calls for MetadataId) happens here, synchronously, + before the multipart body is assembled. + """ + if not items: + return BatchResult() + + resolved = self._resolve_all(items) + + total = sum(len(r.requests) if isinstance(r, _ChangeSetBatchItem) else 1 for r in resolved) + if total > _MAX_BATCH_SIZE: + raise ValidationError( + f"Batch contains {total} operations, which exceeds the limit of " + f"{_MAX_BATCH_SIZE}. Split into multiple batches.", + subcode="batch_size_exceeded", + details={"count": total, "max": _MAX_BATCH_SIZE}, + ) + + batch_boundary = f"batch_{uuid.uuid4()}" + body = self._build_batch_body(resolved, batch_boundary) + + headers: Dict[str, str] = { + "Content-Type": f'multipart/mixed; boundary="{batch_boundary}"', + } + if continue_on_error: + headers["Prefer"] = "odata.continue-on-error" + + url = f"{self._od.api}/$batch" + response = self._od._request( + "post", + url, + data=body.encode("utf-8"), + headers=headers, + # 400 is expected: Dataverse returns 400 for top-level batch + # errors (e.g. malformed body). We parse the response body to + # surface the service error via _parse_batch_response / + # _raise_top_level_batch_error rather than letting _request raise. + expected=(200, 202, 207, 400), + ) + return self._parse_batch_response(response) + + # ------------------------------------------------------------------ + # Intent resolution dispatcher + # ------------------------------------------------------------------ + + def _resolve_all(self, items: List[Any]) -> List[Union[_RawRequest, _ChangeSetBatchItem]]: + result: List[Union[_RawRequest, _ChangeSetBatchItem]] = [] + for item in items: + if isinstance(item, _ChangeSet): + if not item.operations: + # Empty changeset — nothing to send; skip silently. + continue + cs_requests = [self._resolve_one(op) for op in item.operations] + result.append(_ChangeSetBatchItem(requests=cs_requests)) + else: + result.extend(self._resolve_item(item)) + return result + + def _resolve_item(self, item: Any) -> List[_RawRequest]: + """Resolve a single intent to one or more _RawRequest objects.""" + if isinstance(item, _RecordCreate): + return self._resolve_record_create(item) + if isinstance(item, _RecordUpdate): + return self._resolve_record_update(item) + if isinstance(item, _RecordDelete): + return self._resolve_record_delete(item) + if isinstance(item, _RecordGet): + return self._resolve_record_get(item) + if isinstance(item, _RecordUpsert): + return self._resolve_record_upsert(item) + if isinstance(item, _TableCreate): + return self._resolve_table_create(item) + if isinstance(item, _TableDelete): + return self._resolve_table_delete(item) + if isinstance(item, _TableGet): + return self._resolve_table_get(item) + if isinstance(item, _TableList): + return self._resolve_table_list(item) + if isinstance(item, _TableAddColumns): + return self._resolve_table_add_columns(item) + if isinstance(item, _TableRemoveColumns): + return self._resolve_table_remove_columns(item) + if isinstance(item, _TableCreateOneToMany): + return self._resolve_table_create_one_to_many(item) + if isinstance(item, _TableCreateManyToMany): + return self._resolve_table_create_many_to_many(item) + if isinstance(item, _TableDeleteRelationship): + return self._resolve_table_delete_relationship(item) + if isinstance(item, _TableGetRelationship): + return self._resolve_table_get_relationship(item) + if isinstance(item, _TableCreateLookupField): + return self._resolve_table_create_lookup_field(item) + if isinstance(item, _QuerySql): + return self._resolve_query_sql(item) + raise ValidationError( + f"Unknown batch item type: {type(item).__name__}", + subcode="unknown_batch_item", + ) + + def _resolve_one(self, item: Any) -> _RawRequest: + """Resolve a changeset operation to exactly one _RawRequest.""" + resolved = self._resolve_item(item) + if len(resolved) != 1: + raise ValidationError( + "Changeset operations must each produce exactly one HTTP request.", + subcode="changeset_multi_request", + ) + return resolved[0] + + # ------------------------------------------------------------------ + # Record resolvers — delegate to _ODataClient._build_* methods + # ------------------------------------------------------------------ + + def _resolve_record_create(self, op: _RecordCreate) -> List[_RawRequest]: + entity_set = self._od._entity_set_from_schema_name(op.table) + if isinstance(op.data, dict): + return [self._od._build_create(entity_set, op.table, op.data, content_id=op.content_id)] + return [self._od._build_create_multiple(entity_set, op.table, op.data)] + + def _resolve_record_update(self, op: _RecordUpdate) -> List[_RawRequest]: + if isinstance(op.ids, str): + if not isinstance(op.changes, dict): + raise TypeError("For single id, changes must be a dict") + return [self._od._build_update(op.table, op.ids, op.changes, content_id=op.content_id)] + entity_set = self._od._entity_set_from_schema_name(op.table) + return [self._od._build_update_multiple(entity_set, op.table, op.ids, op.changes)] + + def _resolve_record_delete(self, op: _RecordDelete) -> List[_RawRequest]: + if isinstance(op.ids, str): + return [self._od._build_delete(op.table, op.ids, content_id=op.content_id)] + ids = [rid for rid in op.ids if rid] + if not ids: + return [] + if op.use_bulk_delete: + return [self._od._build_delete_multiple(op.table, ids)] + return [self._od._build_delete(op.table, rid) for rid in ids] + + def _resolve_record_get(self, op: _RecordGet) -> List[_RawRequest]: + return [self._od._build_get(op.table, op.record_id, select=op.select)] + + def _resolve_record_upsert(self, op: _RecordUpsert) -> List[_RawRequest]: + entity_set = self._od._entity_set_from_schema_name(op.table) + if len(op.items) == 1: + item = op.items[0] + return [self._od._build_upsert(entity_set, op.table, item.alternate_key, item.record)] + alternate_keys = [i.alternate_key for i in op.items] + records = [i.record for i in op.items] + return [self._od._build_upsert_multiple(entity_set, op.table, alternate_keys, records)] + + # ------------------------------------------------------------------ + # Table resolvers — delegate to _ODataClient._build_* methods + # (pre-resolution GETs for MetadataId remain here; they are batch- + # specific lookups needed before the relevant _build_* call) + # ------------------------------------------------------------------ + + def _require_entity_metadata(self, table: str) -> str: + """Look up MetadataId for *table*, raising MetadataError if not found.""" + ent = self._od._get_entity_by_table_schema_name(table) + if not ent or not ent.get("MetadataId"): + raise MetadataError( + f"Table '{table}' not found.", + subcode=METADATA_TABLE_NOT_FOUND, + ) + return ent["MetadataId"] + + def _resolve_table_create(self, op: _TableCreate) -> List[_RawRequest]: + return [self._od._build_create_entity(op.table, op.columns, op.solution, op.primary_column)] + + def _resolve_table_delete(self, op: _TableDelete) -> List[_RawRequest]: + metadata_id = self._require_entity_metadata(op.table) + return [self._od._build_delete_entity(metadata_id)] + + def _resolve_table_get(self, op: _TableGet) -> List[_RawRequest]: + return [self._od._build_get_entity(op.table)] + + def _resolve_table_list(self, op: _TableList) -> List[_RawRequest]: + return [self._od._build_list_entities(filter=op.filter, select=op.select)] + + def _resolve_table_add_columns(self, op: _TableAddColumns) -> List[_RawRequest]: + metadata_id = self._require_entity_metadata(op.table) + return [self._od._build_create_column(metadata_id, col_name, dtype) for col_name, dtype in op.columns.items()] + + def _resolve_table_remove_columns(self, op: _TableRemoveColumns) -> List[_RawRequest]: + columns = [op.columns] if isinstance(op.columns, str) else list(op.columns) + metadata_id = self._require_entity_metadata(op.table) + requests: List[_RawRequest] = [] + for col_name in columns: + attr_meta = self._od._get_attribute_metadata( + metadata_id, col_name, extra_select="@odata.type,AttributeType" + ) + if not attr_meta or not attr_meta.get("MetadataId"): + raise MetadataError( + f"Column '{col_name}' not found on table '{op.table}'.", + subcode=METADATA_COLUMN_NOT_FOUND, + ) + requests.append(self._od._build_delete_column(metadata_id, attr_meta["MetadataId"])) + return requests + + def _resolve_table_create_one_to_many(self, op: _TableCreateOneToMany) -> List[_RawRequest]: + body = op.relationship.to_dict() + body["Lookup"] = op.lookup.to_dict() + return [self._od._build_create_relationship(body, solution=op.solution)] + + def _resolve_table_create_many_to_many(self, op: _TableCreateManyToMany) -> List[_RawRequest]: + return [self._od._build_create_relationship(op.relationship.to_dict(), solution=op.solution)] + + def _resolve_table_delete_relationship(self, op: _TableDeleteRelationship) -> List[_RawRequest]: + return [self._od._build_delete_relationship(op.relationship_id)] + + def _resolve_table_get_relationship(self, op: _TableGetRelationship) -> List[_RawRequest]: + return [self._od._build_get_relationship(op.schema_name)] + + def _resolve_table_create_lookup_field(self, op: _TableCreateLookupField) -> List[_RawRequest]: + lookup, relationship = self._od._build_lookup_field_models( + referencing_table=op.referencing_table, + lookup_field_name=op.lookup_field_name, + referenced_table=op.referenced_table, + display_name=op.display_name, + description=op.description, + required=op.required, + cascade_delete=op.cascade_delete, + language_code=op.language_code, + ) + body = relationship.to_dict() + body["Lookup"] = lookup.to_dict() + return [self._od._build_create_relationship(body, solution=op.solution)] + + # ------------------------------------------------------------------ + # Query resolvers — delegate to _ODataClient._build_* methods + # ------------------------------------------------------------------ + + def _resolve_query_sql(self, op: _QuerySql) -> List[_RawRequest]: + return [self._od._build_sql(op.sql)] + + # ------------------------------------------------------------------ + # Multipart serialisation + # ------------------------------------------------------------------ + + def _build_batch_body( + self, + resolved: List[Union[_RawRequest, _ChangeSetBatchItem]], + batch_boundary: str, + ) -> str: + parts: List[str] = [] + for item in resolved: + if isinstance(item, _ChangeSetBatchItem): + parts.append(self._serialize_changeset_item(item, batch_boundary)) + else: + parts.append(self._serialize_raw_request(item, batch_boundary)) + return "".join(parts) + f"--{batch_boundary}--{_CRLF}" + + def _serialize_raw_request(self, req: _RawRequest, boundary: str) -> str: + """Serialise a single operation as a multipart/mixed part with CRLF line endings.""" + part_header_lines = [ + f"--{boundary}", + "Content-Type: application/http", + "Content-Transfer-Encoding: binary", + ] + if req.content_id is not None: + part_header_lines.append(f"Content-ID: {req.content_id}") + + inner_lines = [f"{req.method} {req.url} HTTP/1.1"] + if req.body is not None: + inner_lines.append("Content-Type: application/json; type=entry") + if req.headers: + for k, v in req.headers.items(): + inner_lines.append(f"{k}: {v}") + inner_lines.append("") # blank line — end of inner headers + if req.body is not None: + inner_lines.append(req.body) + + part_header_str = _CRLF.join(part_header_lines) + _CRLF + inner_str = _CRLF.join(inner_lines) + return part_header_str + _CRLF + inner_str + _CRLF + + def _serialize_changeset_item(self, cs: _ChangeSetBatchItem, batch_boundary: str) -> str: + cs_boundary = f"changeset_{uuid.uuid4()}" + cs_parts = [self._serialize_raw_request(r, cs_boundary) for r in cs.requests] + cs_parts.append(f"--{cs_boundary}--{_CRLF}") + cs_body = "".join(cs_parts) + + outer = ( + f"--{batch_boundary}{_CRLF}" f'Content-Type: multipart/mixed; boundary="{cs_boundary}"{_CRLF}' f"{_CRLF}" + ) + return outer + cs_body + _CRLF + + # ------------------------------------------------------------------ + # Response parsing (multipart/mixed) + # ------------------------------------------------------------------ + + def _parse_batch_response(self, response: Any) -> BatchResult: + content_type = response.headers.get("Content-Type", "") + boundary = _extract_boundary(content_type) + if not boundary: + # Non-multipart response: the batch request itself was rejected by Dataverse + # (common for top-level 4xx, e.g. malformed body, missing OData headers). + # Returning an empty BatchResult() here would silently hide the error and + # make has_errors=False, which is actively misleading. Raise instead. + _raise_top_level_batch_error(response) + return BatchResult() # unreachable; satisfies type checkers + parts = _split_multipart(response.text or "", boundary) + responses: List[BatchItemResponse] = [] + for part_headers, part_body in parts: + part_ct = part_headers.get("content-type", "") + if "multipart/mixed" in part_ct: + inner_boundary = _extract_boundary(part_ct) + if inner_boundary: + for ih, ib in _split_multipart(part_body, inner_boundary): + item = _parse_http_response_part(ib, ih.get("content-id")) + if item is not None: + responses.append(item) + else: + item = _parse_http_response_part(part_body, content_id=part_headers.get("content-id")) + if item is not None: + responses.append(item) + return BatchResult(responses=responses) + + +# --------------------------------------------------------------------------- +# Multipart parsing helpers +# --------------------------------------------------------------------------- + + +def _raise_top_level_batch_error(response: Any) -> None: + """Parse a non-multipart batch response and raise HttpError with the service message. + + Dataverse returns ``application/json`` with an ``{"error": {...}}`` payload when + it rejects the batch request at the HTTP level (e.g. malformed multipart body, + missing OData headers). This helper surfaces that detail instead of silently + returning an empty ``BatchResult``. + """ + status_code: int = getattr(response, "status_code", 0) + service_error_code: Optional[str] = None + try: + payload = response.json() + error = payload.get("error", {}) + service_error_code = error.get("code") or None + message: str = error.get("message") or response.text or "Unexpected non-multipart response from $batch" + except Exception: + message = (getattr(response, "text", None) or "") or "Unexpected non-multipart response from $batch" + raise HttpError( + message=f"Batch request rejected by Dataverse: {message}", + status_code=status_code, + subcode=_http_subcode(status_code) if status_code else None, + service_error_code=service_error_code, + ) + + +_BOUNDARY_RE = re.compile(r'boundary="?([^";,\s]+)"?', re.IGNORECASE) + + +def _extract_boundary(content_type: str) -> Optional[str]: + m = _BOUNDARY_RE.search(content_type) + return m.group(1) if m else None + + +def _split_multipart(body: str, boundary: str) -> List[Tuple[Dict[str, str], str]]: + delimiter = f"--{boundary}" + parts: List[Tuple[Dict[str, str], str]] = [] + lines = body.replace("\r\n", "\n").split("\n") + current: List[str] = [] + in_part = False + for line in lines: + stripped = line.rstrip("\r") + if stripped == delimiter: + if in_part and current: + parts.append(_parse_mime_part("\n".join(current))) + current = [] + in_part = True + elif stripped == f"{delimiter}--": + if in_part and current: + parts.append(_parse_mime_part("\n".join(current))) + break + elif in_part: + current.append(line) + return parts + + +def _parse_mime_part(raw: str) -> Tuple[Dict[str, str], str]: + if "\n\n" in raw: + header_block, body = raw.split("\n\n", 1) + else: + header_block, body = raw, "" + headers: Dict[str, str] = {} + for line in header_block.splitlines(): + if ":" in line: + name, _, value = line.partition(":") + headers[name.strip().lower()] = value.strip() + return headers, body.strip() + + +def _parse_http_response_part(text: str, content_id: Optional[str]) -> Optional[BatchItemResponse]: + lines = text.replace("\r\n", "\n").splitlines() + if not lines: + return None + status_line = "" + idx = 0 + for i, line in enumerate(lines): + if line.startswith("HTTP/"): + status_line = line + idx = i + 1 + break + if not status_line: + return None + parts = status_line.split(" ", 2) + if len(parts) < 2: + return None + try: + status_code = int(parts[1]) + except ValueError: + return None + resp_headers: Dict[str, str] = {} + body_start = idx + for i in range(idx, len(lines)): + if lines[i] == "": + body_start = i + 1 + break + if ":" in lines[i]: + name, _, value = lines[i].partition(":") + resp_headers[name.strip().lower()] = value.strip() + entity_id: Optional[str] = None + odata_id = resp_headers.get("odata-entityid", "") + if odata_id: + m = _GUID_RE.search(odata_id) + if m: + entity_id = m.group(0) + body_text = "\n".join(lines[body_start:]).strip() + data: Optional[Dict[str, Any]] = None + error_message: Optional[str] = None + error_code: Optional[str] = None + if body_text: + try: + parsed = json.loads(body_text) + if isinstance(parsed, dict): + err = parsed.get("error") + if isinstance(err, dict): + error_message = err.get("message") + error_code = err.get("code") + else: + data = parsed + except (json.JSONDecodeError, ValueError): + pass + return BatchItemResponse( + status_code=status_code, + content_id=content_id, + entity_id=entity_id, + data=data, + error_message=error_message, + error_code=error_code, + ) diff --git a/src/PowerPlatform/Dataverse/data/_odata.py b/src/PowerPlatform/Dataverse/data/_odata.py index 78ce4091..aca42f3d 100644 --- a/src/PowerPlatform/Dataverse/data/_odata.py +++ b/src/PowerPlatform/Dataverse/data/_odata.py @@ -18,15 +18,26 @@ from contextlib import contextmanager from contextvars import ContextVar +from urllib.parse import quote as _url_quote + from ..core._http import _HttpClient from ._upload import _FileUploadMixin from ._relationships import _RelationshipOperationsMixin +from ..models.relationship import ( + LookupAttributeMetadata, + OneToManyRelationshipMetadata, + CascadeConfiguration, +) +from ..models.labels import Label, LocalizedLabel +from ..common.constants import CASCADE_BEHAVIOR_REMOVE_LINK from ..core.errors import * +from ._raw_request import _RawRequest from ..core._error_codes import ( _http_subcode, _is_transient_status, VALIDATION_SQL_NOT_STRING, VALIDATION_SQL_EMPTY, + VALIDATION_UNSUPPORTED_COLUMN_TYPE, METADATA_ENTITYSET_NOT_FOUND, METADATA_ENTITYSET_NAME_MISSING, METADATA_TABLE_NOT_FOUND, @@ -275,6 +286,19 @@ def _request(self, method: str, url: str, *, expected: tuple[int, ...] = _DEFAUL is_transient=is_transient, ) + def _execute_raw(self, req: _RawRequest, *, expected: tuple[int, ...] = _DEFAULT_EXPECTED_STATUSES): + """Execute a ``_RawRequest`` and return the HTTP response. + + Encodes the pre-serialised body (if present) as UTF-8 and merges any + per-request headers into the standard OData header set. + """ + kwargs: Dict[str, Any] = {} + if req.body is not None: + kwargs["data"] = req.body.encode("utf-8") + if req.headers: + kwargs["headers"] = req.headers + return self._request(req.method.lower(), req.url, expected=expected, **kwargs) + # --- CRUD Internal functions --- def _create(self, entity_set: str, table_schema_name: str, record: Dict[str, Any]) -> str: """Create a single record and return its GUID. @@ -292,12 +316,7 @@ def _create(self, entity_set: str, table_schema_name: str, record: Dict[str, Any .. note:: Relies on ``OData-EntityId`` (canonical) or ``Location`` response header. No response body parsing is performed. Raises ``RuntimeError`` if neither header contains a GUID. """ - # Lowercase all keys to match Dataverse LogicalName expectations - record = self._lowercase_keys(record) - record = self._convert_labels_to_ints(table_schema_name, record) - url = f"{self.api}/{entity_set}" - r = self._request("post", url, json=record) - + r = self._execute_raw(self._build_create(entity_set, table_schema_name, record)) ent_loc = r.headers.get("OData-EntityId") or r.headers.get("OData-EntityID") if ent_loc: m = _GUID_RE.search(ent_loc) @@ -331,25 +350,7 @@ def _create_multiple(self, entity_set: str, table_schema_name: str, records: Lis """ if not all(isinstance(r, dict) for r in records): raise TypeError("All items for multi-create must be dicts") - need_logical = any("@odata.type" not in r for r in records) - # @odata.type uses LogicalName (lowercase) - logical_name = table_schema_name.lower() - enriched: List[Dict[str, Any]] = [] - for r in records: - # Lowercase all keys to match Dataverse LogicalName expectations - r = self._lowercase_keys(r) - r = self._convert_labels_to_ints(table_schema_name, r) - if "@odata.type" in r or not need_logical: - enriched.append(r) - else: - nr = r.copy() - nr["@odata.type"] = f"Microsoft.Dynamics.CRM.{logical_name}" - enriched.append(nr) - payload = {"Targets": enriched} - # Bound action form: POST {entity_set}/Microsoft.Dynamics.CRM.CreateMultiple - url = f"{self.api}/{entity_set}/Microsoft.Dynamics.CRM.CreateMultiple" - # The action currently returns only Ids; no need to request representation. - r = self._request("post", url, json=payload) + r = self._execute_raw(self._build_create_multiple(entity_set, table_schema_name, records)) try: body = r.json() if r.text else {} except ValueError: @@ -563,50 +564,10 @@ def _delete_multiple( targets = [rid for rid in ids if rid] if not targets: return None - value_objects = [{"Value": rid, "Type": "System.Guid"} for rid in targets] - - pk_attr = self._primary_id_attr(table_schema_name) - timestamp = datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") - job_label = f"Bulk delete {table_schema_name} records @ {timestamp}" - - # EntityName must use lowercase LogicalName - logical_name = table_schema_name.lower() - - query = { - "@odata.type": "Microsoft.Dynamics.CRM.QueryExpression", - "EntityName": logical_name, - "ColumnSet": { - "@odata.type": "Microsoft.Dynamics.CRM.ColumnSet", - "AllColumns": False, - "Columns": [], - }, - "Criteria": { - "@odata.type": "Microsoft.Dynamics.CRM.FilterExpression", - "FilterOperator": "And", - "Conditions": [ - { - "@odata.type": "Microsoft.Dynamics.CRM.ConditionExpression", - "AttributeName": pk_attr, - "Operator": "In", - "Values": value_objects, - } - ], - }, - } - - payload = { - "JobName": job_label, - "SendEmailNotification": False, - "ToRecipients": [], - "CCRecipients": [], - "RecurrencePattern": "", - "StartDateTime": timestamp, - "QuerySet": [query], - } - - url = f"{self.api}/BulkDelete" - response = self._request("post", url, json=payload, expected=(200, 202, 204)) - + response = self._execute_raw( + self._build_delete_multiple(table_schema_name, targets), + expected=(200, 202, 204), + ) job_id = None try: body = response.json() if response.text else {} @@ -614,7 +575,6 @@ def _delete_multiple( body = {} if isinstance(body, dict): job_id = body.get("JobId") - return job_id def _format_key(self, key: str) -> str: @@ -646,12 +606,7 @@ def _update(self, table_schema_name: str, key: str, data: Dict[str, Any]) -> Non :return: ``None`` :rtype: ``None`` """ - # Lowercase all keys to match Dataverse LogicalName expectations - data = self._lowercase_keys(data) - data = self._convert_labels_to_ints(table_schema_name, data) - entity_set = self._entity_set_from_schema_name(table_schema_name) - url = f"{self.api}/{entity_set}{self._format_key(key)}" - r = self._request("patch", url, headers={"If-Match": "*"}, json=data) + self._execute_raw(self._build_update(table_schema_name, key, data)) def _update_multiple(self, entity_set: str, table_schema_name: str, records: List[Dict[str, Any]]) -> None: """Bulk update existing records via the collection-bound ``UpdateMultiple`` action. @@ -673,27 +628,7 @@ def _update_multiple(self, entity_set: str, table_schema_name: str, records: Lis """ if not isinstance(records, list) or not records or not all(isinstance(r, dict) for r in records): raise TypeError("records must be a non-empty list[dict]") - - # Determine whether we need logical name resolution (@odata.type missing in any payload) - need_logical = any("@odata.type" not in r for r in records) - # @odata.type uses LogicalName (lowercase) - logical_name = table_schema_name.lower() - enriched: List[Dict[str, Any]] = [] - for r in records: - # Lowercase all keys to match Dataverse LogicalName expectations - r = self._lowercase_keys(r) - r = self._convert_labels_to_ints(table_schema_name, r) - if "@odata.type" in r or not need_logical: - enriched.append(r) - else: - nr = r.copy() - nr["@odata.type"] = f"Microsoft.Dynamics.CRM.{logical_name}" - enriched.append(nr) - - payload = {"Targets": enriched} - url = f"{self.api}/{entity_set}/Microsoft.Dynamics.CRM.UpdateMultiple" - r = self._request("post", url, json=payload) - # Intentionally ignore response content: no stable contract for IDs across environments. + self._execute_raw(self._build_update_multiple_from_records(entity_set, table_schema_name, records)) return None def _delete(self, table_schema_name: str, key: str) -> None: @@ -707,9 +642,7 @@ def _delete(self, table_schema_name: str, key: str) -> None: :return: ``None`` :rtype: ``None`` """ - entity_set = self._entity_set_from_schema_name(table_schema_name) - url = f"{self.api}/{entity_set}{self._format_key(key)}" - self._request("delete", url, headers={"If-Match": "*"}) + self._execute_raw(self._build_delete(table_schema_name, key)) def _get(self, table_schema_name: str, key: str, select: Optional[List[str]] = None) -> Dict[str, Any]: """Retrieve a single record. @@ -724,14 +657,7 @@ def _get(self, table_schema_name: str, key: str, select: Optional[List[str]] = N :return: Retrieved record dictionary (may be empty if no selected attributes). :rtype: ``dict[str, Any]`` """ - params = {} - if select: - # Lowercase column names for case-insensitive matching - params["$select"] = ",".join(self._lowercase_list(select)) - entity_set = self._entity_set_from_schema_name(table_schema_name) - url = f"{self.api}/{entity_set}{self._format_key(key)}" - r = self._request("get", url, params=params) - return r.json() + return self._execute_raw(self._build_get(table_schema_name, key, select=select)).json() def _get_multiple( self, @@ -846,15 +772,7 @@ def _query_sql(self, sql: str) -> list[dict[str, Any]]: if not sql.strip(): raise ValidationError("sql must be a non-empty string", subcode=VALIDATION_SQL_EMPTY) sql = sql.strip() - - # Extract logical table name via helper (robust to identifiers ending with 'from') - logical = self._extract_logical_table(sql) - - entity_set = self._entity_set_from_schema_name(logical) - # Issue GET /{entity_set}?sql= - url = f"{self.api}/{entity_set}" - params = {"sql": sql} - r = self._request("get", url, params=params) + r = self._execute_raw(self._build_sql(sql)) try: body = r.json() except ValueError: @@ -1490,19 +1408,7 @@ def _list_tables( :raises HttpError: If the metadata request fails. """ - url = f"{self.api}/EntityDefinitions" - base_filter = "IsPrivate eq false" - if filter: - combined_filter = f"{base_filter} and ({filter})" - else: - combined_filter = base_filter - params: Dict[str, str] = {"$filter": combined_filter} - if select is not None and isinstance(select, str): - raise TypeError("select must be a list of property names, not a bare string") - if select: - params["$select"] = ",".join(select) - r = self._request("get", url, params=params) - return r.json().get("value", []) + return self._execute_raw(self._build_list_entities(filter=filter, select=select)).json().get("value", []) def _delete_table(self, table_schema_name: str) -> None: """Delete a table by schema name. @@ -1522,9 +1428,7 @@ def _delete_table(self, table_schema_name: str) -> None: f"Table '{table_schema_name}' not found.", subcode=METADATA_TABLE_NOT_FOUND, ) - metadata_id = ent["MetadataId"] - url = f"{self.api}/EntityDefinitions({metadata_id})" - r = self._request("delete", url) + self._execute_raw(self._build_delete_entity(ent["MetadataId"])) # ------------------- Alternate key metadata helpers ------------------- @@ -1744,17 +1648,21 @@ def _create_columns( needs_picklist_flush = False for column_name, column_type in columns.items(): - payload = self._attribute_payload(column_name, column_type) - if not payload: - raise ValueError(f"Unsupported column type '{column_type}' for '{column_name}'.") - - url = f"{self.api}/EntityDefinitions({metadata_id})/Attributes" - self._request("post", url, json=payload) - - created.append(column_name) - - if "OptionSet" in payload: + attr = self._attribute_payload(column_name, column_type) + if not attr: + raise ValidationError( + f"Unsupported column type '{column_type}' for column '{column_name}'.", + subcode=VALIDATION_UNSUPPORTED_COLUMN_TYPE, + ) + if "OptionSet" in attr: needs_picklist_flush = True + req = _RawRequest( + method="POST", + url=f"{self.api}/EntityDefinitions({metadata_id})/Attributes", + body=json.dumps(attr, ensure_ascii=False), + ) + self._execute_raw(req) + created.append(column_name) if needs_picklist_flush: self._flush_cache("picklist") @@ -1818,8 +1726,7 @@ def _delete_columns( if not attr_metadata_id: raise RuntimeError(f"Metadata incomplete for column '{column_name}' (missing MetadataId).") - attr_url = f"{self.api}/EntityDefinitions({metadata_id})/Attributes({attr_metadata_id})" - self._request("delete", attr_url, headers={"If-Match": "*"}) + self._execute_raw(self._build_delete_column(metadata_id, attr_metadata_id)) attr_type = attr_meta.get("@odata.type") or attr_meta.get("AttributeType") if isinstance(attr_type, str): @@ -1834,6 +1741,475 @@ def _delete_columns( return deleted + # ---------------------- _build_* methods (no HTTP) --------------- + + def _build_create( + self, + entity_set: str, + table: str, + data: Dict[str, Any], + *, + content_id: Optional[int] = None, + ) -> _RawRequest: + """Build a single-record POST request without sending it.""" + body = self._lowercase_keys(data) + body = self._convert_labels_to_ints(table, body) + return _RawRequest( + method="POST", + url=f"{self.api}/{entity_set}", + body=json.dumps(body, ensure_ascii=False), + content_id=content_id, + ) + + def _build_create_multiple( + self, + entity_set: str, + table: str, + records: List[Dict[str, Any]], + ) -> _RawRequest: + """Build a CreateMultiple POST request without sending it.""" + if not all(isinstance(r, dict) for r in records): + raise TypeError("All items for multi-create must be dicts") + logical_name = table.lower() + enriched = [] + for r in records: + r = self._lowercase_keys(r) + r = self._convert_labels_to_ints(table, r) + if "@odata.type" not in r: + r = {**r, "@odata.type": f"Microsoft.Dynamics.CRM.{logical_name}"} + enriched.append(r) + return _RawRequest( + method="POST", + url=f"{self.api}/{entity_set}/Microsoft.Dynamics.CRM.CreateMultiple", + body=json.dumps({"Targets": enriched}, ensure_ascii=False), + ) + + def _build_update( + self, + table: str, + record_id: str, + changes: Dict[str, Any], + *, + content_id: Optional[int] = None, + ) -> _RawRequest: + """Build a single-record PATCH request without sending it. + + ``record_id`` may be a ``"$n"`` content-ID reference; in that case the + URL is the reference itself (resolved server-side within a changeset). + """ + body = self._lowercase_keys(changes) + body = self._convert_labels_to_ints(table, body) + if record_id.startswith("$"): + url = record_id + else: + entity_set = self._entity_set_from_schema_name(table) + url = f"{self.api}/{entity_set}{self._format_key(record_id)}" + return _RawRequest( + method="PATCH", + url=url, + body=json.dumps(body, ensure_ascii=False), + headers={"If-Match": "*"}, + content_id=content_id, + ) + + def _build_update_multiple_from_records( + self, + entity_set: str, + table: str, + records: List[Dict[str, Any]], + ) -> _RawRequest: + """Build an UpdateMultiple POST request from pre-assembled records. + + Each record must already contain the primary key attribute. This helper + is shared by :meth:`_update_multiple` (which pre-assembles records) and + :meth:`_build_update_multiple` (which assembles from ids + changes). + """ + logical_name = table.lower() + enriched = [] + for r in records: + r = self._lowercase_keys(r) + r = self._convert_labels_to_ints(table, r) + if "@odata.type" not in r: + r = {**r, "@odata.type": f"Microsoft.Dynamics.CRM.{logical_name}"} + enriched.append(r) + return _RawRequest( + method="POST", + url=f"{self.api}/{entity_set}/Microsoft.Dynamics.CRM.UpdateMultiple", + body=json.dumps({"Targets": enriched}, ensure_ascii=False), + ) + + def _build_update_multiple( + self, + entity_set: str, + table: str, + ids: List[str], + changes: Union[Dict[str, Any], List[Dict[str, Any]]], + ) -> _RawRequest: + """Build an UpdateMultiple POST request without sending it.""" + pk_attr = self._primary_id_attr(table) + if isinstance(changes, dict): + records = [{pk_attr: rid, **changes} for rid in ids] + elif isinstance(changes, list): + if len(changes) != len(ids): + raise ValidationError( + "ids and changes lists must have equal length for paired update.", + subcode="ids_changes_length_mismatch", + ) + records = [{pk_attr: rid, **ch} for rid, ch in zip(ids, changes)] + else: + raise ValidationError("changes must be a dict or list[dict].", subcode="invalid_changes_type") + return self._build_update_multiple_from_records(entity_set, table, records) + + def _build_upsert( + self, + entity_set: str, + table: str, + alternate_key: Dict[str, Any], + record: Dict[str, Any], + ) -> _RawRequest: + """Build a single-record PATCH upsert request without sending it. + + Unlike :meth:`_build_update`, no ``If-Match: *`` header is added so the + server creates the record when it does not yet exist. + """ + body = self._lowercase_keys(record) + body = self._convert_labels_to_ints(table, body) + key_str = self._build_alternate_key_str(alternate_key) + url = f"{self.api}/{entity_set}({key_str})" + return _RawRequest( + method="PATCH", + url=url, + body=json.dumps(body, ensure_ascii=False), + ) + + def _build_upsert_multiple( + self, + entity_set: str, + table: str, + alternate_keys: List[Dict[str, Any]], + records: List[Dict[str, Any]], + ) -> _RawRequest: + """Build an UpsertMultiple POST request without sending it.""" + if len(alternate_keys) != len(records): + raise ValidationError( + f"alternate_keys and records must have the same length " f"({len(alternate_keys)} != {len(records)})", + subcode="upsert_length_mismatch", + ) + logical_name = table.lower() + targets: List[Dict[str, Any]] = [] + for alt_key, record in zip(alternate_keys, records): + alt_key_lower = self._lowercase_keys(alt_key) + record_processed = self._lowercase_keys(record) + record_processed = self._convert_labels_to_ints(table, record_processed) + conflicting = { + k for k in set(alt_key_lower) & set(record_processed) if alt_key_lower[k] != record_processed[k] + } + if conflicting: + raise ValidationError( + f"record payload conflicts with alternate_key on fields: {sorted(conflicting)!r}", + subcode="upsert_key_conflict", + ) + if "@odata.type" not in record_processed: + record_processed["@odata.type"] = f"Microsoft.Dynamics.CRM.{logical_name}" + key_str = self._build_alternate_key_str(alt_key) + record_processed["@odata.id"] = f"{entity_set}({key_str})" + targets.append(record_processed) + return _RawRequest( + method="POST", + url=f"{self.api}/{entity_set}/Microsoft.Dynamics.CRM.UpsertMultiple", + body=json.dumps({"Targets": targets}, ensure_ascii=False), + ) + + def _build_delete( + self, + table: str, + record_id: str, + *, + content_id: Optional[int] = None, + ) -> _RawRequest: + """Build a single-record DELETE request without sending it. + + ``record_id`` may be a ``"$n"`` content-ID reference. + """ + if record_id.startswith("$"): + url = record_id + else: + entity_set = self._entity_set_from_schema_name(table) + url = f"{self.api}/{entity_set}{self._format_key(record_id)}" + return _RawRequest( + method="DELETE", + url=url, + headers={"If-Match": "*"}, + content_id=content_id, + ) + + def _build_delete_multiple(self, table: str, ids: List[str]) -> _RawRequest: + """Build a BulkDelete POST request without sending it.""" + pk_attr = self._primary_id_attr(table) + logical_name = table.lower() + timestamp = datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") + payload = { + "JobName": f"Bulk delete {table} records @ {timestamp}", + "SendEmailNotification": False, + "ToRecipients": [], + "CCRecipients": [], + "RecurrencePattern": "", + "StartDateTime": timestamp, + "QuerySet": [ + { + "@odata.type": "Microsoft.Dynamics.CRM.QueryExpression", + "EntityName": logical_name, + "ColumnSet": { + "@odata.type": "Microsoft.Dynamics.CRM.ColumnSet", + "AllColumns": False, + "Columns": [], + }, + "Criteria": { + "@odata.type": "Microsoft.Dynamics.CRM.FilterExpression", + "FilterOperator": "And", + "Conditions": [ + { + "@odata.type": "Microsoft.Dynamics.CRM.ConditionExpression", + "AttributeName": pk_attr, + "Operator": "In", + "Values": [{"Value": rid, "Type": "System.Guid"} for rid in ids], + } + ], + }, + } + ], + } + return _RawRequest( + method="POST", + url=f"{self.api}/BulkDelete", + body=json.dumps(payload, ensure_ascii=False), + ) + + def _build_get( + self, + table: str, + record_id: str, + *, + select: Optional[List[str]] = None, + ) -> _RawRequest: + """Build a single-record GET request without sending it.""" + entity_set = self._entity_set_from_schema_name(table) + url = f"{self.api}/{entity_set}{self._format_key(record_id)}" + if select: + url += "?$select=" + ",".join(self._lowercase_list(select)) + return _RawRequest(method="GET", url=url) + + def _build_create_entity( + self, + table: str, + columns: Dict[str, Any], + solution: Optional[str] = None, + primary_column: Optional[str] = None, + ) -> _RawRequest: + """Build an EntityDefinitions POST request without sending it.""" + if primary_column: + primary_attr = primary_column + else: + primary_attr = f"{table.split('_', 1)[0]}_Name" if "_" in table else "new_Name" + attributes = [self._attribute_payload(primary_attr, "string", is_primary_name=True)] + for col_name, dtype in columns.items(): + attr = self._attribute_payload(col_name, dtype) + if not attr: + raise ValidationError( + f"Unsupported column type '{dtype}' for column '{col_name}'.", + subcode=VALIDATION_UNSUPPORTED_COLUMN_TYPE, + ) + attributes.append(attr) + body = { + "@odata.type": "Microsoft.Dynamics.CRM.EntityMetadata", + "SchemaName": table, + "DisplayName": self._label(table), + "DisplayCollectionName": self._label(table + "s"), + "Description": self._label(f"Custom entity for {table}"), + "OwnershipType": "UserOwned", + "HasActivities": False, + "HasNotes": True, + "IsActivity": False, + "Attributes": attributes, + } + url = f"{self.api}/EntityDefinitions" + if solution: + url += f"?SolutionUniqueName={solution}" + return _RawRequest( + method="POST", + url=url, + body=json.dumps(body, ensure_ascii=False), + ) + + def _build_delete_entity(self, metadata_id: str) -> _RawRequest: + """Build an EntityDefinitions DELETE request without sending it.""" + return _RawRequest( + method="DELETE", + url=f"{self.api}/EntityDefinitions({metadata_id})", + headers={"If-Match": "*"}, + ) + + def _build_get_entity(self, table: str) -> _RawRequest: + """Build an EntityDefinitions GET request without sending it.""" + logical = self._escape_odata_quotes(table.lower()) + return _RawRequest( + method="GET", + url=( + f"{self.api}/EntityDefinitions" + f"?$select=MetadataId,LogicalName,SchemaName,EntitySetName,PrimaryNameAttribute,PrimaryIdAttribute" + f"&$filter=LogicalName eq '{logical}'" + ), + ) + + def _build_list_entities( + self, + *, + filter: Optional[str] = None, + select: Optional[List[str]] = None, + ) -> _RawRequest: + """Build an EntityDefinitions list GET request without sending it.""" + base_filter = "IsPrivate eq false" + if filter: + combined_filter = f"{base_filter} and ({filter})" + else: + combined_filter = base_filter + url = f"{self.api}/EntityDefinitions?$filter={combined_filter}" + if select is not None and isinstance(select, str): + raise TypeError("select must be a list of property names, not a bare string") + if select: + url += "&$select=" + ",".join(select) + return _RawRequest(method="GET", url=url) + + def _build_create_column( + self, + entity_metadata_id: str, + col_name: str, + dtype: Any, + ) -> _RawRequest: + """Build an Attributes POST request for one column without sending it.""" + attr = self._attribute_payload(col_name, dtype) + if not attr: + raise ValidationError( + f"Unsupported column type '{dtype}' for column '{col_name}'.", + subcode=VALIDATION_UNSUPPORTED_COLUMN_TYPE, + ) + return _RawRequest( + method="POST", + url=f"{self.api}/EntityDefinitions({entity_metadata_id})/Attributes", + body=json.dumps(attr, ensure_ascii=False), + ) + + def _build_delete_column( + self, + entity_metadata_id: str, + col_metadata_id: str, + ) -> _RawRequest: + """Build an Attributes DELETE request for one column without sending it.""" + return _RawRequest( + method="DELETE", + url=f"{self.api}/EntityDefinitions({entity_metadata_id})/Attributes({col_metadata_id})", + headers={"If-Match": "*"}, + ) + + @staticmethod + def _build_lookup_field_models( + referencing_table: str, + lookup_field_name: str, + referenced_table: str, + *, + display_name: Optional[str] = None, + description: Optional[str] = None, + required: bool = False, + cascade_delete: str = CASCADE_BEHAVIOR_REMOVE_LINK, + language_code: int = 1033, + ) -> tuple: + """Build a (lookup, relationship) pair for a lookup field creation. + + Returns ``(LookupAttributeMetadata, OneToManyRelationshipMetadata)``. + Used by both the batch resolver and ``TableOperations.create_lookup_field`` + to avoid duplicating the metadata assembly logic. + """ + lookup = LookupAttributeMetadata( + schema_name=lookup_field_name, + display_name=Label( + localized_labels=[ + LocalizedLabel( + label=display_name or referenced_table, + language_code=language_code, + ) + ] + ), + required_level="ApplicationRequired" if required else "None", + ) + if description: + lookup.description = Label( + localized_labels=[LocalizedLabel(label=description, language_code=language_code)] + ) + rel_name = f"{referenced_table}_{referencing_table}_{lookup_field_name}" + relationship = OneToManyRelationshipMetadata( + schema_name=rel_name, + referenced_entity=referenced_table, + referencing_entity=referencing_table, + referenced_attribute=f"{referenced_table}id", + cascade_configuration=CascadeConfiguration(delete=cascade_delete), + ) + return lookup, relationship + + def _build_create_relationship( + self, + body: Dict[str, Any], + *, + solution: Optional[str] = None, + ) -> _RawRequest: + """Build a RelationshipDefinitions POST request without sending it.""" + headers: Dict[str, str] = {} + if solution: + headers["MSCRM.SolutionUniqueName"] = solution + return _RawRequest( + method="POST", + url=f"{self.api}/RelationshipDefinitions", + body=json.dumps(body, ensure_ascii=False), + headers=headers or None, + ) + + def _build_delete_relationship(self, relationship_id: str) -> _RawRequest: + """Build a RelationshipDefinitions DELETE request without sending it.""" + return _RawRequest( + method="DELETE", + url=f"{self.api}/RelationshipDefinitions({relationship_id})", + headers={"If-Match": "*"}, + ) + + def _build_get_relationship(self, schema_name: str) -> _RawRequest: + """Build a RelationshipDefinitions GET request without sending it.""" + escaped = self._escape_odata_quotes(schema_name) + return _RawRequest( + method="GET", + url=f"{self.api}/RelationshipDefinitions?$filter=SchemaName eq '{escaped}'", + ) + + def _build_sql(self, sql: str) -> _RawRequest: + """Build a SQL query GET request without sending it. + + Resolves the entity set from the table name in the SQL statement via + :meth:`_extract_logical_table`, then embeds the SQL as a URL-encoded + ``?sql=`` query parameter. + + Uses ``urllib.parse.quote`` (``%20`` for spaces) rather than + ``urllib.parse.urlencode`` (``+`` for spaces). Both are accepted by + Dataverse and ``%20`` is the canonical RFC 3986 encoding for query- + string values. + + :param sql: SELECT statement (non-empty string; caller is responsible + for validation). + """ + logical = self._extract_logical_table(sql) + entity_set = self._entity_set_from_schema_name(logical) + return _RawRequest( + method="GET", + url=f"{self.api}/{entity_set}?sql={_url_quote(sql, safe='')}", + ) + # ---------------------- Cache maintenance ------------------------- def _flush_cache( self, diff --git a/src/PowerPlatform/Dataverse/data/_raw_request.py b/src/PowerPlatform/Dataverse/data/_raw_request.py new file mode 100644 index 00000000..a5f37679 --- /dev/null +++ b/src/PowerPlatform/Dataverse/data/_raw_request.py @@ -0,0 +1,33 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Resolved HTTP request dataclass shared by _odata.py and _batch.py.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Dict, Optional + +__all__ = [] + + +@dataclass +class _RawRequest: + """A fully-resolved HTTP request ready for execution or multipart serialisation. + + Used by ``_ODataClient._build_*`` methods to return a constructed request + without executing it, and by ``_BatchClient`` to serialise the batch body. + + :param method: HTTP method (``GET``, ``POST``, ``PATCH``, ``DELETE``). + :param url: Absolute URL (``https://org.crm.dynamics.com/api/data/v9.2/...``). + :param body: JSON-serialised request body, or ``None`` for bodyless requests. + :param headers: Extra inner-request headers (e.g. ``{"If-Match": "*"}``). + :param content_id: Emits a ``Content-ID: n`` header in the MIME part when set. + Only relevant for changeset items; enables ``$n`` URI references. + """ + + method: str + url: str + body: Optional[str] = None + headers: Optional[Dict[str, str]] = None + content_id: Optional[int] = None diff --git a/src/PowerPlatform/Dataverse/models/batch.py b/src/PowerPlatform/Dataverse/models/batch.py new file mode 100644 index 00000000..0f2c7c59 --- /dev/null +++ b/src/PowerPlatform/Dataverse/models/batch.py @@ -0,0 +1,106 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Public result types for batch operations.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +__all__ = ["BatchItemResponse", "BatchResult"] + + +@dataclass +class BatchItemResponse: + """ + Response from a single operation within a batch request. + + Responses are returned in submission order. For operations added to a + changeset, responses appear in the changeset's position in that order. + + :param status_code: HTTP status code for this operation (e.g. 204, 200, 400). + :param content_id: ``Content-ID`` value from the changeset response part, if any. + :param entity_id: GUID extracted from the ``OData-EntityId`` response header. + Set for successful create (POST) operations. + :param data: Parsed JSON response body (e.g. for GET operations). + :param error_message: Error message when the operation failed. + :param error_code: Service error code when the operation failed. + + Example:: + + for item in result.responses: + if item.is_success: + print(f"[OK] {item.status_code} entity_id={item.entity_id}") + else: + print(f"[ERR] {item.status_code}: {item.error_message}") + """ + + status_code: int + content_id: Optional[str] = None + entity_id: Optional[str] = None + data: Optional[Dict[str, Any]] = None + error_message: Optional[str] = None + error_code: Optional[str] = None + + @property + def is_success(self) -> bool: + """Return True when status_code is 2xx.""" + return 200 <= self.status_code < 300 + + +@dataclass +class BatchResult: + """ + Result of executing a batch request. + + Contains one :class:`BatchItemResponse` per HTTP operation submitted. + Operations that expand to multiple HTTP requests (e.g. ``add_columns`` + with three columns) contribute three entries. + + :param responses: All responses in submission order. + + Example:: + + result = client.batch.new().execute() + print(f"Succeeded: {len(result.succeeded)}, Failed: {len(result.failed)}") + for guid in result.entity_ids: + print(f"[OK] entity_id: {guid}") + """ + + responses: List[BatchItemResponse] = field(default_factory=list) + + @property + def succeeded(self) -> List[BatchItemResponse]: + """Responses with 2xx status codes.""" + return [r for r in self.responses if r.is_success] + + @property + def failed(self) -> List[BatchItemResponse]: + """Responses with non-2xx status codes.""" + return [r for r in self.responses if not r.is_success] + + @property + def has_errors(self) -> bool: + """True when any response has a non-2xx status code.""" + return any(not r.is_success for r in self.responses) + + @property + def entity_ids(self) -> List[str]: + """GUIDs extracted from ``OData-EntityId`` headers of successful responses. + + Returns entity IDs from any successful (2xx) response that includes an + ``OData-EntityId`` header. Both individual ``POST`` (create) and + ``PATCH`` (update) operations return this header with the record's GUID. + ``GET`` and ``DELETE`` operations do not. + + .. note:: + ``CreateMultiple`` and ``UpsertMultiple`` action responses do **not** + return per-record ``OData-EntityId`` headers. Their IDs are in the + JSON response body (``data["Ids"]``). Access them via:: + + for resp in result.succeeded: + if resp.data and "Ids" in resp.data: + bulk_ids = resp.data["Ids"] + """ + return [r.entity_id for r in self.succeeded if r.entity_id is not None] diff --git a/src/PowerPlatform/Dataverse/operations/batch.py b/src/PowerPlatform/Dataverse/operations/batch.py new file mode 100644 index 00000000..614dc6cb --- /dev/null +++ b/src/PowerPlatform/Dataverse/operations/batch.py @@ -0,0 +1,890 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Batch operation namespaces for the Dataverse SDK.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union + +import pandas as pd + +from ..core.errors import ValidationError +from ..core._error_codes import VALIDATION_SQL_EMPTY +from ..data._batch import ( + _BatchClient, + _ChangeSet, + _RecordCreate, + _RecordUpdate, + _RecordDelete, + _RecordGet, + _RecordUpsert, + _TableCreate, + _TableDelete, + _TableGet, + _TableList, + _TableAddColumns, + _TableRemoveColumns, + _TableCreateOneToMany, + _TableCreateManyToMany, + _TableDeleteRelationship, + _TableGetRelationship, + _TableCreateLookupField, + _QuerySql, +) +from ..models.batch import BatchResult +from ..models.upsert import UpsertItem +from ..models.relationship import ( + LookupAttributeMetadata, + OneToManyRelationshipMetadata, + ManyToManyRelationshipMetadata, +) +from ..common.constants import CASCADE_BEHAVIOR_REMOVE_LINK + +if TYPE_CHECKING: + from ..client import DataverseClient + +__all__ = [ + "BatchRecordOperations", + "BatchTableOperations", + "BatchQueryOperations", + "BatchDataFrameOperations", + "BatchRequest", + "BatchOperations", + "ChangeSet", + "ChangeSetRecordOperations", +] + + +# --------------------------------------------------------------------------- +# Changeset namespaces +# --------------------------------------------------------------------------- + + +class ChangeSetRecordOperations: + """ + Record write operations available inside a :class:`ChangeSet`. + + Mirrors ``client.records`` but restricted to single-record forms (no bulk + create/update/delete). Only write operations are allowed — GET is not + permitted inside a changeset. + + Do not instantiate directly; use :attr:`ChangeSet.records`. + """ + + def __init__(self, cs_internal: _ChangeSet) -> None: + self._cs = cs_internal + + def create(self, table: str, data: Dict[str, Any]) -> str: + """ + Add a single-record create to this changeset. + + :param table: Table schema name (e.g. ``"account"``). + :type table: :class:`str` + :param data: Column values for the new record. + :type data: dict[str, typing.Any] + :returns: A content-ID reference string (e.g. ``"$1"``) usable in + subsequent operations within this changeset as a URI reference + in ``@odata.bind`` fields or as ``record_id`` in + :meth:`update` / :meth:`delete`. + :rtype: :class:`str` + + Example:: + + with batch.changeset() as cs: + lead_ref = cs.records.create("lead", {"firstname": "Ada"}) + cs.records.create("account", { + "name": "Babbage", + "originatingleadid@odata.bind": lead_ref, + }) + """ + return self._cs.add_create(table, data) + + def update(self, table: str, record_id: str, changes: Dict[str, Any]) -> None: + """ + Add a single-record update to this changeset. + + :param table: Table schema name. Ignored when ``record_id`` is a + content-ID reference. + :type table: :class:`str` + :param record_id: GUID or a content-ID reference (e.g. ``"$1"``) + returned by a prior :meth:`create` in this changeset. + :type record_id: :class:`str` + :param changes: Column values to update. + :type changes: dict[str, typing.Any] + """ + self._cs.add_update(table, record_id, changes) + + def delete(self, table: str, record_id: str) -> None: + """ + Add a single-record delete to this changeset. + + :param table: Table schema name. Ignored when ``record_id`` is a + content-ID reference. + :type table: :class:`str` + :param record_id: GUID or a content-ID reference (e.g. ``"$1"``). + :type record_id: :class:`str` + """ + self._cs.add_delete(table, record_id) + + +class ChangeSet: + """ + A transactional group of single-record write operations. + + All operations succeed or are rolled back together. Use as a context + manager or call :attr:`records` to add operations directly. + + Do not instantiate directly; use :meth:`BatchRequest.changeset`. + + Example:: + + with batch.changeset() as cs: + ref = cs.records.create("contact", {"firstname": "Alice"}) + cs.records.update("account", account_id, { + "primarycontactid@odata.bind": ref + }) + """ + + def __init__(self, internal: _ChangeSet) -> None: + self._internal = internal + self.records = ChangeSetRecordOperations(internal) + + def __enter__(self) -> "ChangeSet": + return self + + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + return None + + +# --------------------------------------------------------------------------- +# Batch request namespaces +# --------------------------------------------------------------------------- + + +class BatchRecordOperations: + """ + Record operations on a :class:`BatchRequest`. + + Mirrors ``client.records`` exactly: same method names, same signatures. + All methods return ``None``; results are available via + :class:`~PowerPlatform.Dataverse.models.batch.BatchResult` after + :meth:`BatchRequest.execute`. + + Do not instantiate directly; use ``batch.records``. + """ + + def __init__(self, batch: "BatchRequest") -> None: + self._batch = batch + + def create( + self, + table: str, + data: Union[Dict[str, Any], List[Dict[str, Any]]], + ) -> None: + """ + Add a create operation to the batch. + + A single dict creates one record (POST entity_set). + A list of dicts creates all records via the ``CreateMultiple`` action + (one batch item). + + :param table: Table schema name (e.g. ``"account"``). + :type table: :class:`str` + :param data: Single record dict or list of record dicts. + :type data: dict or list[dict] + """ + self._batch._items.append(_RecordCreate(table=table, data=data)) + + def update( + self, + table: str, + ids: Union[str, List[str]], + changes: Union[Dict[str, Any], List[Dict[str, Any]]], + ) -> None: + """ + Add an update operation to the batch. + + - **Single** ``(table, "guid", {...})`` -> one PATCH request. + - **Broadcast** ``(table, [id1, id2], {...})`` -> one ``UpdateMultiple`` POST. + - **Paired** ``(table, [id1, id2], [{...}, {...}])`` -> one ``UpdateMultiple`` POST. + + :param table: Table schema name. + :type table: :class:`str` + :param ids: Single GUID or list of GUIDs. + :type ids: str or list[str] + :param changes: Single dict (single/broadcast) or list of dicts (paired). + :type changes: dict or list[dict] + """ + self._batch._items.append(_RecordUpdate(table=table, ids=ids, changes=changes)) + + def delete( + self, + table: str, + ids: Union[str, List[str]], + *, + use_bulk_delete: bool = True, + ) -> None: + """ + Add a delete operation to the batch. + + - **Single** ``(table, "guid")`` -> one DELETE request. + - **List + use_bulk_delete=True** (default) -> one ``BulkDelete`` POST. + The async job ID will be available in ``BatchItemResponse.data["JobId"]``. + - **List + use_bulk_delete=False** -> one DELETE per record. + + :param table: Table schema name. + :type table: :class:`str` + :param ids: Single GUID or list of GUIDs. + :type ids: str or list[str] + :param use_bulk_delete: When True (default) and ``ids`` is a list, use the + BulkDelete action. When False, delete records individually. + :type use_bulk_delete: :class:`bool` + """ + self._batch._items.append(_RecordDelete(table=table, ids=ids, use_bulk_delete=use_bulk_delete)) + + def get( + self, + table: str, + record_id: str, + *, + select: Optional[List[str]] = None, + ) -> None: + """ + Add a single-record get operation to the batch. + + Only the single-record overload (``record_id`` provided) is supported. + The paginated/multi-record overload of ``client.records.get()`` + (``filter``, ``orderby``, etc., without ``record_id``) is **not** + supported in batch — pagination requires following + ``@odata.nextLink`` across multiple round-trips, which is + incompatible with a single batch request. + + The response body will be available in + :attr:`~PowerPlatform.Dataverse.models.batch.BatchItemResponse.data` + after :meth:`BatchRequest.execute`. + + :param table: Table schema name. + :type table: :class:`str` + :param record_id: GUID of the record to retrieve. + :type record_id: :class:`str` + :param select: Optional list of column names to include. + :type select: list[str] or None + """ + self._batch._items.append(_RecordGet(table=table, record_id=record_id, select=select)) + + def upsert( + self, + table: str, + items: List[Union[UpsertItem, Dict[str, Any]]], + ) -> None: + """ + Add an upsert operation to the batch. + + Mirrors :meth:`~PowerPlatform.Dataverse.operations.records.RecordOperations.upsert`: + a single item becomes a PATCH request using the alternate key; multiple items + become one ``UpsertMultiple`` POST. + + Each item must be a :class:`~PowerPlatform.Dataverse.models.upsert.UpsertItem` + or a plain ``dict`` with ``"alternate_key"`` and ``"record"`` keys (both dicts). + + :param table: Table schema name (e.g. ``"account"``). + :type table: :class:`str` + :param items: Non-empty list of :class:`~PowerPlatform.Dataverse.models.upsert.UpsertItem` + instances or equivalent dicts. + :type items: list[~PowerPlatform.Dataverse.models.upsert.UpsertItem] + + :raises TypeError: If ``items`` is not a non-empty list, or if any element is + neither a :class:`~PowerPlatform.Dataverse.models.upsert.UpsertItem` nor a + dict with ``"alternate_key"`` and ``"record"`` keys. + + Example:: + + from PowerPlatform.Dataverse.models.upsert import UpsertItem + + batch.records.upsert("account", [ + UpsertItem( + alternate_key={"accountnumber": "ACC-001"}, + record={"name": "Contoso Ltd"}, + ), + UpsertItem( + alternate_key={"accountnumber": "ACC-002"}, + record={"name": "Fabrikam Inc"}, + ), + ]) + """ + if not isinstance(items, list) or not items: + raise TypeError("items must be a non-empty list of UpsertItem or dicts") + normalized: List[UpsertItem] = [] + for i in items: + if isinstance(i, UpsertItem): + normalized.append(i) + elif isinstance(i, dict) and isinstance(i.get("alternate_key"), dict) and isinstance(i.get("record"), dict): + normalized.append(UpsertItem(alternate_key=i["alternate_key"], record=i["record"])) + else: + raise TypeError("Each item must be an UpsertItem or a dict with 'alternate_key' and 'record' keys") + self._batch._items.append(_RecordUpsert(table=table, items=normalized)) + + +class BatchTableOperations: + """ + Table metadata operations on a :class:`BatchRequest`. + + Mirrors ``client.tables`` exactly: same method names, same signatures. + All methods return ``None``; results arrive via + :class:`~PowerPlatform.Dataverse.models.batch.BatchResult`. + + .. note:: + ``tables.delete``, ``tables.add_columns``, and ``tables.remove_columns`` + require a metadata lookup (GET ``EntityDefinitions``) at + :meth:`BatchRequest.execute` time to resolve the table's MetadataId. + This lookup is transparent to the caller. + + .. note:: + ``tables.add_columns`` and ``tables.remove_columns`` each produce one + batch item per column, so they contribute multiple entries to + :attr:`~PowerPlatform.Dataverse.models.batch.BatchResult.responses`. + + Do not instantiate directly; use ``batch.tables``. + """ + + def __init__(self, batch: "BatchRequest") -> None: + self._batch = batch + + def create( + self, + table: str, + columns: Dict[str, Any], + *, + solution: Optional[str] = None, + primary_column: Optional[str] = None, + ) -> None: + """ + Add a table-create operation to the batch. + + .. note:: + The pre-existence check performed by ``client.tables.create`` is skipped + in batch mode. If the table already exists the server returns an error + in the corresponding :class:`~PowerPlatform.Dataverse.models.batch.BatchItemResponse`. + + :param table: Schema name of the new table (e.g. ``"new_Product"``). + :type table: :class:`str` + :param columns: Mapping of column schema names to type strings or Enum subclasses. + :type columns: dict[str, typing.Any] + :param solution: Optional solution unique name. + :type solution: str or None + :param primary_column: Optional primary column schema name. + :type primary_column: str or None + """ + self._batch._items.append( + _TableCreate( + table=table, + columns=columns, + solution=solution, + primary_column=primary_column, + ) + ) + + def delete(self, table: str) -> None: + """ + Add a table-delete operation to the batch. + + The table's ``MetadataId`` is resolved via a GET request at execute time. + + :param table: Schema name of the table to delete. + :type table: :class:`str` + """ + self._batch._items.append(_TableDelete(table=table)) + + def get(self, table: str) -> None: + """ + Add a table-metadata-get operation to the batch. + + The response will be in ``BatchItemResponse.data`` after execute. + + :param table: Schema name of the table. + :type table: :class:`str` + """ + self._batch._items.append(_TableGet(table=table)) + + def list( + self, + *, + filter: Optional[str] = None, + select: Optional[List[str]] = None, + ) -> None: + """ + Add a list-all-tables operation to the batch. + + Mirrors ``client.tables.list()``. Supply an optional OData + ``$filter`` expression to further narrow the results (combined with + ``IsPrivate eq false`` using ``and``). ``select`` projects + specific property names via ``$select``. + + The response will be in ``BatchItemResponse.data`` after execute. + + :param filter: Additional OData ``$filter`` expression. + :type filter: str or None + :param select: List of property names for ``$select``. + :type select: list[str] or None + """ + self._batch._items.append(_TableList(filter=filter, select=select)) + + def add_columns(self, table: str, columns: Dict[str, Any]) -> None: + """ + Add column-create operations to the batch (one per column). + + The table's ``MetadataId`` is resolved at execute time. Each column + produces one entry in :attr:`BatchResult.responses`. + + :param table: Schema name of the target table. + :type table: :class:`str` + :param columns: Mapping of column schema names to type strings or Enum subclasses. + :type columns: dict[str, typing.Any] + """ + self._batch._items.append(_TableAddColumns(table=table, columns=columns)) + + def remove_columns(self, table: str, columns: Union[str, List[str]]) -> None: + """ + Add column-delete operations to the batch (one per column). + + The table's ``MetadataId`` and each column's ``MetadataId`` are resolved + at execute time. Each column produces one entry in + :attr:`BatchResult.responses`. + + :param table: Schema name of the target table. + :type table: :class:`str` + :param columns: Column schema name or list of column schema names to remove. + :type columns: str or list[str] + """ + self._batch._items.append(_TableRemoveColumns(table=table, columns=columns)) + + def create_one_to_many_relationship( + self, + lookup: LookupAttributeMetadata, + relationship: OneToManyRelationshipMetadata, + *, + solution: Optional[str] = None, + ) -> None: + """ + Add a one-to-many relationship creation to the batch. + + :param lookup: Lookup attribute metadata. + :type lookup: ~PowerPlatform.Dataverse.models.relationship.LookupAttributeMetadata + :param relationship: Relationship metadata. + :type relationship: ~PowerPlatform.Dataverse.models.relationship.OneToManyRelationshipMetadata + :param solution: Optional solution unique name. + :type solution: str or None + """ + self._batch._items.append(_TableCreateOneToMany(lookup=lookup, relationship=relationship, solution=solution)) + + def create_many_to_many_relationship( + self, + relationship: ManyToManyRelationshipMetadata, + *, + solution: Optional[str] = None, + ) -> None: + """ + Add a many-to-many relationship creation to the batch. + + :param relationship: Relationship metadata. + :type relationship: ~PowerPlatform.Dataverse.models.relationship.ManyToManyRelationshipMetadata + :param solution: Optional solution unique name. + :type solution: str or None + """ + self._batch._items.append(_TableCreateManyToMany(relationship=relationship, solution=solution)) + + def delete_relationship(self, relationship_id: str) -> None: + """ + Add a relationship-delete operation to the batch. + + :param relationship_id: GUID of the relationship metadata to delete. + :type relationship_id: :class:`str` + """ + self._batch._items.append(_TableDeleteRelationship(relationship_id=relationship_id)) + + def get_relationship(self, schema_name: str) -> None: + """ + Add a relationship-metadata-get operation to the batch. + + The response will be in ``BatchItemResponse.data`` after execute. + + :param schema_name: Schema name of the relationship. + :type schema_name: :class:`str` + """ + self._batch._items.append(_TableGetRelationship(schema_name=schema_name)) + + def create_lookup_field( + self, + referencing_table: str, + lookup_field_name: str, + referenced_table: str, + *, + display_name: Optional[str] = None, + description: Optional[str] = None, + required: bool = False, + cascade_delete: str = CASCADE_BEHAVIOR_REMOVE_LINK, + solution: Optional[str] = None, + language_code: int = 1033, + ) -> None: + """ + Add a lookup field creation to the batch (convenience wrapper for + :meth:`create_one_to_many_relationship`). + + :param referencing_table: Logical name of the child (many) table. + :type referencing_table: :class:`str` + :param lookup_field_name: Schema name for the lookup field. + :type lookup_field_name: :class:`str` + :param referenced_table: Logical name of the parent (one) table. + :type referenced_table: :class:`str` + :param display_name: Display name for the lookup field. + :type display_name: str or None + :param description: Optional description. + :type description: str or None + :param required: Whether the lookup is required. + :type required: :class:`bool` + :param cascade_delete: Delete cascade behaviour. + :type cascade_delete: :class:`str` + :param solution: Optional solution unique name. + :type solution: str or None + :param language_code: Language code for labels (default 1033). + :type language_code: :class:`int` + """ + self._batch._items.append( + _TableCreateLookupField( + referencing_table=referencing_table, + lookup_field_name=lookup_field_name, + referenced_table=referenced_table, + display_name=display_name, + description=description, + required=required, + cascade_delete=cascade_delete, + solution=solution, + language_code=language_code, + ) + ) + + +# --------------------------------------------------------------------------- +# BatchQueryOperations +# --------------------------------------------------------------------------- + + +class BatchQueryOperations: + """ + Query operations on a :class:`BatchRequest`. + + Mirrors ``client.query`` exactly: same method names, same signatures. + All methods return ``None``; results arrive via + :class:`~PowerPlatform.Dataverse.models.batch.BatchResult`. + + Do not instantiate directly; use ``batch.query``. + """ + + def __init__(self, batch: "BatchRequest") -> None: + self._batch = batch + + def sql(self, sql: str) -> None: + """ + Add a SQL SELECT query to the batch. + + Mirrors :meth:`~PowerPlatform.Dataverse.operations.query.QueryOperations.sql`. + The entity set is resolved from the table name in the SQL statement at + :meth:`BatchRequest.execute` time. + + :param sql: A single ``SELECT`` statement within the Dataverse-supported subset. + :type sql: :class:`str` + + :raises ~PowerPlatform.Dataverse.core.errors.ValidationError: + If ``sql`` is not a non-empty string. + + Example:: + + batch.query.sql("SELECT accountid, name FROM account WHERE name = 'Contoso'") + """ + if not isinstance(sql, str) or not sql.strip(): + raise ValidationError("sql must be a non-empty string", subcode=VALIDATION_SQL_EMPTY) + self._batch._items.append(_QuerySql(sql=sql.strip())) + + +# --------------------------------------------------------------------------- +# DataFrame batch operations +# --------------------------------------------------------------------------- + + +class BatchDataFrameOperations: + """DataFrame-oriented wrappers for batch record operations. + + Provides :meth:`create`, :meth:`update`, and :meth:`delete` that accept + ``pandas.DataFrame`` / ``pandas.Series`` inputs and convert them to standard + dicts before enqueueing on the batch. This lets data-science callers feed + DataFrames directly into a batch without manual conversion. + + Accessed via ``batch.dataframe``. + + Example:: + + import pandas as pd + + batch = client.batch.new() + df = pd.DataFrame([ + {"name": "Contoso", "telephone1": "555-0100"}, + {"name": "Fabrikam", "telephone1": "555-0200"}, + ]) + batch.dataframe.create("account", df) + result = batch.execute() + """ + + def __init__(self, batch: "BatchRequest") -> None: + self._batch = batch + + def create(self, table: str, records: pd.DataFrame) -> None: + """Enqueue record creates from a pandas DataFrame. + + Each row becomes a record. All rows are bundled in a single + ``CreateMultiple`` batch item (one HTTP request in the batch). + + :param table: Table schema name (e.g. ``"account"``). + :type table: :class:`str` + :param records: DataFrame where each row is a record to create. + :type records: ~pandas.DataFrame + + :raises TypeError: If ``records`` is not a pandas DataFrame. + :raises ValueError: If ``records`` is empty or any row has no non-null values. + + Example:: + + df = pd.DataFrame([{"name": "Contoso"}, {"name": "Fabrikam"}]) + batch.dataframe.create("account", df) + """ + if not isinstance(records, pd.DataFrame): + raise TypeError("records must be a pandas DataFrame") + if records.empty: + raise ValueError("records must be a non-empty DataFrame") + + from ..utils._pandas import dataframe_to_records + + record_list = dataframe_to_records(records) + empty_rows = [records.index[i] for i, r in enumerate(record_list) if not r] + if empty_rows: + raise ValueError( + f"Records at index(es) {empty_rows} have no non-null values. " + "All rows must contain at least one field to create." + ) + self._batch.records.create(table, record_list) + + def update( + self, + table: str, + changes: pd.DataFrame, + id_column: str, + clear_nulls: bool = False, + ) -> None: + """Enqueue record updates from a pandas DataFrame. + + Each row represents an update. The ``id_column`` specifies which + column contains the record GUIDs. + + :param table: Table schema name (e.g. ``"account"``). + :type table: :class:`str` + :param changes: DataFrame where each row contains a record GUID and + the fields to update. + :type changes: ~pandas.DataFrame + :param id_column: Name of the DataFrame column containing record GUIDs. + :type id_column: :class:`str` + :param clear_nulls: When ``False`` (default), NaN/None values are + skipped. When ``True``, NaN/None sends ``null`` to clear the field. + :type clear_nulls: :class:`bool` + + :raises TypeError: If ``changes`` is not a pandas DataFrame. + :raises ValueError: If ``changes`` is empty, ``id_column`` is missing, + or IDs are invalid. + + Example:: + + df = pd.DataFrame([ + {"accountid": "guid-1", "telephone1": "555-0100"}, + {"accountid": "guid-2", "telephone1": "555-0200"}, + ]) + batch.dataframe.update("account", df, id_column="accountid") + """ + if not isinstance(changes, pd.DataFrame): + raise TypeError("changes must be a pandas DataFrame") + if changes.empty: + raise ValueError("changes must be a non-empty DataFrame") + if id_column not in changes.columns: + raise ValueError(f"id_column '{id_column}' not found in DataFrame columns") + + raw_ids = changes[id_column].tolist() + invalid = [changes.index[i] for i, v in enumerate(raw_ids) if not isinstance(v, str) or not v.strip()] + if invalid: + raise ValueError( + f"id_column '{id_column}' contains invalid values at row index(es) {invalid}. " + "All IDs must be non-empty strings." + ) + ids = [v.strip() for v in raw_ids] + + change_columns = [c for c in changes.columns if c != id_column] + if not change_columns: + raise ValueError( + "No columns to update. The DataFrame must contain at least one column besides the id_column." + ) + + from ..utils._pandas import dataframe_to_records + + change_list = dataframe_to_records(changes[change_columns], na_as_null=clear_nulls) + paired = [(rid, patch) for rid, patch in zip(ids, change_list) if patch] + if not paired: + return + ids_filtered = [p[0] for p in paired] + change_filtered = [p[1] for p in paired] + + self._batch.records.update(table, ids_filtered, change_filtered) + + def delete( + self, + table: str, + ids: pd.Series, + use_bulk_delete: bool = True, + ) -> None: + """Enqueue record deletes from a pandas Series of GUIDs. + + :param table: Table schema name (e.g. ``"account"``). + :type table: :class:`str` + :param ids: Series of record GUIDs to delete. + :type ids: ~pandas.Series + :param use_bulk_delete: When ``True`` (default) and ``ids`` has multiple + values, use the ``BulkDelete`` action. + :type use_bulk_delete: :class:`bool` + + :raises TypeError: If ``ids`` is not a pandas Series. + :raises ValueError: If ``ids`` contains invalid values. + + Example:: + + ids_series = pd.Series(["guid-1", "guid-2", "guid-3"]) + batch.dataframe.delete("account", ids_series) + """ + if not isinstance(ids, pd.Series): + raise TypeError("ids must be a pandas Series") + raw_list = ids.tolist() + if not raw_list: + return + invalid = [ids.index[i] for i, v in enumerate(raw_list) if not isinstance(v, str) or not v.strip()] + if invalid: + raise ValueError(f"ids contains invalid values at index(es) {invalid}. All IDs must be non-empty strings.") + id_list = [v.strip() for v in raw_list] + self._batch.records.delete(table, id_list, use_bulk_delete=use_bulk_delete) + + +# --------------------------------------------------------------------------- +# BatchRequest and BatchOperations +# --------------------------------------------------------------------------- + + +class BatchRequest: + """ + Builder for constructing and executing a Dataverse OData ``$batch`` request. + + Obtain via :meth:`BatchOperations.new` (``client.batch.new()``). Add operations + through :attr:`records`, :attr:`tables`, :attr:`query`, and :attr:`dataframe`, + optionally group writes + into a :meth:`changeset`, then call :meth:`execute`. + + Operations are executed sequentially in the order added. The resulting + :class:`~PowerPlatform.Dataverse.models.batch.BatchResult` contains one + :class:`~PowerPlatform.Dataverse.models.batch.BatchItemResponse` per HTTP + request dispatched (some operations expand to multiple requests). + + .. note:: + Maximum 1000 HTTP operations per batch. + + Example:: + + batch = client.batch.new() + batch.records.create("account", {"name": "Contoso"}) + batch.tables.get("account") + with batch.changeset() as cs: + ref = cs.records.create("contact", {"firstname": "Alice"}) + cs.records.update("account", account_id, { + "primarycontactid@odata.bind": ref + }) + result = batch.execute() + """ + + def __init__(self, client: "DataverseClient") -> None: + self._client = client + self._items: List[Any] = [] + self._content_id_counter: List[int] = [1] # shared across all changesets + self.records = BatchRecordOperations(self) + self.tables = BatchTableOperations(self) + self.query = BatchQueryOperations(self) + self.dataframe = BatchDataFrameOperations(self) + + def changeset(self) -> ChangeSet: + """ + Create a new :class:`ChangeSet` attached to this batch. + + The changeset is added to the batch immediately. Operations added to + the returned :class:`ChangeSet` via ``cs.records.*`` execute atomically. + + :returns: A new :class:`ChangeSet` ready to receive operations. + + Example:: + + with batch.changeset() as cs: + cs.records.create("account", {"name": "ACME"}) + cs.records.create("contact", {"firstname": "Bob"}) + """ + internal = _ChangeSet(_counter=self._content_id_counter) + self._items.append(internal) + return ChangeSet(internal) + + def execute(self, *, continue_on_error: bool = False) -> BatchResult: + """ + Submit the batch to Dataverse and return all responses. + + :param continue_on_error: When False (default), Dataverse stops at the + first failure and returns that operation's error as a 4xx response. + When True, ``Prefer: odata.continue-on-error`` is sent and all + operations are attempted. + :returns: :class:`~PowerPlatform.Dataverse.models.batch.BatchResult` + with one entry per HTTP operation in submission order. + :raises ValidationError: If the batch exceeds 1000 operations or an + unsupported column type is specified. + :raises MetadataError: If metadata pre-resolution fails (table or + column not found) for ``tables.delete``, ``tables.add_columns``, + or ``tables.remove_columns``. + :raises HttpError: On HTTP-level failures (auth, server error, etc.) + that prevent the batch from executing. + """ + with self._client._scoped_odata() as od: + return _BatchClient(od).execute(self._items, continue_on_error=continue_on_error) + + +class BatchOperations: + """ + Namespace for batch operations (``client.batch``). + + Accessed via ``client.batch``. Use :meth:`new` to create a + :class:`BatchRequest` builder. + + :param client: The parent :class:`~PowerPlatform.Dataverse.client.DataverseClient` instance. + + Example:: + + batch = client.batch.new() + batch.records.create("account", {"name": "Fabrikam"}) + result = batch.execute() + """ + + def __init__(self, client: "DataverseClient") -> None: + self._client = client + + def new(self) -> BatchRequest: + """ + Create a new empty :class:`BatchRequest` builder. + + :returns: An empty :class:`BatchRequest`. + """ + return BatchRequest(self._client) diff --git a/src/PowerPlatform/Dataverse/operations/records.py b/src/PowerPlatform/Dataverse/operations/records.py index 7888a52a..ef867f52 100644 --- a/src/PowerPlatform/Dataverse/operations/records.py +++ b/src/PowerPlatform/Dataverse/operations/records.py @@ -548,7 +548,7 @@ def upsert(self, table: str, items: List[Union[UpsertItem, Dict[str, Any]]]) -> elif isinstance(i, dict) and isinstance(i.get("alternate_key"), dict) and isinstance(i.get("record"), dict): normalized.append(UpsertItem(alternate_key=i["alternate_key"], record=i["record"])) else: - raise TypeError("Each item must be a UpsertItem or a dict with 'alternate_key' and 'record' keys") + raise TypeError("Each item must be an UpsertItem or a dict with 'alternate_key' and 'record' keys") with self._client._scoped_odata() as od: entity_set = od._entity_set_from_schema_name(table) if len(normalized) == 1: diff --git a/src/PowerPlatform/Dataverse/operations/tables.py b/src/PowerPlatform/Dataverse/operations/tables.py index 729e0eba..e25c5a14 100644 --- a/src/PowerPlatform/Dataverse/operations/tables.py +++ b/src/PowerPlatform/Dataverse/operations/tables.py @@ -557,33 +557,17 @@ def create_lookup_field( ) print(f"Created lookup: {result['lookup_schema_name']}") """ - localized_labels = [ - LocalizedLabel( - label=display_name or referenced_table, + with self._client._scoped_odata() as od: + lookup, relationship = od._build_lookup_field_models( + referencing_table=referencing_table, + lookup_field_name=lookup_field_name, + referenced_table=referenced_table, + display_name=display_name, + description=description, + required=required, + cascade_delete=cascade_delete, language_code=language_code, ) - ] - - lookup = LookupAttributeMetadata( - schema_name=lookup_field_name, - display_name=Label(localized_labels=localized_labels), - required_level="ApplicationRequired" if required else "None", - ) - - if description: - lookup.description = Label( - localized_labels=[LocalizedLabel(label=description, language_code=language_code)] - ) - - relationship_name = f"{referenced_table}_{referencing_table}_{lookup_field_name}" - - relationship = OneToManyRelationshipMetadata( - schema_name=relationship_name, - referenced_entity=referenced_table, - referencing_entity=referencing_table, - referenced_attribute=f"{referenced_table}id", - cascade_configuration=CascadeConfiguration(delete=cascade_delete), - ) return self.create_one_to_many_relationship(lookup, relationship, solution=solution) diff --git a/tests/unit/data/test_batch_edge_cases.py b/tests/unit/data/test_batch_edge_cases.py new file mode 100644 index 00000000..01b4d629 --- /dev/null +++ b/tests/unit/data/test_batch_edge_cases.py @@ -0,0 +1,1069 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Edge case and corner scenario tests for batch operations. + +Covers OData $batch spec compliance, error handling, and scenarios +derived from the Dataverse Web API public documentation: +https://learn.microsoft.com/en-us/power-apps/developer/data-platform/webapi/execute-batch-operations-using-web-api +""" + +import json +import unittest +from unittest.mock import MagicMock + +from PowerPlatform.Dataverse.data._batch import ( + _BatchClient, + _ChangeSet, + _ChangeSetBatchItem, + _RecordDelete, + _RecordGet, + _extract_boundary, + _raise_top_level_batch_error, + _split_multipart, + _parse_http_response_part, + _CRLF, + _MAX_BATCH_SIZE, +) +from PowerPlatform.Dataverse.core.errors import HttpError, ValidationError +from PowerPlatform.Dataverse.data._raw_request import _RawRequest +from PowerPlatform.Dataverse.models.batch import BatchItemResponse, BatchResult + + +def _make_od(): + """Return a minimal mock _ODataClient.""" + od = MagicMock() + od.api = "https://org.crm.dynamics.com/api/data/v9.2" + return od + + +# --------------------------------------------------------------------------- +# 1. Empty changeset handling +# --------------------------------------------------------------------------- + + +class TestEmptyChangeset(unittest.TestCase): + """An empty changeset (no operations) should be silently skipped.""" + + def test_empty_changeset_skipped_in_resolve(self): + """_resolve_all skips empty changesets rather than producing empty multipart parts.""" + od = _make_od() + client = _BatchClient(od) + cs = _ChangeSet() # no operations + # Also include a non-changeset item to ensure the batch is not entirely empty + get = _RecordGet(table="account", record_id="guid-1") + od._build_get.return_value = _RawRequest(method="GET", url="https://org/api/data/v9.2/accounts(guid-1)") + resolved = client._resolve_all([cs, get]) + # Should only have the GET, no changeset + self.assertEqual(len(resolved), 1) + self.assertIsInstance(resolved[0], _RawRequest) + + def test_empty_changeset_only_batch_returns_empty_result(self): + """A batch with only empty changesets has no items and returns empty BatchResult.""" + od = _make_od() + client = _BatchClient(od) + cs = _ChangeSet() + resolved = client._resolve_all([cs]) + self.assertEqual(len(resolved), 0) + + def test_changeset_with_operations_not_skipped(self): + """A changeset with operations is not skipped.""" + od = _make_od() + client = _BatchClient(od) + cs = _ChangeSet() + cs.add_create("account", {"name": "Test"}) + req = _RawRequest(method="POST", url="https://org/api/data/v9.2/accounts", body='{"name":"Test"}') + od._build_create.return_value = req + resolved = client._resolve_all([cs]) + self.assertEqual(len(resolved), 1) + self.assertIsInstance(resolved[0], _ChangeSetBatchItem) + self.assertEqual(len(resolved[0].requests), 1) + + +# --------------------------------------------------------------------------- +# 2. Changeset error/rollback response parsing +# --------------------------------------------------------------------------- + + +class TestChangeSetRollbackResponse(unittest.TestCase): + """When a changeset fails, Dataverse returns a single error for the entire changeset.""" + + def test_changeset_error_parsed_as_failed_item(self): + """A changeset failure returns one response in the inner changeset boundary.""" + # Simulate a batch response where a changeset within it returned an error + cs_error_body = json.dumps( + { + "error": { + "code": "0x80040237", + "message": "A record with matching key values already exists.", + } + } + ) + inner_response = ( + "HTTP/1.1 409 Conflict\r\n" + "Content-Type: application/json; odata.metadata=minimal\r\n" + "OData-Version: 4.0\r\n" + "\r\n" + f"{cs_error_body}" + ) + cs_boundary = "changesetresponse_abc123" + inner_multipart = ( + f"--{cs_boundary}\r\n" + "Content-Type: application/http\r\n" + "Content-Transfer-Encoding: binary\r\n" + "Content-ID: 1\r\n" + "\r\n" + f"{inner_response}\r\n" + f"--{cs_boundary}--\r\n" + ) + batch_boundary = "batchresponse_xyz789" + full_response = ( + f"--{batch_boundary}\r\n" + f'Content-Type: multipart/mixed; boundary="{cs_boundary}"\r\n' + "\r\n" + f"{inner_multipart}\r\n" + f"--{batch_boundary}--\r\n" + ) + + mock_response = MagicMock() + mock_response.headers = {"Content-Type": f'multipart/mixed; boundary="{batch_boundary}"'} + mock_response.text = full_response + + od = _make_od() + client = _BatchClient(od) + result = client._parse_batch_response(mock_response) + + self.assertTrue(result.has_errors) + self.assertEqual(len(result.failed), 1) + self.assertEqual(result.failed[0].status_code, 409) + self.assertEqual(result.failed[0].error_code, "0x80040237") + self.assertIn("matching key", result.failed[0].error_message) + + def test_successful_changeset_returns_all_items(self): + """A successful changeset returns 204 per create operation.""" + cs_boundary = "changesetresponse_ok123" + inner = "" + for i in range(1, 4): + guid = f"0000000{i}-0000-0000-0000-000000000000" + inner += ( + f"--{cs_boundary}\r\n" + "Content-Type: application/http\r\n" + "Content-Transfer-Encoding: binary\r\n" + f"Content-ID: {i}\r\n" + "\r\n" + "HTTP/1.1 204 No Content\r\n" + "OData-Version: 4.0\r\n" + f"OData-EntityId: https://org.crm.dynamics.com/api/data/v9.2/tasks({guid})\r\n" + "\r\n" + "\r\n" + ) + inner += f"--{cs_boundary}--\r\n" + + batch_boundary = "batchresponse_good789" + full = ( + f"--{batch_boundary}\r\n" + f'Content-Type: multipart/mixed; boundary="{cs_boundary}"\r\n' + "\r\n" + f"{inner}\r\n" + f"--{batch_boundary}--\r\n" + ) + + mock_response = MagicMock() + mock_response.headers = {"Content-Type": f'multipart/mixed; boundary="{batch_boundary}"'} + mock_response.text = full + + od = _make_od() + client = _BatchClient(od) + result = client._parse_batch_response(mock_response) + + self.assertFalse(result.has_errors) + self.assertEqual(len(result.succeeded), 3) + self.assertEqual(len(result.entity_ids), 3) + # Verify content-IDs were extracted + content_ids = [r.content_id for r in result.responses] + self.assertEqual(content_ids, ["1", "2", "3"]) + + +# --------------------------------------------------------------------------- +# 3. Content-ID in non-changeset response parts +# --------------------------------------------------------------------------- + + +class TestContentIdInStandaloneParts(unittest.TestCase): + """Non-changeset parts in the batch response can have content-id headers.""" + + def test_standalone_part_content_id_extracted(self): + """Content-ID from standalone (non-changeset) MIME headers is propagated.""" + batch_boundary = "batchresponse_solo123" + response_text = ( + f"--{batch_boundary}\r\n" + "Content-Type: application/http\r\n" + "Content-Transfer-Encoding: binary\r\n" + "Content-ID: 42\r\n" + "\r\n" + "HTTP/1.1 204 No Content\r\n" + "OData-Version: 4.0\r\n" + "OData-EntityId: https://org.crm.dynamics.com/api/data/v9.2/accounts(11111111-1111-1111-1111-111111111111)\r\n" + "\r\n" + "\r\n" + f"--{batch_boundary}--\r\n" + ) + + mock_response = MagicMock() + mock_response.headers = {"Content-Type": f'multipart/mixed; boundary="{batch_boundary}"'} + mock_response.text = response_text + + od = _make_od() + client = _BatchClient(od) + result = client._parse_batch_response(mock_response) + + self.assertEqual(len(result.responses), 1) + self.assertEqual(result.responses[0].content_id, "42") + + +# --------------------------------------------------------------------------- +# 4. Mixed batch: changesets + standalone GETs +# --------------------------------------------------------------------------- + + +class TestMixedBatch(unittest.TestCase): + """Batch with both changeset writes and standalone reads.""" + + def test_changeset_plus_standalone_get_parsed(self): + """Response with changeset (204s) followed by standalone GET (200 with body).""" + cs_boundary = "changesetresponse_mix1" + cs_part = ( + f"--{cs_boundary}\r\n" + "Content-Type: application/http\r\n" + "Content-Transfer-Encoding: binary\r\n" + "Content-ID: 1\r\n" + "\r\n" + "HTTP/1.1 204 No Content\r\n" + "OData-EntityId: https://org.crm.dynamics.com/api/data/v9.2/tasks(aaaaaaaa-0000-0000-0000-000000000000)\r\n" + "\r\n" + f"--{cs_boundary}--\r\n" + ) + + get_body = json.dumps({"@odata.context": "...", "value": [{"name": "Contoso"}]}) + batch_boundary = "batchresponse_mixed123" + full = ( + f"--{batch_boundary}\r\n" + f'Content-Type: multipart/mixed; boundary="{cs_boundary}"\r\n' + "\r\n" + f"{cs_part}\r\n" + f"--{batch_boundary}\r\n" + "Content-Type: application/http\r\n" + "Content-Transfer-Encoding: binary\r\n" + "\r\n" + "HTTP/1.1 200 OK\r\n" + "Content-Type: application/json; odata.metadata=minimal\r\n" + "OData-Version: 4.0\r\n" + "\r\n" + f"{get_body}\r\n" + f"--{batch_boundary}--\r\n" + ) + + mock_response = MagicMock() + mock_response.headers = {"Content-Type": f'multipart/mixed; boundary="{batch_boundary}"'} + mock_response.text = full + + od = _make_od() + client = _BatchClient(od) + result = client._parse_batch_response(mock_response) + + self.assertEqual(len(result.responses), 2) + # First: changeset create (204) + self.assertEqual(result.responses[0].status_code, 204) + self.assertEqual(result.responses[0].entity_id, "aaaaaaaa-0000-0000-0000-000000000000") + # Second: standalone GET (200 with body data) + self.assertEqual(result.responses[1].status_code, 200) + self.assertIsNotNone(result.responses[1].data) + + +# --------------------------------------------------------------------------- +# 5. Multiple changesets in one batch +# --------------------------------------------------------------------------- + + +class TestMultipleChangesets(unittest.TestCase): + """Batch with multiple changesets — content IDs must be globally unique.""" + + def test_two_changesets_unique_content_ids(self): + """Two changesets in the same batch get unique content IDs.""" + counter = [1] + cs1 = _ChangeSet(_counter=counter) + cs2 = _ChangeSet(_counter=counter) + + ref1 = cs1.add_create("account", {"name": "A"}) + ref2 = cs1.add_create("account", {"name": "B"}) + ref3 = cs2.add_create("contact", {"firstname": "C"}) + ref4 = cs2.add_update("contact", ref3, {"lastname": "D"}) + + self.assertEqual(ref1, "$1") + self.assertEqual(ref2, "$2") + self.assertEqual(ref3, "$3") + # Counter should now be at 5 + self.assertEqual(counter[0], 5) + + # All content IDs across both changesets must be unique + all_cids = [op.content_id for op in cs1.operations + cs2.operations] + self.assertEqual(len(all_cids), len(set(all_cids))) + + +# --------------------------------------------------------------------------- +# 6. Batch size limit with mixed changesets +# --------------------------------------------------------------------------- + + +class TestBatchSizeLimitMixed(unittest.TestCase): + """Max 1000 operations counting across changesets and standalone items.""" + + def test_changeset_ops_counted_toward_limit(self): + """Operations inside changesets count toward the 1000 limit.""" + od = _make_od() + client = _BatchClient(od) + # 999 standalone + 2 in a changeset = 1001 > 1000 + cs = _ChangeSet() + cs.add_create("a", {"name": "x"}) + cs.add_create("a", {"name": "y"}) + + items = [_RecordGet(table="account", record_id=f"guid-{i}") for i in range(999)] + items.append(cs) + + od._build_get.return_value = _RawRequest(method="GET", url="https://org/api/data/v9.2/accounts(x)") + od._build_create.return_value = _RawRequest(method="POST", url="https://org/api/data/v9.2/accounts", body="{}") + + with self.assertRaises(ValidationError) as ctx: + client.execute(items) + self.assertIn("1001", str(ctx.exception)) + self.assertIn("1000", str(ctx.exception)) + + def test_exactly_1000_operations_allowed(self): + """Exactly 1000 operations should not raise.""" + od = _make_od() + client = _BatchClient(od) + + items = [_RecordGet(table="account", record_id=f"guid-{i}") for i in range(1000)] + + od._build_get.return_value = _RawRequest(method="GET", url="https://org/api/data/v9.2/accounts(x)") + mock_resp = MagicMock() + mock_resp.headers = {"Content-Type": 'multipart/mixed; boundary="resp_bnd"'} + mock_resp.text = "--resp_bnd--\r\n" + od._request.return_value = mock_resp + + # Should not raise + result = client.execute(items) + self.assertIsInstance(result, BatchResult) + + +# --------------------------------------------------------------------------- +# 7. Top-level batch error handling +# --------------------------------------------------------------------------- + + +class TestTopLevelBatchError(unittest.TestCase): + """When Dataverse rejects the batch request itself (non-multipart response).""" + + def test_json_error_body_raised_as_http_error(self): + """A 400 with JSON error body raises HttpError with the message.""" + mock_resp = MagicMock() + mock_resp.status_code = 400 + mock_resp.json.return_value = { + "error": { + "code": "0x80048d19", + "message": "The batch request must have Content-Type multipart/mixed.", + } + } + mock_resp.text = json.dumps(mock_resp.json.return_value) + + with self.assertRaises(HttpError) as ctx: + _raise_top_level_batch_error(mock_resp) + self.assertEqual(ctx.exception.status_code, 400) + self.assertIn("multipart/mixed", str(ctx.exception)) + + def test_non_json_body_raised_with_text(self): + """A 500 with non-JSON body raises HttpError with the raw text.""" + mock_resp = MagicMock() + mock_resp.status_code = 500 + mock_resp.json.side_effect = ValueError("not JSON") + mock_resp.text = "Internal Server Error" + + with self.assertRaises(HttpError) as ctx: + _raise_top_level_batch_error(mock_resp) + self.assertEqual(ctx.exception.status_code, 500) + self.assertIn("Internal Server Error", str(ctx.exception)) + + def test_empty_body_raises_generic_error(self): + """A 503 with empty body raises HttpError with a generic message.""" + mock_resp = MagicMock() + mock_resp.status_code = 503 + mock_resp.json.side_effect = ValueError("empty") + mock_resp.text = "" + + with self.assertRaises(HttpError) as ctx: + _raise_top_level_batch_error(mock_resp) + self.assertEqual(ctx.exception.status_code, 503) + + def test_error_code_preserved_as_service_error_code(self): + """The error.code field is preserved in service_error_code.""" + mock_resp = MagicMock() + mock_resp.status_code = 403 + mock_resp.json.return_value = { + "error": {"code": "0x80040220", "message": "Principal user is missing privileges."} + } + mock_resp.text = "" + + with self.assertRaises(HttpError) as ctx: + _raise_top_level_batch_error(mock_resp) + self.assertEqual(ctx.exception.details.get("service_error_code"), "0x80040220") + + +# --------------------------------------------------------------------------- +# 8. Batch response without continue-on-error (first failure stops) +# --------------------------------------------------------------------------- + + +class TestBatchWithoutContinueOnError(unittest.TestCase): + """Without Prefer: odata.continue-on-error, first failure stops the batch.""" + + def test_single_error_response_parsed(self): + """A 400 batch response with a single error in multipart body.""" + error_body = json.dumps( + { + "error": { + "code": "0x80044331", + "message": "The length of the 'subject' attribute exceeded the maximum allowed length of '200'.", + } + } + ) + batch_boundary = "batchresponse_err123" + body = ( + f"--{batch_boundary}\r\n" + "Content-Type: application/http\r\n" + "Content-Transfer-Encoding: binary\r\n" + "\r\n" + "HTTP/1.1 400 Bad Request\r\n" + "Content-Type: application/json; odata.metadata=minimal\r\n" + "OData-Version: 4.0\r\n" + "\r\n" + f"{error_body}\r\n" + f"--{batch_boundary}--\r\n" + ) + + mock_response = MagicMock() + mock_response.headers = {"Content-Type": f'multipart/mixed; boundary="{batch_boundary}"'} + mock_response.text = body + + od = _make_od() + client = _BatchClient(od) + result = client._parse_batch_response(mock_response) + + self.assertTrue(result.has_errors) + self.assertEqual(len(result.responses), 1) + self.assertEqual(result.responses[0].status_code, 400) + self.assertEqual(result.responses[0].error_code, "0x80044331") + + +# --------------------------------------------------------------------------- +# 9. Batch with continue-on-error: mixed success/failure +# --------------------------------------------------------------------------- + + +class TestBatchContinueOnError(unittest.TestCase): + """With continue-on-error, successful items are returned alongside failures.""" + + def test_mixed_success_and_failure(self): + """One 400 error + two 204 successes parsed correctly.""" + error_body = json.dumps({"error": {"code": "0x80040237", "message": "record not found"}}) + batch_boundary = "batchresponse_coe123" + body = ( + f"--{batch_boundary}\r\n" + "Content-Type: application/http\r\n" + "Content-Transfer-Encoding: binary\r\n" + "\r\n" + "HTTP/1.1 400 Bad Request\r\n" + "Content-Type: application/json\r\n" + "\r\n" + f"{error_body}\r\n" + f"--{batch_boundary}\r\n" + "Content-Type: application/http\r\n" + "Content-Transfer-Encoding: binary\r\n" + "\r\n" + "HTTP/1.1 204 No Content\r\n" + "OData-EntityId: https://org.crm.dynamics.com/api/data/v9.2/tasks(11111111-1111-1111-1111-111111111111)\r\n" + "\r\n" + f"--{batch_boundary}\r\n" + "Content-Type: application/http\r\n" + "Content-Transfer-Encoding: binary\r\n" + "\r\n" + "HTTP/1.1 204 No Content\r\n" + "OData-EntityId: https://org.crm.dynamics.com/api/data/v9.2/tasks(22222222-2222-2222-2222-222222222222)\r\n" + "\r\n" + f"--{batch_boundary}--\r\n" + ) + + mock_response = MagicMock() + mock_response.headers = {"Content-Type": f'multipart/mixed; boundary="{batch_boundary}"'} + mock_response.text = body + + od = _make_od() + client = _BatchClient(od) + result = client._parse_batch_response(mock_response) + + self.assertTrue(result.has_errors) + self.assertEqual(len(result.succeeded), 2) + self.assertEqual(len(result.failed), 1) + self.assertEqual(result.failed[0].error_code, "0x80040237") + self.assertEqual(len(result.entity_ids), 2) + + +# --------------------------------------------------------------------------- +# 10. Serialization spec compliance +# --------------------------------------------------------------------------- + + +class TestSerializationCompliance(unittest.TestCase): + """Verify serialized batch body matches OData $batch spec requirements.""" + + def _client(self): + od = _make_od() + return _BatchClient(od) + + def test_crlf_line_endings_in_batch_body(self): + """All line endings in the batch body must be CRLF per OData spec.""" + client = self._client() + req = _RawRequest(method="GET", url="https://org/api/data/v9.2/accounts(guid)") + part = client._serialize_raw_request(req, "batch_test") + # Every newline should be CRLF + lines = part.split("\r\n") + self.assertGreater(len(lines), 1) + # Ensure no bare LFs + for line in lines: + self.assertNotIn("\n", line.rstrip("\n")) + + def test_content_transfer_encoding_binary(self): + """Each batch part must include Content-Transfer-Encoding: binary.""" + client = self._client() + req = _RawRequest(method="POST", url="https://org/api/data/v9.2/accounts", body='{"name":"x"}') + part = client._serialize_raw_request(req, "batch_test") + self.assertIn("Content-Transfer-Encoding: binary", part) + + def test_content_type_application_http(self): + """Each batch part must include Content-Type: application/http.""" + client = self._client() + req = _RawRequest(method="GET", url="https://org/api/data/v9.2/accounts(guid)") + part = client._serialize_raw_request(req, "batch_test") + self.assertIn("Content-Type: application/http", part) + + def test_batch_body_ends_with_closing_boundary(self): + """Batch body must end with --{boundary}-- terminator.""" + client = self._client() + resolved = [_RawRequest(method="GET", url="https://org/api/data/v9.2/accounts(guid)")] + body = client._build_batch_body(resolved, "batch_end_test") + self.assertTrue(body.strip().endswith("--batch_end_test--")) + + def test_changeset_nested_boundary_different_from_batch(self): + """Changeset uses a different boundary than the batch.""" + client = self._client() + cs = _ChangeSetBatchItem( + requests=[ + _RawRequest(method="POST", url="https://org/api/data/v9.2/accounts", body="{}", content_id=1), + ] + ) + body = client._build_batch_body([cs], "batch_outer") + # Should contain both batch_outer and changeset_ + self.assertIn("--batch_outer", body) + self.assertIn("changeset_", body) + + def test_post_body_has_content_type_json_with_type_entry(self): + """POST/PATCH body parts include Content-Type: application/json; type=entry.""" + client = self._client() + req = _RawRequest(method="POST", url="https://org/api/data/v9.2/accounts", body='{"name":"x"}') + part = client._serialize_raw_request(req, "bnd") + self.assertIn("Content-Type: application/json; type=entry", part) + + def test_absolute_urls_in_batch_parts(self): + """Batch parts use absolute URLs (required by Dataverse).""" + client = self._client() + url = "https://org.crm.dynamics.com/api/data/v9.2/accounts(guid)" + req = _RawRequest(method="GET", url=url) + part = client._serialize_raw_request(req, "bnd") + self.assertIn(f"GET {url} HTTP/1.1", part) + + +# --------------------------------------------------------------------------- +# 11. BatchResult computed properties +# --------------------------------------------------------------------------- + + +class TestBatchResultProperties(unittest.TestCase): + """Verify computed properties of BatchResult.""" + + def test_entity_ids_only_from_201_and_204(self): + """entity_ids should include entity_ids from all 2xx (including 201 and 204).""" + responses = [ + BatchItemResponse(status_code=201, entity_id="id-201"), + BatchItemResponse(status_code=204, entity_id="id-204"), + BatchItemResponse(status_code=200, entity_id=None), # GET success, no entity_id + BatchItemResponse(status_code=400, entity_id=None), + ] + result = BatchResult(responses=responses) + self.assertEqual(result.entity_ids, ["id-201", "id-204"]) + + def test_empty_batch_result_properties(self): + """Empty BatchResult has correct defaults.""" + result = BatchResult() + self.assertEqual(result.succeeded, []) + self.assertEqual(result.failed, []) + self.assertFalse(result.has_errors) + self.assertEqual(result.entity_ids, []) + + def test_all_success_no_errors(self): + """All 2xx responses means has_errors is False.""" + responses = [ + BatchItemResponse(status_code=200), + BatchItemResponse(status_code=204), + ] + result = BatchResult(responses=responses) + self.assertFalse(result.has_errors) + self.assertEqual(len(result.succeeded), 2) + self.assertEqual(len(result.failed), 0) + + def test_single_failure_makes_has_errors_true(self): + """Even one 4xx/5xx makes has_errors True.""" + responses = [ + BatchItemResponse(status_code=200), + BatchItemResponse(status_code=404, error_message="not found"), + ] + result = BatchResult(responses=responses) + self.assertTrue(result.has_errors) + self.assertEqual(len(result.failed), 1) + + def test_entity_ids_from_create_multiple_not_in_created_ids(self): + """CreateMultiple IDs are in data['Ids'], NOT in entity_ids property. + + entity_ids only returns entity_id from OData-EntityId headers. + Callers access CreateMultiple IDs via response.data['Ids'] directly. + """ + responses = [ + # CreateMultiple response: 200 OK with {"Ids": [...]} body + BatchItemResponse( + status_code=200, + entity_id=None, + data={"Ids": ["guid-1", "guid-2", "guid-3"]}, + ), + ] + result = BatchResult(responses=responses) + # entity_ids does NOT include bulk IDs (no OData-EntityId header) + self.assertEqual(result.entity_ids, []) + # Callers access them from the response data + self.assertEqual(result.succeeded[0].data["Ids"], ["guid-1", "guid-2", "guid-3"]) + + def test_entity_ids_only_from_odata_entity_id_header(self): + """entity_ids only collects entity_id from OData-EntityId headers.""" + responses = [ + # Single create: entity_id from header + BatchItemResponse(status_code=204, entity_id="single-id"), + # CreateMultiple: Ids from body (not in entity_ids) + BatchItemResponse( + status_code=200, + entity_id=None, + data={"Ids": ["bulk-id-1", "bulk-id-2"]}, + ), + ] + result = BatchResult(responses=responses) + # Only the header-based entity_id + self.assertEqual(result.entity_ids, ["single-id"]) + # Bulk IDs accessed via response.data + self.assertEqual(result.responses[1].data["Ids"], ["bulk-id-1", "bulk-id-2"]) + + def test_bulk_ids_accessible_via_response_data(self): + """Callers iterate responses to access CreateMultiple/UpsertMultiple IDs.""" + responses = [ + BatchItemResponse(status_code=204, entity_id="id-1"), + BatchItemResponse( + status_code=200, + data={"Ids": ["id-2", "id-3"]}, + ), + BatchItemResponse(status_code=204), # delete, no entity_id + ] + result = BatchResult(responses=responses) + # Collect all IDs from both sources (what a caller would do) + all_ids = list(result.entity_ids) + for resp in result.succeeded: + if resp.data and isinstance(resp.data.get("Ids"), list): + all_ids.extend(resp.data["Ids"]) + self.assertEqual(all_ids, ["id-1", "id-2", "id-3"]) + + +# --------------------------------------------------------------------------- +# 12. CreateMultiple response parsing in batch +# --------------------------------------------------------------------------- + + +class TestCreateMultipleInBatch(unittest.TestCase): + """CreateMultiple action returns 200 with {Ids: [...]} in the body.""" + + def test_create_multiple_response_parsed(self): + """A 200 OK CreateMultiple response has IDs in the body, not in headers.""" + ids_body = json.dumps({"Ids": ["aaa-111", "bbb-222", "ccc-333"]}) + batch_boundary = "batchresponse_cm123" + resp_text = ( + f"--{batch_boundary}\r\n" + "Content-Type: application/http\r\n" + "Content-Transfer-Encoding: binary\r\n" + "\r\n" + "HTTP/1.1 200 OK\r\n" + "Content-Type: application/json; odata.metadata=minimal\r\n" + "OData-Version: 4.0\r\n" + "\r\n" + f"{ids_body}\r\n" + f"--{batch_boundary}--\r\n" + ) + + mock_response = MagicMock() + mock_response.headers = {"Content-Type": f'multipart/mixed; boundary="{batch_boundary}"'} + mock_response.text = resp_text + + od = _make_od() + client = _BatchClient(od) + result = client._parse_batch_response(mock_response) + + self.assertFalse(result.has_errors) + self.assertEqual(len(result.succeeded), 1) + # entity_id should be None (no OData-EntityId header for CreateMultiple) + self.assertIsNone(result.succeeded[0].entity_id) + # But data should contain the Ids array + self.assertEqual(result.succeeded[0].data["Ids"], ["aaa-111", "bbb-222", "ccc-333"]) + # created_ids won't have these (no OData-EntityId header) + self.assertEqual(result.entity_ids, []) + # Callers access bulk IDs via response.data["Ids"] + bulk_ids = result.succeeded[0].data["Ids"] + self.assertEqual(len(bulk_ids), 3) + + def test_mixed_single_and_bulk_creates(self): + """Batch with both individual POST create and CreateMultiple.""" + single_guid = "11111111-1111-1111-1111-111111111111" + ids_body = json.dumps({"Ids": ["bulk-1", "bulk-2"]}) + batch_boundary = "batchresponse_mix_cm" + resp_text = ( + # Individual create: 204 with OData-EntityId + f"--{batch_boundary}\r\n" + "Content-Type: application/http\r\n" + "Content-Transfer-Encoding: binary\r\n" + "\r\n" + "HTTP/1.1 204 No Content\r\n" + "OData-Version: 4.0\r\n" + f"OData-EntityId: https://org.crm.dynamics.com/api/data/v9.2/" + f"accounts({single_guid})\r\n" + "\r\n" + "\r\n" + # CreateMultiple: 200 with Ids body + f"--{batch_boundary}\r\n" + "Content-Type: application/http\r\n" + "Content-Transfer-Encoding: binary\r\n" + "\r\n" + "HTTP/1.1 200 OK\r\n" + "Content-Type: application/json\r\n" + "\r\n" + f"{ids_body}\r\n" + f"--{batch_boundary}--\r\n" + ) + + mock_response = MagicMock() + mock_response.headers = {"Content-Type": f'multipart/mixed; boundary="{batch_boundary}"'} + mock_response.text = resp_text + + od = _make_od() + client = _BatchClient(od) + result = client._parse_batch_response(mock_response) + + self.assertFalse(result.has_errors) + self.assertEqual(len(result.succeeded), 2) + # entity_ids only has the individual create's entity_id + self.assertEqual(result.entity_ids, [single_guid]) + # CreateMultiple IDs are in the second response's data + self.assertEqual(result.responses[1].data["Ids"], ["bulk-1", "bulk-2"]) + + +# --------------------------------------------------------------------------- +# 13. Multipart parsing edge cases +# --------------------------------------------------------------------------- + + +class TestMultipartParsingEdgeCases(unittest.TestCase): + """Edge cases in multipart response parsing.""" + + def test_response_with_only_closing_boundary(self): + """A response body with only the closing boundary produces no parts.""" + parts = _split_multipart("--bnd--\r\n", "bnd") + self.assertEqual(len(parts), 0) + + def test_response_with_extra_whitespace_in_parts(self): + """Parts with extra whitespace/blank lines should still parse.""" + body = ( + "--bnd\r\n" + "Content-Type: application/http\r\n" + "\r\n" + "HTTP/1.1 200 OK\r\n" + "Content-Type: application/json\r\n" + "\r\n" + '{"value":[]}\r\n' + "--bnd--\r\n" + ) + parts = _split_multipart(body, "bnd") + self.assertEqual(len(parts), 1) + + def test_parse_response_with_req_id_header(self): + """Dataverse error responses include REQ_ID header — should not break parsing.""" + text = ( + "HTTP/1.1 400 Bad Request\r\n" + "REQ_ID: 5ecd1cb3-1730-4ffc-909c-d44c22270026\r\n" + "Content-Type: application/json; odata.metadata=minimal\r\n" + "OData-Version: 4.0\r\n" + "\r\n" + '{"error":{"code":"0x80044331","message":"validation error"}}' + ) + item = _parse_http_response_part(text, content_id=None) + self.assertIsNotNone(item) + self.assertEqual(item.status_code, 400) + self.assertEqual(item.error_code, "0x80044331") + self.assertEqual(item.error_message, "validation error") + + def test_entity_id_extracted_from_various_guid_formats(self): + """GUID extraction works with different formats.""" + # Standard UUID + text = ( + "HTTP/1.1 204 No Content\r\n" + "OData-EntityId: https://org.crm.dynamics.com/api/data/v9.2/" + "accounts(a1b2c3d4-e5f6-7890-abcd-ef1234567890)\r\n" + "\r\n" + ) + item = _parse_http_response_part(text, content_id=None) + self.assertEqual(item.entity_id, "a1b2c3d4-e5f6-7890-abcd-ef1234567890") + + def test_no_entity_id_for_delete_response(self): + """Delete responses typically have no OData-EntityId.""" + text = "HTTP/1.1 204 No Content\r\n\r\n" + item = _parse_http_response_part(text, content_id=None) + self.assertIsNone(item.entity_id) + + def test_get_response_body_parsed_as_data(self): + """A 200 OK GET response should have body parsed into data.""" + body_data = {"@odata.context": "...", "name": "Contoso", "accountid": "guid-1"} + text = ( + "HTTP/1.1 200 OK\r\n" + "Content-Type: application/json; odata.metadata=minimal\r\n" + "\r\n" + f"{json.dumps(body_data)}" + ) + item = _parse_http_response_part(text, content_id=None) + self.assertEqual(item.status_code, 200) + self.assertEqual(item.data["name"], "Contoso") + self.assertIsNone(item.error_message) + + +# --------------------------------------------------------------------------- +# 13. Changeset content-ID reference validation +# --------------------------------------------------------------------------- + + +class TestContentIdReferences(unittest.TestCase): + """Content-ID references ($n) in changesets.""" + + def test_content_id_ref_format(self): + """add_create returns $n format string starting from 1.""" + cs = _ChangeSet() + ref = cs.add_create("account", {"name": "Test"}) + self.assertEqual(ref, "$1") + self.assertTrue(ref.startswith("$")) + + def test_content_id_usable_in_odata_bind(self): + """Content-ID reference can be used in @odata.bind field.""" + cs = _ChangeSet() + lead_ref = cs.add_create("lead", {"firstname": "Ada"}) + cs.add_create( + "account", + {"name": "Babbage", "originatingleadid@odata.bind": lead_ref}, + ) + # The second create should have the ref in its data + self.assertEqual(cs.operations[1].data["originatingleadid@odata.bind"], "$1") + + def test_content_id_usable_as_record_id_in_update(self): + """Content-ID reference can be used as record_id for update.""" + cs = _ChangeSet() + ref = cs.add_create("contact", {"firstname": "Alice"}) + cs.add_update("contact", ref, {"lastname": "Smith"}) + # The update should use the ref as record_id + self.assertEqual(cs.operations[1].ids, "$1") + + def test_content_id_usable_as_record_id_in_delete(self): + """Content-ID reference can be used as record_id for delete.""" + cs = _ChangeSet() + ref = cs.add_create("temp", {"name": "Delete me"}) + cs.add_delete("temp", ref) + self.assertEqual(cs.operations[1].ids, "$1") + + +# --------------------------------------------------------------------------- +# 14. Intent type validation +# --------------------------------------------------------------------------- + + +class TestIntentValidation(unittest.TestCase): + """_resolve_item rejects unknown types.""" + + def test_unknown_type_raises_validation_error(self): + """An unsupported item type raises ValidationError.""" + od = _make_od() + client = _BatchClient(od) + + with self.assertRaises(ValidationError): + client._resolve_item("not a valid intent type") + + def test_none_item_raises_validation_error(self): + """None as an item type raises ValidationError.""" + od = _make_od() + client = _BatchClient(od) + + with self.assertRaises(ValidationError): + client._resolve_item(None) + + +# --------------------------------------------------------------------------- +# 15. Batch boundary format +# --------------------------------------------------------------------------- + + +class TestBatchBoundaryFormat(unittest.TestCase): + """Boundary identifiers should be unique and follow batch_ prefix convention.""" + + def test_batch_boundary_in_content_type(self): + """execute() sets Content-Type with batch_ prefixed boundary.""" + od = _make_od() + client = _BatchClient(od) + + items = [_RecordGet(table="account", record_id="guid-1")] + od._build_get.return_value = _RawRequest(method="GET", url="https://org/api/data/v9.2/accounts(guid-1)") + + mock_resp = MagicMock() + mock_resp.headers = {"Content-Type": 'multipart/mixed; boundary="resp_bnd"'} + mock_resp.text = "--resp_bnd--\r\n" + od._request.return_value = mock_resp + + client.execute(items) + + # Verify the Content-Type header sent in the POST + call_kwargs = od._request.call_args + headers = call_kwargs.kwargs.get("headers", {}) + ct = headers.get("Content-Type", "") + self.assertIn("multipart/mixed", ct) + self.assertIn("batch_", ct) + + +# --------------------------------------------------------------------------- +# 16. Robustness: malformed inputs and edge cases +# --------------------------------------------------------------------------- + + +class TestRobustnessEdgeCases(unittest.TestCase): + """Edge cases for input validation and malformed data handling.""" + + def test_parse_response_with_malformed_json_body(self): + """A response with invalid JSON in the body should not crash parsing.""" + text = "HTTP/1.1 200 OK\r\n" "Content-Type: application/json\r\n" "\r\n" "{this is not valid json}" + item = _parse_http_response_part(text, content_id=None) + self.assertIsNotNone(item) + self.assertEqual(item.status_code, 200) + # Malformed JSON: data should be None (parsing failed silently) + self.assertIsNone(item.data) + + def test_parse_response_with_truncated_json(self): + """Truncated JSON body should not crash.""" + text = 'HTTP/1.1 200 OK\r\n\r\n{"name": "Contoso", "accou' + item = _parse_http_response_part(text, content_id=None) + self.assertIsNotNone(item) + self.assertEqual(item.status_code, 200) + self.assertIsNone(item.data) + + def test_changeset_exception_in_context_manager(self): + """If user code raises inside with batch.changeset(), batch should still work.""" + from PowerPlatform.Dataverse.operations.batch import BatchRequest + + client = MagicMock() + batch = BatchRequest(client) + + # Exception inside changeset -- changeset is added to items before __enter__ + try: + with batch.changeset() as cs: + cs.records.create("account", {"name": "before error"}) + raise ValueError("user error") + except ValueError: + pass + + # The changeset IS in the items list (added in changeset() call) + # This is correct -- the changeset has 1 create operation + self.assertEqual(len(batch._items), 1) + + def test_empty_string_table_name_in_create(self): + """Empty table name should propagate (validated downstream by OData layer).""" + from PowerPlatform.Dataverse.operations.batch import BatchRequest + + client = MagicMock() + batch = BatchRequest(client) + # Empty string is accepted at batch level -- validated at execute time + batch.records.create("", {"name": "test"}) + self.assertEqual(len(batch._items), 1) + self.assertEqual(batch._items[0].table, "") + + def test_special_chars_in_odata_filter_are_escaped(self): + """OData filter values with single quotes are escaped by _escape_odata_quotes.""" + from PowerPlatform.Dataverse.data._odata import _ODataClient + + mock_auth = MagicMock() + mock_auth._acquire_token.return_value = MagicMock(access_token="token") + od = _ODataClient(mock_auth, "https://example.crm.dynamics.com") + + # _escape_odata_quotes doubles single quotes + escaped = od._escape_odata_quotes("test'table") + self.assertEqual(escaped, "test''table") + + # _build_get_entity uses the escaped value + req = od._build_get_entity("test'Table") + self.assertIn("test''table", req.url) + self.assertNotIn("test'table", req.url.replace("test''table", "")) + + def test_batch_item_response_with_non_dict_json_body(self): + """JSON body that is a list (not dict) should be handled.""" + text = "HTTP/1.1 200 OK\r\n\r\n[1, 2, 3]" + item = _parse_http_response_part(text, content_id=None) + self.assertIsNotNone(item) + self.assertEqual(item.status_code, 200) + # Non-dict JSON: data should be None (only dicts are captured) + self.assertIsNone(item.data) + + def test_batch_response_boundary_with_special_chars(self): + """Boundary strings with special chars should be handled.""" + boundary = "batch_abc+123/xyz" + resp_text = ( + f"--{boundary}\r\n" + "Content-Type: application/http\r\n" + "Content-Transfer-Encoding: binary\r\n" + "\r\n" + "HTTP/1.1 204 No Content\r\n" + "\r\n" + f"--{boundary}--\r\n" + ) + mock_response = MagicMock() + mock_response.headers = {"Content-Type": f'multipart/mixed; boundary="{boundary}"'} + mock_response.text = resp_text + + od = _make_od() + client = _BatchClient(od) + result = client._parse_batch_response(mock_response) + self.assertEqual(len(result.responses), 1) + self.assertTrue(result.responses[0].is_success) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/data/test_batch_serialization.py b/tests/unit/data/test_batch_serialization.py new file mode 100644 index 00000000..561b48b0 --- /dev/null +++ b/tests/unit/data/test_batch_serialization.py @@ -0,0 +1,632 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Unit tests for the internal batch multipart serialisation and response parsing.""" + +import json +import unittest +from unittest.mock import MagicMock + +from PowerPlatform.Dataverse.data._batch import ( + _BatchClient, + _ChangeSet, + _ChangeSetBatchItem, + _RecordCreate, + _RecordDelete, + _RecordGet, + _RecordUpdate, + _RecordUpsert, + _TableGet, + _TableList, + _QuerySql, + _extract_boundary, + _raise_top_level_batch_error, + _split_multipart, + _parse_mime_part, + _parse_http_response_part, + _CRLF, +) +from PowerPlatform.Dataverse.core.errors import HttpError +from PowerPlatform.Dataverse.models.upsert import UpsertItem +from PowerPlatform.Dataverse.data._raw_request import _RawRequest +from PowerPlatform.Dataverse.models.batch import BatchItemResponse + + +def _make_od(): + """Return a minimal mock _ODataClient.""" + od = MagicMock() + od.api = "https://org.crm.dynamics.com/api/data/v9.2" + return od + + +class TestExtractBoundary(unittest.TestCase): + def test_quoted_boundary(self): + ct = 'multipart/mixed; boundary="batch_abc123"' + self.assertEqual(_extract_boundary(ct), "batch_abc123") + + def test_unquoted_boundary(self): + ct = "multipart/mixed; boundary=batch_abc123" + self.assertEqual(_extract_boundary(ct), "batch_abc123") + + def test_no_boundary_returns_none(self): + self.assertIsNone(_extract_boundary("application/json")) + + def test_empty_string_returns_none(self): + self.assertIsNone(_extract_boundary("")) + + def test_boundary_with_uuid(self): + ct = 'multipart/mixed; boundary="batch_11111111-2222-3333-4444-555555555555"' + self.assertEqual( + _extract_boundary(ct), + "batch_11111111-2222-3333-4444-555555555555", + ) + + +class TestParseHttpResponsePart(unittest.TestCase): + def test_no_content_204(self): + text = "HTTP/1.1 204 No Content\r\n\r\n" + item = _parse_http_response_part(text, content_id=None) + self.assertIsNotNone(item) + self.assertEqual(item.status_code, 204) + self.assertTrue(item.is_success) + self.assertIsNone(item.data) + self.assertIsNone(item.entity_id) + + def test_created_with_entity_id(self): + guid = "11111111-2222-3333-4444-555555555555" + text = ( + f"HTTP/1.1 201 Created\r\n" + f"OData-EntityId: https://org.crm.dynamics.com/api/data/v9.2/accounts({guid})\r\n" + f"\r\n" + ) + item = _parse_http_response_part(text, content_id=None) + self.assertEqual(item.status_code, 201) + self.assertEqual(item.entity_id, guid) + + def test_get_response_with_body(self): + body = {"accountid": "abc", "name": "Contoso"} + body_str = json.dumps(body) + text = f"HTTP/1.1 200 OK\r\n" f"Content-Type: application/json\r\n" f"\r\n" f"{body_str}" + item = _parse_http_response_part(text, content_id=None) + self.assertEqual(item.status_code, 200) + self.assertEqual(item.data, body) + self.assertIsNone(item.error_message) + + def test_error_response(self): + error = {"error": {"code": "0x80040217", "message": "Object does not exist"}} + body_str = json.dumps(error) + text = f"HTTP/1.1 404 Not Found\r\n" f"Content-Type: application/json\r\n" f"\r\n" f"{body_str}" + item = _parse_http_response_part(text, content_id=None) + self.assertEqual(item.status_code, 404) + self.assertFalse(item.is_success) + self.assertEqual(item.error_message, "Object does not exist") + self.assertEqual(item.error_code, "0x80040217") + self.assertIsNone(item.data) + + def test_content_id_passed_through(self): + text = "HTTP/1.1 204 No Content\r\n\r\n" + item = _parse_http_response_part(text, content_id="1") + self.assertEqual(item.content_id, "1") + + def test_empty_text_returns_none(self): + self.assertIsNone(_parse_http_response_part("", content_id=None)) + + def test_no_http_status_line_returns_none(self): + self.assertIsNone(_parse_http_response_part("Not an HTTP response", content_id=None)) + + +class TestSerializeRawRequest(unittest.TestCase): + def _client(self): + od = _make_od() + return _BatchClient(od) + + def test_get_request_no_body(self): + req = _RawRequest(method="GET", url="https://org/api/data/v9.2/accounts") + client = self._client() + part = client._serialize_raw_request(req, "boundary_xyz") + self.assertIn("--boundary_xyz", part) + self.assertIn("Content-Type: application/http", part) + self.assertIn("GET https://org/api/data/v9.2/accounts HTTP/1.1", part) + self.assertNotIn("Content-Type: application/json", part) + + def test_post_request_with_body(self): + req = _RawRequest( + method="POST", + url="https://org/api/data/v9.2/accounts", + body='{"name":"Contoso"}', + ) + client = self._client() + part = client._serialize_raw_request(req, "bnd") + self.assertIn("Content-Type: application/json; type=entry", part) + self.assertIn('{"name":"Contoso"}', part) + + def test_delete_request_with_if_match_header(self): + req = _RawRequest( + method="DELETE", + url="https://org/api/data/v9.2/accounts(guid)", + headers={"If-Match": "*"}, + ) + client = self._client() + part = client._serialize_raw_request(req, "bnd") + self.assertIn("If-Match: *", part) + + def test_content_id_header_emitted(self): + req = _RawRequest( + method="POST", + url="https://org/api/data/v9.2/accounts", + body="{}", + content_id=3, + ) + client = self._client() + part = client._serialize_raw_request(req, "bnd") + self.assertIn("Content-ID: 3", part) + + def test_no_content_id_when_none(self): + req = _RawRequest(method="GET", url="https://org/api/data/v9.2/accounts") + client = self._client() + part = client._serialize_raw_request(req, "bnd") + self.assertNotIn("Content-ID", part) + + def test_crlf_line_endings(self): + req = _RawRequest(method="GET", url="https://org/api/data/v9.2/accounts") + client = self._client() + part = client._serialize_raw_request(req, "bnd") + self.assertIn(_CRLF, part) + + +class TestBuildBatchBody(unittest.TestCase): + def _client(self): + od = _make_od() + return _BatchClient(od) + + def test_single_request_body_ends_with_closing_boundary(self): + req = _RawRequest(method="GET", url="https://org/api/data/v9.2/accounts") + client = self._client() + body = client._build_batch_body([req], "batch_bnd") + self.assertIn("--batch_bnd--", body) + + def test_multiple_requests_all_in_body(self): + r1 = _RawRequest(method="GET", url="https://org/api/data/v9.2/accounts") + r2 = _RawRequest( + method="DELETE", + url="https://org/api/data/v9.2/accounts(guid)", + headers={"If-Match": "*"}, + ) + client = self._client() + body = client._build_batch_body([r1, r2], "bnd") + self.assertEqual(body.count("--bnd\r\n"), 2) + + def test_changeset_produces_nested_multipart(self): + r1 = _RawRequest(method="POST", url="https://org/api/data/v9.2/accounts", body="{}") + cs = _ChangeSetBatchItem(requests=[r1]) + client = self._client() + body = client._build_batch_body([cs], "outer_bnd") + self.assertIn("Content-Type: multipart/mixed", body) + self.assertIn("changeset_", body) + + +class TestResolveBatchItems(unittest.TestCase): + """Tests that _BatchClient._resolve_item calls the correct _build_* methods.""" + + def _client_and_od(self): + od = _make_od() + od._entity_set_from_schema_name.return_value = "accounts" + od._primary_id_attr.return_value = "accountid" + client = _BatchClient(od) + return client, od + + def test_resolve_record_create_single(self): + client, od = self._client_and_od() + mock_req = MagicMock() + od._build_create.return_value = mock_req + + op = _RecordCreate(table="account", data={"name": "Contoso"}) + result = client._resolve_record_create(op) + + od._build_create.assert_called_once() + self.assertEqual(result, [mock_req]) + + def test_resolve_record_create_list(self): + client, od = self._client_and_od() + mock_req = MagicMock() + od._build_create_multiple.return_value = mock_req + + op = _RecordCreate(table="account", data=[{"name": "A"}, {"name": "B"}]) + result = client._resolve_record_create(op) + + od._build_create_multiple.assert_called_once() + self.assertEqual(result, [mock_req]) + + def test_resolve_record_get(self): + client, od = self._client_and_od() + mock_req = MagicMock() + od._build_get.return_value = mock_req + + op = _RecordGet(table="account", record_id="guid-1", select=["name"]) + result = client._resolve_record_get(op) + + od._build_get.assert_called_once_with("account", "guid-1", select=["name"]) + self.assertEqual(result, [mock_req]) + + def test_resolve_record_delete_single(self): + client, od = self._client_and_od() + mock_req = MagicMock() + od._build_delete.return_value = mock_req + + op = _RecordDelete(table="account", ids="guid-1") + result = client._resolve_record_delete(op) + + od._build_delete.assert_called_once_with("account", "guid-1", content_id=None) + self.assertEqual(result, [mock_req]) + + def test_resolve_record_update_single(self): + client, od = self._client_and_od() + mock_req = MagicMock() + od._build_update.return_value = mock_req + + op = _RecordUpdate(table="account", ids="guid-1", changes={"name": "Updated"}) + result = client._resolve_record_update(op) + + od._build_update.assert_called_once_with("account", "guid-1", {"name": "Updated"}, content_id=None) + self.assertEqual(result, [mock_req]) + + def test_resolve_record_update_multiple(self): + client, od = self._client_and_od() + mock_req = MagicMock() + od._build_update_multiple.return_value = mock_req + + op = _RecordUpdate( + table="account", + ids=["guid-1", "guid-2"], + changes=[{"name": "A"}, {"name": "B"}], + ) + result = client._resolve_record_update(op) + + od._build_update_multiple.assert_called_once() + self.assertEqual(result, [mock_req]) + + def test_resolve_record_update_single_with_list_changes_raises(self): + client, od = self._client_and_od() + + op = _RecordUpdate(table="account", ids="guid-1", changes=[{"name": "A"}]) + with self.assertRaises(TypeError): + client._resolve_record_update(op) + + def test_resolve_record_delete_multiple_ids(self): + client, od = self._client_and_od() + mock_req = MagicMock() + od._build_delete_multiple.return_value = mock_req + + op = _RecordDelete(table="account", ids=["guid-1", "guid-2", "guid-3"]) + result = client._resolve_record_delete(op) + + od._build_delete_multiple.assert_called_once_with("account", ["guid-1", "guid-2", "guid-3"]) + self.assertEqual(result, [mock_req]) + + def test_resolve_record_delete_multiple_no_bulk(self): + client, od = self._client_and_od() + mock_req = MagicMock() + od._build_delete.return_value = mock_req + + op = _RecordDelete(table="account", ids=["guid-1", "guid-2"], use_bulk_delete=False) + result = client._resolve_record_delete(op) + + self.assertEqual(od._build_delete.call_count, 2) + self.assertEqual(len(result), 2) + + def test_resolve_record_delete_empty_ids_returns_empty(self): + client, od = self._client_and_od() + + op = _RecordDelete(table="account", ids=[]) + result = client._resolve_record_delete(op) + + self.assertEqual(result, []) + + def test_resolve_record_delete_filters_empty_strings(self): + client, od = self._client_and_od() + mock_req = MagicMock() + od._build_delete_multiple.return_value = mock_req + + op = _RecordDelete(table="account", ids=["guid-1", "", "guid-2", ""]) + result = client._resolve_record_delete(op) + + od._build_delete_multiple.assert_called_once_with("account", ["guid-1", "guid-2"]) + + def test_resolve_record_delete_all_empty_strings_returns_empty(self): + client, od = self._client_and_od() + + op = _RecordDelete(table="account", ids=["", "", ""]) + result = client._resolve_record_delete(op) + + self.assertEqual(result, []) + + def test_resolve_table_get(self): + client, od = self._client_and_od() + mock_req = MagicMock() + od._build_get_entity.return_value = mock_req + + op = _TableGet(table="account") + result = client._resolve_table_get(op) + + od._build_get_entity.assert_called_once_with("account") + self.assertEqual(result, [mock_req]) + + def test_resolve_table_list(self): + client, od = self._client_and_od() + mock_req = MagicMock() + od._build_list_entities.return_value = mock_req + + op = _TableList() + result = client._resolve_table_list(op) + + od._build_list_entities.assert_called_once() + self.assertEqual(result, [mock_req]) + + def test_resolve_query_sql(self): + client, od = self._client_and_od() + mock_req = MagicMock() + od._build_sql.return_value = mock_req + + op = _QuerySql(sql="SELECT name FROM account") + result = client._resolve_query_sql(op) + + od._build_sql.assert_called_once_with("SELECT name FROM account") + self.assertEqual(result, [mock_req]) + + def test_resolve_unknown_item_raises(self): + client, od = self._client_and_od() + from PowerPlatform.Dataverse.core.errors import ValidationError + + with self.assertRaises(ValidationError): + client._resolve_item("not_a_valid_intent") + + +class TestBatchSizeLimit(unittest.TestCase): + def test_exceeds_1000_raises(self): + od = _make_od() + od._entity_set_from_schema_name.return_value = "accounts" + od._build_get.return_value = _RawRequest(method="GET", url="https://x/accounts(g)") + client = _BatchClient(od) + + items = [_RecordGet(table="account", record_id=f"guid-{i}") for i in range(1001)] + from PowerPlatform.Dataverse.core.errors import ValidationError + + with self.assertRaises(ValidationError): + client.execute(items) + + +class TestChangeSetInternal(unittest.TestCase): + def test_add_create_returns_dollar_n(self): + cs = _ChangeSet() + ref = cs.add_create("account", {"name": "X"}) + self.assertEqual(ref, "$1") + + def test_add_create_increments_content_id(self): + cs = _ChangeSet() + r1 = cs.add_create("account", {"name": "A"}) + r2 = cs.add_create("contact", {"firstname": "B"}) + self.assertEqual(r1, "$1") + self.assertEqual(r2, "$2") + + def test_add_update_increments_content_id(self): + cs = _ChangeSet() + cs.add_create("account", {"name": "A"}) + cs.add_update("account", "guid-1", {"name": "B"}) + self.assertEqual(cs._counter[0], 3) + + def test_operations_in_order(self): + cs = _ChangeSet() + cs.add_create("account", {"name": "A"}) + cs.add_delete("account", "guid-1") + self.assertEqual(len(cs.operations), 2) + self.assertIsInstance(cs.operations[0], _RecordCreate) + self.assertIsInstance(cs.operations[1], _RecordDelete) + + def test_two_changesets_shared_counter_produce_unique_content_ids(self): + """Two _ChangeSets sharing a counter must emit batch-wide unique Content-IDs.""" + shared = [1] + cs1 = _ChangeSet(_counter=shared) + cs2 = _ChangeSet(_counter=shared) + + cs1.add_create("account", {"name": "A"}) # cid=1 + cs1.add_update("account", "guid-1", {"name": "B"}) # cid=2 + cs2.add_create("contact", {"firstname": "C"}) # cid=3 + cs2.add_update("contact", "guid-2", {"firstname": "D"}) # cid=4 + + ids_cs1 = [op.content_id for op in cs1.operations] + ids_cs2 = [op.content_id for op in cs2.operations] + self.assertEqual(ids_cs1, [1, 2]) + self.assertEqual(ids_cs2, [3, 4]) + # No overlap + self.assertEqual(len(set(ids_cs1) & set(ids_cs2)), 0) + + def test_standalone_changeset_still_starts_at_one(self): + """A _ChangeSet created without a shared counter gets its own [1] counter.""" + cs = _ChangeSet() + ref = cs.add_create("account", {"name": "X"}) + self.assertEqual(ref, "$1") + self.assertEqual(cs._counter[0], 2) + + +class TestResolveBatchUpsert(unittest.TestCase): + """Tests that _BatchClient._resolve_record_upsert calls the correct _build_* methods.""" + + def _client_and_od(self): + od = _make_od() + od._entity_set_from_schema_name.return_value = "accounts" + client = _BatchClient(od) + return client, od + + def test_resolve_single_item_calls_build_upsert(self): + client, od = self._client_and_od() + mock_req = MagicMock() + od._build_upsert.return_value = mock_req + + item = UpsertItem(alternate_key={"accountnumber": "ACC-001"}, record={"name": "Contoso"}) + op = _RecordUpsert(table="account", items=[item]) + result = client._resolve_record_upsert(op) + + od._build_upsert.assert_called_once_with( + "accounts", "account", {"accountnumber": "ACC-001"}, {"name": "Contoso"} + ) + self.assertEqual(result, [mock_req]) + + def test_resolve_multiple_items_calls_build_upsert_multiple(self): + client, od = self._client_and_od() + mock_req = MagicMock() + od._build_upsert_multiple.return_value = mock_req + + items = [ + UpsertItem(alternate_key={"accountnumber": "ACC-001"}, record={"name": "Contoso"}), + UpsertItem(alternate_key={"accountnumber": "ACC-002"}, record={"name": "Fabrikam"}), + ] + op = _RecordUpsert(table="account", items=items) + result = client._resolve_record_upsert(op) + + od._build_upsert_multiple.assert_called_once_with( + "accounts", + "account", + [{"accountnumber": "ACC-001"}, {"accountnumber": "ACC-002"}], + [{"name": "Contoso"}, {"name": "Fabrikam"}], + ) + self.assertEqual(result, [mock_req]) + + def test_resolve_item_dispatch_routes_to_upsert(self): + client, od = self._client_and_od() + mock_req = MagicMock() + od._build_upsert.return_value = mock_req + + item = UpsertItem(alternate_key={"accountnumber": "ACC-001"}, record={"name": "Contoso"}) + op = _RecordUpsert(table="account", items=[item]) + result = client._resolve_item(op) + + self.assertEqual(result, [mock_req]) + + +class TestBatchRecordOperationsUpsert(unittest.TestCase): + """Tests for BatchRecordOperations.upsert (operations/batch.py).""" + + def _make_batch(self): + from PowerPlatform.Dataverse.operations.batch import BatchRecordOperations + + batch = MagicMock() + batch._items = [] + return BatchRecordOperations(batch), batch + + def test_upsert_single_upsert_item_appended(self): + from PowerPlatform.Dataverse.operations.batch import BatchRecordOperations + + rec_ops, batch = self._make_batch() + item = UpsertItem(alternate_key={"accountnumber": "ACC-001"}, record={"name": "Contoso"}) + rec_ops.upsert("account", [item]) + + self.assertEqual(len(batch._items), 1) + intent = batch._items[0] + self.assertIsInstance(intent, _RecordUpsert) + self.assertEqual(intent.table, "account") + self.assertEqual(len(intent.items), 1) + self.assertEqual(intent.items[0].alternate_key, {"accountnumber": "ACC-001"}) + + def test_upsert_plain_dict_normalised_to_upsert_item(self): + from PowerPlatform.Dataverse.operations.batch import BatchRecordOperations + + rec_ops, batch = self._make_batch() + rec_ops.upsert("account", [{"alternate_key": {"accountnumber": "X"}, "record": {"name": "Y"}}]) + + intent = batch._items[0] + self.assertIsInstance(intent.items[0], UpsertItem) + self.assertEqual(intent.items[0].record, {"name": "Y"}) + + def test_upsert_empty_list_raises(self): + from PowerPlatform.Dataverse.operations.batch import BatchRecordOperations + + rec_ops, _ = self._make_batch() + with self.assertRaises(TypeError): + rec_ops.upsert("account", []) + + def test_upsert_invalid_item_raises(self): + from PowerPlatform.Dataverse.operations.batch import BatchRecordOperations + + rec_ops, _ = self._make_batch() + with self.assertRaises(TypeError): + rec_ops.upsert("account", ["not_a_valid_item"]) + + def test_upsert_multiple_items_all_normalised(self): + from PowerPlatform.Dataverse.operations.batch import BatchRecordOperations + + rec_ops, batch = self._make_batch() + rec_ops.upsert( + "account", + [ + UpsertItem(alternate_key={"accountnumber": "A"}, record={"name": "Alpha"}), + UpsertItem(alternate_key={"accountnumber": "B"}, record={"name": "Beta"}), + ], + ) + + intent = batch._items[0] + self.assertEqual(len(intent.items), 2) + self.assertEqual(intent.items[1].alternate_key, {"accountnumber": "B"}) + + +class TestRaiseTopLevelBatchError(unittest.TestCase): + """_raise_top_level_batch_error surfaces Dataverse error details as HttpError.""" + + def _make_response(self, status_code, json_body=None, text=None): + resp = MagicMock() + resp.status_code = status_code + resp.text = text or "" + if json_body is not None: + resp.json.return_value = json_body + else: + resp.json.side_effect = ValueError("no JSON") + return resp + + def test_raises_http_error(self): + """Always raises HttpError, never returns.""" + resp = self._make_response(400, json_body={"error": {"code": "0x0", "message": "Bad batch"}}) + with self.assertRaises(HttpError): + _raise_top_level_batch_error(resp) + + def test_status_code_preserved(self): + """HttpError.status_code matches the response status code.""" + resp = self._make_response(400, json_body={"error": {"code": "0x0", "message": "Bad batch"}}) + with self.assertRaises(HttpError) as ctx: + _raise_top_level_batch_error(resp) + self.assertEqual(ctx.exception.status_code, 400) + + def test_service_message_in_exception(self): + """The Dataverse error message is included in the raised exception.""" + resp = self._make_response(400, json_body={"error": {"code": "BadRequest", "message": "Malformed OData batch"}}) + with self.assertRaises(HttpError) as ctx: + _raise_top_level_batch_error(resp) + self.assertIn("Malformed OData batch", str(ctx.exception)) + + def test_service_error_code_preserved(self): + """The Dataverse error code is forwarded into HttpError.details.""" + resp = self._make_response(400, json_body={"error": {"code": "0x80040216", "message": "..."}}) + with self.assertRaises(HttpError) as ctx: + _raise_top_level_batch_error(resp) + self.assertEqual(ctx.exception.details.get("service_error_code"), "0x80040216") + + def test_falls_back_to_response_text_when_no_json(self): + """Falls back to response.text when the body is not valid JSON.""" + resp = self._make_response(400, text="plain text error body") + with self.assertRaises(HttpError) as ctx: + _raise_top_level_batch_error(resp) + self.assertIn("plain text error body", str(ctx.exception)) + + def test_parse_batch_response_raises_on_missing_boundary(self): + """_BatchClient._parse_batch_response raises HttpError for non-multipart responses.""" + od = _make_od() + client = _BatchClient(od) + resp = MagicMock() + resp.headers = {"Content-Type": "application/json"} + resp.status_code = 400 + resp.text = "" + resp.json.return_value = {"error": {"code": "0x0", "message": "Invalid batch"}} + with self.assertRaises(HttpError): + client._parse_batch_response(resp) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/data/test_format_key.py b/tests/unit/data/test_format_key.py new file mode 100644 index 00000000..2b6216d1 --- /dev/null +++ b/tests/unit/data/test_format_key.py @@ -0,0 +1,58 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Unit tests for _ODataClient._format_key.""" + +import unittest + +from PowerPlatform.Dataverse.data._odata import _ODataClient + +# Create a bare instance that bypasses __init__ — _format_key only needs +# self._escape_odata_quotes, which is a @staticmethod defined on the class. +_CLIENT = object.__new__(_ODataClient) + + +class TestFormatKey(unittest.TestCase): + def test_plain_guid_is_wrapped(self): + guid = "11111111-2222-3333-4444-555555555555" + self.assertEqual(_CLIENT._format_key(guid), f"({guid})") + + def test_already_parenthesised_is_returned_unchanged(self): + key = "(11111111-2222-3333-4444-555555555555)" + self.assertEqual(_CLIENT._format_key(key), key) + + def test_already_parenthesised_is_not_double_wrapped(self): + key = "(some-key)" + result = _CLIENT._format_key(key) + self.assertFalse(result.startswith("(("), f"Double-wrapped: {result!r}") + + def test_leading_trailing_whitespace_is_stripped(self): + guid = "11111111-2222-3333-4444-555555555555" + self.assertEqual(_CLIENT._format_key(f" {guid} "), f"({guid})") + + def test_non_guid_plain_string_is_wrapped(self): + self.assertEqual(_CLIENT._format_key("somevalue"), "(somevalue)") + + def test_alternate_key_no_quotes_is_wrapped(self): + # '=' present but no single quotes — goes straight to final wrap + self.assertEqual(_CLIENT._format_key("myattr=somevalue"), "(myattr=somevalue)") + + def test_alternate_key_with_quoted_value_is_wrapped(self): + result = _CLIENT._format_key("myattr='hello'") + self.assertEqual(result, "(myattr='hello')") + + def test_alternate_key_with_embedded_single_quote_is_escaped(self): + # Value contains an embedded single quote; _escape_odata_quotes doubles it + result = _CLIENT._format_key("myattr='O''Brien'") + # _escape_odata_quotes doubles any ' in the captured value segment; + # the captured value is 'O' (regex stops at first '), then re.sub + # replaces that match → value 'O' has no ' so unchanged → (myattr='O''Brien') + # (the '' between O and Brien is outside the first match; overall the + # parentheses are always added) + self.assertTrue(result.startswith("("), result) + self.assertTrue(result.endswith(")"), result) + + def test_whitespace_only_inside_parens_is_returned_unchanged(self): + # Already parenthesised — content is not inspected + key = "( )" + self.assertEqual(_CLIENT._format_key(key), key) diff --git a/tests/unit/data/test_odata_internal.py b/tests/unit/data/test_odata_internal.py index bea5a3d6..b5b4dd79 100644 --- a/tests/unit/data/test_odata_internal.py +++ b/tests/unit/data/test_odata_internal.py @@ -1,9 +1,11 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +import json import unittest from unittest.mock import MagicMock +from PowerPlatform.Dataverse.core.errors import ValidationError from PowerPlatform.Dataverse.data._odata import _ODataClient @@ -179,9 +181,8 @@ def test_no_filter_uses_default(self): self.od._list_tables() self.od._request.assert_called_once() - call_kwargs = self.od._request.call_args - params = call_kwargs.kwargs.get("params") or call_kwargs[1].get("params", {}) - self.assertEqual(params["$filter"], "IsPrivate eq false") + url = self.od._request.call_args[0][1] + self.assertIn("$filter=IsPrivate eq false", url) def test_filter_combined_with_default(self): """_list_tables(filter=...) combines user filter with IsPrivate eq false.""" @@ -189,12 +190,8 @@ def test_filter_combined_with_default(self): self.od._list_tables(filter="SchemaName eq 'Account'") self.od._request.assert_called_once() - call_kwargs = self.od._request.call_args - params = call_kwargs.kwargs.get("params") or call_kwargs[1].get("params", {}) - self.assertEqual( - params["$filter"], - "IsPrivate eq false and (SchemaName eq 'Account')", - ) + url = self.od._request.call_args[0][1] + self.assertIn("IsPrivate eq false and (SchemaName eq 'Account')", url) def test_filter_none_same_as_no_filter(self): """_list_tables(filter=None) is equivalent to _list_tables().""" @@ -202,9 +199,9 @@ def test_filter_none_same_as_no_filter(self): self.od._list_tables(filter=None) self.od._request.assert_called_once() - call_kwargs = self.od._request.call_args - params = call_kwargs.kwargs.get("params") or call_kwargs[1].get("params", {}) - self.assertEqual(params["$filter"], "IsPrivate eq false") + url = self.od._request.call_args[0][1] + self.assertIn("$filter=IsPrivate eq false", url) + self.assertNotIn("and", url) def test_returns_value_list(self): """_list_tables returns the 'value' array from the response.""" @@ -222,9 +219,8 @@ def test_select_adds_query_param(self): self.od._list_tables(select=["LogicalName", "SchemaName", "DisplayName"]) self.od._request.assert_called_once() - call_kwargs = self.od._request.call_args - params = call_kwargs.kwargs.get("params") or call_kwargs[1].get("params", {}) - self.assertEqual(params["$select"], "LogicalName,SchemaName,DisplayName") + url = self.od._request.call_args[0][1] + self.assertIn("$select=LogicalName,SchemaName,DisplayName", url) def test_select_none_omits_query_param(self): """_list_tables(select=None) does not add $select to params.""" @@ -232,9 +228,8 @@ def test_select_none_omits_query_param(self): self.od._list_tables(select=None) self.od._request.assert_called_once() - call_kwargs = self.od._request.call_args - params = call_kwargs.kwargs.get("params") or call_kwargs[1].get("params", {}) - self.assertNotIn("$select", params) + url = self.od._request.call_args[0][1] + self.assertNotIn("$select", url) def test_select_empty_list_omits_query_param(self): """_list_tables(select=[]) does not add $select (empty list is falsy).""" @@ -242,9 +237,8 @@ def test_select_empty_list_omits_query_param(self): self.od._list_tables(select=[]) self.od._request.assert_called_once() - call_kwargs = self.od._request.call_args - params = call_kwargs.kwargs.get("params") or call_kwargs[1].get("params", {}) - self.assertNotIn("$select", params) + url = self.od._request.call_args[0][1] + self.assertNotIn("$select", url) def test_select_preserves_case(self): """_list_tables does not lowercase select values (PascalCase preserved).""" @@ -252,9 +246,8 @@ def test_select_preserves_case(self): self.od._list_tables(select=["EntitySetName", "LogicalName"]) self.od._request.assert_called_once() - call_kwargs = self.od._request.call_args - params = call_kwargs.kwargs.get("params") or call_kwargs[1].get("params", {}) - self.assertEqual(params["$select"], "EntitySetName,LogicalName") + url = self.od._request.call_args[0][1] + self.assertIn("$select=EntitySetName,LogicalName", url) def test_select_with_filter(self): """_list_tables with both select and filter sends both params.""" @@ -265,13 +258,9 @@ def test_select_with_filter(self): ) self.od._request.assert_called_once() - call_kwargs = self.od._request.call_args - params = call_kwargs.kwargs.get("params") or call_kwargs[1].get("params", {}) - self.assertEqual( - params["$filter"], - "IsPrivate eq false and (SchemaName eq 'Account')", - ) - self.assertEqual(params["$select"], "LogicalName,SchemaName") + url = self.od._request.call_args[0][1] + self.assertIn("IsPrivate eq false and (SchemaName eq 'Account')", url) + self.assertIn("$select=LogicalName,SchemaName", url) def test_select_single_property(self): """_list_tables(select=[...]) with a single property works correctly.""" @@ -279,9 +268,8 @@ def test_select_single_property(self): self.od._list_tables(select=["LogicalName"]) self.od._request.assert_called_once() - call_kwargs = self.od._request.call_args - params = call_kwargs.kwargs.get("params") or call_kwargs[1].get("params", {}) - self.assertEqual(params["$select"], "LogicalName") + url = self.od._request.call_args[0][1] + self.assertIn("$select=LogicalName", url) def test_select_bare_string_raises_type_error(self): """_list_tables(select='LogicalName') raises TypeError for bare str.""" @@ -313,7 +301,7 @@ def test_record_keys_lowercased(self): """Regular record field names are lowercased before sending.""" self.od._create("accounts", "account", {"Name": "Contoso", "AccountNumber": "ACC-001"}) call = self._post_call() - payload = call.kwargs["json"] + payload = json.loads(call.kwargs["data"]) if "data" in call.kwargs else call.kwargs["json"] self.assertIn("name", payload) self.assertIn("accountnumber", payload) self.assertNotIn("Name", payload) @@ -331,7 +319,7 @@ def test_odata_bind_keys_preserve_case(self): }, ) call = self._post_call() - payload = call.kwargs["json"] + payload = json.loads(call.kwargs["data"]) if "data" in call.kwargs else call.kwargs["json"] self.assertIn("new_name", payload) self.assertIn("new_CustomerId@odata.bind", payload) self.assertIn("new_AgentId@odata.bind", payload) @@ -397,7 +385,7 @@ def test_record_keys_lowercased(self): """Regular field names are lowercased in _update.""" self.od._update("new_ticket", "00000000-0000-0000-0000-000000000001", {"New_Status": 100000001}) call = self._patch_call() - payload = call.kwargs["json"] + payload = json.loads(call.kwargs["data"]) if "data" in call.kwargs else call.kwargs["json"] self.assertIn("new_status", payload) self.assertNotIn("New_Status", payload) @@ -412,7 +400,7 @@ def test_odata_bind_keys_preserve_case(self): }, ) call = self._patch_call() - payload = call.kwargs["json"] + payload = json.loads(call.kwargs["data"]) if "data" in call.kwargs else call.kwargs["json"] self.assertIn("new_status", payload) self.assertIn("new_CustomerId@odata.bind", payload) self.assertNotIn("new_customerid@odata.bind", payload) @@ -481,7 +469,7 @@ def test_record_keys_lowercased(self): """Record field names are lowercased before sending.""" self.od._upsert("accounts", "account", {"accountnumber": "ACC-001"}, {"Name": "Contoso"}) call = self._patch_call() - payload = call.kwargs["json"] + payload = json.loads(call.kwargs["data"]) if "data" in call.kwargs else call.kwargs["json"] self.assertIn("name", payload) self.assertNotIn("Name", payload) @@ -497,7 +485,7 @@ def test_odata_bind_keys_preserve_case(self): }, ) call = self._patch_call() - payload = call.kwargs["json"] + payload = json.loads(call.kwargs["data"]) if "data" in call.kwargs else call.kwargs["json"] # Regular field is lowercased self.assertIn("name", payload) # @odata.bind key preserves original casing @@ -530,5 +518,85 @@ def test_returns_none(self): self.assertIsNone(result) +class TestBuildUpsertMultiple(unittest.TestCase): + """Unit tests for _ODataClient._build_upsert_multiple (batch deferred build).""" + + def setUp(self): + self.od = _make_odata_client() + + def _targets(self, alt_keys, records): + import json + + req = self.od._build_upsert_multiple("accounts", "account", alt_keys, records) + return json.loads(req.body)["Targets"] + + def test_payload_excludes_alternate_key_fields(self): + """Alternate key fields must NOT appear in the request body (only in @odata.id).""" + targets = self._targets( + [{"accountnumber": "ACC-001"}], + [{"name": "Contoso"}], + ) + self.assertEqual(len(targets), 1) + target = targets[0] + self.assertNotIn("accountnumber", target) + self.assertIn("name", target) + self.assertIn("@odata.id", target) + self.assertIn("accountnumber", target["@odata.id"]) + + def test_payload_allows_matching_key_field_in_record(self): + """If user passes matching key field in record with same value, it passes through to body.""" + targets = self._targets( + [{"accountnumber": "ACC-001"}], + [{"accountnumber": "ACC-001", "name": "Contoso"}], + ) + target = targets[0] + self.assertIn("name", target) + self.assertIn("@odata.id", target) + self.assertIn("accountnumber", target["@odata.id"]) + + def test_odata_type_added_when_absent(self): + """@odata.type is injected when not provided by caller.""" + targets = self._targets( + [{"accountnumber": "ACC-001"}], + [{"name": "Contoso"}], + ) + self.assertIn("@odata.type", targets[0]) + self.assertEqual(targets[0]["@odata.type"], "Microsoft.Dynamics.CRM.account") + + def test_multiple_targets_all_have_odata_id(self): + """Each target in a multi-item call gets its own @odata.id.""" + targets = self._targets( + [{"accountnumber": "ACC-001"}, {"accountnumber": "ACC-002"}], + [{"name": "Contoso"}, {"name": "Fabrikam"}], + ) + self.assertEqual(len(targets), 2) + self.assertIn("ACC-001", targets[0]["@odata.id"]) + self.assertIn("ACC-002", targets[1]["@odata.id"]) + + def test_conflicting_key_field_raises(self): + """Raises when a record field contradicts its alternate key value.""" + with self.assertRaises(ValidationError) as ctx: + self.od._build_upsert_multiple( + "accounts", + "account", + [{"accountnumber": "ACC-001"}], + [{"accountnumber": "ACC-WRONG", "name": "Contoso"}], + ) + self.assertIn("accountnumber", str(ctx.exception)) + + def test_mismatched_lengths_raises(self): + """Raises when alternate_keys and records lengths differ.""" + with self.assertRaises(ValidationError): + self.od._build_upsert_multiple("accounts", "account", [{"accountnumber": "ACC-001"}], []) + + def test_url_contains_upsert_multiple_action(self): + """POST URL targets the UpsertMultiple bound action.""" + req = self.od._build_upsert_multiple( + "accounts", "account", [{"accountnumber": "ACC-001"}], [{"name": "Contoso"}] + ) + self.assertIn("UpsertMultiple", req.url) + self.assertEqual(req.method, "POST") + + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/data/test_sql_parse.py b/tests/unit/data/test_sql_parse.py index efbf606b..12c25a9c 100644 --- a/tests/unit/data/test_sql_parse.py +++ b/tests/unit/data/test_sql_parse.py @@ -1,6 +1,9 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +from unittest.mock import patch +from urllib.parse import parse_qs, urlparse + import pytest from PowerPlatform.Dataverse.data._odata import _ODataClient @@ -17,6 +20,24 @@ def _client(): return _ODataClient(DummyAuth(), "https://org.example", None) +# --------------------------------------------------------------------------- +# Helpers for _build_sql tests +# --------------------------------------------------------------------------- + +_BARE = object.__new__(_ODataClient) +_BARE.api = "https://org.crm.dynamics.com/api/data/v9.2" +_ENTITY_SET = "accounts" + + +def _build(sql: str) -> str: + with patch.object(_BARE, "_entity_set_from_schema_name", return_value=_ENTITY_SET): + return _BARE._build_sql(sql).url + + +def _sql_param(url: str) -> str: + return parse_qs(urlparse(url).query)["sql"][0] + + def test_basic_from(): c = _client() assert c._extract_logical_table("SELECT a FROM account") == "account" @@ -49,3 +70,42 @@ def test_from_as_value_not_table(): # Table should still be 'incident'; word 'from' earlier shouldn't interfere sql = "SELECT 'from something', col FROM incident" assert c._extract_logical_table(sql) == "incident" + + +# --------------------------------------------------------------------------- +# _build_sql URL encoding +# --------------------------------------------------------------------------- + + +def test_build_sql_plain_select_round_trips(): + sql = "SELECT accountid FROM account" + assert _sql_param(_build(sql)) == sql + + +def test_build_sql_forward_slash_is_percent_encoded(): + sql = "SELECT accountid FROM account WHERE name = 'a/b'" + url = _build(sql) + assert "a/b" not in url.split("?", 1)[1] + assert "%2F" in url + + +def test_build_sql_space_is_percent_encoded(): + sql = "SELECT accountid FROM account WHERE name = 'hello world'" + assert " " not in _build(sql).split("?", 1)[1] + + +def test_build_sql_ampersand_is_percent_encoded(): + sql = "SELECT accountid FROM account WHERE name = 'a&b'" + url = _build(sql) + assert "name=a&b" not in url.split("?", 1)[1] + assert "%26" in url + + +def test_build_sql_equals_in_value_is_percent_encoded(): + sql = "SELECT accountid FROM account WHERE name = 'x=y'" + assert "%3D" in _build(sql) + + +def test_build_sql_decoded_param_matches_input(): + sql = "SELECT accountid, name FROM account WHERE statecode = 0" + assert _sql_param(_build(sql)) == sql diff --git a/tests/unit/test_batch_dataframe.py b/tests/unit/test_batch_dataframe.py new file mode 100644 index 00000000..12c38088 --- /dev/null +++ b/tests/unit/test_batch_dataframe.py @@ -0,0 +1,197 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Unit tests for BatchDataFrameOperations.""" + +import unittest +from unittest.mock import MagicMock + +import pandas as pd + +from PowerPlatform.Dataverse.operations.batch import ( + BatchDataFrameOperations, + BatchRequest, +) +from PowerPlatform.Dataverse.data._batch import _RecordCreate, _RecordUpdate, _RecordDelete + + +def _make_batch(): + """Return a BatchRequest with a mocked client.""" + client = MagicMock() + batch = BatchRequest(client) + return batch + + +class TestBatchDataFrameNamespace(unittest.TestCase): + """BatchRequest should have a .dataframe namespace.""" + + def test_batch_has_dataframe_attribute(self): + batch = _make_batch() + self.assertIsInstance(batch.dataframe, BatchDataFrameOperations) + + +class TestBatchDataFrameCreate(unittest.TestCase): + """batch.dataframe.create() converts DataFrame to records and delegates to batch.records.create.""" + + def test_create_from_dataframe(self): + batch = _make_batch() + df = pd.DataFrame( + [ + {"name": "Contoso", "telephone1": "555-0100"}, + {"name": "Fabrikam", "telephone1": "555-0200"}, + ] + ) + batch.dataframe.create("account", df) + # Should have enqueued a _RecordCreate with a list of dicts + self.assertEqual(len(batch._items), 1) + item = batch._items[0] + self.assertIsInstance(item, _RecordCreate) + self.assertEqual(item.table, "account") + self.assertIsInstance(item.data, list) + self.assertEqual(len(item.data), 2) + self.assertEqual(item.data[0]["name"], "Contoso") + self.assertEqual(item.data[1]["name"], "Fabrikam") + + def test_create_single_row(self): + batch = _make_batch() + df = pd.DataFrame([{"name": "SingleCo"}]) + batch.dataframe.create("account", df) + self.assertEqual(len(batch._items), 1) + self.assertIsInstance(batch._items[0].data, list) + self.assertEqual(len(batch._items[0].data), 1) + + def test_create_rejects_non_dataframe(self): + batch = _make_batch() + with self.assertRaises(TypeError) as ctx: + batch.dataframe.create("account", [{"name": "x"}]) + self.assertIn("DataFrame", str(ctx.exception)) + + def test_create_rejects_empty_dataframe(self): + batch = _make_batch() + df = pd.DataFrame() + with self.assertRaises(ValueError) as ctx: + batch.dataframe.create("account", df) + self.assertIn("non-empty", str(ctx.exception)) + + def test_create_rejects_rows_with_all_nan(self): + batch = _make_batch() + df = pd.DataFrame([{"name": None}]) + with self.assertRaises(ValueError) as ctx: + batch.dataframe.create("account", df) + self.assertIn("no non-null", str(ctx.exception)) + + def test_create_handles_nan_values(self): + """NaN values are dropped from individual records by default.""" + batch = _make_batch() + df = pd.DataFrame( + [ + {"name": "Contoso", "telephone1": "555-0100"}, + {"name": "Fabrikam", "telephone1": None}, + ] + ) + batch.dataframe.create("account", df) + item = batch._items[0] + # Second record should not have telephone1 + self.assertNotIn("telephone1", item.data[1]) + self.assertIn("name", item.data[1]) + + +class TestBatchDataFrameUpdate(unittest.TestCase): + """batch.dataframe.update() converts DataFrame to update operations.""" + + def test_update_enqueues_record_update(self): + batch = _make_batch() + df = pd.DataFrame( + [ + {"accountid": "guid-1", "telephone1": "555-0100"}, + {"accountid": "guid-2", "telephone1": "555-0200"}, + ] + ) + batch.dataframe.update("account", df, id_column="accountid") + self.assertEqual(len(batch._items), 1) + item = batch._items[0] + self.assertIsInstance(item, _RecordUpdate) + self.assertEqual(item.table, "account") + # ids should be a list, changes should be a list + self.assertIsInstance(item.ids, list) + self.assertEqual(len(item.ids), 2) + self.assertEqual(item.ids[0], "guid-1") + + def test_update_rejects_non_dataframe(self): + batch = _make_batch() + with self.assertRaises(TypeError): + batch.dataframe.update("account", [{}], id_column="id") + + def test_update_rejects_empty_dataframe(self): + batch = _make_batch() + with self.assertRaises(ValueError): + batch.dataframe.update("account", pd.DataFrame(), id_column="id") + + def test_update_rejects_missing_id_column(self): + batch = _make_batch() + df = pd.DataFrame([{"name": "x"}]) + with self.assertRaises(ValueError) as ctx: + batch.dataframe.update("account", df, id_column="accountid") + self.assertIn("not found", str(ctx.exception)) + + def test_update_rejects_invalid_ids(self): + batch = _make_batch() + df = pd.DataFrame([{"accountid": 123, "name": "x"}]) + with self.assertRaises(ValueError) as ctx: + batch.dataframe.update("account", df, id_column="accountid") + self.assertIn("invalid", str(ctx.exception)) + + def test_update_skips_all_nan_rows(self): + """Rows where all change values are NaN are silently skipped.""" + batch = _make_batch() + df = pd.DataFrame( + [ + {"accountid": "guid-1", "name": None}, + ] + ) + batch.dataframe.update("account", df, id_column="accountid") + # Nothing enqueued because all change values were NaN + self.assertEqual(len(batch._items), 0) + + +class TestBatchDataFrameDelete(unittest.TestCase): + """batch.dataframe.delete() converts Series to delete operation.""" + + def test_delete_from_series(self): + batch = _make_batch() + ids = pd.Series(["guid-1", "guid-2", "guid-3"]) + batch.dataframe.delete("account", ids) + self.assertEqual(len(batch._items), 1) + item = batch._items[0] + self.assertIsInstance(item, _RecordDelete) + self.assertIsInstance(item.ids, list) + self.assertEqual(len(item.ids), 3) + + def test_delete_rejects_non_series(self): + batch = _make_batch() + with self.assertRaises(TypeError): + batch.dataframe.delete("account", ["guid-1"]) + + def test_delete_empty_series_no_op(self): + batch = _make_batch() + ids = pd.Series([], dtype=str) + batch.dataframe.delete("account", ids) + self.assertEqual(len(batch._items), 0) + + def test_delete_rejects_invalid_ids(self): + batch = _make_batch() + ids = pd.Series([123, 456]) + with self.assertRaises(ValueError): + batch.dataframe.delete("account", ids) + + def test_delete_with_bulk_delete_false(self): + batch = _make_batch() + ids = pd.Series(["guid-1", "guid-2"]) + batch.dataframe.delete("account", ids, use_bulk_delete=False) + item = batch._items[0] + self.assertIsInstance(item, _RecordDelete) + self.assertFalse(item.use_bulk_delete) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_batch_operations.py b/tests/unit/test_batch_operations.py new file mode 100644 index 00000000..f706c6c4 --- /dev/null +++ b/tests/unit/test_batch_operations.py @@ -0,0 +1,463 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import unittest +from unittest.mock import MagicMock, patch + +from azure.core.credentials import TokenCredential + +from PowerPlatform.Dataverse.client import DataverseClient +from PowerPlatform.Dataverse.operations.batch import ( + BatchOperations, + BatchRequest, + BatchRecordOperations, + BatchTableOperations, + BatchQueryOperations, + ChangeSet, + ChangeSetRecordOperations, +) +from PowerPlatform.Dataverse.data._batch import ( + _RecordCreate, + _RecordUpdate, + _RecordDelete, + _RecordGet, + _TableCreate, + _TableDelete, + _TableGet, + _TableList, + _TableAddColumns, + _TableRemoveColumns, + _TableCreateOneToMany, + _TableCreateManyToMany, + _TableDeleteRelationship, + _TableGetRelationship, + _TableCreateLookupField, + _QuerySql, + _ChangeSet, +) +from PowerPlatform.Dataverse.models.batch import BatchResult, BatchItemResponse +from PowerPlatform.Dataverse.core.errors import ValidationError + + +class TestBatchOperationsNamespace(unittest.TestCase): + """Tests for the client.batch namespace.""" + + def setUp(self): + self.mock_credential = MagicMock(spec=TokenCredential) + self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) + + def test_namespace_exists(self): + """client.batch should be a BatchOperations instance.""" + self.assertIsInstance(self.client.batch, BatchOperations) + + def test_new_returns_batch_request(self): + """client.batch.new() should return a BatchRequest.""" + batch = self.client.batch.new() + self.assertIsInstance(batch, BatchRequest) + + def test_new_returns_new_instance_each_call(self): + """Each call to new() should return a distinct BatchRequest.""" + b1 = self.client.batch.new() + b2 = self.client.batch.new() + self.assertIsNot(b1, b2) + + +class TestBatchRequest(unittest.TestCase): + """Tests for BatchRequest builder.""" + + def setUp(self): + self.mock_credential = MagicMock(spec=TokenCredential) + self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) + self.batch = self.client.batch.new() + + def test_has_records_namespace(self): + self.assertIsInstance(self.batch.records, BatchRecordOperations) + + def test_has_tables_namespace(self): + self.assertIsInstance(self.batch.tables, BatchTableOperations) + + def test_has_query_namespace(self): + self.assertIsInstance(self.batch.query, BatchQueryOperations) + + def test_changeset_returns_changeset(self): + cs = self.batch.changeset() + self.assertIsInstance(cs, ChangeSet) + + def test_changeset_added_to_items(self): + cs = self.batch.changeset() + self.assertEqual(len(self.batch._items), 1) + self.assertIsInstance(self.batch._items[0], _ChangeSet) + + def test_execute_calls_batch_client(self): + """execute() should call _BatchClient.execute via _scoped_odata.""" + mock_od = MagicMock() + mock_result = BatchResult(responses=[BatchItemResponse(status_code=204)]) + mock_od._entity_set_from_schema_name.return_value = "accounts" + mock_od._build_create.return_value = MagicMock() + mock_od._request.return_value = MagicMock( + headers={"Content-Type": "multipart/mixed; boundary=batch_xyz"}, + text="", + ) + + self.client._odata = mock_od + self.batch.records.create("account", {"name": "Contoso"}) + + with patch( + "PowerPlatform.Dataverse.data._batch._BatchClient.execute", + return_value=mock_result, + ) as mock_exec: + result = self.batch.execute() + mock_exec.assert_called_once_with(self.batch._items, continue_on_error=False) + self.assertIs(result, mock_result) + + def test_execute_continue_on_error_passed_through(self): + """continue_on_error=True should be forwarded to _BatchClient.execute.""" + mock_result = BatchResult() + with patch( + "PowerPlatform.Dataverse.data._batch._BatchClient.execute", + return_value=mock_result, + ) as mock_exec: + self.client._odata = MagicMock() + self.batch.execute(continue_on_error=True) + mock_exec.assert_called_once_with(self.batch._items, continue_on_error=True) + + +class TestBatchRecordOperations(unittest.TestCase): + """Tests that BatchRecordOperations appends the correct intent objects.""" + + def setUp(self): + self.mock_credential = MagicMock(spec=TokenCredential) + self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) + self.batch = self.client.batch.new() + + def test_create_single_appends_record_create(self): + self.batch.records.create("account", {"name": "Contoso"}) + self.assertEqual(len(self.batch._items), 1) + item = self.batch._items[0] + self.assertIsInstance(item, _RecordCreate) + self.assertEqual(item.table, "account") + self.assertEqual(item.data, {"name": "Contoso"}) + + def test_create_list_appends_record_create(self): + data = [{"name": "A"}, {"name": "B"}] + self.batch.records.create("account", data) + item = self.batch._items[0] + self.assertIsInstance(item, _RecordCreate) + self.assertIs(item.data, data) + + def test_update_single_appends_record_update(self): + self.batch.records.update("account", "guid-1", {"name": "X"}) + item = self.batch._items[0] + self.assertIsInstance(item, _RecordUpdate) + self.assertEqual(item.table, "account") + self.assertEqual(item.ids, "guid-1") + self.assertEqual(item.changes, {"name": "X"}) + + def test_update_list_appends_record_update(self): + ids = ["guid-1", "guid-2"] + changes = {"statecode": 0} + self.batch.records.update("account", ids, changes) + item = self.batch._items[0] + self.assertIsInstance(item, _RecordUpdate) + self.assertIs(item.ids, ids) + + def test_delete_single_appends_record_delete(self): + self.batch.records.delete("account", "guid-to-del") + item = self.batch._items[0] + self.assertIsInstance(item, _RecordDelete) + self.assertEqual(item.table, "account") + self.assertEqual(item.ids, "guid-to-del") + self.assertTrue(item.use_bulk_delete) + + def test_delete_list_use_bulk_delete_false(self): + self.batch.records.delete("account", ["g1", "g2"], use_bulk_delete=False) + item = self.batch._items[0] + self.assertIsInstance(item, _RecordDelete) + self.assertFalse(item.use_bulk_delete) + + def test_get_single_appends_record_get(self): + self.batch.records.get("account", "guid-1", select=["name"]) + item = self.batch._items[0] + self.assertIsInstance(item, _RecordGet) + self.assertEqual(item.table, "account") + self.assertEqual(item.record_id, "guid-1") + self.assertEqual(item.select, ["name"]) + + def test_get_single_no_select(self): + self.batch.records.get("account", "guid-1") + item = self.batch._items[0] + self.assertIsNone(item.select) + + def test_multiple_ops_appended_in_order(self): + self.batch.records.create("account", {"name": "X"}) + self.batch.records.delete("account", "g1") + self.batch.records.get("account", "g2") + self.assertEqual(len(self.batch._items), 3) + self.assertIsInstance(self.batch._items[0], _RecordCreate) + self.assertIsInstance(self.batch._items[1], _RecordDelete) + self.assertIsInstance(self.batch._items[2], _RecordGet) + + +class TestBatchTableOperations(unittest.TestCase): + """Tests that BatchTableOperations appends the correct intent objects.""" + + def setUp(self): + self.mock_credential = MagicMock(spec=TokenCredential) + self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) + self.batch = self.client.batch.new() + + def test_create_appends_table_create(self): + cols = {"new_Price": "decimal"} + self.batch.tables.create("new_Product", cols, solution="Sol", primary_column="new_Name") + item = self.batch._items[0] + self.assertIsInstance(item, _TableCreate) + self.assertEqual(item.table, "new_Product") + self.assertIs(item.columns, cols) + self.assertEqual(item.solution, "Sol") + self.assertEqual(item.primary_column, "new_Name") + + def test_delete_appends_table_delete(self): + self.batch.tables.delete("new_Product") + item = self.batch._items[0] + self.assertIsInstance(item, _TableDelete) + self.assertEqual(item.table, "new_Product") + + def test_get_appends_table_get(self): + self.batch.tables.get("new_Product") + item = self.batch._items[0] + self.assertIsInstance(item, _TableGet) + self.assertEqual(item.table, "new_Product") + + def test_list_appends_table_list(self): + self.batch.tables.list() + item = self.batch._items[0] + self.assertIsInstance(item, _TableList) + + def test_add_columns_appends_table_add_columns(self): + cols = {"new_Notes": "string"} + self.batch.tables.add_columns("new_Product", cols) + item = self.batch._items[0] + self.assertIsInstance(item, _TableAddColumns) + self.assertEqual(item.table, "new_Product") + self.assertIs(item.columns, cols) + + def test_remove_columns_single_string(self): + self.batch.tables.remove_columns("new_Product", "new_Notes") + item = self.batch._items[0] + self.assertIsInstance(item, _TableRemoveColumns) + self.assertEqual(item.columns, "new_Notes") + + def test_remove_columns_list(self): + self.batch.tables.remove_columns("new_Product", ["new_A", "new_B"]) + item = self.batch._items[0] + self.assertIsInstance(item, _TableRemoveColumns) + self.assertEqual(item.columns, ["new_A", "new_B"]) + + def test_create_one_to_many_appends_intent(self): + lookup = MagicMock() + relationship = MagicMock() + self.batch.tables.create_one_to_many_relationship(lookup, relationship, solution="Sol") + item = self.batch._items[0] + self.assertIsInstance(item, _TableCreateOneToMany) + self.assertIs(item.lookup, lookup) + self.assertIs(item.relationship, relationship) + self.assertEqual(item.solution, "Sol") + + def test_create_many_to_many_appends_intent(self): + relationship = MagicMock() + self.batch.tables.create_many_to_many_relationship(relationship) + item = self.batch._items[0] + self.assertIsInstance(item, _TableCreateManyToMany) + self.assertIs(item.relationship, relationship) + self.assertIsNone(item.solution) + + def test_delete_relationship_appends_intent(self): + self.batch.tables.delete_relationship("rel-guid-1") + item = self.batch._items[0] + self.assertIsInstance(item, _TableDeleteRelationship) + self.assertEqual(item.relationship_id, "rel-guid-1") + + def test_get_relationship_appends_intent(self): + self.batch.tables.get_relationship("new_Dept_Emp") + item = self.batch._items[0] + self.assertIsInstance(item, _TableGetRelationship) + self.assertEqual(item.schema_name, "new_Dept_Emp") + + def test_create_lookup_field_appends_intent(self): + self.batch.tables.create_lookup_field( + "new_order", + "new_ProductId", + "new_product", + display_name="Product", + solution="Sol", + ) + item = self.batch._items[0] + self.assertIsInstance(item, _TableCreateLookupField) + self.assertEqual(item.referencing_table, "new_order") + self.assertEqual(item.lookup_field_name, "new_ProductId") + self.assertEqual(item.referenced_table, "new_product") + self.assertEqual(item.display_name, "Product") + self.assertEqual(item.solution, "Sol") + + +class TestBatchQueryOperations(unittest.TestCase): + """Tests that BatchQueryOperations appends the correct intent objects.""" + + def setUp(self): + self.mock_credential = MagicMock(spec=TokenCredential) + self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) + self.batch = self.client.batch.new() + + def test_sql_appends_query_sql(self): + self.batch.query.sql("SELECT accountid FROM account") + item = self.batch._items[0] + self.assertIsInstance(item, _QuerySql) + self.assertEqual(item.sql, "SELECT accountid FROM account") + + def test_sql_strips_whitespace(self): + self.batch.query.sql(" SELECT name FROM account ") + item = self.batch._items[0] + self.assertEqual(item.sql, "SELECT name FROM account") + + def test_sql_empty_raises(self): + with self.assertRaises(ValidationError): + self.batch.query.sql("") + + def test_sql_whitespace_only_raises(self): + with self.assertRaises(ValidationError): + self.batch.query.sql(" ") + + def test_sql_non_string_raises(self): + with self.assertRaises((ValidationError, AttributeError)): + self.batch.query.sql(None) + + +class TestChangeSet(unittest.TestCase): + """Tests for ChangeSet context-manager and ChangeSetRecordOperations.""" + + def setUp(self): + self.mock_credential = MagicMock(spec=TokenCredential) + self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) + self.batch = self.client.batch.new() + + def test_changeset_records_is_change_set_record_ops(self): + cs = self.batch.changeset() + self.assertIsInstance(cs.records, ChangeSetRecordOperations) + + def test_changeset_create_returns_content_id_ref(self): + cs = self.batch.changeset() + ref = cs.records.create("account", {"name": "X"}) + self.assertIsInstance(ref, str) + self.assertTrue(ref.startswith("$")) + + def test_changeset_create_content_ids_increment(self): + cs = self.batch.changeset() + ref1 = cs.records.create("account", {"name": "A"}) + ref2 = cs.records.create("contact", {"firstname": "B"}) + self.assertNotEqual(ref1, ref2) + n1 = int(ref1[1:]) + n2 = int(ref2[1:]) + self.assertGreater(n2, n1) + + def test_changeset_update_adds_operation(self): + cs = self.batch.changeset() + cs.records.update("account", "guid-1", {"name": "Y"}) + internal = self.batch._items[0] + self.assertIsInstance(internal, _ChangeSet) + self.assertEqual(len(internal.operations), 1) + + def test_changeset_delete_adds_operation(self): + cs = self.batch.changeset() + cs.records.delete("account", "guid-del") + internal = self.batch._items[0] + self.assertEqual(len(internal.operations), 1) + + def test_changeset_as_context_manager(self): + with self.batch.changeset() as cs: + cs.records.create("account", {"name": "ACME"}) + internal = self.batch._items[0] + self.assertEqual(len(internal.operations), 1) + + def test_changeset_ops_in_order(self): + cs = self.batch.changeset() + ref = cs.records.create("lead", {"firstname": "Ada"}) + cs.records.update("contact", ref, {"lastname": "L"}) + cs.records.delete("task", "task-guid") + internal = self.batch._items[0] + self.assertEqual(len(internal.operations), 3) + self.assertIsInstance(internal.operations[0], _RecordCreate) + self.assertIsInstance(internal.operations[1], _RecordUpdate) + self.assertIsInstance(internal.operations[2], _RecordDelete) + + +class TestBatchItemResponse(unittest.TestCase): + """Tests for BatchItemResponse model.""" + + def test_is_success_2xx(self): + for code in (200, 201, 204): + item = BatchItemResponse(status_code=code) + self.assertTrue(item.is_success, f"Expected is_success for {code}") + + def test_is_not_success_4xx(self): + for code in (400, 404, 409): + item = BatchItemResponse(status_code=code) + self.assertFalse(item.is_success, f"Expected not is_success for {code}") + + def test_is_not_success_5xx(self): + item = BatchItemResponse(status_code=500) + self.assertFalse(item.is_success) + + +class TestBatchResult(unittest.TestCase): + """Tests for BatchResult model.""" + + def test_succeeded(self): + responses = [ + BatchItemResponse(status_code=204), + BatchItemResponse(status_code=400), + BatchItemResponse(status_code=201, entity_id="guid-1"), + ] + result = BatchResult(responses=responses) + self.assertEqual(len(result.succeeded), 2) + self.assertEqual(len(result.failed), 1) + + def test_has_errors_true(self): + result = BatchResult( + responses=[ + BatchItemResponse(status_code=204), + BatchItemResponse(status_code=404), + ] + ) + self.assertTrue(result.has_errors) + + def test_has_errors_false(self): + result = BatchResult( + responses=[ + BatchItemResponse(status_code=204), + BatchItemResponse(status_code=201), + ] + ) + self.assertFalse(result.has_errors) + + def test_entity_ids(self): + result = BatchResult( + responses=[ + BatchItemResponse(status_code=201, entity_id="guid-1"), + BatchItemResponse(status_code=204), + BatchItemResponse(status_code=201, entity_id="guid-2"), + BatchItemResponse(status_code=400), + ] + ) + self.assertEqual(result.entity_ids, ["guid-1", "guid-2"]) + + def test_empty_result(self): + result = BatchResult() + self.assertEqual(result.responses, []) + self.assertEqual(result.succeeded, []) + self.assertEqual(result.failed, []) + self.assertFalse(result.has_errors) + self.assertEqual(result.entity_ids, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_batch_scenarios.py b/tests/unit/test_batch_scenarios.py new file mode 100644 index 00000000..610dcabb --- /dev/null +++ b/tests/unit/test_batch_scenarios.py @@ -0,0 +1,535 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Batch API user scenario tests. + +Each test documents a real-world scenario a developer might encounter, +explains expected behavior, and verifies the correct access pattern. +These tests serve as executable documentation for SDK consumers. +""" + +import unittest +from unittest.mock import MagicMock + +from PowerPlatform.Dataverse.data._batch import ( + _BatchClient, + _ChangeSet, + _RecordGet, +) +from PowerPlatform.Dataverse.core.errors import ValidationError +from PowerPlatform.Dataverse.data._raw_request import _RawRequest +from PowerPlatform.Dataverse.models.batch import BatchItemResponse, BatchResult + + +def _make_od(): + od = MagicMock() + od.api = "https://org.crm.dynamics.com/api/data/v9.2" + return od + + +# --------------------------------------------------------------------------- +# Scenario 1: Response ordering matches operation order +# --------------------------------------------------------------------------- + + +class TestScenario_ResponseOrdering(unittest.TestCase): + """Scenario: I add create, get, delete to a batch. Are responses in the same order? + + YES -- result.responses[0] corresponds to the first operation, + result.responses[1] to the second, etc. The OData $batch spec guarantees + response order matches request order. + """ + + def test_responses_are_in_submission_order(self): + """Three operations produce three responses in the same order.""" + responses = [ + BatchItemResponse(status_code=204, entity_id="id-from-create"), + BatchItemResponse(status_code=200, data={"name": "Contoso"}), + BatchItemResponse(status_code=204), # delete + ] + result = BatchResult(responses=responses) + # First response is the create + self.assertEqual(result.responses[0].entity_id, "id-from-create") + # Second response is the get + self.assertEqual(result.responses[1].data["name"], "Contoso") + # Third response is the delete + self.assertIsNone(result.responses[2].entity_id) + self.assertIsNone(result.responses[2].data) + + +# --------------------------------------------------------------------------- +# Scenario 2: CreateMultiple IDs are NOT in entity_ids +# --------------------------------------------------------------------------- + + +class TestScenario_CreateMultipleIDs(unittest.TestCase): + """Scenario: I create 100 records via batch.records.create(table, [list]). + Where are the IDs? + + CreateMultiple is a Dataverse bound action (POST to .../CreateMultiple). + It returns 200 OK with {"Ids": ["guid-1", "guid-2", ...]} in the body. + entity_ids only collects from OData-EntityId headers, which CreateMultiple + does NOT return. + + Access via: result.succeeded[n].data["Ids"] + """ + + def test_bulk_create_ids_in_response_data(self): + """CreateMultiple IDs are in data['Ids'], not in entity_ids.""" + resp = BatchItemResponse( + status_code=200, + data={"Ids": ["aaa-111", "bbb-222", "ccc-333"]}, + ) + result = BatchResult(responses=[resp]) + # entity_ids is EMPTY for CreateMultiple + self.assertEqual(result.entity_ids, []) + # Access IDs from the response body + self.assertEqual(resp.data["Ids"], ["aaa-111", "bbb-222", "ccc-333"]) + + def test_single_create_id_in_entity_ids(self): + """Individual POST create returns entity_id via OData-EntityId header.""" + resp = BatchItemResponse(status_code=204, entity_id="single-guid") + result = BatchResult(responses=[resp]) + self.assertEqual(result.entity_ids, ["single-guid"]) + + +# --------------------------------------------------------------------------- +# Scenario 3: Update responses also have entity_id +# --------------------------------------------------------------------------- + + +class TestScenario_UpdateReturnsEntityId(unittest.TestCase): + """Scenario: I do a create + update in a batch. entity_ids has BOTH GUIDs. + + PATCH (update) also returns OData-EntityId with the updated record GUID. + So entity_ids contains IDs from both creates AND updates. + """ + + def test_entity_ids_includes_both_creates_and_updates(self): + """entity_ids has GUIDs from POST creates AND PATCH updates.""" + responses = [ + BatchItemResponse(status_code=204, entity_id="created-guid"), + BatchItemResponse(status_code=204, entity_id="updated-guid"), + ] + result = BatchResult(responses=responses) + self.assertEqual(result.entity_ids, ["created-guid", "updated-guid"]) + + +# --------------------------------------------------------------------------- +# Scenario 4: GET response -- data, not entity_id +# --------------------------------------------------------------------------- + + +class TestScenario_GetResponse(unittest.TestCase): + """Scenario: I add a GET to my batch. How do I access the record data? + + GET returns 200 OK with the record JSON in the body. + data contains the parsed JSON. entity_id is None. + """ + + def test_get_response_has_data_not_entity_id(self): + """GET response: data has the record, entity_id is None.""" + resp = BatchItemResponse( + status_code=200, + data={"name": "Contoso", "accountid": "guid-1"}, + ) + result = BatchResult(responses=[resp]) + # No entity_id for GETs + self.assertEqual(result.entity_ids, []) + # Data has the record + self.assertEqual(resp.data["name"], "Contoso") + + +# --------------------------------------------------------------------------- +# Scenario 5: DELETE response -- no data, no entity_id +# --------------------------------------------------------------------------- + + +class TestScenario_DeleteResponse(unittest.TestCase): + """Scenario: I delete records in a batch. What does the response look like? + + DELETE returns 204 No Content. No entity_id, no data. + Check is_success to verify the delete worked. + """ + + def test_delete_response_is_empty(self): + """DELETE response: 204, no entity_id, no data.""" + resp = BatchItemResponse(status_code=204) + self.assertTrue(resp.is_success) + self.assertIsNone(resp.entity_id) + self.assertIsNone(resp.data) + + +# --------------------------------------------------------------------------- +# Scenario 6: SQL query result -- how to get rows +# --------------------------------------------------------------------------- + + +class TestScenario_SqlQueryResult(unittest.TestCase): + """Scenario: I add a SQL query to a batch. How do I get the result rows? + + SQL returns 200 OK with {"value": [row1, row2, ...]} in the body. + Access via: result.responses[n].data["value"] + """ + + def test_sql_query_result_in_data_value(self): + """SQL query: rows are in data['value'].""" + rows = [{"name": "Contoso"}, {"name": "Fabrikam"}] + resp = BatchItemResponse(status_code=200, data={"value": rows}) + result = BatchResult(responses=[resp]) + self.assertEqual(len(resp.data["value"]), 2) + self.assertEqual(resp.data["value"][0]["name"], "Contoso") + + +# --------------------------------------------------------------------------- +# Scenario 7: Empty batch -- execute with no operations +# --------------------------------------------------------------------------- + + +class TestScenario_EmptyBatch(unittest.TestCase): + """Scenario: I create a batch but add no operations. What happens? + + execute() returns an empty BatchResult. No HTTP request is sent. + """ + + def test_empty_batch_returns_empty_result(self): + """Empty batch returns empty result without HTTP call.""" + od = _make_od() + client = _BatchClient(od) + result = client.execute([], continue_on_error=False) + self.assertEqual(len(result.responses), 0) + self.assertFalse(result.has_errors) + self.assertEqual(result.entity_ids, []) + # No HTTP call was made + od._request.assert_not_called() + + +# --------------------------------------------------------------------------- +# Scenario 8: Double execute -- calling execute() twice +# --------------------------------------------------------------------------- + + +class TestScenario_DoubleExecute(unittest.TestCase): + """Scenario: I call batch.execute() twice. Is it safe? + + YES -- each execute() builds a fresh multipart body from the items list. + The items are still in the batch so the same operations execute again. + This is safe (idempotent for GETs, creates new records for POSTs). + """ + + def test_execute_twice_sends_two_requests(self): + """Calling execute twice makes two HTTP requests.""" + od = _make_od() + od._build_get.return_value = _RawRequest(method="GET", url="https://org/api/data/v9.2/accounts(g)") + mock_resp = MagicMock() + mock_resp.headers = {"Content-Type": 'multipart/mixed; boundary="b"'} + mock_resp.text = "--b--\r\n" + od._request.return_value = mock_resp + + client = _BatchClient(od) + items = [_RecordGet(table="account", record_id="g")] + client.execute(items) + client.execute(items) + self.assertEqual(od._request.call_count, 2) + + +# --------------------------------------------------------------------------- +# Scenario 9: Content-ID scope -- only within same changeset +# --------------------------------------------------------------------------- + + +class TestScenario_ContentIdScope(unittest.TestCase): + """Scenario: Can I use a $ref from changeset 1 in changeset 2? + + NO -- Content-ID references ($n) are only valid within the same changeset. + The OData spec says: "The link can only be to an entity created earlier + in the same change set." Using $1 from CS1 in CS2 causes a 400 error. + + The SDK enforces this by design: ChangeSetRecordOperations.create() + returns $n, but that reference is only meaningful within the same + changeset context manager. + """ + + def test_content_ids_are_per_changeset_scope(self): + """Content-ID refs from one changeset are not usable in another.""" + counter = [1] + cs1 = _ChangeSet(_counter=counter) + cs2 = _ChangeSet(_counter=counter) + + ref1 = cs1.add_create("account", {"name": "A"}) + # ref1 is "$1" -- only valid in cs1 + self.assertEqual(ref1, "$1") + + # If cs2 tried to use "$1", it would be a different operation + # The SDK doesn't prevent this at build time (it can't know the intent), + # but Dataverse will return: "Content-ID Reference: '$1' does not exist" + ref2 = cs2.add_create("contact", {"firstname": "B"}) + # cs2 gets "$2" (unique due to shared counter) + self.assertEqual(ref2, "$2") + + +# --------------------------------------------------------------------------- +# Scenario 10: add_columns contributes multiple responses +# --------------------------------------------------------------------------- + + +class TestScenario_AddColumnsMultipleResponses(unittest.TestCase): + """Scenario: I call batch.tables.add_columns(table, {"col1": "string", "col2": "int"}). + How many responses will I get? + + TWO -- add_columns creates one HTTP request per column. + result.responses will have 2 entries for 2 columns. + """ + + def test_add_columns_response_count_matches_column_count(self): + """N columns = N responses in the result.""" + # Simulate 3 column-creates, each returning 204 + responses = [ + BatchItemResponse(status_code=204), + BatchItemResponse(status_code=204), + BatchItemResponse(status_code=204), + ] + result = BatchResult(responses=responses) + self.assertEqual(len(result.succeeded), 3) + + +# --------------------------------------------------------------------------- +# Scenario 11: tables.create returns 204, no metadata +# --------------------------------------------------------------------------- + + +class TestScenario_TableCreateNoMetadata(unittest.TestCase): + """Scenario: I create a table in a batch. Can I get its MetadataId? + + NO -- tables.create in a batch returns 204 No Content. + The response has no body (data is None). You need a follow-up + batch.tables.get() to retrieve the table metadata. + """ + + def test_table_create_returns_no_data(self): + """Table create response: 204, no data.""" + resp = BatchItemResponse(status_code=204) + self.assertTrue(resp.is_success) + self.assertIsNone(resp.data) + + +# --------------------------------------------------------------------------- +# Scenario 12: continue_on_error behavior +# --------------------------------------------------------------------------- + + +class TestScenario_ContinueOnError(unittest.TestCase): + """Scenario: Without continue_on_error, first failure stops the batch. + With it, all operations are attempted. + + Without: Batch returns HTTP 400. Only the failed op's response is present. + With: Batch returns HTTP 200. All operations attempted. Check result.failed. + """ + + def test_without_continue_on_error_one_failure(self): + """Without continue_on_error, only the error response is returned.""" + responses = [ + BatchItemResponse( + status_code=404, + error_message="Record not found", + error_code="0x80040217", + ), + ] + result = BatchResult(responses=responses) + self.assertTrue(result.has_errors) + self.assertEqual(len(result.failed), 1) + self.assertEqual(len(result.succeeded), 0) + + def test_with_continue_on_error_mixed(self): + """With continue_on_error, all ops attempted, mixed results.""" + responses = [ + BatchItemResponse(status_code=404, error_message="not found"), + BatchItemResponse(status_code=204, entity_id="good-id"), + BatchItemResponse(status_code=200, data={"value": []}), + ] + result = BatchResult(responses=responses) + self.assertTrue(result.has_errors) + self.assertEqual(len(result.failed), 1) + self.assertEqual(len(result.succeeded), 2) + + +# --------------------------------------------------------------------------- +# Scenario 13: Changeset rollback -- what the error looks like +# --------------------------------------------------------------------------- + + +class TestScenario_ChangesetRollback(unittest.TestCase): + """Scenario: One op in my changeset fails. What happens to the others? + + ALL operations in the changeset are rolled back. No records are created. + The response contains a single error for the failed operation. + entity_ids will be empty (nothing was persisted). + """ + + def test_changeset_failure_produces_single_error(self): + """Failed changeset: one error response, no entity_ids.""" + responses = [ + BatchItemResponse( + status_code=404, + error_message="referenced record not found", + content_id="2", + ), + ] + result = BatchResult(responses=responses) + self.assertTrue(result.has_errors) + self.assertEqual(result.entity_ids, []) + self.assertEqual(result.failed[0].content_id, "2") + + +# --------------------------------------------------------------------------- +# Scenario 14: DataFrame create -- entity_ids will be empty +# --------------------------------------------------------------------------- + + +class TestScenario_DataFrameCreateIds(unittest.TestCase): + """Scenario: I use batch.dataframe.create(table, df). Where are the IDs? + + batch.dataframe.create() calls batch.records.create(table, list_of_dicts), + which uses CreateMultiple. The response is 200 OK with {"Ids": [...]}. + entity_ids is empty. Access IDs via result.succeeded[n].data["Ids"]. + """ + + def test_dataframe_create_ids_pattern(self): + """DataFrame create: IDs in data['Ids'], NOT in entity_ids.""" + resp = BatchItemResponse( + status_code=200, + data={"Ids": ["df-id-1", "df-id-2", "df-id-3"]}, + ) + result = BatchResult(responses=[resp]) + # entity_ids is empty + self.assertEqual(result.entity_ids, []) + # Access pattern for callers + ids = [] + for r in result.succeeded: + if r.data and "Ids" in r.data: + ids.extend(r.data["Ids"]) + self.assertEqual(ids, ["df-id-1", "df-id-2", "df-id-3"]) + + +# --------------------------------------------------------------------------- +# Scenario 15: Mixed batch with changeset + standalone ops +# --------------------------------------------------------------------------- + + +class TestScenario_MixedBatchOrdering(unittest.TestCase): + """Scenario: I have a changeset (2 creates) then a standalone GET. + How are responses ordered? + + Changeset responses come first (in their position), then standalone. + responses[0] and [1] are from the changeset, responses[2] is the GET. + """ + + def test_changeset_responses_then_standalone(self): + """Changeset responses first, then standalone ops.""" + responses = [ + BatchItemResponse(status_code=204, entity_id="cs-create-1", content_id="1"), + BatchItemResponse(status_code=204, entity_id="cs-create-2", content_id="2"), + BatchItemResponse(status_code=200, data={"name": "Existing"}), + ] + result = BatchResult(responses=responses) + # Changeset creates + self.assertEqual(result.responses[0].content_id, "1") + self.assertEqual(result.responses[1].content_id, "2") + # Standalone GET + self.assertIsNone(result.responses[2].content_id) + self.assertEqual(result.responses[2].data["name"], "Existing") + + +# --------------------------------------------------------------------------- +# Scenario 16: Checking individual response status +# --------------------------------------------------------------------------- + + +class TestScenario_IndividualResponseStatus(unittest.TestCase): + """Scenario: I need to check if each specific operation succeeded or failed. + + Iterate result.responses and use is_success, status_code, error_message. + """ + + def test_iterate_and_check_each_response(self): + """Check individual response status codes and errors.""" + responses = [ + BatchItemResponse(status_code=204, entity_id="id-1"), + BatchItemResponse(status_code=400, error_message="bad field", error_code="0x80044331"), + BatchItemResponse(status_code=200, data={"value": []}), + ] + result = BatchResult(responses=responses) + + # Pattern: iterate with index to correlate with operations + for i, resp in enumerate(result.responses): + if resp.is_success: + if resp.entity_id: + pass # create/update succeeded + elif resp.data: + pass # GET/SQL query succeeded + else: + pass # delete succeeded + else: + self.assertEqual(resp.error_code, "0x80044331") + self.assertIn("bad field", resp.error_message) + + +# --------------------------------------------------------------------------- +# Scenario 17: Batch max size validation +# --------------------------------------------------------------------------- + + +class TestScenario_BatchMaxSize(unittest.TestCase): + """Scenario: I add 1001 operations. What happens? + + ValidationError is raised BEFORE any HTTP request is sent. + The error message includes the count and the 1000 limit. + """ + + def test_over_1000_raises_before_sending(self): + """1001+ operations raise ValidationError pre-flight.""" + od = _make_od() + od._build_get.return_value = _RawRequest(method="GET", url="https://org/x") + client = _BatchClient(od) + items = [_RecordGet(table="account", record_id=f"g-{i}") for i in range(1001)] + with self.assertRaises(ValidationError) as ctx: + client.execute(items) + self.assertIn("1001", str(ctx.exception)) + self.assertIn("1000", str(ctx.exception)) + # No HTTP request was made + od._request.assert_not_called() + + +# --------------------------------------------------------------------------- +# Scenario 18: Error response parsing +# --------------------------------------------------------------------------- + + +class TestScenario_ErrorResponseFields(unittest.TestCase): + """Scenario: An operation fails. What fields are available on the error? + + error_message: Human-readable error text from Dataverse + error_code: Hex error code (e.g. "0x80040217") + status_code: HTTP status (e.g. 404, 400, 500) + is_success: False + data: None (error responses don't have data) + """ + + def test_error_response_fields(self): + """Failed response has all error fields populated.""" + resp = BatchItemResponse( + status_code=404, + error_message="account With Id = 00000000 Does Not Exist", + error_code="0x80040217", + ) + self.assertFalse(resp.is_success) + self.assertEqual(resp.status_code, 404) + self.assertEqual(resp.error_code, "0x80040217") + self.assertIn("Does Not Exist", resp.error_message) + self.assertIsNone(resp.data) + self.assertIsNone(resp.entity_id) + + +if __name__ == "__main__": + unittest.main() From 78cd852c977c784f763da33a7eeff8db0395e67d Mon Sep 17 00:00:00 2001 From: abelmilash-msft Date: Wed, 8 Apr 2026 14:38:52 -0700 Subject: [PATCH 18/20] Optimize picklist label resolution with bulk PicklistAttributeMetadata fetch (#154) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Reduces API calls during picklist label-to-integer resolution by fetching all picklist attributes and their options for the entire table in a single API call using the `PicklistAttributeMetadata` cast, instead of checking each attribute individually. Results are cached with a 1-hour TTL. ## Changes **`src/PowerPlatform/Dataverse/data/_odata.py`** - Add `_bulk_fetch_picklists()` — single API call to fetch all picklist attributes and their options for a table - Add `_request_metadata_with_retry()` — exponential backoff on transient metadata errors - Simplify `_convert_labels_to_ints()` — calls `_bulk_fetch_picklists` then resolves labels from cache **`tests/unit/data/test_odata_internal.py`** - Rewrite `TestPicklistLabelResolution` class with 50 unit tests covering `_bulk_fetch_picklists`, `_request_metadata_with_retry`, `_convert_labels_to_ints`, integration through `_create`/`_update`/`_upsert`, and edge cases **`examples/advanced/walkthrough.py`** - Add picklist label update test to Section 10 (verifies both create and update with string labels) ## Performance impact Cold cache API calls reduced from `n + p` to always `1`, where `p` = picklist fields, `n` = string fields. | Picklist Columns | Before Calls | Before Time | After Calls | After Time | Speedup | |-----------------|-------------|-------------|-------------|------------|---------| | 1 | 2 | 0.6s | 1 | 0.3s | 2x | | 10 | 11 | 3.3s | 1 | 0.4s | 9x | | 100 | 101 | 34s | 1 | 0.6s | 55x | | 250 | 251 | 79s | 1 | 1.2s | 64x | | 400 | 401 | 119s | 1 | 1.3s | 92x | Repeat operations use a 1-hour TTL cache (0 API calls, <5ms). ## Testing - 660 unit tests passing - Performance benchmarks verified against live Dataverse environment (5 runs each) --------- Co-authored-by: Abel Milash Co-authored-by: Claude Sonnet 4.6 --- examples/advanced/walkthrough.py | 10 + src/PowerPlatform/Dataverse/data/_odata.py | 190 +++-- tests/unit/data/test_odata_internal.py | 775 ++++++++++++++++++++- tests/unit/test_context_manager.py | 2 +- 4 files changed, 858 insertions(+), 119 deletions(-) diff --git a/examples/advanced/walkthrough.py b/examples/advanced/walkthrough.py index 52c4d7a3..7740bff7 100644 --- a/examples/advanced/walkthrough.py +++ b/examples/advanced/walkthrough.py @@ -445,6 +445,16 @@ def _run_walkthrough(client): print(f" new_Priority stored as integer: {retrieved.get('new_priority')}") print(f" new_Priority@FormattedValue: {retrieved.get('new_priority@OData.Community.Display.V1.FormattedValue')}") + # Update with a string label + log_call(f"client.records.update('{table_name}', label_id, {{'new_Priority': 'Low'}})") + backoff(lambda: client.records.update(table_name, label_id, {"new_Priority": "Low"})) + updated_label = backoff(lambda: client.records.get(table_name, label_id)) + print(f"[OK] Updated record with string label 'Low' for new_Priority") + print(f" new_Priority stored as integer: {updated_label.get('new_priority')}") + print( + f" new_Priority@FormattedValue: {updated_label.get('new_priority@OData.Community.Display.V1.FormattedValue')}" + ) + # ============================================================================ # 11. COLUMN MANAGEMENT # ============================================================================ diff --git a/src/PowerPlatform/Dataverse/data/_odata.py b/src/PowerPlatform/Dataverse/data/_odata.py index aca42f3d..a3ccaefe 100644 --- a/src/PowerPlatform/Dataverse/data/_odata.py +++ b/src/PowerPlatform/Dataverse/data/_odata.py @@ -171,8 +171,7 @@ def __init__( self._logical_to_entityset_cache: dict[str, str] = {} # Cache: normalized table_schema_name (lowercase) -> primary id attribute (e.g. accountid) self._logical_primaryid_cache: dict[str, str] = {} - # Picklist label cache: (normalized_table_schema_name, normalized_attribute) -> {'map': {...}, 'ts': epoch_seconds} - self._picklist_label_cache = {} + self._picklist_label_cache: dict[str, dict] = {} self._picklist_cache_ttl_seconds = 3600 # 1 hour TTL @contextmanager @@ -1134,141 +1133,118 @@ def _normalize_picklist_label(self, label: str) -> str: norm = re.sub(r"\s+", " ", norm).strip().lower() return norm - def _optionset_map(self, table_schema_name: str, attr_logical: str) -> Optional[Dict[str, int]]: - """Build or return cached mapping of normalized label -> value for a picklist attribute. - - Returns empty dict if attribute is not a picklist or has no options. Returns None only - for invalid inputs or unexpected metadata parse failures. - - Notes - ----- - - This method calls the Web API twice per attribute so it could have perf impact when there are lots of columns on the entity. - """ - if not table_schema_name or not attr_logical: - return None - # Normalize cache key for case-insensitive lookups - cache_key = (self._normalize_cache_key(table_schema_name), self._normalize_cache_key(attr_logical)) - now = time.time() - entry = self._picklist_label_cache.get(cache_key) - if isinstance(entry, dict) and "map" in entry and (now - entry.get("ts", 0)) < self._picklist_cache_ttl_seconds: - return entry["map"] - - # LogicalNames in Dataverse are stored in lowercase, so we need to lowercase for filters - attr_esc = self._escape_odata_quotes(attr_logical.lower()) - table_schema_name_esc = self._escape_odata_quotes(table_schema_name.lower()) - - # Step 1: lightweight fetch (no expand) to determine attribute type - url_type = ( - f"{self.api}/EntityDefinitions(LogicalName='{table_schema_name_esc}')/Attributes" - f"?$filter=LogicalName eq '{attr_esc}'&$select=LogicalName,AttributeType" - ) - # Retry on 404 (metadata not yet published) before surfacing the error. - r_type = None + def _request_metadata_with_retry(self, method: str, url: str, **kwargs): + """Fetch metadata with retries on transient errors.""" max_attempts = 5 backoff_seconds = 0.4 for attempt in range(1, max_attempts + 1): try: - r_type = self._request("get", url_type) - break + return self._request(method, url, **kwargs) except HttpError as err: if getattr(err, "status_code", None) == 404: if attempt < max_attempts: - # Exponential backoff: 0.4s, 0.8s, 1.6s, 3.2s time.sleep(backoff_seconds * (2 ** (attempt - 1))) continue - raise RuntimeError( - f"Picklist attribute metadata not found after retries: entity='{table_schema_name}' attribute='{attr_logical}' (404)" - ) from err + raise RuntimeError(f"Metadata request failed after {max_attempts} retries (404): {url}") from err raise - if r_type is None: - raise RuntimeError("Failed to retrieve attribute metadata due to repeated request failures.") - body_type = r_type.json() - items = body_type.get("value", []) if isinstance(body_type, dict) else [] - if not items: - return None - attr_md = items[0] - if attr_md.get("AttributeType") not in ("Picklist", "PickList"): - self._picklist_label_cache[cache_key] = {"map": {}, "ts": now} - return {} - - # Step 2: fetch with expand only now that we know it's a picklist - # Need to cast to the derived PicklistAttributeMetadata type; OptionSet is not a nav on base AttributeMetadata. - cast_url = ( - f"{self.api}/EntityDefinitions(LogicalName='{table_schema_name_esc}')/Attributes(LogicalName='{attr_esc}')/" - "Microsoft.Dynamics.CRM.PicklistAttributeMetadata?$select=LogicalName&$expand=OptionSet($select=Options)" + def _bulk_fetch_picklists(self, table_schema_name: str) -> None: + """Fetch all picklist attributes and their options for a table in one API call. + + Uses collection-level PicklistAttributeMetadata cast to retrieve every picklist + attribute on the table, including its OptionSet options. Populates the nested + cache so that ``_convert_labels_to_ints`` resolves labels without further API calls. + The Dataverse metadata API does not page results. + """ + table_key = self._normalize_cache_key(table_schema_name) + now = time.time() + table_entry = self._picklist_label_cache.get(table_key) + if isinstance(table_entry, dict) and (now - table_entry.get("ts", 0)) < self._picklist_cache_ttl_seconds: + return + + table_esc = self._escape_odata_quotes(table_schema_name.lower()) + url = ( + f"{self.api}/EntityDefinitions(LogicalName='{table_esc}')" + f"/Attributes/Microsoft.Dynamics.CRM.PicklistAttributeMetadata" + f"?$select=LogicalName&$expand=OptionSet($select=Options)" ) - # Step 2 fetch with retries: expanded OptionSet (cast form first) - r_opts = None - for attempt in range(1, max_attempts + 1): - try: - r_opts = self._request("get", cast_url) - break - except HttpError as err: - if getattr(err, "status_code", None) == 404: - if attempt < max_attempts: - time.sleep(backoff_seconds * (2 ** (attempt - 1))) - continue - raise RuntimeError( - f"Picklist OptionSet metadata not found after retries: entity='{table_schema_name}' attribute='{attr_logical}' (404)" - ) from err - raise - if r_opts is None: - raise RuntimeError("Failed to retrieve picklist OptionSet metadata due to repeated request failures.") + response = self._request_metadata_with_retry("get", url) + body = response.json() + items = body.get("value", []) if isinstance(body, dict) else [] - attr_full = {} - try: - attr_full = r_opts.json() if r_opts.text else {} - except ValueError: - return None - option_set = attr_full.get("OptionSet") or {} - options = option_set.get("Options") if isinstance(option_set, dict) else None - if not isinstance(options, list): - return None - mapping: Dict[str, int] = {} - for opt in options: - if not isinstance(opt, dict): + picklists: Dict[str, Dict[str, int]] = {} + for item in items: + if not isinstance(item, dict): continue - val = opt.get("Value") - if not isinstance(val, int): + ln = item.get("LogicalName", "").lower() + if not ln: continue - label_def = opt.get("Label") or {} - locs = label_def.get("LocalizedLabels") - if isinstance(locs, list): - for loc in locs: - if isinstance(loc, dict): - lab = loc.get("Label") - if isinstance(lab, str) and lab.strip(): - normalized = self._normalize_picklist_label(lab) - mapping.setdefault(normalized, val) - if mapping: - self._picklist_label_cache[cache_key] = {"map": mapping, "ts": now} - return mapping - # No options available - self._picklist_label_cache[cache_key] = {"map": {}, "ts": now} - return {} + option_set = item.get("OptionSet") or {} + options = option_set.get("Options") if isinstance(option_set, dict) else None + mapping: Dict[str, int] = {} + if isinstance(options, list): + for opt in options: + if not isinstance(opt, dict): + continue + val = opt.get("Value") + if not isinstance(val, int): + continue + label_def = opt.get("Label") or {} + locs = label_def.get("LocalizedLabels") + if isinstance(locs, list): + for loc in locs: + if isinstance(loc, dict): + lab = loc.get("Label") + if isinstance(lab, str) and lab.strip(): + normalized = self._normalize_picklist_label(lab) + mapping.setdefault(normalized, val) + picklists[ln] = mapping + + self._picklist_label_cache[table_key] = {"ts": now, "picklists": picklists} def _convert_labels_to_ints(self, table_schema_name: str, record: Dict[str, Any]) -> Dict[str, Any]: """Return a copy of record with any labels converted to option ints. Heuristic: For each string value, attempt to resolve against picklist metadata. If attribute isn't a picklist or label not found, value left unchanged. + + On first encounter of a table, bulk-fetches all picklist attributes and + their options in a single API call, then resolves labels from the warm cache. """ - out = record.copy() - for k, v in list(out.items()): + resolved_record = record.copy() + + # Check if there are any string-valued candidates worth resolving + has_candidates = any( + isinstance(v, str) and v.strip() and isinstance(k, str) and "@odata." not in k + for k, v in resolved_record.items() + ) + if not has_candidates: + return resolved_record + + # Bulk-fetch all picklists for this table (1 API call, cached for TTL) + self._bulk_fetch_picklists(table_schema_name) + + # Resolve labels from the nested cache + table_key = self._normalize_cache_key(table_schema_name) + table_entry = self._picklist_label_cache.get(table_key) + if not isinstance(table_entry, dict): + return resolved_record + picklists = table_entry.get("picklists", {}) + + for k, v in resolved_record.items(): if not isinstance(v, str) or not v.strip(): continue - # Skip OData annotations — they are not attribute names if isinstance(k, str) and "@odata." in k: continue - mapping = self._optionset_map(table_schema_name, k) - if not mapping: + attr_key = self._normalize_cache_key(k) + mapping = picklists.get(attr_key) + if not isinstance(mapping, dict) or not mapping: continue norm = self._normalize_picklist_label(v) val = mapping.get(norm) if val is not None: - out[k] = val - return out + resolved_record[k] = val + return resolved_record def _attribute_payload( self, column_schema_name: str, dtype: Any, *, is_primary_name: bool = False diff --git a/tests/unit/data/test_odata_internal.py b/tests/unit/data/test_odata_internal.py index b5b4dd79..2ed49791 100644 --- a/tests/unit/data/test_odata_internal.py +++ b/tests/unit/data/test_odata_internal.py @@ -3,7 +3,7 @@ import json import unittest -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch from PowerPlatform.Dataverse.core.errors import ValidationError from PowerPlatform.Dataverse.data._odata import _ODataClient @@ -494,23 +494,24 @@ def test_odata_bind_keys_preserve_case(self): def test_convert_labels_skips_odata_keys(self): """_convert_labels_to_ints should skip @odata.bind keys (no metadata lookup).""" - # Patch _optionset_map to track calls - calls = [] - original = self.od._optionset_map + import time - def tracking_optionset_map(table, attr): - calls.append(attr) - return original(table, attr) + # Pre-populate cache so no API call needed + self.od._picklist_label_cache["account"] = { + "ts": time.time(), + "picklists": {}, + } - self.od._optionset_map = tracking_optionset_map record = { "name": "Contoso", "new_CustomerId@odata.bind": "/contacts(00000000-0000-0000-0000-000000000001)", "@odata.type": "Microsoft.Dynamics.CRM.account", } - self.od._convert_labels_to_ints("account", record) - # Only "name" should be checked, not the @odata keys - self.assertEqual(calls, ["name"]) + result = self.od._convert_labels_to_ints("account", record) + # @odata keys must be left unchanged + self.assertEqual(result["new_CustomerId@odata.bind"], "/contacts(00000000-0000-0000-0000-000000000001)") + self.assertEqual(result["@odata.type"], "Microsoft.Dynamics.CRM.account") + self.assertEqual(result["name"], "Contoso") def test_returns_none(self): """_upsert always returns None.""" @@ -518,6 +519,758 @@ def test_returns_none(self): self.assertIsNone(result) +class TestPicklistLabelResolution(unittest.TestCase): + """Tests for picklist label-to-integer resolution. + + Covers _bulk_fetch_picklists, _request_metadata_with_retry, + _convert_labels_to_ints, and their integration through _create / _update / _upsert. + + Cache structure (nested): + _picklist_label_cache = { + "table_key": {"ts": epoch, "picklists": {"attr": {norm_label: int}}} + } + """ + + def setUp(self): + self.od = _make_odata_client() + + # ---- Helper to build a bulk-fetch API response ---- + @staticmethod + def _bulk_response(*picklists): + """Build a mock response for _bulk_fetch_picklists. + + Each picklist is (logical_name, [(value, label), ...]). + """ + items = [] + for ln, options in picklists: + opts = [{"Value": val, "Label": {"LocalizedLabels": [{"Label": lab}]}} for val, lab in options] + items.append({"LogicalName": ln, "OptionSet": {"Options": opts}}) + resp = MagicMock() + resp.json.return_value = {"value": items} + return resp + + # ---- _bulk_fetch_picklists ---- + + def test_bulk_fetch_populates_nested_cache(self): + """Bulk fetch stores picklists in nested {table: {ts, picklists: {...}}} format.""" + import time + + resp = self._bulk_response( + ("industrycode", [(6, "Technology"), (12, "Consulting")]), + ) + self.od._request.return_value = resp + + self.od._bulk_fetch_picklists("account") + + entry = self.od._picklist_label_cache.get("account") + self.assertIsNotNone(entry) + self.assertIn("ts", entry) + self.assertIn("picklists", entry) + self.assertEqual(entry["picklists"]["industrycode"], {"technology": 6, "consulting": 12}) + + def test_bulk_fetch_multiple_picklists(self): + """Multiple picklist attributes are all stored under the same table entry.""" + resp = self._bulk_response( + ("industrycode", [(6, "Technology")]), + ("statuscode", [(1, "Active"), (2, "Inactive")]), + ) + self.od._request.return_value = resp + + self.od._bulk_fetch_picklists("account") + + picklists = self.od._picklist_label_cache["account"]["picklists"] + self.assertEqual(picklists["industrycode"], {"technology": 6}) + self.assertEqual(picklists["statuscode"], {"active": 1, "inactive": 2}) + + def test_bulk_fetch_no_picklists_caches_empty(self): + """Table with no picklist attributes gets cached with empty picklists dict.""" + resp = MagicMock() + resp.json.return_value = {"value": []} + self.od._request.return_value = resp + + self.od._bulk_fetch_picklists("account") + + entry = self.od._picklist_label_cache.get("account") + self.assertIsNotNone(entry) + self.assertEqual(entry["picklists"], {}) + + def test_bulk_fetch_skips_when_cache_fresh(self): + """Warm cache within TTL should skip the API call.""" + import time + + self.od._picklist_label_cache["account"] = { + "ts": time.time(), + "picklists": {"industrycode": {"technology": 6}}, + } + + self.od._bulk_fetch_picklists("account") + self.od._request.assert_not_called() + + def test_bulk_fetch_refreshes_when_cache_expired(self): + """Expired cache should trigger a new API call.""" + import time + + self.od._picklist_label_cache["account"] = { + "ts": time.time() - 7200, # 2 hours ago, beyond 1h TTL + "picklists": {"industrycode": {"technology": 6}}, + } + + resp = self._bulk_response(("industrycode", [(6, "Tech"), (12, "Consulting")])) + self.od._request.return_value = resp + + self.od._bulk_fetch_picklists("account") + self.od._request.assert_called_once() + self.assertEqual( + self.od._picklist_label_cache["account"]["picklists"]["industrycode"], + {"tech": 6, "consulting": 12}, + ) + + def test_bulk_fetch_case_insensitive_table_key(self): + """Table key is normalized to lowercase.""" + resp = self._bulk_response(("industrycode", [(6, "Tech")])) + self.od._request.return_value = resp + + self.od._bulk_fetch_picklists("Account") + + self.assertIn("account", self.od._picklist_label_cache) + self.assertNotIn("Account", self.od._picklist_label_cache) + + def test_bulk_fetch_uses_picklist_cast_url(self): + """API call uses PicklistAttributeMetadata cast segment.""" + resp = self._bulk_response() + self.od._request.return_value = resp + + self.od._bulk_fetch_picklists("account") + + call_url = self.od._request.call_args.args[1] + self.assertIn("PicklistAttributeMetadata", call_url) + self.assertIn("OptionSet", call_url) + + def test_bulk_fetch_makes_single_api_call(self): + """Bulk fetch uses exactly one API call regardless of picklist count.""" + resp = self._bulk_response( + ("a", [(1, "X")]), + ("b", [(2, "Y")]), + ("c", [(3, "Z")]), + ) + self.od._request.return_value = resp + + self.od._bulk_fetch_picklists("account") + self.assertEqual(self.od._request.call_count, 1) + + def test_bulk_fetch_stress_large_workload(self): + """Bulk fetch correctly parses a response with a large number of picklist attributes.""" + num_picklists = 5000 + picklists = [(f"new_pick{i}", [(100000000 + j, f"Option {j}") for j in range(4)]) for i in range(num_picklists)] + resp = self._bulk_response(*picklists) + self.od._request.return_value = resp + + self.od._bulk_fetch_picklists("account") + + self.assertEqual(self.od._request.call_count, 1) + cached = self.od._picklist_label_cache["account"]["picklists"] + self.assertEqual(len(cached), num_picklists) + self.assertEqual(cached["new_pick0"]["option 0"], 100000000) + self.assertEqual(cached[f"new_pick{num_picklists - 1}"]["option 3"], 100000003) + + # ---- _request_metadata_with_retry ---- + + def test_retry_succeeds_on_first_try(self): + """No retry needed when first call succeeds.""" + mock_resp = MagicMock() + self.od._request.return_value = mock_resp + + result = self.od._request_metadata_with_retry("get", "https://example.com/test") + self.assertIs(result, mock_resp) + self.assertEqual(self.od._request.call_count, 1) + + @patch("PowerPlatform.Dataverse.data._odata.time.sleep") + def test_retry_retries_on_404(self, mock_sleep): + """Should retry on 404 and succeed on later attempt.""" + from PowerPlatform.Dataverse.core.errors import HttpError + + err_404 = HttpError("Not Found", status_code=404) + mock_resp = MagicMock() + self.od._request.side_effect = [err_404, mock_resp] + + result = self.od._request_metadata_with_retry("get", "https://example.com/test") + self.assertIs(result, mock_resp) + self.assertEqual(self.od._request.call_count, 2) + mock_sleep.assert_called_once() + + @patch("PowerPlatform.Dataverse.data._odata.time.sleep") + def test_retry_raises_after_max_attempts(self, mock_sleep): + """Should raise RuntimeError after all retries exhausted.""" + from PowerPlatform.Dataverse.core.errors import HttpError + + err_404 = HttpError("Not Found", status_code=404) + self.od._request.side_effect = err_404 + + with self.assertRaises(RuntimeError) as ctx: + self.od._request_metadata_with_retry("get", "https://example.com/test") + self.assertIn("404", str(ctx.exception)) + self.assertTrue(mock_sleep.called) + + def test_retry_does_not_retry_non_404(self): + """Non-404 errors should be raised immediately without retry.""" + from PowerPlatform.Dataverse.core.errors import HttpError + + err_500 = HttpError("Server Error", status_code=500) + self.od._request.side_effect = err_500 + + with self.assertRaises(HttpError): + self.od._request_metadata_with_retry("get", "https://example.com/test") + self.assertEqual(self.od._request.call_count, 1) + + # ---- _convert_labels_to_ints ---- + + def test_convert_no_string_values_skips_fetch(self): + """Record with no string values should not trigger any API call.""" + record = {"quantity": 5, "amount": 99.99, "completed": False} + result = self.od._convert_labels_to_ints("account", record) + self.assertEqual(result, record) + self.od._request.assert_not_called() + + def test_convert_empty_record_returns_copy(self): + """Empty record returns empty dict without API calls.""" + result = self.od._convert_labels_to_ints("account", {}) + self.assertEqual(result, {}) + self.od._request.assert_not_called() + + def test_convert_whitespace_only_string_skipped(self): + """String values that are only whitespace should not be candidates.""" + record = {"name": " ", "description": ""} + result = self.od._convert_labels_to_ints("account", record) + self.assertEqual(result, record) + self.od._request.assert_not_called() + + def test_convert_odata_keys_skipped(self): + """@odata.bind keys must not be resolved.""" + import time + + self.od._picklist_label_cache["account"] = { + "ts": time.time(), + "picklists": {}, + } + + record = { + "name": "Contoso", + "new_CustomerId@odata.bind": "/contacts(guid)", + "@odata.type": "Microsoft.Dynamics.CRM.account", + } + result = self.od._convert_labels_to_ints("account", record) + # @odata keys left unchanged + self.assertEqual(result["new_CustomerId@odata.bind"], "/contacts(guid)") + self.assertEqual(result["@odata.type"], "Microsoft.Dynamics.CRM.account") + + def test_convert_warm_cache_no_api_calls(self): + """Warm cache should resolve labels without any API calls.""" + import time + + now = time.time() + self.od._picklist_label_cache["account"] = { + "ts": now, + "picklists": { + "industrycode": {"technology": 6}, + }, + } + + record = {"name": "Contoso", "industrycode": "Technology"} + result = self.od._convert_labels_to_ints("account", record) + + self.assertEqual(result["industrycode"], 6) + self.assertEqual(result["name"], "Contoso") + self.od._request.assert_not_called() + + def test_convert_resolves_picklist_label_to_int(self): + """Full flow: bulk fetch returns picklists, label resolved to int.""" + resp = self._bulk_response( + ("industrycode", [(6, "Technology")]), + ) + self.od._request.return_value = resp + + record = {"name": "Contoso", "industrycode": "Technology"} + result = self.od._convert_labels_to_ints("account", record) + + self.assertEqual(result["industrycode"], 6) + self.assertEqual(result["name"], "Contoso") + # Single bulk fetch call + self.assertEqual(self.od._request.call_count, 1) + + def test_convert_non_picklist_leaves_string_unchanged(self): + """Non-picklist string fields are left as strings (no picklist entry in cache).""" + resp = self._bulk_response() # no picklists on table + self.od._request.return_value = resp + + record = {"name": "Contoso", "telephone1": "555-0100"} + result = self.od._convert_labels_to_ints("account", record) + + self.assertEqual(result["name"], "Contoso") + self.assertEqual(result["telephone1"], "555-0100") + + def test_convert_unmatched_label_left_unchanged(self): + """If a picklist label doesn't match any option, value stays as string.""" + import time + + self.od._picklist_label_cache["account"] = { + "ts": time.time(), + "picklists": { + "industrycode": {"technology": 6, "consulting": 12}, + }, + } + + record = {"industrycode": "UnknownIndustry"} + result = self.od._convert_labels_to_ints("account", record) + self.assertEqual(result["industrycode"], "UnknownIndustry") + + def test_convert_does_not_mutate_original_record(self): + """_convert_labels_to_ints must return a copy, not mutate the input.""" + import time + + self.od._picklist_label_cache["account"] = { + "ts": time.time(), + "picklists": {"industrycode": {"technology": 6}}, + } + + original = {"industrycode": "Technology"} + result = self.od._convert_labels_to_ints("account", original) + + self.assertEqual(result["industrycode"], 6) + self.assertEqual(original["industrycode"], "Technology") + + def test_convert_multiple_picklists_in_one_record(self): + """Multiple picklist fields in the same record are all resolved.""" + resp = self._bulk_response( + ("industrycode", [(6, "Tech")]), + ("statuscode", [(1, "Active")]), + ) + self.od._request.return_value = resp + + record = {"industrycode": "Tech", "statuscode": "Active"} + result = self.od._convert_labels_to_ints("account", record) + + self.assertEqual(result["industrycode"], 6) + self.assertEqual(result["statuscode"], 1) + # Single bulk fetch call + self.assertEqual(self.od._request.call_count, 1) + + def test_convert_mixed_picklists_and_non_picklists(self): + """Picklists resolved, non-picklist strings left unchanged, 1 API call.""" + resp = self._bulk_response( + ("industrycode", [(6, "Tech")]), + ("statuscode", [(1, "Active")]), + ) + self.od._request.return_value = resp + + record = { + "name": "Contoso", + "industrycode": "Tech", + "description": "A company", + "statuscode": "Active", + } + result = self.od._convert_labels_to_ints("account", record) + + self.assertEqual(result["industrycode"], 6) + self.assertEqual(result["statuscode"], 1) + self.assertEqual(result["name"], "Contoso") + self.assertEqual(result["description"], "A company") + self.assertEqual(self.od._request.call_count, 1) + + def test_convert_all_non_picklist_makes_one_api_call(self): + """All non-picklist string fields: 1 bulk fetch call, labels unchanged.""" + resp = self._bulk_response() # no picklists + self.od._request.return_value = resp + + record = {"name": "Contoso", "description": "A company", "telephone1": "555-0100"} + result = self.od._convert_labels_to_ints("account", record) + + self.assertEqual(self.od._request.call_count, 1) + self.assertEqual(result["name"], "Contoso") + + def test_convert_no_string_values_makes_zero_api_calls(self): + """All non-string values: 0 API calls total.""" + record = {"revenue": 1000000, "quantity": 5, "active": True} + self.od._convert_labels_to_ints("account", record) + + self.assertEqual(self.od._request.call_count, 0) + + def test_convert_bulk_fetch_failure_propagates(self): + """Server error during bulk fetch propagates to caller.""" + from PowerPlatform.Dataverse.core.errors import HttpError + + self.od._request.side_effect = HttpError("Server Error", status_code=500) + + with self.assertRaises(HttpError): + self.od._convert_labels_to_ints("account", {"name": "Contoso"}) + + def test_convert_single_picklist_makes_one_api_call(self): + """Single picklist field (cold cache): 1 bulk fetch total.""" + resp = self._bulk_response(("industrycode", [(6, "Tech")])) + self.od._request.return_value = resp + + record = {"industrycode": "Tech"} + result = self.od._convert_labels_to_ints("account", record) + + self.assertEqual(result["industrycode"], 6) + self.assertEqual(self.od._request.call_count, 1) + + def test_convert_integer_values_passed_through(self): + """Integer values (already resolved) are left unchanged.""" + import time + + self.od._picklist_label_cache["account"] = { + "ts": time.time(), + "picklists": {"industrycode": {"technology": 6}}, + } + + record = {"industrycode": 6, "name": "Contoso"} + result = self.od._convert_labels_to_ints("account", record) + self.assertEqual(result["industrycode"], 6) + + def test_convert_case_insensitive_label_matching(self): + """Picklist label matching is case-insensitive.""" + import time + + self.od._picklist_label_cache["account"] = { + "ts": time.time(), + "picklists": {"industrycode": {"technology": 6}}, + } + + record = {"industrycode": "TECHNOLOGY"} + result = self.od._convert_labels_to_ints("account", record) + self.assertEqual(result["industrycode"], 6) + + def test_convert_second_call_same_table_no_api(self): + """Second convert call for same table uses cached bulk fetch, no API call.""" + resp = self._bulk_response(("industrycode", [(6, "Tech")])) + self.od._request.return_value = resp + + self.od._convert_labels_to_ints("account", {"industrycode": "Tech"}) + self.assertEqual(self.od._request.call_count, 1) + + # Second call -- cache warm + self.od._request.reset_mock() + result = self.od._convert_labels_to_ints("account", {"industrycode": "Tech"}) + self.assertEqual(result["industrycode"], 6) + self.od._request.assert_not_called() + + def test_convert_different_tables_separate_fetches(self): + """Different tables each get their own bulk fetch.""" + resp1 = self._bulk_response(("industrycode", [(6, "Tech")])) + resp2 = self._bulk_response(("new_status", [(100, "Open")])) + self.od._request.side_effect = [resp1, resp2] + + r1 = self.od._convert_labels_to_ints("account", {"industrycode": "Tech"}) + r2 = self.od._convert_labels_to_ints("new_ticket", {"new_status": "Open"}) + + self.assertEqual(r1["industrycode"], 6) + self.assertEqual(r2["new_status"], 100) + self.assertEqual(self.od._request.call_count, 2) + + def test_convert_only_odata_and_non_strings_skips_fetch(self): + """Record with only @odata keys and non-string values should skip fetch entirely.""" + record = { + "@odata.type": "Microsoft.Dynamics.CRM.account", + "new_CustomerId@odata.bind": "/contacts(guid)", + "quantity": 5, + "active": True, + } + result = self.od._convert_labels_to_ints("account", record) + self.assertEqual(result, record) + self.od._request.assert_not_called() + + def test_convert_partial_picklist_match(self): + """Some picklists match, some don't -- matched ones resolved, unmatched left as string.""" + import time + + self.od._picklist_label_cache["account"] = { + "ts": time.time(), + "picklists": { + "industrycode": {"technology": 6, "consulting": 12}, + "statuscode": {"active": 1, "inactive": 2}, + }, + } + + record = {"industrycode": "Technology", "statuscode": "UnknownStatus"} + result = self.od._convert_labels_to_ints("account", record) + + self.assertEqual(result["industrycode"], 6) + self.assertEqual(result["statuscode"], "UnknownStatus") + + def test_convert_mixed_int_and_label_in_same_record(self): + """One picklist already int, another is a label string -- only label resolved.""" + import time + + self.od._picklist_label_cache["account"] = { + "ts": time.time(), + "picklists": { + "industrycode": {"technology": 6}, + "statuscode": {"active": 1}, + }, + } + + record = {"industrycode": 6, "statuscode": "Active"} + result = self.od._convert_labels_to_ints("account", record) + + self.assertEqual(result["industrycode"], 6) + self.assertEqual(result["statuscode"], 1) + + def test_convert_same_label_different_picklists(self): + """Same label text in two different picklist columns resolves to different values.""" + import time + + self.od._picklist_label_cache["new_ticket"] = { + "ts": time.time(), + "picklists": { + "new_priority": {"high": 3}, + "new_severity": {"high": 100}, + }, + } + + record = {"new_priority": "High", "new_severity": "High"} + result = self.od._convert_labels_to_ints("new_ticket", record) + + self.assertEqual(result["new_priority"], 3) + self.assertEqual(result["new_severity"], 100) + + def test_convert_picklist_with_empty_options(self): + """Picklist attribute with zero defined options: label stays as string.""" + import time + + self.od._picklist_label_cache["account"] = { + "ts": time.time(), + "picklists": { + "customcode": {}, # picklist exists but has no options + }, + } + + record = {"customcode": "SomeValue"} + result = self.od._convert_labels_to_ints("account", record) + self.assertEqual(result["customcode"], "SomeValue") + + def test_convert_full_realistic_record(self): + """Realistic record: mix of strings, ints, bools, @odata keys, and picklists.""" + resp = self._bulk_response( + ("industrycode", [(6, "Technology"), (12, "Consulting")]), + ("statuscode", [(1, "Active"), (2, "Inactive")]), + ) + self.od._request.return_value = resp + + record = { + "name": "Contoso Ltd", + "industrycode": "Technology", + "statuscode": "Active", + "revenue": 1000000, + "telephone1": "555-0100", + "emailaddress1": "info@contoso.com", + "new_completed": True, + "new_quantity": 42, + "description": "A technology company", + "@odata.type": "Microsoft.Dynamics.CRM.account", + "new_CustomerId@odata.bind": "/contacts(00000000-0000-0000-0000-000000000001)", + } + result = self.od._convert_labels_to_ints("account", record) + + # Picklists resolved + self.assertEqual(result["industrycode"], 6) + self.assertEqual(result["statuscode"], 1) + # Non-picklist strings unchanged + self.assertEqual(result["name"], "Contoso Ltd") + self.assertEqual(result["telephone1"], "555-0100") + self.assertEqual(result["emailaddress1"], "info@contoso.com") + self.assertEqual(result["description"], "A technology company") + # Non-strings unchanged + self.assertEqual(result["revenue"], 1000000) + self.assertEqual(result["new_completed"], True) + self.assertEqual(result["new_quantity"], 42) + # @odata keys unchanged + self.assertEqual(result["@odata.type"], "Microsoft.Dynamics.CRM.account") + self.assertEqual( + result["new_CustomerId@odata.bind"], + "/contacts(00000000-0000-0000-0000-000000000001)", + ) + self.assertEqual(self.od._request.call_count, 1) + + def test_bulk_fetch_skips_malformed_items(self): + """Bulk fetch ignores items that aren't dicts or lack LogicalName.""" + resp = MagicMock() + resp.json.return_value = { + "value": [ + "not-a-dict", + {"LogicalName": "", "OptionSet": {"Options": []}}, + { + "LogicalName": "industrycode", + "OptionSet": {"Options": [{"Value": 6, "Label": {"LocalizedLabels": [{"Label": "Tech"}]}}]}, + }, + {"no_logical_name_key": True}, + ] + } + self.od._request.return_value = resp + + self.od._bulk_fetch_picklists("account") + + picklists = self.od._picklist_label_cache["account"]["picklists"] + self.assertEqual(len(picklists), 1) + self.assertEqual(picklists["industrycode"], {"tech": 6}) + + def test_bulk_fetch_first_label_wins_for_same_value(self): + """When multiple localized labels exist, first label wins via setdefault.""" + resp = MagicMock() + resp.json.return_value = { + "value": [ + { + "LogicalName": "industrycode", + "OptionSet": { + "Options": [ + { + "Value": 6, + "Label": { + "LocalizedLabels": [ + {"Label": "Technology"}, + {"Label": "Technologie"}, + ] + }, + } + ] + }, + } + ] + } + self.od._request.return_value = resp + + self.od._bulk_fetch_picklists("account") + + picklists = self.od._picklist_label_cache["account"]["picklists"] + # Both labels should be present, mapping to the same value + self.assertEqual(picklists["industrycode"]["technology"], 6) + self.assertEqual(picklists["industrycode"]["technologie"], 6) + + # ---- Integration: through _create ---- + + def test_create_resolves_picklist_in_payload(self): + """_create resolves a picklist label to its integer in the POST payload.""" + bulk_resp = self._bulk_response( + ("industrycode", [(6, "Technology")]), + ) + post_resp = MagicMock() + post_resp.headers = { + "OData-EntityId": "https://example.crm.dynamics.com/api/data/v9.2/accounts(00000000-0000-0000-0000-000000000001)" + } + self.od._request.side_effect = [bulk_resp, post_resp] + + result = self.od._create("accounts", "account", {"name": "Contoso", "industrycode": "Technology"}) + self.assertEqual(result, "00000000-0000-0000-0000-000000000001") + post_calls = [c for c in self.od._request.call_args_list if c.args[0] == "post"] + payload = json.loads(post_calls[0].kwargs["data"]) + self.assertEqual(payload["industrycode"], 6) + self.assertEqual(payload["name"], "Contoso") + + def test_create_warm_cache_skips_fetch(self): + """_create with warm cache makes only the POST call.""" + import time + + now = time.time() + self.od._picklist_label_cache["account"] = { + "ts": now, + "picklists": {"industrycode": {"technology": 6}}, + } + + post_resp = MagicMock() + post_resp.headers = { + "OData-EntityId": "https://example.crm.dynamics.com/api/data/v9.2/accounts(00000000-0000-0000-0000-000000000001)" + } + self.od._request.return_value = post_resp + + result = self.od._create("accounts", "account", {"name": "Contoso", "industrycode": "Technology"}) + self.assertEqual(result, "00000000-0000-0000-0000-000000000001") + self.assertEqual(self.od._request.call_count, 1) + payload = json.loads(self.od._request.call_args.kwargs["data"]) + self.assertEqual(payload["industrycode"], 6) + + # ---- Integration: through _update ---- + + def test_update_resolves_picklist_in_payload(self): + """_update resolves a picklist label to its integer in the PATCH payload.""" + self.od._entity_set_from_schema_name = MagicMock(return_value="new_tickets") + + bulk_resp = self._bulk_response( + ("new_status", [(100000001, "In Progress")]), + ) + patch_resp = MagicMock() + self.od._request.side_effect = [bulk_resp, patch_resp] + + self.od._update( + "new_ticket", + "00000000-0000-0000-0000-000000000001", + {"new_status": "In Progress"}, + ) + patch_calls = [c for c in self.od._request.call_args_list if c.args[0] == "patch"] + payload = json.loads(patch_calls[0].kwargs["data"]) + self.assertEqual(payload["new_status"], 100000001) + + def test_update_warm_cache_skips_fetch(self): + """_update with warm cache makes only the PATCH call.""" + import time + + self.od._entity_set_from_schema_name = MagicMock(return_value="new_tickets") + self.od._picklist_label_cache["new_ticket"] = { + "ts": time.time(), + "picklists": {"new_status": {"in progress": 100000001}}, + } + + self.od._update( + "new_ticket", + "00000000-0000-0000-0000-000000000001", + {"new_status": "In Progress"}, + ) + self.assertEqual(self.od._request.call_count, 1) + self.assertEqual(self.od._request.call_args.args[0], "patch") + payload = json.loads(self.od._request.call_args.kwargs["data"]) + self.assertEqual(payload["new_status"], 100000001) + + # ---- Integration: through _upsert ---- + + def test_upsert_resolves_picklist_in_payload(self): + """_upsert resolves a picklist label to its integer in the PATCH payload.""" + bulk_resp = self._bulk_response( + ("industrycode", [(6, "Technology")]), + ) + patch_resp = MagicMock() + self.od._request.side_effect = [bulk_resp, patch_resp] + + self.od._upsert( + "accounts", + "account", + {"accountnumber": "ACC-001"}, + {"name": "Contoso", "industrycode": "Technology"}, + ) + patch_calls = [c for c in self.od._request.call_args_list if c.args[0] == "patch"] + payload = patch_calls[0].kwargs["json"] + self.assertEqual(payload["industrycode"], 6) + self.assertEqual(payload["name"], "Contoso") + + def test_upsert_warm_cache_skips_fetch(self): + """_upsert with warm cache makes only the PATCH call.""" + import time + + now = time.time() + self.od._picklist_label_cache["account"] = { + "ts": now, + "picklists": {"industrycode": {"technology": 6}}, + } + + self.od._upsert( + "accounts", + "account", + {"accountnumber": "ACC-001"}, + {"name": "Contoso", "industrycode": "Technology"}, + ) + self.assertEqual(self.od._request.call_count, 1) + patch_calls = [c for c in self.od._request.call_args_list if c.args[0] == "patch"] + payload = patch_calls[0].kwargs["json"] + self.assertEqual(payload["industrycode"], 6) + + class TestBuildUpsertMultiple(unittest.TestCase): """Unit tests for _ODataClient._build_upsert_multiple (batch deferred build).""" diff --git a/tests/unit/test_context_manager.py b/tests/unit/test_context_manager.py index df916583..2f1aab9c 100644 --- a/tests/unit/test_context_manager.py +++ b/tests/unit/test_context_manager.py @@ -158,7 +158,7 @@ def test_close_clears_odata_caches(self): odata = client._get_odata() odata._logical_to_entityset_cache["test"] = "value" odata._logical_primaryid_cache["test"] = "value" - odata._picklist_label_cache[("test", "attr")] = {"map": {}, "ts": 0} + odata._picklist_label_cache["test"] = {"ts": 0, "picklists": {"attr": {}}} client.close() From 29eabaec9f93244feb20b98fe8f54f9ccda6fb87 Mon Sep 17 00:00:00 2001 From: abelmilash-msft Date: Thu, 9 Apr 2026 11:41:29 -0700 Subject: [PATCH 19/20] Add memo/multiline column type support (#155) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Adds support for `"memo"` (or `"multiline"`) column type, enabling creation of multiline text columns on Dataverse tables. Users can specify `"memo"` as a column type in `client.tables.create()` and `client.tables.add_columns()`. ## Changes **`src/PowerPlatform/Dataverse/data/_odata.py`** - Add `"memo"` / `"multiline"` handling in `_attribute_payload()`, generating `MemoAttributeMetadata` with `MaxLength: 4000`, `Format: Text`, `ImeMode: Auto` **`src/PowerPlatform/Dataverse/operations/tables.py`** - Document `"memo"` / `"multiline"` in `create()` docstring **`examples/advanced/walkthrough.py`** - Add `"new_Notes": "memo"` column to walkthrough table - Include memo field in record creation with multiline content - Read back and display memo field in Section 4 - Update memo with new multiline content in Section 5 **`tests/unit/data/test_odata_internal.py`** - `test_memo_type()` — validates MemoAttributeMetadata payload - `test_multiline_alias()` — validates `"multiline"` produces identical result **`tests/unit/test_tables_operations.py`** - `test_add_columns_memo()` — validates memo type through `add_columns()` **`README.md`** / **SKILL docs** - List `"memo"` in supported column types ## Testing - 620 unit tests passing - E2E memo walkthrough verified against live Dataverse (10 assertions): multiline create/read/update, empty string, None, special characters, long text (4000 chars), memo not mistaken for picklist label, triple-quoted strings, clearing memo to None --------- Co-authored-by: Abel Milash Co-authored-by: Claude Sonnet 4.6 --- .claude/skills/dataverse-sdk-use/SKILL.md | 1 + README.md | 3 +- examples/advanced/walkthrough.py | 25 ++++++++++---- .../claude_skill/dataverse-sdk-use/SKILL.md | 1 + src/PowerPlatform/Dataverse/data/_odata.py | 10 ++++++ .../Dataverse/operations/tables.py | 3 +- tests/unit/data/test_odata_internal.py | 34 +++++++++++++++++++ tests/unit/test_tables_operations.py | 10 ++++++ 8 files changed, 79 insertions(+), 8 deletions(-) diff --git a/.claude/skills/dataverse-sdk-use/SKILL.md b/.claude/skills/dataverse-sdk-use/SKILL.md index 569caec0..72677468 100644 --- a/.claude/skills/dataverse-sdk-use/SKILL.md +++ b/.claude/skills/dataverse-sdk-use/SKILL.md @@ -250,6 +250,7 @@ table_info = client.tables.create( #### Supported Column Types Types on the same line map to the same exact format under the hood - `"string"` or `"text"` - Single line of text +- `"memo"` or `"multiline"` - Multiple lines of text (4000 character default) - `"int"` or `"integer"` - Whole number - `"decimal"` or `"money"` - Decimal number - `"float"` or `"double"` - Floating point number diff --git a/README.md b/README.md index d6c6403d..551f6dcf 100644 --- a/README.md +++ b/README.md @@ -404,6 +404,7 @@ for page in client.records.get( # Create a custom table, including the customization prefix value in the schema names for the table and columns. table_info = client.tables.create("new_Product", { "new_Code": "string", + "new_Description": "memo", "new_Price": "decimal", "new_Active": "bool" }) @@ -674,7 +675,7 @@ For optimal performance in production environments: ### Limitations - SQL queries are **read-only** and support a limited subset of SQL syntax -- Create Table supports a limited number of column types (string, int, decimal, bool, datetime, picklist) +- Create Table supports the following column types: string, memo, int, decimal, float, bool, datetime, file, and picklist (Enum subclass) - File uploads are limited by Dataverse file size restrictions (default 128MB per file) ## Contributing diff --git a/examples/advanced/walkthrough.py b/examples/advanced/walkthrough.py index 7740bff7..5e4d0a4e 100644 --- a/examples/advanced/walkthrough.py +++ b/examples/advanced/walkthrough.py @@ -120,6 +120,7 @@ def _run_walkthrough(client): "new_Quantity": "int", "new_Amount": "decimal", "new_Completed": "bool", + "new_Notes": "memo", "new_Priority": Priority, } table_info = backoff(lambda: client.tables.create(table_name, columns)) @@ -140,6 +141,7 @@ def _run_walkthrough(client): "new_Quantity": 5, "new_Amount": 1250.50, "new_Completed": False, + "new_Notes": "This is a multiline memo field.\nIt supports longer text content.", "new_Priority": Priority.MEDIUM, } id1 = backoff(lambda: client.records.create(table_name, single_record)) @@ -192,6 +194,7 @@ def _run_walkthrough(client): "new_quantity": record.get("new_quantity"), "new_amount": record.get("new_amount"), "new_completed": record.get("new_completed"), + "new_notes": record.get("new_notes"), "new_priority": record.get("new_priority"), "new_priority@FormattedValue": record.get("new_priority@OData.Community.Display.V1.FormattedValue"), }, @@ -218,9 +221,19 @@ def _run_walkthrough(client): # Single update log_call(f"client.records.update('{table_name}', '{id1}', {{...}})") - backoff(lambda: client.records.update(table_name, id1, {"new_Quantity": 100})) + backoff( + lambda: client.records.update( + table_name, + id1, + { + "new_Quantity": 100, + "new_Notes": "Updated memo field.\nNow with revised content across multiple lines.", + }, + ) + ) updated = backoff(lambda: client.records.get(table_name, id1)) print(f"[OK] Updated single record new_Quantity: {updated.get('new_quantity')}") + print(f" new_Notes: {repr(updated.get('new_notes'))}") # Multiple update (broadcast same change) log_call(f"client.records.update('{table_name}', [{len(ids)} IDs], {{...}})") @@ -462,14 +475,14 @@ def _run_walkthrough(client): print("11. Column Management") print("=" * 80) - log_call(f"client.tables.add_columns('{table_name}', {{'new_Notes': 'string'}})") - created_cols = backoff(lambda: client.tables.add_columns(table_name, {"new_Notes": "string"})) + log_call(f"client.tables.add_columns('{table_name}', {{'new_Tags': 'string'}})") + created_cols = backoff(lambda: client.tables.add_columns(table_name, {"new_Tags": "string"})) print(f"[OK] Added column: {created_cols[0]}") # Delete the column we just added - log_call(f"client.tables.remove_columns('{table_name}', ['new_Notes'])") - backoff(lambda: client.tables.remove_columns(table_name, ["new_Notes"])) - print(f"[OK] Deleted column: new_Notes") + log_call(f"client.tables.remove_columns('{table_name}', ['new_Tags'])") + backoff(lambda: client.tables.remove_columns(table_name, ["new_Tags"])) + print(f"[OK] Deleted column: new_Tags") # ============================================================================ # 12. DELETE OPERATIONS diff --git a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md index 569caec0..72677468 100644 --- a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md +++ b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md @@ -250,6 +250,7 @@ table_info = client.tables.create( #### Supported Column Types Types on the same line map to the same exact format under the hood - `"string"` or `"text"` - Single line of text +- `"memo"` or `"multiline"` - Multiple lines of text (4000 character default) - `"int"` or `"integer"` - Whole number - `"decimal"` or `"money"` - Decimal number - `"float"` or `"double"` - Floating point number diff --git a/src/PowerPlatform/Dataverse/data/_odata.py b/src/PowerPlatform/Dataverse/data/_odata.py index a3ccaefe..a0bf2709 100644 --- a/src/PowerPlatform/Dataverse/data/_odata.py +++ b/src/PowerPlatform/Dataverse/data/_odata.py @@ -1268,6 +1268,16 @@ def _attribute_payload( "FormatName": {"Value": "Text"}, "IsPrimaryName": bool(is_primary_name), } + if dtype_l in ("memo", "multiline"): + return { + "@odata.type": "Microsoft.Dynamics.CRM.MemoAttributeMetadata", + "SchemaName": column_schema_name, + "DisplayName": self._label(label), + "RequiredLevel": {"Value": "None"}, + "MaxLength": 4000, + "FormatName": {"Value": "Text"}, + "ImeMode": "Auto", + } if dtype_l in ("int", "integer"): return { "@odata.type": "Microsoft.Dynamics.CRM.IntegerAttributeMetadata", diff --git a/src/PowerPlatform/Dataverse/operations/tables.py b/src/PowerPlatform/Dataverse/operations/tables.py index e25c5a14..6fffd692 100644 --- a/src/PowerPlatform/Dataverse/operations/tables.py +++ b/src/PowerPlatform/Dataverse/operations/tables.py @@ -82,7 +82,8 @@ def create( :type table: :class:`str` :param columns: Mapping of column schema names (with customization prefix) to their types. Supported types include ``"string"`` - (or ``"text"``), ``"int"`` (or ``"integer"``), ``"decimal"`` + (or ``"text"``), ``"memo"`` (or ``"multiline"``), + ``"int"`` (or ``"integer"``), ``"decimal"`` (or ``"money"``), ``"float"`` (or ``"double"``), ``"datetime"`` (or ``"date"``), ``"bool"`` (or ``"boolean"``), ``"file"``, and ``Enum`` subclasses diff --git a/tests/unit/data/test_odata_internal.py b/tests/unit/data/test_odata_internal.py index 2ed49791..0f8873f4 100644 --- a/tests/unit/data/test_odata_internal.py +++ b/tests/unit/data/test_odata_internal.py @@ -519,6 +519,40 @@ def test_returns_none(self): self.assertIsNone(result) +class TestAttributePayload(unittest.TestCase): + """Unit tests for _ODataClient._attribute_payload.""" + + def setUp(self): + self.od = _make_odata_client() + + def test_memo_type(self): + """'memo' should produce MemoAttributeMetadata with MaxLength 4000.""" + result = self.od._attribute_payload("new_Notes", "memo") + self.assertEqual(result["@odata.type"], "Microsoft.Dynamics.CRM.MemoAttributeMetadata") + self.assertEqual(result["SchemaName"], "new_Notes") + self.assertEqual(result["MaxLength"], 4000) + self.assertEqual(result["FormatName"], {"Value": "Text"}) + self.assertNotIn("IsPrimaryName", result) + + def test_multiline_alias(self): + """'multiline' should produce identical payload to 'memo'.""" + memo_result = self.od._attribute_payload("new_Description", "memo") + multiline_result = self.od._attribute_payload("new_Description", "multiline") + self.assertEqual(multiline_result, memo_result) + + def test_string_type(self): + """'string' should produce StringAttributeMetadata with MaxLength 200.""" + result = self.od._attribute_payload("new_Title", "string") + self.assertEqual(result["@odata.type"], "Microsoft.Dynamics.CRM.StringAttributeMetadata") + self.assertEqual(result["MaxLength"], 200) + self.assertEqual(result["FormatName"], {"Value": "Text"}) + + def test_unsupported_type_returns_none(self): + """An unknown type string should return None.""" + result = self.od._attribute_payload("new_Col", "unknown_type") + self.assertIsNone(result) + + class TestPicklistLabelResolution(unittest.TestCase): """Tests for picklist label-to-integer resolution. diff --git a/tests/unit/test_tables_operations.py b/tests/unit/test_tables_operations.py index 69f57a58..d04cef24 100644 --- a/tests/unit/test_tables_operations.py +++ b/tests/unit/test_tables_operations.py @@ -193,6 +193,16 @@ def test_add_columns(self): self.client._odata._create_columns.assert_called_once_with("new_Product", columns) self.assertEqual(result, ["new_Notes", "new_Active"]) + def test_add_columns_memo(self): + """add_columns() with memo type should pass through correctly.""" + self.client._odata._create_columns.return_value = ["new_Description"] + + columns = {"new_Description": "memo"} + result = self.client.tables.add_columns("new_Product", columns) + + self.client._odata._create_columns.assert_called_once_with("new_Product", columns) + self.assertEqual(result, ["new_Description"]) + # --------------------------------------------------------- remove_columns def test_remove_columns_single(self): From 9cff47f53459d0ec069a5957c388b1da65f589a7 Mon Sep 17 00:00:00 2001 From: abelmilash-msft Date: Thu, 9 Apr 2026 17:48:14 -0700 Subject: [PATCH 20/20] Add unit test coverage and CI coverage reporting (#158) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Adds comprehensive unit tests across 13 test files and enables coverage reporting and enforcement in both CI pipelines and `pyproject.toml`. Coverage increased from 72% to 93%. Coverage enforcement is now built into both CI pipelines and the project config: overall coverage must stay at or above 90% (`fail_under = 90` in `pyproject.toml`), and any new code introduced in a PR must also meet 90% (`diff-cover`). ## Test Results - **1127 passed**, 0 failures - **93% line coverage** (up from ~72%) ## Changes ### Configuration (2 files) - `pyproject.toml`: Added `[tool.coverage.run]` (`source = ["src/PowerPlatform"]`) and `[tool.coverage.report]` (`fail_under = 90`, `show_missing = true`) to centralize coverage config - `pyproject.toml`: Added `diff-cover` dev dependency for PR-level coverage enforcement ### CI Pipelines (2 files) - `.github/workflows/python-package.yml`: Added `PYTHONPATH=src pytest --cov --cov-report=xml --junitxml=test-results.xml`, `diff-cover` step to enforce 90% on new changes, `fetch-depth: 0` for full git history, and artifact upload steps - `.azdo/ci-pr.yaml`: Same pytest and diff-cover steps, added `PublishTestResults@2` and `PublishCodeCoverageResults@2` tasks ### New Test Files (6 files) - `test_auth.py`: Credential validation, token acquisition (2 tests) - `test_http_client.py`: Timeout selection, session routing, retry behavior (15 tests) - `test_http_errors.py`: Error response parsing, correlation IDs, transient detection (9 tests) - `test_upload.py`: File upload orchestrator, small upload, chunked streaming (50+ tests) - `test_relationships.py`: 1:N and M:N relationship CRUD (25+ tests) - `test_records_operations.py`: Public records API delegation (30+ tests) ### Expanded Test Files (7 files) - `test_odata_internal.py`: 36 test classes, 219 tests covering all `_ODataClient` methods — CRUD, upsert, bulk operations, metadata, caching, picklist resolution, pagination, SQL queries, alternate keys, column management. Includes kwarg correctness audit (`json=` vs `data=`), response content validation, and assertion strengthening. - `test_batch_serialization.py`: 7 new test classes (26 tests) — dispatch routing for all intent types, changeset validation, metadata resolution, continue-on-error header, MIME parsing edge cases - `test_query_builder.py`, `test_table_info.py`, `test_client.py`, `test_context_manager.py`, `test_records_operations.py`: Docstrings added to existing tests --------- Co-authored-by: Abel Milash Co-authored-by: Claude Sonnet 4.6 --- .azdo/ci-pr.yaml | 24 +- .github/workflows/python-package.yml | 29 +- pyproject.toml | 8 + tests/unit/core/test_auth.py | 35 + tests/unit/core/test_http_client.py | 123 ++ tests/unit/core/test_http_errors.py | 37 + tests/unit/data/test_batch_serialization.py | 325 ++++- tests/unit/data/test_odata_internal.py | 1462 ++++++++++++++++++- tests/unit/data/test_relationships.py | 13 + tests/unit/data/test_upload.py | 430 ++++++ tests/unit/models/test_query_builder.py | 6 + tests/unit/models/test_table_info.py | 6 + tests/unit/test_client.py | 5 + tests/unit/test_context_manager.py | 11 + tests/unit/test_records_operations.py | 46 + 15 files changed, 2523 insertions(+), 37 deletions(-) create mode 100644 tests/unit/core/test_auth.py create mode 100644 tests/unit/core/test_http_client.py create mode 100644 tests/unit/data/test_upload.py diff --git a/.azdo/ci-pr.yaml b/.azdo/ci-pr.yaml index eeb64f83..80fc4b4d 100644 --- a/.azdo/ci-pr.yaml +++ b/.azdo/ci-pr.yaml @@ -42,7 +42,7 @@ extends: - script: | python -m pip install --upgrade pip - python -m pip install flake8 black build + python -m pip install flake8 black build diff-cover python -m pip install -e .[dev] displayName: 'Install dependencies' @@ -60,18 +60,30 @@ extends: - script: | python -m build displayName: 'Build package' - + - script: | python -m pip install dist/*.whl displayName: 'Install wheel' - + - script: | - pytest + PYTHONPATH=src pytest --junitxml=test-results.xml --cov --cov-report=xml displayName: 'Test with pytest' - + + - script: | + git fetch origin main + diff-cover coverage.xml --compare-branch=origin/main --fail-under=90 + displayName: 'Diff coverage (90% for new changes)' + - task: PublishTestResults@2 condition: succeededOrFailed() inputs: - testResultsFiles: '**/test-*.xml' + testResultsFiles: '**/test-results.xml' testRunTitle: 'Python 3.12' displayName: 'Publish test results' + + - task: PublishCodeCoverageResults@2 + condition: succeededOrFailed() + inputs: + summaryFileLocation: '**/coverage.xml' + pathToSources: '$(Build.SourcesDirectory)/src' + displayName: 'Publish code coverage' diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 26209bf7..886bc72b 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -18,6 +18,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Set up Python 3.12 uses: actions/setup-python@v5 @@ -27,7 +29,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install flake8 black build + python -m pip install flake8 black build diff-cover python -m pip install -e .[dev] - name: Check format with black @@ -44,11 +46,30 @@ jobs: - name: Build package run: | python -m build - + - name: Install wheel run: | python -m pip install dist/*.whl - + - name: Test with pytest run: | - pytest + PYTHONPATH=src pytest --junitxml=test-results.xml --cov --cov-report=xml + + - name: Diff coverage (90% for new changes) + run: | + git fetch origin ${{ github.base_ref }} + diff-cover coverage.xml --compare-branch=origin/${{ github.base_ref }} --fail-under=90 + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-results + path: test-results.xml + + - name: Upload coverage report + if: always() + uses: actions/upload-artifact@v4 + with: + name: coverage-report + path: coverage.xml diff --git a/pyproject.toml b/pyproject.toml index 3df59c9d..3e26347e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,6 +93,14 @@ select = [ [tool.pytest.ini_options] testpaths = ["tests/unit"] + +[tool.coverage.run] +source = ["src/PowerPlatform"] + +[tool.coverage.report] +fail_under = 90 +show_missing = true + markers = [ "e2e: end-to-end tests requiring a live Dataverse environment (DATAVERSE_URL)", ] diff --git a/tests/unit/core/test_auth.py b/tests/unit/core/test_auth.py new file mode 100644 index 00000000..b82bb9bd --- /dev/null +++ b/tests/unit/core/test_auth.py @@ -0,0 +1,35 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import unittest +from unittest.mock import MagicMock + +from azure.core.credentials import TokenCredential + +from PowerPlatform.Dataverse.core._auth import _AuthManager, _TokenPair + + +class TestAuthManager(unittest.TestCase): + """Tests for _AuthManager credential validation and token acquisition.""" + + def test_non_token_credential_raises(self): + """_AuthManager raises TypeError when credential does not implement TokenCredential.""" + with self.assertRaises(TypeError) as ctx: + _AuthManager("not-a-credential") + self.assertEqual( + str(ctx.exception), + "credential must implement azure.core.credentials.TokenCredential.", + ) + + def test_acquire_token_returns_token_pair(self): + """_acquire_token calls get_token and returns a _TokenPair with scope and token.""" + mock_credential = MagicMock(spec=TokenCredential) + mock_credential.get_token.return_value = MagicMock(token="my-access-token") + + manager = _AuthManager(mock_credential) + result = manager._acquire_token("https://org.crm.dynamics.com/.default") + + mock_credential.get_token.assert_called_once_with("https://org.crm.dynamics.com/.default") + self.assertIsInstance(result, _TokenPair) + self.assertEqual(result.resource, "https://org.crm.dynamics.com/.default") + self.assertEqual(result.access_token, "my-access-token") diff --git a/tests/unit/core/test_http_client.py b/tests/unit/core/test_http_client.py new file mode 100644 index 00000000..200b8be9 --- /dev/null +++ b/tests/unit/core/test_http_client.py @@ -0,0 +1,123 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import unittest +from unittest.mock import MagicMock, patch, call + +import requests + +from PowerPlatform.Dataverse.core._http import _HttpClient + + +class TestHttpClientTimeout(unittest.TestCase): + """Tests for automatic timeout selection in _HttpClient._request.""" + + def _make_response(self, status=200): + resp = MagicMock(spec=requests.Response) + resp.status_code = status + return resp + + def test_get_uses_10s_default_timeout(self): + """GET requests use 10s default when no timeout is specified.""" + client = _HttpClient(retries=1) + with patch("requests.request", return_value=self._make_response()) as mock_req: + client._request("get", "https://example.com/data") + _, kwargs = mock_req.call_args + self.assertEqual(kwargs["timeout"], 10) + + def test_post_uses_120s_default_timeout(self): + """POST requests use 120s default when no timeout is specified.""" + client = _HttpClient(retries=1) + with patch("requests.request", return_value=self._make_response()) as mock_req: + client._request("post", "https://example.com/data") + _, kwargs = mock_req.call_args + self.assertEqual(kwargs["timeout"], 120) + + def test_delete_uses_120s_default_timeout(self): + """DELETE requests use 120s default when no timeout is specified.""" + client = _HttpClient(retries=1) + with patch("requests.request", return_value=self._make_response()) as mock_req: + client._request("delete", "https://example.com/data") + _, kwargs = mock_req.call_args + self.assertEqual(kwargs["timeout"], 120) + + def test_default_timeout_overrides_per_method_default(self): + """Explicit default_timeout on the client overrides per-method defaults.""" + client = _HttpClient(retries=1, timeout=30.0) + with patch("requests.request", return_value=self._make_response()) as mock_req: + client._request("get", "https://example.com/data") + _, kwargs = mock_req.call_args + self.assertEqual(kwargs["timeout"], 30.0) + + def test_explicit_timeout_kwarg_takes_precedence(self): + """If timeout is already in kwargs it is passed through unchanged.""" + client = _HttpClient(retries=1, timeout=30.0) + with patch("requests.request", return_value=self._make_response()) as mock_req: + client._request("get", "https://example.com/data", timeout=5) + _, kwargs = mock_req.call_args + self.assertEqual(kwargs["timeout"], 5) + + +class TestHttpClientRequester(unittest.TestCase): + """Tests for session vs direct requests.request routing.""" + + def _make_response(self): + resp = MagicMock(spec=requests.Response) + resp.status_code = 200 + return resp + + def test_uses_direct_request_without_session(self): + """Without a session, _request uses requests.request directly.""" + client = _HttpClient(retries=1) + with patch("requests.request", return_value=self._make_response()) as mock_req: + client._request("get", "https://example.com/data") + mock_req.assert_called_once() + + def test_uses_session_request_when_session_provided(self): + """With a session, _request uses session.request instead of requests.request.""" + mock_session = MagicMock(spec=requests.Session) + mock_session.request.return_value = self._make_response() + client = _HttpClient(retries=1, session=mock_session) + with patch("requests.request") as mock_req: + client._request("get", "https://example.com/data") + mock_session.request.assert_called_once() + mock_req.assert_not_called() + + +class TestHttpClientRetry(unittest.TestCase): + """Tests for retry behavior on RequestException.""" + + def test_retries_on_request_exception_and_succeeds(self): + """Retries after a RequestException and returns response on second attempt.""" + resp = MagicMock(spec=requests.Response) + resp.status_code = 200 + client = _HttpClient(retries=2, backoff=0) + with patch("requests.request", side_effect=[requests.exceptions.ConnectionError(), resp]) as mock_req: + with patch("time.sleep"): + result = client._request("get", "https://example.com/data") + self.assertEqual(mock_req.call_count, 2) + self.assertIs(result, resp) + + def test_raises_after_all_retries_exhausted(self): + """Raises RequestException after all retry attempts fail.""" + client = _HttpClient(retries=3, backoff=0) + with patch("requests.request", side_effect=requests.exceptions.ConnectionError("timeout")): + with patch("time.sleep"): + with self.assertRaises(requests.exceptions.RequestException): + client._request("get", "https://example.com/data") + + def test_backoff_delay_between_retries(self): + """Sleeps with exponential backoff between retry attempts.""" + resp = MagicMock(spec=requests.Response) + resp.status_code = 200 + client = _HttpClient(retries=3, backoff=1.0) + side_effects = [ + requests.exceptions.ConnectionError(), + requests.exceptions.ConnectionError(), + resp, + ] + with patch("requests.request", side_effect=side_effects): + with patch("time.sleep") as mock_sleep: + client._request("get", "https://example.com/data") + # First retry: delay = 1.0 * 2^0 = 1.0, second retry: 1.0 * 2^1 = 2.0 + mock_sleep.assert_has_calls([call(1.0), call(2.0)]) diff --git a/tests/unit/core/test_http_errors.py b/tests/unit/core/test_http_errors.py index 729ebae3..39373e05 100644 --- a/tests/unit/core/test_http_errors.py +++ b/tests/unit/core/test_http_errors.py @@ -180,3 +180,40 @@ def test_correlation_id_shared_inside_call_scope(): h1, h2 = recorder.recorded_headers assert h1["x-ms-client-request-id"] != h2["x-ms-client-request-id"] assert h1["x-ms-correlation-id"] == h2["x-ms-correlation-id"] + + +def test_validation_error_instantiates(): + """ValidationError can be raised and carries the correct code.""" + from PowerPlatform.Dataverse.core.errors import ValidationError + + err = ValidationError("bad input", subcode="missing_field", details={"field": "name"}) + assert err.code == "validation_error" + assert err.subcode == "missing_field" + assert err.details["field"] == "name" + assert err.source == "client" + + +def test_sql_parse_error_instantiates(): + """SQLParseError can be raised and carries the correct code.""" + from PowerPlatform.Dataverse.core.errors import SQLParseError + + err = SQLParseError("unexpected token", subcode="syntax_error") + assert err.code == "sql_parse_error" + assert err.subcode == "syntax_error" + assert err.source == "client" + + +def test_http_error_optional_diagnostic_fields(): + """HttpError stores correlation_id, service_request_id, and traceparent in details.""" + from PowerPlatform.Dataverse.core.errors import HttpError + + err = HttpError( + "Server error", + status_code=500, + correlation_id="corr-123", + service_request_id="svc-456", + traceparent="00-abc-def-01", + ) + assert err.details["correlation_id"] == "corr-123" + assert err.details["service_request_id"] == "svc-456" + assert err.details["traceparent"] == "00-abc-def-01" diff --git a/tests/unit/data/test_batch_serialization.py b/tests/unit/data/test_batch_serialization.py index 561b48b0..b194a341 100644 --- a/tests/unit/data/test_batch_serialization.py +++ b/tests/unit/data/test_batch_serialization.py @@ -16,20 +16,27 @@ _RecordGet, _RecordUpdate, _RecordUpsert, + _TableCreate, + _TableDelete, _TableGet, _TableList, + _TableAddColumns, + _TableRemoveColumns, + _TableCreateOneToMany, + _TableCreateManyToMany, + _TableDeleteRelationship, + _TableGetRelationship, + _TableCreateLookupField, _QuerySql, _extract_boundary, _raise_top_level_batch_error, - _split_multipart, _parse_mime_part, _parse_http_response_part, _CRLF, ) -from PowerPlatform.Dataverse.core.errors import HttpError +from PowerPlatform.Dataverse.core.errors import HttpError, MetadataError, ValidationError from PowerPlatform.Dataverse.models.upsert import UpsertItem from PowerPlatform.Dataverse.data._raw_request import _RawRequest -from PowerPlatform.Dataverse.models.batch import BatchItemResponse def _make_od(): @@ -186,19 +193,19 @@ def test_single_request_body_ends_with_closing_boundary(self): self.assertIn("--batch_bnd--", body) def test_multiple_requests_all_in_body(self): - r1 = _RawRequest(method="GET", url="https://org/api/data/v9.2/accounts") - r2 = _RawRequest( + req1 = _RawRequest(method="GET", url="https://org/api/data/v9.2/accounts") + req2 = _RawRequest( method="DELETE", url="https://org/api/data/v9.2/accounts(guid)", headers={"If-Match": "*"}, ) client = self._client() - body = client._build_batch_body([r1, r2], "bnd") + body = client._build_batch_body([req1, req2], "bnd") self.assertEqual(body.count("--bnd\r\n"), 2) def test_changeset_produces_nested_multipart(self): - r1 = _RawRequest(method="POST", url="https://org/api/data/v9.2/accounts", body="{}") - cs = _ChangeSetBatchItem(requests=[r1]) + req1 = _RawRequest(method="POST", url="https://org/api/data/v9.2/accounts", body="{}") + cs = _ChangeSetBatchItem(requests=[req1]) client = self._client() body = client._build_batch_body([cs], "outer_bnd") self.assertIn("Content-Type: multipart/mixed", body) @@ -389,12 +396,36 @@ def test_exceeds_1000_raises(self): client = _BatchClient(od) items = [_RecordGet(table="account", record_id=f"guid-{i}") for i in range(1001)] - from PowerPlatform.Dataverse.core.errors import ValidationError - with self.assertRaises(ValidationError): client.execute(items) +class TestContinueOnError(unittest.TestCase): + """execute() sends Prefer: odata.continue-on-error when requested.""" + + def setUp(self): + self.od = _make_od() + self.od._build_get.return_value = _RawRequest(method="GET", url="https://x/accounts(g)") + mock_resp = MagicMock() + mock_resp.headers = {"Content-Type": 'multipart/mixed; boundary="batch_x"'} + mock_resp.status_code = 200 + mock_resp.text = "--batch_x\r\n\r\nHTTP/1.1 204 No Content\r\n\r\n\r\n--batch_x--" + self.od._request.return_value = mock_resp + self.client = _BatchClient(self.od) + + def test_continue_on_error_header_sent(self): + """Prefer: odata.continue-on-error header is included when continue_on_error=True.""" + self.client.execute([_RecordGet(table="account", record_id="guid-1")], continue_on_error=True) + _, kwargs = self.od._request.call_args + self.assertEqual(kwargs.get("headers", {}).get("Prefer"), "odata.continue-on-error") + + def test_no_continue_on_error_header_by_default(self): + """Prefer header is absent when continue_on_error is not set.""" + self.client.execute([_RecordGet(table="account", record_id="guid-1")]) + _, kwargs = self.od._request.call_args + self.assertNotIn("Prefer", kwargs.get("headers", {})) + + class TestChangeSetInternal(unittest.TestCase): def test_add_create_returns_dollar_n(self): cs = _ChangeSet() @@ -628,5 +659,279 @@ def test_parse_batch_response_raises_on_missing_boundary(self): client._parse_batch_response(resp) +class TestResolveItemDispatch(unittest.TestCase): + """_resolve_item() routes each intent type to the correct resolver.""" + + def _client_and_od(self): + od = _make_od() + client = _BatchClient(od) + return client, od + + def test_dispatch_record_update(self): + """_resolve_item routes _RecordUpdate to _resolve_record_update.""" + client, od = self._client_and_od() + od._build_update.return_value = MagicMock() + op = _RecordUpdate(table="account", ids="guid-1", changes={"name": "X"}) + result = client._resolve_item(op) + od._build_update.assert_called_once_with("account", "guid-1", {"name": "X"}, content_id=None) + self.assertEqual(len(result), 1) + + def test_dispatch_record_delete(self): + """_resolve_item routes _RecordDelete to _resolve_record_delete.""" + client, od = self._client_and_od() + od._build_delete.return_value = MagicMock() + op = _RecordDelete(table="account", ids="guid-1") + result = client._resolve_item(op) + od._build_delete.assert_called_once_with("account", "guid-1", content_id=None) + self.assertEqual(len(result), 1) + + def test_dispatch_table_create(self): + """_resolve_item routes _TableCreate to _build_create_entity.""" + client, od = self._client_and_od() + od._build_create_entity.return_value = MagicMock() + op = _TableCreate(table="new_Widget", columns={"new_name": str}) + result = client._resolve_item(op) + od._build_create_entity.assert_called_once_with("new_Widget", {"new_name": str}, None, None) + self.assertEqual(len(result), 1) + + def test_dispatch_table_delete(self): + """_resolve_item routes _TableDelete, resolving MetadataId before calling _build_delete_entity.""" + client, od = self._client_and_od() + od._get_entity_by_table_schema_name.return_value = {"MetadataId": "meta-1"} + od._build_delete_entity.return_value = MagicMock() + op = _TableDelete(table="new_Widget") + result = client._resolve_item(op) + od._build_delete_entity.assert_called_once_with("meta-1") + self.assertEqual(len(result), 1) + + def test_dispatch_table_get(self): + """_resolve_item routes _TableGet to _build_get_entity.""" + client, od = self._client_and_od() + od._build_get_entity.return_value = MagicMock() + op = _TableGet(table="account") + result = client._resolve_item(op) + od._build_get_entity.assert_called_once_with("account") + self.assertEqual(len(result), 1) + + def test_dispatch_table_list(self): + """_resolve_item routes _TableList to _build_list_entities, passing filter and select.""" + client, od = self._client_and_od() + od._build_list_entities.return_value = MagicMock() + op = _TableList() + result = client._resolve_item(op) + od._build_list_entities.assert_called_once_with(filter=None, select=None) + self.assertEqual(len(result), 1) + + def test_dispatch_table_add_columns(self): + """_resolve_item routes _TableAddColumns, emitting one request per column.""" + client, od = self._client_and_od() + od._get_entity_by_table_schema_name.return_value = {"MetadataId": "meta-1"} + od._build_create_column.return_value = MagicMock() + op = _TableAddColumns(table="account", columns={"new_col": str}) + result = client._resolve_item(op) + od._build_create_column.assert_called_once_with("meta-1", "new_col", str) + self.assertEqual(len(result), 1) + + def test_dispatch_table_remove_columns(self): + """_resolve_item routes _TableRemoveColumns, fetching attribute metadata before deleting.""" + client, od = self._client_and_od() + od._get_entity_by_table_schema_name.return_value = {"MetadataId": "meta-1"} + od._get_attribute_metadata.return_value = {"MetadataId": "attr-1"} + od._build_delete_column.return_value = MagicMock() + op = _TableRemoveColumns(table="account", columns="new_col") + result = client._resolve_item(op) + od._build_delete_column.assert_called_once_with("meta-1", "attr-1") + self.assertEqual(len(result), 1) + + def test_dispatch_table_create_one_to_many(self): + """_resolve_item routes _TableCreateOneToMany, merging lookup into relationship body.""" + client, od = self._client_and_od() + od._build_create_relationship.return_value = MagicMock() + lookup = MagicMock() + lookup.to_dict.return_value = {"SchemaName": "new_account_contact"} + relationship = MagicMock() + relationship.to_dict.return_value = {"ReferencedEntity": "account"} + op = _TableCreateOneToMany(lookup=lookup, relationship=relationship) + result = client._resolve_item(op) + od._build_create_relationship.assert_called_once_with( + {"ReferencedEntity": "account", "Lookup": {"SchemaName": "new_account_contact"}}, + solution=None, + ) + self.assertEqual(len(result), 1) + + def test_dispatch_table_create_many_to_many(self): + """_resolve_item routes _TableCreateManyToMany to _build_create_relationship.""" + client, od = self._client_and_od() + od._build_create_relationship.return_value = MagicMock() + relationship = MagicMock() + relationship.to_dict.return_value = {"SchemaName": "new_account_contact"} + op = _TableCreateManyToMany(relationship=relationship) + result = client._resolve_item(op) + od._build_create_relationship.assert_called_once_with({"SchemaName": "new_account_contact"}, solution=None) + self.assertEqual(len(result), 1) + + def test_dispatch_table_delete_relationship(self): + """_resolve_item routes _TableDeleteRelationship, passing relationship_id.""" + client, od = self._client_and_od() + od._build_delete_relationship.return_value = MagicMock() + op = _TableDeleteRelationship(relationship_id="rel-guid-1") + result = client._resolve_item(op) + od._build_delete_relationship.assert_called_once_with("rel-guid-1") + self.assertEqual(len(result), 1) + + def test_dispatch_table_get_relationship(self): + """_resolve_item routes _TableGetRelationship, passing schema_name.""" + client, od = self._client_and_od() + od._build_get_relationship.return_value = MagicMock() + op = _TableGetRelationship(schema_name="new_account_contact") + result = client._resolve_item(op) + od._build_get_relationship.assert_called_once_with("new_account_contact") + self.assertEqual(len(result), 1) + + def test_dispatch_table_create_lookup_field(self): + """_resolve_item routes _TableCreateLookupField, building lookup and relationship models.""" + client, od = self._client_and_od() + lookup = MagicMock() + lookup.to_dict.return_value = {"SchemaName": "new_accountid"} + relationship = MagicMock() + relationship.to_dict.return_value = {"ReferencedEntity": "account"} + od._build_lookup_field_models.return_value = (lookup, relationship) + od._build_create_relationship.return_value = MagicMock() + op = _TableCreateLookupField( + referencing_table="new_Widget", + lookup_field_name="new_accountid", + referenced_table="account", + ) + result = client._resolve_item(op) + od._build_lookup_field_models.assert_called_once_with( + referencing_table="new_Widget", + lookup_field_name="new_accountid", + referenced_table="account", + display_name=None, + description=None, + required=False, + cascade_delete="RemoveLink", + language_code=1033, + ) + od._build_create_relationship.assert_called_once_with( + {"ReferencedEntity": "account", "Lookup": {"SchemaName": "new_accountid"}}, + solution=None, + ) + self.assertEqual(len(result), 1) + + def test_dispatch_query_sql(self): + """_resolve_item routes _QuerySql to _build_sql, passing the SQL string.""" + client, od = self._client_and_od() + od._build_sql.return_value = MagicMock() + op = _QuerySql(sql="SELECT name FROM account") + result = client._resolve_item(op) + od._build_sql.assert_called_once_with("SELECT name FROM account") + self.assertEqual(len(result), 1) + + +class TestResolveOneChangeset(unittest.TestCase): + """_resolve_one() raises ValidationError when operation produces != 1 request.""" + + def test_multi_request_op_in_changeset_raises(self): + """use_bulk_delete=False with 2 ids produces 2 requests — not allowed in a changeset.""" + od = _make_od() + client = _BatchClient(od) + od._build_delete.return_value = MagicMock() + op = _RecordDelete(table="account", ids=["guid-1", "guid-2"], use_bulk_delete=False) + with self.assertRaises(ValidationError): + client._resolve_one(op) + + +class TestRequireEntityMetadata(unittest.TestCase): + """_require_entity_metadata raises MetadataError when table not found.""" + + def test_missing_entity_raises_metadata_error(self): + """MetadataError raised when _get_entity_by_table_schema_name returns None.""" + od = _make_od() + od._get_entity_by_table_schema_name.return_value = None + client = _BatchClient(od) + with self.assertRaises(MetadataError): + client._require_entity_metadata("new_Missing") + + def test_entity_without_metadata_id_raises(self): + """MetadataError raised when entity exists but has no MetadataId field.""" + od = _make_od() + od._get_entity_by_table_schema_name.return_value = {"LogicalName": "new_missing"} + client = _BatchClient(od) + with self.assertRaises(MetadataError): + client._require_entity_metadata("new_Missing") + + def test_valid_entity_returns_metadata_id(self): + """Returns MetadataId string when entity is found and has a MetadataId.""" + od = _make_od() + od._get_entity_by_table_schema_name.return_value = {"MetadataId": "meta-abc"} + client = _BatchClient(od) + result = client._require_entity_metadata("account") + self.assertEqual(result, "meta-abc") + + +class TestTableRemoveColumnsResolver(unittest.TestCase): + """_resolve_table_remove_columns covers string input and missing column error.""" + + def _client_and_od(self): + od = _make_od() + client = _BatchClient(od) + return client, od + + def test_single_string_column_resolved(self): + """A single string column name is accepted and resolved to one delete request.""" + client, od = self._client_and_od() + od._get_entity_by_table_schema_name.return_value = {"MetadataId": "meta-1"} + od._get_attribute_metadata.return_value = {"MetadataId": "attr-1"} + od._build_delete_column.return_value = MagicMock() + op = _TableRemoveColumns(table="account", columns="new_col") + result = client._resolve_table_remove_columns(op) + od._build_delete_column.assert_called_once_with("meta-1", "attr-1") + self.assertEqual(len(result), 1) + + def test_missing_column_raises_metadata_error(self): + """MetadataError raised when attribute metadata is not found for the column.""" + client, od = self._client_and_od() + od._get_entity_by_table_schema_name.return_value = {"MetadataId": "meta-1"} + od._get_attribute_metadata.return_value = None + op = _TableRemoveColumns(table="account", columns="new_missing") + with self.assertRaises(MetadataError): + client._resolve_table_remove_columns(op) + + def test_column_without_metadata_id_raises(self): + """MetadataError raised when attribute metadata exists but has no MetadataId.""" + client, od = self._client_and_od() + od._get_entity_by_table_schema_name.return_value = {"MetadataId": "meta-1"} + od._get_attribute_metadata.return_value = {"AttributeType": "String"} + op = _TableRemoveColumns(table="account", columns="new_col") + with self.assertRaises(MetadataError): + client._resolve_table_remove_columns(op) + + +class TestParseMimePartNoSeparator(unittest.TestCase): + """_parse_mime_part handles raw string with no blank-line separator.""" + + def test_no_double_newline_returns_empty_body(self): + """When raw part has no blank-line separator, headers are parsed and body is empty.""" + raw = "Content-Type: application/http" + headers, body = _parse_mime_part(raw) + self.assertEqual(headers.get("content-type"), "application/http") + self.assertEqual(body, "") + + +class TestParseHttpResponsePartMalformed(unittest.TestCase): + """_parse_http_response_part returns None for malformed status lines.""" + + def test_status_line_too_short_returns_none(self): + """Returns None when status line has fewer than 2 tokens (no status code).""" + result = _parse_http_response_part("HTTP/1.1\r\n\r\n", content_id=None) + self.assertIsNone(result) + + def test_non_integer_status_code_returns_none(self): + """Returns None when status code token is not a valid integer.""" + result = _parse_http_response_part("HTTP/1.1 XYZ OK\r\n\r\n", content_id=None) + self.assertIsNone(result) + + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/data/test_odata_internal.py b/tests/unit/data/test_odata_internal.py index 0f8873f4..1aa50c0a 100644 --- a/tests/unit/data/test_odata_internal.py +++ b/tests/unit/data/test_odata_internal.py @@ -2,10 +2,12 @@ # Licensed under the MIT license. import json +import time import unittest +from enum import Enum from unittest.mock import MagicMock, patch -from PowerPlatform.Dataverse.core.errors import ValidationError +from PowerPlatform.Dataverse.core.errors import HttpError, MetadataError, ValidationError from PowerPlatform.Dataverse.data._odata import _ODataClient @@ -18,6 +20,33 @@ def _make_odata_client() -> _ODataClient: return client +def _mock_response(json_data=None, text="", status_code=200, headers=None): + """Create a mock HTTP response.""" + response = MagicMock() + response.status_code = status_code + response.text = text or (str(json_data) if json_data else "") + response.json.return_value = json_data or {} + response.headers = headers or {} + return response + + +def _entity_def_response(entity_set_name="accounts", primary_id="accountid", metadata_id="meta-001"): + """Simulate a successful EntityDefinitions response.""" + return _mock_response( + json_data={ + "value": [ + { + "LogicalName": "account", + "EntitySetName": entity_set_name, + "PrimaryIdAttribute": primary_id, + "MetadataId": metadata_id, + "SchemaName": "Account", + } + ] + } + ) + + class TestUpsertMultipleValidation(unittest.TestCase): """Unit tests for _ODataClient._upsert_multiple internal validation.""" @@ -58,6 +87,10 @@ def test_equal_lengths_does_not_raise(self): post_calls = [c for c in self.od._request.call_args_list if c.args[0] == "post"] self.assertEqual(len(post_calls), 1) self.assertIn("UpsertMultiple", post_calls[0].args[1]) + payload = post_calls[0].kwargs.get("json", {}) + self.assertEqual(len(payload["Targets"]), 2) + self.assertIn("@odata.type", payload["Targets"][0]) + self.assertIn("@odata.id", payload["Targets"][0]) def test_payload_excludes_alternate_key_fields(self): """Alternate key fields must NOT appear in the request body (only in @odata.id).""" @@ -301,7 +334,7 @@ def test_record_keys_lowercased(self): """Regular record field names are lowercased before sending.""" self.od._create("accounts", "account", {"Name": "Contoso", "AccountNumber": "ACC-001"}) call = self._post_call() - payload = json.loads(call.kwargs["data"]) if "data" in call.kwargs else call.kwargs["json"] + payload = json.loads(call.kwargs["data"]) self.assertIn("name", payload) self.assertIn("accountnumber", payload) self.assertNotIn("Name", payload) @@ -319,7 +352,7 @@ def test_odata_bind_keys_preserve_case(self): }, ) call = self._post_call() - payload = json.loads(call.kwargs["data"]) if "data" in call.kwargs else call.kwargs["json"] + payload = json.loads(call.kwargs["data"]) self.assertIn("new_name", payload) self.assertIn("new_CustomerId@odata.bind", payload) self.assertIn("new_AgentId@odata.bind", payload) @@ -385,7 +418,7 @@ def test_record_keys_lowercased(self): """Regular field names are lowercased in _update.""" self.od._update("new_ticket", "00000000-0000-0000-0000-000000000001", {"New_Status": 100000001}) call = self._patch_call() - payload = json.loads(call.kwargs["data"]) if "data" in call.kwargs else call.kwargs["json"] + payload = json.loads(call.kwargs["data"]) self.assertIn("new_status", payload) self.assertNotIn("New_Status", payload) @@ -400,7 +433,7 @@ def test_odata_bind_keys_preserve_case(self): }, ) call = self._patch_call() - payload = json.loads(call.kwargs["data"]) if "data" in call.kwargs else call.kwargs["json"] + payload = json.loads(call.kwargs["data"]) self.assertIn("new_status", payload) self.assertIn("new_CustomerId@odata.bind", payload) self.assertNotIn("new_customerid@odata.bind", payload) @@ -469,7 +502,7 @@ def test_record_keys_lowercased(self): """Record field names are lowercased before sending.""" self.od._upsert("accounts", "account", {"accountnumber": "ACC-001"}, {"Name": "Contoso"}) call = self._patch_call() - payload = json.loads(call.kwargs["data"]) if "data" in call.kwargs else call.kwargs["json"] + payload = call.kwargs["json"] self.assertIn("name", payload) self.assertNotIn("Name", payload) @@ -485,7 +518,7 @@ def test_odata_bind_keys_preserve_case(self): }, ) call = self._patch_call() - payload = json.loads(call.kwargs["data"]) if "data" in call.kwargs else call.kwargs["json"] + payload = call.kwargs["json"] # Regular field is lowercased self.assertIn("name", payload) # @odata.bind key preserves original casing @@ -519,14 +552,1029 @@ def test_returns_none(self): self.assertIsNone(result) +class TestStaticHelpers(unittest.TestCase): + """Unit tests for _ODataClient static helper methods.""" + + def test_normalize_cache_key_non_string_returns_empty(self): + """_normalize_cache_key with non-string returns empty string.""" + self.assertEqual(_ODataClient._normalize_cache_key(None), "") + self.assertEqual(_ODataClient._normalize_cache_key(42), "") + + def test_lowercase_list_none_returns_none(self): + """_lowercase_list(None) returns None.""" + self.assertIsNone(_ODataClient._lowercase_list(None)) + + def test_lowercase_list_empty_returns_empty(self): + """_lowercase_list([]) returns [].""" + self.assertFalse(_ODataClient._lowercase_list([])) + + def test_lowercase_keys_non_dict_returned_as_is(self): + """_lowercase_keys with non-dict input returns it unchanged.""" + self.assertEqual(_ODataClient._lowercase_keys("a string"), "a string") + self.assertIsNone(_ODataClient._lowercase_keys(None)) + + def test_lowercase_keys_preserves_odata_bind_casing(self): + """_lowercase_keys lowercases regular keys but preserves @odata.bind key casing.""" + result = _ODataClient._lowercase_keys( + { + "Name": "Contoso", + "new_CustomerId@odata.bind": "/contacts(id-1)", + "@odata.type": "Microsoft.Dynamics.CRM.account", + } + ) + self.assertIn("name", result) + self.assertNotIn("Name", result) + self.assertIn("new_CustomerId@odata.bind", result) + self.assertNotIn("new_customerid@odata.bind", result) + self.assertIn("@odata.type", result) + + def test_to_pascal_basic(self): + """_to_pascal converts snake_case to PascalCase.""" + client = _make_odata_client() + self.assertEqual(client._to_pascal("hello_world"), "HelloWorld") + self.assertEqual(client._to_pascal("my_table_name"), "MyTableName") + self.assertEqual(client._to_pascal("single"), "Single") + + +class TestRequestErrorParsing(unittest.TestCase): + """Unit tests for _ODataClient._request error response handling.""" + + def setUp(self): + mock_auth = MagicMock() + mock_auth._acquire_token.return_value = MagicMock(access_token="token") + self.client = _ODataClient(mock_auth, "https://example.crm.dynamics.com") + + def _make_raw_response(self, status_code, json_data=None, headers=None): + response = MagicMock() + response.status_code = status_code + response.text = "body" + response.json.return_value = json_data or {} + response.headers = headers or {} + return response + + def test_message_key_fallback_used_when_no_error_key(self): + """_request uses 'message' key when 'error' key is absent.""" + response = self._make_raw_response(400, json_data={"message": "Bad input received"}) + self.client._raw_request = MagicMock(return_value=response) + with self.assertRaises(HttpError) as ctx: + self.client._request("get", "http://example.com/test") + self.assertIn("Bad input received", str(ctx.exception)) + + def test_retry_after_non_int_not_stored_in_details(self): + """Retry-After header that is non-numeric results in retry_after absent from details.""" + response = self._make_raw_response(429, headers={"Retry-After": "not-a-number"}) + self.client._raw_request = MagicMock(return_value=response) + with self.assertRaises(HttpError) as ctx: + self.client._request("get", "http://example.com/test") + self.assertIsNone(ctx.exception.details.get("retry_after")) + + def test_retry_after_int_stored_in_details(self): + """Retry-After header that is numeric is stored in exception details.""" + response = self._make_raw_response(429, headers={"Retry-After": "30"}) + self.client._raw_request = MagicMock(return_value=response) + with self.assertRaises(HttpError) as ctx: + self.client._request("get", "http://example.com/test") + self.assertEqual(ctx.exception.details.get("retry_after"), 30) + + +class TestCreateMultiple(unittest.TestCase): + """Unit tests for _ODataClient._create_multiple.""" + + def setUp(self): + self.od = _make_odata_client() + + def test_non_dict_items_raise_type_error(self): + """_create_multiple raises TypeError for non-dict items.""" + with self.assertRaises(TypeError): + self.od._create_multiple("accounts", "account", ["not a dict"]) + + def test_odata_type_already_present_not_duplicated(self): + """If @odata.type already in record, it is preserved as-is.""" + self.od._request.return_value = _mock_response( + json_data={"Ids": ["id-1"]}, + text='{"Ids": ["id-1"]}', + ) + self.od._create_multiple( + "accounts", + "account", + [{"@odata.type": "Microsoft.Dynamics.CRM.account", "name": "Test"}], + ) + post_calls = [c for c in self.od._request.call_args_list if c.args[0] == "post"] + target = json.loads(post_calls[0].kwargs["data"])["Targets"][0] + self.assertEqual(target["@odata.type"], "Microsoft.Dynamics.CRM.account") + + def test_body_not_dict_returns_empty_list(self): + """When response body is not a dict, returns empty list.""" + response = _mock_response(text='["id1", "id2"]') + response.json.return_value = ["id1", "id2"] + self.od._request.return_value = response + result = self.od._create_multiple("accounts", "account", [{"name": "A"}]) + self.assertEqual(result, []) + + def test_value_key_path_extracts_ids(self): + """Falls back to 'value' key to extract IDs via heuristic.""" + long_guid = "a" * 32 + response = _mock_response( + json_data={"value": [{"accountid": long_guid, "name": "Test"}]}, + text="...", + ) + self.od._request.return_value = response + result = self.od._create_multiple("accounts", "account", [{"name": "Test"}]) + self.assertEqual(result, [long_guid]) + + def test_value_key_with_non_dict_items_returns_empty(self): + """'value' list with non-dict items returns empty list.""" + response = _mock_response(json_data={"value": ["not-a-dict"]}, text="...") + self.od._request.return_value = response + self.od._convert_labels_to_ints = MagicMock(side_effect=lambda _, rec: rec) + result = self.od._create_multiple("accounts", "account", [{"name": "Test"}]) + self.assertEqual(result, []) + + def test_no_ids_or_value_key_returns_empty_list(self): + """When body has neither 'Ids' nor 'value' keys, returns empty list.""" + response = _mock_response(json_data={"something_else": "data"}, text="...") + self.od._request.return_value = response + self.od._convert_labels_to_ints = MagicMock(side_effect=lambda _, rec: rec) + result = self.od._create_multiple("accounts", "account", [{"name": "Test"}]) + self.assertEqual(result, []) + + def test_value_parse_error_returns_empty_list(self): + """ValueError in body.json() returns empty list.""" + response = MagicMock() + response.text = "invalid json" + response.json.side_effect = ValueError("bad json") + self.od._request.return_value = response + self.od._convert_labels_to_ints = MagicMock(side_effect=lambda _, rec: rec) + result = self.od._create_multiple("accounts", "account", [{"name": "Test"}]) + self.assertEqual(result, []) + + def test_multiple_records_returns_all_ids(self): + """All IDs from the Ids response key are returned for multiple input records.""" + self.od._request.return_value = _mock_response( + json_data={"Ids": ["id-1", "id-2", "id-3"]}, + text='{"Ids": ["id-1", "id-2", "id-3"]}', + ) + result = self.od._create_multiple( + "accounts", + "account", + [{"name": "A"}, {"name": "B"}, {"name": "C"}], + ) + self.assertEqual(result, ["id-1", "id-2", "id-3"]) + + +class TestPrimaryIdAttr(unittest.TestCase): + """Unit tests for _ODataClient._primary_id_attr cache-miss behavior.""" + + def setUp(self): + self.od = _make_odata_client() + + def test_cache_miss_resolves_via_entity_set_lookup(self): + """Cache miss triggers entity set lookup and populates primary ID cache.""" + + def mock_entity_set(table_schema_name): + cache_key = table_schema_name.lower() + self.od._logical_to_entityset_cache[cache_key] = "accounts" + self.od._logical_primaryid_cache[cache_key] = "accountid" + return "accounts" + + self.od._entity_set_from_schema_name = MagicMock(side_effect=mock_entity_set) + result = self.od._primary_id_attr("account") + self.assertEqual(result, "accountid") + + def test_cache_miss_no_primary_id_raises_runtime_error(self): + """Cache miss with no PrimaryIdAttribute in metadata raises RuntimeError.""" + + def mock_entity_set_no_pid(table_schema_name): + cache_key = table_schema_name.lower() + self.od._logical_to_entityset_cache[cache_key] = "accounts" + return "accounts" + + self.od._entity_set_from_schema_name = MagicMock(side_effect=mock_entity_set_no_pid) + with self.assertRaises(RuntimeError) as ctx: + self.od._primary_id_attr("account") + self.assertIn("PrimaryIdAttribute not resolved", str(ctx.exception)) + + def test_cache_hit_returns_without_lookup(self): + """Cache hit returns primary ID immediately without issuing any request.""" + self.od._logical_primaryid_cache["account"] = "accountid" + result = self.od._primary_id_attr("account") + self.assertEqual(result, "accountid") + self.od._request.assert_not_called() + + +class TestUpdateByIds(unittest.TestCase): + """Unit tests for _ODataClient._update_by_ids.""" + + def setUp(self): + self.od = _make_odata_client() + + def test_non_list_ids_raises_type_error(self): + """_update_by_ids raises TypeError when ids is not a list.""" + with self.assertRaises(TypeError): + self.od._update_by_ids("account", "not-a-list", {"name": "X"}) + + def test_empty_ids_returns_none(self): + """_update_by_ids returns None immediately for empty ids list.""" + result = self.od._update_by_ids("account", [], {"name": "X"}) + self.assertIsNone(result) + self.od._request.assert_not_called() + + def test_non_list_non_dict_changes_raises_type_error(self): + """_update_by_ids raises TypeError for changes that is not dict or list.""" + self.od._primary_id_attr = MagicMock(return_value="accountid") + self.od._entity_set_from_schema_name = MagicMock(return_value="accounts") + with self.assertRaises(TypeError) as ctx: + self.od._update_by_ids("account", ["id-1"], "bad-changes") + self.assertIn("changes must be dict or list[dict]", str(ctx.exception)) + + def test_list_changes_length_mismatch_raises_value_error(self): + """_update_by_ids raises ValueError when changes list length != ids length.""" + self.od._primary_id_attr = MagicMock(return_value="accountid") + self.od._entity_set_from_schema_name = MagicMock(return_value="accounts") + with self.assertRaises(ValueError) as ctx: + self.od._update_by_ids("account", ["id-1", "id-2"], [{"name": "A"}]) + self.assertIn("Length of changes list must match", str(ctx.exception)) + + def test_non_dict_patch_in_list_raises_type_error(self): + """_update_by_ids raises TypeError when a patch in the list is not a dict.""" + self.od._primary_id_attr = MagicMock(return_value="accountid") + self.od._entity_set_from_schema_name = MagicMock(return_value="accounts") + self.od._update_multiple = MagicMock() + with self.assertRaises(TypeError) as ctx: + self.od._update_by_ids("account", ["id-1"], ["not-a-dict"]) + self.assertIn("Each patch must be a dict", str(ctx.exception)) + + def test_dict_changes_broadcasts_to_all_ids(self): + """_update_by_ids with dict changes builds one batch record per ID.""" + self.od._primary_id_attr = MagicMock(return_value="accountid") + self.od._entity_set_from_schema_name = MagicMock(return_value="accounts") + self.od._update_multiple = MagicMock() + self.od._update_by_ids("account", ["id-1", "id-2"], {"name": "X"}) + self.od._update_multiple.assert_called_once() + _, _, batch = self.od._update_multiple.call_args.args + self.assertEqual(len(batch), 2) + self.assertEqual(batch[0]["accountid"], "id-1") + self.assertEqual(batch[1]["accountid"], "id-2") + self.assertEqual(batch[0]["name"], "X") + self.assertEqual(batch[1]["name"], "X") + + def test_list_changes_merges_per_record(self): + """_update_by_ids with list changes merges each patch with its corresponding ID.""" + self.od._primary_id_attr = MagicMock(return_value="accountid") + self.od._entity_set_from_schema_name = MagicMock(return_value="accounts") + self.od._update_multiple = MagicMock() + self.od._update_by_ids("account", ["id-1", "id-2"], [{"name": "A"}, {"name": "B"}]) + _, _, batch = self.od._update_multiple.call_args.args + self.assertEqual(batch[0], {"accountid": "id-1", "name": "A"}) + self.assertEqual(batch[1], {"accountid": "id-2", "name": "B"}) + + +class TestUpdateMultiple(unittest.TestCase): + """Unit tests for _ODataClient._update_multiple.""" + + def setUp(self): + self.od = _make_odata_client() + + def test_non_list_records_raises_type_error(self): + """_update_multiple raises TypeError for non-list records.""" + with self.assertRaises(TypeError): + self.od._update_multiple("accounts", "account", "not-a-list") + + def test_empty_list_raises_type_error(self): + """_update_multiple raises TypeError for empty list.""" + with self.assertRaises(TypeError): + self.od._update_multiple("accounts", "account", []) + + def test_odata_type_already_present_not_overridden(self): + """If all records have @odata.type, it is preserved.""" + self.od._request.return_value = _mock_response() + records = [{"@odata.type": "Microsoft.Dynamics.CRM.CustomType", "accountid": "id-1", "name": "A"}] + self.od._update_multiple("accounts", "account", records) + payload = json.loads(self.od._request.call_args.kwargs["data"]) + self.assertEqual(payload["Targets"][0]["@odata.type"], "Microsoft.Dynamics.CRM.CustomType") + + def test_posts_to_update_multiple_endpoint(self): + """_update_multiple POSTs to {entity_set}/Microsoft.Dynamics.CRM.UpdateMultiple.""" + self.od._request.return_value = _mock_response() + self.od._update_multiple("accounts", "account", [{"accountid": "id-1", "name": "X"}]) + method, url = self.od._request.call_args.args + self.assertEqual(method, "post") + self.assertIn("accounts/Microsoft.Dynamics.CRM.UpdateMultiple", url) + + def test_payload_contains_targets_array(self): + """_update_multiple sends {"Targets": [...]} with @odata.type injected per record.""" + self.od._request.return_value = _mock_response() + self.od._update_multiple("accounts", "account", [{"accountid": "id-1", "name": "X"}]) + payload = json.loads(self.od._request.call_args.kwargs["data"]) + self.assertIn("Targets", payload) + self.assertEqual(len(payload["Targets"]), 1) + self.assertIn("@odata.type", payload["Targets"][0]) + + def test_multiple_records_all_in_targets(self): + """All records are included in the Targets payload for multiple inputs.""" + self.od._request.return_value = _mock_response() + records = [ + {"accountid": "id-1", "name": "A"}, + {"accountid": "id-2", "name": "B"}, + {"accountid": "id-3", "name": "C"}, + ] + self.od._update_multiple("accounts", "account", records) + payload = json.loads(self.od._request.call_args.kwargs["data"]) + self.assertEqual(len(payload["Targets"]), 3) + self.assertEqual(payload["Targets"][0]["accountid"], "id-1") + self.assertEqual(payload["Targets"][2]["accountid"], "id-3") + + +class TestDeleteMultiple(unittest.TestCase): + """Unit tests for _ODataClient._delete_multiple.""" + + def setUp(self): + self.od = _make_odata_client() + self.od._primary_id_attr = MagicMock(return_value="accountid") + + def test_empty_ids_returns_none(self): + """_delete_multiple returns None for empty IDs.""" + result = self.od._delete_multiple("account", []) + self.assertIsNone(result) + self.od._request.assert_not_called() + + def test_filters_out_falsy_ids(self): + """_delete_multiple filters None/empty strings from ids.""" + result = self.od._delete_multiple("account", [None, "", None]) + self.assertIsNone(result) + self.od._request.assert_not_called() + + def test_posts_bulk_delete_payload(self): + """_delete_multiple issues POST to BulkDelete with correct payload.""" + self.od._request.return_value = _mock_response(json_data={"JobId": "job-001"}, text='{"JobId": "job-001"}') + result = self.od._delete_multiple("account", ["id-1", "id-2"]) + self.assertEqual(result, "job-001") + call_args = self.od._request.call_args + self.assertEqual(call_args.args[0], "post") + self.assertIn("BulkDelete", call_args.args[1]) + payload = json.loads(call_args.kwargs["data"]) + self.assertIn("QuerySet", payload) + self.assertIn("JobName", payload) + query = payload["QuerySet"][0] + self.assertEqual(query["EntityName"], "account") + conditions = query["Criteria"]["Conditions"] + self.assertEqual(len(conditions), 1) + self.assertEqual(conditions[0]["AttributeName"], "accountid") + values = conditions[0]["Values"] + self.assertEqual(len(values), 2) + self.assertEqual({v["Value"] for v in values}, {"id-1", "id-2"}) + + def test_returns_none_when_no_job_id_in_body(self): + """_delete_multiple returns None when response body has no JobId.""" + self.od._request.return_value = _mock_response(json_data={}, text="{}") + result = self.od._delete_multiple("account", ["id-1"]) + self.assertIsNone(result) + + def test_handles_value_error_in_json_parsing(self): + """_delete_multiple handles ValueError in response JSON parsing gracefully.""" + response = MagicMock() + response.text = "invalid" + response.json.side_effect = ValueError + self.od._request.return_value = response + result = self.od._delete_multiple("account", ["id-1"]) + self.assertIsNone(result) + + +class TestFormatKey(unittest.TestCase): + """Unit tests for _ODataClient._format_key.""" + + def setUp(self): + self.od = _make_odata_client() + + def test_guid_wrapped_in_parens(self): + """_format_key wraps 36-char GUID in parentheses.""" + guid = "11111111-2222-3333-4444-555555555555" + self.assertEqual(self.od._format_key(guid), f"({guid})") + + def test_already_wrapped_key_returned_as_is(self): + """_format_key returns already-parenthesized key unchanged.""" + key = "(some-key)" + self.assertEqual(self.od._format_key(key), key) + + def test_alternate_key_with_quotes_is_escaped(self): + """_format_key wraps alternate key with single-quoted value in parentheses.""" + result = self.od._format_key("mykey='it''s value'") + self.assertEqual(result, "(mykey='it''s value')") + + +class TestGetMultiple(unittest.TestCase): + """Unit tests for _ODataClient._get_multiple query parameter handling.""" + + def setUp(self): + self.od = _make_odata_client() + self.od._entity_set_from_schema_name = MagicMock(return_value="accounts") + + def _single_page_response(self, items=None): + data = {"value": items or [{"accountid": "id-1"}]} + response = _mock_response(json_data=data, text=str(data)) + self.od._request.return_value = response + + def test_filter_param_passed(self): + """_get_multiple passes $filter to params.""" + self._single_page_response() + list(self.od._get_multiple("account", filter="statecode eq 0")) + params = self.od._request.call_args.kwargs["params"] + self.assertEqual(params["$filter"], "statecode eq 0") + + def test_orderby_param_passed(self): + """_get_multiple passes $orderby to params.""" + self._single_page_response() + list(self.od._get_multiple("account", orderby=["name asc", "createdon desc"])) + params = self.od._request.call_args.kwargs["params"] + self.assertEqual(params["$orderby"], "name asc,createdon desc") + + def test_expand_param_passed(self): + """_get_multiple passes $expand to params.""" + self._single_page_response() + list(self.od._get_multiple("account", expand=["contact_customer_accounts"])) + params = self.od._request.call_args.kwargs["params"] + self.assertEqual(params["$expand"], "contact_customer_accounts") + + def test_top_param_passed(self): + """_get_multiple passes $top to params.""" + self._single_page_response() + list(self.od._get_multiple("account", top=5)) + params = self.od._request.call_args.kwargs["params"] + self.assertEqual(params["$top"], 5) + + def test_count_param_passed(self): + """_get_multiple passes $count=true when count=True.""" + self._single_page_response() + list(self.od._get_multiple("account", count=True)) + params = self.od._request.call_args.kwargs["params"] + self.assertEqual(params["$count"], "true") + + def test_include_annotations_sets_prefer_header(self): + """_get_multiple sets Prefer header with include-annotations.""" + self._single_page_response() + list(self.od._get_multiple("account", include_annotations="*")) + headers = self.od._request.call_args.kwargs.get("headers") or {} + self.assertIn("Prefer", headers) + self.assertIn("include-annotations", headers["Prefer"]) + + def test_page_size_sets_prefer_header(self): + """_get_multiple sets Prefer odata.maxpagesize when page_size > 0.""" + self._single_page_response() + list(self.od._get_multiple("account", page_size=50)) + headers = self.od._request.call_args.kwargs.get("headers") or {} + self.assertIn("odata.maxpagesize=50", headers.get("Prefer", "")) + + def test_value_error_in_json_returns_empty(self): + """ValueError in page JSON parsing yields nothing.""" + response = MagicMock() + response.text = "bad json" + response.json.side_effect = ValueError + self.od._request.return_value = response + pages = list(self.od._get_multiple("account")) + self.assertEqual(pages, []) + + def test_yields_value_items_as_page(self): + """_get_multiple yields the 'value' list as a page of dicts.""" + items = [{"accountid": "id-1", "name": "A"}, {"accountid": "id-2", "name": "B"}] + self._single_page_response(items) + pages = list(self.od._get_multiple("account")) + self.assertEqual(len(pages), 1) + self.assertEqual(pages[0], items) + + def test_follows_nextlink_pagination(self): + """_get_multiple follows @odata.nextLink across multiple pages.""" + page1 = _mock_response( + json_data={ + "value": [{"accountid": "id-1"}], + "@odata.nextLink": "https://example.crm.dynamics.com/next-page", + }, + text="...", + ) + page2 = _mock_response( + json_data={"value": [{"accountid": "id-2"}]}, + text="...", + ) + self.od._request.side_effect = [page1, page2] + pages = list(self.od._get_multiple("account")) + self.assertEqual(len(pages), 2) + self.assertEqual(pages[0][0]["accountid"], "id-1") + self.assertEqual(pages[1][0]["accountid"], "id-2") + + def test_stops_when_no_nextlink(self): + """_get_multiple stops after a page without nextLink.""" + self._single_page_response([{"accountid": "id-1"}]) + pages = list(self.od._get_multiple("account")) + self.assertEqual(len(pages), 1) + self.od._request.assert_called_once() + + def test_filters_non_dict_items_from_page(self): + """_get_multiple filters out non-dict items from each page.""" + data = {"value": [{"accountid": "id-1"}, "not-a-dict", 42]} + response = _mock_response(json_data=data, text=str(data)) + self.od._request.return_value = response + pages = list(self.od._get_multiple("account")) + self.assertEqual(len(pages), 1) + self.assertEqual(len(pages[0]), 1) + self.assertEqual(pages[0][0]["accountid"], "id-1") + + def test_empty_value_list_yields_nothing(self): + """_get_multiple yields nothing when value list is empty.""" + data = {"value": []} + response = _mock_response(json_data=data, text=str(data)) + self.od._request.return_value = response + pages = list(self.od._get_multiple("account")) + self.assertEqual(pages, []) + + +class TestQuerySql(unittest.TestCase): + """Unit tests for _ODataClient._query_sql.""" + + def setUp(self): + self.od = _make_odata_client() + self.od._entity_set_from_schema_name = MagicMock(return_value="accounts") + + def test_non_string_sql_raises_validation_error(self): + """_query_sql raises ValidationError for non-string sql.""" + with self.assertRaises(ValidationError): + self.od._query_sql(123) + + def test_empty_sql_raises_validation_error(self): + """_query_sql raises ValidationError for empty sql.""" + with self.assertRaises(ValidationError): + self.od._query_sql(" ") + + def test_returns_value_list(self): + """_query_sql returns rows from response 'value' key.""" + self.od._request.return_value = _mock_response( + json_data={"value": [{"accountid": "id-1", "name": "Contoso"}]}, + text="...", + ) + result = self.od._query_sql("SELECT name FROM account") + self.assertEqual(len(result), 1) + self.assertEqual(result[0]["name"], "Contoso") + + def test_filters_non_dict_rows(self): + """_query_sql filters out non-dict rows from 'value' list.""" + self.od._request.return_value = _mock_response( + json_data={"value": [{"name": "A"}, "not-a-dict", 42]}, text="..." + ) + result = self.od._query_sql("SELECT name FROM account") + self.assertEqual(len(result), 1) + + def test_body_as_list_fallback(self): + """_query_sql handles body being a list directly.""" + response = _mock_response(text="...") + response.json.return_value = [{"name": "A"}, {"name": "B"}] + self.od._request.return_value = response + result = self.od._query_sql("SELECT name FROM account") + self.assertEqual(len(result), 2) + + def test_value_error_in_json_returns_empty(self): + """_query_sql returns empty list when JSON parsing fails.""" + response = MagicMock() + response.text = "bad json" + response.json.side_effect = ValueError + self.od._request.return_value = response + result = self.od._query_sql("SELECT name FROM account") + self.assertEqual(result, []) + + def test_unexpected_body_returns_empty(self): + """_query_sql returns empty list for non-dict, non-list body.""" + response = _mock_response(text="...") + response.json.return_value = "unexpected" + self.od._request.return_value = response + result = self.od._query_sql("SELECT name FROM account") + self.assertEqual(result, []) + + def test_extract_non_string_raises_value_error(self): + """_extract_logical_table with non-string raises ValueError.""" + with self.assertRaises(ValueError): + _ODataClient._extract_logical_table(123) + + def test_extract_no_from_clause_raises_value_error(self): + """_extract_logical_table without FROM raises ValueError.""" + with self.assertRaises(ValueError) as ctx: + _ODataClient._extract_logical_table("SELECT name, surname") + self.assertIn("FROM", str(ctx.exception)) + + +class TestEntitySetFromSchemaName(unittest.TestCase): + """Unit tests for _ODataClient._entity_set_from_schema_name.""" + + def setUp(self): + self.od = _make_odata_client() + + def test_empty_table_schema_name_raises_value_error(self): + """_entity_set_from_schema_name raises ValueError for empty input.""" + with self.assertRaises(ValueError) as ctx: + self.od._entity_set_from_schema_name("") + self.assertIn("table schema name required", str(ctx.exception)) + + def test_json_value_error_in_response_treated_as_empty(self): + """_entity_set_from_schema_name handles ValueError in JSON parsing.""" + response = MagicMock() + response.text = "invalid json" + response.json.side_effect = ValueError + self.od._request.return_value = response + with self.assertRaises(MetadataError): + self.od._entity_set_from_schema_name("account") + + def test_plural_hint_when_name_ends_with_s(self): + """Error message includes plural hint when name ends with 's' (not 'ss').""" + self.od._request.return_value = _mock_response(json_data={"value": []}, text="{}") + with self.assertRaises(MetadataError) as ctx: + self.od._entity_set_from_schema_name("accounts") + self.assertIn("plural", str(ctx.exception).lower()) + + def test_no_plural_hint_when_name_ends_with_ss(self): + """No plural hint when name ends with 'ss'.""" + self.od._request.return_value = _mock_response(json_data={"value": []}, text="{}") + with self.assertRaises(MetadataError) as ctx: + self.od._entity_set_from_schema_name("address") + self.assertNotIn("plural", str(ctx.exception).lower()) + + def test_missing_entity_set_name_raises_metadata_error(self): + """MetadataError raised when EntitySetName is absent from metadata.""" + self.od._request.return_value = _mock_response( + json_data={"value": [{"LogicalName": "account", "EntitySetName": None, "PrimaryIdAttribute": "accountid"}]}, + text="...", + ) + with self.assertRaises(MetadataError) as ctx: + self.od._entity_set_from_schema_name("account") + self.assertIn("EntitySetName", str(ctx.exception)) + + def test_cache_hit_returns_without_request(self): + """Cache hit returns entity set name immediately without issuing any request.""" + self.od._logical_to_entityset_cache["account"] = "accounts" + result = self.od._entity_set_from_schema_name("account") + self.assertEqual(result, "accounts") + self.od._request.assert_not_called() + + def test_success_populates_entityset_cache(self): + """Successful API response populates _logical_to_entityset_cache.""" + self.od._request.return_value = _entity_def_response(entity_set_name="accounts", primary_id="accountid") + result = self.od._entity_set_from_schema_name("account") + self.assertEqual(result, "accounts") + self.assertEqual(self.od._logical_to_entityset_cache["account"], "accounts") + + def test_success_populates_primaryid_cache(self): + """Successful API response populates _logical_primaryid_cache.""" + self.od._request.return_value = _entity_def_response(entity_set_name="accounts", primary_id="accountid") + self.od._entity_set_from_schema_name("account") + self.assertEqual(self.od._logical_primaryid_cache["account"], "accountid") + + def test_success_without_primary_id_does_not_populate_primaryid_cache(self): + """When PrimaryIdAttribute is missing, _logical_primaryid_cache is not populated.""" + self.od._request.return_value = _mock_response( + json_data={"value": [{"LogicalName": "account", "EntitySetName": "accounts"}]}, + text="...", + ) + self.od._entity_set_from_schema_name("account") + self.assertNotIn("account", self.od._logical_primaryid_cache) + + +class TestGetEntityByTableSchemaName(unittest.TestCase): + """Unit tests for _ODataClient._get_entity_by_table_schema_name.""" + + def setUp(self): + self.od = _make_odata_client() + + def test_returns_first_match(self): + """_get_entity_by_table_schema_name returns first entity when found.""" + self.od._request.return_value = _entity_def_response() + result = self.od._get_entity_by_table_schema_name("account") + self.assertIsNotNone(result) + self.assertEqual(result["EntitySetName"], "accounts") + + def test_returns_none_when_not_found(self): + """_get_entity_by_table_schema_name returns None when no match.""" + self.od._request.return_value = _mock_response(json_data={"value": []}, text="{}") + result = self.od._get_entity_by_table_schema_name("nonexistent") + self.assertIsNone(result) + + +class TestCreateEntity(unittest.TestCase): + """Unit tests for _ODataClient._create_entity.""" + + def setUp(self): + self.od = _make_odata_client() + + def _setup_entity_creation(self, get_response=None): + """Mock _request: POST returns 201, GET returns entity definition.""" + + def side_effect(method, url, **kwargs): + if method == "post": + return _mock_response(status_code=201) + else: + return get_response or _entity_def_response() + + self.od._request.side_effect = side_effect + + def test_successful_entity_creation(self): + """_create_entity returns metadata on success.""" + self._setup_entity_creation() + result = self.od._create_entity("new_TestTable", "Test Table", [], solution_unique_name=None) + self.assertEqual(result["EntitySetName"], "accounts") + + def test_entity_set_name_missing_raises_runtime_error(self): + """_create_entity raises RuntimeError when EntitySetName not available after create.""" + get_response = _mock_response(json_data={"value": []}, text="{}") + + def side_effect(method, url, **kwargs): + return _mock_response(status_code=201) if method == "post" else get_response + + self.od._request.side_effect = side_effect + with self.assertRaises(RuntimeError) as ctx: + self.od._create_entity("new_TestTable", "Test Table", []) + self.assertIn("EntitySetName not available", str(ctx.exception)) + + def test_metadata_id_missing_raises_runtime_error(self): + """_create_entity raises RuntimeError when MetadataId missing after create.""" + get_response = _mock_response( + json_data={"value": [{"EntitySetName": "new_testtables", "SchemaName": "new_TestTable"}]}, + text="...", + ) + + def side_effect(method, url, **kwargs): + return _mock_response(status_code=201) if method == "post" else get_response + + self.od._request.side_effect = side_effect + with self.assertRaises(RuntimeError) as ctx: + self.od._create_entity("new_TestTable", "Test Table", []) + self.assertIn("MetadataId missing", str(ctx.exception)) + + def test_solution_unique_name_passed_as_param(self): + """_create_entity passes SolutionUniqueName as query param when provided.""" + self._setup_entity_creation() + self.od._create_entity("new_TestTable", "Test Table", [], solution_unique_name="MySolution") + post_call = next(c for c in self.od._request.call_args_list if c.args[0] == "post") + self.assertEqual(post_call.kwargs.get("params"), {"SolutionUniqueName": "MySolution"}) + + +class TestGetAttributeMetadata(unittest.TestCase): + """Unit tests for _ODataClient._get_attribute_metadata.""" + + def setUp(self): + self.od = _make_odata_client() + + def test_returns_attribute_when_found(self): + """_get_attribute_metadata returns attribute dict when found.""" + self.od._request.return_value = _mock_response( + json_data={"value": [{"MetadataId": "attr-001", "LogicalName": "name", "SchemaName": "Name"}]}, + text="...", + ) + result = self.od._get_attribute_metadata("meta-001", "name") + self.assertIsNotNone(result) + self.assertEqual(result["MetadataId"], "attr-001") + + def test_returns_none_when_not_found(self): + """_get_attribute_metadata returns None when attribute not in response.""" + self.od._request.return_value = _mock_response(json_data={"value": []}, text="{}") + result = self.od._get_attribute_metadata("meta-001", "name") + self.assertIsNone(result) + + def test_extra_select_fields_included(self): + """_get_attribute_metadata appends extra_select fields to $select.""" + self.od._request.return_value = _mock_response(json_data={"value": []}, text="{}") + self.od._get_attribute_metadata("meta-001", "name", extra_select="AttributeType,MaxLength") + params = self.od._request.call_args.kwargs["params"] + self.assertIn("AttributeType", params["$select"]) + self.assertIn("MaxLength", params["$select"]) + + def test_extra_select_skips_empty_pieces(self): + """_get_attribute_metadata skips empty pieces in extra_select.""" + self.od._request.return_value = _mock_response(json_data={"value": []}, text="{}") + self.od._get_attribute_metadata("meta-001", "name", extra_select=",AttributeType,") + params = self.od._request.call_args.kwargs["params"] + self.assertIn("AttributeType", params["$select"]) + + def test_extra_select_skips_odata_annotation_pieces(self): + """_get_attribute_metadata skips pieces starting with @.""" + self.od._request.return_value = _mock_response(json_data={"value": []}, text="{}") + self.od._get_attribute_metadata("meta-001", "name", extra_select="@odata.type,MaxLength") + params = self.od._request.call_args.kwargs["params"] + self.assertNotIn("@odata.type", params["$select"]) + self.assertIn("MaxLength", params["$select"]) + + def test_value_error_in_json_returns_none(self): + """_get_attribute_metadata returns None on JSON parse failure.""" + response = MagicMock() + response.text = "bad json" + response.json.side_effect = ValueError + self.od._request.return_value = response + result = self.od._get_attribute_metadata("meta-001", "name") + self.assertIsNone(result) + + +class TestWaitForAttributeVisibility(unittest.TestCase): + """Unit tests for _ODataClient._wait_for_attribute_visibility.""" + + def setUp(self): + self.od = _make_odata_client() + + def test_returns_immediately_on_success(self): + """_wait_for_attribute_visibility returns immediately when first probe succeeds.""" + self.od._request.return_value = _mock_response() + self.od._wait_for_attribute_visibility("accounts", "name", delays=(0,)) + self.od._request.assert_called_once() + + def test_retries_on_failure_then_succeeds(self): + """_wait_for_attribute_visibility retries after initial failure.""" + self.od._request.side_effect = [RuntimeError("not ready"), _mock_response()] + self.od._wait_for_attribute_visibility("accounts", "name", delays=(0, 0)) + self.assertEqual(self.od._request.call_count, 2) + + def test_sleep_is_called_for_nonzero_delays(self): + """_wait_for_attribute_visibility calls time.sleep for non-zero delays.""" + self.od._request.side_effect = [RuntimeError("not ready"), _mock_response()] + with patch("PowerPlatform.Dataverse.data._odata.time.sleep") as mock_sleep: + self.od._wait_for_attribute_visibility("accounts", "name", delays=(0, 5)) + mock_sleep.assert_called_once_with(5) + + def test_raises_runtime_error_after_all_retries_exhausted(self): + """_wait_for_attribute_visibility raises RuntimeError when all retries fail.""" + self.od._request.side_effect = RuntimeError("not ready") + with self.assertRaises(RuntimeError) as ctx: + self.od._wait_for_attribute_visibility("accounts", "name", delays=(0, 0)) + self.assertIn("did not become visible", str(ctx.exception)) + + +class TestLocalizedLabelsPayload(unittest.TestCase): + """Unit tests for _ODataClient._build_localizedlabels_payload.""" + + def setUp(self): + self.od = _make_odata_client() + + def test_non_int_lang_raises_value_error(self): + """_build_localizedlabels_payload raises ValueError for non-int language code.""" + with self.assertRaises(ValueError) as ctx: + self.od._build_localizedlabels_payload({"1033": "English"}) + self.assertIn("must be int", str(ctx.exception)) + + def test_non_string_label_raises_value_error(self): + """_build_localizedlabels_payload raises ValueError for non-string label.""" + with self.assertRaises(ValueError) as ctx: + self.od._build_localizedlabels_payload({1033: 42}) + self.assertIn("non-empty string", str(ctx.exception)) + + def test_empty_translations_raises_value_error(self): + """_build_localizedlabels_payload raises ValueError for empty translations.""" + with self.assertRaises(ValueError) as ctx: + self.od._build_localizedlabels_payload({}) + self.assertIn("At least one translation", str(ctx.exception)) + + def test_empty_string_label_raises_value_error(self): + """_build_localizedlabels_payload raises ValueError for empty string label.""" + with self.assertRaises(ValueError): + self.od._build_localizedlabels_payload({1033: " "}) + + +class TestEnumOptionSetPayload(unittest.TestCase): + """Unit tests for _ODataClient._enum_optionset_payload.""" + + def setUp(self): + self.od = _make_odata_client() + + def test_empty_enum_raises_value_error(self): + """_enum_optionset_payload raises ValueError for enum with no members.""" + + class EmptyEnum(Enum): + pass + + with self.assertRaises(ValueError) as ctx: + self.od._enum_optionset_payload("new_Status", EmptyEnum) + self.assertIn("no members", str(ctx.exception)) + + def test_int_key_in_labels_resolved_to_member_name(self): + """__labels__ with int keys (matching enum values) are resolved to member names.""" + + class Status(Enum): + Active = 1 + Inactive = 2 + + Status.__labels__ = {1033: {1: "Active", 2: "Inactive"}} + result = self.od._enum_optionset_payload("new_Status", Status) + self.assertEqual(len(result["OptionSet"]["Options"]), 2) + + def test_enum_member_object_as_labels_key(self): + """__labels__ with enum member objects as keys resolves member name.""" + + class Status(Enum): + Active = 1 + Inactive = 2 + + Status.__labels__ = {1033: {Status.Active: "Active Label", Status.Inactive: "Inactive Label"}} + result = self.od._enum_optionset_payload("new_Status", Status) + options = result["OptionSet"]["Options"] + self.assertEqual(len(options), 2) + active_opt = next(o for o in options if o["Value"] == 1) + active_label = next( + loc["Label"] for loc in active_opt["Label"]["LocalizedLabels"] if loc["LanguageCode"] == 1033 + ) + self.assertEqual(active_label, "Active Label") + + def test_int_key_not_matching_any_member_raises_value_error(self): + """__labels__ with int key not matching any member raises ValueError.""" + + class Status(Enum): + Active = 1 + + Status.__labels__ = {1033: {99: "Unknown"}} + with self.assertRaises(ValueError) as ctx: + self.od._enum_optionset_payload("new_Status", Status) + self.assertIn("int key", str(ctx.exception)) + + def test_duplicate_enum_values_raises_value_error(self): + """_enum_optionset_payload raises ValueError when two members share the same int value.""" + + # Python treats second definition as an alias; __members__ exposes both names + class Status(Enum): + Active = 1 + DuplicateActive = 1 # alias for Active in Python Enum + + with self.assertRaises(ValueError) as ctx: + self.od._enum_optionset_payload("new_Status", Status) + self.assertIn("Duplicate", str(ctx.exception)) + + def test_non_int_enum_value_raises_value_error(self): + """_enum_optionset_payload raises ValueError for enum member with a non-int value.""" + + class Status(Enum): + Active = "active" + + with self.assertRaises(ValueError) as ctx: + self.od._enum_optionset_payload("new_Status", Status) + self.assertIn("non-int", str(ctx.exception)) + + class TestAttributePayload(unittest.TestCase): """Unit tests for _ODataClient._attribute_payload.""" def setUp(self): self.od = _make_odata_client() + def test_int_dtype(self): + """'int' produces IntegerAttributeMetadata.""" + result = self.od._attribute_payload("new_Count", "int") + self.assertEqual(result["@odata.type"], "Microsoft.Dynamics.CRM.IntegerAttributeMetadata") + + def test_integer_dtype_alias(self): + """'integer' is an alias for 'int'.""" + result = self.od._attribute_payload("new_Count", "integer") + self.assertIn("Integer", result["@odata.type"]) + + def test_decimal_dtype(self): + """'decimal' produces DecimalAttributeMetadata.""" + result = self.od._attribute_payload("new_Price", "decimal") + self.assertEqual(result["@odata.type"], "Microsoft.Dynamics.CRM.DecimalAttributeMetadata") + + def test_money_dtype_alias(self): + """'money' is an alias for 'decimal'.""" + result = self.od._attribute_payload("new_Revenue", "money") + self.assertIn("Decimal", result["@odata.type"]) + + def test_float_dtype(self): + """'float' produces DoubleAttributeMetadata.""" + result = self.od._attribute_payload("new_Score", "float") + self.assertEqual(result["@odata.type"], "Microsoft.Dynamics.CRM.DoubleAttributeMetadata") + + def test_double_dtype_alias(self): + """'double' is an alias for 'float'.""" + result = self.od._attribute_payload("new_Score", "double") + self.assertIn("Double", result["@odata.type"]) + + def test_datetime_dtype(self): + """'datetime' produces DateTimeAttributeMetadata.""" + result = self.od._attribute_payload("new_CreatedDate", "datetime") + self.assertEqual(result["@odata.type"], "Microsoft.Dynamics.CRM.DateTimeAttributeMetadata") + + def test_date_dtype_alias(self): + """'date' is an alias for 'datetime'.""" + result = self.od._attribute_payload("new_BirthDate", "date") + self.assertIn("DateTime", result["@odata.type"]) + + def test_bool_dtype(self): + """'bool' produces BooleanAttributeMetadata.""" + result = self.od._attribute_payload("new_IsActive", "bool") + self.assertEqual(result["@odata.type"], "Microsoft.Dynamics.CRM.BooleanAttributeMetadata") + + def test_boolean_dtype_alias(self): + """'boolean' is an alias for 'bool'.""" + result = self.od._attribute_payload("new_IsActive", "boolean") + self.assertIn("Boolean", result["@odata.type"]) + + def test_file_dtype(self): + """'file' produces FileAttributeMetadata.""" + result = self.od._attribute_payload("new_Attachment", "file") + self.assertEqual(result["@odata.type"], "Microsoft.Dynamics.CRM.FileAttributeMetadata") + + def test_non_string_dtype_raises_value_error(self): + """Non-string dtype raises ValueError.""" + with self.assertRaises(ValueError): + self.od._attribute_payload("new_Field", 42) + def test_memo_type(self): - """'memo' should produce MemoAttributeMetadata with MaxLength 4000.""" + """'memo' produces MemoAttributeMetadata with MaxLength 4000.""" result = self.od._attribute_payload("new_Notes", "memo") self.assertEqual(result["@odata.type"], "Microsoft.Dynamics.CRM.MemoAttributeMetadata") self.assertEqual(result["SchemaName"], "new_Notes") @@ -535,13 +1583,13 @@ def test_memo_type(self): self.assertNotIn("IsPrimaryName", result) def test_multiline_alias(self): - """'multiline' should produce identical payload to 'memo'.""" + """'multiline' produces identical payload to 'memo'.""" memo_result = self.od._attribute_payload("new_Description", "memo") multiline_result = self.od._attribute_payload("new_Description", "multiline") self.assertEqual(multiline_result, memo_result) - def test_string_type(self): - """'string' should produce StringAttributeMetadata with MaxLength 200.""" + def test_string_type_max_length(self): + """'string' produces StringAttributeMetadata with MaxLength 200.""" result = self.od._attribute_payload("new_Title", "string") self.assertEqual(result["@odata.type"], "Microsoft.Dynamics.CRM.StringAttributeMetadata") self.assertEqual(result["MaxLength"], 200) @@ -553,6 +1601,388 @@ def test_unsupported_type_returns_none(self): self.assertIsNone(result) +class TestGetTableInfo(unittest.TestCase): + """Unit tests for _ODataClient._get_table_info.""" + + def setUp(self): + self.od = _make_odata_client() + + def test_returns_none_when_entity_not_found(self): + """_get_table_info returns None when entity does not exist.""" + self.od._request.return_value = _mock_response(json_data={"value": []}, text="{}") + self.assertIsNone(self.od._get_table_info("new_NonExistent")) + + def test_returns_metadata_when_found(self): + """_get_table_info returns metadata dict when entity exists.""" + self.od._request.return_value = _entity_def_response() + result = self.od._get_table_info("account") + self.assertIsNotNone(result) + self.assertIn("entity_set_name", result) + + def test_returns_full_dict_shape(self): + """_get_table_info returns all expected keys from metadata.""" + self.od._request.return_value = _entity_def_response( + entity_set_name="accounts", primary_id="accountid", metadata_id="meta-001" + ) + result = self.od._get_table_info("account") + self.assertEqual(result["table_schema_name"], "Account") + self.assertEqual(result["table_logical_name"], "account") + self.assertEqual(result["entity_set_name"], "accounts") + self.assertEqual(result["metadata_id"], "meta-001") + self.assertEqual(result["primary_id_attribute"], "accountid") + self.assertIsInstance(result["columns_created"], list) + self.assertEqual(result["columns_created"], []) + + +class TestDeleteTable(unittest.TestCase): + """Unit tests for _ODataClient._delete_table.""" + + def setUp(self): + self.od = _make_odata_client() + + def test_deletes_table_by_metadata_id(self): + """_delete_table issues DELETE to EntityDefinitions({MetadataId}).""" + self.od._request.return_value = _mock_response() + self.od._get_entity_by_table_schema_name = MagicMock( + return_value={"MetadataId": "meta-001", "SchemaName": "new_Test"} + ) + self.od._delete_table("new_Test") + delete_call = next(c for c in self.od._request.call_args_list if c.args[0] == "delete") + self.assertIn("meta-001", delete_call.args[1]) + + def test_raises_metadata_error_when_not_found(self): + """_delete_table raises MetadataError when entity does not exist.""" + self.od._get_entity_by_table_schema_name = MagicMock(return_value=None) + with self.assertRaises(MetadataError): + self.od._delete_table("new_NonExistent") + + def test_raises_metadata_error_when_metadata_id_missing(self): + """_delete_table raises MetadataError when MetadataId is absent from entity.""" + self.od._get_entity_by_table_schema_name = MagicMock(return_value={"SchemaName": "new_Test"}) + with self.assertRaises(MetadataError): + self.od._delete_table("new_Test") + + +class TestCreateAlternateKey(unittest.TestCase): + """Unit tests for _ODataClient._create_alternate_key.""" + + def setUp(self): + self.od = _make_odata_client() + + def test_creates_alternate_key(self): + """_create_alternate_key posts to Keys endpoint and returns metadata.""" + post_response = MagicMock() + post_response.headers = {"OData-EntityId": "https://example.com/Keys(key-meta-001)"} + self.od._get_entity_by_table_schema_name = MagicMock( + return_value={"MetadataId": "meta-001", "LogicalName": "account", "SchemaName": "Account"} + ) + self.od._request.return_value = post_response + result = self.od._create_alternate_key("account", "new_AccountNumKey", ["accountnumber"]) + self.assertEqual(result["schema_name"], "new_AccountNumKey") + self.assertEqual(result["key_attributes"], ["accountnumber"]) + + def test_display_name_label_passed_to_payload(self): + """_create_alternate_key includes DisplayName when display_name_label is provided.""" + post_response = MagicMock() + post_response.headers = {"OData-EntityId": "https://example.com/Keys(key-id)"} + self.od._get_entity_by_table_schema_name = MagicMock( + return_value={"MetadataId": "meta-001", "LogicalName": "account"} + ) + self.od._request.return_value = post_response + mock_label = MagicMock() + mock_label.to_dict.return_value = {"LocalizedLabels": [{"Label": "Account Number Key", "LanguageCode": 1033}]} + self.od._create_alternate_key("account", "new_AccNumKey", ["accountnumber"], display_name_label=mock_label) + payload = self.od._request.call_args.kwargs["json"] + self.assertIn("DisplayName", payload) + mock_label.to_dict.assert_called_once() + + def test_raises_metadata_error_when_table_not_found(self): + """_create_alternate_key raises MetadataError when table not found.""" + self.od._get_entity_by_table_schema_name = MagicMock(return_value=None) + with self.assertRaises(MetadataError): + self.od._create_alternate_key("nonexistent", "key", ["col"]) + + +class TestGetAlternateKeys(unittest.TestCase): + """Unit tests for _ODataClient._get_alternate_keys.""" + + def setUp(self): + self.od = _make_odata_client() + + def test_returns_keys_list(self): + """_get_alternate_keys returns list of alternate keys.""" + self.od._get_entity_by_table_schema_name = MagicMock( + return_value={"MetadataId": "meta-001", "LogicalName": "account"} + ) + self.od._request.return_value = _mock_response( + json_data={"value": [{"SchemaName": "new_AccountNumKey"}]}, + text="...", + ) + result = self.od._get_alternate_keys("account") + self.assertEqual(len(result), 1) + self.assertEqual(result[0]["SchemaName"], "new_AccountNumKey") + + def test_raises_metadata_error_when_table_not_found(self): + """_get_alternate_keys raises MetadataError when table not found.""" + self.od._get_entity_by_table_schema_name = MagicMock(return_value=None) + with self.assertRaises(MetadataError): + self.od._get_alternate_keys("nonexistent") + + +class TestDeleteAlternateKey(unittest.TestCase): + """Unit tests for _ODataClient._delete_alternate_key.""" + + def setUp(self): + self.od = _make_odata_client() + + def test_deletes_alternate_key(self): + """_delete_alternate_key issues DELETE to Keys({key_id}).""" + self.od._get_entity_by_table_schema_name = MagicMock( + return_value={"MetadataId": "meta-001", "LogicalName": "account"} + ) + self.od._request.return_value = _mock_response() + self.od._delete_alternate_key("account", "key-meta-001") + delete_call = self.od._request.call_args + self.assertEqual(delete_call.args[0], "delete") + self.assertIn("key-meta-001", delete_call.args[1]) + + def test_raises_metadata_error_when_table_not_found(self): + """_delete_alternate_key raises MetadataError when table not found.""" + self.od._get_entity_by_table_schema_name = MagicMock(return_value=None) + with self.assertRaises(MetadataError): + self.od._delete_alternate_key("nonexistent", "key-id") + + +class TestCreateTable(unittest.TestCase): + """Unit tests for _ODataClient._create_table.""" + + def setUp(self): + self.od = _make_odata_client() + + def _setup_for_create(self, entity_exists=False): + """Mock helpers for _create_table.""" + existing = {"MetadataId": "meta-001", "EntitySetName": "accounts"} if entity_exists else None + created = { + "MetadataId": "meta-001", + "EntitySetName": "new_testtables", + "LogicalName": "new_testtable", + "SchemaName": "new_TestTable", + "PrimaryNameAttribute": "new_name", + "PrimaryIdAttribute": "new_testtableid", + } + call_count = [0] + + def mock_get_entity(table_schema_name, headers=None): + call_count[0] += 1 + if entity_exists: + return existing + return None if call_count[0] == 1 else created + + self.od._get_entity_by_table_schema_name = MagicMock(side_effect=mock_get_entity) + self.od._request.return_value = _mock_response(status_code=201) + + def test_creates_table_successfully(self): + """_create_table returns metadata dict on success.""" + self._setup_for_create() + result = self.od._create_table("new_TestTable", {"new_Name": "string", "new_Age": "int"}) + self.assertEqual(result["table_schema_name"], "new_TestTable") + self.assertIn("new_Name", result["columns_created"]) + self.assertIn("new_Age", result["columns_created"]) + + def test_raises_metadata_error_when_table_already_exists(self): + """_create_table raises MetadataError when table already exists.""" + self._setup_for_create(entity_exists=True) + with self.assertRaises(MetadataError): + self.od._create_table("new_TestTable", {"new_Name": "string"}) + + def test_raises_value_error_for_unsupported_column_type(self): + """_create_table raises ValueError for unsupported column type.""" + self._setup_for_create() + with self.assertRaises(ValueError) as ctx: + self.od._create_table("new_TestTable", {"new_Col": "unsupported_type"}) + self.assertIn("Unsupported column type", str(ctx.exception)) + + def test_raises_type_error_for_non_string_solution_name(self): + """_create_table raises TypeError when solution_unique_name is not str.""" + self._setup_for_create() + with self.assertRaises(TypeError): + self.od._create_table("new_TestTable", {}, solution_unique_name=123) + + def test_raises_value_error_for_empty_solution_name(self): + """_create_table raises ValueError when solution_unique_name is empty string.""" + self._setup_for_create() + with self.assertRaises(ValueError): + self.od._create_table("new_TestTable", {}, solution_unique_name="") + + def test_primary_column_schema_name_used_when_provided(self): + """_create_table uses provided primary_column_schema_name in the POST payload.""" + self._setup_for_create() + self.od._create_table("new_TestTable", {}, primary_column_schema_name="new_CustomName") + post_json = self.od._request.call_args.kwargs["json"] + attrs = post_json["Attributes"] + primary_attr = next((a for a in attrs if a.get("IsPrimaryName")), None) + self.assertIsNotNone(primary_attr) + self.assertEqual(primary_attr["SchemaName"], "new_CustomName") + + +class TestCreateColumns(unittest.TestCase): + """Unit tests for _ODataClient._create_columns.""" + + def setUp(self): + self.od = _make_odata_client() + self.od._get_entity_by_table_schema_name = MagicMock( + return_value={"MetadataId": "meta-001", "SchemaName": "new_Test"} + ) + self.od._request.return_value = _mock_response(status_code=201) + + def test_creates_columns_successfully(self): + """_create_columns returns list of created column names.""" + result = self.od._create_columns("new_Test", {"new_Name": "string", "new_Age": "int"}) + self.assertIn("new_Name", result) + self.assertIn("new_Age", result) + + def test_empty_columns_raises_type_error(self): + """_create_columns raises TypeError for empty columns dict.""" + with self.assertRaises(TypeError): + self.od._create_columns("new_Test", {}) + + def test_non_dict_columns_raises_type_error(self): + """_create_columns raises TypeError for non-dict columns.""" + with self.assertRaises(TypeError): + self.od._create_columns("new_Test", None) + + def test_table_not_found_raises_metadata_error(self): + """_create_columns raises MetadataError when table does not exist.""" + self.od._get_entity_by_table_schema_name = MagicMock(return_value=None) + with self.assertRaises(MetadataError): + self.od._create_columns("new_NonExistent", {"new_Col": "string"}) + + def test_unsupported_column_type_raises_validation_error(self): + """Raises ValidationError for unsupported column type.""" + from PowerPlatform.Dataverse.core.errors import ValidationError + + with self.assertRaises(ValidationError): + self.od._create_columns("new_Test", {"new_Col": "unsupported"}) + + def test_picklist_column_flushes_cache(self): + """_create_columns calls _flush_cache when a picklist column is created.""" + self.od._flush_cache = MagicMock(return_value=0) + + class Status(Enum): + Active = 1 + + result = self.od._create_columns("new_Test", {"new_Status": Status}) + self.assertIn("new_Status", result) + self.od._flush_cache.assert_called_once_with("picklist") + + def test_posts_to_correct_endpoint(self): + """_create_columns POSTs each column to EntityDefinitions({metadata_id})/Attributes.""" + self.od._create_columns("new_Test", {"new_Name": "string"}) + call_args = self.od._request.call_args + self.assertEqual(call_args.args[0], "post") + self.assertIn("EntityDefinitions(meta-001)/Attributes", call_args.args[1]) + + +class TestDeleteColumns(unittest.TestCase): + """Unit tests for _ODataClient._delete_columns.""" + + def setUp(self): + self.od = _make_odata_client() + self.od._get_entity_by_table_schema_name = MagicMock( + return_value={"MetadataId": "meta-001", "SchemaName": "new_Test"} + ) + self.od._get_attribute_metadata = MagicMock( + return_value={"MetadataId": "attr-001", "LogicalName": "new_name", "@odata.type": "StringAttributeMetadata"} + ) + self.od._request.return_value = _mock_response(status_code=204) + + def test_deletes_single_column(self): + """_delete_columns accepts a string column name and issues DELETE.""" + result = self.od._delete_columns("new_Test", "new_Name") + self.assertIn("new_Name", result) + delete_calls = [c for c in self.od._request.call_args_list if c.args[0] == "delete"] + self.assertEqual(len(delete_calls), 1) + self.assertIn("attr-001", delete_calls[0].args[1]) + + def test_deletes_list_of_columns(self): + """_delete_columns accepts a list of column names and issues DELETE for each.""" + result = self.od._delete_columns("new_Test", ["new_Name1", "new_Name2"]) + self.assertEqual(len(result), 2) + delete_calls = [c for c in self.od._request.call_args_list if c.args[0] == "delete"] + self.assertEqual(len(delete_calls), 2) + + def test_non_string_non_list_raises_type_error(self): + """_delete_columns raises TypeError for invalid columns type.""" + with self.assertRaises(TypeError): + self.od._delete_columns("new_Test", 42) + + def test_empty_column_name_raises_value_error(self): + """_delete_columns raises ValueError for empty column name.""" + with self.assertRaises(ValueError): + self.od._delete_columns("new_Test", "") + + def test_table_not_found_raises_metadata_error(self): + """_delete_columns raises MetadataError when table not found.""" + self.od._get_entity_by_table_schema_name = MagicMock(return_value=None) + with self.assertRaises(MetadataError): + self.od._delete_columns("new_NonExistent", "new_Col") + + def test_column_not_found_raises_metadata_error(self): + """_delete_columns raises MetadataError when column not found.""" + self.od._get_attribute_metadata = MagicMock(return_value=None) + with self.assertRaises(MetadataError) as ctx: + self.od._delete_columns("new_Test", "new_Missing") + self.assertIn("not found", str(ctx.exception)) + + def test_missing_metadata_id_raises_runtime_error(self): + """_delete_columns raises RuntimeError when column MetadataId is missing.""" + self.od._get_attribute_metadata = MagicMock(return_value={"LogicalName": "new_name"}) + with self.assertRaises(RuntimeError) as ctx: + self.od._delete_columns("new_Test", "new_Name") + self.assertIn("MetadataId", str(ctx.exception)) + + def test_picklist_column_deletion_flushes_cache(self): + """_delete_columns flushes picklist cache when a picklist column is deleted.""" + self.od._get_attribute_metadata = MagicMock( + return_value={ + "MetadataId": "attr-001", + "LogicalName": "new_status", + "@odata.type": "PicklistAttributeMetadata", + } + ) + self.od._flush_cache = MagicMock(return_value=0) + self.od._delete_columns("new_Test", "new_Status") + self.od._flush_cache.assert_called_once_with("picklist") + + +class TestFlushCache(unittest.TestCase): + """Unit tests for _ODataClient._flush_cache.""" + + def setUp(self): + self.od = _make_odata_client() + + def test_flush_picklist_clears_cache(self): + """_flush_cache('picklist') clears _picklist_label_cache.""" + self.od._picklist_label_cache = {("account", "statuscode"): {"map": {}, "ts": 0.0}} + removed = self.od._flush_cache("picklist") + self.assertEqual(removed, 1) + self.assertEqual(len(self.od._picklist_label_cache), 0) + + def test_flush_empty_cache_returns_zero(self): + """_flush_cache returns 0 when cache is already empty.""" + self.assertEqual(self.od._flush_cache("picklist"), 0) + + def test_unsupported_cache_kind_raises_validation_error(self): + """_flush_cache raises ValidationError for unsupported kind.""" + with self.assertRaises(ValidationError): + self.od._flush_cache("entityset") + + def test_none_kind_raises_validation_error(self): + """_flush_cache raises ValidationError for None kind.""" + with self.assertRaises(ValidationError): + self.od._flush_cache(None) + + class TestPicklistLabelResolution(unittest.TestCase): """Tests for picklist label-to-integer resolution. @@ -994,11 +2424,11 @@ def test_convert_different_tables_separate_fetches(self): resp2 = self._bulk_response(("new_status", [(100, "Open")])) self.od._request.side_effect = [resp1, resp2] - r1 = self.od._convert_labels_to_ints("account", {"industrycode": "Tech"}) - r2 = self.od._convert_labels_to_ints("new_ticket", {"new_status": "Open"}) + result1 = self.od._convert_labels_to_ints("account", {"industrycode": "Tech"}) + result2 = self.od._convert_labels_to_ints("new_ticket", {"new_status": "Open"}) - self.assertEqual(r1["industrycode"], 6) - self.assertEqual(r2["new_status"], 100) + self.assertEqual(result1["industrycode"], 6) + self.assertEqual(result2["new_status"], 100) self.assertEqual(self.od._request.call_count, 2) def test_convert_only_odata_and_non_strings_skips_fetch(self): @@ -1312,8 +2742,6 @@ def setUp(self): self.od = _make_odata_client() def _targets(self, alt_keys, records): - import json - req = self.od._build_upsert_multiple("accounts", "account", alt_keys, records) return json.loads(req.body)["Targets"] diff --git a/tests/unit/data/test_relationships.py b/tests/unit/data/test_relationships.py index c67b3f1e..6636b184 100644 --- a/tests/unit/data/test_relationships.py +++ b/tests/unit/data/test_relationships.py @@ -196,6 +196,19 @@ def test_create_m2m_relationship_returns_result(self): self.assertEqual(result["entity1_logical_name"], "account") self.assertEqual(result["entity2_logical_name"], "contact") + def test_create_m2m_relationship_with_solution(self): + """Solution name is added as MSCRM.SolutionUniqueName header.""" + mock_response = Mock() + mock_response.headers = { + "OData-EntityId": "https://example.crm.dynamics.com/api/data/v9.2/RelationshipDefinitions(abcd1234-abcd-1234-abcd-1234abcd5678)" + } + self.client._mock_request.return_value = mock_response + + self.client._create_many_to_many_relationship(self.relationship, solution="MySolution") + + headers = self.client._mock_request.call_args.kwargs["headers"] + self.assertEqual(headers["MSCRM.SolutionUniqueName"], "MySolution") + class TestDeleteRelationship(unittest.TestCase): """Tests for _delete_relationship method.""" diff --git a/tests/unit/data/test_upload.py b/tests/unit/data/test_upload.py new file mode 100644 index 00000000..2cf3b751 --- /dev/null +++ b/tests/unit/data/test_upload.py @@ -0,0 +1,430 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import tempfile +import unittest +from unittest.mock import MagicMock, patch + +from PowerPlatform.Dataverse.data._odata import _ODataClient + + +def _make_odata_client() -> _ODataClient: + """Return an _ODataClient with HTTP calls mocked out.""" + mock_auth = MagicMock() + mock_auth._acquire_token.return_value = MagicMock(access_token="token") + client = _ODataClient(mock_auth, "https://example.crm.dynamics.com") + client._request = MagicMock() + return client + + +def _make_temp_file(content: bytes = b"test content", suffix: str = ".bin") -> str: + """Create a temporary file and return its path. Caller must delete.""" + with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as f: + f.write(content) + return f.name + + +class TestUploadFile(unittest.TestCase): + """Tests for _upload_file() mode selection, column auto-creation, and argument forwarding.""" + + def setUp(self): + self.od = _make_odata_client() + self.od._entity_set_from_schema_name = MagicMock(return_value="accounts") + self.od._get_entity_by_table_schema_name = MagicMock( + return_value={"MetadataId": "meta-1", "LogicalName": "account"} + ) + self.od._get_attribute_metadata = MagicMock(return_value={"LogicalName": "new_document"}) + + def test_auto_mode_small_file(self): + """Auto mode routes files <128MB to _upload_file_small.""" + path = _make_temp_file() + self.addCleanup(os.unlink, path) + self.od._upload_file_small = MagicMock() + self.od._upload_file("account", "guid-1", "new_Document", path, mode="auto") + self.od._upload_file_small.assert_called_once() + + def test_auto_mode_large_file_routes_to_chunk(self): + """Auto mode routes files >=128MB to _upload_file_chunk.""" + path = _make_temp_file() + self.addCleanup(os.unlink, path) + self.od._upload_file_chunk = MagicMock() + with patch("os.path.getsize", return_value=128 * 1024 * 1024): + self.od._upload_file("account", "guid-1", "new_Document", path, mode="auto") + self.od._upload_file_chunk.assert_called_once() + + def test_default_mode_is_auto(self): + """mode=None is treated as 'auto'.""" + path = _make_temp_file() + self.addCleanup(os.unlink, path) + self.od._upload_file_small = MagicMock() + self.od._upload_file("account", "guid-1", "new_Document", path) + self.od._upload_file_small.assert_called_once() + + def test_auto_mode_file_not_found(self): + """Auto mode raises FileNotFoundError for missing file.""" + with self.assertRaises(FileNotFoundError): + self.od._upload_file("account", "guid-1", "new_Document", "/nonexistent/file.pdf") + + def test_explicit_small_mode(self): + """Explicit 'small' mode calls _upload_file_small.""" + path = _make_temp_file() + self.addCleanup(os.unlink, path) + self.od._upload_file_small = MagicMock() + self.od._upload_file("account", "guid-1", "new_Document", path, mode="small") + self.od._upload_file_small.assert_called_once() + + def test_explicit_chunk_mode(self): + """Explicit 'chunk' mode calls _upload_file_chunk.""" + path = _make_temp_file() + self.addCleanup(os.unlink, path) + self.od._upload_file_chunk = MagicMock() + self.od._upload_file("account", "guid-1", "new_Document", path, mode="chunk") + self.od._upload_file_chunk.assert_called_once() + + def test_invalid_mode_raises(self): + """Invalid mode raises ValueError.""" + path = _make_temp_file() + self.addCleanup(os.unlink, path) + with self.assertRaises(ValueError) as ctx: + self.od._upload_file("account", "guid-1", "new_Document", path, mode="invalid") + self.assertIn("invalid", str(ctx.exception).lower()) + + def test_column_auto_creation_when_missing(self): + """Creates file column when attribute metadata not found.""" + self.od._get_attribute_metadata = MagicMock(return_value=None) + self.od._create_columns = MagicMock() + self.od._wait_for_attribute_visibility = MagicMock() + self.od._upload_file_small = MagicMock() + path = _make_temp_file() + self.addCleanup(os.unlink, path) + self.od._upload_file("account", "guid-1", "new_Document", path, mode="small") + self.od._create_columns.assert_called_once_with("account", {"new_Document": "file"}) + self.od._wait_for_attribute_visibility.assert_called_once_with("accounts", "new_Document") + + def test_column_exists_skips_creation(self): + """Does not create column when attribute already exists.""" + self.od._create_columns = MagicMock() + self.od._upload_file_small = MagicMock() + path = _make_temp_file() + self.addCleanup(os.unlink, path) + self.od._upload_file("account", "guid-1", "new_Document", path, mode="small") + self.od._create_columns.assert_not_called() + + def test_no_entity_metadata_skips_column_check(self): + """Skips column check entirely when entity metadata is None.""" + self.od._get_entity_by_table_schema_name = MagicMock(return_value=None) + self.od._get_attribute_metadata = MagicMock() + self.od._upload_file_small = MagicMock() + path = _make_temp_file() + self.addCleanup(os.unlink, path) + self.od._upload_file("account", "guid-1", "new_Document", path, mode="small") + self.od._get_attribute_metadata.assert_not_called() + + def test_entity_metadata_without_metadata_id_skips_column_check(self): + """Skips attribute check when entity metadata has no MetadataId.""" + self.od._get_entity_by_table_schema_name = MagicMock(return_value={"LogicalName": "account"}) + self.od._get_attribute_metadata = MagicMock() + self.od._upload_file_small = MagicMock() + path = _make_temp_file() + self.addCleanup(os.unlink, path) + self.od._upload_file("account", "guid-1", "new_Document", path, mode="small") + self.od._get_attribute_metadata.assert_not_called() + + def test_lowercases_attribute_name(self): + """File name attribute is lowercased for URL usage.""" + self.od._upload_file_small = MagicMock() + path = _make_temp_file() + self.addCleanup(os.unlink, path) + self.od._upload_file("account", "guid-1", "new_Document", path, mode="small") + # Third positional arg to _upload_file_small is the logical_name (lowercased) + self.assertEqual(self.od._upload_file_small.call_args.args[2], "new_document") + + def test_passes_mime_type_to_small(self): + """mime_type is forwarded as content_type to _upload_file_small.""" + self.od._upload_file_small = MagicMock() + path = _make_temp_file() + self.addCleanup(os.unlink, path) + self.od._upload_file("account", "guid-1", "new_Document", path, mode="small", mime_type="text/csv") + self.assertEqual(self.od._upload_file_small.call_args.kwargs["content_type"], "text/csv") + + def test_passes_if_none_match_to_small(self): + """if_none_match is forwarded to _upload_file_small.""" + self.od._upload_file_small = MagicMock() + path = _make_temp_file() + self.addCleanup(os.unlink, path) + self.od._upload_file("account", "guid-1", "new_Document", path, mode="small", if_none_match=False) + self.assertFalse(self.od._upload_file_small.call_args.kwargs["if_none_match"]) + + def test_passes_if_none_match_to_chunk(self): + """if_none_match is forwarded to _upload_file_chunk.""" + self.od._upload_file_chunk = MagicMock() + path = _make_temp_file() + self.addCleanup(os.unlink, path) + self.od._upload_file("account", "guid-1", "new_Document", path, mode="chunk", if_none_match=False) + self.assertFalse(self.od._upload_file_chunk.call_args.kwargs["if_none_match"]) + + +class TestUploadFileSmall(unittest.TestCase): + """Tests for _upload_file_small() single PATCH upload.""" + + def setUp(self): + self.od = _make_odata_client() + + def test_successful_upload(self): + """Sends PATCH with correct URL, headers and file data.""" + path = _make_temp_file(b"PDF file content here", suffix=".pdf") + self.addCleanup(os.unlink, path) + self.od._upload_file_small("accounts", "guid-1", "new_document", path) + self.od._request.assert_called_once() + call = self.od._request.call_args + self.assertEqual(call.args[0], "patch") + self.assertIn("new_document", call.args[1]) + self.assertEqual(call.kwargs["data"], b"PDF file content here") + + def test_url_contains_entity_set_and_record_id(self): + """URL is constructed from entity_set, record_id, and attribute.""" + path = _make_temp_file() + self.addCleanup(os.unlink, path) + self.od._upload_file_small("accounts", "guid-1", "new_document", path) + url = self.od._request.call_args.args[1] + self.assertIn("accounts", url) + self.assertIn("guid-1", url) + self.assertIn("new_document", url) + + def test_if_none_match_header(self): + """if_none_match=True sends If-None-Match: null.""" + path = _make_temp_file() + self.addCleanup(os.unlink, path) + self.od._upload_file_small("accounts", "guid-1", "col", path, if_none_match=True) + headers = self.od._request.call_args.kwargs["headers"] + self.assertEqual(headers["If-None-Match"], "null") + self.assertNotIn("If-Match", headers) + + def test_if_match_overwrite_header(self): + """if_none_match=False sends If-Match: *.""" + path = _make_temp_file() + self.addCleanup(os.unlink, path) + self.od._upload_file_small("accounts", "guid-1", "col", path, if_none_match=False) + headers = self.od._request.call_args.kwargs["headers"] + self.assertEqual(headers["If-Match"], "*") + self.assertNotIn("If-None-Match", headers) + + def test_custom_mime_type(self): + """Custom content_type is used in Content-Type header.""" + path = _make_temp_file(b"{}", suffix=".json") + self.addCleanup(os.unlink, path) + self.od._upload_file_small("accounts", "guid-1", "col", path, content_type="application/json") + headers = self.od._request.call_args.kwargs["headers"] + self.assertEqual(headers["Content-Type"], "application/json") + + def test_default_mime_type(self): + """Default Content-Type is application/octet-stream.""" + path = _make_temp_file() + self.addCleanup(os.unlink, path) + self.od._upload_file_small("accounts", "guid-1", "col", path) + headers = self.od._request.call_args.kwargs["headers"] + self.assertEqual(headers["Content-Type"], "application/octet-stream") + + def test_file_not_found_raises(self): + """Raises FileNotFoundError for missing file.""" + with self.assertRaises(FileNotFoundError): + self.od._upload_file_small("accounts", "guid-1", "col", "/no/such/file.txt") + + def test_empty_record_id_raises(self): + """Raises ValueError for empty record_id.""" + with self.assertRaises(ValueError): + self.od._upload_file_small("accounts", "", "col", "/any/path") + + def test_file_name_in_header(self): + """x-ms-file-name header contains the basename of the file.""" + path = _make_temp_file(b"a,b,c", suffix=".csv") + self.addCleanup(os.unlink, path) + self.od._upload_file_small("accounts", "guid-1", "col", path) + headers = self.od._request.call_args.kwargs["headers"] + self.assertEqual(headers["x-ms-file-name"], os.path.basename(path)) + + def test_file_exceeds_small_upload_limit_raises(self): + """Raises ValueError when file exceeds 128MB single-upload limit.""" + path = _make_temp_file() + self.addCleanup(os.unlink, path) + with patch("os.path.getsize", return_value=128 * 1024 * 1024 + 1): + with self.assertRaises(ValueError) as ctx: + self.od._upload_file_small("accounts", "guid-1", "col", path) + self.assertIn("chunk", str(ctx.exception).lower()) + + +class TestUploadFileChunk(unittest.TestCase): + """Tests for _upload_file_chunk() streaming chunked upload.""" + + def setUp(self): + self.od = _make_odata_client() + + @staticmethod + def _mock_init_response(location="https://example.com/session?token=abc", chunk_size=None): + """Create a mock init PATCH response with Location and optional chunk-size headers.""" + resp = MagicMock() + headers = {"Location": location} + if chunk_size is not None: + headers["x-ms-chunk-size"] = str(chunk_size) + resp.headers = headers + return resp + + def test_init_patch_sends_chunked_header(self): + """Initial PATCH sends x-ms-transfer-mode: chunked.""" + self.od._request.return_value = self._mock_init_response() + path = _make_temp_file(b"x" * 100) + self.addCleanup(os.unlink, path) + self.od._upload_file_chunk("accounts", "guid-1", "col", path) + init_call = self.od._request.call_args_list[0] + self.assertEqual(init_call.kwargs["headers"]["x-ms-transfer-mode"], "chunked") + + def test_init_url_contains_file_name(self): + """Init PATCH URL includes x-ms-file-name query parameter.""" + self.od._request.return_value = self._mock_init_response() + path = _make_temp_file(b"data", suffix=".pdf") + self.addCleanup(os.unlink, path) + self.od._upload_file_chunk("accounts", "guid-1", "col", path) + init_url = self.od._request.call_args_list[0].args[1] + self.assertIn("x-ms-file-name=", init_url) + + def test_missing_location_header_raises(self): + """Raises RuntimeError when init response lacks Location header.""" + resp = MagicMock() + resp.headers = {} + self.od._request.return_value = resp + path = _make_temp_file() + self.addCleanup(os.unlink, path) + with self.assertRaises(RuntimeError) as ctx: + self.od._upload_file_chunk("accounts", "guid-1", "col", path) + self.assertIn("Location", str(ctx.exception)) + + def test_lowercase_location_header_accepted(self): + """Accepts lowercase 'location' header as fallback.""" + resp = MagicMock() + resp.headers = {"location": "https://example.com/session?token=abc"} + self.od._request.return_value = resp + path = _make_temp_file(b"data") + self.addCleanup(os.unlink, path) + self.od._upload_file_chunk("accounts", "guid-1", "col", path) + # 1 init + 1 chunk = 2 total calls + self.assertEqual(self.od._request.call_count, 2) + + def test_uses_chunk_size_from_response(self): + """Uses x-ms-chunk-size from init response to determine chunk size.""" + self.od._request.return_value = self._mock_init_response(chunk_size=50) + path = _make_temp_file(b"x" * 120) # 120 bytes / 50-byte chunks = 3 chunks + self.addCleanup(os.unlink, path) + self.od._upload_file_chunk("accounts", "guid-1", "col", path) + # 1 init + 3 chunk calls = 4 total + self.assertEqual(self.od._request.call_count, 4) + + def test_default_chunk_size_when_header_missing(self): + """Falls back to 4MB chunk size when x-ms-chunk-size header missing.""" + self.od._request.return_value = self._mock_init_response() # no chunk_size + path = _make_temp_file(b"x" * 100) # 100 bytes < 4MB = single chunk + self.addCleanup(os.unlink, path) + self.od._upload_file_chunk("accounts", "guid-1", "col", path) + # 1 init + 1 chunk = 2 total + self.assertEqual(self.od._request.call_count, 2) + + def test_malformed_chunk_size_header_falls_back_to_default(self): + """Non-integer x-ms-chunk-size falls back to 4MB default.""" + resp = MagicMock() + resp.headers = {"Location": "https://example.com/session", "x-ms-chunk-size": "not-a-number"} + self.od._request.return_value = resp + path = _make_temp_file(b"x" * 100) # 100 bytes < 4MB = single chunk + self.addCleanup(os.unlink, path) + self.od._upload_file_chunk("accounts", "guid-1", "col", path) + # Falls back to 4MB default → 100 bytes = 1 chunk → 2 total calls + self.assertEqual(self.od._request.call_count, 2) + + def test_negative_chunk_size_raises(self): + """Negative x-ms-chunk-size raises ValueError (zero falls back to 4MB default).""" + resp = MagicMock() + resp.headers = {"Location": "https://example.com/session", "x-ms-chunk-size": "-1"} + self.od._request.return_value = resp + path = _make_temp_file(b"data") + self.addCleanup(os.unlink, path) + with self.assertRaises(ValueError): + self.od._upload_file_chunk("accounts", "guid-1", "col", path) + + def test_empty_file_completes_without_chunk_requests(self): + """Zero-byte file sends only the init PATCH, no chunk PATCHes.""" + self.od._request.return_value = self._mock_init_response() + path = _make_temp_file(b"") + self.addCleanup(os.unlink, path) + self.od._upload_file_chunk("accounts", "guid-1", "col", path) + # Only the init PATCH is sent + self.assertEqual(self.od._request.call_count, 1) + + def test_content_range_headers(self): + """Each chunk has correct Content-Range header.""" + self.od._request.return_value = self._mock_init_response(chunk_size=10) + path = _make_temp_file(b"A" * 10 + b"B" * 10 + b"C" * 5) # 25 bytes -> 3 chunks + self.addCleanup(os.unlink, path) + self.od._upload_file_chunk("accounts", "guid-1", "col", path) + chunk_calls = self.od._request.call_args_list[1:] # skip init + self.assertEqual(len(chunk_calls), 3) + self.assertEqual(chunk_calls[0].kwargs["headers"]["Content-Range"], "bytes 0-9/25") + self.assertEqual(chunk_calls[1].kwargs["headers"]["Content-Range"], "bytes 10-19/25") + self.assertEqual(chunk_calls[2].kwargs["headers"]["Content-Range"], "bytes 20-24/25") + + def test_chunk_content_length_header(self): + """Each chunk includes correct Content-Length header.""" + self.od._request.return_value = self._mock_init_response(chunk_size=10) + path = _make_temp_file(b"A" * 10 + b"B" * 5) # 15 bytes -> 2 chunks (10 + 5) + self.addCleanup(os.unlink, path) + self.od._upload_file_chunk("accounts", "guid-1", "col", path) + chunk_calls = self.od._request.call_args_list[1:] + self.assertEqual(chunk_calls[0].kwargs["headers"]["Content-Length"], "10") + self.assertEqual(chunk_calls[1].kwargs["headers"]["Content-Length"], "5") + + def test_chunk_sends_to_location_url(self): + """Chunk PATCHes go to the Location URL, not the original URL.""" + session_url = "https://example.com/upload?session=xyz" + self.od._request.return_value = self._mock_init_response(location=session_url) + path = _make_temp_file() + self.addCleanup(os.unlink, path) + self.od._upload_file_chunk("accounts", "guid-1", "col", path) + chunk_call = self.od._request.call_args_list[1] + self.assertEqual(chunk_call.args[1], session_url) + + def test_if_none_match_on_init(self): + """if_none_match=True sends If-None-Match on init PATCH.""" + self.od._request.return_value = self._mock_init_response() + path = _make_temp_file() + self.addCleanup(os.unlink, path) + self.od._upload_file_chunk("accounts", "guid-1", "col", path, if_none_match=True) + init_headers = self.od._request.call_args_list[0].kwargs["headers"] + self.assertEqual(init_headers["If-None-Match"], "null") + self.assertNotIn("If-Match", init_headers) + + def test_if_match_overwrite_on_init(self): + """if_none_match=False sends If-Match on init PATCH.""" + self.od._request.return_value = self._mock_init_response() + path = _make_temp_file() + self.addCleanup(os.unlink, path) + self.od._upload_file_chunk("accounts", "guid-1", "col", path, if_none_match=False) + init_headers = self.od._request.call_args_list[0].kwargs["headers"] + self.assertEqual(init_headers["If-Match"], "*") + self.assertNotIn("If-None-Match", init_headers) + + def test_empty_record_id_raises(self): + """Raises ValueError for empty record_id.""" + with self.assertRaises(ValueError): + self.od._upload_file_chunk("accounts", "", "col", "/any/path") + + def test_file_not_found_raises(self): + """Raises FileNotFoundError for missing file.""" + with self.assertRaises(FileNotFoundError): + self.od._upload_file_chunk("accounts", "guid-1", "col", "/no/such/file.bin") + + def test_chunk_requests_accept_206_and_204(self): + """Chunk requests use expected=(206, 204).""" + self.od._request.return_value = self._mock_init_response(chunk_size=50) + path = _make_temp_file(b"x" * 100) + self.addCleanup(os.unlink, path) + self.od._upload_file_chunk("accounts", "guid-1", "col", path) + for chunk_call in self.od._request.call_args_list[1:]: + self.assertEqual(chunk_call.kwargs["expected"], (206, 204)) diff --git a/tests/unit/models/test_query_builder.py b/tests/unit/models/test_query_builder.py index 47bb28fc..9f094912 100644 --- a/tests/unit/models/test_query_builder.py +++ b/tests/unit/models/test_query_builder.py @@ -344,6 +344,12 @@ def test_filter_raw_returns_self(self): qb = QueryBuilder("account") self.assertIs(qb.filter_raw("a eq 1"), qb) + def test_build_with_plain_string_filter_part(self): + """build() handles plain string entries in _filter_parts (internal path).""" + qb = QueryBuilder("account") + qb._filter_parts.append("name eq 'Contoso'") + self.assertEqual(qb.build()["filter"], "name eq 'Contoso'") + class TestWhere(unittest.TestCase): """Tests for the where() method with composable expressions.""" diff --git a/tests/unit/models/test_table_info.py b/tests/unit/models/test_table_info.py index b3da07fa..7842c266 100644 --- a/tests/unit/models/test_table_info.py +++ b/tests/unit/models/test_table_info.py @@ -65,9 +65,15 @@ def test_len(self): def test_keys_values_items(self): self.assertEqual(list(self.info.keys()), list(self.info._LEGACY_KEY_MAP.keys())) + self.assertEqual(self.info.values()[0], "new_Product") items = dict(self.info.items()) self.assertEqual(items["table_schema_name"], "new_Product") + def test_contains_non_string_key_returns_false(self): + """__contains__ returns False for non-string keys.""" + self.assertNotIn(42, self.info) + self.assertNotIn(None, self.info) + def test_to_dict(self): d = self.info.to_dict() self.assertIsInstance(d, dict) diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index cfad101e..440643a3 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -134,6 +134,11 @@ def test_get_multiple(self): self.assertEqual(results[0][0]["name"], "A") self.assertEqual(results[0][1]["name"], "B") + def test_empty_base_url_raises(self): + """DataverseClient raises ValueError when base_url is empty.""" + with self.assertRaises(ValueError): + DataverseClient("", self.mock_credential) + class TestCreateLookupField(unittest.TestCase): """Tests for client.tables.create_lookup_field convenience method.""" diff --git a/tests/unit/test_context_manager.py b/tests/unit/test_context_manager.py index 2f1aab9c..054f0395 100644 --- a/tests/unit/test_context_manager.py +++ b/tests/unit/test_context_manager.py @@ -268,6 +268,17 @@ def test_close_available_without_context_manager(self): with self.assertRaises(RuntimeError): client.records.create("account", {"name": "test"}) + def test_flush_cache_delegates_to_odata(self): + """flush_cache() calls _flush_cache on the OData client and returns its result.""" + client = DataverseClient(self.base_url, self.mock_credential) + client._odata = MagicMock() + client._odata._flush_cache.return_value = 3 + + result = client.flush_cache("picklist") + + client._odata._flush_cache.assert_called_once_with("picklist") + self.assertEqual(result, 3) + class TestExceptionHandling(unittest.TestCase): """Tests for exception handling during context manager usage.""" diff --git a/tests/unit/test_records_operations.py b/tests/unit/test_records_operations.py index 97da5a40..09df6869 100644 --- a/tests/unit/test_records_operations.py +++ b/tests/unit/test_records_operations.py @@ -64,6 +64,27 @@ def test_create_single_returns_string(self): self.assertNotIsInstance(result, list) self.assertEqual(result, "single-guid") + def test_create_single_non_string_return_raises(self): + """create() raises TypeError if _create returns a non-string.""" + self.client._odata._entity_set_from_schema_name.return_value = "accounts" + self.client._odata._create.return_value = 12345 + + with self.assertRaises(TypeError): + self.client.records.create("account", {"name": "Contoso"}) + + def test_create_bulk_non_list_return_raises(self): + """create() raises TypeError if _create_multiple returns a non-list.""" + self.client._odata._entity_set_from_schema_name.return_value = "accounts" + self.client._odata._create_multiple.return_value = "not-a-list" + + with self.assertRaises(TypeError): + self.client.records.create("account", [{"name": "Contoso"}]) + + def test_create_invalid_data_type_raises(self): + """create() raises TypeError if data is neither dict nor list.""" + with self.assertRaises(TypeError): + self.client.records.create("account", "invalid") + # ------------------------------------------------------------------ update def test_update_single(self): @@ -96,6 +117,16 @@ def test_update_paired(self): self.client._odata._update_by_ids.assert_called_once_with("account", ids, changes) + def test_update_single_non_dict_changes_raises(self): + """update() raises TypeError if ids is str but changes is not a dict.""" + with self.assertRaises(TypeError): + self.client.records.update("account", "guid-1", ["not", "a", "dict"]) + + def test_update_invalid_ids_type_raises(self): + """update() raises TypeError if ids is neither str nor list.""" + with self.assertRaises(TypeError): + self.client.records.update("account", 12345, {"name": "X"}) + # ------------------------------------------------------------------ delete def test_delete_single(self): @@ -137,6 +168,16 @@ def test_delete_empty_list(self): self.client._odata._delete_multiple.assert_not_called() self.assertIsNone(result) + def test_delete_invalid_ids_type_raises(self): + """delete() raises TypeError if ids is neither str nor list.""" + with self.assertRaises(TypeError): + self.client.records.delete("account", 12345) + + def test_delete_list_with_non_string_guids_raises(self): + """delete() raises TypeError if the ids list contains non-string entries.""" + with self.assertRaises(TypeError): + self.client.records.delete("account", ["valid-guid", 42]) + # --------------------------------------------------------------------- get def test_get_single(self): @@ -158,6 +199,11 @@ def test_get_single_with_query_params_raises(self): with self.assertRaises(ValueError): self.client.records.get("account", "guid-1", filter="statecode eq 0") + def test_get_non_string_record_id_raises(self): + """get() raises TypeError if record_id is not a string.""" + with self.assertRaises(TypeError): + self.client.records.get("account", 12345) + def test_get_paginated(self): """get() without record_id should yield pages of Record objects.""" page_1 = [{"accountid": "1", "name": "A"}]