diff --git a/bigquery/google/cloud/bigquery/client.py b/bigquery/google/cloud/bigquery/client.py index 0723e9133af9..2c967fa09d7e 100644 --- a/bigquery/google/cloud/bigquery/client.py +++ b/bigquery/google/cloud/bigquery/client.py @@ -338,34 +338,53 @@ def _call_api(self, retry, **kwargs): def get_dataset(self, dataset_ref, retry=DEFAULT_RETRY): """Fetch the dataset referenced by ``dataset_ref`` - :type dataset_ref: - :class:`google.cloud.bigquery.dataset.DatasetReference` - :param dataset_ref: the dataset to use. - - :type retry: :class:`google.api_core.retry.Retry` - :param retry: (Optional) How to retry the RPC. + Args: + dataset_ref (Union[ \ + :class:`~google.cloud.bigquery.dataset.DatasetReference`, \ + str, \ + ]): + A reference to the dataset to fetch from the BigQuery API. + If a string is passed in, this method attempts to create a + dataset reference from a string using + :func:`~google.cloud.bigquery.dataset.DatasetReference.from_string`. + retry (:class:`google.api_core.retry.Retry`): + (Optional) How to retry the RPC. - :rtype: :class:`google.cloud.bigquery.dataset.Dataset` - :returns: a ``Dataset`` instance + Returns: + google.cloud.bigquery.dataset.Dataset: + A ``Dataset`` instance. """ - api_response = self._call_api(retry, - method='GET', - path=dataset_ref.path) + if isinstance(dataset_ref, str): + dataset_ref = DatasetReference.from_string( + dataset_ref, default_project=self.project) + + api_response = self._call_api( + retry, method='GET', path=dataset_ref.path) return Dataset.from_api_repr(api_response) def get_table(self, table_ref, retry=DEFAULT_RETRY): - """Fetch the table referenced by ``table_ref`` + """Fetch the table referenced by ``table_ref``. - :type table_ref: - :class:`google.cloud.bigquery.table.TableReference` - :param table_ref: the table to use. - - :type retry: :class:`google.api_core.retry.Retry` - :param retry: (Optional) How to retry the RPC. + Args: + table_ref (Union[ \ + :class:`~google.cloud.bigquery.table.TableReference`, \ + str, \ + ]): + A reference to the table to fetch from the BigQuery API. + If a string is passed in, this method attempts to create a + table reference from a string using + :func:`google.cloud.bigquery.table.TableReference.from_string`. + retry (:class:`google.api_core.retry.Retry`): + (Optional) How to retry the RPC. - :rtype: :class:`google.cloud.bigquery.table.Table` - :returns: a ``Table`` instance + Returns: + google.cloud.bigquery.table.Table: + A ``Table`` instance. """ + if isinstance(table_ref, str): + table_ref = TableReference.from_string( + table_ref, default_project=self.project) + api_response = self._call_api(retry, method='GET', path=table_ref.path) return Table.from_api_repr(api_response) @@ -450,34 +469,43 @@ def list_tables(self, dataset, max_results=None, page_token=None, See https://cloud.google.com/bigquery/docs/reference/rest/v2/tables/list - :type dataset: One of: - :class:`~google.cloud.bigquery.dataset.Dataset` - :class:`~google.cloud.bigquery.dataset.DatasetReference` - :param dataset: the dataset whose tables to list, or a reference to it. - - :type max_results: int - :param max_results: (Optional) Maximum number of tables to return. - If not passed, defaults to a value set by the API. - - :type page_token: str - :param page_token: - (Optional) Token representing a cursor into the tables. If not - passed, the API will return the first page of tables. The - token marks the beginning of the iterator to be returned and - the value of the ``page_token`` can be accessed at - ``next_page_token`` of the - :class:`~google.api_core.page_iterator.HTTPIterator`. - - :type retry: :class:`google.api_core.retry.Retry` - :param retry: (Optional) How to retry the RPC. + Args: + dataset (Union[ \ + :class:`~google.cloud.bigquery.dataset.Dataset`, \ + :class:`~google.cloud.bigquery.dataset.DatasetReference`, \ + str, \ + ]): + A reference to the dataset whose tables to list from the + BigQuery API. If a string is passed in, this method attempts + to create a dataset reference from a string using + :func:`google.cloud.bigquery.dataset.DatasetReference.from_string`. + max_results (int): + (Optional) Maximum number of tables to return. If not passed, + defaults to a value set by the API. + page_token (str): + (Optional) Token representing a cursor into the tables. If + not passed, the API will return the first page of tables. The + token marks the beginning of the iterator to be returned and + the value of the ``page_token`` can be accessed at + ``next_page_token`` of the + :class:`~google.api_core.page_iterator.HTTPIterator`. + retry (:class:`google.api_core.retry.Retry`): + (Optional) How to retry the RPC. - :rtype: :class:`~google.api_core.page_iterator.Iterator` - :returns: - Iterator of :class:`~google.cloud.bigquery.table.TableListItem` - contained within the current dataset. + Returns: + google.api_core.page_iterator.Iterator: + Iterator of + :class:`~google.cloud.bigquery.table.TableListItem` contained + within the requested dataset. """ + if isinstance(dataset, str): + dataset = DatasetReference.from_string( + dataset, default_project=self.project) + if not isinstance(dataset, (Dataset, DatasetReference)): - raise TypeError('dataset must be a Dataset or a DatasetReference') + raise TypeError( + 'dataset must be a Dataset, DatasetReference, or string') + path = '%s/tables' % dataset.path result = page_iterator.HTTPIterator( client=self, @@ -497,19 +525,27 @@ def delete_dataset(self, dataset, delete_contents=False, See https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets/delete - :type dataset: One of: - :class:`~google.cloud.bigquery.dataset.Dataset` - :class:`~google.cloud.bigquery.dataset.DatasetReference` - :param dataset: the dataset to delete, or a reference to it. - - :type retry: :class:`google.api_core.retry.Retry` - :param retry: (Optional) How to retry the RPC. - - :type delete_contents: boolean - :param delete_contents: (Optional) If True, delete all the tables - in the dataset. If False and the dataset contains tables, the - request will fail. Default is False + Args + dataset (Union[ \ + :class:`~google.cloud.bigquery.dataset.Dataset`, \ + :class:`~google.cloud.bigquery.dataset.DatasetReference`, \ + str, \ + ]): + A reference to the dataset to delete. If a string is passed + in, this method attempts to create a dataset reference from a + string using + :func:`google.cloud.bigquery.dataset.DatasetReference.from_string`. + retry (:class:`google.api_core.retry.Retry`): + (Optional) How to retry the RPC. + delete_contents (boolean): + (Optional) If True, delete all the tables in the dataset. If + False and the dataset contains tables, the request will fail. + Default is False. """ + if isinstance(dataset, str): + dataset = DatasetReference.from_string( + dataset, default_project=self.project) + if not isinstance(dataset, (Dataset, DatasetReference)): raise TypeError('dataset must be a Dataset or a DatasetReference') @@ -528,14 +564,23 @@ def delete_table(self, table, retry=DEFAULT_RETRY): See https://cloud.google.com/bigquery/docs/reference/rest/v2/tables/delete - :type table: One of: - :class:`~google.cloud.bigquery.table.Table` - :class:`~google.cloud.bigquery.table.TableReference` - :param table: the table to delete, or a reference to it. - - :type retry: :class:`google.api_core.retry.Retry` - :param retry: (Optional) How to retry the RPC. + Args: + table (Union[ \ + :class:`~google.cloud.bigquery.table.Table`, \ + :class:`~google.cloud.bigquery.table.TableReference`, \ + str, \ + ]): + A reference to the table to delete. If a string is passed in, + this method attempts to create a table reference from a + string using + :func:`google.cloud.bigquery.table.TableReference.from_string`. + retry (:class:`google.api_core.retry.Retry`): + (Optional) How to retry the RPC. """ + if isinstance(table, str): + table = TableReference.from_string( + table, default_project=self.project) + if not isinstance(table, (Table, TableReference)): raise TypeError('table must be a Table or a TableReference') self._call_api(retry, method='DELETE', path=table.path) @@ -785,8 +830,14 @@ def load_table_from_uri( source_uris (Union[str, Sequence[str]]): URIs of data files to be loaded; in format ``gs:///``. - destination (google.cloud.bigquery.table.TableReference): - Table into which data is to be loaded. + destination (Union[ \ + :class:`~google.cloud.bigquery.table.TableReference`, \ + str, \ + ]): + Table into which data is to be loaded. If a string is passed + in, this method attempts to create a table reference from a + string using + :func:`google.cloud.bigquery.table.TableReference.from_string`. Keyword Arguments: job_id (str): (Optional) Name of the job. @@ -821,6 +872,10 @@ def load_table_from_uri( if isinstance(source_uris, six.string_types): source_uris = [source_uris] + if isinstance(destination, str): + destination = TableReference.from_string( + destination, default_project=self.project) + load_job = job.LoadJob( job_ref, source_uris, destination, self, job_config) load_job._begin(retry=retry) @@ -839,8 +894,14 @@ def load_table_from_file( Arguments: file_obj (file): A file handle opened in binary mode for reading. - destination (google.cloud.bigquery.table.TableReference): - Table into which data is to be loaded. + destination (Union[ \ + :class:`~google.cloud.bigquery.table.TableReference`, \ + str, \ + ]): + Table into which data is to be loaded. If a string is passed + in, this method attempts to create a table reference from a + string using + :func:`google.cloud.bigquery.table.TableReference.from_string`. Keyword Arguments: rewind (bool): @@ -882,6 +943,10 @@ def load_table_from_file( if location is None: location = self.location + if isinstance(destination, str): + destination = TableReference.from_string( + destination, default_project=self.project) + job_ref = job._JobReference(job_id, project=project, location=location) load_job = job.LoadJob(job_ref, None, destination, self, job_config) job_resource = load_job._build_resource() @@ -923,6 +988,10 @@ def load_table_from_dataframe(self, dataframe, destination, does not yet exist, the schema is inferred from the :class:`~pandas.DataFrame`. + If a string is passed in, this method attempts to create a + table reference from a string using + :func:`google.cloud.bigquery.table.TableReference.from_string`. + Keyword Arguments: num_retries (int, optional): Number of upload retries. job_id (str, optional): Name of the job. @@ -1085,10 +1154,17 @@ def copy_table( https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.copy Arguments: - sources (Union[google.cloud.bigquery.table.TableReference, \ - Sequence[google.cloud.bigquery.table.TableReference]]): + sources (Union[ \ + :class:`~google.cloud.bigquery.table.TableReference`, \ + str, \ + Sequence[ \ + :class:`~google.cloud.bigquery.table.TableReference`], \ + ]): Table or tables to be copied. - destination (google.cloud.bigquery.table.TableReference): + destination (Union[ + :class:`~google.cloud.bigquery.table.TableReference`, \ + str, \ + ]): Table into which data is to be copied. Keyword Arguments: @@ -1121,6 +1197,14 @@ def copy_table( job_ref = job._JobReference(job_id, project=project, location=location) + if isinstance(sources, str): + sources = TableReference.from_string( + sources, default_project=self.project) + + if isinstance(destination, str): + destination = TableReference.from_string( + destination, default_project=self.project) + if not isinstance(sources, collections.Sequence): sources = [sources] @@ -1141,7 +1225,10 @@ def extract_table( https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.extract Arguments: - source (google.cloud.bigquery.table.TableReference): + source (Union[ \ + :class:`google.cloud.bigquery.table.TableReference`, \ + src, \ + ]): Table to be extracted. destination_uris (Union[str, Sequence[str]]): URIs of Cloud Storage file(s) into which table data is to be @@ -1181,6 +1268,10 @@ def extract_table( job_ref = job._JobReference(job_id, project=project, location=location) + if isinstance(source, str): + source = TableReference.from_string( + source, default_project=self.project) + if isinstance(destination_uris, six.string_types): destination_uris = [destination_uris] @@ -1261,40 +1352,45 @@ def insert_rows(self, table, rows, selected_fields=None, **kwargs): See https://cloud.google.com/bigquery/docs/reference/rest/v2/tabledata/insertAll - :type table: One of: - :class:`~google.cloud.bigquery.table.Table` - :class:`~google.cloud.bigquery.table.TableReference` - :param table: the destination table for the row data, or a reference - to it. - - :type rows: One of: - list of tuples - list of dictionaries - :param rows: Row data to be inserted. If a list of tuples is given, - each tuple should contain data for each schema field on - the current table and in the same order as the schema - fields. If a list of dictionaries is given, the keys must - include all required fields in the schema. Keys which do - not correspond to a field in the schema are ignored. - - :type selected_fields: - list of :class:`~google.cloud.bigquery.schema.SchemaField` - :param selected_fields: - The fields to return. Required if ``table`` is a - :class:`~google.cloud.bigquery.table.TableReference`. - - :type kwargs: dict - :param kwargs: - Keyword arguments to - :meth:`~google.cloud.bigquery.client.Client.insert_rows_json` - - :rtype: list of mappings - :returns: One mapping per row with insert errors: the "index" key - identifies the row, and the "errors" key contains a list - of the mappings describing one or more problems with the - row. - :raises: ValueError if table's schema is not set + Args: + table (Union[ \ + :class:`~google.cloud.bigquery.table.Table`, \ + :class:`~google.cloud.bigquery.table.TableReference`, \ + str, \ + ]): + The destination table for the row data, or a reference to it. + rows (Union[ \ + Sequence[Tuple], \ + Sequence[dict], \ + ]): + Row data to be inserted. If a list of tuples is given, each + tuple should contain data for each schema field on the + current table and in the same order as the schema fields. If + a list of dictionaries is given, the keys must include all + required fields in the schema. Keys which do not correspond + to a field in the schema are ignored. + selected_fields (Sequence[ \ + :class:`~google.cloud.bigquery.schema.SchemaField`, \ + ]): + The fields to return. Required if ``table`` is a + :class:`~google.cloud.bigquery.table.TableReference`. + kwargs (dict): + Keyword arguments to + :meth:`~google.cloud.bigquery.client.Client.insert_rows_json`. + + Returns: + Sequence[Mappings]: + One mapping per row with insert errors: the "index" key + identifies the row, and the "errors" key contains a list of + the mappings describing one or more problems with the row. + + Raises: + ValueError: if table's schema is not set """ + if isinstance(table, str): + table = TableReference.from_string( + table, default_project=self.project) + if selected_fields is not None: schema = selected_fields elif isinstance(table, TableReference): @@ -1331,51 +1427,44 @@ def insert_rows_json(self, table, json_rows, row_ids=None, See https://cloud.google.com/bigquery/docs/reference/rest/v2/tabledata/insertAll - :type table: One of: - :class:`~google.cloud.bigquery.table.Table` - :class:`~google.cloud.bigquery.table.TableReference` - :param table: the destination table for the row data, or a reference - to it. - - :type json_rows: list of dictionaries - :param json_rows: Row data to be inserted. Keys must match the table - schema fields and values must be JSON-compatible - representations. - - :type row_ids: list of string - :param row_ids: (Optional) Unique ids, one per row being inserted. - If omitted, unique IDs are created. - - :type skip_invalid_rows: bool - :param skip_invalid_rows: (Optional) Insert all valid rows of a - request, even if invalid rows exist. - The default value is False, which causes - the entire request to fail if any invalid - rows exist. - - :type ignore_unknown_values: bool - :param ignore_unknown_values: (Optional) Accept rows that contain - values that do not match the schema. - The unknown values are ignored. Default - is False, which treats unknown values as - errors. - - :type template_suffix: str - :param template_suffix: + table (Union[ \ + :class:`~google.cloud.bigquery.table.Table` \ + :class:`~google.cloud.bigquery.table.TableReference`, \ + str, \ + ]): + The destination table for the row data, or a reference to it. + json_rows (Sequence[dict]): + Row data to be inserted. Keys must match the table schema fields + and values must be JSON-compatible representations. + row_ids (Sequence[str]): + (Optional) Unique ids, one per row being inserted. If omitted, + unique IDs are created. + skip_invalid_rows (bool): + (Optional) Insert all valid rows of a request, even if invalid + rows exist. The default value is False, which causes the entire + request to fail if any invalid rows exist. + ignore_unknown_values (bool): + (Optional) Accept rows that contain values that do not match the + schema. The unknown values are ignored. Default is False, which + treats unknown values as errors. + template_suffix (str): (Optional) treat ``name`` as a template table and provide a suffix. BigQuery will create the table `` + `` based on the schema of the template table. See https://cloud.google.com/bigquery/streaming-data-into-bigquery#template-tables + retry (:class:`google.api_core.retry.Retry`): + (Optional) How to retry the RPC. - :type retry: :class:`google.api_core.retry.Retry` - :param retry: (Optional) How to retry the RPC. - - :rtype: list of mappings - :returns: One mapping per row with insert errors: the "index" key - identifies the row, and the "errors" key contains a list - of the mappings describing one or more problems with the - row. + Returns: + Sequence[Mappings]: + One mapping per row with insert errors: the "index" key + identifies the row, and the "errors" key contains a list of + the mappings describing one or more problems with the row. """ + if isinstance(table, str): + table = TableReference.from_string( + table, default_project=self.project) + rows_info = [] data = {'rows': rows_info} @@ -1414,8 +1503,11 @@ def list_partitions(self, table, retry=DEFAULT_RETRY): """List the partitions in a table. Arguments: - table (Union[google.cloud.bigquery.table.Table, - google.cloud.bigquery.table.TableReference]): + table (Union[ \ + :class:`~google.cloud.bigquery.table.Table`, \ + :class:`~google.cloud.bigquery.table.TableReference`, \ + str, \ + ]): The table or reference from which to get partition info retry (google.api_core.retry.Retry): (Optional) How to retry the RPC. @@ -1424,6 +1516,10 @@ def list_partitions(self, table, retry=DEFAULT_RETRY): List[str]: A list of the partition ids present in the partitioned table """ + if isinstance(table, str): + table = TableReference.from_string( + table, default_project=self.project) + meta_table = self.get_table( TableReference( self.dataset(table.dataset_id, project=table.project), @@ -1450,49 +1546,48 @@ def list_rows(self, table, selected_fields=None, max_results=None, identical, the values returned may be incomplete. To ensure that the local copy of the schema is up-to-date, call ``client.get_table``. - :type table: One of: - :class:`~google.cloud.bigquery.table.Table` - :class:`~google.cloud.bigquery.table.TableReference` - :param table: the table to list, or a reference to it. - - :type selected_fields: - list of :class:`~google.cloud.bigquery.schema.SchemaField` - :param selected_fields: - The fields to return. Required if ``table`` is a - :class:`~google.cloud.bigquery.table.TableReference`. - - :type max_results: int - :param max_results: (Optional) maximum number of rows to return. - - :type page_token: str - :param page_token: (Optional) Token representing a cursor into the - table's rows. If not passed, the API will return - the first page of the rows. The token marks the - beginning of the iterator to be returned and the - value of the ``page_token`` can be accessed at - ``next_page_token`` of the - :class:`~google.cloud.bigquery.table.RowIterator`. - - :type start_index: int - :param start_index: (Optional) The zero-based index of the starting - row to read. - - :type page_size: int - :param page_size: (Optional) The maximum number of items to return - per page in the iterator. - - :type retry: :class:`google.api_core.retry.Retry` - :param retry: (Optional) How to retry the RPC. - - :rtype: :class:`~google.cloud.bigquery.table.RowIterator` - :returns: Iterator of row data - :class:`~google.cloud.bigquery.table.Row`-s. During each - page, the iterator will have the ``total_rows`` attribute - set, which counts the total number of rows **in the table** - (this is distinct from the total number of rows in the - current page: ``iterator.page.num_items``). + Args: + table (Union[ \ + :class:`~google.cloud.bigquery.table.Table`, \ + :class:`~google.cloud.bigquery.table.TableReference`, \ + str, \ + ]): + The table to list, or a reference to it. + selected_fields (Sequence[ \ + :class:`~google.cloud.bigquery.schema.SchemaField` \ + ]): + The fields to return. Required if ``table`` is a + :class:`~google.cloud.bigquery.table.TableReference`. + max_results (int): + (Optional) maximum number of rows to return. + page_token (str): + (Optional) Token representing a cursor into the table's rows. + If not passed, the API will return the first page of the + rows. The token marks the beginning of the iterator to be + returned and the value of the ``page_token`` can be accessed + at ``next_page_token`` of the + :class:`~google.cloud.bigquery.table.RowIterator`. + start_index (int): + (Optional) The zero-based index of the starting row to read. + page_size (int): + (Optional) The maximum number of items to return per page in + the iterator. + retry (:class:`google.api_core.retry.Retry`): + (Optional) How to retry the RPC. + Returns: + google.cloud.bigquery.table.RowIterator: + Iterator of row data + :class:`~google.cloud.bigquery.table.Row`-s. During each + page, the iterator will have the ``total_rows`` attribute + set, which counts the total number of rows **in the table** + (this is distinct from the total number of rows in the + current page: ``iterator.page.num_items``). """ + if isinstance(table, str): + table = TableReference.from_string( + table, default_project=self.project) + if selected_fields is not None: schema = selected_fields elif isinstance(table, TableReference): diff --git a/bigquery/google/cloud/bigquery/dataset.py b/bigquery/google/cloud/bigquery/dataset.py index 9927af4e9a7a..5679ed76744b 100644 --- a/bigquery/google/cloud/bigquery/dataset.py +++ b/bigquery/google/cloud/bigquery/dataset.py @@ -211,18 +211,21 @@ def from_api_repr(cls, resource): return cls(project, dataset_id) @classmethod - def from_string(cls, full_dataset_id): - """Construct a dataset reference from fully-qualified dataset ID. + def from_string(cls, dataset_id, default_project=None): + """Construct a dataset reference from dataset ID string. Args: - full_dataset_id (str): - A fully-qualified dataset ID in standard SQL format. Must - included both the project ID and the dataset ID, separated by - ``.``. + dataset_id (str): + A dataset ID in standard SQL format. If ``default_project`` + is not specified, this must included both the project ID and + the dataset ID, separated by ``.``. + default_project (str): + Optional. The project ID to use when ``dataset_id`` does not + include a project ID. Returns: DatasetReference: - Dataset reference parsed from ``full_dataset_id``. + Dataset reference parsed from ``dataset_id``. Examples: >>> DatasetReference.from_string('my-project-id.some_dataset') @@ -230,16 +233,27 @@ def from_string(cls, full_dataset_id): Raises: ValueError: - If ``full_dataset_id`` is not a fully-qualified dataset ID in + If ``dataset_id`` is not a fully-qualified dataset ID in standard SQL format. """ - parts = full_dataset_id.split('.') - if len(parts) != 2: + output_dataset_id = dataset_id + output_project_id = default_project + parts = dataset_id.split('.') + + if len(parts) == 1 and not default_project: raise ValueError( - 'full_dataset_id must be a fully-qualified dataset ID in ' - 'standard SQL format. e.g. "project.dataset_id", got ' - '{}'.format(full_dataset_id)) - return cls(*parts) + 'When default_project is not set, dataset_id must be a ' + 'fully-qualified dataset ID in standard SQL format. ' + 'e.g. "project.dataset_id", got {}'.format(dataset_id)) + elif len(parts) == 2: + output_project_id, output_dataset_id = parts + elif len(parts) > 2: + raise ValueError( + 'Too many parts in dataset_id. Expected a fully-qualified ' + 'dataset ID in standard SQL format. e.g. ' + '"project.dataset_id", got {}'.format(dataset_id)) + + return cls(output_project_id, output_dataset_id) def to_api_repr(self): """Construct the API resource representation of this dataset reference diff --git a/bigquery/google/cloud/bigquery/table.py b/bigquery/google/cloud/bigquery/table.py index d4edbd875792..57e06e7f2258 100644 --- a/bigquery/google/cloud/bigquery/table.py +++ b/bigquery/google/cloud/bigquery/table.py @@ -172,17 +172,20 @@ def path(self): self._project, self._dataset_id, self._table_id) @classmethod - def from_string(cls, full_table_id): - """Construct a table reference from fully-qualified table ID. + def from_string(cls, table_id, default_project=None): + """Construct a table reference from table ID string. Args: - full_table_id (str): - A fully-qualified table ID in standard SQL format. Must - included a project ID, dataset ID, and table ID, each - separated by ``.``. + table_id (str): + A table ID in standard SQL format. If ``default_project`` + is not specified, this must included a project ID, dataset + ID, and table ID, each separated by ``.``. + default_project (str): + Optional. The project ID to use when ``table_id`` does not + include a project ID. Returns: - TableReference: Table reference parsed from ``full_table_id``. + TableReference: Table reference parsed from ``table_id``. Examples: >>> TableReference.from_string('my-project.mydataset.mytable') @@ -190,19 +193,41 @@ def from_string(cls, full_table_id): Raises: ValueError: - If ``full_table_id`` is not a fully-qualified table ID in + If ``table_id`` is not a fully-qualified table ID in standard SQL format. """ from google.cloud.bigquery.dataset import DatasetReference - parts = full_table_id.split('.') - if len(parts) != 3: + output_project_id = default_project + output_dataset_id = None + output_table_id = None + parts = table_id.split('.') + + if len(parts) < 2: raise ValueError( - 'full_table_id must be a fully-qualified table ID in ' + 'table_id must be a fully-qualified table ID in ' 'standard SQL format. e.g. "project.dataset.table", got ' - '{}'.format(full_table_id)) + '{}'.format(table_id)) + elif len(parts) == 2: + if not default_project: + raise ValueError( + 'When default_project is not set, table_id must be a ' + 'fully-qualified table ID in standard SQL format. ' + 'e.g. "project.dataset_id.table_id", got {}'.format( + table_id)) + output_dataset_id, output_table_id = parts + elif len(parts) == 3: + output_project_id, output_dataset_id, output_table_id = parts + if len(parts) > 3: + raise ValueError( + 'Too many parts in table_id. Must be a fully-qualified table ' + 'ID in standard SQL format. e.g. "project.dataset.table", ' + 'got {}'.format(table_id)) - return cls(DatasetReference(parts[0], parts[1]), parts[2]) + return cls( + DatasetReference(output_project_id, output_dataset_id), + output_table_id, + ) @classmethod def from_api_repr(cls, resource): diff --git a/bigquery/tests/system.py b/bigquery/tests/system.py index 1cfc46c4736e..650d1a12bf55 100644 --- a/bigquery/tests/system.py +++ b/bigquery/tests/system.py @@ -175,17 +175,27 @@ def test_create_dataset(self): self.assertEqual(dataset.project, Config.CLIENT.project) def test_get_dataset(self): - DATASET_ID = _make_dataset_id('get_dataset') + dataset_id = _make_dataset_id('get_dataset') client = Config.CLIENT - dataset_arg = Dataset(client.dataset(DATASET_ID)) + dataset_arg = Dataset(client.dataset(dataset_id)) dataset_arg.friendly_name = 'Friendly' dataset_arg.description = 'Description' dataset = retry_403(client.create_dataset)(dataset_arg) self.to_delete.append(dataset) - dataset_ref = client.dataset(DATASET_ID) + dataset_ref = client.dataset(dataset_id) + # Get with a reference. got = client.get_dataset(dataset_ref) + self.assertEqual(got.friendly_name, 'Friendly') + self.assertEqual(got.description, 'Description') + + # Get with a string. + got = client.get_dataset(dataset_id) + self.assertEqual(got.friendly_name, 'Friendly') + self.assertEqual(got.description, 'Description') + # Get with a fully-qualified string. + got = client.get_dataset('{}.{}'.format(client.project, dataset_id)) self.assertEqual(got.friendly_name, 'Friendly') self.assertEqual(got.description, 'Description') @@ -281,6 +291,14 @@ def test_create_table_w_time_partitioning_w_clustering_fields(self): self.assertEqual(time_partitioning.field, 'transaction_time') self.assertEqual(table.clustering_fields, ['user_email', 'store_code']) + def test_delete_dataset_with_string(self): + dataset_id = _make_dataset_id('delete_table_true') + dataset_ref = Config.CLIENT.dataset(dataset_id) + retry_403(Config.CLIENT.create_dataset)(Dataset(dataset_ref)) + self.assertTrue(_dataset_exists(dataset_ref)) + Config.CLIENT.delete_dataset(dataset_id) + self.assertFalse(_dataset_exists(dataset_ref)) + def test_delete_dataset_delete_contents_true(self): dataset_id = _make_dataset_id('delete_table_true') dataset = retry_403(Config.CLIENT.create_dataset)( @@ -304,20 +322,27 @@ def test_delete_dataset_delete_contents_false(self): Config.CLIENT.delete_dataset(dataset) def test_get_table_w_public_dataset(self): - PUBLIC = 'bigquery-public-data' - DATASET_ID = 'samples' - TABLE_ID = 'shakespeare' - table_ref = DatasetReference(PUBLIC, DATASET_ID).table(TABLE_ID) + public = 'bigquery-public-data' + dataset_id = 'samples' + table_id = 'shakespeare' + table_ref = DatasetReference(public, dataset_id).table(table_id) + # Get table with reference. table = Config.CLIENT.get_table(table_ref) - - self.assertEqual(table.table_id, TABLE_ID) - self.assertEqual(table.dataset_id, DATASET_ID) - self.assertEqual(table.project, PUBLIC) + self.assertEqual(table.table_id, table_id) + self.assertEqual(table.dataset_id, dataset_id) + self.assertEqual(table.project, public) schema_names = [field.name for field in table.schema] self.assertEqual( schema_names, ['word', 'word_count', 'corpus', 'corpus_date']) + # Get table with string. + table = Config.CLIENT.get_table( + '{}.{}.{}'.format(public, dataset_id, table_id)) + self.assertEqual(table.table_id, table_id) + self.assertEqual(table.dataset_id, dataset_id) + self.assertEqual(table.project, public) + def test_list_partitions(self): table_ref = DatasetReference( 'bigquery-public-data', @@ -327,8 +352,8 @@ def test_list_partitions(self): self.assertGreater(len(all_rows), 1000) def test_list_tables(self): - DATASET_ID = _make_dataset_id('list_tables') - dataset = self.temp_dataset(DATASET_ID) + dataset_id = _make_dataset_id('list_tables') + dataset = self.temp_dataset(dataset_id) # Retrieve tables before any are created for the dataset. iterator = Config.CLIENT.list_tables(dataset) all_tables = list(iterator) @@ -352,9 +377,18 @@ def test_list_tables(self): self.assertIsNone(iterator.next_page_token) created = [table for table in all_tables if (table.table_id in tables_to_create and - table.dataset_id == DATASET_ID)] + table.dataset_id == dataset_id)] self.assertEqual(len(created), len(tables_to_create)) + # List tables with a string ID. + iterator = Config.CLIENT.list_tables(dataset_id) + self.assertGreater(len(list(iterator)), 0) + + # List tables with a fully-qualified string ID. + iterator = Config.CLIENT.list_tables( + '{}.{}'.format(Config.CLIENT.project, dataset_id)) + self.assertGreater(len(list(iterator)), 0) + def test_update_table(self): dataset = self.temp_dataset(_make_dataset_id('update_table')) diff --git a/bigquery/tests/unit/test_client.py b/bigquery/tests/unit/test_client.py index e49ec3670956..73b14b506244 100644 --- a/bigquery/tests/unit/test_client.py +++ b/bigquery/tests/unit/test_client.py @@ -53,6 +53,25 @@ def _make_connection(*responses): return mock_conn +def _make_list_partitons_meta_info(project, dataset_id, table_id, num_rows=0): + return { + 'tableReference': + { + 'projectId': project, + 'datasetId': dataset_id, + 'tableId': '{}$__PARTITIONS_SUMMARY__'.format(table_id), + }, + 'schema': {'fields': [ + {'name': 'project_id', 'type': 'STRING', 'mode': 'NULLABLE'}, + {'name': 'dataset_id', 'type': 'STRING', 'mode': 'NULLABLE'}, + {'name': 'table_id', 'type': 'STRING', 'mode': 'NULLABLE'}, + {'name': 'partition_id', 'type': 'STRING', 'mode': 'NULLABLE'} + ]}, + 'etag': 'ETAG', + 'numRows': num_rows, + } + + class TestClient(unittest.TestCase): PROJECT = 'PROJECT' @@ -449,7 +468,10 @@ def test_get_dataset(self): client._connection = _make_connection( ServerError('', errors=[{'reason': 'backendError'}]), resource) - dataset = client.get_dataset(dataset_ref) + dataset = client.get_dataset( + # Test with a string for dataset ID. + dataset_ref.dataset_id, + ) self.assertEqual(dataset.dataset_id, self.DS_ID) def test_create_dataset_minimal(self): @@ -1342,7 +1364,14 @@ def test_update_table_w_schema_None(self): creds = _make_credentials() client = self._make_one(project=self.PROJECT, credentials=creds) conn = client._connection = _make_connection(resource1, resource2) - table = client.get_table(self.TABLE_REF) + table = client.get_table( + # Test with string for table ID + '{}.{}.{}'.format( + self.TABLE_REF.project, + self.TABLE_REF.dataset_id, + self.TABLE_REF.table_id, + ) + ) table.schema = None updated_table = client.update_table(table, ['schema']) @@ -1495,8 +1524,10 @@ def test_list_tables_explicit(self): dataset = client.dataset(self.DS_ID) iterator = client.list_tables( - dataset, max_results=3, page_token=TOKEN) - self.assertIs(iterator.dataset, dataset) + # Test with string for dataset ID. + self.DS_ID, + max_results=3, page_token=TOKEN) + self.assertEqual(iterator.dataset, dataset) page = six.next(iterator.pages) tables = list(page) token = iterator.next_page_token @@ -1521,13 +1552,19 @@ def test_list_tables_wrong_type(self): def test_delete_dataset(self): from google.cloud.bigquery.dataset import Dataset + from google.cloud.bigquery.dataset import DatasetReference + ds_ref = DatasetReference(self.PROJECT, self.DS_ID) + datasets = ( + ds_ref, + Dataset(ds_ref), + '{}.{}'.format(self.PROJECT, self.DS_ID), + ) PATH = 'projects/%s/datasets/%s' % (self.PROJECT, self.DS_ID) creds = _make_credentials() client = self._make_one(project=self.PROJECT, credentials=creds) - conn = client._connection = _make_connection({}, {}) - ds_ref = client.dataset(self.DS_ID) - for arg in (ds_ref, Dataset(ds_ref)): + conn = client._connection = _make_connection(*([{}] * len(datasets))) + for arg in datasets: client.delete_dataset(arg) conn.api_request.assert_called_with( method='DELETE', @@ -1558,15 +1595,24 @@ def test_delete_dataset_wrong_type(self): def test_delete_table(self): from google.cloud.bigquery.table import Table + tables = ( + self.TABLE_REF, + Table(self.TABLE_REF), + '{}.{}.{}'.format( + self.TABLE_REF.project, + self.TABLE_REF.dataset_id, + self.TABLE_REF.table_id, + ), + ) path = 'projects/%s/datasets/%s/tables/%s' % ( self.PROJECT, self.DS_ID, self.TABLE_ID) creds = _make_credentials() http = object() client = self._make_one(project=self.PROJECT, credentials=creds, _http=http) - conn = client._connection = _make_connection({}, {}) + conn = client._connection = _make_connection(*([{}] * len(tables))) - for arg in (self.TABLE_REF, Table(self.TABLE_REF)): + for arg in tables: client.delete_table(arg) conn.api_request.assert_called_with( method='DELETE', path='/%s' % path) @@ -2115,10 +2161,14 @@ def test_load_table_from_uri_w_client_location(self): project=self.PROJECT, credentials=creds, _http=http, location=self.LOCATION) conn = client._connection = _make_connection(resource) - destination = client.dataset(self.DS_ID).table(destination_id) client.load_table_from_uri( - source_uri, destination, + source_uri, + # Test with string for table ID. + '{}.{}'.format( + self.DS_ID, + destination_id, + ), job_id=job_id, project='other-project') @@ -2410,12 +2460,12 @@ def test_copy_table_w_client_location(self): project=self.PROJECT, credentials=creds, _http=http, location=self.LOCATION) conn = client._connection = _make_connection(resource) - dataset = client.dataset(self.DS_ID) - source = dataset.table(source_id) - destination = dataset.table(destination_id) client.copy_table( - source, destination, job_id=job_id, project='other-project') + # Test with string for table IDs. + '{}.{}'.format(self.DS_ID, source_id), + '{}.{}'.format(self.DS_ID, destination_id), + job_id=job_id, project='other-project') # Check that copy_table actually starts the job. conn.api_request.assert_called_once_with( @@ -2536,11 +2586,11 @@ def test_extract_table_w_client_location(self): project=self.PROJECT, credentials=creds, _http=http, location=self.LOCATION) conn = client._connection = _make_connection(resource) - dataset = client.dataset(self.DS_ID) - source = dataset.table(source_id) client.extract_table( - source, destination, job_id=job_id, project='other-project') + # Test with string for table ID. + '{}.{}'.format(self.DS_ID, source_id), + destination, job_id=job_id, project='other-project') # Check that extract_table actually starts the job. conn.api_request.assert_called_once_with( @@ -3081,7 +3131,7 @@ def test_insert_rows_w_schema(self): from google.cloud._helpers import UTC from google.cloud._helpers import _datetime_to_rfc3339 from google.cloud._helpers import _microseconds_from_datetime - from google.cloud.bigquery.table import Table, SchemaField + from google.cloud.bigquery.table import SchemaField WHEN_TS = 1437767599.006 WHEN = datetime.datetime.utcfromtimestamp(WHEN_TS).replace( @@ -3098,7 +3148,6 @@ def test_insert_rows_w_schema(self): SchemaField('age', 'INTEGER', mode='REQUIRED'), SchemaField('joined', 'TIMESTAMP', mode='NULLABLE'), ] - table = Table(self.TABLE_REF, schema=schema) ROWS = [ ('Phred Phlyntstone', 32, _datetime_to_rfc3339(WHEN)), ('Bharney Rhubble', 33, WHEN + datetime.timedelta(seconds=1)), @@ -3122,7 +3171,11 @@ def _row_data(row): } with mock.patch('uuid.uuid4', side_effect=map(str, range(len(ROWS)))): - errors = client.insert_rows(table, ROWS) + # Test with using string IDs for the table. + errors = client.insert_rows( + '{}.{}'.format(self.DS_ID, self.TABLE_ID), + ROWS, + selected_fields=schema) self.assertEqual(len(errors), 0) conn.api_request.assert_called_once() @@ -3519,24 +3572,35 @@ def test_insert_rows_json(self): path='/%s' % PATH, data=SENT) + def test_insert_rows_json_with_string_id(self): + rows = [{'col1': 'val1'}] + creds = _make_credentials() + http = object() + client = self._make_one( + project='default-project', credentials=creds, _http=http) + conn = client._connection = _make_connection({}) + + with mock.patch('uuid.uuid4', side_effect=map(str, range(len(rows)))): + errors = client.insert_rows_json('proj.dset.tbl', rows) + + self.assertEqual(len(errors), 0) + expected = { + 'rows': [{ + 'json': row, + 'insertId': str(i), + } for i, row in enumerate(rows)], + } + conn.api_request.assert_called_once_with( + method='POST', + path='/projects/proj/datasets/dset/tables/tbl/insertAll', + data=expected) + def test_list_partitions(self): from google.cloud.bigquery.table import Table rows = 3 - meta_info = { - 'tableReference': - {'projectId': self.PROJECT, - 'datasetId': self.DS_ID, - 'tableId': '%s$__PARTITIONS_SUMMARY__' % self.TABLE_ID}, - 'schema': {'fields': [ - {'name': 'project_id', 'type': 'STRING', 'mode': 'NULLABLE'}, - {'name': 'dataset_id', 'type': 'STRING', 'mode': 'NULLABLE'}, - {'name': 'table_id', 'type': 'STRING', 'mode': 'NULLABLE'}, - {'name': 'partition_id', 'type': 'STRING', 'mode': 'NULLABLE'} - ]}, - 'etag': 'ETAG', - 'numRows': rows, - } + meta_info = _make_list_partitons_meta_info( + self.PROJECT, self.DS_ID, self.TABLE_ID, rows) data = { 'totalRows': str(rows), @@ -3563,6 +3627,21 @@ def test_list_partitions(self): self.assertEqual(len(partition_list), rows) self.assertIn('20180102', partition_list) + def test_list_partitions_with_string_id(self): + meta_info = _make_list_partitons_meta_info( + self.PROJECT, self.DS_ID, self.TABLE_ID, 0) + + creds = _make_credentials() + http = object() + client = self._make_one( + project=self.PROJECT, credentials=creds, _http=http) + client._connection = _make_connection(meta_info, {}) + + partition_list = client.list_partitions( + '{}.{}'.format(self.DS_ID, self.TABLE_ID)) + + self.assertEquals(len(partition_list), 0) + def test_list_rows(self): import datetime from google.cloud._helpers import UTC @@ -3641,8 +3720,6 @@ def _bigquery_timestamp_float_repr(ts_float): query_params={}) def test_list_rows_empty_table(self): - from google.cloud.bigquery.table import Table - response = { 'totalRows': '0', 'rows': [], @@ -3654,9 +3731,17 @@ def test_list_rows_empty_table(self): client._connection = _make_connection(response, response) # Table that has no schema because it's an empty table. - table = Table(self.TABLE_REF) - table._properties['creationTime'] = '1234567890' - rows = tuple(client.list_rows(table)) + rows = tuple( + client.list_rows( + # Test with using a string for the table ID. + '{}.{}.{}'.format( + self.TABLE_REF.project, + self.TABLE_REF.dataset_id, + self.TABLE_REF.table_id, + ), + selected_fields=[], + ) + ) self.assertEqual(rows, ()) def test_list_rows_query_params(self): @@ -3977,7 +4062,14 @@ def test_load_table_from_file_w_client_location(self): client, '_do_resumable_upload', self.EXPECTED_CONFIGURATION) with do_upload_patch as do_upload: client.load_table_from_file( - file_obj, self.TABLE_REF, job_id='job_id', + file_obj, + # Test with string for table ID. + '{}.{}.{}'.format( + self.TABLE_REF.project, + self.TABLE_REF.dataset_id, + self.TABLE_REF.table_id, + ), + job_id='job_id', project='other-project', job_config=self._make_config()) diff --git a/bigquery/tests/unit/test_dataset.py b/bigquery/tests/unit/test_dataset.py index d43687e32244..d438c1d478a9 100644 --- a/bigquery/tests/unit/test_dataset.py +++ b/bigquery/tests/unit/test_dataset.py @@ -187,6 +187,27 @@ def test_from_string_legacy_string(self): with self.assertRaises(ValueError): cls.from_string('string-project:string_dataset') + def test_from_string_not_fully_qualified(self): + cls = self._get_target_class() + with self.assertRaises(ValueError): + cls.from_string('string_dataset') + with self.assertRaises(ValueError): + cls.from_string('a.b.c') + + def test_from_string_with_default_project(self): + cls = self._get_target_class() + got = cls.from_string( + 'string_dataset', default_project='default-project') + self.assertEqual(got.project, 'default-project') + self.assertEqual(got.dataset_id, 'string_dataset') + + def test_from_string_ignores_default_project(self): + cls = self._get_target_class() + got = cls.from_string( + 'string-project.string_dataset', default_project='default-project') + self.assertEqual(got.project, 'string-project') + self.assertEqual(got.dataset_id, 'string_dataset') + def test___eq___wrong_type(self): dataset = self._make_one('project_1', 'dataset_1') other = object() diff --git a/bigquery/tests/unit/test_table.py b/bigquery/tests/unit/test_table.py index 9e7b7340ac6f..1d57abbd47cd 100644 --- a/bigquery/tests/unit/test_table.py +++ b/bigquery/tests/unit/test_table.py @@ -148,9 +148,32 @@ def test_from_string_legacy_string(self): def test_from_string_not_fully_qualified(self): cls = self._get_target_class() + with self.assertRaises(ValueError): + cls.from_string('string_table') + with self.assertRaises(ValueError): cls.from_string('string_dataset.string_table') + with self.assertRaises(ValueError): + cls.from_string('a.b.c.d') + + def test_from_string_with_default_project(self): + cls = self._get_target_class() + got = cls.from_string( + 'string_dataset.string_table', default_project='default-project') + self.assertEqual(got.project, 'default-project') + self.assertEqual(got.dataset_id, 'string_dataset') + self.assertEqual(got.table_id, 'string_table') + + def test_from_string_ignores_default_project(self): + cls = self._get_target_class() + got = cls.from_string( + 'string-project.string_dataset.string_table', + default_project='default-project') + self.assertEqual(got.project, 'string-project') + self.assertEqual(got.dataset_id, 'string_dataset') + self.assertEqual(got.table_id, 'string_table') + def test___eq___wrong_type(self): from google.cloud.bigquery.dataset import DatasetReference dataset_ref = DatasetReference('project_1', 'dataset_1')