Skip to content
This repository was archived by the owner on Mar 6, 2026. It is now read-only.

Commit 9868518

Browse files
chalmerlowechalmerlowe
committed
Feat: Adds foreign_type_info attribute to table class and adds unit tests. (#2126)
* adds foreign_type_info attribute to table * feat: Adds foreign_type_info attribute and tests * updates docstrings for foreign_type_info * Updates property handling, especially as regards set/get_sub_prop * Removes extraneous comments and debug expressions * Refactors build_resource_from_properties w get/set_sub_prop * updates to foreign_type_info, tests and wiring * Adds logic to detect non-Sequence schema.fields value * updates assorted tests and logic
1 parent b2d6e0a commit 9868518

7 files changed

Lines changed: 398 additions & 104 deletions

File tree

‎google/cloud/bigquery/_helpers.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -978,11 +978,11 @@ def _build_resource_from_properties(obj, filter_fields):
978978
"""
979979
partial = {}
980980
for filter_field in filter_fields:
981-
api_field = obj._PROPERTY_TO_API_FIELD.get(filter_field)
981+
api_field = _get_sub_prop(obj._PROPERTY_TO_API_FIELD, filter_field)
982982
if api_field is None and filter_field not in obj._properties:
983983
raise ValueError("No property %s" % filter_field)
984984
elif api_field is not None:
985-
partial[api_field] = obj._properties.get(api_field)
985+
_set_sub_prop(partial, api_field, _get_sub_prop(obj._properties, api_field))
986986
else:
987987
# allows properties that are not defined in the library
988988
# and properties that have the same name as API resource key

‎google/cloud/bigquery/schema.py‎

Lines changed: 32 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,9 @@
1515
"""Schemas for BigQuery tables / queries."""
1616

1717
from __future__ import annotations
18-
import collections
1918
import enum
2019
import typing
21-
from typing import Any, cast, Dict, Iterable, Optional, Union
20+
from typing import Any, cast, Dict, Iterable, Optional, Union, Sequence
2221

2322
from google.cloud.bigquery import _helpers
2423
from google.cloud.bigquery import standard_sql
@@ -493,6 +492,8 @@ def _parse_schema_resource(info):
493492
Optional[Sequence[google.cloud.bigquery.schema.SchemaField`]:
494493
A list of parsed fields, or ``None`` if no "fields" key found.
495494
"""
495+
if isinstance(info, list):
496+
return [SchemaField.from_api_repr(f) for f in info]
496497
return [SchemaField.from_api_repr(f) for f in info.get("fields", ())]
497498

498499

@@ -505,40 +506,46 @@ def _build_schema_resource(fields):
505506
Returns:
506507
Sequence[Dict]: Mappings describing the schema of the supplied fields.
507508
"""
508-
return [field.to_api_repr() for field in fields]
509+
if isinstance(fields, Sequence):
510+
# Input is a Sequence (e.g. a list): Process and return a list of SchemaFields
511+
return [field.to_api_repr() for field in fields]
512+
513+
else:
514+
raise TypeError("Schema must be a Sequence (e.g. a list) or None.")
509515

510516

511517
def _to_schema_fields(schema):
512-
"""Coerce `schema` to a list of schema field instances.
518+
"""Coerces schema to a list of SchemaField instances while
519+
preserving the original structure as much as possible.
513520
514521
Args:
515-
schema(Sequence[Union[ \
516-
:class:`~google.cloud.bigquery.schema.SchemaField`, \
517-
Mapping[str, Any] \
518-
]]):
519-
Table schema to convert. If some items are passed as mappings,
520-
their content must be compatible with
521-
:meth:`~google.cloud.bigquery.schema.SchemaField.from_api_repr`.
522+
schema (Sequence[Union[ \
523+
:class:`~google.cloud.bigquery.schema.SchemaField`, \
524+
Mapping[str, Any] \
525+
]
526+
]
527+
)::
528+
Table schema to convert. Can be a list of SchemaField
529+
objects or mappings.
522530
523531
Returns:
524-
Sequence[:class:`~google.cloud.bigquery.schema.SchemaField`]
532+
A list of SchemaField objects.
525533
526534
Raises:
527-
Exception: If ``schema`` is not a sequence, or if any item in the
528-
sequence is not a :class:`~google.cloud.bigquery.schema.SchemaField`
529-
instance or a compatible mapping representation of the field.
535+
TypeError: If schema is not a Sequence.
530536
"""
531-
for field in schema:
532-
if not isinstance(field, (SchemaField, collections.abc.Mapping)):
533-
raise ValueError(
534-
"Schema items must either be fields or compatible "
535-
"mapping representations."
536-
)
537537

538-
return [
539-
field if isinstance(field, SchemaField) else SchemaField.from_api_repr(field)
540-
for field in schema
541-
]
538+
if isinstance(schema, Sequence):
539+
# Input is a Sequence (e.g. a list): Process and return a list of SchemaFields
540+
return [
541+
field
542+
if isinstance(field, SchemaField)
543+
else SchemaField.from_api_repr(field)
544+
for field in schema
545+
]
546+
547+
else:
548+
raise TypeError("Schema must be a Sequence (e.g. a list) or None.")
542549

543550

544551
class PolicyTagList(object):

‎google/cloud/bigquery/table.py‎

Lines changed: 69 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@
2121
import functools
2222
import operator
2323
import typing
24-
from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple, Union
24+
from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple, Union, Sequence
25+
2526
import warnings
2627

2728
try:
@@ -66,6 +67,7 @@
6667
from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration
6768
from google.cloud.bigquery.enums import DefaultPandasDTypes
6869
from google.cloud.bigquery.external_config import ExternalConfig
70+
from google.cloud.bigquery import schema as _schema
6971
from google.cloud.bigquery.schema import _build_schema_resource
7072
from google.cloud.bigquery.schema import _parse_schema_resource
7173
from google.cloud.bigquery.schema import _to_schema_fields
@@ -401,7 +403,7 @@ class Table(_TableBase):
401403
"partitioning_type": "timePartitioning",
402404
"range_partitioning": "rangePartitioning",
403405
"time_partitioning": "timePartitioning",
404-
"schema": "schema",
406+
"schema": ["schema", "fields"],
405407
"snapshot_definition": "snapshotDefinition",
406408
"clone_definition": "cloneDefinition",
407409
"streaming_buffer": "streamingBuffer",
@@ -414,6 +416,7 @@ class Table(_TableBase):
414416
"max_staleness": "maxStaleness",
415417
"resource_tags": "resourceTags",
416418
"external_catalog_table_options": "externalCatalogTableOptions",
419+
"foreign_type_info": ["schema", "foreignTypeInfo"],
417420
}
418421

419422
def __init__(self, table_ref, schema=None) -> None:
@@ -457,8 +460,20 @@ def schema(self):
457460
If ``schema`` is not a sequence, or if any item in the sequence
458461
is not a :class:`~google.cloud.bigquery.schema.SchemaField`
459462
instance or a compatible mapping representation of the field.
463+
464+
.. Note::
465+
If you are referencing a schema for an external catalog table such
466+
as a Hive table, it will also be necessary to populate the foreign_type_info
467+
attribute. This is not necessary if defining the schema for a BigQuery table.
468+
469+
For details, see:
470+
https://cloud.google.com/bigquery/docs/external-tables
471+
https://cloud.google.com/bigquery/docs/datasets-intro#external_datasets
472+
460473
"""
461-
prop = self._properties.get(self._PROPERTY_TO_API_FIELD["schema"])
474+
prop = _helpers._get_sub_prop(
475+
self._properties, self._PROPERTY_TO_API_FIELD["schema"]
476+
)
462477
if not prop:
463478
return []
464479
else:
@@ -469,10 +484,21 @@ def schema(self, value):
469484
api_field = self._PROPERTY_TO_API_FIELD["schema"]
470485

471486
if value is None:
472-
self._properties[api_field] = None
473-
else:
487+
_helpers._set_sub_prop(
488+
self._properties,
489+
api_field,
490+
None,
491+
)
492+
elif isinstance(value, Sequence):
474493
value = _to_schema_fields(value)
475-
self._properties[api_field] = {"fields": _build_schema_resource(value)}
494+
value = _build_schema_resource(value)
495+
_helpers._set_sub_prop(
496+
self._properties,
497+
api_field,
498+
value,
499+
)
500+
else:
501+
raise TypeError("Schema must be a Sequence (e.g. a list) or None.")
476502

477503
@property
478504
def labels(self):
@@ -1081,6 +1107,43 @@ def external_catalog_table_options(
10811107
self._PROPERTY_TO_API_FIELD["external_catalog_table_options"]
10821108
] = value
10831109

1110+
@property
1111+
def foreign_type_info(self) -> Optional[_schema.ForeignTypeInfo]:
1112+
"""Optional. Specifies metadata of the foreign data type definition in
1113+
field schema (TableFieldSchema.foreign_type_definition).
1114+
1115+
Returns:
1116+
Optional[schema.ForeignTypeInfo]:
1117+
Foreign type information, or :data:`None` if not set.
1118+
1119+
.. Note::
1120+
foreign_type_info is only required if you are referencing an
1121+
external catalog such as a Hive table.
1122+
For details, see:
1123+
https://cloud.google.com/bigquery/docs/external-tables
1124+
https://cloud.google.com/bigquery/docs/datasets-intro#external_datasets
1125+
"""
1126+
1127+
prop = _helpers._get_sub_prop(
1128+
self._properties, self._PROPERTY_TO_API_FIELD["foreign_type_info"]
1129+
)
1130+
if prop is not None:
1131+
return _schema.ForeignTypeInfo.from_api_repr(prop)
1132+
return None
1133+
1134+
@foreign_type_info.setter
1135+
def foreign_type_info(self, value: Union[_schema.ForeignTypeInfo, dict, None]):
1136+
value = _helpers._isinstance_or_raise(
1137+
value,
1138+
(_schema.ForeignTypeInfo, dict),
1139+
none_allowed=True,
1140+
)
1141+
if isinstance(value, _schema.ForeignTypeInfo):
1142+
value = value.to_api_repr()
1143+
_helpers._set_sub_prop(
1144+
self._properties, self._PROPERTY_TO_API_FIELD["foreign_type_info"], value
1145+
)
1146+
10841147
@classmethod
10851148
def from_string(cls, full_table_id: str) -> "Table":
10861149
"""Construct a table from fully-qualified table ID.

‎tests/unit/job/test_load.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -272,7 +272,7 @@ def test_schema_setter_invalid_field(self):
272272

273273
config = LoadJobConfig()
274274
full_name = SchemaField("full_name", "STRING", mode="REQUIRED")
275-
with self.assertRaises(ValueError):
275+
with self.assertRaises(TypeError):
276276
config.schema = [full_name, object()]
277277

278278
def test_schema_setter(self):

‎tests/unit/test_client.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2051,7 +2051,7 @@ def test_update_dataset(self):
20512051
ds.labels = LABELS
20522052
ds.access_entries = [AccessEntry("OWNER", "userByEmail", "phred@example.com")]
20532053
ds.resource_tags = RESOURCE_TAGS
2054-
fields = [
2054+
filter_fields = [
20552055
"description",
20562056
"friendly_name",
20572057
"location",
@@ -2065,12 +2065,12 @@ def test_update_dataset(self):
20652065
) as final_attributes:
20662066
ds2 = client.update_dataset(
20672067
ds,
2068-
fields=fields,
2068+
fields=filter_fields,
20692069
timeout=7.5,
20702070
)
20712071

20722072
final_attributes.assert_called_once_with(
2073-
{"path": "/%s" % PATH, "fields": fields}, client, None
2073+
{"path": "/%s" % PATH, "fields": filter_fields}, client, None
20742074
)
20752075

20762076
conn.api_request.assert_called_once_with(
@@ -2615,7 +2615,7 @@ def test_update_table_w_schema_None(self):
26152615
self.assertEqual(len(conn.api_request.call_args_list), 2)
26162616
req = conn.api_request.call_args_list[1]
26172617
self.assertEqual(req[1]["method"], "PATCH")
2618-
sent = {"schema": None}
2618+
sent = {"schema": {"fields": None}}
26192619
self.assertEqual(req[1]["data"], sent)
26202620
self.assertEqual(req[1]["path"], "/%s" % path)
26212621
self.assertEqual(len(updated_table.schema), 0)

0 commit comments

Comments
 (0)