Skip to content

Commit efaa148

Browse files
partheaparthea
committed
Merge remote-tracking branch 'migration/main' into python-bigquery-migration
2 parents 574a568 + 56e7ee7 commit efaa148

273 files changed

Lines changed: 12417 additions & 0 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎bigquery/AUTHORING_GUIDE.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
See https://github.com/GoogleCloudPlatform/python-docs-samples/blob/main/AUTHORING_GUIDE.md

‎bigquery/CONTRIBUTING.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
See https://github.com/GoogleCloudPlatform/python-docs-samples/blob/main/CONTRIBUTING.md

‎bigquery/__init__.py‎

Whitespace-only changes.

‎bigquery/add_empty_column.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# Copyright 2019 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
16+
def add_empty_column(table_id: str) -> None:
17+
# [START bigquery_add_empty_column]
18+
from google.cloud import bigquery
19+
20+
# Construct a BigQuery client object.
21+
client = bigquery.Client()
22+
23+
# TODO(developer): Set table_id to the ID of the table
24+
# to add an empty column.
25+
# table_id = "your-project.your_dataset.your_table_name"
26+
27+
table = client.get_table(table_id) # Make an API request.
28+
29+
original_schema = table.schema
30+
new_schema = original_schema[:] # Creates a copy of the schema.
31+
new_schema.append(bigquery.SchemaField("phone", "STRING"))
32+
33+
table.schema = new_schema
34+
table = client.update_table(table, ["schema"]) # Make an API request.
35+
36+
if len(table.schema) == len(original_schema) + 1 == len(new_schema):
37+
print("A new column has been added.")
38+
else:
39+
print("The column has not been added.")
40+
# [END bigquery_add_empty_column]

‎bigquery/browse_table_data.py‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
# Copyright 2019 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
16+
def browse_table_data(table_id: str) -> None:
17+
# [START bigquery_browse_table]
18+
19+
from google.cloud import bigquery
20+
21+
# Construct a BigQuery client object.
22+
client = bigquery.Client()
23+
24+
# TODO(developer): Set table_id to the ID of the table to browse data rows.
25+
# table_id = "your-project.your_dataset.your_table_name"
26+
27+
# Download all rows from a table.
28+
rows_iter = client.list_rows(table_id) # Make an API request.
29+
30+
# Iterate over rows to make the API requests to fetch row data.
31+
rows = list(rows_iter)
32+
print("Downloaded {} rows from table {}".format(len(rows), table_id))
33+
34+
# Download at most 10 rows.
35+
rows_iter = client.list_rows(table_id, max_results=10)
36+
rows = list(rows_iter)
37+
print("Downloaded {} rows from table {}".format(len(rows), table_id))
38+
39+
# Specify selected fields to limit the results to certain columns.
40+
table = client.get_table(table_id) # Make an API request.
41+
fields = table.schema[:2] # First two columns.
42+
rows_iter = client.list_rows(table_id, selected_fields=fields, max_results=10)
43+
print("Selected {} columns from table {}.".format(len(rows_iter.schema), table_id))
44+
45+
rows = list(rows_iter)
46+
print("Downloaded {} rows from table {}".format(len(rows), table_id))
47+
48+
# Print row data in tabular format.
49+
rows_iter = client.list_rows(table_id, max_results=10)
50+
format_string = "{!s:<16} " * len(rows_iter.schema)
51+
field_names = [field.name for field in rows_iter.schema]
52+
print(format_string.format(*field_names)) # Prints column headers.
53+
54+
for row in rows_iter:
55+
print(format_string.format(*row)) # Prints row data.
56+
# [END bigquery_browse_table]

‎bigquery/client_list_jobs.py‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# Copyright 2019 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
16+
def client_list_jobs() -> None:
17+
# [START bigquery_list_jobs]
18+
19+
from google.cloud import bigquery
20+
21+
import datetime
22+
23+
# Construct a BigQuery client object.
24+
client = bigquery.Client()
25+
26+
# List the 10 most recent jobs in reverse chronological order.
27+
# Omit the max_results parameter to list jobs from the past 6 months.
28+
print("Last 10 jobs:")
29+
for job in client.list_jobs(max_results=10): # API request(s)
30+
print("{}".format(job.job_id))
31+
32+
# The following are examples of additional optional parameters:
33+
34+
# Use min_creation_time and/or max_creation_time to specify a time window.
35+
print("Jobs from the last ten minutes:")
36+
ten_mins_ago = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
37+
for job in client.list_jobs(min_creation_time=ten_mins_ago):
38+
print("{}".format(job.job_id))
39+
40+
# Use all_users to include jobs run by all users in the project.
41+
print("Last 10 jobs run by all users:")
42+
for job in client.list_jobs(max_results=10, all_users=True):
43+
print("{} run by user: {}".format(job.job_id, job.user_email))
44+
45+
# Use state_filter to filter by job state.
46+
print("Last 10 jobs done:")
47+
for job in client.list_jobs(max_results=10, state_filter="DONE"):
48+
print("{}".format(job.job_id))
49+
# [END bigquery_list_jobs]

‎bigquery/client_load_partitioned_table.py‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# Copyright 2019 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
16+
def client_load_partitioned_table(table_id: str) -> None:
17+
# [START bigquery_load_table_partitioned]
18+
from google.cloud import bigquery
19+
20+
# Construct a BigQuery client object.
21+
client = bigquery.Client()
22+
23+
# TODO(developer): Set table_id to the ID of the table to create.
24+
# table_id = "your-project.your_dataset.your_table_name"
25+
26+
job_config = bigquery.LoadJobConfig(
27+
schema=[
28+
bigquery.SchemaField("name", "STRING"),
29+
bigquery.SchemaField("post_abbr", "STRING"),
30+
bigquery.SchemaField("date", "DATE"),
31+
],
32+
skip_leading_rows=1,
33+
time_partitioning=bigquery.TimePartitioning(
34+
type_=bigquery.TimePartitioningType.DAY,
35+
field="date", # Name of the column to use for partitioning.
36+
expiration_ms=7776000000, # 90 days.
37+
),
38+
)
39+
uri = "gs://cloud-samples-data/bigquery/us-states/us-states-by-date.csv"
40+
41+
load_job = client.load_table_from_uri(
42+
uri, table_id, job_config=job_config
43+
) # Make an API request.
44+
45+
load_job.result() # Wait for the job to complete.
46+
47+
table = client.get_table(table_id)
48+
print("Loaded {} rows to table {}".format(table.num_rows, table_id))
49+
# [END bigquery_load_table_partitioned]

‎bigquery/client_query_add_column.py‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
# Copyright 2019 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
16+
def client_query_add_column(table_id: str) -> None:
17+
# [START bigquery_add_column_query_append]
18+
from google.cloud import bigquery
19+
20+
# Construct a BigQuery client object.
21+
client = bigquery.Client()
22+
23+
# TODO(developer): Set table_id to the ID of the destination table.
24+
# table_id = "your-project.your_dataset.your_table_name"
25+
26+
# Retrieves the destination table and checks the length of the schema.
27+
table = client.get_table(table_id) # Make an API request.
28+
print("Table {} contains {} columns".format(table_id, len(table.schema)))
29+
30+
# Configures the query to append the results to a destination table,
31+
# allowing field addition.
32+
job_config = bigquery.QueryJobConfig(
33+
destination=table_id,
34+
schema_update_options=[bigquery.SchemaUpdateOption.ALLOW_FIELD_ADDITION],
35+
write_disposition=bigquery.WriteDisposition.WRITE_APPEND,
36+
)
37+
38+
# Start the query, passing in the extra configuration.
39+
client.query_and_wait(
40+
# In this example, the existing table contains only the 'full_name' and
41+
# 'age' columns, while the results of this query will contain an
42+
# additional 'favorite_color' column.
43+
'SELECT "Timmy" as full_name, 85 as age, "Blue" as favorite_color;',
44+
job_config=job_config,
45+
) # Make an API request and wait for job to complete.
46+
47+
# Checks the updated length of the schema.
48+
table = client.get_table(table_id) # Make an API request.
49+
print("Table {} now contains {} columns".format(table_id, len(table.schema)))
50+
# [END bigquery_add_column_query_append]

‎bigquery/client_query_batch.py‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
# Copyright 2019 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import typing
16+
17+
if typing.TYPE_CHECKING:
18+
from google.cloud import bigquery
19+
20+
21+
def client_query_batch() -> "bigquery.QueryJob":
22+
# [START bigquery_query_batch]
23+
from google.cloud import bigquery
24+
25+
# Construct a BigQuery client object.
26+
client = bigquery.Client()
27+
28+
job_config = bigquery.QueryJobConfig(
29+
# Run at batch priority, which won't count toward concurrent rate limit.
30+
priority=bigquery.QueryPriority.BATCH
31+
)
32+
33+
sql = """
34+
SELECT corpus
35+
FROM `bigquery-public-data.samples.shakespeare`
36+
GROUP BY corpus;
37+
"""
38+
39+
# Start the query, passing in the extra configuration.
40+
query_job = client.query(sql, job_config=job_config) # Make an API request.
41+
42+
# Check on the progress by getting the job's updated state. Once the state
43+
# is `DONE`, the results are ready.
44+
query_job = typing.cast(
45+
"bigquery.QueryJob",
46+
client.get_job(
47+
query_job.job_id, location=query_job.location
48+
), # Make an API request.
49+
)
50+
51+
print("Job {} is currently in state {}".format(query_job.job_id, query_job.state))
52+
# [END bigquery_query_batch]
53+
return query_job

‎bigquery/client_query_destination_table.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# Copyright 2019 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
16+
def client_query_destination_table(table_id: str) -> None:
17+
# [START bigquery_query_destination_table]
18+
from google.cloud import bigquery
19+
20+
# Construct a BigQuery client object.
21+
client = bigquery.Client()
22+
23+
# TODO(developer): Set table_id to the ID of the destination table.
24+
# table_id = "your-project.your_dataset.your_table_name"
25+
26+
job_config = bigquery.QueryJobConfig(destination=table_id)
27+
28+
sql = """
29+
SELECT corpus
30+
FROM `bigquery-public-data.samples.shakespeare`
31+
GROUP BY corpus;
32+
"""
33+
34+
# Start the query, passing in the extra configuration.
35+
client.query_and_wait(
36+
sql, job_config=job_config
37+
) # Make an API request and wait for the query to finish.
38+
39+
print("Query results loaded to the table {}".format(table_id))
40+
# [END bigquery_query_destination_table]

0 commit comments

Comments
 (0)