{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# README\n",
    "\n",
    "This Notebook runs requiring the following environent variable:\n",
    "1. GOOGLE_CLOUD_PROJECT - The google cloud project id.\n",
    "1. BIGQUERY_LOCATION - can take values as per https://cloud.google.com/bigquery/docs/locations, e.g. `us`, `asia-east1`."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Infer location and set up data in that location if needed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/google/home/shobs/code/bigframes1/venv/lib/python3.10/site-packages/google/auth/_default.py:78: UserWarning: Your application has authenticated using end user credentials from Google Cloud SDK without a quota project. You might receive a \"quota exceeded\" or \"API not enabled\" error. See the following page for troubleshooting: https://cloud.google.com/docs/authentication/adc-troubleshooting/user-creds. \n",
      "  warnings.warn(_CLOUD_SDK_CREDENTIALS_WARNING)\n",
      "/usr/local/google/home/shobs/code/bigframes1/venv/lib/python3.10/site-packages/google/auth/_default.py:78: UserWarning: Your application has authenticated using end user credentials from Google Cloud SDK without a quota project. You might receive a \"quota exceeded\" or \"API not enabled\" error. See the following page for troubleshooting: https://cloud.google.com/docs/authentication/adc-troubleshooting/user-creds. \n",
      "  warnings.warn(_CLOUD_SDK_CREDENTIALS_WARNING)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BigQuery project: bigframes-dev\n",
      "BigQuery location: us\n",
      "Penguins Table: bigquery-public-data.ml_datasets.penguins\n",
      "ML Model Dataset: bigframes-dev.bigframes_testing\n"
     ]
    }
   ],
   "source": [
    "# Take multi-region US as the default BQ location, where most of the BQ data lies including the BQ public datasets\n",
    "import os\n",
    "\n",
    "PROJECT_ID = os.environ.get(\"GOOGLE_CLOUD_PROJECT\")\n",
    "BQ_LOCATION = os.environ.get(\"BIGQUERY_LOCATION\")\n",
    "\n",
    "if not PROJECT_ID:\n",
    "    raise ValueError(\"Project must be set via environment variable GOOGLE_CLOUD_PROJECT\")\n",
    "if not BQ_LOCATION:\n",
    "    raise ValueError(\"BQ location must be set via environment variable BIGQUERY_LOCATION\")\n",
    "\n",
    "DATASET = \"bigframes_testing\"\n",
    "PENGUINS_TABLE = \"bigquery-public-data.ml_datasets.penguins\"\n",
    "\n",
    "\n",
    "# Check for a location set in the environment and do location-specific setup if needed\n",
    "\n",
    "import google.api_core.exceptions\n",
    "from google.cloud import bigquery\n",
    "import bigframes\n",
    "\n",
    "client = bigquery.Client()\n",
    "\n",
    "BQ_LOCATION = BQ_LOCATION.lower()\n",
    "if BQ_LOCATION != \"us\":\n",
    "    bq_location_normalized = BQ_LOCATION.replace('-', '_')\n",
    "\n",
    "    # Nominate a local penguins table\n",
    "    penguins_table_ref = bigquery.TableReference.from_string(PENGUINS_TABLE)\n",
    "    penguins_local_dataset_name = f\"{DATASET}_{bq_location_normalized}\"\n",
    "    penguins_local_dataset_ref = bigquery.DatasetReference(project=PROJECT_ID, dataset_id=penguins_local_dataset_name)\n",
    "    penguins_local_dataset = bigquery.Dataset(penguins_local_dataset_ref)\n",
    "    penguins_local_dataset.location = BQ_LOCATION\n",
    "    penguins_local_table_ref= bigquery.TableReference(penguins_local_dataset, penguins_table_ref.table_id)\n",
    "    penguins_local_table = str(penguins_local_table_ref)\n",
    "    try:\n",
    "        client.get_table(penguins_local_table_ref)\n",
    "    except google.api_core.exceptions.NotFound:\n",
    "        client.create_dataset(penguins_local_dataset, exists_ok=True)\n",
    "\n",
    "        # Read the public table as an in-memory dataframe and then write to the local table\n",
    "        session_us = bigframes.connect()\n",
    "        df = session_us.read_gbq(PENGUINS_TABLE).to_pandas()\n",
    "        df.to_gbq(penguins_local_table)\n",
    "\n",
    "    # Finally point the penguins table to the local table\n",
    "    PENGUINS_TABLE=penguins_local_table\n",
    "\n",
    "    # Also update the dataset name used for test artifacts\n",
    "    DATASET = f\"{DATASET}_{bq_location_normalized}\"\n",
    "\n",
    "# Create the dataset to store the model if it doesn't exist    \n",
    "model_local_dataset = bigquery.Dataset(bigquery.DatasetReference(project=PROJECT_ID, dataset_id=DATASET))\n",
    "model_local_dataset.location = BQ_LOCATION\n",
    "model_dataset = client.create_dataset(model_local_dataset, exists_ok=True)\n",
    "\n",
    "# Finally log the variables driving the core notebook execution\n",
    "log = ('\\n'.join(f\"{name}: {str(value)}\" for name, value in {\n",
    "    \"BigQuery project\" : PROJECT_ID,\n",
    "    \"BigQuery location\" : BQ_LOCATION,\n",
    "    \"Penguins Table\" : PENGUINS_TABLE,\n",
    "    \"ML Model Dataset\" : model_dataset.reference\n",
    "}.items())) \n",
    "print(log)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using the BigQuery DataFrames API"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set BigQuery DataFrames options"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import bigframes.pandas\n",
    "\n",
    "# Note: The project option is not required in all environments.\n",
    "# On BigQuery Studio, the project ID is automatically detected.\n",
    "bigframes.pandas.options.bigquery.project = PROJECT_ID\n",
    "\n",
    "# Note: The location option is not required.\n",
    "# It defaults to the location of the first table or query\n",
    "# passed to read_gbq(). For APIs where a location can't be\n",
    "# auto-detected, the location defaults to the \"US\" location.\n",
    "bigframes.pandas.options.bigquery.location = BQ_LOCATION"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialize a dataframe for a BigQuery table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/google/home/shobs/code/bigframes1/venv/lib/python3.10/site-packages/google/auth/_default.py:78: UserWarning: Your application has authenticated using end user credentials from Google Cloud SDK without a quota project. You might receive a \"quota exceeded\" or \"API not enabled\" error. See the following page for troubleshooting: https://cloud.google.com/docs/authentication/adc-troubleshooting/user-creds. \n",
      "  warnings.warn(_CLOUD_SDK_CREDENTIALS_WARNING)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "02af1103261a4e63a4c15efd26b1bc9a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job e1a62d56-8cab-4bc1-9ad3-457f48b71d9c is RUNNING. <a target=\"_blank\" href=\"https://consol…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df = bigframes.pandas.read_gbq(PENGUINS_TABLE)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## View the DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7596fa1f558149bbb01067f13958ca0b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job 88571869-2c86-4605-942d-21f11909f16b is DONE. 0 Bytes processed. <a target=\"_blank\" href…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "327df899d66a43eabc8b9f217079032b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job ba0deab9-f576-46ad-adb8-ec22f6a88029 is DONE. 31.7 kB processed. <a target=\"_blank\" href…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e27df5c8c9d04ad7b60110db7c935400",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job a4ec65ff-4672-4b28-8300-3269bbbc516a is DONE. 0 Bytes processed. <a target=\"_blank\" href…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>species</th>\n",
       "      <th>island</th>\n",
       "      <th>culmen_length_mm</th>\n",
       "      <th>culmen_depth_mm</th>\n",
       "      <th>flipper_length_mm</th>\n",
       "      <th>body_mass_g</th>\n",
       "      <th>sex</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>36.6</td>\n",
       "      <td>18.4</td>\n",
       "      <td>184.0</td>\n",
       "      <td>3475.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>39.8</td>\n",
       "      <td>19.1</td>\n",
       "      <td>184.0</td>\n",
       "      <td>4650.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>40.9</td>\n",
       "      <td>18.9</td>\n",
       "      <td>184.0</td>\n",
       "      <td>3900.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>46.5</td>\n",
       "      <td>17.9</td>\n",
       "      <td>192.0</td>\n",
       "      <td>3500.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>37.3</td>\n",
       "      <td>16.8</td>\n",
       "      <td>192.0</td>\n",
       "      <td>3000.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>43.2</td>\n",
       "      <td>18.5</td>\n",
       "      <td>192.0</td>\n",
       "      <td>4100.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>46.9</td>\n",
       "      <td>16.6</td>\n",
       "      <td>192.0</td>\n",
       "      <td>2700.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>50.5</td>\n",
       "      <td>18.4</td>\n",
       "      <td>200.0</td>\n",
       "      <td>3400.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>49.5</td>\n",
       "      <td>19.0</td>\n",
       "      <td>200.0</td>\n",
       "      <td>3800.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>40.2</td>\n",
       "      <td>20.1</td>\n",
       "      <td>200.0</td>\n",
       "      <td>3975.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>40.8</td>\n",
       "      <td>18.9</td>\n",
       "      <td>208.0</td>\n",
       "      <td>4300.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>39.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>185.0</td>\n",
       "      <td>3650.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>37.0</td>\n",
       "      <td>16.9</td>\n",
       "      <td>185.0</td>\n",
       "      <td>3000.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>47.0</td>\n",
       "      <td>17.3</td>\n",
       "      <td>185.0</td>\n",
       "      <td>3700.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>34.0</td>\n",
       "      <td>17.1</td>\n",
       "      <td>185.0</td>\n",
       "      <td>3400.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>37.0</td>\n",
       "      <td>16.5</td>\n",
       "      <td>185.0</td>\n",
       "      <td>3400.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>45.7</td>\n",
       "      <td>17.3</td>\n",
       "      <td>193.0</td>\n",
       "      <td>3600.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>50.6</td>\n",
       "      <td>19.4</td>\n",
       "      <td>193.0</td>\n",
       "      <td>3800.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>39.7</td>\n",
       "      <td>17.9</td>\n",
       "      <td>193.0</td>\n",
       "      <td>4250.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>37.8</td>\n",
       "      <td>18.1</td>\n",
       "      <td>193.0</td>\n",
       "      <td>3750.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>46.6</td>\n",
       "      <td>17.8</td>\n",
       "      <td>193.0</td>\n",
       "      <td>3800.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>51.3</td>\n",
       "      <td>19.2</td>\n",
       "      <td>193.0</td>\n",
       "      <td>3650.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>40.2</td>\n",
       "      <td>17.1</td>\n",
       "      <td>193.0</td>\n",
       "      <td>3400.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>36.8</td>\n",
       "      <td>18.5</td>\n",
       "      <td>193.0</td>\n",
       "      <td>3500.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>49.6</td>\n",
       "      <td>18.2</td>\n",
       "      <td>193.0</td>\n",
       "      <td>3775.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>25 rows × 7 columns</p>\n",
       "</div>[344 rows x 7 columns in total]"
      ],
      "text/plain": [
       "                                      species island  culmen_length_mm  \\\n",
       "0         Adelie Penguin (Pygoscelis adeliae)  Dream              36.6   \n",
       "1         Adelie Penguin (Pygoscelis adeliae)  Dream              39.8   \n",
       "2         Adelie Penguin (Pygoscelis adeliae)  Dream              40.9   \n",
       "3   Chinstrap penguin (Pygoscelis antarctica)  Dream              46.5   \n",
       "4         Adelie Penguin (Pygoscelis adeliae)  Dream              37.3   \n",
       "5         Adelie Penguin (Pygoscelis adeliae)  Dream              43.2   \n",
       "6   Chinstrap penguin (Pygoscelis antarctica)  Dream              46.9   \n",
       "7   Chinstrap penguin (Pygoscelis antarctica)  Dream              50.5   \n",
       "8   Chinstrap penguin (Pygoscelis antarctica)  Dream              49.5   \n",
       "9         Adelie Penguin (Pygoscelis adeliae)  Dream              40.2   \n",
       "10        Adelie Penguin (Pygoscelis adeliae)  Dream              40.8   \n",
       "11        Adelie Penguin (Pygoscelis adeliae)  Dream              39.0   \n",
       "12        Adelie Penguin (Pygoscelis adeliae)  Dream              37.0   \n",
       "13  Chinstrap penguin (Pygoscelis antarctica)  Dream              47.0   \n",
       "14        Adelie Penguin (Pygoscelis adeliae)  Dream              34.0   \n",
       "15        Adelie Penguin (Pygoscelis adeliae)  Dream              37.0   \n",
       "16  Chinstrap penguin (Pygoscelis antarctica)  Dream              45.7   \n",
       "17  Chinstrap penguin (Pygoscelis antarctica)  Dream              50.6   \n",
       "18        Adelie Penguin (Pygoscelis adeliae)  Dream              39.7   \n",
       "19        Adelie Penguin (Pygoscelis adeliae)  Dream              37.8   \n",
       "20  Chinstrap penguin (Pygoscelis antarctica)  Dream              46.6   \n",
       "21  Chinstrap penguin (Pygoscelis antarctica)  Dream              51.3   \n",
       "22        Adelie Penguin (Pygoscelis adeliae)  Dream              40.2   \n",
       "23        Adelie Penguin (Pygoscelis adeliae)  Dream              36.8   \n",
       "24  Chinstrap penguin (Pygoscelis antarctica)  Dream              49.6   \n",
       "\n",
       "    culmen_depth_mm  flipper_length_mm  body_mass_g     sex  \n",
       "0              18.4              184.0       3475.0  FEMALE  \n",
       "1              19.1              184.0       4650.0    MALE  \n",
       "2              18.9              184.0       3900.0    MALE  \n",
       "3              17.9              192.0       3500.0  FEMALE  \n",
       "4              16.8              192.0       3000.0  FEMALE  \n",
       "5              18.5              192.0       4100.0    MALE  \n",
       "6              16.6              192.0       2700.0  FEMALE  \n",
       "7              18.4              200.0       3400.0  FEMALE  \n",
       "8              19.0              200.0       3800.0    MALE  \n",
       "9              20.1              200.0       3975.0    MALE  \n",
       "10             18.9              208.0       4300.0    MALE  \n",
       "11             18.7              185.0       3650.0    MALE  \n",
       "12             16.9              185.0       3000.0  FEMALE  \n",
       "13             17.3              185.0       3700.0  FEMALE  \n",
       "14             17.1              185.0       3400.0  FEMALE  \n",
       "15             16.5              185.0       3400.0  FEMALE  \n",
       "16             17.3              193.0       3600.0  FEMALE  \n",
       "17             19.4              193.0       3800.0    MALE  \n",
       "18             17.9              193.0       4250.0    MALE  \n",
       "19             18.1              193.0       3750.0    MALE  \n",
       "20             17.8              193.0       3800.0  FEMALE  \n",
       "21             19.2              193.0       3650.0    MALE  \n",
       "22             17.1              193.0       3400.0  FEMALE  \n",
       "23             18.5              193.0       3500.0  FEMALE  \n",
       "24             18.2              193.0       3775.0    MALE  \n",
       "...\n",
       "\n",
       "[344 rows x 7 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### View the column names in the dataframe (aka columns names in the table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['species', 'island', 'culmen_length_mm', 'culmen_depth_mm',\n",
       "       'flipper_length_mm', 'body_mass_g', 'sex'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### View the table schema"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "species              string[pyarrow]\n",
       "island               string[pyarrow]\n",
       "culmen_length_mm             Float64\n",
       "culmen_depth_mm              Float64\n",
       "flipper_length_mm            Float64\n",
       "body_mass_g                  Float64\n",
       "sex                  string[pyarrow]\n",
       "dtype: object"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Select a subset of columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e8d0aab7b7284b2fb2b5931c8461b7f5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job ee2bae68-7c4b-49f4-b5c2-8818cec33a74 is DONE. 0 Bytes processed. <a target=\"_blank\" href…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b99bbe9e2fa341789507e87a53a6c1e1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job 2568d815-1e92-407c-8c58-27aed7093a6c is DONE. 21.2 kB processed. <a target=\"_blank\" href…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b5f1bff88ab340c8ad5e6dc05893cea8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job 01d36b39-7d27-4b25-850e-d362cb705c1f is DONE. 0 Bytes processed. <a target=\"_blank\" href…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>species</th>\n",
       "      <th>island</th>\n",
       "      <th>body_mass_g</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>3475.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>4650.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>3900.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>3500.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>3000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>4100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>2700.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>3400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>3800.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>3975.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>4300.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>3650.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>3000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>3700.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>3400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>3400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>3600.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>3800.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>4250.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>3750.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>3800.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>3650.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>3400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>3500.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>3775.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>25 rows × 3 columns</p>\n",
       "</div>[344 rows x 3 columns in total]"
      ],
      "text/plain": [
       "                                      species island  body_mass_g\n",
       "0         Adelie Penguin (Pygoscelis adeliae)  Dream       3475.0\n",
       "1         Adelie Penguin (Pygoscelis adeliae)  Dream       4650.0\n",
       "2         Adelie Penguin (Pygoscelis adeliae)  Dream       3900.0\n",
       "3   Chinstrap penguin (Pygoscelis antarctica)  Dream       3500.0\n",
       "4         Adelie Penguin (Pygoscelis adeliae)  Dream       3000.0\n",
       "5         Adelie Penguin (Pygoscelis adeliae)  Dream       4100.0\n",
       "6   Chinstrap penguin (Pygoscelis antarctica)  Dream       2700.0\n",
       "7   Chinstrap penguin (Pygoscelis antarctica)  Dream       3400.0\n",
       "8   Chinstrap penguin (Pygoscelis antarctica)  Dream       3800.0\n",
       "9         Adelie Penguin (Pygoscelis adeliae)  Dream       3975.0\n",
       "10        Adelie Penguin (Pygoscelis adeliae)  Dream       4300.0\n",
       "11        Adelie Penguin (Pygoscelis adeliae)  Dream       3650.0\n",
       "12        Adelie Penguin (Pygoscelis adeliae)  Dream       3000.0\n",
       "13  Chinstrap penguin (Pygoscelis antarctica)  Dream       3700.0\n",
       "14        Adelie Penguin (Pygoscelis adeliae)  Dream       3400.0\n",
       "15        Adelie Penguin (Pygoscelis adeliae)  Dream       3400.0\n",
       "16  Chinstrap penguin (Pygoscelis antarctica)  Dream       3600.0\n",
       "17  Chinstrap penguin (Pygoscelis antarctica)  Dream       3800.0\n",
       "18        Adelie Penguin (Pygoscelis adeliae)  Dream       4250.0\n",
       "19        Adelie Penguin (Pygoscelis adeliae)  Dream       3750.0\n",
       "20  Chinstrap penguin (Pygoscelis antarctica)  Dream       3800.0\n",
       "21  Chinstrap penguin (Pygoscelis antarctica)  Dream       3650.0\n",
       "22        Adelie Penguin (Pygoscelis adeliae)  Dream       3400.0\n",
       "23        Adelie Penguin (Pygoscelis adeliae)  Dream       3500.0\n",
       "24  Chinstrap penguin (Pygoscelis antarctica)  Dream       3775.0\n",
       "...\n",
       "\n",
       "[344 rows x 3 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df[[\n",
    "    \"species\",\n",
    "    \"island\",\n",
    "    \"body_mass_g\",\n",
    "]]\n",
    "df"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### View the first ten values of a series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "38a3a113ceec4bd7a3acd29c60a1af34",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job 31a62db3-8550-43e9-8707-ed80de1382d3 is DONE. 2.8 kB processed. <a target=\"_blank\" href=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3fbd42886e454063ba071cb051a04dda",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job 9cf7ee94-1ff3-4362-a30f-a7cc732717e1 is DONE. 5.5 kB processed. <a target=\"_blank\" href=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "0    3475.0\n",
       "1    4650.0\n",
       "2    3900.0\n",
       "3    3500.0\n",
       "4    3000.0\n",
       "5    4100.0\n",
       "6    2700.0\n",
       "7    3400.0\n",
       "8    3800.0\n",
       "9    3975.0\n",
       "Name: body_mass_g, dtype: Float64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['body_mass_g'].head(10)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute the mean of a series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "feca17550c9e49e194fa6ddbe7635828",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job 792dfdf3-0c5d-4acf-b809-2247ff897bb2 is DONE. 2.7 kB processed. <a target=\"_blank\" href=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "4201.754385964912"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['body_mass_g'].mean()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Filter the DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "30199ccb2dc0496c994e880c8e7470c9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job a5a4fb7a-5570-4ca3-93e5-7dcefee3d137 is DONE. 2.7 kB processed. <a target=\"_blank\" href=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e4da4c5bc1af40e79da1f108a06a4382",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job 06f79d18-0bd6-48cd-b7a7-465efb48f651 is DONE. 21.2 kB processed. <a target=\"_blank\" href…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "85ae699d5bce4ff4ab435d5a3bb1d6c8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job a6917951-c692-4de6-b03e-67250d8fedc9 is RUNNING. <a target=\"_blank\" href=\"https://consol…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>species</th>\n",
       "      <th>island</th>\n",
       "      <th>body_mass_g</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>4650.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>4100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>4300.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>4250.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>4050.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>4000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>4050.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>4300.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>4450.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>4450.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>4100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>4800.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>4400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>4150.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>4050.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>4050.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>4100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>4350.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>4150.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>4400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>4300.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>4150.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>4500.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>4300.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>4550.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>25 rows × 3 columns</p>\n",
       "</div>[177 rows x 3 columns in total]"
      ],
      "text/plain": [
       "                                      species island  body_mass_g\n",
       "1         Adelie Penguin (Pygoscelis adeliae)  Dream       4650.0\n",
       "5         Adelie Penguin (Pygoscelis adeliae)  Dream       4100.0\n",
       "10        Adelie Penguin (Pygoscelis adeliae)  Dream       4300.0\n",
       "18        Adelie Penguin (Pygoscelis adeliae)  Dream       4250.0\n",
       "25  Chinstrap penguin (Pygoscelis antarctica)  Dream       4050.0\n",
       "26        Adelie Penguin (Pygoscelis adeliae)  Dream       4000.0\n",
       "27  Chinstrap penguin (Pygoscelis antarctica)  Dream       4050.0\n",
       "28  Chinstrap penguin (Pygoscelis antarctica)  Dream       4300.0\n",
       "30  Chinstrap penguin (Pygoscelis antarctica)  Dream       4450.0\n",
       "36        Adelie Penguin (Pygoscelis adeliae)  Dream       4450.0\n",
       "44  Chinstrap penguin (Pygoscelis antarctica)  Dream       4100.0\n",
       "45  Chinstrap penguin (Pygoscelis antarctica)  Dream       4800.0\n",
       "57  Chinstrap penguin (Pygoscelis antarctica)  Dream       4400.0\n",
       "61  Chinstrap penguin (Pygoscelis antarctica)  Dream       4150.0\n",
       "66  Chinstrap penguin (Pygoscelis antarctica)  Dream       4050.0\n",
       "67  Chinstrap penguin (Pygoscelis antarctica)  Dream       4050.0\n",
       "68  Chinstrap penguin (Pygoscelis antarctica)  Dream       4100.0\n",
       "74        Adelie Penguin (Pygoscelis adeliae)  Dream       4350.0\n",
       "77        Adelie Penguin (Pygoscelis adeliae)  Dream       4150.0\n",
       "78        Adelie Penguin (Pygoscelis adeliae)  Dream       4400.0\n",
       "80  Chinstrap penguin (Pygoscelis antarctica)  Dream       4300.0\n",
       "90  Chinstrap penguin (Pygoscelis antarctica)  Dream       4150.0\n",
       "92  Chinstrap penguin (Pygoscelis antarctica)  Dream       4500.0\n",
       "93        Adelie Penguin (Pygoscelis adeliae)  Dream       4300.0\n",
       "94  Chinstrap penguin (Pygoscelis antarctica)  Dream       4550.0\n",
       "...\n",
       "\n",
       "[177 rows x 3 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['body_mass_g'] >= 4000.0]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using the Remote Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "# Python 3.13 is not yet a supported runtime for remote functions.\n",
    "# See: https://cloud.google.com/functions/docs/runtime-support#python for the supported runtimes.\n",
    "if sys.version_info >= (3, 13, 0):\n",
    "    sys.exit(0)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BigQuery DataFrames gives you the ability to turn your custom scalar functions into a BigQuery remote function.\n",
    "\n",
    "It requires the GCP project to be set up appropriately and the user having sufficient privileges to use them. One can find more details on it via `help` command."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on function remote_function in module bigframes.pandas:\n",
      "\n",
      "remote_function(input_types: 'List[type]', output_type: 'type', dataset: 'Optional[str]' = None, bigquery_connection: 'Optional[str]' = None, reuse: 'bool' = True)\n",
      "    Decorator to turn a user defined function into a BigQuery remote function.\n",
      "    \n",
      "    .. note::\n",
      "        Please make sure following is setup before using this API:\n",
      "    \n",
      "        1. Have the below APIs enabled for your project:\n",
      "              a. BigQuery Connection API\n",
      "              b. Cloud Functions API\n",
      "              c. Cloud Run API\n",
      "              d. Cloud Build API\n",
      "              e. Artifact Registry API\n",
      "              f. Cloud Resource Manager API\n",
      "    \n",
      "          This can be done from the cloud console (change PROJECT_ID to yours):\n",
      "              https://console.cloud.google.com/apis/enableflow?apiid=bigqueryconnection.googleapis.com,cloudfunctions.googleapis.com,run.googleapis.com,cloudbuild.googleapis.com,artifactregistry.googleapis.com,cloudresourcemanager.googleapis.com&project=PROJECT_ID\n",
      "          Or from the gcloud CLI:\n",
      "              $ gcloud services enable bigqueryconnection.googleapis.com cloudfunctions.googleapis.com run.googleapis.com cloudbuild.googleapis.com artifactregistry.googleapis.com cloudresourcemanager.googleapis.com\n",
      "    \n",
      "        2. Have following IAM roles enabled for you:\n",
      "              a. BigQuery Data Editor (roles/bigquery.dataEditor)\n",
      "              b. BigQuery Connection Admin (roles/bigquery.connectionAdmin)\n",
      "              c. Cloud Functions Developer (roles/cloudfunctions.developer)\n",
      "              d. Service Account User (roles/iam.serviceAccountUser)\n",
      "              e. Storage Object Viewer (roles/storage.objectViewer)\n",
      "              f. Project IAM Admin (roles/resourcemanager.projectIamAdmin)\n",
      "                 (Only required if the bigquery connection being used is not pre-created and is created dynamically with user credentials.)\n",
      "    \n",
      "        3. Either the user has setIamPolicy privilege on the project, or a BigQuery connection is pre-created with necessary IAM role set:\n",
      "              a. To create a connection, follow https://cloud.google.com/bigquery/docs/reference/standard-sql/remote-functions#create_a_connection\n",
      "              b. To set up IAM, follow https://cloud.google.com/bigquery/docs/reference/standard-sql/remote-functions#grant_permission_on_function\n",
      "           Alternatively, the IAM could also be setup via the gcloud CLI:\n",
      "              $ gcloud projects add-iam-policy-binding PROJECT_ID --member=\"serviceAccount:CONNECTION_SERVICE_ACCOUNT_ID\" --role=\"roles/run.invoker\"\n",
      "    \n",
      "    Args:\n",
      "        input_types (list(type)):\n",
      "            List of input data types in the user defined function.\n",
      "        output_type (type):\n",
      "            Data type of the output in the user defined function.\n",
      "        dataset (str, Optional):\n",
      "            Dataset to use to create a BigQuery function. It should be in\n",
      "            `<project_id>.<dataset_name>` or `<dataset_name>` format. If this\n",
      "            param is not provided then session dataset id would be used.\n",
      "        bigquery_connection (str, Optional):\n",
      "            Name of the BigQuery connection. If it is pre created in the same\n",
      "            location as the `bigquery_client.location` then it would be used,\n",
      "            otherwise it would be created dynamically assuming the user has\n",
      "            necessary priviliges. If this param is not provided then the\n",
      "            bigquery connection from the session would be used.\n",
      "        reuse (bool, Optional):\n",
      "            Reuse the remote function if already exists.\n",
      "            `True` by default, which will result in reusing an existing remote\n",
      "            function (if any) that was previously created for the same udf.\n",
      "            Setting it to false would force creating a unique remote function.\n",
      "            If the required remote function does not exist then it would be\n",
      "            created irrespective of this param.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import bigframes.pandas as bpd\n",
    "help(bpd.remote_function)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define a custom function, and specify the intent to turn it into a remote function.\n",
    "\n",
    "It requires a BigQuery connection. If the connection is not already created,\n",
    "the BigQuery DataFrames package attempts to create one assuming the necessary\n",
    "APIs and IAM permissions are setup in the project."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[INFO][2023-08-05 23:12:12,870][bigframes.remote_function] Creating new cloud function: gcloud functions deploy bigframes-f9320ad496b5aeca2d7f343cbab03e2f --gen2 --runtime=python310 --project=bigframes-dev --region=us-central1 --source=/tmp/tmps5m0qu4z --entry-point=udf_http --trigger-http --no-allow-unauthenticated\n",
      "[INFO][2023-08-05 23:13:20,660][bigframes.remote_function] Successfully created cloud function bigframes-f9320ad496b5aeca2d7f343cbab03e2f with uri (https://bigframes-f9320ad496b5aeca2d7f343cbab03e2f-7krlje3eoq-uc.a.run.app)\n",
      "[INFO][2023-08-05 23:13:32,717][bigframes.remote_function] Connector bigframes-rf-conn already exists\n",
      "[INFO][2023-08-05 23:13:32,719][bigframes.remote_function] Creating BQ remote function: \n",
      "    CREATE OR REPLACE FUNCTION `bigframes-dev.bigframes_temp_us`.bigframes_f9320ad496b5aeca2d7f343cbab03e2f(num FLOAT64)\n",
      "    RETURNS STRING\n",
      "    REMOTE WITH CONNECTION `bigframes-dev.us.bigframes-rf-conn`\n",
      "    OPTIONS (\n",
      "      endpoint = \"https://bigframes-f9320ad496b5aeca2d7f343cbab03e2f-7krlje3eoq-uc.a.run.app\"\n",
      "    )\n",
      "[INFO][2023-08-05 23:13:33,697][bigframes.remote_function] Created remote function bigframes-dev.bigframes_temp_us.bigframes_f9320ad496b5aeca2d7f343cbab03e2f\n"
     ]
    }
   ],
   "source": [
    "@bpd.remote_function(bigquery_connection='bigframes-rf-conn', cloud_function_service_account=\"default\")\n",
    "def get_bucket(num: float) -> str:\n",
    "    if not num: return \"NA\"\n",
    "    boundary = 4000\n",
    "    return \"at_or_above_4000\" if num >= boundary else \"below_4000\""
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run the custom function on the BigQuery-backed dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0dba87f5bcb74dca9efebe8f522beeff",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job 5f30816f-f4d0-4063-bb9e-2905b89f717d is DONE. 2.8 kB processed. <a target=\"_blank\" href=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f83d37f1196f4cf780572b0064b745dc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job 447748be-7eaa-486a-a67c-65a595f413cb is RUNNING. <a target=\"_blank\" href=\"https://consol…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a6f6a287d4c64f4fa40756aff1e6490d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job 0ea806ba-d42d-4464-897b-f60210695d69 is DONE. 5.5 kB processed. <a target=\"_blank\" href=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>body_mass_g</th>\n",
       "      <th>body_mass_bucket</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3475.0</td>\n",
       "      <td>below_4000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4650.0</td>\n",
       "      <td>at_or_above_4000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3900.0</td>\n",
       "      <td>below_4000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3500.0</td>\n",
       "      <td>below_4000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3000.0</td>\n",
       "      <td>below_4000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>4100.0</td>\n",
       "      <td>at_or_above_4000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2700.0</td>\n",
       "      <td>below_4000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>3400.0</td>\n",
       "      <td>below_4000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>3800.0</td>\n",
       "      <td>below_4000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>3975.0</td>\n",
       "      <td>below_4000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 2 columns</p>\n",
       "</div>[10 rows x 2 columns in total]"
      ],
      "text/plain": [
       "   body_mass_g  body_mass_bucket\n",
       "0       3475.0        below_4000\n",
       "1       4650.0  at_or_above_4000\n",
       "2       3900.0        below_4000\n",
       "3       3500.0        below_4000\n",
       "4       3000.0        below_4000\n",
       "5       4100.0  at_or_above_4000\n",
       "6       2700.0        below_4000\n",
       "7       3400.0        below_4000\n",
       "8       3800.0        below_4000\n",
       "9       3975.0        below_4000\n",
       "\n",
       "[10 rows x 2 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.assign(body_mass_bucket=df['body_mass_g'].apply(get_bucket))\n",
    "df[['body_mass_g', 'body_mass_bucket']].head(10)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using the ML API"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialize a DataFrame from a BigQuery table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "89bf7ae3fd8641bcbdc5a4614a82f48a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job acd770bb-5ccb-463f-beec-2386132ded6b is RUNNING. <a target=\"_blank\" href=\"https://consol…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "955b432680f44a30a30dcda0f8fbe2a9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job a4529f0b-cc46-498b-8c70-74e0ba660674 is DONE. 0 Bytes processed. <a target=\"_blank\" href…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5d1d6e9dcbf548a88361f164f9f3d707",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job 5b7b38a5-0720-4797-8401-179668134038 is DONE. 31.7 kB processed. <a target=\"_blank\" href…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4599df789445477f85004d0371194578",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job c8370cb9-4f79-4766-a43c-e24db7ba6b50 is DONE. 0 Bytes processed. <a target=\"_blank\" href…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>species</th>\n",
       "      <th>island</th>\n",
       "      <th>culmen_length_mm</th>\n",
       "      <th>culmen_depth_mm</th>\n",
       "      <th>flipper_length_mm</th>\n",
       "      <th>body_mass_g</th>\n",
       "      <th>sex</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>36.6</td>\n",
       "      <td>18.4</td>\n",
       "      <td>184.0</td>\n",
       "      <td>3475.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>39.8</td>\n",
       "      <td>19.1</td>\n",
       "      <td>184.0</td>\n",
       "      <td>4650.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>40.9</td>\n",
       "      <td>18.9</td>\n",
       "      <td>184.0</td>\n",
       "      <td>3900.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>46.5</td>\n",
       "      <td>17.9</td>\n",
       "      <td>192.0</td>\n",
       "      <td>3500.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>37.3</td>\n",
       "      <td>16.8</td>\n",
       "      <td>192.0</td>\n",
       "      <td>3000.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>43.2</td>\n",
       "      <td>18.5</td>\n",
       "      <td>192.0</td>\n",
       "      <td>4100.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>46.9</td>\n",
       "      <td>16.6</td>\n",
       "      <td>192.0</td>\n",
       "      <td>2700.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>50.5</td>\n",
       "      <td>18.4</td>\n",
       "      <td>200.0</td>\n",
       "      <td>3400.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>49.5</td>\n",
       "      <td>19.0</td>\n",
       "      <td>200.0</td>\n",
       "      <td>3800.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>40.2</td>\n",
       "      <td>20.1</td>\n",
       "      <td>200.0</td>\n",
       "      <td>3975.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>40.8</td>\n",
       "      <td>18.9</td>\n",
       "      <td>208.0</td>\n",
       "      <td>4300.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>39.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>185.0</td>\n",
       "      <td>3650.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>37.0</td>\n",
       "      <td>16.9</td>\n",
       "      <td>185.0</td>\n",
       "      <td>3000.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>47.0</td>\n",
       "      <td>17.3</td>\n",
       "      <td>185.0</td>\n",
       "      <td>3700.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>34.0</td>\n",
       "      <td>17.1</td>\n",
       "      <td>185.0</td>\n",
       "      <td>3400.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>37.0</td>\n",
       "      <td>16.5</td>\n",
       "      <td>185.0</td>\n",
       "      <td>3400.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>45.7</td>\n",
       "      <td>17.3</td>\n",
       "      <td>193.0</td>\n",
       "      <td>3600.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>50.6</td>\n",
       "      <td>19.4</td>\n",
       "      <td>193.0</td>\n",
       "      <td>3800.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>39.7</td>\n",
       "      <td>17.9</td>\n",
       "      <td>193.0</td>\n",
       "      <td>4250.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>37.8</td>\n",
       "      <td>18.1</td>\n",
       "      <td>193.0</td>\n",
       "      <td>3750.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>46.6</td>\n",
       "      <td>17.8</td>\n",
       "      <td>193.0</td>\n",
       "      <td>3800.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>51.3</td>\n",
       "      <td>19.2</td>\n",
       "      <td>193.0</td>\n",
       "      <td>3650.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>40.2</td>\n",
       "      <td>17.1</td>\n",
       "      <td>193.0</td>\n",
       "      <td>3400.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Adelie Penguin (Pygoscelis adeliae)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>36.8</td>\n",
       "      <td>18.5</td>\n",
       "      <td>193.0</td>\n",
       "      <td>3500.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Chinstrap penguin (Pygoscelis antarctica)</td>\n",
       "      <td>Dream</td>\n",
       "      <td>49.6</td>\n",
       "      <td>18.2</td>\n",
       "      <td>193.0</td>\n",
       "      <td>3775.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>25 rows × 7 columns</p>\n",
       "</div>[344 rows x 7 columns in total]"
      ],
      "text/plain": [
       "                                      species island  culmen_length_mm  \\\n",
       "0         Adelie Penguin (Pygoscelis adeliae)  Dream              36.6   \n",
       "1         Adelie Penguin (Pygoscelis adeliae)  Dream              39.8   \n",
       "2         Adelie Penguin (Pygoscelis adeliae)  Dream              40.9   \n",
       "3   Chinstrap penguin (Pygoscelis antarctica)  Dream              46.5   \n",
       "4         Adelie Penguin (Pygoscelis adeliae)  Dream              37.3   \n",
       "5         Adelie Penguin (Pygoscelis adeliae)  Dream              43.2   \n",
       "6   Chinstrap penguin (Pygoscelis antarctica)  Dream              46.9   \n",
       "7   Chinstrap penguin (Pygoscelis antarctica)  Dream              50.5   \n",
       "8   Chinstrap penguin (Pygoscelis antarctica)  Dream              49.5   \n",
       "9         Adelie Penguin (Pygoscelis adeliae)  Dream              40.2   \n",
       "10        Adelie Penguin (Pygoscelis adeliae)  Dream              40.8   \n",
       "11        Adelie Penguin (Pygoscelis adeliae)  Dream              39.0   \n",
       "12        Adelie Penguin (Pygoscelis adeliae)  Dream              37.0   \n",
       "13  Chinstrap penguin (Pygoscelis antarctica)  Dream              47.0   \n",
       "14        Adelie Penguin (Pygoscelis adeliae)  Dream              34.0   \n",
       "15        Adelie Penguin (Pygoscelis adeliae)  Dream              37.0   \n",
       "16  Chinstrap penguin (Pygoscelis antarctica)  Dream              45.7   \n",
       "17  Chinstrap penguin (Pygoscelis antarctica)  Dream              50.6   \n",
       "18        Adelie Penguin (Pygoscelis adeliae)  Dream              39.7   \n",
       "19        Adelie Penguin (Pygoscelis adeliae)  Dream              37.8   \n",
       "20  Chinstrap penguin (Pygoscelis antarctica)  Dream              46.6   \n",
       "21  Chinstrap penguin (Pygoscelis antarctica)  Dream              51.3   \n",
       "22        Adelie Penguin (Pygoscelis adeliae)  Dream              40.2   \n",
       "23        Adelie Penguin (Pygoscelis adeliae)  Dream              36.8   \n",
       "24  Chinstrap penguin (Pygoscelis antarctica)  Dream              49.6   \n",
       "\n",
       "    culmen_depth_mm  flipper_length_mm  body_mass_g     sex  \n",
       "0              18.4              184.0       3475.0  FEMALE  \n",
       "1              19.1              184.0       4650.0    MALE  \n",
       "2              18.9              184.0       3900.0    MALE  \n",
       "3              17.9              192.0       3500.0  FEMALE  \n",
       "4              16.8              192.0       3000.0  FEMALE  \n",
       "5              18.5              192.0       4100.0    MALE  \n",
       "6              16.6              192.0       2700.0  FEMALE  \n",
       "7              18.4              200.0       3400.0  FEMALE  \n",
       "8              19.0              200.0       3800.0    MALE  \n",
       "9              20.1              200.0       3975.0    MALE  \n",
       "10             18.9              208.0       4300.0    MALE  \n",
       "11             18.7              185.0       3650.0    MALE  \n",
       "12             16.9              185.0       3000.0  FEMALE  \n",
       "13             17.3              185.0       3700.0  FEMALE  \n",
       "14             17.1              185.0       3400.0  FEMALE  \n",
       "15             16.5              185.0       3400.0  FEMALE  \n",
       "16             17.3              193.0       3600.0  FEMALE  \n",
       "17             19.4              193.0       3800.0    MALE  \n",
       "18             17.9              193.0       4250.0    MALE  \n",
       "19             18.1              193.0       3750.0    MALE  \n",
       "20             17.8              193.0       3800.0  FEMALE  \n",
       "21             19.2              193.0       3650.0    MALE  \n",
       "22             17.1              193.0       3400.0  FEMALE  \n",
       "23             18.5              193.0       3500.0  FEMALE  \n",
       "24             18.2              193.0       3775.0    MALE  \n",
       "...\n",
       "\n",
       "[344 rows x 7 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = bigframes.pandas.read_gbq(PENGUINS_TABLE)\n",
    "df"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clean and prepare the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3b6cf41c4b2843ee81aa536863f02f75",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job 6cb59d7e-091b-4c08-833d-3e189972c28e is DONE. 28.9 kB processed. <a target=\"_blank\" href…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fef8c12077e84b478d842851f1858081",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job b42f2b25-a040-414f-afc0-dfdfde0e358b is DONE. 31.7 kB processed. <a target=\"_blank\" href…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e824317d1255490db89eed9ced439009",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job 9a68cc3f-eade-4c5c-8d18-1eb3277678b6 is DONE. 0 Bytes processed. <a target=\"_blank\" href…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>island</th>\n",
       "      <th>culmen_length_mm</th>\n",
       "      <th>culmen_depth_mm</th>\n",
       "      <th>flipper_length_mm</th>\n",
       "      <th>body_mass_g</th>\n",
       "      <th>sex</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Dream</td>\n",
       "      <td>36.6</td>\n",
       "      <td>18.4</td>\n",
       "      <td>184.0</td>\n",
       "      <td>3475.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Dream</td>\n",
       "      <td>39.8</td>\n",
       "      <td>19.1</td>\n",
       "      <td>184.0</td>\n",
       "      <td>4650.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Dream</td>\n",
       "      <td>40.9</td>\n",
       "      <td>18.9</td>\n",
       "      <td>184.0</td>\n",
       "      <td>3900.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Dream</td>\n",
       "      <td>37.3</td>\n",
       "      <td>16.8</td>\n",
       "      <td>192.0</td>\n",
       "      <td>3000.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Dream</td>\n",
       "      <td>43.2</td>\n",
       "      <td>18.5</td>\n",
       "      <td>192.0</td>\n",
       "      <td>4100.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Dream</td>\n",
       "      <td>40.2</td>\n",
       "      <td>20.1</td>\n",
       "      <td>200.0</td>\n",
       "      <td>3975.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Dream</td>\n",
       "      <td>40.8</td>\n",
       "      <td>18.9</td>\n",
       "      <td>208.0</td>\n",
       "      <td>4300.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Dream</td>\n",
       "      <td>39.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>185.0</td>\n",
       "      <td>3650.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Dream</td>\n",
       "      <td>37.0</td>\n",
       "      <td>16.9</td>\n",
       "      <td>185.0</td>\n",
       "      <td>3000.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Dream</td>\n",
       "      <td>34.0</td>\n",
       "      <td>17.1</td>\n",
       "      <td>185.0</td>\n",
       "      <td>3400.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Dream</td>\n",
       "      <td>37.0</td>\n",
       "      <td>16.5</td>\n",
       "      <td>185.0</td>\n",
       "      <td>3400.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Dream</td>\n",
       "      <td>39.7</td>\n",
       "      <td>17.9</td>\n",
       "      <td>193.0</td>\n",
       "      <td>4250.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Dream</td>\n",
       "      <td>37.8</td>\n",
       "      <td>18.1</td>\n",
       "      <td>193.0</td>\n",
       "      <td>3750.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Dream</td>\n",
       "      <td>40.2</td>\n",
       "      <td>17.1</td>\n",
       "      <td>193.0</td>\n",
       "      <td>3400.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Dream</td>\n",
       "      <td>36.8</td>\n",
       "      <td>18.5</td>\n",
       "      <td>193.0</td>\n",
       "      <td>3500.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Dream</td>\n",
       "      <td>41.5</td>\n",
       "      <td>18.5</td>\n",
       "      <td>201.0</td>\n",
       "      <td>4000.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>Dream</td>\n",
       "      <td>33.1</td>\n",
       "      <td>16.1</td>\n",
       "      <td>178.0</td>\n",
       "      <td>2900.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>Dream</td>\n",
       "      <td>37.2</td>\n",
       "      <td>18.1</td>\n",
       "      <td>178.0</td>\n",
       "      <td>3900.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>Dream</td>\n",
       "      <td>39.5</td>\n",
       "      <td>16.7</td>\n",
       "      <td>178.0</td>\n",
       "      <td>3250.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>Dream</td>\n",
       "      <td>36.0</td>\n",
       "      <td>18.5</td>\n",
       "      <td>186.0</td>\n",
       "      <td>3100.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>Dream</td>\n",
       "      <td>39.6</td>\n",
       "      <td>18.1</td>\n",
       "      <td>186.0</td>\n",
       "      <td>4450.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>Dream</td>\n",
       "      <td>41.3</td>\n",
       "      <td>20.3</td>\n",
       "      <td>194.0</td>\n",
       "      <td>3550.0</td>\n",
       "      <td>MALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>Dream</td>\n",
       "      <td>35.7</td>\n",
       "      <td>18.0</td>\n",
       "      <td>202.0</td>\n",
       "      <td>3550.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>Dream</td>\n",
       "      <td>38.1</td>\n",
       "      <td>17.6</td>\n",
       "      <td>187.0</td>\n",
       "      <td>3425.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>Dream</td>\n",
       "      <td>36.0</td>\n",
       "      <td>17.1</td>\n",
       "      <td>187.0</td>\n",
       "      <td>3700.0</td>\n",
       "      <td>FEMALE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>25 rows × 6 columns</p>\n",
       "</div>[146 rows x 6 columns in total]"
      ],
      "text/plain": [
       "   island  culmen_length_mm  culmen_depth_mm  flipper_length_mm  body_mass_g  \\\n",
       "0   Dream              36.6             18.4              184.0       3475.0   \n",
       "1   Dream              39.8             19.1              184.0       4650.0   \n",
       "2   Dream              40.9             18.9              184.0       3900.0   \n",
       "4   Dream              37.3             16.8              192.0       3000.0   \n",
       "5   Dream              43.2             18.5              192.0       4100.0   \n",
       "9   Dream              40.2             20.1              200.0       3975.0   \n",
       "10  Dream              40.8             18.9              208.0       4300.0   \n",
       "11  Dream              39.0             18.7              185.0       3650.0   \n",
       "12  Dream              37.0             16.9              185.0       3000.0   \n",
       "14  Dream              34.0             17.1              185.0       3400.0   \n",
       "15  Dream              37.0             16.5              185.0       3400.0   \n",
       "18  Dream              39.7             17.9              193.0       4250.0   \n",
       "19  Dream              37.8             18.1              193.0       3750.0   \n",
       "22  Dream              40.2             17.1              193.0       3400.0   \n",
       "23  Dream              36.8             18.5              193.0       3500.0   \n",
       "26  Dream              41.5             18.5              201.0       4000.0   \n",
       "31  Dream              33.1             16.1              178.0       2900.0   \n",
       "32  Dream              37.2             18.1              178.0       3900.0   \n",
       "33  Dream              39.5             16.7              178.0       3250.0   \n",
       "35  Dream              36.0             18.5              186.0       3100.0   \n",
       "36  Dream              39.6             18.1              186.0       4450.0   \n",
       "38  Dream              41.3             20.3              194.0       3550.0   \n",
       "41  Dream              35.7             18.0              202.0       3550.0   \n",
       "51  Dream              38.1             17.6              187.0       3425.0   \n",
       "53  Dream              36.0             17.1              187.0       3700.0   \n",
       "\n",
       "       sex  \n",
       "0   FEMALE  \n",
       "1     MALE  \n",
       "2     MALE  \n",
       "4   FEMALE  \n",
       "5     MALE  \n",
       "9     MALE  \n",
       "10    MALE  \n",
       "11    MALE  \n",
       "12  FEMALE  \n",
       "14  FEMALE  \n",
       "15  FEMALE  \n",
       "18    MALE  \n",
       "19    MALE  \n",
       "22  FEMALE  \n",
       "23  FEMALE  \n",
       "26    MALE  \n",
       "31  FEMALE  \n",
       "32    MALE  \n",
       "33  FEMALE  \n",
       "35  FEMALE  \n",
       "36    MALE  \n",
       "38    MALE  \n",
       "41  FEMALE  \n",
       "51  FEMALE  \n",
       "53  FEMALE  \n",
       "...\n",
       "\n",
       "[146 rows x 6 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# filter down to the data we want to analyze\n",
    "adelie_data = df[df.species == \"Adelie Penguin (Pygoscelis adeliae)\"]\n",
    "\n",
    "# drop the columns we don't care about\n",
    "adelie_data = adelie_data.drop(columns=[\"species\"])\n",
    "\n",
    "# drop rows with nulls to get our training data\n",
    "training_data = adelie_data.dropna()\n",
    "\n",
    "# take a peek at the training data\n",
    "training_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pick feature columns and label column\n",
    "feature_columns = training_data[['island', 'culmen_length_mm', 'culmen_depth_mm', 'flipper_length_mm', 'sex']]\n",
    "label_columns = training_data[['body_mass_g']]\n",
    "\n",
    "# also get the rows that we want to make predictions for (i.e. where the feature column is null)\n",
    "missing_body_mass = adelie_data[adelie_data.body_mass_g.isnull()]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train and evaluate a linear regression model using the ML API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "865f6bb75c5b48e4a52a3183fe3c2582",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job 4b0c58e4-4752-4b96-b490-a95e3ae326c0 is DONE. 31.9 kB processed. <a target=\"_blank\" href…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8c50d8a1cc414e5aab4b5e2255dc0247",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job 741394fb-1599-44d2-9ba2-7e179c906dc5 is RUNNING. <a target=\"_blank\" href=\"https://consol…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "28fc43c965c2439db96095194c597f73",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job 015ccf47-1d52-465f-8201-2a24122d4006 is DONE. 0 Bytes processed. <a target=\"_blank\" href…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4de281419ede4b009d235b57bba1c2bc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job eaefd749-80bf-49e0-a0f9-93ea5315462f is DONE. 56 Bytes processed. <a target=\"_blank\" hre…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9d05dea3dd654b74b8ae73ddc9e48b4e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job e931ec39-9c02-4a04-bd47-fe82d299780b is DONE. 0 Bytes processed. <a target=\"_blank\" href…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_absolute_error</th>\n",
       "      <th>mean_squared_error</th>\n",
       "      <th>mean_squared_log_error</th>\n",
       "      <th>median_absolute_error</th>\n",
       "      <th>r2_score</th>\n",
       "      <th>explained_variance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>223.878763</td>\n",
       "      <td>78553.601634</td>\n",
       "      <td>0.005614</td>\n",
       "      <td>181.330911</td>\n",
       "      <td>0.623951</td>\n",
       "      <td>0.623951</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 6 columns</p>\n",
       "</div>[1 rows x 6 columns in total]"
      ],
      "text/plain": [
       "   mean_absolute_error  mean_squared_error  mean_squared_log_error  \\\n",
       "0           223.878763        78553.601634                0.005614   \n",
       "\n",
       "   median_absolute_error  r2_score  explained_variance  \n",
       "0             181.330911  0.623951            0.623951  \n",
       "\n",
       "[1 rows x 6 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from bigframes.ml.linear_model import LinearRegression\n",
    "\n",
    "# as in scikit-learn, a newly created model is just a bundle of parameters\n",
    "# default parameters are fine here\n",
    "model = LinearRegression()\n",
    "\n",
    "# this will train a temporary model in BigQuery Machine Learning\n",
    "model.fit(feature_columns, label_columns)\n",
    "\n",
    "# check how the model performed\n",
    "model.score(feature_columns, label_columns)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Make predictions using the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "caf8cefe06a14da2a02f31aa1e12c23a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job fe2fe252-8433-4d20-861c-681a8dfbf2c4 is RUNNING. <a target=\"_blank\" href=\"https://consol…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "88cc6713b3dc4252b770c196eb3e6368",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job 929db8ef-fc3b-41ae-a646-102e6ef94c5f is DONE. 8 Bytes processed. <a target=\"_blank\" href…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e3a2e93b205246e4884399ab00afda2c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job 95a08810-311e-4cb9-aad5-ba867b384ad7 is DONE. 0 Bytes processed. <a target=\"_blank\" href…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a2d7f6cff5314c15abc8d6f3f57a187b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job ff0d54a9-3e5f-4f45-a511-17f05c2a5101 is DONE. 16 Bytes processed. <a target=\"_blank\" hre…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e9089a1e055e4667ba6a3ea5b260e91b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='Query job f56133e3-e97f-4413-a909-5ff150b70b9a is DONE. 0 Bytes processed. <a target=\"_blank\" href…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>predicted_body_mass_g</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>292</th>\n",
       "      <td>3459.735118</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 1 columns</p>\n",
       "</div>[1 rows x 1 columns in total]"
      ],
      "text/plain": [
       "     predicted_body_mass_g\n",
       "292            3459.735118\n",
       "\n",
       "[1 rows x 1 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.predict(missing_body_mass)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save the trained model to BigQuery, so we can load it later"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearRegression()"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.to_gbq(f\"{DATASET}.penguins_model\", replace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Clean Up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bpd.close_session()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
