{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "02766e9b-e62c-4887-a507-1b3c4cab54b9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-07T03:49:25.410896Z",
     "iopub.status.busy": "2022-06-07T03:49:25.410698Z",
     "iopub.status.idle": "2022-06-07T03:50:06.533546Z",
     "shell.execute_reply": "2022-06-07T03:50:06.532940Z",
     "shell.execute_reply.started": "2022-06-07T03:49:25.410874Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Spark application\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n<tbody><tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>User</th><th>Current session?</th></tr><tr><td>3</td><td>application_1654573204738_0004</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://ip-172-31-11-179.ap-northeast-1.compute.internal:20888/proxy/application_1654573204738_0004/\" class=\"emr-proxy-link j-36BHYNX80DXQY application_1654573204738_0004\" emr-resource=\"j-36BHYNX80DXQY\n\" application-id=\"application_1654573204738_0004\">Link</a></td><td><a target=\"_blank\" href=\"http://ip-172-31-13-230.ap-northeast-1.compute.internal:8042/node/containerlogs/container_1654573204738_0004_01_000001/livy\">Link</a></td><td>None</td><td>✔</td></tr></tbody></table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SparkSession available as 'spark'.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Current session configs: <tt>{'conf': {'spark.jars.packages': 'io.delta:delta-core_2.12:1.2.0,io.delta:delta-storage:1.2.0', 'spark.sql.extensions': 'io.delta.sql.DeltaSparkSessionExtension', 'spark.sql.catalog.spark_catalog': 'org.apache.spark.sql.delta.catalog.DeltaCatalog'}, 'proxyUser': 'user_Administrator', 'kind': 'pyspark'}</tt><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n<tbody><tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>User</th><th>Current session?</th></tr><tr><td>3</td><td>application_1654573204738_0004</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://ip-172-31-11-179.ap-northeast-1.compute.internal:20888/proxy/application_1654573204738_0004/\" class=\"emr-proxy-link j-36BHYNX80DXQY application_1654573204738_0004\" emr-resource=\"j-36BHYNX80DXQY\n\" application-id=\"application_1654573204738_0004\">Link</a></td><td><a target=\"_blank\" href=\"http://ip-172-31-13-230.ap-northeast-1.compute.internal:8042/node/containerlogs/container_1654573204738_0004_01_000001/livy\">Link</a></td><td>None</td><td>✔</td></tr></tbody></table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%configure -f\n",
    "\n",
    "{\n",
    "  \"conf\": {\n",
    "    \"spark.jars.packages\": \"io.delta:delta-core_2.12:1.2.0,io.delta:delta-storage:1.2.0\",\n",
    "    \"spark.sql.extensions\": \"io.delta.sql.DeltaSparkSessionExtension\",\n",
    "    \"spark.sql.catalog.spark_catalog\": \"org.apache.spark.sql.delta.catalog.DeltaCatalog\"\n",
    "  }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a7c53db2-e446-44d3-9295-e1b024cd891e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-07T03:50:08.735234Z",
     "iopub.status.busy": "2022-06-07T03:50:08.734993Z",
     "iopub.status.idle": "2022-06-07T03:50:08.791446Z",
     "shell.execute_reply": "2022-06-07T03:50:08.790944Z",
     "shell.execute_reply.started": "2022-06-07T03:50:08.735209Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a6b40fc6f6da4c17ad76cdfa48e65044",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from pyspark.sql import SparkSession"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ea024554-0483-4baf-87e0-7ac636809a7c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-07T03:53:24.911728Z",
     "iopub.status.busy": "2022-06-07T03:53:24.911508Z",
     "iopub.status.idle": "2022-06-07T03:54:09.101817Z",
     "shell.execute_reply": "2022-06-07T03:54:09.101120Z",
     "shell.execute_reply.started": "2022-06-07T03:53:24.911705Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "53c39279afbb480eabd2714e19fda568",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "716f5a5499fb444bb3cb4a884c9af796",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Interrupted by user\n"
     ]
    }
   ],
   "source": [
    "from delta.tables import *\n",
    "serverless_outcome = spark.read.format(\"delta\") \\\n",
    "    .load(\"s3a://emr-serverless-630778274080/emr-serverless-spark/delta-lake/nyx-tlc-2021\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e5b2e27-ef57-40d3-affc-cd5f24c9f680",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-07T03:54:47.686065Z",
     "iopub.status.busy": "2022-06-07T03:54:47.685812Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2d52f8a961dd4331ae339973b54e1c95",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d7b846908c5e49bfb54ad510ef00923c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "parquet_input = spark.read.parquet(\"s3a://emr-serverless-630778274080/nyc-taxi/*.parquet\")\n",
    "parquet_input.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "02cf81f5-4e27-4b99-9942-a13d8c75ed96",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "PySpark",
   "language": "python",
   "name": "pysparkkernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "python",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "pyspark",
   "pygments_lexer": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}