From 783e14ce020009bdd14186d3ce333bbec09786d0 Mon Sep 17 00:00:00 2001 From: "user1-63761607@lodsprodmca.onmicrosoft.com" Date: Mon, 27 Jul 2026 16:26:35 +0000 Subject: [PATCH 1/5] adding exercise --- Allfiles/01-explore-azure-databricks.ipynb | 192 ++++++++++++++++++--- 1 file changed, 170 insertions(+), 22 deletions(-) diff --git a/Allfiles/01-explore-azure-databricks.ipynb b/Allfiles/01-explore-azure-databricks.ipynb index af6e358..8b46625 100644 --- a/Allfiles/01-explore-azure-databricks.ipynb +++ b/Allfiles/01-explore-azure-databricks.ipynb @@ -2,8 +2,16 @@ "cells": [ { "cell_type": "markdown", - "id": "82c6d5ed", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "7a601857-78c8-4bab-b2a6-4389ddc11845", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "## Exercise 1: Running Python Code\n", "\n", @@ -17,25 +25,80 @@ "2. Prints a formatted message that summarises the route. For example:\n", " `Route: City Express | From: Central Station → To: Airport Terminal`\n", "\n", - "> 💬 **Genie Code tip:** Open Genie Code panel (click the ![assistant-icon](https://raw.githubusercontent.com/MicrosoftLearning/DP-750T00-Implement-Data-Engineering-Solutions-using-Azure-Databricks/refs/heads/main/Allfiles/media/databricks-assistant.svg) icon on the cell) and type:\n", + "> \uD83D\uDCAC **Genie Code tip:** Open Genie Code panel (click the ![assistant-icon](https://raw.githubusercontent.com/MicrosoftLearning/DP-750T00-Implement-Data-Engineering-Solutions-using-Azure-Databricks/refs/heads/main/Allfiles/media/databricks-assistant.svg) icon on the cell) and type:\n", "> *\"Write Python code to define three route variables and print a formatted summary message\"*\n", "> Use the suggestion as a starting point, then personalise the values." ] }, { "cell_type": "code", - "execution_count": null, - "id": "ef6b149a", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "784b33d8-e96a-4ecf-912c-b4c1ea50a4cc", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 2" + } + }, "outputs": [], "source": [ - "# TODO: Define your route variables and print a formatted summary message" + "%python\n", + "\n", + "route_data =[]\n", + "route_id = ['R001' 'R002', 'R003']\n", + "route_name = ['City Express', 'Harbor Line', 'Metro']\n", + "origin = ['Central Station', 'Downtown', 'Airport Terminal']\n", + "destination = ['Airport Terminal', 'Airport Terminal', 'City Center']\n", + "\n", + "for i in range(len(route_id)):\n", + " route_data.append([route_id[i],route_name[i],origin[i],destination[i]])\n", + "\n", + "route_data\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "374888a6-5fa5-4ad1-a90a-6dab15ad7252", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 3" + } + }, + "outputs": [], + "source": [ + "for i in range(len(route_data)):\n", + " print(f\"Route: {route_data[i][1]} | From: {route_data[i][2]} → To: {route_data[i][3]}\")" ] }, { "cell_type": "markdown", - "id": "0d2df199", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "55d3b919-ace8-4586-9f93-ae3b06e46e27", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "## Exercise 2: Running SQL Commands\n", "\n", @@ -50,26 +113,46 @@ "\n", "The result should be a single-row table with at least two columns.\n", "\n", - "> 💬 **Genie Code tip:** Ask Genie Code:\n", + "> \uD83D\uDCAC **Genie Code tip:** Ask Genie Code:\n", "> *\"Write a SQL SELECT statement in a Databricks notebook that returns the current timestamp and a string literal as named columns\"*\n", "> Remember to start your cell with `%sql`." ] }, { "cell_type": "code", - "execution_count": null, - "id": "01a150db", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "57a32746-f97d-44ec-9832-fcb1b9f76ebd", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ - "# TODO: Use the %sql magic command and write a SQL query\n", - "# Hint: start with %sql on the first line, then write your SELECT statement" + "%sql\n", + "SELECT current_timestamp() AS current_datetime, 'CityMoves Transit' AS system_name\n" ] }, { "cell_type": "markdown", - "id": "2a577b30", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "63912b42-24eb-43e2-8b7f-3ad3fc22e35a", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "## Exercise 3: Documenting with Markdown\n", "\n", @@ -83,23 +166,88 @@ "2. A short paragraph (2–3 sentences) describing the purpose of the notebook in the context of CityMoves Transit.\n", "3. A bulleted list (`-`) of at least three things a reader will find in this notebook.\n", "\n", - "> 💬 **Genie Code tip:** Select this markdown cell and ask Genie Code:\n", + "> \uD83D\uDCAC **Genie Code tip:** Select this markdown cell and ask Genie Code:\n", "> *\"Write a Markdown introduction cell for a data engineering notebook about a public transit system called CityMoves Transit\"*\n", "> Refine the generated text as you see fit." ] }, { "cell_type": "markdown", - "id": "974bf85b", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "6d007e6b-f716-477d-9bc1-1c92b36fdd3b", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "# CityMoves Transit — Data Engineering Notebook\n", + "\n", + "This notebook serves as an introductory exploration of the CityMoves Transit data platform built on Azure Databricks. It demonstrates core notebook capabilities that the data engineering team will use daily for route analytics and operational reporting.\n", + "\n", + "- Python code for defining and summarising transit route data\n", + "- SQL queries for retrieving system metadata and timestamps\n", + "- Markdown documentation for clear, shareable notebook structure" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "f398ad50-590e-48ac-874e-bba7e6217801", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "ee66acac-aec1-451c-a96b-c4498421ece1", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [] } ], "metadata": { + "application/vnd.databricks.v1+notebook": { + "computePreferences": null, + "dashboards": [], + "environmentMetadata": { + "base_environment": "", + "environment_version": "5" + }, + "inputWidgetPreferences": null, + "language": "python", + "notebookMetadata": { + "mostRecentlyExecutedCommandWithImplicitDF": { + "commandId": 3202781778147245, + "dataframes": [ + "_sqldf" + ] + }, + "pythonIndentUnit": 4 + }, + "notebookName": "01-explore-azure-databricks (1)", + "widgets": {} + }, "language_info": { "name": "python" } }, "nbformat": 4, - "nbformat_minor": 5 -} + "nbformat_minor": 0 +} \ No newline at end of file From a92adda794aaf3e7b4a891a3399f865f04fdadaa Mon Sep 17 00:00:00 2001 From: "gb.burcea@gmail.com" Date: Thu, 30 Jul 2026 16:49:18 +0000 Subject: [PATCH 2/5] second exercise --- .../02-select-and-configure-compute.ipynb | 196 +++++++++++++++--- 1 file changed, 162 insertions(+), 34 deletions(-) diff --git a/Allfiles/02-select-and-configure-compute.ipynb b/Allfiles/02-select-and-configure-compute.ipynb index 62deb25..516fe49 100644 --- a/Allfiles/02-select-and-configure-compute.ipynb +++ b/Allfiles/02-select-and-configure-compute.ipynb @@ -2,8 +2,16 @@ "cells": [ { "cell_type": "markdown", - "id": "f611b9a4", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "47edc795-82c4-4438-a096-8eaa26cc876a", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "## Exercise 3: Install Libraries Notebook-Scoped\n", "\n", @@ -16,14 +24,22 @@ }, { "cell_type": "markdown", - "id": "47d5e4b7", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "6e2f07ae-d264-45b2-abfc-b9d960754fc3", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 3.1 — Install the `faker` library notebook-scoped\n", "\n", "Your team relies on `faker` to generate synthetic patient records for pipeline testing. Install it as a notebook-scoped library so the package is available to this notebook only and doesn't affect other users sharing the same compute.\n", "\n", - "> 🤖 **Genie Code tip:** Open Genie Code panel (click the ![assistant-icon](https://raw.githubusercontent.com/MicrosoftLearning/DP-750T00-Implement-Data-Engineering-Solutions-using-Azure-Databricks/refs/heads/main/Allfiles/media/genie-code.svg) icon on the cell) and use the prompt below to get started:\n", + "> \uD83E\uDD16 **Genie Code tip:** Open Genie Code panel (click the ![assistant-icon](https://raw.githubusercontent.com/MicrosoftLearning/DP-750T00-Implement-Data-Engineering-Solutions-using-Azure-Databricks/refs/heads/main/Allfiles/media/genie-code.svg) icon on the cell) and use the prompt below to get started:\n", "> *\"How do I install a Python package notebook-scoped in Databricks using a magic command?\"*\n", "\n", "**Hint:** Use a `%pip` magic command to install the package. Pinning an exact version (e.g., `faker==40.8.0`) is a good practice for reproducibility." @@ -31,18 +47,37 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "667ecc3f", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "8ef69aa3-d5a0-44d9-ab99-1bb0cc8aa8a2", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ - "# TODO: Install faker==40.8.0 notebook-scoped" + "%pip install faker==40.8.0" ] }, { "cell_type": "markdown", - "id": "5145c763", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "d90646dc-cf99-442e-852c-dcd3ad8f1ae9", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 3.2 — Verify the installation\n", "\n", @@ -52,24 +87,48 @@ "2. Creating a `Faker` instance.\n", "3. Printing a randomly generated **full name** and **date of birth**.\n", "\n", - "> 🤖 **Genie Code tip:**\n", + "> \uD83E\uDD16 **Genie Code tip:**\n", "> *\"Show me a Python example of generating a random name and date of birth using the Faker library.\"*" ] }, { "cell_type": "code", - "execution_count": null, - "id": "df305ccc", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "916c5cad-4b4b-4946-ac40-11bd951ddea6", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ - "# TODO: Import Faker, create an instance, and print a generated name and date of birth" + "from faker import Faker\n", + "\n", + "fake = Faker()\n", + "\n", + "print(f\"Full Name: {fake.name()}\")\n", + "print(f\"Date of Birth: {fake.date_of_birth(minimum_age=20, maximum_age=80)}\")" ] }, { "cell_type": "markdown", - "id": "537b38a5", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "3f613c1f-763b-44d4-a070-d283dd24b382", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "## Exercise 4: Generate and Analyze Synthetic Patient Data\n", "\n", @@ -80,8 +139,16 @@ }, { "cell_type": "markdown", - "id": "ee221a7b", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "677df061-0d6b-40d3-a2ef-d2de6128f607", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 4.1 — Generate a synthetic patient admissions DataFrame\n", "\n", @@ -97,7 +164,7 @@ "\n", "Display the first 10 rows of the resulting DataFrame.\n", "\n", - "> 🤖 **Genie Code tip:**\n", + "> \uD83E\uDD16 **Genie Code tip:**\n", "> *\"How do I generate a list of Python dictionaries using the Faker library, then create a PySpark DataFrame from that list?\"*\n", ">\n", "> **Hint:** Use `faker.date_of_birth()` with `minimum_age` and `maximum_age` parameters, or `faker.date_between()` with `start_date` and `end_date`. Use `random.choice()` for the diagnosis code." @@ -105,19 +172,53 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "b38ec3d4", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "9d3b3ee7-b41b-477f-844e-e9237abd9be0", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ - "# TODO: Generate 100 synthetic patient records and create a Spark DataFrame\n", - "# Display the first 10 rows" + "from faker import Faker\n", + "import random\n", + "from datetime import date\n", + "fake = Faker()\n", + "diagnosis_codes = [\"I21\", 'J18', 'E11', 'K80', 'N39']\n", + "\n", + "records = []\n", + "for i in range(1, 1001):\n", + " records.append({\n", + " \"patient_id\": i, \n", + " \"full_name\": fake.name(), \n", + " \"date_of_birth\": str(fake.date_between(start_date = date(1940, 1, 1), end_date = date(2005, 12, 31))), \n", + " \"admission_date\":str(fake.date_between(start_date=date(2023, 1, 1), end_date= date(2025, 12, 31))), \n", + " \"diagnosis_code\": random.choice(diagnosis_codes)\n", + " })\n", + "df = spark.createDataFrame(records)\n", + "display(df)" ] }, { "cell_type": "markdown", - "id": "034453d6", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "947491cd-51e9-48c7-8f48-f342f6c12935", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 4.2 — Analyze admissions by diagnosis code\n", "\n", @@ -127,26 +228,53 @@ "2. Order the results from **most to least** admissions.\n", "3. Display the result.\n", "\n", - "> 🤖 **Genie Code tip:**\n", + "> \uD83E\uDD16 **Genie Code tip:**\n", "> *\"How do I group by a column, count occurrences, and sort the result in descending order using PySpark DataFrame API?\"*" ] }, { "cell_type": "code", - "execution_count": null, - "id": "0115d27c", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "cba9771b-976e-40e3-ba2c-11c7d771481b", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ - "# TODO: Count admissions per diagnosis_code, ordered from most to least" + "from pyspark.sql.functions import col \n", + "\n", + "\n", + "diagnosis_counts = df.groupBy(\"diagnosis_code\").count().orderBy(col(\"count\").desc())\n", + "display(diagnosis_counts)" ] } ], "metadata": { + "application/vnd.databricks.v1+notebook": { + "computePreferences": null, + "dashboards": [], + "environmentMetadata": null, + "inputWidgetPreferences": null, + "language": "python", + "notebookMetadata": { + "pythonIndentUnit": 4 + }, + "notebookName": "02-select-and-configure-compute", + "widgets": {} + }, "language_info": { "name": "python" } }, "nbformat": 4, - "nbformat_minor": 5 + "nbformat_minor": 0 } From 73bb60ff350b14451b03798662d4aef871f9b24d Mon Sep 17 00:00:00 2001 From: "user1-63853039@lodsprodmca.onmicrosoft.com" Date: Thu, 30 Jul 2026 19:04:17 +0000 Subject: [PATCH 3/5] adding 3rd exercise --- ...nd-organize-objects-in-unity-catalog.ipynb | 979 +++++++++++++++--- 1 file changed, 814 insertions(+), 165 deletions(-) diff --git a/Allfiles/03-create-and-organize-objects-in-unity-catalog.ipynb b/Allfiles/03-create-and-organize-objects-in-unity-catalog.ipynb index 7e49032..fddca91 100644 --- a/Allfiles/03-create-and-organize-objects-in-unity-catalog.ipynb +++ b/Allfiles/03-create-and-organize-objects-in-unity-catalog.ipynb @@ -2,8 +2,19 @@ "cells": [ { "cell_type": "markdown", - "id": "86081d58", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "d85d70ca-f48a-4fd3-9fd6-d4f3d60167fc", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "## Exercise 1: Set Up the Catalog Structure\n", "\n", @@ -16,7 +27,7 @@ "```txt\n", "\n", "Unity Catalog\n", - "└── edu_dev [catalog]\n", + "└── edu_dev63853039 [catalog]\n", " │ tags: environment=development | university=lakeside | data_classification=internal\n", " │\n", " ├── bronze [schema]\n", @@ -61,18 +72,29 @@ }, { "cell_type": "markdown", - "id": "293e733e", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "bf42b44b-1e8b-4b2a-930f-d5c2f97b88d1", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 1.1 — Create the development catalog\n", "\n", - "Create a catalog called `edu_dev` with a descriptive comment. This catalog serves as the **development environment** for Lakeside University's data platform.\n", + "Create a catalog called `edu_dev63853039` with a descriptive comment. This catalog serves as the **development environment** for Lakeside University's data platform.\n", "\n", "**Naming convention reminders:**\n", "- Use lowercase with underscores\n", "- No periods, spaces, forward slashes, or control characters\n", "\n", - "> 🤖 **Genie Code tip:** Ask Genie Code:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask Genie Code:\n", "> *\"How do I create a Unity Catalog catalog in Databricks SQL with a comment?\"*\n", "\n", "**Hint:** Use the SQL command `CREATE CATALOG`." @@ -80,9 +102,20 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "9933f5b5", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "56373874-bc35-479c-8712-45bb01547a79", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "# Since no default storage is enabled, we are inheriting the storage path from the default catalog's root.\n", @@ -101,23 +134,48 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "e38aeda3", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "47ea25a1-9872-4855-9ad6-6c41b049141e", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 5" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Create the edu_dev catalog with a descriptive comment" + "CREATE CATALOG IF NOT EXISTS edu_dev63853039\n", + "MANAGED LOCATION 'abfss://unity-catalog-storage@dbstoragep3bbxu5v55osy.dfs.core.windows.net/7405616271049399'\n", + "COMMENT 'Development catalog for Lakeside University data platform';" ] }, { "cell_type": "markdown", - "id": "3470e517", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "f6b3c97c-ac3b-433b-8114-ecc45c3aefd2", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 1.2 — Create medallion schemas\n", "\n", - "Within the `edu_dev` catalog, create three schemas that implement the **medallion architecture**:\n", + "Within the `edu_dev63853039` catalog, create three schemas that implement the **medallion architecture**:\n", "\n", "| Schema | Purpose |\n", "|--------|---------|\n", @@ -125,7 +183,7 @@ "| `silver` | Cleaned, validated, and enriched data |\n", "| `gold` | Aggregated, analytics-ready data |\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"Show me how to create multiple schemas in a Databricks Unity Catalog catalog using SQL.\"*\n", "\n", "**Hint:** Use `CREATE SCHEMA IF NOT EXISTS catalog.schema_name` for each schema." @@ -133,19 +191,49 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "f82b0009", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "d668e74c-1ef6-4a77-b33e-c496f517ee52", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Create the bronze, silver, and gold schemas inside edu_dev" + "CREATE SCHEMA IF NOT EXISTS edu_dev63853039.bronze\n", + "COMMENT 'Raw ingested data - unmodified source files'; \n", + "\n", + "CREATE SCHEMA IF NOT EXISTS edu_dev63853039.silver\n", + "COMMENT 'Cleaned, validated, and enriched data';\n", + "\n", + "CREATE SCHEMA IF NOT EXISTS edu_dev63853039.gold\n", + "COMMENT 'Aggregated and ready for analysis';\n" ] }, { "cell_type": "markdown", - "id": "fc2c333a", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "b14d3471-b8a1-4906-a503-ea9e4ab61181", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "## Exercise 2: Create Tables with Constraints\n", "\n", @@ -154,12 +242,23 @@ }, { "cell_type": "markdown", - "id": "43cc1d95", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "55175eca-57fa-4eda-bd30-203cc34d41ac", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 2.1 — Create the `students` table\n", "\n", - "Create a managed Delta table `edu_dev.silver.students` to store student records with the following columns:\n", + "Create a managed Delta table `edu_dev63853039.silver.students` to store student records with the following columns:\n", "\n", "| Column | Type | Description |\n", "|--------|------|-------------|\n", @@ -170,7 +269,7 @@ "| `enrollment_year` | INT | Year the student enrolled |\n", "| `program` | STRING | Degree program |\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I create a managed Delta table with a primary key constraint in Unity Catalog?\"*\n", "\n", "**Hint:** Add `CONSTRAINT students_pk PRIMARY KEY (student_id)` at the end of the column list." @@ -178,23 +277,55 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "c84aa6e8", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "545e9034-88b2-4640-89f1-b511280c027e", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Create edu_dev.silver.students with a primary key constraint" + "CREATE TABLE IF NOT EXISTS edu_dev63853039.silver.students (\n", + "\n", + " student_id BIGINT NOT NULL,\n", + " first_name STRING, \n", + " last_name STRING, \n", + " email STRING, \n", + " enrolled_date TIMESTAMP, \n", + " program STRING, \n", + " CONSTRAINT students_pk PRIMARY KEY(student_id)\n", + ");" ] }, { "cell_type": "markdown", - "id": "62473788", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "2142e2db-871d-442e-a8cb-018e428970cc", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 2.2 — Create the `courses` table\n", "\n", - "Create `edu_dev.silver.courses` with the following columns:\n", + "Create `edu_dev63853039.silver.courses` with the following columns:\n", "\n", "| Column | Type | Description |\n", "|--------|------|-------------|\n", @@ -203,29 +334,58 @@ "| `department` | STRING | Academic department offering the course |\n", "| `credits` | INT | Number of academic credits |\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"Write a CREATE TABLE statement for a courses table in Databricks Unity Catalog with a primary key constraint.\"*" ] }, { "cell_type": "code", - "execution_count": null, - "id": "12121b41", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "a684072c-2646-4b8d-91ff-eb898d9411c8", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Create edu_dev.silver.courses with a primary key constraint" + "CREATE TABLE IF NOT EXISTS edu_dev63853039.silver.courses (\n", + " course_id BIGINT NOT NULL, \n", + " course_name STRING, \n", + " department STRING, \n", + " credits INT, \n", + " CONSTRAINT courses_pk PRIMARY KEY(course_id)\n", + "); " ] }, { "cell_type": "markdown", - "id": "1a2238fd", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "b0465fe0-40c2-4322-9d9e-dcf776f63955", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 2.3 — Create the `enrollments` table with foreign keys\n", "\n", - "Create `edu_dev.silver.enrollments` to record which students are enrolled in which courses. This table must reference both `students` and `courses` via foreign key constraints.\n", + "Create `edu_dev63853039.silver.enrollments` to record which students are enrolled in which courses. This table must reference both `students` and `courses` via foreign key constraints.\n", "\n", "| Column | Type | Description |\n", "|--------|------|-------------|\n", @@ -235,7 +395,7 @@ "| `semester` | STRING | Academic semester (e.g., `Spring 2023`) |\n", "| `grade` | DECIMAL(4,2) | Numerical grade on a 0.0–10.0 scale |\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I add foreign key constraints referencing other tables when creating a table in Databricks Unity Catalog?\"*\n", "\n", "**Hint:** Use `CONSTRAINT FOREIGN KEY () REFERENCES ..()` for each foreign key." @@ -243,19 +403,51 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "aaf6a330", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "cbed427e-ab8d-41f7-89b7-a5c83975fe0c", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Create edu_dev.silver.enrollments with primary key and two foreign key constraints" + "CREATE TABLE IF NOT EXISTS edu_dev63853039.silver.enrollments (\n", + " enrollment_id BIGINT NOT NULL, \n", + " student_id BIGINT, \n", + " course_id BIGINT, \n", + " semester STRING, \n", + " grade DECIMAL(4,2), \n", + " CONSTRAINT enrollments_pk PRIMARY KEY(enrollment_id),\n", + " CONSTRAINT enrollments_students_fk FOREIGN KEY(student_id) REFERENCES edu_dev63853039.silver.students(student_id),\n", + " CONSTRAINT enrollments_courses_fk FOREIGN KEY(course_id) REFERENCES edu_dev63853039.silver.courses(course_id)\n", + ");" ] }, { "cell_type": "markdown", - "id": "e06d70c9", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "eac7be32-e47e-4ae5-9a7b-b1d802893636", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Sample data — run this cell\n", "\n", @@ -264,14 +456,26 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "3de44411", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "5a612473-233c-4115-a415-2da3a4da9fc4", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", "-- Populate students\n", - "INSERT INTO edu_dev.silver.students VALUES\n", + "INSERT INTO edu_dev63853039.silver.students VALUES\n", "(1001, 'Emma', 'Watson', 'emma.watson@lakeside.edu', 2022, 'Computer Science'),\n", "(1002, 'Liam', 'Ahmed', 'liam.ahmed@lakeside.edu', 2023, 'Data Science'),\n", "(1003, 'Sofia', 'Chen', 'sofia.chen@lakeside.edu', 2021, 'Mathematics'),\n", @@ -284,7 +488,7 @@ "(1010, 'Jake', 'OBrien', 'jake.obrien@lakeside.edu', 2020, 'Computer Science');\n", "\n", "-- Populate courses\n", - "INSERT INTO edu_dev.silver.courses VALUES\n", + "INSERT INTO edu_dev63853039.silver.courses VALUES\n", "(101, 'Introduction to Programming', 'Computer Science', 3),\n", "(102, 'Data Structures', 'Computer Science', 4),\n", "(103, 'Statistics for Data Science', 'Data Science', 3),\n", @@ -293,7 +497,7 @@ "(106, 'Linear Algebra', 'Mathematics', 3);\n", "\n", "-- Populate enrollments\n", - "INSERT INTO edu_dev.silver.enrollments VALUES\n", + "INSERT INTO edu_dev63853039.silver.enrollments VALUES\n", "(1, 1001, 101, 'Spring 2023', 8.50),\n", "(2, 1001, 102, 'Fall 2023', 7.80),\n", "(3, 1002, 103, 'Spring 2023', 9.20),\n", @@ -313,8 +517,19 @@ }, { "cell_type": "markdown", - "id": "4333ac8a", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "497190f2-178f-4375-b070-04575c16b6a7", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "## Exercise 3: Create Views\n", "\n", @@ -323,12 +538,23 @@ }, { "cell_type": "markdown", - "id": "da6bf6d7", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "945011ce-9172-404c-a691-5a918c45884a", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 3.1 — Create a standard view\n", "\n", - "Create a view `edu_dev.silver.vw_student_enrollments` that presents a combined, human-readable view of student enrollment records. It should include:\n", + "Create a view `edu_dev63853039.silver.vw_student_enrollments` that presents a combined, human-readable view of student enrollment records. It should include:\n", "\n", "- Student full name (concatenate `first_name` and `last_name` with a space between them)\n", "- Student `email`\n", @@ -337,7 +563,7 @@ "- `semester`\n", "- `grade`\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"Write a SQL view in Azure Databricks that joins three tables: students, courses, and enrollments\"*\n", "\n", "**Hint:** Use `CONCAT(first_name, ' ', last_name)` or the `||` operator to combine names. You need to join all three silver tables." @@ -345,24 +571,70 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "36092d06", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "f708c615-da8a-49c1-a866-3f3e2b3841ea", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 18" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Create the edu_dev.silver.vw_student_enrollments view\n", - "-- Join students, courses, and enrollments; include full_name, email, course_name, department, semester, grade" + "-- TODO: Create the edu_dev63853039.silver.vw_student_enrollments view\n", + "-- Join students, courses, and enrollments; include full_name, email, course_name, department, semester, grade\n", + "\n", + "DROP TABLE IF EXISTS edu_dev63853039.silver.vw_student_enrollments;\n", + "\n", + "CREATE OR REPLACE VIEW edu_dev63853039.silver.vw_student_enrollments AS (\n", + " SELECT \n", + " CONCAT(s.first_name, ' ', s.last_name) \n", + " AS full_name,\n", + " s.email,\n", + " c.course_name,\n", + " c.department,\n", + " e.semester,\n", + " e.grade\n", + " FROM \n", + " edu_dev63853039.silver.enrollments e \n", + " INNER JOIN \n", + " edu_dev63853039.silver.students s\n", + " ON \n", + " e.student_id = s.student_id\n", + " INNER JOIN \n", + " edu_dev63853039.silver.courses c\n", + " ON \n", + " e.course_id = c.course_id\n", + ");" ] }, { "cell_type": "markdown", - "id": "304bd8cb", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "489032f4-79bb-40b0-8811-6184c425096d", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 3.2 — Create a materialized view for department statistics\n", "\n", - "Create a materialized view `edu_dev.gold.vw_department_enrollment_stats` that pre-computes the following per academic department:\n", + "Create a materialized view `edu_dev63853039.gold.vw_department_enrollment_stats` that pre-computes the following per academic department:\n", "\n", "| Output column | Description |\n", "|--------------|-------------|\n", @@ -371,7 +643,7 @@ "| `avg_grade` | Average grade, rounded to 2 decimal places |\n", "| `distinct_students` | Number of unique students enrolled |\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"What is the syntax for creating a materialized view in Azure Databricks that aggregates data from two joined tables?\"*\n", "\n", "**Hint:** Use `CREATE MATERIALIZED VIEW`, join `enrollments` with `courses`, and use `COUNT()`, `ROUND(AVG(...), 2)`, and `COUNT(DISTINCT ...)`.\n", @@ -382,19 +654,58 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "589ad9da", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "b607649d-b674-4d19-b938-02c58e26869c", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 20" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Create the edu_dev.gold.vw_department_enrollment_stats materialized view" + "-- TODO: Create the edu_dev63853039.gold.vw_department_enrollment_stats materialized view\n", + "\n", + "CREATE MATERIALIZED VIEW IF NOT EXISTS edu_dev63853039.gold.vw_department_enrollment_stats AS (\n", + " SELECT \n", + " c.department,\n", + " COUNT(*) AS total_enrollments,\n", + " ROUND(AVG(e.grade), 2) AS avg_grade,\n", + " COUNT(DISTINCT e.student_id) AS distinct_students\n", + " FROM \n", + " edu_dev63853039.silver.enrollments e\n", + " INNER JOIN \n", + " edu_dev63853039.silver.courses c\n", + " ON \n", + " e.course_id = c.course_id\n", + " GROUP BY \n", + " c.department\n", + ");" ] }, { "cell_type": "markdown", - "id": "4ab4b66e", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "fc765d60-2062-416e-9db2-62509ffdcc14", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "**✅ Test your materialized view** — Run the following query to verify it returns per-department aggregates:\n", "\n", @@ -403,19 +714,42 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "d3b2354e", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "f38a40de-a840-4b78-9af6-9230ae7584cb", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", - "SELECT * FROM edu_dev.gold.vw_department_enrollment_stats;" + "SELECT * FROM edu_dev63853039.gold.vw_department_enrollment_stats;" ] }, { "cell_type": "markdown", - "id": "77ee534b", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "fc9a15af-bfb9-4244-b301-23b82b2fdfc8", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "## Exercise 4: Create a Volume and Read Files\n", "\n", @@ -424,14 +758,25 @@ }, { "cell_type": "markdown", - "id": "a8fca075", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "be2a335a-742a-49e5-8b43-5ebaee21e67c", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 4.1 — Create a managed volume\n", "\n", - "Create a managed volume named `raw_files` in the `edu_dev.bronze` schema. This will serve as the landing area for raw data files.\n", + "Create a managed volume named `raw_files` in the `edu_dev63853039.bronze` schema. This will serve as the landing area for raw data files.\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I create a managed volume in Unity Catalog using a SQL statement?\"*\n", "\n", "**Hint:** Use `CREATE VOLUME IF NOT EXISTS ..`." @@ -439,19 +784,43 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "c21a6290", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "c67111e9-a135-47e9-a880-9e643608a798", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Create the raw_files managed volume in edu_dev.bronze" + "-- TODO: Create the raw_files managed volume in edu_dev63853039.bronze\n", + "\n", + "CREATE VOLUME IF NOT EXISTS edu_dev63853039.bronze.raw_files" ] }, { "cell_type": "markdown", - "id": "5252fe12", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "d2132d9c-6c9c-462b-abfd-a6be4afbae5c", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 4.2 — Write a CSV file to the volume\n", "\n", @@ -460,9 +829,20 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "6c348308", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "a047fe3b-0fb4-4ca3-92ac-85789743bd24", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "# Write a sample student CSV file to the volume — run this cell without modification\n", @@ -478,20 +858,31 @@ "1009,Mei,Lindqvist,mei.lindqvist@lakeside.edu,2023,Mathematics\n", "1010,Jake,OBrien,jake.obrien@lakeside.edu,2020,Computer Science\"\"\"\n", "\n", - "dbutils.fs.put(\"/Volumes/edu_dev/bronze/raw_files/students.csv\", csv_content, overwrite=True)\n", - "print(\"students.csv written to /Volumes/edu_dev/bronze/raw_files/\")" + "dbutils.fs.put(\"/Volumes/edu_dev63853039/bronze/raw_files/students.csv\", csv_content, overwrite=True)\n", + "print(\"students.csv written to /Volumes/edu_dev63853039/bronze/raw_files/\")" ] }, { "cell_type": "markdown", - "id": "6823ffb1", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "d7554b95-5c1c-4cd9-8b49-a7d779f40c67", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 4.3 — Read the CSV from the volume into a DataFrame\n", "\n", - "Read the CSV file you just placed in the volume into a Spark DataFrame and display its contents. The file is at `/Volumes/edu_dev/bronze/raw_files/students.csv`.\n", + "Read the CSV file you just placed in the volume into a Spark DataFrame and display its contents. The file is at `/Volumes/edu_dev63853039/bronze/raw_files/students.csv`.\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I read a CSV file with a header row from a Unity Catalog volume path in PySpark?\"*\n", "\n", "**Hint:** Use `spark.read.csv(path, header=True, inferSchema=True)` then call `.display()` or `.show()` on the result." @@ -499,18 +890,41 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "4f2b6ef8", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "2e2ecc7c-205e-4959-a66e-ece8c2b5a98c", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ - "# TODO: Read /Volumes/edu_dev/bronze/raw_files/students.csv into a Spark DataFrame and display it" + "# TODO: Read /Volumes/edu_dev63853039/bronze/raw_files/students.csv into a Spark DataFrame and display it\n", + "spark.read.csv('/Volumes/edu_dev63853039/bronze/raw_files/students.csv', header=True, inferSchema=True).display()" ] }, { "cell_type": "markdown", - "id": "d9848d3c", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "f6988ea6-f83f-4725-94ca-e19b2c64a3dc", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "## Exercise 5: Create a Reusable SQL Function\n", "\n", @@ -519,12 +933,23 @@ }, { "cell_type": "markdown", - "id": "efc0eb82", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "cf215b86-565c-4e54-ae30-9db6468e76c2", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 5.1 — Create a grade classification function\n", "\n", - "Create a SQL scalar function `edu_dev.silver.get_grade_classification` that accepts a `grade DECIMAL(4,2)` and returns a `STRING` letter classification based on the scale used at Lakeside University:\n", + "Create a SQL scalar function `edu_dev63853039.silver.get_grade_classification` that accepts a `grade DECIMAL(4,2)` and returns a `STRING` letter classification based on the scale used at Lakeside University:\n", "\n", "| Grade range | Classification |\n", "|-------------|----------------|\n", @@ -534,7 +959,7 @@ "| ≥ 4.0 | `'D'` |\n", "| < 4.0 | `'F'` |\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I create a SQL scalar function in Unity Catalog that uses a CASE expression to return a letter grade based on a numeric score?\"*\n", "\n", "**Hint:** Use `CREATE FUNCTION ... RETURNS STRING RETURN CASE WHEN ... END`." @@ -542,61 +967,140 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "abd7299f", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "4e00efc2-8578-4d14-9191-aca2527edc74", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Create the edu_dev.silver.get_grade_classification function" + "-- TODO: Create the edu_dev63853039.silver.get_grade_classification function\n", + "\n", + "CREATE OR REPLACE FUNCTION get_grade_classification(grade\n", + "DECIMAL(4, 2))\n", + "RETURNS STRING\n", + "RETURN CASE\n", + " WHEN grade >= 8.5 THEN 'A'\n", + " WHEN grade >= 7.0 THEN 'B'\n", + " WHEN grade >= 5.5 THEN 'C'\n", + " WHEN grade >= 4.0 THEN 'D'\n", + " ELSE 'F'\n", + "END;" ] }, { "cell_type": "markdown", - "id": "c96c404a", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "a1a7118b-7f8b-4741-a35b-05fcfbca6d46", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 5.2 — Test the function\n", "\n", - "Query `edu_dev.silver.enrollments` and apply your new function to produce a `grade_classification` column for each row. Include: `enrollment_id`, `student_id`, `course_id`, `semester`, `grade`, and `grade_classification`.\n", + "Query `edu_dev63853039.silver.enrollments` and apply your new function to produce a `grade_classification` column for each row. Include: `enrollment_id`, `student_id`, `course_id`, `semester`, `grade`, and `grade_classification`.\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I call a Unity Catalog user-defined function in a SELECT statement in Azure Databricks?\"*" ] }, { "cell_type": "code", - "execution_count": null, - "id": "60f10718", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "27ab739a-3b97-4770-b159-26d561be1a1c", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Query edu_dev.silver.enrollments and call get_grade_classification on the grade column" + "-- TODO: Query edu_dev63853039.silver.enrollments and call get_grade_classification on the grade column\n", + "\n", + "SELECT \n", + " enrollment_id, \n", + " student_id, \n", + " course_id,\n", + " semester,\n", + " grade,\n", + " get_grade_classification(grade) AS grade_classification\n", + "FROM \n", + " edu_dev63853039.silver.enrollments;\n", + " \n" ] }, { "cell_type": "markdown", - "id": "ed144885", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "ccef6d90-975b-442d-b5c3-7d92c3c855d5", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "## Exercise 6: DDL Operations\n", "\n", "Unity Catalog objects evolve over time as requirements change. In this exercise, you use `ALTER` statements to:\n", "1. Extend the `students` table with a new column.\n", - "2. Apply governance **tags** to the `edu_dev` catalog." + "2. Apply governance **tags** to the `edu_dev63853039` catalog." ] }, { "cell_type": "markdown", - "id": "c6b30ba9", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "4b8e4332-a4df-4e0e-b126-d0bdae092aec", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 6.1 — Add a column to the students table\n", "\n", - "The student registration system has been updated to capture phone numbers. Add a nullable `phone_number` column of type `STRING` to `edu_dev.silver.students`.\n", + "The student registration system has been updated to capture phone numbers. Add a nullable `phone_number` column of type `STRING` to `edu_dev63853039.silver.students`.\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I add a new nullable column to an existing Delta table in Azure Databricks using SQL?\"*\n", "\n", "**Hint:** Use `ALTER TABLE`." @@ -604,23 +1108,49 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "85c7df17", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "461fd0b3-e8fd-40cb-8127-ac459821105e", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Add a phone_number (STRING) column to edu_dev.silver.students" + "-- TODO: Add a phone_number (STRING) column to edu_dev63853039.silver.students\n", + "\n", + "ALTER TABLE edu_dev63853039.silver.students\n", + "ADD COLUMN phone_number STRING" ] }, { "cell_type": "markdown", - "id": "706ba53d", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "d7eb1f67-364e-454f-bfb0-16e35961221d", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 6.2 — Apply tags to the catalog\n", "\n", - "Apply the following tags to the `edu_dev` catalog to support governance and discoverability:\n", + "Apply the following tags to the `edu_dev63853039` catalog to support governance and discoverability:\n", "\n", "| Tag key | Tag value |\n", "|---------|----------|\n", @@ -628,25 +1158,55 @@ "| `university` | `'lakeside'` |\n", "| `data_classification` | `'internal'` |\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I set metadata tags on a Unity Catalog catalog using the ALTER CATALOG statement in Databricks?\"*" ] }, { "cell_type": "code", - "execution_count": null, - "id": "bae85a69", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "681f838d-681a-4e71-bd6d-07f6cbf3a46e", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 39" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Apply the three tags to the edu_dev catalog using ALTER CATALOG ... SET TAGS" + "-- TODO: Apply the three tags to the edu_dev63853039 catalog using ALTER CATALOG ... SET TAGS\n", + "\n", + "ALTER CATALOG edu_dev63853039\n", + "SET TAGS (\n", + " 'environment' = 'education',\n", + " 'university' = 'lakeside',\n", + " 'data_classification' = 'internal'\n", + ")" ] }, { "cell_type": "markdown", - "id": "338f23f8", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "4acd1fc3-97b3-4d32-b730-26a9bd7b5ffa", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 6.3 — Verify your work\n", "\n", @@ -655,58 +1215,117 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "178ae537", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "f51485bd-a21a-4d69-b815-80ae0a822029", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", "-- Show all tables and views in the silver schema\n", - "SHOW TABLES IN edu_dev.silver;" + "SHOW TABLES IN edu_dev63853039.silver;" ] }, { "cell_type": "code", - "execution_count": null, - "id": "24d433ae", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "68b77758-4e42-48d6-b6e5-4d310a89bf7b", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", "-- Describe the students table — verify the new phone_number column is present\n", - "DESCRIBE TABLE edu_dev.silver.students;" + "DESCRIBE TABLE edu_dev63853039.silver.students;" ] }, { "cell_type": "code", - "execution_count": null, - "id": "44e96056", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "bbadd540-0ecf-467b-aa16-7314ad053899", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", "-- Show catalog details\n", - "DESCRIBE CATALOG EXTENDED edu_dev;" + "DESCRIBE CATALOG EXTENDED edu_dev63853039;" ] }, { "cell_type": "code", - "execution_count": null, - "id": "28dab7de", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "47b34975-b8fb-4526-ba4f-8325f5c218c2", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql \n", "-- Query tags via information schema (structured way)\n", "SELECT *\n", "FROM system.information_schema.catalog_tags\n", - "WHERE catalog_name = 'edu_dev';" + "WHERE catalog_name = 'edu_dev63853039';" ] }, { "cell_type": "markdown", - "id": "5b9e4d0e", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "c58cba83-b616-4c8f-b8fe-6fd2bdb0217f", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "---\n", "\n", @@ -719,10 +1338,40 @@ } ], "metadata": { + "application/vnd.databricks.v1+notebook": { + "computePreferences": { + "hardware": { + "accelerator": null, + "gpuPoolId": null, + "memory": null + }, + "software": { + "pinSparkToX86": null + } + }, + "dashboards": [], + "environmentMetadata": { + "base_environment": "", + "environment_version": "5" + }, + "inputWidgetPreferences": null, + "language": "python", + "notebookMetadata": { + "mostRecentlyExecutedCommandWithImplicitDF": { + "commandId": -1, + "dataframes": [ + "_sqldf" + ] + }, + "pythonIndentUnit": 4 + }, + "notebookName": "03-create-and-organize-objects-in-unity-catalog", + "widgets": {} + }, "language_info": { "name": "python" } }, "nbformat": 4, - "nbformat_minor": 5 -} + "nbformat_minor": 0 +} \ No newline at end of file From b55a53cd3a88455f6781efb9257e2ad06e21ad5b Mon Sep 17 00:00:00 2001 From: "user1-63874195@lodsprodmca.onmicrosoft.com" Date: Fri, 31 Jul 2026 19:03:06 +0000 Subject: [PATCH 4/5] adding 4th exercise --- .../04-secure-unity-catalog-objects.ipynb | 714 ++++++++++++++---- 1 file changed, 571 insertions(+), 143 deletions(-) diff --git a/Allfiles/04-secure-unity-catalog-objects.ipynb b/Allfiles/04-secure-unity-catalog-objects.ipynb index af16699..8a3ed0b 100644 --- a/Allfiles/04-secure-unity-catalog-objects.ipynb +++ b/Allfiles/04-secure-unity-catalog-objects.ipynb @@ -2,8 +2,16 @@ "cells": [ { "cell_type": "markdown", - "id": "246cbfa5", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "89d52cf3-35b6-48d5-8389-3b7bfd0ddf16", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "## Exercise 1: Access Control\n", "\n", @@ -14,15 +22,23 @@ }, { "cell_type": "markdown", - "id": "7109cd52", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "c27b7625-8cf2-4dc7-b412-1a26451f9b60", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 1.1 — Create the catalog, schema, and customers table\n", "\n", "The cell below sets up all the objects needed for this exercise:\n", - "- Creates a catalog called `retail_catalog` to hold all NorthMart data assets.\n", + "- Creates a catalog called `retail_catalog63874195` to hold all NorthMart data assets.\n", "- Creates a schema called `security_lab` inside it.\n", - "- Creates a managed Delta table `retail_catalog.security_lab.customers` and populates it with six sample customer records. These rows will be used throughout the rest of this exercise to demonstrate row filtering and column masking.\n", + "- Creates a managed Delta table `retail_catalog63874195.security_lab.customers` and populates it with six sample customer records. These rows will be used throughout the rest of this exercise to demonstrate row filtering and column masking.\n", "\n", "| customer_id | customer_name | email | region | total_spend | tier |\n", "|-------------|-----------------|-------------------------|--------|-------------|----------|\n", @@ -38,9 +54,20 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "cb0ec373", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "c3f1f958-9a9f-4ca6-8607-b1e8dc9e28a5", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "# Since no default storage is enabled, we are inheriting the storage path from the default catalog's root.\n", @@ -57,7 +84,7 @@ "print (f\"Storage root: {storage_root}\")\n", "\n", "spark.sql(f\"\"\"\n", - " CREATE CATALOG IF NOT EXISTS retail_catalog\n", + " CREATE CATALOG IF NOT EXISTS retail_catalog63874195\n", " MANAGED LOCATION '{storage_root}'\n", " COMMENT 'Catalog for NorthMart Retail data assets'\n", "\"\"\")" @@ -65,16 +92,28 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "6bbd6143", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "05a04dbf-9f24-4421-a355-4d91c904de4d", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", - "CREATE SCHEMA IF NOT EXISTS retail_catalog.security_lab\n", + "CREATE SCHEMA IF NOT EXISTS retail_catalog63874195.security_lab\n", " COMMENT 'Schema used for the security lab exercises';\n", "\n", - "CREATE OR REPLACE TABLE retail_catalog.security_lab.customers AS\n", + "CREATE OR REPLACE TABLE retail_catalog63874195.security_lab.customers AS\n", "SELECT * FROM VALUES\n", " (1, 'Alice Johnson', 'alice@northmart.com', 'North', 5200.00, 'Standard'),\n", " (2, 'Bob Martinez', 'bob@northmart.com', 'South', 8100.00, 'Premium'),\n", @@ -87,18 +126,26 @@ }, { "cell_type": "markdown", - "id": "e28ea3dd", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "034db656-bd70-4761-bab3-3178e6cc65c2", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 1.2 — Grant permissions to the retail-analysts group\n", "\n", "The `retail-analysts` group needs to query the `customers` table. Grant the necessary privileges at all required levels of the Unity Catalog hierarchy:\n", "\n", - "- `USE CATALOG` on `retail_catalog`\n", - "- `USE SCHEMA` on `retail_catalog.security_lab`\n", - "- `SELECT` on `retail_catalog.security_lab.customers`\n", + "- `USE CATALOG` on `retail_catalog63874195`\n", + "- `USE SCHEMA` on `retail_catalog63874195.security_lab`\n", + "- `SELECT` on `retail_catalog63874195.security_lab.customers`\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"What SQL grants are needed to allow a Databricks group to SELECT from a Unity Catalog table?\"*\n", "\n", "**Hint:** Use `GRANT ON TO \\`retail-analysts\\``." @@ -106,25 +153,49 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "7d849128", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "3452e891-b644-4b57-b582-f3d462fff5df", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Verify the granted permissions" + "-- TODO: Verify the granted permissions\n", + "\n", + "GRANT ALL PRIVILEGES ON CATALOG retail_catalog63874195 TO `retail-analysts`;\n", + "GRANT USE SCHEMA ON SCHEMA retail_catalog63874195.security_lab TO `retail-analysts`;\n", + "GRANT SELECT ON TABLE retail_catalog63874195.security_lab.customers TO `retail-analysts`;" ] }, { "cell_type": "markdown", - "id": "bed7d1e5", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "8105adca-4565-4a47-ab58-f616e80015ce", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 1.3 — Verify the granted permissions\n", "\n", "Verify that the privileges were correctly assigned. Use `SHOW GRANTS` to display all privileges on the `customers` table.\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I use SHOW GRANTS to verify permissions on a Unity Catalog table?\"*\n", "\n", "**Expected result:** You should see `retail-analysts` listed with the `SELECT` privilege." @@ -132,19 +203,41 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "4a4dc08c", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "0403d8f2-86c4-4c05-90d3-9b0d2bedad8e", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Show all grants on retail_catalog.security_lab.customers" + "-- TODO: Show all grants on retail_catalog63874195.security_lab.customers\n", + "\n", + "SHOW GRANTS ON TABLE retail_catalog63874195.security_lab.customers" ] }, { "cell_type": "markdown", - "id": "54004c07", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "58417853-8983-4f8d-8e9d-76692dd1315a", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "## Exercise 2: Row Filtering\n", "\n", @@ -155,18 +248,26 @@ }, { "cell_type": "markdown", - "id": "5c40704e", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "3fe59099-d4a5-44e0-9117-a86580226d04", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 2.1 — Create a row filter function\n", "\n", - "Create a SQL function `retail_catalog.security_lab.region_filter` that takes a `region` parameter of type `STRING` and returns `TRUE` if:\n", + "Create a SQL function `retail_catalog63874195.security_lab.region_filter` that takes a `region` parameter of type `STRING` and returns `TRUE` if:\n", "- the `region` is `'North'`, **OR**\n", "- the current user is your own account (replace `your@email.com` with your actual email).\n", "\n", "This means: unless you are the privileged user, only `North` region rows will be visible.\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I create a row filter function in Databricks Unity Catalog that uses current_user() to control access?\"*\n", "\n", "**Hint:** Use `CREATE OR REPLACE FUNCTION ... RETURN `." @@ -174,26 +275,51 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "1aa7f33a", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "3aac1aca-fcce-49f6-819d-5a0f1388b8f8", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Create the region_filter function in retail_catalog.security_lab\n", - "-- Replace 'your@email.com' with your actual Databricks user email" + "-- TODO: Create the region_filter function in retail_catalog63874195.security_lab\n", + "-- Replace 'your@email.com' with your actual Databricks user email\n", + "\n", + "CREATE OR REPLACE FUNCTION retail_catalog63874195.security_lab.region_filter(region STRING)\n", + "RETURNS BOOLEAN \n", + "RETURN (region = 'North' OR current_user() = 'user1-63874195@lodsprodmca.onmicrosoft.com'\n", + ");" ] }, { "cell_type": "markdown", - "id": "f8c13df8", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "b5cd7b09-be18-454c-84f5-77333b83a39c", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 2.2 — Apply the row filter to the customers table\n", "\n", "Attach the `region_filter` function to the `customers` table so that it is automatically applied to every query against that table.\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I set a row filter on a Unity Catalog table in Azure Databricks?\"*\n", "\n", "**Hint:** Use `ALTER TABLE ... SET ROW FILTER ON ()`." @@ -201,19 +327,42 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "44a27dc5", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "918736d0-4782-4546-8e1f-e5aca51f07be", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Apply region_filter to retail_catalog.security_lab.customers on the region column" + "-- TODO: Apply region_filter to retail_catalog63874195.security_lab.customers on the region column\n", + "\n", + "ALTER TABLE retail_catalog63874195.security_lab.customers\n", + "SET ROW FILTER retail_catalog63874195.security_lab.region_filter ON(region);\n" ] }, { "cell_type": "markdown", - "id": "064a0a5c", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "4e1c00df-b483-4744-8929-78b7c7c2333c", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 2.3 — Test the row filter\n", "\n", @@ -221,42 +370,72 @@ "\n", "If you had logged in as someone else (not your email), you would only see the 2 `North` region rows.\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How can I verify which user is currently running a query in Databricks using SQL?\"*" ] }, { "cell_type": "code", - "execution_count": null, - "id": "d8081299", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "b66d2c2b-7512-49fe-b366-7a77037b3dc5", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", "-- Verify the current user\n", "SELECT current_user();\n", "\n", - "-- TODO: Query retail_catalog.security_lab.customers and observe the results" + "-- TODO: Query retail_catalog63874195.security_lab.customers and observe the results\n", + "\n", + "SELECT * FROM retail_catalog63874195.security_lab.customers;\n" ] }, { "cell_type": "markdown", - "id": "07c44057", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "cf9eaf07-ce75-46fc-99bf-54de53d42e6a", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "➡️ Tip: update the email address in the row filter function you created earlier, and test again. Observe the output when querying the customers table." ] }, { "cell_type": "markdown", - "id": "e2ae227c", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "37dd7f85-a769-4f3c-a532-fe28396f0d44", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 2.4 — Remove the row filter\n", "\n", "Now remove the row filter from the `customers` table. After removing it, query the table again to confirm **all 6 rows** are now returned regardless of user context.\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I remove a row filter from a Unity Catalog table?\"*\n", "\n", "**Hint:** Use `ALTER TABLE ... DROP ROW FILTER`." @@ -264,21 +443,44 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "1ade042c", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "719f44ea-4111-498b-9a8c-40e312c45dc5", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Remove the row filter from retail_catalog.security_lab.customers\n", + "-- TODO: Remove the row filter from retail_catalog63874195.security_lab.customers\n", + "ALTER TABLE retail_catalog63874195.security_lab.customers DROP ROW FILTER;\n", "\n", - "-- TODO: Query the table to confirm all 6 rows are visible" + " \n", + "-- TODO: Create the mask_email function in retail_catalog63874195.security_lab\n", + "SELECT * FROM retail_catalog63874195.security_lab.customers;" ] }, { "cell_type": "markdown", - "id": "32632e1e", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "1a9021db-4a77-4d9b-8cce-a4211002e6be", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "## Exercise 3: Column Masking\n", "\n", @@ -287,16 +489,24 @@ }, { "cell_type": "markdown", - "id": "60241d37", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "df68fcab-38e8-44e9-998a-4e0675f18fd4", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 3.1 — Create a column mask function\n", "\n", - "Create a SQL function `retail_catalog.security_lab.mask_email` that takes an `email` parameter of type `STRING` and:\n", + "Create a SQL function `retail_catalog63874195.security_lab.mask_email` that takes an `email` parameter of type `STRING` and:\n", "- Returns the **full email** if the current user is your own account.\n", "- Returns a **masked value** (e.g. `'***@***.com'`) for all other users.\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I create a column masking function in Databricks Unity Catalog that uses current_user() to decide whether to return the real or masked value?\"*\n", "\n", "**Hint:** Use a `CASE WHEN current_user() = '...' THEN email ELSE '***@***.com' END` pattern inside the function body." @@ -304,26 +514,51 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "f7e64496", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "5ce679c5-046b-475c-b814-024440f12b64", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Create the mask_email function in retail_catalog.security_lab\n", - "-- Replace 'your@email.com' with your actual Databricks user email" + "-- TODO: Create the mask_email function in retail_catalog63874195.security_lab\n", + "-- Replace 'your@email.com' with your actual Databricks user email\n", + "\n", + "CREATE OR REPLACE FUNCTION retail_catalog63874195.security_lab.mask_email(email STRING)\n", + "RETURNS STRING\n", + "RETURN CASE WHEN current_user = 'user1-63874195@lodsprodmca.onmicrosoft.com'\n", + "THEN email ELSE '***@***.com' END;" ] }, { "cell_type": "markdown", - "id": "f0f0d2f8", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "da0d4fa7-f5a9-45af-af89-008f937e8fdf", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 3.2 — Apply the column mask to the email column\n", "\n", "Apply the `mask_email` function as a column mask on the `email` column of the `customers` table.\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I set a column mask on a Unity Catalog table column in Azure Databricks using SQL?\"*\n", "\n", "**Hint:** Use `ALTER TABLE ... ALTER COLUMN email SET MASK `." @@ -331,19 +566,42 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "7c3bd607", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "c83e8bdd-f8e8-43da-afce-59d89b004af6", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Apply mask_email to the email column of retail_catalog.security_lab.customers" + "-- TODO: Apply mask_email to the email column of retail_catalog63874195.security_lab.customers\n", + "\n", + "ALTER TABLE retail_catalog63874195.security_lab.customers\n", + "ALTER COLUMN email SET MASK retail_catalog63874195.security_lab.mask_email" ] }, { "cell_type": "markdown", - "id": "06673f52", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "fdc63555-9014-4165-8fa4-c0807909b838", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 3.3 — Test the column mask\n", "\n", @@ -354,25 +612,46 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "9557db8c", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "16ae39da-525e-476d-a804-8a282c6b2be6", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Query retail_catalog.security_lab.customers and observe the email column" + "-- TODO: Query retail_catalog63874195.security_lab.customers and observe the email column\n", + "SELECT * FROM retail_catalog63874195.security_lab.customers" ] }, { "cell_type": "markdown", - "id": "b268d979", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "6a2377ea-ab09-43ac-a29f-4c34f74c3c03", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 3.4 — Remove the column mask\n", "\n", "Remove the column mask from the `email` column, then query the table again to confirm emails are now fully visible to all users.\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I drop a column mask from a Unity Catalog table column?\"*\n", "\n", "**Hint:** Use `ALTER TABLE ... ALTER COLUMN email DROP MASK`." @@ -380,21 +659,43 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "7d425b20", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "c329740e-d69e-40f4-831e-fd5ccf6b4ee1", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", "-- TODO: Remove the column mask from the email column\n", - "\n", - "-- TODO: Query the table to confirm email addresses are now unmasked" + "ALTER TABLE retail_catalog63874195.security_lab.customers\n", + "ALTER COLUMN email DROP MASK;\n", + "-- TODO: Query the table to confirm email addresses are now unmasked\n", + "SELECT * FROM retail_catalog63874195.security_lab.customers" ] }, { "cell_type": "markdown", - "id": "75a2120a", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "b216559c-c988-4f7a-a738-007fbf60c949", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "## Exercise 4: Azure Key Vault Secrets\n", "\n", @@ -405,14 +706,22 @@ }, { "cell_type": "markdown", - "id": "bc54e26e", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "d5406ec4-8d3a-4323-92d5-f3b2dc187922", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 4.1 — List available secrets in the scope\n", "\n", "List all secrets available in the `retail-kv-scope` secret scope to confirm that `loyalty-api-key` is accessible.\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I list the secrets in a Databricks secret scope using dbutils?\"*\n", "\n", "**Hint:** Use `dbutils.secrets.list(\"scope-name\")`. This returns key names only — never secret values." @@ -420,18 +729,38 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "7b36e969", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "981c6f36-d0c6-4307-8a59-18a755dd7c88", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ - "# TODO: List all secrets in the retail-kv-scope and display the output" + "# TODO: List all secrets in the retail-kv-scope and display the output\n", + "dbutils.secrets.list(\"retail-kv-scope\")" ] }, { "cell_type": "markdown", - "id": "49ba2ffb", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "575831f4-a306-4386-8879-33ae67caddd4", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 4.2 — Retrieve the API key securely in Python\n", "\n", @@ -439,50 +768,95 @@ "\n", "Observe how Azure Databricks **automatically redacts** the value in the notebook output — you will see `[REDACTED]` instead of the actual key.\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I retrieve a secret from a Databricks secret scope using dbutils.secrets.get and why is the output redacted?\"*" ] }, { "cell_type": "code", - "execution_count": null, - "id": "47000f12", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "3b519b1b-4b98-49aa-90e9-0895863d20b1", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "# TODO: Retrieve the loyalty-api-key from retail-kv-scope\n", - "# TODO: Print the api_key variable and observe that the output is [REDACTED]" + "\n", + "api_key = dbutils.secrets.get(\"retail-kv-scope\", \"loyalty-api-key\")\n", + "\n", + "# TODO: Print the api_key variable and observe that the output is [REDACTED]\n", + "print(api_key)" ] }, { "cell_type": "markdown", - "id": "5d8d2940", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "2062d8dd-542c-442e-b927-bbd8a55aef43", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 4.3 — Use the secret to simulate a connection string\n", "\n", "Construct a connection string that embeds the API key, simulating how it would be used to connect to NorthMart's loyalty platform. Print the connection string and observe that the secret value is still redacted.\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I safely use a Databricks secret inside an f-string for a connection string without exposing the value?\"*" ] }, { "cell_type": "code", - "execution_count": null, - "id": "61c36ca0", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "9b94e389-73ac-49b9-b207-28af7d2f5492", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "# TODO: Build a connection string using the api_key variable\n", "# Example format: \"https://loyalty.northmart.com/api?key=\"\n", - "# TODO: Print the connection string" + "\n", + "connection_string = f\"https://loyalty.northmart.com/api?key={api_key}\"\n", + "# TODO: Print the connection string\n", + "print(connection_string)" ] }, { "cell_type": "markdown", - "id": "0b3f910f", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "7a5b1dfc-2b08-4bbc-b0fd-432b64ead4bc", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 4.4 — Retrieve the secret using SQL\n", "\n", @@ -490,25 +864,46 @@ "\n", "Note that the output is also redacted in SQL results.\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I use the secret() function in Databricks SQL to retrieve a secret from a scope?\"*" ] }, { "cell_type": "code", - "execution_count": null, - "id": "c7cb70ca", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "27afa2d8-cb47-4f6b-a840-08ac7aee8089", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Use the secret() SQL function to retrieve loyalty-api-key from retail-kv-scope" + "-- TODO: Use the secret() SQL function to retrieve loyalty-api-key from retail-kv-scope\n", + "SELECT secret('retail-kv-scope','loyalty-api-key')" ] }, { "cell_type": "markdown", - "id": "e8c931f1", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "64d34933-9eea-4f18-82fd-43cec5c39427", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "## Cleanup\n", "\n", @@ -517,28 +912,61 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "0e4e500a", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "816389ad-198b-4cb8-b084-0fa4573d65ca", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", "-- Remove all lab objects\n", - "ALTER TABLE retail_catalog.security_lab.customers DROP ROW FILTER;\n", - "ALTER TABLE retail_catalog.security_lab.customers ALTER COLUMN email DROP MASK;\n", - "DROP TABLE IF EXISTS retail_catalog.security_lab.customers;\n", - "DROP FUNCTION IF EXISTS retail_catalog.security_lab.region_filter;\n", - "DROP FUNCTION IF EXISTS retail_catalog.security_lab.mask_email;\n", - "DROP SCHEMA IF EXISTS retail_catalog.security_lab;\n", - "DROP CATALOG IF EXISTS retail_catalog CASCADE;" + "ALTER TABLE retail_catalog63874195.security_lab.customers DROP ROW FILTER;\n", + "ALTER TABLE retail_catalog63874195.security_lab.customers ALTER COLUMN email DROP MASK;\n", + "DROP TABLE IF EXISTS retail_catalog63874195.security_lab.customers;\n", + "DROP FUNCTION IF EXISTS retail_catalog63874195.security_lab.region_filter;\n", + "DROP FUNCTION IF EXISTS retail_catalog63874195.security_lab.mask_email;\n", + "DROP SCHEMA IF EXISTS retail_catalog63874195.security_lab;\n", + "DROP CATALOG IF EXISTS retail_catalog63874195 CASCADE;" ] } ], "metadata": { + "application/vnd.databricks.v1+notebook": { + "computePreferences": null, + "dashboards": [], + "environmentMetadata": { + "base_environment": "", + "environment_version": "5" + }, + "inputWidgetPreferences": null, + "language": "python", + "notebookMetadata": { + "mostRecentlyExecutedCommandWithImplicitDF": { + "commandId": 4629336142089078, + "dataframes": [ + "_sqldf" + ] + }, + "pythonIndentUnit": 4 + }, + "notebookName": "04-secure-unity-catalog-objects", + "widgets": {} + }, "language_info": { "name": "python" } }, "nbformat": 4, - "nbformat_minor": 5 -} + "nbformat_minor": 0 +} \ No newline at end of file From 941c72e82968427ae98b3153a9f408055114b661 Mon Sep 17 00:00:00 2001 From: "user1-63885547@lodsprodmca.onmicrosoft.com" Date: Sat, 1 Aug 2026 19:06:37 +0000 Subject: [PATCH 5/5] adding 5th exercise --- .../05-govern-unity-catalog-objects.ipynb | 751 ++++++++++++++---- 1 file changed, 606 insertions(+), 145 deletions(-) diff --git a/Allfiles/05-govern-unity-catalog-objects.ipynb b/Allfiles/05-govern-unity-catalog-objects.ipynb index f8fbe1f..2d62b00 100644 --- a/Allfiles/05-govern-unity-catalog-objects.ipynb +++ b/Allfiles/05-govern-unity-catalog-objects.ipynb @@ -2,8 +2,16 @@ "cells": [ { "cell_type": "markdown", - "id": "5ce4abca", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "fc2d7e7e-0450-40fe-8b20-75c03074bcf2", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "## Exercise 1: Set Up the AutoSphere Data Platform\n", "\n", @@ -14,8 +22,16 @@ }, { "cell_type": "markdown", - "id": "dc2a60c0", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "8f0966c1-e3fc-458e-9b49-7bf1b5a9c12e", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Setup — Create the catalog, schema, and tables\n", "\n", @@ -31,15 +47,26 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "4ad1a39f", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "ba5fe687-96bc-4c93-b771-f11e5c051e4f", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 3" + } + }, "outputs": [], "source": [ "# Since no default storage is enabled, we are inheriting the storage path from the default catalog's root.\n", "# Use the current catalog to reliably find the workspace default catalog,\n", "# regardless of its naming convention.\n", - "default_catalog = spark.catalog.currentCatalog()\n", + "default_catalog = [c.catalog for c in spark.sql(\"SHOW CATALOGS\").collect() if c.catalog.startswith(\"adb_\")][0]\n", "\n", "storage_root = (\n", " spark.sql(f\"DESCRIBE CATALOG EXTENDED {default_catalog}\")\n", @@ -50,7 +77,7 @@ "print (f\"Storage root: {storage_root}\")\n", "\n", "spark.sql(f\"\"\"\n", - " CREATE CATALOG IF NOT EXISTS automotive_catalog\n", + " CREATE CATALOG IF NOT EXISTS automotive_catalog2\n", " MANAGED LOCATION '{storage_root}'\n", " COMMENT 'Connected vehicle platform catalog for AutoSphere AG — stores registration, telemetry, and service data.'\n", "\"\"\")" @@ -58,52 +85,84 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "8e55df83", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "eec52119-9b86-4f40-8097-fa9bdf875a81", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 4" + } + }, "outputs": [], "source": [ - "%sql\n", - "CREATE SCHEMA IF NOT EXISTS automotive_catalog.governance_lab\n", - " COMMENT 'Governance lab schema for hands-on Unity Catalog exercises.';\n", + "spark.sql(\"USE CATALOG automotive_catalog2\")\n", + "\n", + "spark.sql(\"\"\"\n", + " CREATE SCHEMA IF NOT EXISTS automotive_catalog2.governance_lab\n", + " COMMENT 'Governance lab schema for hands-on Unity Catalog exercises.'\n", + "\"\"\")\n", + "print(\"✓ Schema created\")\n", "\n", - "-- Create the customer_registrations table\n", - "CREATE TABLE IF NOT EXISTS automotive_catalog.governance_lab.customer_registrations\n", + "spark.sql(\"\"\"\n", + " CREATE TABLE IF NOT EXISTS automotive_catalog2.governance_lab.customer_registrations\n", " COMMENT 'One row per registered AutoSphere customer and their associated vehicle.'\n", - "AS SELECT * FROM VALUES\n", - " (1, 'Lukas Bauer', 'lukas.bauer@autosphere.de', 'DE-LB-4821', 'DE', 'VH-001'),\n", - " (2, 'Sophie Martin', 'sophie.martin@autosphere.fr', 'FR-SM-9034', 'FR', 'VH-002'),\n", - " (3, 'James Clarke', 'james.clarke@autosphere.uk', 'UK-JC-1172', 'UK', 'VH-003'),\n", - " (4, 'Yuki Tanaka', 'yuki.tanaka@autosphere.jp', 'JP-YT-5561', 'JP', 'VH-004'),\n", - " (5, 'Maria Santos', 'maria.santos@autosphere.pt', 'PT-MS-8843', 'PT', 'VH-005'),\n", - " (6, 'Carlos Rivera', 'carlos.rivera@autosphere.es', 'ES-CR-3309', 'ES', 'VH-006')\n", - "AS t(customer_id, full_name, email, driver_license_no, country, vehicle_id);\n", - "\n", - "-- Create the vehicle_telemetry table\n", - "CREATE TABLE IF NOT EXISTS automotive_catalog.governance_lab.vehicle_telemetry (\n", - " event_id BIGINT COMMENT 'Unique telemetry event identifier',\n", - " vehicle_id STRING,\n", - " event_time TIMESTAMP,\n", - " speed_kmh INT COMMENT 'Instantaneous vehicle speed in kilometres per hour',\n", - " battery_level_pct INT COMMENT 'State of charge as a percentage (0–100)',\n", - " latitude DOUBLE,\n", - " longitude DOUBLE,\n", - " country STRING\n", - ");\n", - "\n", - "INSERT INTO automotive_catalog.governance_lab.vehicle_telemetry VALUES\n", - " (1, 'VH-001', TIMESTAMP '2026-03-01 08:15:00', 112, 78, 48.8566, 2.3522, 'DE'),\n", - " (2, 'VH-002', TIMESTAMP '2026-03-01 08:16:00', 95, 55, 51.5074, -0.1278, 'FR'),\n", - " (3, 'VH-003', TIMESTAMP '2026-03-01 08:17:00', 130, 91, 40.4168, -3.7038, 'UK'),\n", - " (4, 'VH-004', TIMESTAMP '2026-03-01 08:18:00', 88, 34, 35.6762, 139.6503, 'JP'),\n", - " (5, 'VH-005', TIMESTAMP '2026-03-01 08:19:00', 105, 62, 38.7223, -9.1393, 'PT'),\n", - " (6, 'VH-006', TIMESTAMP '2026-03-01 08:20:00', 77, 47, 41.3851, 2.1734, 'ES');" + " AS SELECT * FROM VALUES\n", + " (1, 'Lukas Bauer', 'lukas.bauer@autosphere.de', 'DE-LB-4821', 'DE', 'VH-001'),\n", + " (2, 'Sophie Martin', 'sophie.martin@autosphere.fr', 'FR-SM-9034', 'FR', 'VH-002'),\n", + " (3, 'James Clarke', 'james.clarke@autosphere.uk', 'UK-JC-1172', 'UK', 'VH-003'),\n", + " (4, 'Yuki Tanaka', 'yuki.tanaka@autosphere.jp', 'JP-YT-5561', 'JP', 'VH-004'),\n", + " (5, 'Maria Santos', 'maria.santos@autosphere.pt', 'PT-MS-8843', 'PT', 'VH-005'),\n", + " (6, 'Carlos Rivera', 'carlos.rivera@autosphere.es', 'ES-CR-3309', 'ES', 'VH-006')\n", + " AS t(customer_id, full_name, email, driver_license_no, country, vehicle_id)\n", + "\"\"\")\n", + "print(\"✓ customer_registrations table created\")\n", + "\n", + "spark.sql(\"\"\"\n", + " CREATE TABLE IF NOT EXISTS automotive_catalog2.governance_lab.vehicle_telemetry (\n", + " event_id BIGINT COMMENT 'Unique telemetry event identifier',\n", + " vehicle_id STRING,\n", + " event_time TIMESTAMP,\n", + " speed_kmh INT COMMENT 'Instantaneous vehicle speed in kilometres per hour',\n", + " battery_level_pct INT COMMENT 'State of charge as a percentage (0-100)',\n", + " latitude DOUBLE,\n", + " longitude DOUBLE,\n", + " country STRING\n", + " )\n", + "\"\"\")\n", + "print(\"✓ vehicle_telemetry table created\")\n", + "\n", + "spark.sql(\"\"\"\n", + " INSERT INTO automotive_catalog2.governance_lab.vehicle_telemetry VALUES\n", + " (1, 'VH-001', TIMESTAMP '2026-03-01 08:15:00', 112, 78, 48.8566, 2.3522, 'DE'),\n", + " (2, 'VH-002', TIMESTAMP '2026-03-01 08:16:00', 95, 55, 51.5074, -0.1278, 'FR'),\n", + " (3, 'VH-003', TIMESTAMP '2026-03-01 08:17:00', 130, 91, 40.4168, -3.7038, 'UK'),\n", + " (4, 'VH-004', TIMESTAMP '2026-03-01 08:18:00', 88, 34, 35.6762, 139.6503, 'JP'),\n", + " (5, 'VH-005', TIMESTAMP '2026-03-01 08:19:00', 105, 62, 38.7223, -9.1393, 'PT'),\n", + " (6, 'VH-006', TIMESTAMP '2026-03-01 08:20:00', 77, 47, 41.3851, 2.1734, 'ES')\n", + "\"\"\")\n", + "print(\"✓ Telemetry data inserted\")\n", + "print(\"\\nAll objects created successfully in automotive_catalog2.governance_lab\")" ] }, { "cell_type": "markdown", - "id": "7bd11ac1", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "2e01185f-414b-4d54-b97c-9f5e65a400e3", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "## Exercise 2: Tag Data Assets for Governance\n", "\n", @@ -114,8 +173,16 @@ }, { "cell_type": "markdown", - "id": "00c87158", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "f010d9b9-e1d9-4220-a2e9-260147c038cc", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 2.1 — Add table-level tags\n", "\n", @@ -124,7 +191,7 @@ "- `domain` = `customer`\n", "- `data_classification` = `confidential`\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I add key-value tags to a Unity Catalog table in Databricks SQL?\"*\n", "\n", "**Hint:** Use `ALTER TABLE ... SET TAGS ('key' = 'value', 'key2' = 'value2')`." @@ -132,20 +199,45 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "ae5833f4", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "ce6078ba-dfd5-4ee8-a338-e002d6d66044", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", "-- TODO: Add table-level tags to automotive_catalog.governance_lab.customer_registrations\n", + "\n", + "ALTER TABLE automotive_catalog2.governance_lab.customer_registrations SET TAGS (\n", + " 'domain'= 'customer',\n", + " 'data_classification' = 'confidential'\n", + ")\n", "-- Tags: domain = customer, data_classification = confidential" ] }, { "cell_type": "markdown", - "id": "8d96e379", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "f7033da3-5c4d-4a2e-b328-d41a9d241cd2", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 2.2 — Add column-level PII tags\n", "\n", @@ -154,7 +246,7 @@ "- Column `email` → `pii` = `email`\n", "- Column `driver_license_no` → `pii` = `driver_license`\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I add a tag to a specific column in a Unity Catalog table using SQL in Databricks?\"*\n", "\n", "**Hint:** Use `ALTER TABLE ... ALTER COLUMN SET TAGS (...)`." @@ -162,56 +254,117 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "32304672", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "bd034de2-6011-4bca-9665-9d80c93b2b4a", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", "-- TODO: Tag the email column with pii = email\n", - "\n", - "-- TODO: Tag the driver_license_no column with pii = driver_license" + "ALTER TABLE automotive_catalog2.governance_lab.customer_registrations ALTER COLUMN email SET TAGS ('pii'='email');\n", + "-- TODO: Tag the driver_license_no column with pii = driver_license\n", + "ALTER TABLE automotive_catalog2.governance_lab.customer_registrations ALTER COLUMN driver_license_no SET TAGS ('pii' = 'driver_license')" ] }, { "cell_type": "markdown", - "id": "154ded38", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "856185cd-26a0-4b02-8173-79db505ff9be", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 2.3 — Verify the tags\n", "\n", "Use `system.information_schema.table_tags` to confirm the table-level tags have been applied. Then describe the columns to confirm the column tags are present. Use `system.information_schema.column_tags` for this.\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How can I verify tags on a Unity Catalog table and its columns in Databricks SQL?\"*" ] }, { "cell_type": "code", - "execution_count": null, - "id": "d6913f7e", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "ca3a2ff1-f7cc-440a-8211-41715342f06d", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", - "-- Show table-level tags for customer_registrations" + "-- Show table-level tags for customer_registrations\n", + "SELECT * FROM system.information_schema.table_tags \n", + "WHERE catalog_name = 'automotive_catalog2'\n", + "AND schema_name = 'governance_lab'\n", + "AND table_name = 'customer_registrations'" ] }, { "cell_type": "code", - "execution_count": null, - "id": "2ec89502", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "b6cd9020-36c0-4ebc-900e-c95a7e47dfa2", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", - "-- Show column-level tags for customer_registrations" + "-- Show column-level tags for customer_registrations\n", + "SELECT * FROM system.information_schema.column_tags\n", + "WHERE catalog_name = 'automotive_catalog2'\n", + "AND schema_name = 'governance_lab'\n", + "AND table_name = 'customer_registrations';" ] }, { "cell_type": "markdown", - "id": "660110e6", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "93dbbb3f-50a1-4c8b-841e-7172ed78b2e3", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "## Exercise 3: Configure Data Retention\n", "\n", @@ -226,8 +379,16 @@ }, { "cell_type": "markdown", - "id": "0ed2394b", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "57c148b2-bb9b-4f01-9c4e-fce12f7003dc", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 3.1 — Configure retention properties\n", "\n", @@ -236,7 +397,7 @@ "- `delta.logRetentionDuration` = `interval 14 days`\n", "- `delta.deletedFileRetentionDuration` = `interval 14 days`\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I set Delta Lake retention duration properties on a Unity Catalog table in Databricks?\"*\n", "\n", "**Hint:** Use `ALTER TABLE ... SET TBLPROPERTIES (...)`." @@ -244,20 +405,44 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "23ccba01", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "fc3bc7ee-e30d-4aeb-89eb-5dd17d15840c", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", "-- TODO: Set logRetentionDuration and deletedFileRetentionDuration to 14 days\n", - "-- on automotive_catalog.governance_lab.vehicle_telemetry" + "-- on automotive_catalog.governance_lab.vehicle_telemetry\n", + "ALTER TABLE automotive_catalog2.governance_lab.vehicle_telemetry SET TBLPROPERTIES (\n", + " delta.logRetentionDuration = '14 days',\n", + " delta.deletedFileRetentionDuration = '14 days'\n", + ")\n" ] }, { "cell_type": "markdown", - "id": "98e8791e", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "acca0119-a28b-41ef-a6b3-0f18c647fdbe", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 3.2 — Simulate a GDPR deletion request\n", "\n", @@ -265,27 +450,49 @@ "\n", "After deletion, verify the records are gone by querying the table.\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I delete specific rows from a Delta Lake table in Databricks SQL?\"*" ] }, { "cell_type": "code", - "execution_count": null, - "id": "fcee14fb", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "d07d6dde-e3c3-4408-a17f-114514c659c5", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", "-- TODO: Delete all rows for vehicle_id = 'VH-003' from vehicle_telemetry\n", - "\n", - "-- TODO: Verify the deletion by querying the table" + "DELETE FROM automotive_catalog2.governance_lab.vehicle_telemetry\n", + "WHERE vehicle_id = 'VH-003'; \n", + "-- TODO: Verify the deletion by querying the table\n", + "SELECT * FROM automotive_catalog2.governance_lab.vehicle_telemetry;" ] }, { "cell_type": "markdown", - "id": "7ad2c419", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "98eeea53-6b66-40fd-8e07-6501177688cd", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 3.3 — Run VACUUM to purge deleted data files\n", "\n", @@ -293,7 +500,7 @@ "\n", "Use the default retention of **168 hours (7 days)**. This is safe, works on Serverless compute, and is appropriate for a lab environment.\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I run VACUUM on a Delta Lake table in Databricks, and what does the RETAIN option do?\"*\n", "\n", "**Note:** On Serverless compute, overriding the minimum retention duration is not supported. Always specify a `RETAIN` value of at least 168 hours, or omit `RETAIN` entirely to use the default." @@ -301,26 +508,47 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "1f9654e4", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "14f4a0bc-3a2b-4fe3-971d-84b1fbe63236", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", "-- TODO: Run VACUUM on automotive_catalog.governance_lab.vehicle_telemetry\n", - "-- Use RETAIN 168 HOURS (7 days) or omit RETAIN to use the default" + "VACUUM automotive_catalog2.governance_lab.vehicle_telemetry RETAIN 168 HOURS\n", + "-- Use RETAIN 168 HOURS (7 days) or omit RETAIN to use the default\n" ] }, { "cell_type": "markdown", - "id": "5b382bbd", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "7bab7d93-fc33-4715-aa1c-02946f51cee3", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 3.4 — Enable predictive optimization\n", "\n", "Enable **predictive optimization** on the `governance_lab` schema so that future `VACUUM` and `OPTIMIZE` maintenance runs automatically without manual scheduling.\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I enable predictive optimization on a Unity Catalog schema in Databricks?\"*\n", "\n", "**Hint:** Use `ALTER SCHEMA ... ENABLE PREDICTIVE OPTIMIZATION`." @@ -328,37 +556,67 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "881c3b88", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "e0abc3fe-e693-44d2-9ff1-9afdbcd6b82b", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Enable predictive optimization on automotive_catalog.governance_lab" + "-- TODO: Enable predictive optimization on automotive_catalog.governance_lab\n", + "\n", + "ALTER SCHEMA automotive_catalog2.governance_lab ENABLE PREDICTIVE OPTIMIZATION;" ] }, { "cell_type": "markdown", - "id": "c280c455", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "7e9ec799-d1fe-44c5-856d-49cf30346a5a", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "## Exercise 4: Query Data Lineage Programmatically\n", "\n", - "> 📋 **Before running this exercise**: If you haven't already done so, pause here and follow the **Catalog Explorer lineage steps** in the lab setup instructions to explore lineage visually in the UI. Then return here to query lineage data using SQL.\n", + "> \uD83D\uDCCB **Before running this exercise**: If you haven't already done so, pause here and follow the **Catalog Explorer lineage steps** in the lab setup instructions to explore lineage visually in the UI. Then return here to query lineage data using SQL.\n", "\n", "Unity Catalog captures all read and write events and stores them in the `system.access.table_lineage` system table. This lets you programmatically answer questions like: *\"Which tables in our automotive catalog were most queried this week?\"*" ] }, { "cell_type": "markdown", - "id": "2c4f5d42", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "734e68ba-1259-43fa-8da6-f0fc5ab07cab", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 4.1 — Find recently accessed tables in your catalog\n", "\n", "Query `system.access.table_lineage` to find all **write events** (source tables) within the past 7 days that relate to `automotive_catalog`. Return the table name and the count of distinct events, ordered by the most active tables first.\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I query system.access.table_lineage to find recently accessed Unity Catalog tables in Databricks?\"*\n", "\n", "**Hint:** Filter on `source_table_full_name LIKE 'automotive_catalog%'` and group by table name." @@ -366,45 +624,123 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "363d78dd", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "e4edea03-8f98-400f-9910-ba2092573325", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 24" + } + }, + "outputs": [], + "source": [ + "%sql\n", + "GRANT USE SCHEMA ON SCHEMA system.access TO `user1-63885547@lodsprodmca.onmicrosoft.com`;\n", + "GRANT SELECT ON SCHEMA system.access TO `user1-63885547@lodsprodmca.onmicrosoft.com`;" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "6de3b319-3871-4159-809b-72412f420a82", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", "-- TODO: Query system.access.table_lineage for tables in automotive_catalog accessed in the last 7 days\n", "-- Return: source_table_full_name, count of events\n", - "-- Order by most active first" + "-- Order by most active first\n", + "SELECT \n", + " source_table_full_name,\n", + " COUNT(DISTINCT event_id) AS event_count, \n", + " MAX(event_time)\n", + "FROM \n", + " system.access.table_lineage\n", + "WHERE \n", + " event_date > CURRENT_DATE() - INTERVAL 7 DAYS\n", + " AND(\n", + " source_table_full_name LIKE 'automotive_catalog2%'\n", + " )\n", + "GROUP BY source_table_full_name\n", + "ORDER BY last_event_time DESC;" ] }, { "cell_type": "markdown", - "id": "f6acecab", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "e4fac3dd-7b10-45e2-aa1d-e2bad4b1b794", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 4.2 — View the history of vehicle_telemetry\n", "\n", "Use `DESCRIBE HISTORY` to see the full Delta Lake operation history for `vehicle_telemetry`. Identify the version where the deletion occurred (`DELETE` operation).\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I view the full version history of a Delta Lake table in Databricks SQL?\"*" ] }, { "cell_type": "code", - "execution_count": null, - "id": "eaafacd9", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "3a77b86e-4b2a-4443-9434-e2dff2e65511", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Describe the history of automotive_catalog.governance_lab.vehicle_telemetry" + "-- TODO: Describe the history of automotive_catalog.governance_lab.vehicle_telemetry\n", + "DESCRIBE HISTORY automotive_catalog2.governance_lab.vehicle_telemetry;" ] }, { "cell_type": "markdown", - "id": "1a505bac", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "54b037d9-0bfb-4453-a81c-a6b1a8051b96", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "## Exercise 5: Audit Logging\n", "\n", @@ -415,14 +751,22 @@ }, { "cell_type": "markdown", - "id": "dfb2a448", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "7e900a68-a3d6-48e3-b5b8-774aef38e2e4", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 5.1 — Find who accessed the customer_registrations table\n", "\n", "Query `system.access.audit` to list all users who accessed `automotive_catalog.governance_lab.customer_registrations` in the past 30 days. Include the action name and the time of each access, ordered by most recent first.\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I query the Databricks audit log system table to find who accessed a specific Unity Catalog table?\"*\n", "\n", "**Hint:** Filter on `request_params.full_name_arg` and `action_name IN ('getTable', 'createTable')`. Use `user_identity.email` for the user column." @@ -430,27 +774,59 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "4b237cd6", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "ff234ae9-9cb2-4566-8aab-b62700ba8e50", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", "-- TODO: Query system.access.audit for access events on automotive_catalog.governance_lab.customer_registrations\n", "-- Return: user email, action_name, event_time\n", - "-- Filter to the past 30 days, order by event_time DESC" + "-- Filter to the past 30 days, order by event_time DESC\n", + "\n", + "SELECT \n", + " user_identity.email AS\n", + " user_email,\n", + " action_name,\n", + " event_time\n", + "FROM system.access.audit\n", + "WHERE request_params.full_name_arg = 'automotive_catalog2.governance_lab.customer_registrations'\n", + "AND action_name IN\n", + "('getTable', 'createTable')\n", + "AND event_date > CURRENT_DATE() - INTERVAL 30 DAYS\n", + "ORDER BY event_time DESC;\n" ] }, { "cell_type": "markdown", - "id": "dbd2e60c", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "e49aafea-5ba4-4723-8d86-fa715806d7f5", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 5.2 — Detect permission changes across your catalog\n", "\n", "Query `system.access.audit` to list all **permission change events** (`updatePermissions` action) on securable objects in `automotive_catalog` during the past 30 days. Include who made the change and what object was affected.\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I find all permission changes in the Databricks audit log for a specific Unity Catalog catalog?\"*\n", "\n", "**Hint:** Filter on `service_name = 'unityCatalog'`, `action_name = 'updatePermissions'`, and `request_params.securable_full_name LIKE 'automotive_catalog%'`." @@ -458,50 +834,135 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "bde1da29", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "dcae3ef8-26f4-4994-af5e-6b56875335c1", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", "-- TODO: Query system.access.audit for permission change events on automotive_catalog objects\n", "-- Return: event_time, user email, securable_type, securable_full_name, changes\n", - "-- Filter to the past 30 days, order by event_time DESC" + "-- Filter to the past 30 days, order by event_time DESC\n", + "\n", + "SELECT \n", + " event_time,\n", + " user_identity.email AS user_email,\n", + " request_params.securable_type,\n", + " request_params.securable_full_name,\n", + " request_params.changes\n", + "FROM system.access.audit\n", + "WHERE service_name = 'unityCatalog'\n", + " AND action_name = 'updatePermissions'\n", + " AND request_params.securable_full_name LIKE 'automotive_catalog2%'\n", + " AND event_date > CURRENT_DATE() - INTERVAL 30 DAYS\n", + "ORDER BY event_time DESC;\n" ] }, { "cell_type": "markdown", - "id": "f9458263", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "26a948ce-f28b-4dc7-8af7-aa035e69466d", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 5.3 — Summarise audit activity by user\n", "\n", "Create a summary query that counts the total number of audit events per user for Unity Catalog actions in the past 7 days. This gives the compliance team a quick overview of who was most active on the platform.\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I aggregate the Databricks audit log to count events per user for Unity Catalog activity?\"*" ] }, { "cell_type": "code", - "execution_count": null, - "id": "2e3e9bb9", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "2f679f7a-425d-46a2-89ec-ee52ccafc8c5", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", "-- TODO: Count total audit events per user from system.access.audit\n", "-- Filter to service_name = 'unityCatalog' and the last 7 days\n", "-- Return: user email, event count\n", - "-- Order by most active user first" + "-- Order by most active user first\n", + "\n", + "\n", + "SELECT \n", + " user_identity.email AS user_email,\n", + " COUNT(*) AS event_count\n", + "FROM system.access.audit\n", + "WHERE service_name = 'unityCatalog'\n", + " AND event_date > CURRENT_DATE() - INTERVAL 7 DAYS\n", + "GROUP BY user_identity.email\n", + "ORDER BY event_count DESC;" ] } ], "metadata": { + "application/vnd.databricks.v1+notebook": { + "computePreferences": { + "hardware": { + "accelerator": null, + "gpuPoolId": null, + "memory": null + }, + "software": { + "pinSparkToX86": null + } + }, + "dashboards": [], + "environmentMetadata": { + "base_environment": "", + "environment_version": "5" + }, + "inputWidgetPreferences": null, + "language": "python", + "notebookMetadata": { + "mostRecentlyExecutedCommandWithImplicitDF": { + "commandId": 8684257317553162, + "dataframes": [ + "_sqldf" + ] + }, + "pythonIndentUnit": 4 + }, + "notebookName": "05-govern-unity-catalog-objects", + "widgets": {} + }, "language_info": { "name": "python" } }, "nbformat": 4, - "nbformat_minor": 5 -} + "nbformat_minor": 0 +} \ No newline at end of file