diff --git a/dbt_sql/AGENTS.md b/dbt_sql/AGENTS.md new file mode 100644 index 00000000..fdcca98a --- /dev/null +++ b/dbt_sql/AGENTS.md @@ -0,0 +1,25 @@ +# Declarative Automation Bundles Project + +This project uses Declarative Automation Bundles (DABs) for deployment. Add project-specific instructions below. + +## For AI Agents: Use Databricks AI Tools + +**BEFORE any other action, read the `databricks-core` skill.** + +It sets you up to work with this project reliably: CLI authentication, profile +selection, data discovery, and the bundle deployment workflow. Without it, +results are often slower and less accurate. + +If this skill is not available (Databricks AI Tools are not installed), you can install them for your coding agent in seconds: + +```bash +databricks aitools install +``` + +If the CLI is not installed, see: https://docs.databricks.com/dev-tools/cli/install + +--- + +## Project Instructions + + diff --git a/dbt_sql/CLAUDE.md b/dbt_sql/CLAUDE.md new file mode 100644 index 00000000..5612c9bd --- /dev/null +++ b/dbt_sql/CLAUDE.md @@ -0,0 +1,6 @@ +# CLAUDE.md + +Project guidance for AI agents lives in AGENTS.md. +Claude Code loads it via the import below. + +@AGENTS.md diff --git a/dbt_sql/README.md b/dbt_sql/README.md index ef22b54a..78f3cfe9 100644 --- a/dbt_sql/README.md +++ b/dbt_sql/README.md @@ -102,7 +102,7 @@ on CI/CD setup. ## Manually deploying to Databricks with Declarative Automation Bundles Declarative Automation Bundles can be used to deploy to Databricks and to execute -dbt commands as a job using Databricks Workflows. See +dbt commands as a job using Databricks Jobs. See https://docs.databricks.com/dev-tools/bundles/index.html to learn more. Use the Databricks CLI to deploy a development copy of this project to a workspace: @@ -117,7 +117,7 @@ is optional here.) This deploys everything that's defined for this project. For example, the default template would deploy a job called `[dev yourname] dbt_sql_job` to your workspace. -You can find that job by opening your workpace and clicking on **Workflows**. +You can find that job by opening your workpace and clicking on **Jobs & Pipelines**. You can also deploy to your production target directly from the command-line. The warehouse, catalog, and schema for that target are configured in `dbt_profiles/profiles.yml`. diff --git a/dbt_sql/dbt_project.yml b/dbt_sql/dbt_project.yml index 947412e3..2be6b44a 100644 --- a/dbt_sql/dbt_project.yml +++ b/dbt_sql/dbt_project.yml @@ -6,7 +6,7 @@ config-version: 2 profile: 'dbt_sql' # These configurations specify where dbt should look for different types of files. -# For Databricks asset bundles, we put everything in src, as you may have +# For Declarative Automation Bundles, we put everything in src, as you may have # non-dbt resources in your project. model-paths: ["src/models"] analysis-paths: ["src/analyses"] diff --git a/dbt_sql/resources/dbt_sql.job.yml b/dbt_sql/resources/dbt_sql.job.yml index 3af98c24..3c1a7e3a 100644 --- a/dbt_sql/resources/dbt_sql.job.yml +++ b/dbt_sql/resources/dbt_sql.job.yml @@ -29,6 +29,6 @@ resources: environments: - environment_key: default spec: - environment_version: "4" + environment_version: "5" dependencies: - dbt-databricks>=1.8.0,<2.0.0 diff --git a/default_minimal/.gitignore b/default_minimal/.gitignore index e566c51f..622a8811 100644 --- a/default_minimal/.gitignore +++ b/default_minimal/.gitignore @@ -1,9 +1,4 @@ .databricks/ -build/ -dist/ -__pycache__/ -*.egg-info -.venv/ scratch/** !scratch/README.md **/explorations/** diff --git a/default_minimal/.vscode/__builtins__.pyi b/default_minimal/.vscode/__builtins__.pyi deleted file mode 100644 index 0edd5181..00000000 --- a/default_minimal/.vscode/__builtins__.pyi +++ /dev/null @@ -1,3 +0,0 @@ -# Typings for Pylance in Visual Studio Code -# see https://github.com/microsoft/pyright/blob/main/docs/builtins.md -from databricks.sdk.runtime import * diff --git a/default_minimal/.vscode/extensions.json b/default_minimal/.vscode/extensions.json index 5ba48e79..1f39c330 100644 --- a/default_minimal/.vscode/extensions.json +++ b/default_minimal/.vscode/extensions.json @@ -1,7 +1,6 @@ { "recommendations": [ "databricks.databricks", - "redhat.vscode-yaml", - "charliermarsh.ruff" + "redhat.vscode-yaml" ] } diff --git a/default_minimal/.vscode/settings.json b/default_minimal/.vscode/settings.json index d73c73b5..d3e81488 100644 --- a/default_minimal/.vscode/settings.json +++ b/default_minimal/.vscode/settings.json @@ -1,39 +1,5 @@ { - "jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])", - "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------", - "python.testing.pytestArgs": [ - "." - ], - "files.exclude": { - "**/*.egg-info": true, - "**/__pycache__": true, - ".pytest_cache": true, - "dist": true, - }, "files.associations": { "**/.gitkeep": "markdown" }, - - // Pylance settings (VS Code) - // Set typeCheckingMode to "basic" to enable type checking! - "python.analysis.typeCheckingMode": "off", - "python.analysis.extraPaths": ["src", "lib", "resources"], - "python.analysis.diagnosticMode": "workspace", - "python.analysis.stubPath": ".vscode", - - // Pyright settings (Cursor) - // Set typeCheckingMode to "basic" to enable type checking! - "cursorpyright.analysis.typeCheckingMode": "off", - "cursorpyright.analysis.extraPaths": ["src", "lib", "resources"], - "cursorpyright.analysis.diagnosticMode": "workspace", - "cursorpyright.analysis.stubPath": ".vscode", - - // General Python settings - "python.defaultInterpreterPath": "./.venv/bin/python", - "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true, - "[python]": { - "editor.defaultFormatter": "charliermarsh.ruff", - "editor.formatOnSave": true, - }, } diff --git a/default_minimal/AGENTS.md b/default_minimal/AGENTS.md new file mode 100644 index 00000000..fdcca98a --- /dev/null +++ b/default_minimal/AGENTS.md @@ -0,0 +1,25 @@ +# Declarative Automation Bundles Project + +This project uses Declarative Automation Bundles (DABs) for deployment. Add project-specific instructions below. + +## For AI Agents: Use Databricks AI Tools + +**BEFORE any other action, read the `databricks-core` skill.** + +It sets you up to work with this project reliably: CLI authentication, profile +selection, data discovery, and the bundle deployment workflow. Without it, +results are often slower and less accurate. + +If this skill is not available (Databricks AI Tools are not installed), you can install them for your coding agent in seconds: + +```bash +databricks aitools install +``` + +If the CLI is not installed, see: https://docs.databricks.com/dev-tools/cli/install + +--- + +## Project Instructions + + diff --git a/default_minimal/CLAUDE.md b/default_minimal/CLAUDE.md new file mode 100644 index 00000000..5612c9bd --- /dev/null +++ b/default_minimal/CLAUDE.md @@ -0,0 +1,6 @@ +# CLAUDE.md + +Project guidance for AI agents lives in AGENTS.md. +Claude Code loads it via the import below. + +@AGENTS.md diff --git a/default_minimal/README.md b/default_minimal/README.md index a56a0a1d..4fe3b9fe 100644 --- a/default_minimal/README.md +++ b/default_minimal/README.md @@ -17,13 +17,6 @@ Choose how you want to work on this project: (c) With command line tools, see https://docs.databricks.com/dev-tools/cli/databricks-cli.html -If you're developing with an IDE, dependencies for this project should be installed using uv: - -* Make sure you have the UV package manager installed. - It's an alternative to tools like pip: https://docs.astral.sh/uv/getting-started/installation/. -* Run `uv sync --dev` to install the project's dependencies. - - # Using this project using the CLI The Databricks workspace and IDE extensions provide a graphical interface for working diff --git a/default_minimal/databricks.yml b/default_minimal/databricks.yml index c788c8df..2c93242e 100644 --- a/default_minimal/databricks.yml +++ b/default_minimal/databricks.yml @@ -1,4 +1,4 @@ -# This is a Databricks asset bundle definition for default_minimal. +# This is a Declarative Automation Bundle definition for default_minimal. # See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. bundle: name: default_minimal diff --git a/default_python/.vscode/extensions.json b/default_python/.vscode/extensions.json index 5ba48e79..b958aacf 100644 --- a/default_python/.vscode/extensions.json +++ b/default_python/.vscode/extensions.json @@ -1,7 +1,7 @@ { "recommendations": [ + "charliermarsh.ruff", "databricks.databricks", - "redhat.vscode-yaml", - "charliermarsh.ruff" + "redhat.vscode-yaml" ] } diff --git a/default_python/AGENTS.md b/default_python/AGENTS.md new file mode 100644 index 00000000..fdcca98a --- /dev/null +++ b/default_python/AGENTS.md @@ -0,0 +1,25 @@ +# Declarative Automation Bundles Project + +This project uses Declarative Automation Bundles (DABs) for deployment. Add project-specific instructions below. + +## For AI Agents: Use Databricks AI Tools + +**BEFORE any other action, read the `databricks-core` skill.** + +It sets you up to work with this project reliably: CLI authentication, profile +selection, data discovery, and the bundle deployment workflow. Without it, +results are often slower and less accurate. + +If this skill is not available (Databricks AI Tools are not installed), you can install them for your coding agent in seconds: + +```bash +databricks aitools install +``` + +If the CLI is not installed, see: https://docs.databricks.com/dev-tools/cli/install + +--- + +## Project Instructions + + diff --git a/default_python/CLAUDE.md b/default_python/CLAUDE.md new file mode 100644 index 00000000..5612c9bd --- /dev/null +++ b/default_python/CLAUDE.md @@ -0,0 +1,6 @@ +# CLAUDE.md + +Project guidance for AI agents lives in AGENTS.md. +Claude Code loads it via the import below. + +@AGENTS.md diff --git a/default_python/databricks.yml b/default_python/databricks.yml index 1c1578b2..b5879653 100644 --- a/default_python/databricks.yml +++ b/default_python/databricks.yml @@ -1,4 +1,4 @@ -# This is a Databricks asset bundle definition for default_python. +# This is a Declarative Automation Bundle definition for default_python. # See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. bundle: name: default_python diff --git a/default_python/pyproject.toml b/default_python/pyproject.toml index d89e336a..8552fdce 100644 --- a/default_python/pyproject.toml +++ b/default_python/pyproject.toml @@ -2,7 +2,7 @@ name = "default_python" version = "0.0.1" authors = [{ name = "user@company.com" }] -requires-python = ">=3.10,<3.13" +requires-python = ">=3.12,<3.13" dependencies = [ # Any dependencies for jobs and pipelines in this project can be added here # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies @@ -16,7 +16,7 @@ dev = [ "pytest", "ruff", "databricks-dlt", - "databricks-connect>=15.4,<15.5", + "databricks-connect>=16.4,<16.5", "ipykernel", ] diff --git a/default_python/resources/sample_job.job.yml b/default_python/resources/sample_job.job.yml index da5e0eaa..0af1458f 100644 --- a/default_python/resources/sample_job.job.yml +++ b/default_python/resources/sample_job.job.yml @@ -46,7 +46,7 @@ resources: environments: - environment_key: default spec: - environment_version: "4" + environment_version: "5" dependencies: # By default we just include the .whl file generated for the default_python package. # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html diff --git a/default_python/src/default_python/main.py b/default_python/src/default_python/main.py index bbe33994..f51ef0b0 100644 --- a/default_python/src/default_python/main.py +++ b/default_python/src/default_python/main.py @@ -13,8 +13,8 @@ def main(): args = parser.parse_args() # Set the default catalog and schema - spark.sql(f"USE CATALOG {args.catalog}") - spark.sql(f"USE SCHEMA {args.schema}") + spark.sql(f"USE CATALOG `{args.catalog}`") + spark.sql(f"USE SCHEMA `{args.schema}`") # Example: just find all taxis from a sample catalog taxis.find_all_taxis().show(5) diff --git a/default_python/src/default_python_etl/README.md b/default_python/src/default_python_etl/README.md index de5e26a4..aaab1805 100644 --- a/default_python/src/default_python_etl/README.md +++ b/default_python/src/default_python_etl/README.md @@ -15,6 +15,6 @@ To get started, go to the `transformations` folder -- most of the relevant sourc * Take a look at the sample called "sample_trips_default_python.py" to get familiar with the syntax. Read more about the syntax at https://docs.databricks.com/dlt/python-ref.html. * If you're using the workspace UI, use `Run file` to run and preview a single transformation. -* If you're using the CLI, use `databricks bundle run default_python_etl --select sample_trips_default_python` to run a single transformation. +* If you're using the CLI, use `databricks bundle run default_python_etl --refresh sample_trips_default_python` to run a single transformation. For more tutorials and reference material, see https://docs.databricks.com/dlt. diff --git a/default_python/src/sample_notebook.ipynb b/default_python/src/sample_notebook.ipynb index 63723b5e..9c39fc7a 100644 --- a/default_python/src/sample_notebook.ipynb +++ b/default_python/src/sample_notebook.ipynb @@ -34,8 +34,8 @@ "# Set default catalog and schema\n", "catalog = dbutils.widgets.get(\"catalog\")\n", "schema = dbutils.widgets.get(\"schema\")\n", - "spark.sql(f\"USE CATALOG {catalog}\")\n", - "spark.sql(f\"USE SCHEMA {schema}\")" + "spark.sql(f\"USE CATALOG `{catalog}`\")\n", + "spark.sql(f\"USE SCHEMA `{schema}`\")" ] }, { @@ -72,7 +72,7 @@ "dependencies": [ "--editable .." ], - "environment_version": "4" + "environment_version": "5" }, "language": "python", "notebookMetadata": { diff --git a/default_sql/AGENTS.md b/default_sql/AGENTS.md new file mode 100644 index 00000000..fdcca98a --- /dev/null +++ b/default_sql/AGENTS.md @@ -0,0 +1,25 @@ +# Declarative Automation Bundles Project + +This project uses Declarative Automation Bundles (DABs) for deployment. Add project-specific instructions below. + +## For AI Agents: Use Databricks AI Tools + +**BEFORE any other action, read the `databricks-core` skill.** + +It sets you up to work with this project reliably: CLI authentication, profile +selection, data discovery, and the bundle deployment workflow. Without it, +results are often slower and less accurate. + +If this skill is not available (Databricks AI Tools are not installed), you can install them for your coding agent in seconds: + +```bash +databricks aitools install +``` + +If the CLI is not installed, see: https://docs.databricks.com/dev-tools/cli/install + +--- + +## Project Instructions + + diff --git a/default_sql/CLAUDE.md b/default_sql/CLAUDE.md new file mode 100644 index 00000000..5612c9bd --- /dev/null +++ b/default_sql/CLAUDE.md @@ -0,0 +1,6 @@ +# CLAUDE.md + +Project guidance for AI agents lives in AGENTS.md. +Claude Code loads it via the import below. + +@AGENTS.md diff --git a/default_sql/README.md b/default_sql/README.md index 3f1e5d43..e44cf6f4 100644 --- a/default_sql/README.md +++ b/default_sql/README.md @@ -21,7 +21,7 @@ The 'default_sql' project was generated by using the default-sql template. This deploys everything that's defined for this project. For example, the default template would deploy a job called `[dev yourname] default_sql_job` to your workspace. - You can find that job by opening your workpace and clicking on **Workflows**. + You can find that job by opening your workpace and clicking on **Jobs & Pipelines**. 4. Similarly, to deploy a production copy, type: ``` diff --git a/default_sql/databricks.yml b/default_sql/databricks.yml index db1c948b..08add18a 100644 --- a/default_sql/databricks.yml +++ b/default_sql/databricks.yml @@ -1,4 +1,4 @@ -# This is a Databricks asset bundle definition for default_sql. +# This is a Declarative Automation Bundle definition for default_sql. # See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. bundle: name: default_sql diff --git a/default_sql/src/orders_daily.sql b/default_sql/src/orders_daily.sql index 101bdd67..1d479985 100644 --- a/default_sql/src/orders_daily.sql +++ b/default_sql/src/orders_daily.sql @@ -1,6 +1,6 @@ --- This query is executed using Databricks Workflows (see resources/default_sql_sql.job.yml) +-- This query is executed using Databricks Jobs (see resources/default_sql_sql.job.yml) -USE CATALOG {{catalog}}; +USE CATALOG `{{catalog}}`; USE IDENTIFIER({{schema}}); CREATE OR REPLACE MATERIALIZED VIEW diff --git a/default_sql/src/orders_raw.sql b/default_sql/src/orders_raw.sql index dfb1ce69..1fc12524 100644 --- a/default_sql/src/orders_raw.sql +++ b/default_sql/src/orders_raw.sql @@ -1,9 +1,9 @@ --- This query is executed using Databricks Workflows (see resources/default_sql_sql.job.yml) +-- This query is executed using Databricks Jobs (see resources/default_sql_sql.job.yml) -- -- The streaming table below ingests all JSON files in /databricks-datasets/retail-org/sales_orders/ -- See also https://docs.databricks.com/sql/language-manual/sql-ref-syntax-ddl-create-streaming-table.html -USE CATALOG {{catalog}}; +USE CATALOG `{{catalog}}`; USE IDENTIFIER({{schema}}); CREATE OR REFRESH STREAMING TABLE diff --git a/lakeflow_pipelines_python/.vscode/extensions.json b/lakeflow_pipelines_python/.vscode/extensions.json index 5ba48e79..b958aacf 100644 --- a/lakeflow_pipelines_python/.vscode/extensions.json +++ b/lakeflow_pipelines_python/.vscode/extensions.json @@ -1,7 +1,7 @@ { "recommendations": [ + "charliermarsh.ruff", "databricks.databricks", - "redhat.vscode-yaml", - "charliermarsh.ruff" + "redhat.vscode-yaml" ] } diff --git a/lakeflow_pipelines_python/AGENTS.md b/lakeflow_pipelines_python/AGENTS.md new file mode 100644 index 00000000..fdcca98a --- /dev/null +++ b/lakeflow_pipelines_python/AGENTS.md @@ -0,0 +1,25 @@ +# Declarative Automation Bundles Project + +This project uses Declarative Automation Bundles (DABs) for deployment. Add project-specific instructions below. + +## For AI Agents: Use Databricks AI Tools + +**BEFORE any other action, read the `databricks-core` skill.** + +It sets you up to work with this project reliably: CLI authentication, profile +selection, data discovery, and the bundle deployment workflow. Without it, +results are often slower and less accurate. + +If this skill is not available (Databricks AI Tools are not installed), you can install them for your coding agent in seconds: + +```bash +databricks aitools install +``` + +If the CLI is not installed, see: https://docs.databricks.com/dev-tools/cli/install + +--- + +## Project Instructions + + diff --git a/lakeflow_pipelines_python/CLAUDE.md b/lakeflow_pipelines_python/CLAUDE.md new file mode 100644 index 00000000..5612c9bd --- /dev/null +++ b/lakeflow_pipelines_python/CLAUDE.md @@ -0,0 +1,6 @@ +# CLAUDE.md + +Project guidance for AI agents lives in AGENTS.md. +Claude Code loads it via the import below. + +@AGENTS.md diff --git a/lakeflow_pipelines_python/databricks.yml b/lakeflow_pipelines_python/databricks.yml index ef70b307..58eff5e1 100644 --- a/lakeflow_pipelines_python/databricks.yml +++ b/lakeflow_pipelines_python/databricks.yml @@ -1,4 +1,4 @@ -# This is a Databricks asset bundle definition for lakeflow_pipelines_python. +# This is a Declarative Automation Bundle definition for lakeflow_pipelines_python. # See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. bundle: name: lakeflow_pipelines_python diff --git a/lakeflow_pipelines_python/pyproject.toml b/lakeflow_pipelines_python/pyproject.toml index 91cc59b7..fdd915f2 100644 --- a/lakeflow_pipelines_python/pyproject.toml +++ b/lakeflow_pipelines_python/pyproject.toml @@ -2,7 +2,7 @@ name = "lakeflow_pipelines_python" version = "0.0.1" authors = [{ name = "user@company.com" }] -requires-python = ">=3.10,<3.13" +requires-python = ">=3.12,<3.13" dependencies = [ # Any dependencies for jobs and pipelines in this project can be added here # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies @@ -16,7 +16,7 @@ dev = [ "pytest", "ruff", "databricks-dlt", - "databricks-connect>=15.4,<15.5", + "databricks-connect>=16.4,<16.5", "ipykernel", ] diff --git a/lakeflow_pipelines_python/resources/sample_job.job.yml b/lakeflow_pipelines_python/resources/sample_job.job.yml index a7628601..99f05e06 100644 --- a/lakeflow_pipelines_python/resources/sample_job.job.yml +++ b/lakeflow_pipelines_python/resources/sample_job.job.yml @@ -29,4 +29,4 @@ resources: environments: - environment_key: default spec: - environment_version: "4" + environment_version: "5" diff --git a/lakeflow_pipelines_python/src/lakeflow_pipelines_python_etl/README.md b/lakeflow_pipelines_python/src/lakeflow_pipelines_python_etl/README.md index aac5d43e..dcd72efc 100644 --- a/lakeflow_pipelines_python/src/lakeflow_pipelines_python_etl/README.md +++ b/lakeflow_pipelines_python/src/lakeflow_pipelines_python_etl/README.md @@ -15,6 +15,6 @@ To get started, go to the `transformations` folder -- most of the relevant sourc * Take a look at the sample called "sample_trips_lakeflow_pipelines_python.py" to get familiar with the syntax. Read more about the syntax at https://docs.databricks.com/dlt/python-ref.html. * If you're using the workspace UI, use `Run file` to run and preview a single transformation. -* If you're using the CLI, use `databricks bundle run lakeflow_pipelines_python_etl --select sample_trips_lakeflow_pipelines_python` to run a single transformation. +* If you're using the CLI, use `databricks bundle run lakeflow_pipelines_python_etl --refresh sample_trips_lakeflow_pipelines_python` to run a single transformation. For more tutorials and reference material, see https://docs.databricks.com/dlt. diff --git a/lakeflow_pipelines_sql/.gitignore b/lakeflow_pipelines_sql/.gitignore index e566c51f..622a8811 100644 --- a/lakeflow_pipelines_sql/.gitignore +++ b/lakeflow_pipelines_sql/.gitignore @@ -1,9 +1,4 @@ .databricks/ -build/ -dist/ -__pycache__/ -*.egg-info -.venv/ scratch/** !scratch/README.md **/explorations/** diff --git a/lakeflow_pipelines_sql/.vscode/__builtins__.pyi b/lakeflow_pipelines_sql/.vscode/__builtins__.pyi deleted file mode 100644 index 0edd5181..00000000 --- a/lakeflow_pipelines_sql/.vscode/__builtins__.pyi +++ /dev/null @@ -1,3 +0,0 @@ -# Typings for Pylance in Visual Studio Code -# see https://github.com/microsoft/pyright/blob/main/docs/builtins.md -from databricks.sdk.runtime import * diff --git a/lakeflow_pipelines_sql/.vscode/extensions.json b/lakeflow_pipelines_sql/.vscode/extensions.json index 5ba48e79..1f39c330 100644 --- a/lakeflow_pipelines_sql/.vscode/extensions.json +++ b/lakeflow_pipelines_sql/.vscode/extensions.json @@ -1,7 +1,6 @@ { "recommendations": [ "databricks.databricks", - "redhat.vscode-yaml", - "charliermarsh.ruff" + "redhat.vscode-yaml" ] } diff --git a/lakeflow_pipelines_sql/.vscode/settings.json b/lakeflow_pipelines_sql/.vscode/settings.json index d73c73b5..d3e81488 100644 --- a/lakeflow_pipelines_sql/.vscode/settings.json +++ b/lakeflow_pipelines_sql/.vscode/settings.json @@ -1,39 +1,5 @@ { - "jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])", - "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------", - "python.testing.pytestArgs": [ - "." - ], - "files.exclude": { - "**/*.egg-info": true, - "**/__pycache__": true, - ".pytest_cache": true, - "dist": true, - }, "files.associations": { "**/.gitkeep": "markdown" }, - - // Pylance settings (VS Code) - // Set typeCheckingMode to "basic" to enable type checking! - "python.analysis.typeCheckingMode": "off", - "python.analysis.extraPaths": ["src", "lib", "resources"], - "python.analysis.diagnosticMode": "workspace", - "python.analysis.stubPath": ".vscode", - - // Pyright settings (Cursor) - // Set typeCheckingMode to "basic" to enable type checking! - "cursorpyright.analysis.typeCheckingMode": "off", - "cursorpyright.analysis.extraPaths": ["src", "lib", "resources"], - "cursorpyright.analysis.diagnosticMode": "workspace", - "cursorpyright.analysis.stubPath": ".vscode", - - // General Python settings - "python.defaultInterpreterPath": "./.venv/bin/python", - "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true, - "[python]": { - "editor.defaultFormatter": "charliermarsh.ruff", - "editor.formatOnSave": true, - }, } diff --git a/lakeflow_pipelines_sql/AGENTS.md b/lakeflow_pipelines_sql/AGENTS.md new file mode 100644 index 00000000..fdcca98a --- /dev/null +++ b/lakeflow_pipelines_sql/AGENTS.md @@ -0,0 +1,25 @@ +# Declarative Automation Bundles Project + +This project uses Declarative Automation Bundles (DABs) for deployment. Add project-specific instructions below. + +## For AI Agents: Use Databricks AI Tools + +**BEFORE any other action, read the `databricks-core` skill.** + +It sets you up to work with this project reliably: CLI authentication, profile +selection, data discovery, and the bundle deployment workflow. Without it, +results are often slower and less accurate. + +If this skill is not available (Databricks AI Tools are not installed), you can install them for your coding agent in seconds: + +```bash +databricks aitools install +``` + +If the CLI is not installed, see: https://docs.databricks.com/dev-tools/cli/install + +--- + +## Project Instructions + + diff --git a/lakeflow_pipelines_sql/CLAUDE.md b/lakeflow_pipelines_sql/CLAUDE.md new file mode 100644 index 00000000..5612c9bd --- /dev/null +++ b/lakeflow_pipelines_sql/CLAUDE.md @@ -0,0 +1,6 @@ +# CLAUDE.md + +Project guidance for AI agents lives in AGENTS.md. +Claude Code loads it via the import below. + +@AGENTS.md diff --git a/lakeflow_pipelines_sql/databricks.yml b/lakeflow_pipelines_sql/databricks.yml index bd42ad0e..eade6907 100644 --- a/lakeflow_pipelines_sql/databricks.yml +++ b/lakeflow_pipelines_sql/databricks.yml @@ -1,4 +1,4 @@ -# This is a Databricks asset bundle definition for lakeflow_pipelines_sql. +# This is a Declarative Automation Bundle definition for lakeflow_pipelines_sql. # See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. bundle: name: lakeflow_pipelines_sql diff --git a/lakeflow_pipelines_sql/resources/sample_job.job.yml b/lakeflow_pipelines_sql/resources/sample_job.job.yml index ea0f8e1b..974844e2 100644 --- a/lakeflow_pipelines_sql/resources/sample_job.job.yml +++ b/lakeflow_pipelines_sql/resources/sample_job.job.yml @@ -29,4 +29,4 @@ resources: environments: - environment_key: default spec: - environment_version: "4" + environment_version: "5" diff --git a/lakeflow_pipelines_sql/src/lakeflow_pipelines_sql_etl/README.md b/lakeflow_pipelines_sql/src/lakeflow_pipelines_sql_etl/README.md index e994a917..5a5ba1de 100644 --- a/lakeflow_pipelines_sql/src/lakeflow_pipelines_sql_etl/README.md +++ b/lakeflow_pipelines_sql/src/lakeflow_pipelines_sql_etl/README.md @@ -15,6 +15,6 @@ To get started, go to the `transformations` folder -- most of the relevant sourc * Take a look at the sample called "sample_trips_lakeflow_pipelines_sql.py" to get familiar with the syntax. Read more about the syntax at https://docs.databricks.com/dlt/python-ref.html. * If you're using the workspace UI, use `Run file` to run and preview a single transformation. -* If you're using the CLI, use `databricks bundle run lakeflow_pipelines_sql_etl --select sample_trips_lakeflow_pipelines_sql` to run a single transformation. +* If you're using the CLI, use `databricks bundle run lakeflow_pipelines_sql_etl --refresh sample_trips_lakeflow_pipelines_sql` to run a single transformation. For more tutorials and reference material, see https://docs.databricks.com/dlt. diff --git a/pydabs/.vscode/extensions.json b/pydabs/.vscode/extensions.json index 5ba48e79..b958aacf 100644 --- a/pydabs/.vscode/extensions.json +++ b/pydabs/.vscode/extensions.json @@ -1,7 +1,7 @@ { "recommendations": [ + "charliermarsh.ruff", "databricks.databricks", - "redhat.vscode-yaml", - "charliermarsh.ruff" + "redhat.vscode-yaml" ] } diff --git a/pydabs/AGENTS.md b/pydabs/AGENTS.md new file mode 100644 index 00000000..fdcca98a --- /dev/null +++ b/pydabs/AGENTS.md @@ -0,0 +1,25 @@ +# Declarative Automation Bundles Project + +This project uses Declarative Automation Bundles (DABs) for deployment. Add project-specific instructions below. + +## For AI Agents: Use Databricks AI Tools + +**BEFORE any other action, read the `databricks-core` skill.** + +It sets you up to work with this project reliably: CLI authentication, profile +selection, data discovery, and the bundle deployment workflow. Without it, +results are often slower and less accurate. + +If this skill is not available (Databricks AI Tools are not installed), you can install them for your coding agent in seconds: + +```bash +databricks aitools install +``` + +If the CLI is not installed, see: https://docs.databricks.com/dev-tools/cli/install + +--- + +## Project Instructions + + diff --git a/pydabs/CLAUDE.md b/pydabs/CLAUDE.md new file mode 100644 index 00000000..5612c9bd --- /dev/null +++ b/pydabs/CLAUDE.md @@ -0,0 +1,6 @@ +# CLAUDE.md + +Project guidance for AI agents lives in AGENTS.md. +Claude Code loads it via the import below. + +@AGENTS.md diff --git a/pydabs/databricks.yml b/pydabs/databricks.yml index 8bf72636..733979f3 100644 --- a/pydabs/databricks.yml +++ b/pydabs/databricks.yml @@ -1,4 +1,4 @@ -# This is a Databricks asset bundle definition for pydabs. +# This is a Declarative Automation Bundle definition for pydabs. # See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. bundle: name: pydabs diff --git a/pydabs/pyproject.toml b/pydabs/pyproject.toml index 44c02cc2..8de47ecb 100644 --- a/pydabs/pyproject.toml +++ b/pydabs/pyproject.toml @@ -2,7 +2,7 @@ name = "pydabs" version = "0.0.1" authors = [{ name = "user@company.com" }] -requires-python = ">=3.10,<3.13" +requires-python = ">=3.12,<3.13" dependencies = [ # Any dependencies for jobs and pipelines in this project can be added here # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies @@ -16,9 +16,9 @@ dev = [ "pytest", "ruff", "databricks-dlt", - "databricks-connect>=15.4,<15.5", + "databricks-connect>=16.4,<16.5", "ipykernel", - "databricks-bundles==0.295.0", + "databricks-bundles==1.18.0", ] [project.scripts] diff --git a/pydabs/resources/sample_job.py b/pydabs/resources/sample_job.py index 68093c7e..a9da7d2f 100644 --- a/pydabs/resources/sample_job.py +++ b/pydabs/resources/sample_job.py @@ -67,7 +67,7 @@ { "environment_key": "default", "spec": { - "environment_version": "4", + "environment_version": "5", "dependencies": [ # By default we just include the .whl file generated for the pydabs package. # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html diff --git a/pydabs/src/pydabs/main.py b/pydabs/src/pydabs/main.py index 0b7d9a00..2451094c 100644 --- a/pydabs/src/pydabs/main.py +++ b/pydabs/src/pydabs/main.py @@ -13,8 +13,8 @@ def main(): args = parser.parse_args() # Set the default catalog and schema - spark.sql(f"USE CATALOG {args.catalog}") - spark.sql(f"USE SCHEMA {args.schema}") + spark.sql(f"USE CATALOG `{args.catalog}`") + spark.sql(f"USE SCHEMA `{args.schema}`") # Example: just find all taxis from a sample catalog taxis.find_all_taxis().show(5) diff --git a/pydabs/src/pydabs_etl/README.md b/pydabs/src/pydabs_etl/README.md index 9211406d..4d3ab06e 100644 --- a/pydabs/src/pydabs_etl/README.md +++ b/pydabs/src/pydabs_etl/README.md @@ -15,6 +15,6 @@ To get started, go to the `transformations` folder -- most of the relevant sourc * Take a look at the sample called "sample_trips_pydabs.py" to get familiar with the syntax. Read more about the syntax at https://docs.databricks.com/dlt/python-ref.html. * If you're using the workspace UI, use `Run file` to run and preview a single transformation. -* If you're using the CLI, use `databricks bundle run pydabs_etl --select sample_trips_pydabs` to run a single transformation. +* If you're using the CLI, use `databricks bundle run pydabs_etl --refresh sample_trips_pydabs` to run a single transformation. For more tutorials and reference material, see https://docs.databricks.com/dlt. diff --git a/pydabs/src/sample_notebook.ipynb b/pydabs/src/sample_notebook.ipynb index a5ce76cd..2449125a 100644 --- a/pydabs/src/sample_notebook.ipynb +++ b/pydabs/src/sample_notebook.ipynb @@ -34,8 +34,8 @@ "# Set default catalog and schema\n", "catalog = dbutils.widgets.get(\"catalog\")\n", "schema = dbutils.widgets.get(\"schema\")\n", - "spark.sql(f\"USE CATALOG {catalog}\")\n", - "spark.sql(f\"USE SCHEMA {schema}\")" + "spark.sql(f\"USE CATALOG `{catalog}`\")\n", + "spark.sql(f\"USE SCHEMA `{schema}`\")" ] }, { @@ -72,7 +72,7 @@ "dependencies": [ "--editable .." ], - "environment_version": "4" + "environment_version": "5" }, "language": "python", "notebookMetadata": {