The Databricks Command Cheat Sheet Every Data Engineer Should Bookmark

The Spark SQL, Delta Lake, and PySpark commands that come up constantly in day-to-day Databricks work.

Delta Table Basics

SQL
-- Create a managed Delta table
CREATE TABLE sales.orders (
    order_id    BIGINT,
    customer_id BIGINT,
    order_date  DATE,
    amount      DECIMAL(12,2)
)
USING DELTA
PARTITIONED BY (order_date);

-- Create from a query
CREATE TABLE sales.orders_summary
USING DELTA
AS SELECT order_date, SUM(amount) AS total FROM sales.orders GROUP BY order_date;

Auto Loader (Incremental File Ingestion)

Code
df = (spark.readStream
      .format("cloudFiles")
      .option("cloudFiles.format", "parquet")
      .option("cloudFiles.schemaLocation", "/mnt/schema/orders")
      .load("/mnt/raw/orders/"))

(df.writeStream
   .format("delta")
   .option("checkpointLocation", "/mnt/checkpoints/orders")
   .trigger(availableNow=True)
   .table("sales.orders_bronze"))

MERGE INTO (Upserts)

SQL
MERGE INTO sales.orders AS target
USING staging.orders_incremental AS source
ON target.order_id = source.order_id
WHEN MATCHED THEN UPDATE SET *
WHEN NOT MATCHED THEN INSERT *;

Code
# PySpark equivalent
from delta.tables import DeltaTable

target = DeltaTable.forName(spark, "sales.orders")
(target.alias("t")
 .merge(source_df.alias("s"), "t.order_id = s.order_id")
 .whenMatchedUpdateAll()
 .whenNotMatchedInsertAll()
 .execute())

Time Travel and History

SQL
-- Query a previous version
SELECT * FROM sales.orders VERSION AS OF 12;
SELECT * FROM sales.orders TIMESTAMP AS OF '2026-08-01';

-- View change history
DESCRIBE HISTORY sales.orders;

-- Restore to a previous version
RESTORE TABLE sales.orders TO VERSION AS OF 12;

Table Maintenance

SQL
-- Compact small files
OPTIMIZE sales.orders;

-- Z-order for multi-column filter performance
OPTIMIZE sales.orders ZORDER BY (customer_id, order_date);

-- Remove old, unreferenced files (default retention: 7 days)
VACUUM sales.orders RETAIN 168 HOURS;

-- Check table detail and size
DESCRIBE DETAIL sales.orders;

Unity Catalog

SQL
-- Grants
GRANT SELECT ON TABLE sales.orders TO `analyst_group`;
GRANT USAGE ON CATALOG main TO `analyst_group`;
GRANT ALL PRIVILEGES ON SCHEMA sales TO `etl_group`;

-- Three-level namespace
SELECT * FROM main.sales.orders;

-- Row filters and column masks (Unity Catalog)
CREATE FUNCTION main.sales.region_filter(region STRING)
RETURN IS_ACCOUNT_GROUP_MEMBER('admin') OR region = current_user();

ALTER TABLE sales.orders SET ROW FILTER main.sales.region_filter ON (region);

Job and Cluster Diagnostics

Code
# List running jobs (Databricks CLI / SDK)
# databricks jobs list

# Get cluster event log for troubleshooting slow starts
# databricks clusters events --cluster-id <id>

SQL
-- Query history and performance via system tables
SELECT
  statement_id, execution_status, total_duration_ms, statement_text
FROM system.query.history
WHERE start_time >= current_timestamp() - INTERVAL 1 DAY
ORDER BY total_duration_ms DESC
LIMIT 20;

-- Cluster utilization
SELECT * FROM system.compute.clusters;

Databricks CLI Essentials

Shell
# Configure
databricks configure --token

# Run a job
databricks jobs run-now --job-id 12345

# Upload a notebook
databricks workspace import notebook.py /Workspace/Shared/notebook

# Deploy via Asset Bundles
databricks bundle deploy -t prod

OPTIMIZE

ZORDER

VACUUM