# Data Engineering with Python - course dependencies
# Python 3.10+
# Pinned upper bounds keep the labs reproducible; raise them deliberately.

# --- Module 1 (Local Pipeline) ---
requests>=2.31,<3            # HTTP client for gharchive.org downloads
pandas>=2.1,<3               # DataFrame ergonomics in L01-L02
pyarrow>=14.0,<17            # Parquet + Arrow in L02
great-expectations>=0.18,<1  # Data quality suite in L03

# --- Module 2 (Orchestration) - exercised from L04 onward ---
apache-airflow>=2.9,<3       # Pinned to 2.x; 3.x will be addressed in a later refresh

# --- Module 3 (SQL Foundations) ---
dbt-core>=1.7,<2             # dbt project in L08
dbt-duckdb>=1.7,<2           # DuckDB adapter for dbt (lets the dbt lab run offline)

# --- Module 4 (Modern Lakehouse) ---
duckdb>=1.0,<2               # Local analytical engine in L09-L11
pyiceberg[sql-sqlite,pyarrow,duckdb]>=0.7,<1   # L10: SqlCatalog + write_from_parquet + DuckDB-on-Iceberg reads
# Cloud warehouse adapters (optional; L11 Track B works without these)
snowflake-connector-python>=3.10 ; python_version >= "3.10"
google-cloud-bigquery>=3.20 ; python_version >= "3.10"

# --- Module 5 (Production) ---
openlineage-python>=1.10,<2  # Lineage / observability in L12