Data Lakehouse (Medallion Architecture)
Four source systems through CDC and streaming ingest into bronze / silver / gold S3 zones with an Iceberg catalogue, transformed by Spark and dbt, served to BI, ML and a data API under Lake Formation governance.
Rendering…
Make it your own.
{
"nodes": [
{ "id": "fing", "type": "container", "label": "Ingest", "x": 240, "y": 40, "width": 220, "height": 620, "fill": "rgba(224,242,254,0.28)", "stroke": "#0369a1" },
{ "id": "flak", "type": "container", "label": "S3 lake — medallion", "x": 500, "y": 40, "width": 260, "height": 620, "fill": "rgba(254,243,199,0.28)", "stroke": "#b45309" },
{ "id": "fpro", "type": "container", "label": "Process & serve", "x": 800, "y": 40, "width": 260, "height": 620, "fill": "rgba(219,234,254,0.28)", "stroke": "#1d4ed8" },
{ "id": "src1", "type": "cloud-sql", "label": "Postgres — orders", "x": 40, "y": 60, "width": 160, "height": 104, "fill": "#e0f2fe", "stroke": "#0369a1" },
{ "id": "src2", "type": "aws-dynamo", "label": "DynamoDB — events", "x": 40, "y": 210, "width": 140, "height": 110, "fill": "#fff7ed", "stroke": "#f97316" },
{ "id": "src3", "type": "cloud", "label": "SaaS APIs — CRM, ads","x": 40, "y": 360, "width": 160, "height": 104, "fill": "#e0f2fe", "stroke": "#0369a1" },
{ "id": "src4", "type": "iot-sensor", "label": "Store telemetry", "x": 40, "y": 510, "width": 150, "height": 104, "fill": "#e0f2fe", "stroke": "#0369a1" },
{ "id": "dms", "type": "conveyor", "label": "DMS — CDC stream", "x": 270, "y": 90, "width": 160, "height": 104, "fill": "#e0f2fe", "stroke": "#0369a1" },
{ "id": "kds", "type": "message-queue","label": "Kinesis Firehose", "x": 270, "y": 240, "width": 160, "height": 104, "fill": "#e0f2fe", "stroke": "#0369a1" },
{ "id": "air", "type": "cog-cluster", "label": "Airbyte connectors", "x": 270, "y": 390, "width": 160, "height": 104, "fill": "#e0f2fe", "stroke": "#0369a1" },
{ "id": "iotc", "type": "antenna", "label": "IoT Core → Firehose", "x": 270, "y": 540, "width": 160, "height": 104, "fill": "#e0f2fe", "stroke": "#0369a1" },
{ "id": "bron", "type": "data-lake", "label": "Bronze — raw, partitioned", "x": 530, "y": 90, "width": 190, "height": 104, "fill": "#fef3c7", "stroke": "#b45309" },
{ "id": "silv", "type": "data-lake", "label": "Silver — cleaned, deduped", "x": 530, "y": 250, "width": 190, "height": 104, "fill": "#fef3c7", "stroke": "#b45309" },
{ "id": "gold", "type": "data-warehouse", "label": "Gold — conformed marts", "x": 530, "y": 410, "width": 190, "height": 104, "fill": "#fef3c7", "stroke": "#b45309", "bold": true },
{ "id": "cat", "type": "entity-table", "label": "Glue catalogue + Iceberg", "x": 530, "y": 540, "width": 190, "height": 104, "fill": "#fef3c7", "stroke": "#b45309" },
{ "id": "emr", "type": "cog-cluster", "label": "EMR Spark — batch ETL", "x": 830, "y": 90, "width": 190, "height": 104, "fill": "#dbeafe", "stroke": "#1d4ed8" },
{ "id": "dbt", "type": "gear", "label": "dbt — transformations", "x": 830, "y": 250, "width": 190, "height": 104, "fill": "#dbeafe", "stroke": "#1d4ed8" },
{ "id": "ath", "type": "funnel", "label": "Athena — ad-hoc SQL", "x": 830, "y": 410, "width": 190, "height": 104, "fill": "#dbeafe", "stroke": "#1d4ed8" },
{ "id": "rs", "type": "data-warehouse", "label": "Redshift Serverless", "x": 830, "y": 540, "width": 190, "height": 104, "fill": "#dbeafe", "stroke": "#1d4ed8" },
{ "id": "bi", "type": "bar-chart", "label": "Looker dashboards", "x": 1100, "y": 90, "width": 170, "height": 104, "fill": "#dcfce7", "stroke": "#15803d" },
{ "id": "ml", "type": "neural-net", "label": "Feature store + ML", "x": 1100, "y": 250, "width": 170, "height": 104, "fill": "#dcfce7", "stroke": "#15803d" },
{ "id": "api", "type": "microservice", "label": "Data API", "x": 1100, "y": 410, "width": 170, "height": 104, "fill": "#dcfce7", "stroke": "#15803d" },
{ "id": "gov", "type": "shield-check", "label": "Lake Formation — row/column ACLs", "x": 1100, "y": 540, "width": 180, "height": 104, "fill": "#fee2e2", "stroke": "#b91c1c" }
],
"edges": [
{ "id": "d1", "source": "src1", "target": "dms" },
{ "id": "d2", "source": "src2", "target": "kds" },
{ "id": "d3", "source": "src3", "target": "air" },
{ "id": "d4", "source": "src4", "target": "iotc" },
{ "id": "d5", "source": "dms", "target": "bron", "label": "append-only", "thickness": 2 },
{ "id": "d6", "source": "kds", "target": "bron" },
{ "id": "d7", "source": "air", "target": "bron" },
{ "id": "d8", "source": "iotc", "target": "bron" },
{ "id": "d9", "source": "bron", "target": "emr", "label": "read raw" },
{ "id": "d10", "source": "emr", "target": "silv", "label": "quality rules", "thickness": 2 },
{ "id": "d11", "source": "silv", "target": "dbt" },
{ "id": "d12", "source": "dbt", "target": "gold", "label": "marts + tests", "thickness": 2 },
{ "id": "d13", "source": "gold", "target": "ath" },
{ "id": "d14", "source": "gold", "target": "rs", "label": "materialise" },
{ "id": "d15", "source": "gold", "target": "cat", "label": "register tables", "style": "dashed" },
{ "id": "d16", "source": "cat", "target": "ath", "label": "schema", "style": "dashed" },
{ "id": "d17", "source": "ath", "target": "bi" },
{ "id": "d18", "source": "rs", "target": "bi" },
{ "id": "d19", "source": "gold", "target": "ml", "label": "training sets" },
{ "id": "d20", "source": "rs", "target": "api" },
{ "id": "d21", "source": "gov", "target": "gold", "label": "governed access", "style": "dashed" }
]
}