{
"description": "Data Lakehouse at-least-once delivery profile for AidboxTopicDestination. Default writeMode 'managed-zerobus' streams writes directly into a Databricks Unity Catalog managed Delta table via the Zerobus gRPC streaming-ingest SDK (no SQL warehouse on the hot path; pay-per-row ingest). Initial bulk export uses a temporary staging external Delta table on the customer's bucket and merges via the SQL warehouse on the resource id. 'managed-sql' targets the same managed table but routes every batch through the Databricks SQL warehouse (Statement Execution API) — same staging+MERGE for initial bulk; use this when Zerobus isn't available on the customer's SKU. 'external-direct' falls back to writing Parquet directly to an external Delta table on S3/GCS/ADLS via the Delta Kernel writer (no SQL warehouse, restart-safe-idempotent via Delta txn action).",
"_filename": "StructureDefinition-aidboxtopicdestination-dataLakehouseAtLeastOnceProfile.json",
"package_name": "io.health-samurai.core.r5",
"date": null,
"derivation": "constraint",
"publisher": "Health Samurai",
"fhirVersion": "5.0.0",
"name": "DataLakehouseAtLeastOnceProfile",
"abstract": false,
"type": "AidboxTopicDestination",
"experimental": null,
"resourceType": "StructureDefinition",
"title": "Data Lakehouse At Least Once Profile",
"package_version": "0.2605.0",
"status": "active",
"id": "12639360-9494-44bc-85e8-9312c40874bd",
"kind": "resource",
"url": "http://health-samurai.io/fhir/core/StructureDefinition/aidboxtopicdestination-dataLakehouseAtLeastOnceProfile",
"version": "0.2605.0",
"differential": {
"element": [ {
"id": "AidboxTopicDestination.kind",
"path": "AidboxTopicDestination.kind",
"fixedString": "data-lakehouse-at-least-once"
}, {
"id": "AidboxTopicDestination.parameter",
"path": "AidboxTopicDestination.parameter",
"slicing": {
"rules": "closed",
"discriminator": [ {
"path": "name",
"type": "pattern"
} ]
}
}, {
"id": "AidboxTopicDestination.parameter:tablePath",
"max": "1",
"min": 0,
"path": "AidboxTopicDestination.parameter",
"short": "Delta table URI, e.g. s3://my-bucket/path/to/table. Must exist (created via Databricks UI / SQL). Required when databricksWorkspaceUrl is absent; ignored when set (table location resolved from Unity Catalog instead).",
"sliceName": "tablePath"
}, {
"id": "AidboxTopicDestination.parameter:tablePath.name",
"path": "AidboxTopicDestination.parameter.name",
"fixedString": "tablePath"
}, {
"id": "AidboxTopicDestination.parameter:tablePath.value[x]",
"min": 1,
"path": "AidboxTopicDestination.parameter.value[x]",
"type": [ {
"code": "string"
} ]
}, {
"id": "AidboxTopicDestination.parameter:tablePath.resource",
"max": "0",
"path": "AidboxTopicDestination.parameter.resource"
}, {
"id": "AidboxTopicDestination.parameter:tablePath.part",
"max": "0",
"path": "AidboxTopicDestination.parameter.part"
}, {
"id": "AidboxTopicDestination.parameter:viewDefinition",
"max": "1",
"min": 1,
"path": "AidboxTopicDestination.parameter",
"short": "Name of the ViewDefinition resource for data transformation",
"sliceName": "viewDefinition"
}, {
"id": "AidboxTopicDestination.parameter:viewDefinition.name",
"path": "AidboxTopicDestination.parameter.name",
"fixedString": "viewDefinition"
}, {
"id": "AidboxTopicDestination.parameter:viewDefinition.value[x]",
"min": 1,
"path": "AidboxTopicDestination.parameter.value[x]",
"type": [ {
"code": "string"
} ]
}, {
"id": "AidboxTopicDestination.parameter:viewDefinition.resource",
"max": "0",
"path": "AidboxTopicDestination.parameter.resource"
}, {
"id": "AidboxTopicDestination.parameter:viewDefinition.part",
"max": "0",
"path": "AidboxTopicDestination.parameter.part"
}, {
"id": "AidboxTopicDestination.parameter:awsRegion",
"max": "1",
"min": 0,
"path": "AidboxTopicDestination.parameter",
"short": "AWS region for the S3 bucket (required for real AWS; MinIO/Garage accept us-east-1)",
"sliceName": "awsRegion"
}, {
"id": "AidboxTopicDestination.parameter:awsRegion.name",
"path": "AidboxTopicDestination.parameter.name",
"fixedString": "awsRegion"
}, {
"id": "AidboxTopicDestination.parameter:awsRegion.value[x]",
"min": 1,
"path": "AidboxTopicDestination.parameter.value[x]",
"type": [ {
"code": "string"
} ]
}, {
"id": "AidboxTopicDestination.parameter:awsRegion.resource",
"max": "0",
"path": "AidboxTopicDestination.parameter.resource"
}, {
"id": "AidboxTopicDestination.parameter:awsRegion.part",
"max": "0",
"path": "AidboxTopicDestination.parameter.part"
}, {
"id": "AidboxTopicDestination.parameter:awsAccessKeyId",
"max": "1",
"min": 0,
"path": "AidboxTopicDestination.parameter",
"short": "AWS access key ID (omit to use default provider chain: env / EC2 / EKS IRSA). Supports vault refs.",
"sliceName": "awsAccessKeyId"
}, {
"id": "AidboxTopicDestination.parameter:awsAccessKeyId.name",
"path": "AidboxTopicDestination.parameter.name",
"fixedString": "awsAccessKeyId"
}, {
"id": "AidboxTopicDestination.parameter:awsAccessKeyId.value[x]",
"min": 1,
"path": "AidboxTopicDestination.parameter.value[x]",
"type": [ {
"code": "string"
} ]
}, {
"id": "AidboxTopicDestination.parameter:awsAccessKeyId.resource",
"max": "0",
"path": "AidboxTopicDestination.parameter.resource"
}, {
"id": "AidboxTopicDestination.parameter:awsAccessKeyId.part",
"max": "0",
"path": "AidboxTopicDestination.parameter.part"
}, {
"id": "AidboxTopicDestination.parameter:awsSecretAccessKey",
"max": "1",
"min": 0,
"path": "AidboxTopicDestination.parameter",
"short": "AWS secret access key. Pair with awsAccessKeyId. Supports vault refs.",
"sliceName": "awsSecretAccessKey"
}, {
"id": "AidboxTopicDestination.parameter:awsSecretAccessKey.name",
"path": "AidboxTopicDestination.parameter.name",
"fixedString": "awsSecretAccessKey"
}, {
"id": "AidboxTopicDestination.parameter:awsSecretAccessKey.value[x]",
"min": 1,
"path": "AidboxTopicDestination.parameter.value[x]",
"type": [ {
"code": "string"
} ]
}, {
"id": "AidboxTopicDestination.parameter:awsSecretAccessKey.resource",
"max": "0",
"path": "AidboxTopicDestination.parameter.resource"
}, {
"id": "AidboxTopicDestination.parameter:awsSecretAccessKey.part",
"max": "0",
"path": "AidboxTopicDestination.parameter.part"
}, {
"id": "AidboxTopicDestination.parameter:s3Endpoint",
"max": "1",
"min": 0,
"path": "AidboxTopicDestination.parameter",
"short": "Custom S3 endpoint (MinIO / Garage / LocalStack). Forces path-style URLs.",
"sliceName": "s3Endpoint"
}, {
"id": "AidboxTopicDestination.parameter:s3Endpoint.name",
"path": "AidboxTopicDestination.parameter.name",
"fixedString": "s3Endpoint"
}, {
"id": "AidboxTopicDestination.parameter:s3Endpoint.value[x]",
"min": 1,
"path": "AidboxTopicDestination.parameter.value[x]",
"type": [ {
"code": "string"
} ]
}, {
"id": "AidboxTopicDestination.parameter:s3Endpoint.resource",
"max": "0",
"path": "AidboxTopicDestination.parameter.resource"
}, {
"id": "AidboxTopicDestination.parameter:s3Endpoint.part",
"max": "0",
"path": "AidboxTopicDestination.parameter.part"
}, {
"id": "AidboxTopicDestination.parameter:batchSize",
"max": "1",
"min": 1,
"path": "AidboxTopicDestination.parameter",
"short": "Number of messages per Delta commit batch",
"sliceName": "batchSize"
}, {
"id": "AidboxTopicDestination.parameter:batchSize.name",
"path": "AidboxTopicDestination.parameter.name",
"fixedString": "batchSize"
}, {
"id": "AidboxTopicDestination.parameter:batchSize.value[x]",
"min": 1,
"path": "AidboxTopicDestination.parameter.value[x]",
"type": [ {
"code": "unsignedInt"
} ]
}, {
"id": "AidboxTopicDestination.parameter:batchSize.resource",
"max": "0",
"path": "AidboxTopicDestination.parameter.resource"
}, {
"id": "AidboxTopicDestination.parameter:batchSize.part",
"max": "0",
"path": "AidboxTopicDestination.parameter.part"
}, {
"id": "AidboxTopicDestination.parameter:sendIntervalMs",
"max": "1",
"min": 1,
"path": "AidboxTopicDestination.parameter",
"short": "Maximum time between commits in milliseconds",
"sliceName": "sendIntervalMs"
}, {
"id": "AidboxTopicDestination.parameter:sendIntervalMs.name",
"path": "AidboxTopicDestination.parameter.name",
"fixedString": "sendIntervalMs"
}, {
"id": "AidboxTopicDestination.parameter:sendIntervalMs.value[x]",
"min": 1,
"path": "AidboxTopicDestination.parameter.value[x]",
"type": [ {
"code": "unsignedInt"
} ]
}, {
"id": "AidboxTopicDestination.parameter:sendIntervalMs.resource",
"max": "0",
"path": "AidboxTopicDestination.parameter.resource"
}, {
"id": "AidboxTopicDestination.parameter:sendIntervalMs.part",
"max": "0",
"path": "AidboxTopicDestination.parameter.part"
}, {
"id": "AidboxTopicDestination.parameter:targetFileSizeMb",
"max": "1",
"min": 0,
"path": "AidboxTopicDestination.parameter",
"short": "Target Parquet file size in MiB during initial export (default 128). Avoid tiny files.",
"sliceName": "targetFileSizeMb"
}, {
"id": "AidboxTopicDestination.parameter:targetFileSizeMb.name",
"path": "AidboxTopicDestination.parameter.name",
"fixedString": "targetFileSizeMb"
}, {
"id": "AidboxTopicDestination.parameter:targetFileSizeMb.value[x]",
"min": 1,
"path": "AidboxTopicDestination.parameter.value[x]",
"type": [ {
"code": "unsignedInt"
} ]
}, {
"id": "AidboxTopicDestination.parameter:targetFileSizeMb.resource",
"max": "0",
"path": "AidboxTopicDestination.parameter.resource"
}, {
"id": "AidboxTopicDestination.parameter:targetFileSizeMb.part",
"max": "0",
"path": "AidboxTopicDestination.parameter.part"
}, {
"id": "AidboxTopicDestination.parameter:initialExportChunkCount",
"max": "1",
"min": 0,
"path": "AidboxTopicDestination.parameter",
"short": "Number of hash-partitioned chunks the initial export splits into. Default 1 (single cursor). Chunks distribute across Aidbox pods and write into per-chunk staging Delta tables that are merged into the managed target in one MERGE INTO at finalize.",
"sliceName": "initialExportChunkCount"
}, {
"id": "AidboxTopicDestination.parameter:initialExportChunkCount.name",
"path": "AidboxTopicDestination.parameter.name",
"fixedString": "initialExportChunkCount"
}, {
"id": "AidboxTopicDestination.parameter:initialExportChunkCount.value[x]",
"min": 1,
"path": "AidboxTopicDestination.parameter.value[x]",
"type": [ {
"code": "unsignedInt"
} ]
}, {
"id": "AidboxTopicDestination.parameter:initialExportChunkCount.resource",
"max": "0",
"path": "AidboxTopicDestination.parameter.resource"
}, {
"id": "AidboxTopicDestination.parameter:initialExportChunkCount.part",
"max": "0",
"path": "AidboxTopicDestination.parameter.part"
}, {
"id": "AidboxTopicDestination.parameter:skipInitialExport",
"max": "1",
"min": 0,
"path": "AidboxTopicDestination.parameter",
"short": "Skip initial export of existing data (default: false)",
"sliceName": "skipInitialExport"
}, {
"id": "AidboxTopicDestination.parameter:skipInitialExport.name",
"path": "AidboxTopicDestination.parameter.name",
"fixedString": "skipInitialExport"
}, {
"id": "AidboxTopicDestination.parameter:skipInitialExport.value[x]",
"min": 1,
"path": "AidboxTopicDestination.parameter.value[x]",
"type": [ {
"code": "boolean"
} ]
}, {
"id": "AidboxTopicDestination.parameter:skipInitialExport.resource",
"max": "0",
"path": "AidboxTopicDestination.parameter.resource"
}, {
"id": "AidboxTopicDestination.parameter:skipInitialExport.part",
"max": "0",
"path": "AidboxTopicDestination.parameter.part"
}, {
"id": "AidboxTopicDestination.parameter:databricksWorkspaceUrl",
"max": "1",
"min": 0,
"path": "AidboxTopicDestination.parameter",
"short": "Databricks workspace URL, e.g. https://workspace.cloud.databricks.com. PRESENCE of this parameter activates Unity Catalog credential vending — tablePath and AWS credentials are then resolved from UC.",
"sliceName": "databricksWorkspaceUrl"
}, {
"id": "AidboxTopicDestination.parameter:databricksWorkspaceUrl.name",
"path": "AidboxTopicDestination.parameter.name",
"fixedString": "databricksWorkspaceUrl"
}, {
"id": "AidboxTopicDestination.parameter:databricksWorkspaceUrl.value[x]",
"min": 1,
"path": "AidboxTopicDestination.parameter.value[x]",
"type": [ {
"code": "string"
} ]
}, {
"id": "AidboxTopicDestination.parameter:databricksWorkspaceUrl.resource",
"max": "0",
"path": "AidboxTopicDestination.parameter.resource"
}, {
"id": "AidboxTopicDestination.parameter:databricksWorkspaceUrl.part",
"max": "0",
"path": "AidboxTopicDestination.parameter.part"
}, {
"id": "AidboxTopicDestination.parameter:tableName",
"max": "1",
"min": 0,
"path": "AidboxTopicDestination.parameter",
"short": "Unity Catalog table full name: catalog.schema.table. Required when databricksWorkspaceUrl is set.",
"sliceName": "tableName"
}, {
"id": "AidboxTopicDestination.parameter:tableName.name",
"path": "AidboxTopicDestination.parameter.name",
"fixedString": "tableName"
}, {
"id": "AidboxTopicDestination.parameter:tableName.value[x]",
"min": 1,
"path": "AidboxTopicDestination.parameter.value[x]",
"type": [ {
"code": "string"
} ]
}, {
"id": "AidboxTopicDestination.parameter:tableName.resource",
"max": "0",
"path": "AidboxTopicDestination.parameter.resource"
}, {
"id": "AidboxTopicDestination.parameter:tableName.part",
"max": "0",
"path": "AidboxTopicDestination.parameter.part"
}, {
"id": "AidboxTopicDestination.parameter:writeMode",
"max": "1",
"min": 0,
"path": "AidboxTopicDestination.parameter",
"short": "Write path strategy. Default managed-zerobus (gRPC streaming-ingest direct into a Unity Catalog managed Delta table; no warehouse hours). Set managed-sql to route inserts through a Databricks SQL warehouse (same target, requires warm warehouse). Set external-direct to write directly into a non-managed external Delta table (S3 / MinIO / GCS / ADLS) via Kernel.",
"sliceName": "writeMode"
}, {
"id": "AidboxTopicDestination.parameter:writeMode.name",
"path": "AidboxTopicDestination.parameter.name",
"fixedString": "writeMode"
}, {
"id": "AidboxTopicDestination.parameter:writeMode.value[x]",
"min": 1,
"path": "AidboxTopicDestination.parameter.value[x]",
"type": [ {
"code": "string"
} ]
}, {
"id": "AidboxTopicDestination.parameter:writeMode.resource",
"max": "0",
"path": "AidboxTopicDestination.parameter.resource"
}, {
"id": "AidboxTopicDestination.parameter:writeMode.part",
"max": "0",
"path": "AidboxTopicDestination.parameter.part"
}, {
"id": "AidboxTopicDestination.parameter:databricksWarehouseId",
"max": "1",
"min": 0,
"path": "AidboxTopicDestination.parameter",
"short": "Databricks SQL warehouse ID. Required when writeMode=managed-sql; the warehouse executes INSERT / COPY INTO / ALTER on behalf of this destination. Also required for managed-zerobus when initial-export is enabled (the staging-table schema sync still uses INFORMATION_SCHEMA via the warehouse).",
"sliceName": "databricksWarehouseId"
}, {
"id": "AidboxTopicDestination.parameter:databricksWarehouseId.name",
"path": "AidboxTopicDestination.parameter.name",
"fixedString": "databricksWarehouseId"
}, {
"id": "AidboxTopicDestination.parameter:databricksWarehouseId.value[x]",
"min": 1,
"path": "AidboxTopicDestination.parameter.value[x]",
"type": [ {
"code": "string"
} ]
}, {
"id": "AidboxTopicDestination.parameter:databricksWarehouseId.resource",
"max": "0",
"path": "AidboxTopicDestination.parameter.resource"
}, {
"id": "AidboxTopicDestination.parameter:databricksWarehouseId.part",
"max": "0",
"path": "AidboxTopicDestination.parameter.part"
}, {
"id": "AidboxTopicDestination.parameter:stagingTablePath",
"max": "1",
"min": 0,
"path": "AidboxTopicDestination.parameter",
"short": "S3/GCS/ADLS URI for an external Delta table used as staging during managed-sql or managed-zerobus mode's initial-export phase. Required when writeMode is managed-sql or managed-zerobus AND skipInitialExport is not true. The customer must pre-configure a UC external location at this path.",
"sliceName": "stagingTablePath"
}, {
"id": "AidboxTopicDestination.parameter:stagingTablePath.name",
"path": "AidboxTopicDestination.parameter.name",
"fixedString": "stagingTablePath"
}, {
"id": "AidboxTopicDestination.parameter:stagingTablePath.value[x]",
"min": 1,
"path": "AidboxTopicDestination.parameter.value[x]",
"type": [ {
"code": "string"
} ]
}, {
"id": "AidboxTopicDestination.parameter:stagingTablePath.resource",
"max": "0",
"path": "AidboxTopicDestination.parameter.resource"
}, {
"id": "AidboxTopicDestination.parameter:stagingTablePath.part",
"max": "0",
"path": "AidboxTopicDestination.parameter.part"
}, {
"id": "AidboxTopicDestination.parameter:databricksWorkspaceId",
"max": "1",
"min": 0,
"path": "AidboxTopicDestination.parameter",
"short": "Numeric Databricks workspace ID (e.g. 1234567890123456). Required when writeMode=managed-zerobus; used together with databricksRegion to compose the Zerobus gRPC server endpoint host.",
"sliceName": "databricksWorkspaceId"
}, {
"id": "AidboxTopicDestination.parameter:databricksWorkspaceId.name",
"path": "AidboxTopicDestination.parameter.name",
"fixedString": "databricksWorkspaceId"
}, {
"id": "AidboxTopicDestination.parameter:databricksWorkspaceId.value[x]",
"min": 1,
"path": "AidboxTopicDestination.parameter.value[x]",
"type": [ {
"code": "string"
} ]
}, {
"id": "AidboxTopicDestination.parameter:databricksWorkspaceId.resource",
"max": "0",
"path": "AidboxTopicDestination.parameter.resource"
}, {
"id": "AidboxTopicDestination.parameter:databricksWorkspaceId.part",
"max": "0",
"path": "AidboxTopicDestination.parameter.part"
}, {
"id": "AidboxTopicDestination.parameter:databricksRegion",
"max": "1",
"min": 0,
"path": "AidboxTopicDestination.parameter",
"short": "AWS region of the Databricks workspace, e.g. us-east-1. Required when writeMode=managed-zerobus; used together with databricksWorkspaceId to compose the Zerobus gRPC server endpoint host.",
"sliceName": "databricksRegion"
}, {
"id": "AidboxTopicDestination.parameter:databricksRegion.name",
"path": "AidboxTopicDestination.parameter.name",
"fixedString": "databricksRegion"
}, {
"id": "AidboxTopicDestination.parameter:databricksRegion.value[x]",
"min": 1,
"path": "AidboxTopicDestination.parameter.value[x]",
"type": [ {
"code": "string"
} ]
}, {
"id": "AidboxTopicDestination.parameter:databricksRegion.resource",
"max": "0",
"path": "AidboxTopicDestination.parameter.resource"
}, {
"id": "AidboxTopicDestination.parameter:databricksRegion.part",
"max": "0",
"path": "AidboxTopicDestination.parameter.part"
} ]
},
"baseDefinition": "http://health-samurai.io/fhir/core/StructureDefinition/AidboxTopicDestination"
}