external_lineage

Creates, updates, deletes, gets or lists an external_lineage resource.

Overview

Name	external_lineage
Type	Resource
Id	databricks_workspace.catalog.external_lineage

Fields

The following fields are returned by SELECT queries:

list

Name	Datatype	Description
▶external_lineage_info	object	Information about the edge metadata of the external lineage relationship.
▶external_metadata_info	object	Information about external metadata involved in the lineage relationship.
▶file_info	object	Information about the file involved in the lineage relationship.
▶model_info	object	Information about the model version involved in the lineage relationship.
▶table_info	object	Information about the table involved in the lineage relationship.

Methods

The following methods are available for this resource:

Name	Accessible by	Required Params	Optional Params	Description
list	select	object_info, lineage_direction, deployment_name	page_size, page_token	Lists external lineage relationships of a Databricks object or external metadata given a supplied
create	insert	deployment_name, external_lineage_relationship		Creates an external lineage relationship between a Databricks or external metadata object and another
update	update	update_mask, deployment_name, external_lineage_relationship		Updates an external lineage relationship between a Databricks or external metadata object and another
delete	delete	external_lineage_relationship, deployment_name		Deletes an external lineage relationship between a Databricks or external metadata object and another

Parameters

Parameters can be passed in the WHERE clause of a query. Check the Methods section to see which parameters are required or optional for each operation.

Name	Datatype	Description
deployment_name	string	The Databricks Workspace Deployment Name (default: dbc-abcd0123-a1bc)
external_lineage_relationship	object	:class:DeleteRequestExternalLineage
lineage_direction	string	The lineage direction to filter on.
object_info	object	The object to query external lineage relationships for. Since this field is a query parameter, please flatten the nested fields. For example, if the object is a table, the query parameter should look like: object_info.table.name=main.sales.customers
update_mask	string
page_size	integer	Specifies the maximum number of external lineage relationships to return in a single response. The value must be less than or equal to 1000.
page_token	string	Opaque pagination token to go to next page based on previous query.

SELECT examples

list

Lists external lineage relationships of a Databricks object or external metadata given a supplied

SELECT
external_lineage_info,
external_metadata_info,
file_info,
model_info,
table_info
FROM databricks_workspace.catalog.external_lineage
WHERE object_info = '{{ object_info }}' -- required
AND lineage_direction = '{{ lineage_direction }}' -- required
AND deployment_name = '{{ deployment_name }}' -- required
AND page_size = '{{ page_size }}'
AND page_token = '{{ page_token }}'
;

INSERT examples

create

Creates an external lineage relationship between a Databricks or external metadata object and another

INSERT INTO databricks_workspace.catalog.external_lineage (
external_lineage_relationship,
deployment_name
)
SELECT 
'{{ external_lineage_relationship }}' /* required */,
'{{ deployment_name }}'
RETURNING
id,
columns,
properties,
source,
target
;

Manifest

# Description fields are for documentation purposes
- name: external_lineage
props:
  - name: deployment_name
    value: "{{ deployment_name }}"
    description: Required parameter for the external_lineage resource.
  - name: external_lineage_relationship
    description: |
      :returns: :class:`ExternalLineageRelationship`
    value:
      source:
        external_metadata:
          name: "{{ name }}"
        model_version:
          name: "{{ name }}"
          version: "{{ version }}"
        path:
          url: "{{ url }}"
        table:
          name: "{{ name }}"
      target:
        external_metadata:
          name: "{{ name }}"
        model_version:
          name: "{{ name }}"
          version: "{{ version }}"
        path:
          url: "{{ url }}"
        table:
          name: "{{ name }}"
      columns:
        - source: "{{ source }}"
          target: "{{ target }}"
      id: "{{ id }}"
      properties: "{{ properties }}"

UPDATE examples

update

Updates an external lineage relationship between a Databricks or external metadata object and another

UPDATE databricks_workspace.catalog.external_lineage
SET 
external_lineage_relationship = '{{ external_lineage_relationship }}'
WHERE 
update_mask = '{{ update_mask }}' --required
AND deployment_name = '{{ deployment_name }}' --required
AND external_lineage_relationship = '{{ external_lineage_relationship }}' --required
RETURNING
id,
columns,
properties,
source,
target;

DELETE examples

delete

Deletes an external lineage relationship between a Databricks or external metadata object and another

DELETE FROM databricks_workspace.catalog.external_lineage
WHERE external_lineage_relationship = '{{ external_lineage_relationship }}' --required
AND deployment_name = '{{ deployment_name }}' --required
;