feature_engineering_streams

Creates, updates, deletes, gets or lists a feature_engineering_streams resource.

Overview

Name	feature_engineering_streams
Type	Resource
Id	databricks_workspace.ml.feature_engineering_streams

Fields

The following fields are returned by SELECT queries:

get

Name	Datatype	Description
name	string	Full three-part (catalog.schema.stream) name of the stream.
browse_only	boolean	Indicates whether the principal is limited to retrieving metadata for the associated object through the BROWSE privilege when include_browse is enabled in the request.
▶connection_config	object	Specifies how to connect and authenticate to the stream platform.
create_time	string (date-time)	Time at which this Stream was created.
created_by	string	Username of the Stream creator.
description	string	User-provided description.
▶ingestion_config	object	Configuration for streaming data ingestion: the managed table storing an offline copy of forward fill data and optional historical backfill.
▶schema_config	object	Schema definitions for the stream, provided either directly on the Stream or resolved from an external schema registry through a UC Connection.
▶source_config	object	Source-specific configuration. Determines the streaming platform source.
update_time	string (date-time)	Time at which this Stream was last modified.
updated_by	string	Username of user who last modified the Stream.

list

Name	Datatype	Description
name	string	Full three-part (catalog.schema.stream) name of the stream.
browse_only	boolean	Indicates whether the principal is limited to retrieving metadata for the associated object through the BROWSE privilege when include_browse is enabled in the request.
▶connection_config	object	Specifies how to connect and authenticate to the stream platform.
create_time	string (date-time)	Time at which this Stream was created.
created_by	string	Username of the Stream creator.
description	string	User-provided description.
▶ingestion_config	object	Configuration for streaming data ingestion: the managed table storing an offline copy of forward fill data and optional historical backfill.
▶schema_config	object	Schema definitions for the stream, provided either directly on the Stream or resolved from an external schema registry through a UC Connection.
▶source_config	object	Source-specific configuration. Determines the streaming platform source.
update_time	string (date-time)	Time at which this Stream was last modified.
updated_by	string	Username of user who last modified the Stream.

Methods

The following methods are available for this resource:

Name	Accessible by	Required Params	Optional Params	Description
get	select	name, deployment_name		Get a Stream by its full three-part name (catalog.schema.stream).
list	select	deployment_name	page_size, page_token, parent	List Streams under a given catalog.schema parent.
create	insert	deployment_name, stream		Create a Stream, a governed UC entity representing an external streaming data source.
update	update	name, update_mask, deployment_name, stream		Update a Stream. Only fields listed in update_mask are mutated.
delete	delete	name, deployment_name		Delete a Stream by its full three-part name (catalog.schema.stream).

Parameters

Parameters can be passed in the WHERE clause of a query. Check the Methods section to see which parameters are required or optional for each operation.

Name	Datatype	Description
deployment_name	string	The Databricks Workspace Deployment Name (default: dbc-abcd0123-a1bc)
name	string	Full three-part name (catalog.schema.stream) of the Stream to delete.
update_mask	object	The list of fields to update.
page_size	integer	The maximum number of results to return.
page_token	string	Pagination token to go to the next page based on a previous query.
parent	string	Two-part name (catalog.schema) of the parent under which to list Streams.

SELECT examples

get

Get a Stream by its full three-part name (catalog.schema.stream).

SELECT
name,
browse_only,
connection_config,
create_time,
created_by,
description,
ingestion_config,
schema_config,
source_config,
update_time,
updated_by
FROM databricks_workspace.ml.feature_engineering_streams
WHERE name = '{{ name }}' -- required
AND deployment_name = '{{ deployment_name }}' -- required
;

list

List Streams under a given catalog.schema parent.

SELECT
name,
browse_only,
connection_config,
create_time,
created_by,
description,
ingestion_config,
schema_config,
source_config,
update_time,
updated_by
FROM databricks_workspace.ml.feature_engineering_streams
WHERE deployment_name = '{{ deployment_name }}' -- required
AND page_size = '{{ page_size }}'
AND page_token = '{{ page_token }}'
AND parent = '{{ parent }}'
;

INSERT examples

create

Create a Stream, a governed UC entity representing an external streaming data source.

INSERT INTO databricks_workspace.ml.feature_engineering_streams (
stream,
deployment_name
)
SELECT 
'{{ stream }}' /* required */,
'{{ deployment_name }}'
RETURNING
name,
browse_only,
connection_config,
create_time,
created_by,
description,
ingestion_config,
schema_config,
source_config,
update_time,
updated_by
;

Manifest

# Description fields are for documentation purposes
- name: feature_engineering_streams
props:
  - name: deployment_name
    value: "{{ deployment_name }}"
    description: Required parameter for the feature_engineering_streams resource.
  - name: stream
    description: |
      The Stream to create.
    value:
      name: "{{ name }}"
      source_config:
        kafka_stream_config:
          subscription_mode:
            assign: "{{ assign }}"
            subscribe: "{{ subscribe }}"
            subscribe_pattern: "{{ subscribe_pattern }}"
          extra_options: "{{ extra_options }}"
        kinesis_stream_config:
          extra_options: "{{ extra_options }}"
          stream_arns:
            arns:
              - "{{ arns }}"
          stream_names:
            names:
              - "{{ names }}"
      connection_config:
        direct_mtls_config:
          bootstrap_servers: "{{ bootstrap_servers }}"
          mtls_config:
            keystore_location: "{{ keystore_location }}"
            keystore_password_ref:
              scope: "{{ scope }}"
              key: "{{ key }}"
            key_password_ref:
              scope: "{{ scope }}"
              key: "{{ key }}"
            truststore_location: "{{ truststore_location }}"
            truststore_password_ref:
              scope: "{{ scope }}"
              key: "{{ key }}"
            disable_hostname_verification: {{ disable_hostname_verification }}
        uc_connection_name: "{{ uc_connection_name }}"
      schema_config:
        direct_schemas:
          key_schema:
            avro_schema: "{{ avro_schema }}"
            json_schema: "{{ json_schema }}"
            proto_schema:
              schema_text: "{{ schema_text }}"
              message_name: "{{ message_name }}"
          payload_schema:
            avro_schema: "{{ avro_schema }}"
            json_schema: "{{ json_schema }}"
            proto_schema:
              schema_text: "{{ schema_text }}"
              message_name: "{{ message_name }}"
        schema_registry_config:
          api_secret_ref:
            scope: "{{ scope }}"
            key: "{{ key }}"
          key_schema_locator:
            format: "{{ format }}"
            confluent_schema:
              subject: "{{ subject }}"
          payload_schema_locator:
            format: "{{ format }}"
            confluent_schema:
              subject: "{{ subject }}"
          uc_connection: "{{ uc_connection }}"
      ingestion_config:
        ingestion_destination:
          delta_table_name: "{{ delta_table_name }}"
        backfill_job_id: {{ backfill_job_id }}
        backfill_source:
          delta_table_name: "{{ delta_table_name }}"
          delta_table_source:
            full_name: "{{ full_name }}"
            dataframe_schema: "{{ dataframe_schema }}"
            entity_columns:
              - "{{ entity_columns }}"
            filter_condition: "{{ filter_condition }}"
            timeseries_column: "{{ timeseries_column }}"
            transformation_sql: "{{ transformation_sql }}"
        deduplication_columns:
          - "{{ deduplication_columns }}"
        ingestion_job_id: {{ ingestion_job_id }}
        ingestion_pipeline_id: "{{ ingestion_pipeline_id }}"
      browse_only: {{ browse_only }}
      create_time: "{{ create_time }}"
      created_by: "{{ created_by }}"
      description: "{{ description }}"
      update_time: "{{ update_time }}"
      updated_by: "{{ updated_by }}"

UPDATE examples

update

Update a Stream. Only fields listed in update_mask are mutated.

UPDATE databricks_workspace.ml.feature_engineering_streams
SET 
stream = '{{ stream }}'
WHERE 
name = '{{ name }}' --required
AND update_mask = '{{ update_mask }}' --required
AND deployment_name = '{{ deployment_name }}' --required
AND stream = '{{ stream }}' --required
RETURNING
name,
browse_only,
connection_config,
create_time,
created_by,
description,
ingestion_config,
schema_config,
source_config,
update_time,
updated_by;

DELETE examples

delete

Delete a Stream by its full three-part name (catalog.schema.stream).

DELETE FROM databricks_workspace.ml.feature_engineering_streams
WHERE name = '{{ name }}' --required
AND deployment_name = '{{ deployment_name }}' --required
;