feature_engineering

Creates, updates, deletes, gets or lists a feature_engineering resource.

Overview

Name	feature_engineering
Type	Resource
Id	databricks_workspace.ml.feature_engineering

Fields

The following fields are returned by SELECT queries:

get

Name	Datatype	Description
full_name	string
description	string	The description of the feature.
▶entities	array	The entity columns for the feature, used as aggregation keys and for query-time lookup.
filter_condition	string	Deprecated: Use DeltaTableSource.filter_condition or KafkaSource.filter_condition instead. Kept for backwards compatibility. The filter condition applied to the source data before aggregation.
▶function	object	The function by which the feature is computed.
inputs	array	Deprecated: Use AggregationFunction.inputs instead. Kept for backwards compatibility. The input columns from which the feature is computed.
▶lineage_context	object	Lineage context information for this feature. WARNING: This field is primarily intended for internal use by Databricks systems and is automatically populated when features are created through Databricks notebooks or jobs. Users should not manually set this field as incorrect values may lead to inaccurate lineage tracking or unexpected behavior. This field will be set by feature-engineering client and should be left unset by SDK and terraform users.
▶source	object	The data source of the feature.
▶time_window	object	Deprecated: Use Function.aggregation_function.time_window instead. Kept for backwards compatibility. The time window in which the feature is computed.
▶timeseries_column	object	Column recording time, used for point-in-time joins, backfills, and aggregations.

list

Name	Datatype	Description
full_name	string
description	string	The description of the feature.
▶entities	array	The entity columns for the feature, used as aggregation keys and for query-time lookup.
filter_condition	string	Deprecated: Use DeltaTableSource.filter_condition or KafkaSource.filter_condition instead. Kept for backwards compatibility. The filter condition applied to the source data before aggregation.
▶function	object	The function by which the feature is computed.
inputs	array	Deprecated: Use AggregationFunction.inputs instead. Kept for backwards compatibility. The input columns from which the feature is computed.
▶lineage_context	object	Lineage context information for this feature. WARNING: This field is primarily intended for internal use by Databricks systems and is automatically populated when features are created through Databricks notebooks or jobs. Users should not manually set this field as incorrect values may lead to inaccurate lineage tracking or unexpected behavior. This field will be set by feature-engineering client and should be left unset by SDK and terraform users.
▶source	object	The data source of the feature.
▶time_window	object	Deprecated: Use Function.aggregation_function.time_window instead. Kept for backwards compatibility. The time window in which the feature is computed.
▶timeseries_column	object	Column recording time, used for point-in-time joins, backfills, and aggregations.

Methods

The following methods are available for this resource:

Name	Accessible by	Required Params	Optional Params	Description
get	select	full_name, deployment_name		Get a Feature.
list	select	deployment_name	page_size, page_token	List Features.
create	insert	deployment_name, feature		Create a Feature.
update	update	full_name, update_mask, deployment_name, feature		Update a Feature.
delete	delete	full_name, deployment_name		Delete a Feature.

Parameters

Parameters can be passed in the WHERE clause of a query. Check the Methods section to see which parameters are required or optional for each operation.

Name	Datatype	Description
deployment_name	string	The Databricks Workspace Deployment Name (default: dbc-abcd0123-a1bc)
full_name	string	Name of the feature to delete.
update_mask	string	The list of fields to update.
page_size	integer	The maximum number of results to return.
page_token	string	Pagination token to go to the next page based on a previous query.

SELECT examples

get

Get a Feature.

SELECT
full_name,
description,
entities,
filter_condition,
function,
inputs,
lineage_context,
source,
time_window,
timeseries_column
FROM databricks_workspace.ml.feature_engineering
WHERE full_name = '{{ full_name }}' -- required
AND deployment_name = '{{ deployment_name }}' -- required
;

list

List Features.

SELECT
full_name,
description,
entities,
filter_condition,
function,
inputs,
lineage_context,
source,
time_window,
timeseries_column
FROM databricks_workspace.ml.feature_engineering
WHERE deployment_name = '{{ deployment_name }}' -- required
AND page_size = '{{ page_size }}'
AND page_token = '{{ page_token }}'
;

INSERT examples

create

Create a Feature.

INSERT INTO databricks_workspace.ml.feature_engineering (
feature,
deployment_name
)
SELECT 
'{{ feature }}' /* required */,
'{{ deployment_name }}'
RETURNING
full_name,
description,
entities,
filter_condition,
function,
inputs,
lineage_context,
source,
time_window,
timeseries_column
;

Manifest

# Description fields are for documentation purposes
- name: feature_engineering
props:
  - name: deployment_name
    value: "{{ deployment_name }}"
    description: Required parameter for the feature_engineering resource.
  - name: feature
    description: |
      Feature to create.
    value:
      full_name: "{{ full_name }}"
      source:
        delta_table_source:
          full_name: "{{ full_name }}"
          dataframe_schema: "{{ dataframe_schema }}"
          entity_columns:
            - "{{ entity_columns }}"
          filter_condition: "{{ filter_condition }}"
          timeseries_column: "{{ timeseries_column }}"
          transformation_sql: "{{ transformation_sql }}"
        kafka_source:
          name: "{{ name }}"
          entity_column_identifiers:
            - variant_expr_path: "{{ variant_expr_path }}"
          filter_condition: "{{ filter_condition }}"
          timeseries_column_identifier:
            variant_expr_path: "{{ variant_expr_path }}"
      function:
        aggregation_function:
          approx_count_distinct:
            input: "{{ input }}"
            relative_sd: {{ relative_sd }}
          approx_percentile:
            input: "{{ input }}"
            percentile: {{ percentile }}
            accuracy: {{ accuracy }}
          avg:
            input: "{{ input }}"
          count_function:
            input: "{{ input }}"
          first:
            input: "{{ input }}"
          last:
            input: "{{ input }}"
          max:
            input: "{{ input }}"
          min:
            input: "{{ input }}"
          stddev_pop:
            input: "{{ input }}"
          stddev_samp:
            input: "{{ input }}"
          sum:
            input: "{{ input }}"
          time_window:
            continuous:
              window_duration: "{{ window_duration }}"
              offset: "{{ offset }}"
            sliding:
              window_duration: "{{ window_duration }}"
              slide_duration: "{{ slide_duration }}"
            tumbling:
              window_duration: "{{ window_duration }}"
          var_pop:
            input: "{{ input }}"
          var_samp:
            input: "{{ input }}"
        extra_parameters:
          - key: "{{ key }}"
            value: "{{ value }}"
        function_type: "{{ function_type }}"
      description: "{{ description }}"
      entities:
        - name: "{{ name }}"
      filter_condition: "{{ filter_condition }}"
      inputs:
        - "{{ inputs }}"
      lineage_context:
        job_context:
          job_id: {{ job_id }}
          job_run_id: {{ job_run_id }}
        notebook_id: {{ notebook_id }}
      time_window:
        continuous:
          window_duration: "{{ window_duration }}"
          offset: "{{ offset }}"
        sliding:
          window_duration: "{{ window_duration }}"
          slide_duration: "{{ slide_duration }}"
        tumbling:
          window_duration: "{{ window_duration }}"
      timeseries_column:
        name: "{{ name }}"

UPDATE examples

update

Update a Feature.

UPDATE databricks_workspace.ml.feature_engineering
SET 
feature = '{{ feature }}'
WHERE 
full_name = '{{ full_name }}' --required
AND update_mask = '{{ update_mask }}' --required
AND deployment_name = '{{ deployment_name }}' --required
AND feature = '{{ feature }}' --required
RETURNING
full_name,
description,
entities,
filter_condition,
function,
inputs,
lineage_context,
source,
time_window,
timeseries_column;

DELETE examples

delete

Delete a Feature.

DELETE FROM databricks_workspace.ml.feature_engineering
WHERE full_name = '{{ full_name }}' --required
AND deployment_name = '{{ deployment_name }}' --required
;