indexes

Creates, updates, deletes, gets or lists an indexes resource.

Overview

Name	indexes
Type	Resource
Id	databricks_workspace.vectorsearch.indexes

Fields

The following fields are returned by SELECT queries:

get

Name	Datatype	Description
name	string	Name of the index
endpoint_id	string	ID of the endpoint associated with the index.
endpoint_name	string	Name of the endpoint associated with the index
creator	string
▶delta_sync_index_spec	object
▶direct_access_index_spec	object
index_subtype	string	The subtype of the index. (FULL_TEXT, HYBRID, VECTOR)
index_type	string	There are 2 types of AI Search indexes:<br /><br />- ``DELTA_SYNC``: An index that automatically syncs with a source Delta Table, automatically and<br /> incrementally updating the index as the underlying data in the Delta Table changes.<br />- ``DIRECT_ACCESS``: An index that supports direct read and write of vectors and metadata<br /> through our REST and SDK APIs. With this model, the user manages index updates. (DELTA_SYNC, DIRECT_ACCESS)
primary_key	string	Primary key of the index
▶status	object

list

Name	Datatype	Description
name	string	Name of the index
endpoint_id	string	ID of the endpoint associated with the index.
endpoint_name	string	Name of the endpoint associated with the index
creator	string
index_subtype	string	The subtype of the index. (FULL_TEXT, HYBRID, VECTOR)
index_type	string	There are 2 types of AI Search indexes:<br /><br />- ``DELTA_SYNC``: An index that automatically syncs with a source Delta Table, automatically and<br /> incrementally updating the index as the underlying data in the Delta Table changes.<br />- ``DIRECT_ACCESS``: An index that supports direct read and write of vectors and metadata<br /> through our REST and SDK APIs. With this model, the user manages index updates. (DELTA_SYNC, DIRECT_ACCESS)
primary_key	string	Primary key of the index

Methods

The following methods are available for this resource:

Name	Accessible by	Required Params	Optional Params	Description
get	select	index_name, deployment_name, workspace_domain	ensure_reranker_compatible	Get an index.
list	select	endpoint_name, deployment_name, workspace_domain	page_token	List all indexes in the given endpoint.
create	insert	deployment_name, workspace_domain, name, endpoint_name, primary_key, index_type		Create a new index.
delete	delete	index_name, deployment_name, workspace_domain		Delete an index.
delete_data_vector_index	exec	index_name, primary_keys, deployment_name, workspace_domain		Handles the deletion of data from a specified vector index.
query_index	exec	index_name, deployment_name, workspace_domain, columns		Query the specified vector index.
query_next_page	exec	index_name, deployment_name, workspace_domain		Use next_page_token returned from previous QueryVectorIndex or QueryVectorIndexNextPage
scan_index	exec	index_name, deployment_name, workspace_domain		Scan the specified vector index and return the first num_results entries after the exclusive
sync_index	exec	index_name, deployment_name, workspace_domain		Triggers a synchronization process for a specified vector index.
upsert_data_vector_index	exec	index_name, deployment_name, workspace_domain, inputs_json		Handles the upserting of data into a specified vector index.

Parameters

Parameters can be passed in the WHERE clause of a query. Check the Methods section to see which parameters are required or optional for each operation.

Name	Datatype	Description
deployment_name	string	The Databricks workspace deployment name: the host prefix of the workspace URL (dbc-abcd0123-a1bc on AWS, adb-1234567890123456.7 on Azure, 1234567890123456.7 on GCP). Set DATABRICKS_DEPLOYMENT_NAME to default it for the session. (default: dbc-abcd0123-a1bc, x-stackQL-envVar: DATABRICKS_DEPLOYMENT_NAME)
endpoint_name	string	Name of the endpoint
index_name	string	Name of the vector index where data is to be upserted. Must be a Direct Vector Access Index.
primary_keys	array	List of primary keys for the data to be deleted.
workspace_domain	string	The Databricks workspace domain suffix for the cloud. Selected by DATABRICKS_CLOUD (aws, azure or gcp; default aws). (default: cloud.databricks.com, x-stackQL-envVarSelector: {"envVar":"DATABRICKS_CLOUD","default":"aws","values":{"aws":"cloud.databricks.com","azure":"azuredatabricks.net","gcp":"gcp.databricks.com"}})
ensure_reranker_compatible	boolean	If true, the URL returned for the index is guaranteed to be compatible with the reranker. Currently this means we return the CP URL regardless of how the index is being accessed. If not set or set to false, the URL may still be compatible with the reranker depending on what URL we return.
page_token	string	Token for pagination

SELECT examples

get

Get an index.

SELECT
name,
endpoint_id,
endpoint_name,
creator,
delta_sync_index_spec,
direct_access_index_spec,
index_subtype,
index_type,
primary_key,
status
FROM databricks_workspace.vectorsearch.indexes
WHERE index_name = '{{ index_name }}' -- required
AND deployment_name = '{{ deployment_name }}' -- required
AND workspace_domain = '{{ workspace_domain }}' -- required
AND ensure_reranker_compatible = '{{ ensure_reranker_compatible }}'
;

list

List all indexes in the given endpoint.

SELECT
name,
endpoint_id,
endpoint_name,
creator,
index_subtype,
index_type,
primary_key
FROM databricks_workspace.vectorsearch.indexes
WHERE endpoint_name = '{{ endpoint_name }}' -- required
AND deployment_name = '{{ deployment_name }}' -- required
AND workspace_domain = '{{ workspace_domain }}' -- required
AND page_token = '{{ page_token }}'
;

INSERT examples

create

Create a new index.

INSERT INTO databricks_workspace.vectorsearch.indexes (
name,
endpoint_name,
primary_key,
index_type,
delta_sync_index_spec,
direct_access_index_spec,
index_subtype,
deployment_name,
workspace_domain
)
SELECT 
'{{ name }}' /* required */,
'{{ endpoint_name }}' /* required */,
'{{ primary_key }}' /* required */,
'{{ index_type }}' /* required */,
'{{ delta_sync_index_spec }}',
'{{ direct_access_index_spec }}',
'{{ index_subtype }}',
'{{ deployment_name }}',
'{{ workspace_domain }}'
RETURNING
name,
endpoint_id,
endpoint_name,
creator,
delta_sync_index_spec,
direct_access_index_spec,
index_subtype,
index_type,
primary_key,
status
;

Manifest

# Description fields are for documentation purposes
- name: indexes
props:
  - name: deployment_name
    value: "{{ deployment_name }}"
    description: Required parameter for the indexes resource.
  - name: workspace_domain
    value: "{{ workspace_domain }}"
    description: Required parameter for the indexes resource.
  - name: name
    value: "{{ name }}"
    description: |
      Name of the index
  - name: endpoint_name
    value: "{{ endpoint_name }}"
    description: |
      Name of the endpoint to be used for serving the index
  - name: primary_key
    value: "{{ primary_key }}"
    description: |
      Primary key of the index
  - name: index_type
    value: "{{ index_type }}"
    description: |
      There are 2 types of AI Search indexes:
      - ``DELTA_SYNC``: An index that automatically syncs with a source Delta Table, automatically and
      incrementally updating the index as the underlying data in the Delta Table changes.
      - ``DIRECT_ACCESS``: An index that supports direct read and write of vectors and metadata
      through our REST and SDK APIs. With this model, the user manages index updates.
  - name: delta_sync_index_spec
    description: |
      Specification for Delta Sync Index. Required if ``index_type`` is ``DELTA_SYNC``.
    value:
      columns_to_index:
        - "{{ columns_to_index }}"
      columns_to_sync:
        - "{{ columns_to_sync }}"
      embedding_source_columns:
        - embedding_model_endpoint_name: "{{ embedding_model_endpoint_name }}"
          model_endpoint_name_for_query: "{{ model_endpoint_name_for_query }}"
          name: "{{ name }}"
      embedding_vector_columns:
        - embedding_dimension: {{ embedding_dimension }}
          name: "{{ name }}"
      embedding_writeback_table: "{{ embedding_writeback_table }}"
      pipeline_type: "{{ pipeline_type }}"
      source_table: "{{ source_table }}"
  - name: direct_access_index_spec
    description: |
      Specification for Direct Vector Access Index. Required if ``index_type`` is ``DIRECT_ACCESS``.
    value:
      embedding_source_columns:
        - embedding_model_endpoint_name: "{{ embedding_model_endpoint_name }}"
          model_endpoint_name_for_query: "{{ model_endpoint_name_for_query }}"
          name: "{{ name }}"
      embedding_vector_columns:
        - embedding_dimension: {{ embedding_dimension }}
          name: "{{ name }}"
      schema_json: "{{ schema_json }}"
  - name: index_subtype
    value: "{{ index_subtype }}"
    description: |
      The subtype of the index. Use ``HYBRID`` or ``FULL_TEXT``. ``VECTOR`` is not supported.

DELETE examples

delete

Delete an index.

DELETE FROM databricks_workspace.vectorsearch.indexes
WHERE index_name = '{{ index_name }}' --required
AND deployment_name = '{{ deployment_name }}' --required
AND workspace_domain = '{{ workspace_domain }}' --required
;

Lifecycle Methods

delete_data_vector_index

Handles the deletion of data from a specified vector index.

EXEC databricks_workspace.vectorsearch.indexes.delete_data_vector_index 
@index_name='{{ index_name }}' --required, 
@primary_keys='{{ primary_keys }}' --required, 
@deployment_name='{{ deployment_name }}' --required, 
@workspace_domain='{{ workspace_domain }}' --required
;

query_index

Query the specified vector index.

EXEC databricks_workspace.vectorsearch.indexes.query_index 
@index_name='{{ index_name }}' --required, 
@deployment_name='{{ deployment_name }}' --required, 
@workspace_domain='{{ workspace_domain }}' --required 
@@json=
'{
"columns": "{{ columns }}", 
"columns_to_rerank": "{{ columns_to_rerank }}", 
"facets": "{{ facets }}", 
"filters_json": "{{ filters_json }}", 
"num_results": {{ num_results }}, 
"query_columns": "{{ query_columns }}", 
"query_text": "{{ query_text }}", 
"query_type": "{{ query_type }}", 
"query_vector": "{{ query_vector }}", 
"reranker": "{{ reranker }}", 
"score_threshold": {{ score_threshold }}, 
"sort_columns": "{{ sort_columns }}"
}'
;

query_next_page

Use next_page_token returned from previous QueryVectorIndex or QueryVectorIndexNextPage

EXEC databricks_workspace.vectorsearch.indexes.query_next_page 
@index_name='{{ index_name }}' --required, 
@deployment_name='{{ deployment_name }}' --required, 
@workspace_domain='{{ workspace_domain }}' --required 
@@json=
'{
"endpoint_name": "{{ endpoint_name }}", 
"page_token": "{{ page_token }}"
}'
;

scan_index

Scan the specified vector index and return the first num_results entries after the exclusive

EXEC databricks_workspace.vectorsearch.indexes.scan_index 
@index_name='{{ index_name }}' --required, 
@deployment_name='{{ deployment_name }}' --required, 
@workspace_domain='{{ workspace_domain }}' --required 
@@json=
'{
"last_primary_key": "{{ last_primary_key }}", 
"num_results": {{ num_results }}
}'
;

sync_index

Triggers a synchronization process for a specified vector index.

EXEC databricks_workspace.vectorsearch.indexes.sync_index 
@index_name='{{ index_name }}' --required, 
@deployment_name='{{ deployment_name }}' --required, 
@workspace_domain='{{ workspace_domain }}' --required
;

upsert_data_vector_index

Handles the upserting of data into a specified vector index.

EXEC databricks_workspace.vectorsearch.indexes.upsert_data_vector_index 
@index_name='{{ index_name }}' --required, 
@deployment_name='{{ deployment_name }}' --required, 
@workspace_domain='{{ workspace_domain }}' --required 
@@json=
'{
"inputs_json": "{{ inputs_json }}"
}'
;