indexes

Creates, updates, deletes, gets or lists an indexes resource.

Overview

Name	indexes
Type	Resource
Id	databricks_workspace.vectorsearch.indexes

Fields

The following fields are returned by SELECT queries:

get

Name	Datatype	Description
name	string	Name of the index
endpoint_name	string	Name of the endpoint associated with the index
creator	string
▶delta_sync_index_spec	object
▶direct_access_index_spec	object
index_type	string	There are 2 types of Vector Search indexes: - `DELTA_SYNC`: An index that automatically syncs<br />with a source Delta Table, automatically and incrementally updating the index as the underlying<br />data in the Delta Table changes. - `DIRECT_ACCESS`: An index that supports direct read and write<br />of vectors and metadata through our REST and SDK APIs. With this model, the user manages index<br />updates. (DELTA_SYNC, DIRECT_ACCESS)
primary_key	string	Primary key of the index
▶status	object

list

Name	Datatype	Description
name	string	Name of the index
endpoint_name	string	Name of the endpoint associated with the index
creator	string
index_type	string	There are 2 types of Vector Search indexes: - `DELTA_SYNC`: An index that automatically syncs<br />with a source Delta Table, automatically and incrementally updating the index as the underlying<br />data in the Delta Table changes. - `DIRECT_ACCESS`: An index that supports direct read and write<br />of vectors and metadata through our REST and SDK APIs. With this model, the user manages index<br />updates. (DELTA_SYNC, DIRECT_ACCESS)
primary_key	string	Primary key of the index

Methods

The following methods are available for this resource:

Name	Accessible by	Required Params	Optional Params	Description
get	select	index_name, deployment_name	ensure_reranker_compatible	Get an index.
list	select	endpoint_name, deployment_name	page_token	List all indexes in the given endpoint.
create	insert	deployment_name, name, endpoint_name, primary_key, index_type		Create a new index.
delete	delete	index_name, deployment_name		Delete an index.
delete_data_vector_index	exec	index_name, primary_keys, deployment_name		Handles the deletion of data from a specified vector index.
query_index	exec	index_name, deployment_name, columns		Query the specified vector index.
query_next_page	exec	index_name, deployment_name		Use next_page_token returned from previous QueryVectorIndex or QueryVectorIndexNextPage request
scan_index	exec	index_name, deployment_name		Scan the specified vector index and return the first num_results entries after the exclusive
sync_index	exec	index_name, deployment_name		Triggers a synchronization process for a specified vector index.
upsert_data_vector_index	exec	index_name, deployment_name, inputs_json		Handles the upserting of data into a specified vector index.

Parameters

Parameters can be passed in the WHERE clause of a query. Check the Methods section to see which parameters are required or optional for each operation.

Name	Datatype	Description
deployment_name	string	The Databricks Workspace Deployment Name (default: dbc-abcd0123-a1bc)
endpoint_name	string	Name of the endpoint
index_name	string	Name of the vector index where data is to be upserted. Must be a Direct Vector Access Index.
primary_keys	array	List of primary keys for the data to be deleted.
ensure_reranker_compatible	boolean	If true, the URL returned for the index is guaranteed to be compatible with the reranker. Currently this means we return the CP URL regardless of how the index is being accessed. If not set or set to false, the URL may still be compatible with the reranker depending on what URL we return.
page_token	string	Token for pagination

SELECT examples

get

Get an index.

SELECT
name,
endpoint_name,
creator,
delta_sync_index_spec,
direct_access_index_spec,
index_type,
primary_key,
status
FROM databricks_workspace.vectorsearch.indexes
WHERE index_name = '{{ index_name }}' -- required
AND deployment_name = '{{ deployment_name }}' -- required
AND ensure_reranker_compatible = '{{ ensure_reranker_compatible }}'
;

list

List all indexes in the given endpoint.

SELECT
name,
endpoint_name,
creator,
index_type,
primary_key
FROM databricks_workspace.vectorsearch.indexes
WHERE endpoint_name = '{{ endpoint_name }}' -- required
AND deployment_name = '{{ deployment_name }}' -- required
AND page_token = '{{ page_token }}'
;

INSERT examples

create

Create a new index.

INSERT INTO databricks_workspace.vectorsearch.indexes (
name,
endpoint_name,
primary_key,
index_type,
delta_sync_index_spec,
direct_access_index_spec,
deployment_name
)
SELECT 
'{{ name }}' /* required */,
'{{ endpoint_name }}' /* required */,
'{{ primary_key }}' /* required */,
'{{ index_type }}' /* required */,
'{{ delta_sync_index_spec }}',
'{{ direct_access_index_spec }}',
'{{ deployment_name }}'
RETURNING
name,
endpoint_name,
creator,
delta_sync_index_spec,
direct_access_index_spec,
index_type,
primary_key,
status
;

Manifest

# Description fields are for documentation purposes
- name: indexes
props:
  - name: deployment_name
    value: "{{ deployment_name }}"
    description: Required parameter for the indexes resource.
  - name: name
    value: "{{ name }}"
    description: |
      Name of the index
  - name: endpoint_name
    value: "{{ endpoint_name }}"
    description: |
      Name of the endpoint to be used for serving the index
  - name: primary_key
    value: "{{ primary_key }}"
    description: |
      Primary key of the index
  - name: index_type
    value: "{{ index_type }}"
    description: |
      There are 2 types of Vector Search indexes: - `DELTA_SYNC`: An index that automatically syncs
      with a source Delta Table, automatically and incrementally updating the index as the underlying
      data in the Delta Table changes. - `DIRECT_ACCESS`: An index that supports direct read and write
      of vectors and metadata through our REST and SDK APIs. With this model, the user manages index
      updates.
  - name: delta_sync_index_spec
    description: |
      Specification for Delta Sync Index. Required if `index_type` is `DELTA_SYNC`.
    value:
      columns_to_sync:
        - "{{ columns_to_sync }}"
      embedding_source_columns:
        - embedding_model_endpoint_name: "{{ embedding_model_endpoint_name }}"
          model_endpoint_name_for_query: "{{ model_endpoint_name_for_query }}"
          name: "{{ name }}"
      embedding_vector_columns:
        - embedding_dimension: {{ embedding_dimension }}
          name: "{{ name }}"
      embedding_writeback_table: "{{ embedding_writeback_table }}"
      pipeline_type: "{{ pipeline_type }}"
      source_table: "{{ source_table }}"
  - name: direct_access_index_spec
    description: |
      Specification for Direct Vector Access Index. Required if `index_type` is `DIRECT_ACCESS`.
    value:
      embedding_source_columns:
        - embedding_model_endpoint_name: "{{ embedding_model_endpoint_name }}"
          model_endpoint_name_for_query: "{{ model_endpoint_name_for_query }}"
          name: "{{ name }}"
      embedding_vector_columns:
        - embedding_dimension: {{ embedding_dimension }}
          name: "{{ name }}"
      schema_json: "{{ schema_json }}"

DELETE examples

delete

Delete an index.

DELETE FROM databricks_workspace.vectorsearch.indexes
WHERE index_name = '{{ index_name }}' --required
AND deployment_name = '{{ deployment_name }}' --required
;

Lifecycle Methods

delete_data_vector_index

Handles the deletion of data from a specified vector index.

EXEC databricks_workspace.vectorsearch.indexes.delete_data_vector_index 
@index_name='{{ index_name }}' --required, 
@primary_keys='{{ primary_keys }}' --required, 
@deployment_name='{{ deployment_name }}' --required
;

query_index

Query the specified vector index.

EXEC databricks_workspace.vectorsearch.indexes.query_index 
@index_name='{{ index_name }}' --required, 
@deployment_name='{{ deployment_name }}' --required 
@@json=
'{
"columns": "{{ columns }}", 
"columns_to_rerank": "{{ columns_to_rerank }}", 
"filters_json": "{{ filters_json }}", 
"num_results": {{ num_results }}, 
"query_text": "{{ query_text }}", 
"query_type": "{{ query_type }}", 
"query_vector": "{{ query_vector }}", 
"reranker": "{{ reranker }}", 
"score_threshold": {{ score_threshold }}
}'
;

query_next_page

Use next_page_token returned from previous QueryVectorIndex or QueryVectorIndexNextPage request

EXEC databricks_workspace.vectorsearch.indexes.query_next_page 
@index_name='{{ index_name }}' --required, 
@deployment_name='{{ deployment_name }}' --required 
@@json=
'{
"endpoint_name": "{{ endpoint_name }}", 
"page_token": "{{ page_token }}"
}'
;

scan_index

Scan the specified vector index and return the first num_results entries after the exclusive

EXEC databricks_workspace.vectorsearch.indexes.scan_index 
@index_name='{{ index_name }}' --required, 
@deployment_name='{{ deployment_name }}' --required 
@@json=
'{
"last_primary_key": "{{ last_primary_key }}", 
"num_results": {{ num_results }}
}'
;

sync_index

Triggers a synchronization process for a specified vector index.

EXEC databricks_workspace.vectorsearch.indexes.sync_index 
@index_name='{{ index_name }}' --required, 
@deployment_name='{{ deployment_name }}' --required
;

upsert_data_vector_index

Handles the upserting of data into a specified vector index.

EXEC databricks_workspace.vectorsearch.indexes.upsert_data_vector_index 
@index_name='{{ index_name }}' --required, 
@deployment_name='{{ deployment_name }}' --required 
@@json=
'{
"inputs_json": "{{ inputs_json }}"
}'
;