Skip to main content

Download notebook · View source · Open in Colab

Work with Descriptors (Embeddings)

A Descriptor stores an embedding: a numeric vector with optional metadata. A descriptor class groups embeddings in a common search space and defines their dimensions, index engine, and distance metric. This example stores a recipe embedding, searches for it, updates its metadata, and removes it.

The examples require Python 3.10 or later, an ApertureDB connection profile or environment credentials, and permission to create, read, update, and delete objects. A unique documentation_run property separates the example data from other records. Cleanup uses that value, so keep the setup variables available until cleanup completes.

Connect to ApertureDB​

ApertureDB Cloud The SDK connects with an API key from your Cloud instance. Obtain the key from Connect > Generate API Key and add it to a .env file in this directory:

APERTUREDB_KEY=your_key_here

Local evaluation: follow the deployment guide, then configure your connection profile.

%pip install --quiet "aperturedb>=1.0.1,<2" "python-dotenv==1.1.1"
# Option A: ApertureDB Cloud
from dotenv import load_dotenv
load_dotenv() # loads APERTUREDB_KEY from .env into the environment
# For local evaluation, configure the profile from your versioned deployment package.
# https://docs.aperturedata.io/administration/deployment/DockerCompose
from aperturedb.CommonLibrary import create_connector

client = create_connector()

import uuid

documentation_run = uuid.uuid4().hex
ingredient_class = "CookbookIngredient_" + documentation_run
connection_class = "CookbookIngredientLink_" + documentation_run

def checked_query(query, input_blobs=None):
response, output_blobs = client.query(query, input_blobs or [])
assert isinstance(response, list) and len(response) == len(query), response
for command in response:
result = next(iter(command.values()))
assert result.get("status") == 0, response
return response, output_blobs

response, _ = checked_query([{"GetStatus": {}}])
client.print_last_response()

Define the search space for recipes​

CreateIndex creates a named search space for 1,024-dimensional recipe embeddings. This example selects the Flat engine and L2 distance metric.

import uuid
descriptor_class = "recipe_search_" + uuid.uuid4().hex[:12]
q = [{"CreateIndex": {"target": "descriptor", "class": descriptor_class, "property": "_blob", "kind": "flat", "params": {"dimensions": 1024, "metric": "L2"}, "metadata": {
"year_created": 2023,
"source": "ApertureDB cookbook dataset",
"model": "embed-english-v3.0",
"provider": "cohere"
}}}]

responses, blobs = checked_query(q)

print(client.get_last_response_str())

Prepare the embedding to be added​

Embeddings can be loaded from a NumPy file or generated by an embedding model. The supplied file contains one 1,024-dimensional vector, which the example converts to float32 bytes for ingestion.

import numpy as np

# Download the sample file
from pathlib import Path
from urllib.request import urlopen
import hashlib

fixture_path = Path("data/embeddings.npy")
fixture_path.parent.mkdir(exist_ok=True)
fixture_url = "https://raw.githubusercontent.com/aperture-data/Cookbook/e333f6c59070b9165033d9ddd5af852a6b9624ba/notebooks/simple/data/embeddings.npy"
fixture_sha256 = "5ec479dd18d5a407c5081616279f4684a50910b3e8d51bd60b7cfdf827ec9d8f"
if not fixture_path.exists() or hashlib.sha256(fixture_path.read_bytes()).hexdigest() != fixture_sha256:
with urlopen(fixture_url, timeout=60) as response:
fixture_path.write_bytes(response.read())
assert hashlib.sha256(fixture_path.read_bytes()).hexdigest() == fixture_sha256
print("Verified fixture:", fixture_path.name)

# File can contain multiple descriptors of matching dimensions. So you can load them
# by passing the right index
embedding_npy_array = np.load("data/embeddings.npy")
index = 0 # We currently have only one embedding in the file
embedding = embedding_npy_array[index]

# expected byte array format for adding descriptor in ApertureDB
embedding_bytes = embedding.astype('float32').tobytes()

assert embedding.shape == (1024,)
assert np.isfinite(embedding).all()

Add a Recipe Embedding for Similarity Search Later​

AddDescriptor stores the vector in the named set with a label and metadata. The vector bytes accompany the command as an input blob.

The Python SDK loaders handle bulk ingestion.

q = [{
"AddDescriptor": {
'class': descriptor_class, # descriptors / embeddings are always added to a set / collection
"label": "dinner",
"properties": {
'documentation_run': documentation_run,
"id": 75,
"year_created": 2023,
"source": "Cookbook recipe",
"model": "embed-english-v3.0",
"provider": "cohere"
},
"if_not_found": ['all', ['$id', '==', 75], ['$documentation_run', '==', documentation_run]]
}
}]

responses, blobs = checked_query(q, [embedding_bytes])

print(client.get_last_response_str())

K-NN Search For Matching Embedding​

FindDescriptor searches the set using the same vector that was added. Because the set contains one descriptor, the response contains one match, including its label, properties, distance, and vector bytes.

q = [{
"FindDescriptor": {"engine": 'Flat', "metric": 'L2',
'constraints': ['$documentation_run', '==', documentation_run],
# Specify the descriptor set in which to search.
'with_class': descriptor_class,
"k_neighbors": 6,
"distances": True,
"labels": True,
"blobs": True,
"results": {
"all_properties": True
}
}
}]

responses, blobs = checked_query(q, [embedding_bytes])

print(client.get_last_response_str())

assert responses[0]["FindDescriptor"]["returned"] == 1
assert len(blobs) == 1
np.testing.assert_array_equal(np.frombuffer(blobs[0], dtype="float32"), embedding.astype("float32"))

Remove Extra Properties with Update​

UpdateDescriptor with remove_props removes the year_created property from the matching descriptor. The vector remains in the set.

q = [{
"UpdateDescriptor": {
"constraints": ['all', ['$id', '==', 75], ['$documentation_run', '==', documentation_run]],
"remove_props": ["year_created"]
}
}]


responses, blobs = checked_query(q)

print(client.get_last_response_str())

Read the updated descriptor​

The descriptor is still returned, but its properties no longer include year_created.

q = [{
"FindDescriptor": {
# Specify the descriptor set in which to search.
'with_class': descriptor_class,
"constraints": ['all', ['$id', '==', 75], ['$documentation_run', '==', documentation_run]],
"results": {
"all_properties": True
}
}
}]

responses, blobs = checked_query(q)

print(client.get_last_response_str())

assert responses[0]["FindDescriptor"]["returned"] == 1
assert "year_created" not in responses[0]["FindDescriptor"]["entities"][0]

Remove the Embedding​

DeleteDescriptor removes the descriptor selected by its metadata and this run's identifier.

q = [{
"DeleteDescriptor": {
'with_class': descriptor_class,
"constraints": ['all', ['$provider', '==', 'cohere'], ['$documentation_run', '==', documentation_run]]
}
}]

responses, blobs = checked_query(q)

print(client.get_last_response_str())

Verify Deletion​

The same descriptor query returns no matches after deletion.

q = [{
"FindDescriptor": {
# Specify the descriptor set in which to search.
'with_class': descriptor_class,
"constraints": ['all', ['$id', '==', 75], ['$documentation_run', '==', documentation_run]],
"results": {
"all_properties": True
}
}
}]

responses, blobs = checked_query(q)

print(client.get_last_response_str())

assert responses[0]["FindDescriptor"]["returned"] == 0

Remove the descriptor class​

descriptor-class removal removes the set created for this run.

from aperturedb.Utils import Utils

assert Utils(client).remove_descriptorset(descriptor_class), client.get_last_response_str()
remaining, _ = client.query([{"GetIndexes": {"target": "descriptor", "class": descriptor_class, "property": "_blob"}}])
assert remaining[0]["GetIndexes"]["status"] == 0, remaining
assert remaining[0]["GetIndexes"]["returned"] == 0
print("Descriptor class and vector index removed")

What's next?​