pythonadvanced

pgvector Semantic Search in Python

Store OpenAI embeddings in PostgreSQL with pgvector extension for scalable semantic search.

python
from openai import OpenAI
import psycopg2
import numpy as np

client = OpenAI()
conn   = psycopg2.connect('postgresql://user:password@localhost:5432/mydb')
cur    = conn.cursor()

cur.execute('CREATE EXTENSION IF NOT EXISTS vector')
cur.execute('CREATE TABLE IF NOT EXISTS documents (id SERIAL PRIMARY KEY, content TEXT, embedding vector(1536))')
cur.execute('CREATE INDEX IF NOT EXISTS doc_emb_idx ON documents USING ivfflat (embedding vector_cosine_ops) WITH (lists=100)')
conn.commit()

def add_document(text: str) -> None:
    resp = client.embeddings.create(input=text, model='text-embedding-3-small')
    emb  = resp.data[0].embedding
    cur.execute('INSERT INTO documents (content, embedding) VALUES (%s, %s)', (text, emb))
    conn.commit()

def semantic_search(query: str, top_k: int = 5) -> list[tuple]:
    resp = client.embeddings.create(input=query, model='text-embedding-3-small')
    qemb = resp.data[0].embedding
    cur.execute('SELECT content, 1-(embedding <=> %s::vector) AS score FROM documents ORDER BY embedding <=> %s::vector LIMIT %s', (qemb, qemb, top_k))
    return cur.fetchall()

add_document('Python is great for AI applications.')
results = semantic_search('machine learning with Python')
for content, score in results:
    print(f'{score:.3f}: {content}')

Use Cases

  • semantic search
  • vector database
  • PostgreSQL AI

Tags

Related Snippets

Similar patterns you can reuse in the same workflow.