Iterative Retrieval Pattern

Overview

Iterative Retrieval improves results through multiple rounds:

  1. Initial query and retrieval
  2. Analysis of initial results
  3. Query refinement based on analysis
  4. Subsequent retrievals with refined queries

Implementation Example


from transformers import pipeline
from sentence_transformers import SentenceTransformer
import numpy as np

class IterativeRetriever:
    def __init__(self, documents):
        self.documents = documents
        self.embedder = SentenceTransformer('all-MiniLM-L6-v2')
        self.embeddings = self.embedder.encode(documents)
        self.qa = pipeline("question-answering")
        
    def retrieve(self, query, max_iterations=3):
        results = []
        current_query = query
        
        for i in range(max_iterations):
            # Retrieve documents
            query_embedding = self.embedder.encode(current_query)
            scores = cosine_similarity(
                [query_embedding],
                self.embeddings
            )[0]
            top_indices = np.argsort(scores)[::-1][:5]
            retrieved = [self.documents[i] for i in top_indices]
            
            # Analyze results
            analysis = self.qa(
                question=current_query,
                context=" ".join(retrieved)
            )
            
            # Store results
            results.append({
                'iteration': i + 1,
                'query': current_query,
                'documents': retrieved,
                'analysis': analysis
            })
            
            # Refine query
            current_query = f"{current_query} {analysis['answer']}"
            
        return results

# Usage
documents = [...]  # Your document collection
retriever = IterativeRetriever(documents)
results = retriever.retrieve("Explain RAG architecture")

When to Use