splitter

Class info

Classes

Name	Children	Inherits
TextChunk llmling_agent.splitter Chunk of text with metadata.		object
TextChunker llmling_agent.splitter Base class for text chunkers.		ABC

🛈 DocStrings

Base classes for text chunking implementations.

TextChunk

Chunk of text with metadata.

Source code in src/llmling_agent/splitter.py

class TextChunk:
    """Chunk of text with metadata."""

    def __init__(
        self,
        text: str,
        metadata: dict[str, Any] | None = None,
    ):
        self.text = text
        self.metadata = metadata or {}

TextChunker

Bases: ABC

Base class for text chunkers.

Source code in src/llmling_agent/splitter.py

class TextChunker(ABC):
    """Base class for text chunkers."""

    @abstractmethod
    def split(
        self, text: str, document_metadata: dict[str, Any] | None = None
    ) -> list[TextChunk]:
        """Split text into chunks."""
        raise NotImplementedError

    async def split_async(
        self, text: str, document_metadata: dict[str, Any] | None = None
    ) -> AsyncIterable[TextChunk]:
        """Split text asynchronously into chunks."""
        # Default implementation, override for true async processing
        for chunk in self.split(text, document_metadata):
            yield chunk

    def split_texts(
        self, texts: Iterable[str | tuple[str, dict[str, Any]]]
    ) -> list[TextChunk]:
        """Split multiple texts into chunks."""
        result: list[TextChunk] = []

        for item in texts:
            if isinstance(item, tuple):
                text, metadata = item
            else:
                text, metadata = item, None

            result.extend(self.split(text, metadata))

        return result

split abstractmethod

split(text: str, document_metadata: dict[str, Any] | None = None) -> list[TextChunk]

Split text into chunks.

Source code in src/llmling_agent/splitter.py

@abstractmethod
def split(
    self, text: str, document_metadata: dict[str, Any] | None = None
) -> list[TextChunk]:
    """Split text into chunks."""
    raise NotImplementedError

split_async async

split_async(
    text: str, document_metadata: dict[str, Any] | None = None
) -> AsyncIterable[TextChunk]

Split text asynchronously into chunks.

Source code in src/llmling_agent/splitter.py

async def split_async(
    self, text: str, document_metadata: dict[str, Any] | None = None
) -> AsyncIterable[TextChunk]:
    """Split text asynchronously into chunks."""
    # Default implementation, override for true async processing
    for chunk in self.split(text, document_metadata):
        yield chunk

split_texts

split_texts(texts: Iterable[str | tuple[str, dict[str, Any]]]) -> list[TextChunk]

Split multiple texts into chunks.

Source code in src/llmling_agent/splitter.py

def split_texts(
    self, texts: Iterable[str | tuple[str, dict[str, Any]]]
) -> list[TextChunk]:
    """Split multiple texts into chunks."""
    result: list[TextChunk] = []

    for item in texts:
        if isinstance(item, tuple):
            text, metadata = item
        else:
            text, metadata = item, None

        result.extend(self.split(text, metadata))

    return result