""" Comprehensive test suite for text splitter functionality. This module provides extensive testing coverage for text splitting operations used in RAG (Retrieval-Augmented Generation) systems. Text splitters are crucial for breaking down large documents into manageable chunks while preserving context and semantic meaning. ## Test Coverage Overview ### Core Splitter Types Tested: 1. **RecursiveCharacterTextSplitter**: Main splitter that recursively tries different separators (paragraph -> line -> word -> character) to split text appropriately. 2. **TokenTextSplitter**: Splits text based on token count using tiktoken library, useful for LLM context window management. 3. **EnhanceRecursiveCharacterTextSplitter**: Enhanced version with custom token counting support via embedding models or GPT2 tokenizer. 4. **FixedRecursiveCharacterTextSplitter**: Prioritizes a fixed separator before falling back to recursive splitting, useful for structured documents. ### Test Categories: #### Helper Functions (TestSplitTextWithRegex, TestSplitTextOnTokens) - Tests low-level splitting utilities - Regex pattern handling - Token-based splitting mechanics #### Core Functionality (TestRecursiveCharacterTextSplitter, TestTokenTextSplitter) - Initialization and configuration - Basic splitting operations - Separator hierarchy behavior - Chunk size and overlap handling #### Enhanced Splitters (TestEnhanceRecursiveCharacterTextSplitter, TestFixedRecursiveCharacterTextSplitter) - Custom encoder integration - Fixed separator prioritization - Character-level splitting with overlap - Multilingual separator support #### Metadata Preservation (TestMetadataPreservation) - Metadata copying across chunks - Start index tracking - Multiple document processing - Complex metadata types (strings, lists, dicts) #### Edge Cases (TestEdgeCases) - Empty text, single characters, whitespace - Unicode and emoji handling - Very small/large chunk sizes - Zero overlap scenarios - Mixed separator types #### Advanced Scenarios (TestAdvancedSplittingScenarios) - Markdown, HTML, JSON document splitting - Technical documentation - Code and mixed content - Lists, tables, quotes - URLs and email content #### Configuration Testing (TestSplitterConfiguration) - Custom length functions - Different separator orderings - Extreme overlap ratios - Start index accuracy - Regex pattern separators #### Error Handling (TestErrorHandlingAndRobustness) - Invalid inputs (None, empty) - Extreme parameters - Special characters (unicode, control chars) - Repeated separators - Empty separator lists #### Performance (TestPerformanceCharacteristics) - Chunk size consistency - Information preservation - Deterministic behavior - Chunk count estimation ## Usage Examples ```python # Basic recursive splitting splitter = RecursiveCharacterTextSplitter( chunk_size=1000, chunk_overlap=200, separators=["\n\n", "\n", " ", ""] ) chunks = splitter.split_text(long_text) # With metadata preservation documents = splitter.create_documents( texts=[text1, text2], metadatas=[{"source": "doc1.pdf"}, {"source": "doc2.pdf"}] ) # Token-based splitting token_splitter = TokenTextSplitter( encoding_name="gpt2", chunk_size=500, chunk_overlap=50 ) token_chunks = token_splitter.split_text(text) ``` ## Test Execution Run all tests: pytest tests/unit_tests/core/rag/splitter/test_text_splitter.py -v Run specific test class: pytest tests/unit_tests/core/rag/splitter/test_text_splitter.py::TestRecursiveCharacterTextSplitter -v Run with coverage: pytest tests/unit_tests/core/rag/splitter/test_text_splitter.py --cov=core.rag.splitter ## Notes - Some tests are skipped if tiktoken library is not installed (TokenTextSplitter tests) - Tests use pytest fixtures for reusable test data - All tests follow Arrange-Act-Assert pattern - Tests are organized by functionality in classes for better organization """ import asyncio import string import sys import types from inspect import currentframe from unittest.mock import Mock, patch import pytest from core.rag.models.document import Document from core.rag.splitter.fixed_text_splitter import ( EnhanceRecursiveCharacterTextSplitter, FixedRecursiveCharacterTextSplitter, ) from core.rag.splitter.text_splitter import ( RecursiveCharacterTextSplitter, Tokenizer, TokenTextSplitter, _split_text_with_regex, split_text_on_tokens, ) # ============================================================================ # Test Fixtures # ============================================================================ @pytest.fixture def sample_text(): """Provide sample text for testing.""" return """This is the first paragraph. It contains multiple sentences. This is the second paragraph. It also has several sentences. This is the third paragraph with more content.""" @pytest.fixture def long_text(): """Provide long text for testing chunking.""" return " ".join([f"Sentence number {i}." for i in range(100)]) @pytest.fixture def multilingual_text(): """Provide multilingual text for testing.""" return "This is English. 这是中文。日本語です。한국어입니다。" @pytest.fixture def code_text(): """Provide code snippet for testing.""" return """def hello_world(): print("Hello, World!") return True def another_function(): x = 10 y = 20 return x + y""" @pytest.fixture def markdown_text(): """ Provide markdown formatted text for testing. This fixture simulates a typical markdown document with headers, paragraphs, and code blocks. """ return """# Main Title This is an introduction paragraph with some content. ## Section 1 Content for section 1 with multiple sentences. This should be split appropriately. ### Subsection 1.1 More detailed content here. ## Section 2 Another section with different content. ```python def example(): return "code" ``` Final paragraph.""" @pytest.fixture def html_text(): """ Provide HTML formatted text for testing. Tests how splitters handle structured markup content. """ return """ Test

Header

First paragraph with content.

Second paragraph with more content.

Nested content here.
""" @pytest.fixture def json_text(): """ Provide JSON formatted text for testing. Tests splitting of structured data formats. """ return """{ "name": "Test Document", "content": "This is the main content", "metadata": { "author": "John Doe", "date": "2024-01-01" }, "sections": [ {"title": "Section 1", "text": "Content 1"}, {"title": "Section 2", "text": "Content 2"} ] }""" @pytest.fixture def technical_text(): """ Provide technical documentation text. Simulates API documentation or technical writing with specific terminology and formatting. """ return """API Endpoint: /api/v1/users Description: Retrieves user information from the database. Parameters: - user_id (required): The unique identifier for the user - include_metadata (optional): Boolean flag to include additional metadata Response Format: { "user_id": "12345", "name": "John Doe", "email": "john@example.com" } Error Codes: - 404: User not found - 401: Unauthorized access - 500: Internal server error""" # ============================================================================ # Test Helper Functions # ============================================================================ class TestSplitTextWithRegex: """ Test the _split_text_with_regex helper function. This helper function is used internally by text splitters to split text using regex patterns. It supports keeping or removing separators and handles special regex characters properly. """ def test_split_with_separator_keep(self): """ Test splitting text with separator kept. When keep_separator=True, the separator should be appended to each chunk (except possibly the last one). This is useful for maintaining document structure like paragraph breaks. """ text = "Hello\nWorld\nTest" result = _split_text_with_regex(text, "\n", keep_separator=True) # Each line should keep its newline character assert result == ["Hello\n", "World\n", "Test"] def test_split_with_separator_no_keep(self): """Test splitting text without keeping separator.""" text = "Hello\nWorld\nTest" result = _split_text_with_regex(text, "\n", keep_separator=False) assert result == ["Hello", "World", "Test"] def test_split_empty_separator(self): """Test splitting with empty separator (character by character).""" text = "ABC" result = _split_text_with_regex(text, "", keep_separator=False) assert result == ["A", "B", "C"] def test_split_filters_empty_strings(self): """Test that empty strings and newlines are filtered out.""" text = "Hello\n\nWorld" result = _split_text_with_regex(text, "\n", keep_separator=False) # Empty strings between consecutive separators should be filtered assert "" not in result assert result == ["Hello", "World"] def test_split_with_special_regex_chars(self): """Test splitting with special regex characters in separator.""" text = "Hello.World.Test" result = _split_text_with_regex(text, ".", keep_separator=False) # The function escapes regex chars, so it should split correctly # But empty strings are filtered, so we get the parts assert len(result) >= 0 # May vary based on regex escaping assert isinstance(result, list) class TestSplitTextOnTokens: """Test the split_text_on_tokens function.""" def test_basic_token_splitting(self): """Test basic token-based splitting.""" # Mock tokenizer def mock_encode(text: str) -> list[int]: return [ord(c) for c in text] def mock_decode(tokens: list[int]) -> str: return "".join([chr(t) for t in tokens]) tokenizer = Tokenizer(chunk_overlap=2, tokens_per_chunk=5, decode=mock_decode, encode=mock_encode) text = "ABCDEFGHIJ" result = split_text_on_tokens(text=text, tokenizer=tokenizer) # Should split into chunks of 5 with overlap of 2 assert len(result) > 1 assert all(isinstance(chunk, str) for chunk in result) def test_token_splitting_with_overlap(self): """Test that overlap is correctly applied in token splitting.""" def mock_encode(text: str) -> list[int]: return list(range(len(text))) def mock_decode(tokens: list[int]) -> str: return "".join([str(t) for t in tokens]) tokenizer = Tokenizer(chunk_overlap=2, tokens_per_chunk=5, decode=mock_decode, encode=mock_encode) text = string.digits result = split_text_on_tokens(text=text, tokenizer=tokenizer) # Verify we get multiple chunks assert len(result) >= 2 def test_token_splitting_short_text(self): """Test token splitting with text shorter than chunk size.""" def mock_encode(text: str) -> list[int]: return [ord(c) for c in text] def mock_decode(tokens: list[int]) -> str: return "".join([chr(t) for t in tokens]) tokenizer = Tokenizer(chunk_overlap=2, tokens_per_chunk=100, decode=mock_decode, encode=mock_encode) text = "Short" result = split_text_on_tokens(text=text, tokenizer=tokenizer) # Should return single chunk for short text assert len(result) == 1 assert result[0] == text # ============================================================================ # Test RecursiveCharacterTextSplitter # ============================================================================ class TestRecursiveCharacterTextSplitter: """ Test RecursiveCharacterTextSplitter functionality. RecursiveCharacterTextSplitter is the main text splitting class that recursively tries different separators (paragraph -> line -> word -> character) to split text into chunks of appropriate size. This is the most commonly used splitter for general text processing. """ def test_initialization(self): """ Test splitter initialization with default parameters. Verifies that the splitter is properly initialized with the correct chunk size, overlap, and default separator hierarchy. """ splitter = RecursiveCharacterTextSplitter(chunk_size=100, chunk_overlap=10) assert splitter._chunk_size == 100 assert splitter._chunk_overlap == 10 # Default separators: paragraph, line, space, character assert splitter._separators == ["\n\n", "\n", " ", ""] def test_initialization_custom_separators(self): """Test splitter initialization with custom separators.""" custom_separators = ["\n\n\n", "\n\n", "\n", " "] splitter = RecursiveCharacterTextSplitter(chunk_size=100, chunk_overlap=10, separators=custom_separators) assert splitter._separators == custom_separators def test_chunk_overlap_validation(self): """Test that chunk overlap cannot exceed chunk size.""" with pytest.raises(ValueError, match="larger chunk overlap"): RecursiveCharacterTextSplitter(chunk_size=100, chunk_overlap=150) def test_split_by_paragraph(self, sample_text): """Test splitting text by paragraphs.""" splitter = RecursiveCharacterTextSplitter(chunk_size=100, chunk_overlap=10) result = splitter.split_text(sample_text) assert len(result) > 0 assert all(isinstance(chunk, str) for chunk in result) # Verify chunks respect size limit (with some tolerance for overlap) assert all(len(chunk) <= 150 for chunk in result) def test_split_by_newline(self): """Test splitting by newline when paragraphs are too large.""" text = "Line 1\nLine 2\nLine 3\nLine 4\nLine 5" splitter = RecursiveCharacterTextSplitter(chunk_size=20, chunk_overlap=5) result = splitter.split_text(text) assert len(result) > 0 assert all(isinstance(chunk, str) for chunk in result) def test_split_by_space(self): """Test splitting by space when lines are too large.""" text = "word1 word2 word3 word4 word5 word6 word7 word8" splitter = RecursiveCharacterTextSplitter(chunk_size=15, chunk_overlap=3) result = splitter.split_text(text) assert len(result) > 1 assert all(isinstance(chunk, str) for chunk in result) def test_split_by_character(self): """Test splitting by character when words are too large.""" text = "verylongwordthatcannotbesplit" splitter = RecursiveCharacterTextSplitter(chunk_size=10, chunk_overlap=2) result = splitter.split_text(text) assert len(result) > 1 assert all(len(chunk) <= 12 for chunk in result) # Allow for overlap def test_keep_separator_true(self): """Test that separators are kept when keep_separator=True.""" text = "Para1\n\nPara2\n\nPara3" splitter = RecursiveCharacterTextSplitter(chunk_size=50, chunk_overlap=5, keep_separator=True) result = splitter.split_text(text) # At least one chunk should contain the separator combined = "".join(result) assert "Para1" in combined assert "Para2" in combined def test_keep_separator_false(self): """Test that separators are removed when keep_separator=False.""" text = "Para1\n\nPara2\n\nPara3" splitter = RecursiveCharacterTextSplitter(chunk_size=50, chunk_overlap=5, keep_separator=False) result = splitter.split_text(text) assert len(result) > 0 # Verify text content is preserved combined = " ".join(result) assert "Para1" in combined assert "Para2" in combined def test_overlap_handling(self): """ Test that chunk overlap is correctly handled. Overlap ensures that context is preserved between chunks by having some content appear in consecutive chunks. This is crucial for maintaining semantic continuity in RAG applications. """ text = "A B C D E F G H I J K L M N O P" splitter = RecursiveCharacterTextSplitter(chunk_size=10, chunk_overlap=3) result = splitter.split_text(text) # Verify we have multiple chunks assert len(result) > 1 # Verify overlap exists between consecutive chunks # The end of one chunk should have some overlap with the start of the next for i in range(len(result) - 1): # Some content should overlap assert len(result[i]) > 0 assert len(result[i + 1]) > 0 def test_empty_text(self): """Test splitting empty text.""" splitter = RecursiveCharacterTextSplitter(chunk_size=100, chunk_overlap=10) result = splitter.split_text("") assert result == [] def test_single_word(self): """Test splitting single word.""" splitter = RecursiveCharacterTextSplitter(chunk_size=100, chunk_overlap=10) result = splitter.split_text("Hello") assert len(result) == 1 assert result[0] == "Hello" def test_create_documents(self): """Test creating documents from texts.""" splitter = RecursiveCharacterTextSplitter(chunk_size=50, chunk_overlap=5) texts = ["Text 1 with some content", "Text 2 with more content"] metadatas = [{"source": "doc1"}, {"source": "doc2"}] documents = splitter.create_documents(texts, metadatas) assert len(documents) > 0 assert all(isinstance(doc, Document) for doc in documents) assert all(hasattr(doc, "page_content") for doc in documents) assert all(hasattr(doc, "metadata") for doc in documents) def test_create_documents_with_start_index(self): """Test creating documents with start_index in metadata.""" splitter = RecursiveCharacterTextSplitter(chunk_size=20, chunk_overlap=5, add_start_index=True) texts = ["This is a longer text that will be split into chunks"] documents = splitter.create_documents(texts) # Verify start_index is added to metadata assert any("start_index" in doc.metadata for doc in documents) # First chunk should start at index 0 if documents: assert documents[0].metadata.get("start_index") == 0 def test_split_documents(self): """Test splitting existing documents.""" splitter = RecursiveCharacterTextSplitter(chunk_size=30, chunk_overlap=5) docs = [ Document(page_content="First document content", metadata={"id": 1}), Document(page_content="Second document content", metadata={"id": 2}), ] result = splitter.split_documents(docs) assert len(result) > 0 assert all(isinstance(doc, Document) for doc in result) # Verify metadata is preserved assert any(doc.metadata.get("id") == 1 for doc in result) def test_transform_documents(self): """Test transform_documents interface.""" splitter = RecursiveCharacterTextSplitter(chunk_size=30, chunk_overlap=5) docs = [Document(page_content="Document to transform", metadata={"key": "value"})] result = splitter.transform_documents(docs) assert len(result) > 0 assert all(isinstance(doc, Document) for doc in result) def test_long_text_splitting(self, long_text): """Test splitting very long text.""" splitter = RecursiveCharacterTextSplitter(chunk_size=100, chunk_overlap=20) result = splitter.split_text(long_text) assert len(result) > 5 # Should create multiple chunks assert all(isinstance(chunk, str) for chunk in result) # Verify all chunks are within reasonable size assert all(len(chunk) <= 150 for chunk in result) def test_code_splitting(self, code_text): """Test splitting code with proper structure preservation.""" splitter = RecursiveCharacterTextSplitter(chunk_size=80, chunk_overlap=10) result = splitter.split_text(code_text) assert len(result) > 0 # Verify code content is preserved combined = "\n".join(result) assert "def hello_world" in combined or "hello_world" in combined class TestTextSplitterBasePaths: """Target uncovered base TextSplitter paths.""" def test_from_huggingface_tokenizer_success_path(self): """Cover from_huggingface_tokenizer success branch with mocked transformers.""" class _FakePreTrainedTokenizerBase: pass class _FakeTokenizer(_FakePreTrainedTokenizerBase): def encode(self, text: str): return [ord(c) for c in text] fake_transformers = types.SimpleNamespace(PreTrainedTokenizerBase=_FakePreTrainedTokenizerBase) with patch.dict(sys.modules, {"transformers": fake_transformers}): splitter = RecursiveCharacterTextSplitter.from_huggingface_tokenizer( tokenizer=_FakeTokenizer(), chunk_size=5, chunk_overlap=1, ) chunks = splitter.split_text("abcdef") assert chunks def test_from_huggingface_tokenizer_import_error(self): """Cover from_huggingface_tokenizer import-error branch.""" with patch.dict(sys.modules, {"transformers": None}): with pytest.raises(ValueError, match="Could not import transformers"): RecursiveCharacterTextSplitter.from_huggingface_tokenizer(tokenizer=object(), chunk_size=5) def test_atransform_documents_raises_not_implemented(self): """Cover atransform_documents NotImplemented branch.""" splitter = RecursiveCharacterTextSplitter(chunk_size=20, chunk_overlap=5) with pytest.raises(NotImplementedError): asyncio.run(splitter.atransform_documents([Document(page_content="x", metadata={})])) def test_merge_splits_logs_warning_for_oversized_total(self): """Cover logger.warning path in _merge_splits.""" splitter = RecursiveCharacterTextSplitter(chunk_size=5, chunk_overlap=1) with patch("core.rag.splitter.text_splitter.logger.warning") as mock_warning: merged = splitter._merge_splits(["abcdefghij", "b"], "", [10, 1]) assert merged mock_warning.assert_called_once() # ============================================================================ # Test TokenTextSplitter # ============================================================================ class TestTokenTextSplitter: """Test TokenTextSplitter functionality.""" @pytest.mark.skipif(True, reason="Requires tiktoken library which may not be installed") def test_initialization_with_encoding(self): """Test TokenTextSplitter initialization with encoding name.""" try: splitter = TokenTextSplitter(encoding_name="gpt2", chunk_size=100, chunk_overlap=10) assert splitter._chunk_size == 100 assert splitter._chunk_overlap == 10 except ImportError: pytest.skip("tiktoken not installed") @pytest.mark.skipif(True, reason="Requires tiktoken library which may not be installed") def test_initialization_with_model(self): """Test TokenTextSplitter initialization with model name.""" try: splitter = TokenTextSplitter(model_name="gpt-3.5-turbo", chunk_size=100, chunk_overlap=10) assert splitter._chunk_size == 100 except ImportError: pytest.skip("tiktoken not installed") def test_initialization_without_tiktoken(self): """Test that proper error is raised when tiktoken is not installed.""" with patch("core.rag.splitter.text_splitter.TokenTextSplitter.__init__") as mock_init: mock_init.side_effect = ImportError("Could not import tiktoken") with pytest.raises(ImportError, match="tiktoken"): TokenTextSplitter(chunk_size=100) @pytest.mark.skipif(True, reason="Requires tiktoken library which may not be installed") def test_split_text_by_tokens(self, sample_text): """Test splitting text by token count.""" try: splitter = TokenTextSplitter(encoding_name="gpt2", chunk_size=50, chunk_overlap=10) result = splitter.split_text(sample_text) assert len(result) > 0 assert all(isinstance(chunk, str) for chunk in result) except ImportError: pytest.skip("tiktoken not installed") @pytest.mark.skipif(True, reason="Requires tiktoken library which may not be installed") def test_token_overlap(self): """Test that token overlap works correctly.""" try: splitter = TokenTextSplitter(encoding_name="gpt2", chunk_size=20, chunk_overlap=5) text = " ".join([f"word{i}" for i in range(50)]) result = splitter.split_text(text) assert len(result) > 1 except ImportError: pytest.skip("tiktoken not installed") def test_initialization_and_split_with_mocked_tiktoken_encoding(self): """Cover TokenTextSplitter __init__ else-path and split_text logic.""" class _FakeEncoding: def encode(self, text: str, allowed_special=None, disallowed_special=None): return [ord(c) for c in text] def decode(self, token_ids: list[int]) -> str: return "".join(chr(i) for i in token_ids) fake_tiktoken = types.SimpleNamespace(get_encoding=lambda name: _FakeEncoding()) with patch.dict(sys.modules, {"tiktoken": fake_tiktoken}): splitter = TokenTextSplitter(encoding_name="gpt2", chunk_size=4, chunk_overlap=1) result = splitter.split_text("abcdefgh") assert result assert all(isinstance(chunk, str) for chunk in result) def test_initialization_with_model_name_uses_encoding_for_model(self): """Cover TokenTextSplitter model_name init branch.""" class _FakeEncoding: def encode(self, text: str, allowed_special=None, disallowed_special=None): return [ord(c) for c in text] def decode(self, token_ids: list[int]) -> str: return "".join(chr(i) for i in token_ids) fake_encoding = _FakeEncoding() fake_tiktoken = types.SimpleNamespace( encoding_for_model=lambda model_name: fake_encoding, get_encoding=lambda name: _FakeEncoding(), ) with patch.dict(sys.modules, {"tiktoken": fake_tiktoken}): splitter = TokenTextSplitter(model_name="gpt-4", chunk_size=5, chunk_overlap=1) assert splitter._tokenizer is fake_encoding # ============================================================================ # Test EnhanceRecursiveCharacterTextSplitter # ============================================================================ class TestEnhanceRecursiveCharacterTextSplitter: """Test EnhanceRecursiveCharacterTextSplitter functionality.""" def test_from_encoder_without_model(self): """Test creating splitter from encoder without embedding model.""" splitter = EnhanceRecursiveCharacterTextSplitter.from_encoder( embedding_model_instance=None, chunk_size=100, chunk_overlap=10 ) assert splitter._chunk_size == 100 assert splitter._chunk_overlap == 10 def test_from_encoder_with_mock_model(self): """Test creating splitter from encoder with mock embedding model.""" mock_model = Mock() mock_model.get_text_embedding_num_tokens = Mock(return_value=[10, 20, 30]) splitter = EnhanceRecursiveCharacterTextSplitter.from_encoder( embedding_model_instance=mock_model, chunk_size=100, chunk_overlap=10 ) assert splitter._chunk_size == 100 assert splitter._chunk_overlap == 10 def test_split_text_basic(self, sample_text): """Test basic text splitting with EnhanceRecursiveCharacterTextSplitter.""" splitter = EnhanceRecursiveCharacterTextSplitter.from_encoder( embedding_model_instance=None, chunk_size=100, chunk_overlap=10 ) result = splitter.split_text(sample_text) assert len(result) > 0 assert all(isinstance(chunk, str) for chunk in result) def test_character_encoder_length_function(self): """Test that character encoder correctly counts characters.""" splitter = EnhanceRecursiveCharacterTextSplitter.from_encoder( embedding_model_instance=None, chunk_size=50, chunk_overlap=5 ) text = "A" * 100 result = splitter.split_text(text) # Should split into multiple chunks assert len(result) >= 2 def test_with_embedding_model_token_counting(self): """Test token counting with embedding model.""" mock_model = Mock() # Mock returns token counts for input texts mock_model.get_text_embedding_num_tokens = Mock(side_effect=lambda texts: [len(t) // 2 for t in texts]) splitter = EnhanceRecursiveCharacterTextSplitter.from_encoder( embedding_model_instance=mock_model, chunk_size=50, chunk_overlap=5 ) text = "This is a test text that should be split" result = splitter.split_text(text) assert len(result) > 0 assert all(isinstance(chunk, str) for chunk in result) def test_from_encoder_internal_token_encoder_paths(self): """ Test internal _token_encoder branches by capturing local closure from frame. This validates: - empty texts path - embedding model path - GPT2Tokenizer fallback path - _character_encoder empty-path branch """ class _SpySplitter(EnhanceRecursiveCharacterTextSplitter): captured_token_encoder = None captured_character_encoder = None def __init__(self, **kwargs): frame = currentframe() if frame and frame.f_back: _SpySplitter.captured_token_encoder = frame.f_back.f_locals.get("_token_encoder") _SpySplitter.captured_character_encoder = frame.f_back.f_locals.get("_character_encoder") super().__init__(**kwargs) mock_model = Mock() mock_model.get_text_embedding_num_tokens.return_value = [3, 5] _SpySplitter.from_encoder(embedding_model_instance=mock_model, chunk_size=10, chunk_overlap=1) token_encoder = _SpySplitter.captured_token_encoder character_encoder = _SpySplitter.captured_character_encoder assert token_encoder is not None assert character_encoder is not None assert token_encoder([]) == [] assert token_encoder(["abc", "defgh"]) == [3, 5] assert character_encoder([]) == [] with patch( "core.rag.splitter.fixed_text_splitter.GPT2Tokenizer.get_num_tokens", side_effect=lambda text: len(text) + 1, ): _SpySplitter.from_encoder(embedding_model_instance=None, chunk_size=10, chunk_overlap=1) token_encoder_without_model = _SpySplitter.captured_token_encoder assert token_encoder_without_model is not None assert token_encoder_without_model(["ab", "cdef"]) == [3, 5] # ============================================================================ # Test FixedRecursiveCharacterTextSplitter # ============================================================================ class TestFixedRecursiveCharacterTextSplitter: """Test FixedRecursiveCharacterTextSplitter functionality.""" def test_initialization_with_fixed_separator(self): """Test initialization with fixed separator.""" splitter = FixedRecursiveCharacterTextSplitter(fixed_separator="\n\n", chunk_size=100, chunk_overlap=10) assert splitter._fixed_separator == "\n\n" assert splitter._chunk_size == 100 assert splitter._chunk_overlap == 10 def test_split_by_fixed_separator(self): """Test splitting by fixed separator first.""" text = "Part 1\n\nPart 2\n\nPart 3" splitter = FixedRecursiveCharacterTextSplitter(fixed_separator="\n\n", chunk_size=100, chunk_overlap=10) result = splitter.split_text(text) assert len(result) >= 3 assert all(isinstance(chunk, str) for chunk in result) def test_recursive_split_when_chunk_too_large(self): """Test recursive splitting when chunks exceed size limit.""" # Create text with large chunks separated by fixed separator large_chunk = " ".join([f"word{i}" for i in range(50)]) text = f"{large_chunk}\n\n{large_chunk}" splitter = FixedRecursiveCharacterTextSplitter(fixed_separator="\n\n", chunk_size=50, chunk_overlap=5) result = splitter.split_text(text) # Should split into more than 2 chunks due to size limit assert len(result) > 2 def test_custom_separators(self): """Test with custom separator list.""" text = "Sentence 1. Sentence 2. Sentence 3." splitter = FixedRecursiveCharacterTextSplitter( fixed_separator=".", separators=[".", " ", ""], chunk_size=30, chunk_overlap=5, ) result = splitter.split_text(text) assert len(result) > 0 assert all(isinstance(chunk, str) for chunk in result) def test_no_fixed_separator(self): """Test behavior when no fixed separator is provided.""" text = "This is a test text without fixed separator" splitter = FixedRecursiveCharacterTextSplitter(fixed_separator="", chunk_size=20, chunk_overlap=5) result = splitter.split_text(text) assert len(result) > 0 def test_chinese_separator(self): """Test with Chinese period separator.""" text = "这是第一句。这是第二句。这是第三句。" splitter = FixedRecursiveCharacterTextSplitter(fixed_separator="。", chunk_size=50, chunk_overlap=5) result = splitter.split_text(text) assert len(result) > 0 assert all(isinstance(chunk, str) for chunk in result) def test_space_separator_handling(self): """Test special handling of space separator.""" text = "word1 word2 word3 word4" # Multiple spaces splitter = FixedRecursiveCharacterTextSplitter( fixed_separator=" ", separators=[" ", ""], chunk_size=15, chunk_overlap=3 ) result = splitter.split_text(text) assert len(result) > 0 # Verify words are present combined = " ".join(result) assert "word1" in combined assert "word2" in combined def test_character_level_splitting(self): """Test character-level splitting when no separator works.""" text = "verylongwordwithoutspaces" splitter = FixedRecursiveCharacterTextSplitter( fixed_separator="", separators=[""], chunk_size=10, chunk_overlap=2 ) result = splitter.split_text(text) assert len(result) > 1 # Verify chunks respect size with overlap for chunk in result: assert len(chunk) <= 12 # chunk_size + some tolerance for overlap def test_overlap_in_character_splitting(self): """Test that overlap is correctly applied in character-level splitting.""" text = string.ascii_uppercase splitter = FixedRecursiveCharacterTextSplitter( fixed_separator="", separators=[""], chunk_size=10, chunk_overlap=3 ) result = splitter.split_text(text) assert len(result) > 1 # Verify overlap exists for i in range(len(result) - 1): # Check that some characters appear in consecutive chunks assert len(result[i]) > 0 assert len(result[i + 1]) > 0 def test_metadata_preservation_in_documents(self): """Test that metadata is preserved when splitting documents.""" splitter = FixedRecursiveCharacterTextSplitter(fixed_separator="\n\n", chunk_size=50, chunk_overlap=5) docs = [ Document( page_content="First part\n\nSecond part\n\nThird part", metadata={"source": "test.txt", "page": 1}, ) ] result = splitter.split_documents(docs) assert len(result) > 0 # Verify all chunks have the original metadata for doc in result: assert doc.metadata.get("source") == "test.txt" assert doc.metadata.get("page") == 1 def test_empty_text_handling(self): """Test handling of empty text.""" splitter = FixedRecursiveCharacterTextSplitter(fixed_separator="\n\n", chunk_size=100, chunk_overlap=10) result = splitter.split_text("") # May return empty list or list with empty string depending on implementation assert isinstance(result, list) assert len(result) <= 1 def test_single_chunk_text(self): """Test text that fits in a single chunk.""" text = "Short text" splitter = FixedRecursiveCharacterTextSplitter(fixed_separator="\n\n", chunk_size=100, chunk_overlap=10) result = splitter.split_text(text) assert len(result) == 1 assert result[0] == text def test_newline_filtering(self): """Test that newlines are properly filtered in splits.""" text = "Line 1\nLine 2\n\nLine 3" splitter = FixedRecursiveCharacterTextSplitter( fixed_separator="", separators=["\n", ""], chunk_size=50, chunk_overlap=5 ) result = splitter.split_text(text) # Verify no empty chunks assert all(len(chunk) > 0 for chunk in result) def test_double_slash_n(self): data = "chunk 1\n\nsubchunk 1.\nsubchunk 2.\n\n---\n\nchunk 2\n\nsubchunk 1\nsubchunk 2." separator = "\\n\\n---\\n\\n" splitter = FixedRecursiveCharacterTextSplitter(fixed_separator=separator) chunks = splitter.split_text(data) assert chunks == ["chunk 1\n\nsubchunk 1.\nsubchunk 2.", "chunk 2\n\nsubchunk 1\nsubchunk 2."] def test_recursive_split_keep_separator_and_recursive_fallback(self): """Cover keep-separator split branch and recursive _split_text fallback.""" text = "short." + ("x" * 60) splitter = FixedRecursiveCharacterTextSplitter( fixed_separator="", separators=[".", " ", ""], chunk_size=10, chunk_overlap=2, keep_separator=True, ) chunks = splitter.recursive_split_text(text) assert chunks assert any("short." in chunk for chunk in chunks) assert any(len(chunk) <= 12 for chunk in chunks) def test_recursive_split_newline_separator_filtering(self): """Cover newline-specific empty filtering branch.""" text = "line1\n\nline2\n\nline3" splitter = FixedRecursiveCharacterTextSplitter( fixed_separator="", separators=["\n", ""], chunk_size=50, chunk_overlap=5, ) chunks = splitter.recursive_split_text(text) assert chunks assert all(chunk != "" for chunk in chunks) assert "line1" in "".join(chunks) assert "line2" in "".join(chunks) assert "line3" in "".join(chunks) def test_recursive_split_without_new_separator_appends_long_chunk(self): """Cover branch where no further separators exist and long split is appended directly.""" text = "aa\n" + ("b" * 40) splitter = FixedRecursiveCharacterTextSplitter( fixed_separator="", separators=["\n"], chunk_size=10, chunk_overlap=2, ) chunks = splitter.recursive_split_text(text) assert "aa" in chunks assert any(len(chunk) >= 40 for chunk in chunks) # ============================================================================ # Test Metadata Preservation # ============================================================================ class TestMetadataPreservation: """ Test metadata preservation across different splitters. Metadata preservation is critical for RAG systems as it allows tracking the source, author, timestamps, and other contextual information for each chunk. All chunks derived from a document should inherit its metadata. """ def test_recursive_splitter_metadata(self): """ Test metadata preservation with RecursiveCharacterTextSplitter. When a document is split into multiple chunks, each chunk should receive a copy of the original document's metadata. This ensures that we can trace each chunk back to its source. """ splitter = RecursiveCharacterTextSplitter(chunk_size=30, chunk_overlap=5) texts = ["Text content here"] # Metadata includes various types: strings, dates, lists metadatas = [{"author": "John", "date": "2024-01-01", "tags": ["test"]}] documents = splitter.create_documents(texts, metadatas) # Every chunk should have the same metadata as the original for doc in documents: assert doc.metadata.get("author") == "John" assert doc.metadata.get("date") == "2024-01-01" assert doc.metadata.get("tags") == ["test"] def test_enhance_splitter_metadata(self): """Test metadata preservation with EnhanceRecursiveCharacterTextSplitter.""" splitter = EnhanceRecursiveCharacterTextSplitter.from_encoder( embedding_model_instance=None, chunk_size=30, chunk_overlap=5 ) docs = [ Document( page_content="Content to split", metadata={"id": 123, "category": "test"}, ) ] result = splitter.split_documents(docs) for doc in result: assert doc.metadata.get("id") == 123 assert doc.metadata.get("category") == "test" def test_fixed_splitter_metadata(self): """Test metadata preservation with FixedRecursiveCharacterTextSplitter.""" splitter = FixedRecursiveCharacterTextSplitter(fixed_separator="\n", chunk_size=30, chunk_overlap=5) docs = [ Document( page_content="Line 1\nLine 2\nLine 3", metadata={"version": "1.0", "status": "active"}, ) ] result = splitter.split_documents(docs) for doc in result: assert doc.metadata.get("version") == "1.0" assert doc.metadata.get("status") == "active" def test_metadata_with_start_index(self): """Test that start_index is added to metadata when requested.""" splitter = RecursiveCharacterTextSplitter(chunk_size=20, chunk_overlap=5, add_start_index=True) texts = ["This is a test text that will be split"] metadatas = [{"original": "metadata"}] documents = splitter.create_documents(texts, metadatas) # Verify both original metadata and start_index are present for doc in documents: assert "start_index" in doc.metadata assert doc.metadata.get("original") == "metadata" assert isinstance(doc.metadata["start_index"], int) assert doc.metadata["start_index"] >= 0 # ============================================================================ # Test Edge Cases # ============================================================================ class TestEdgeCases: """Test edge cases and boundary conditions.""" def test_chunk_size_equals_text_length(self): """Test when chunk size equals text length.""" text = "Exact size text" splitter = RecursiveCharacterTextSplitter(chunk_size=len(text), chunk_overlap=0) result = splitter.split_text(text) assert len(result) == 1 assert result[0] == text def test_very_small_chunk_size(self): """Test with very small chunk size.""" text = "Test text" splitter = RecursiveCharacterTextSplitter(chunk_size=3, chunk_overlap=1) result = splitter.split_text(text) assert len(result) > 1 assert all(len(chunk) <= 5 for chunk in result) # Allow for overlap def test_zero_overlap(self): """Test splitting with zero overlap.""" text = "Word1 Word2 Word3 Word4" splitter = RecursiveCharacterTextSplitter(chunk_size=12, chunk_overlap=0) result = splitter.split_text(text) assert len(result) > 0 # Verify no overlap between chunks combined_length = sum(len(chunk) for chunk in result) # Should be close to original length (accounting for separators) assert combined_length >= len(text) - 10 def test_unicode_text(self): """Test splitting text with unicode characters.""" text = "Hello 世界 🌍 مرحبا" splitter = RecursiveCharacterTextSplitter(chunk_size=20, chunk_overlap=3) result = splitter.split_text(text) assert len(result) > 0 # Verify unicode is preserved combined = " ".join(result) assert "世界" in combined or "世" in combined def test_only_separators(self): """Test text containing only separators.""" text = "\n\n\n\n" splitter = RecursiveCharacterTextSplitter(chunk_size=10, chunk_overlap=2) result = splitter.split_text(text) # Should return empty list or handle gracefully assert isinstance(result, list) def test_mixed_separators(self): """Test text with mixed separator types.""" text = "Para1\n\nPara2\nLine\n\n\nPara3" splitter = RecursiveCharacterTextSplitter(chunk_size=50, chunk_overlap=5) result = splitter.split_text(text) assert len(result) > 0 combined = "".join(result) assert "Para1" in combined assert "Para2" in combined assert "Para3" in combined def test_whitespace_only_text(self): """Test text containing only whitespace.""" text = " " splitter = RecursiveCharacterTextSplitter(chunk_size=10, chunk_overlap=2) result = splitter.split_text(text) # Should handle whitespace-only text assert isinstance(result, list) def test_single_character_text(self): """Test splitting single character.""" text = "A" splitter = RecursiveCharacterTextSplitter(chunk_size=10, chunk_overlap=2) result = splitter.split_text(text) assert len(result) == 1 assert result[0] == "A" def test_multiple_documents_different_sizes(self): """Test splitting multiple documents of different sizes.""" splitter = RecursiveCharacterTextSplitter(chunk_size=30, chunk_overlap=5) docs = [ Document(page_content="Short", metadata={"id": 1}), Document( page_content="This is a much longer document that will be split", metadata={"id": 2}, ), Document(page_content="Medium length doc", metadata={"id": 3}), ] result = splitter.split_documents(docs) # Verify all documents are processed assert len(result) >= 3 # Verify metadata is preserved ids = [doc.metadata.get("id") for doc in result] assert 1 in ids assert 2 in ids assert 3 in ids # ============================================================================ # Test Integration Scenarios # ============================================================================ class TestIntegrationScenarios: """Test realistic integration scenarios.""" def test_document_processing_pipeline(self): """Test complete document processing pipeline.""" # Simulate a document processing workflow splitter = RecursiveCharacterTextSplitter(chunk_size=100, chunk_overlap=20, add_start_index=True) # Original documents with metadata original_docs = [ Document( page_content="First document with multiple paragraphs.\n\nSecond paragraph here.\n\nThird paragraph.", metadata={"source": "doc1.txt", "author": "Alice"}, ), Document( page_content="Second document content.\n\nMore content here.", metadata={"source": "doc2.txt", "author": "Bob"}, ), ] # Split documents split_docs = splitter.split_documents(original_docs) # Verify results - documents may fit in single chunks if small enough assert len(split_docs) >= len(original_docs) # At least as many chunks as original docs assert all(isinstance(doc, Document) for doc in split_docs) assert all("start_index" in doc.metadata for doc in split_docs) assert all("source" in doc.metadata for doc in split_docs) assert all("author" in doc.metadata for doc in split_docs) def test_multilingual_document_splitting(self, multilingual_text): """Test splitting multilingual documents.""" splitter = RecursiveCharacterTextSplitter(chunk_size=30, chunk_overlap=5) result = splitter.split_text(multilingual_text) assert len(result) > 0 # Verify content is preserved combined = " ".join(result) assert "English" in combined or "Eng" in combined def test_code_documentation_splitting(self, code_text): """Test splitting code documentation.""" splitter = FixedRecursiveCharacterTextSplitter(fixed_separator="\n\n", chunk_size=100, chunk_overlap=10) result = splitter.split_text(code_text) assert len(result) > 0 # Verify code structure is somewhat preserved combined = "\n".join(result) assert "def" in combined def test_large_document_chunking(self): """Test chunking of large documents.""" # Create a large document large_text = "\n\n".join([f"Paragraph {i} with some content." for i in range(100)]) splitter = RecursiveCharacterTextSplitter(chunk_size=200, chunk_overlap=50) result = splitter.split_text(large_text) # Verify efficient chunking assert len(result) > 10 assert all(len(chunk) <= 250 for chunk in result) # Allow some tolerance def test_semantic_chunking_simulation(self): """Test semantic-like chunking by using paragraph separators.""" text = """Introduction paragraph. Main content paragraph with details. Conclusion paragraph with summary. Additional notes and references.""" splitter = RecursiveCharacterTextSplitter(chunk_size=100, chunk_overlap=20, keep_separator=True) result = splitter.split_text(text) # Verify paragraph structure is somewhat maintained assert len(result) > 0 assert all(isinstance(chunk, str) for chunk in result) # ============================================================================ # Test Performance and Limits # ============================================================================ class TestPerformanceAndLimits: """Test performance characteristics and limits.""" def test_max_chunk_size_warning(self): """Test that warning is logged for chunks exceeding size.""" # Create text with a very long word long_word = "a" * 200 text = f"Short {long_word} text" splitter = RecursiveCharacterTextSplitter(chunk_size=50, chunk_overlap=10) # Should handle gracefully and log warning result = splitter.split_text(text) assert len(result) > 0 # Long word may be split into multiple chunks at character level # Verify all content is preserved combined = "".join(result) assert "a" * 100 in combined # At least part of the long word is preserved def test_many_small_chunks(self): """Test creating many small chunks.""" text = " ".join([f"w{i}" for i in range(1000)]) splitter = RecursiveCharacterTextSplitter(chunk_size=20, chunk_overlap=5) result = splitter.split_text(text) # Should create many chunks assert len(result) > 50 assert all(isinstance(chunk, str) for chunk in result) def test_deeply_nested_splitting(self): """ Test that recursive splitting works for deeply nested cases. This test verifies that the splitter can handle text that requires multiple levels of recursive splitting (paragraph -> line -> word -> character). """ # Text that requires multiple levels of splitting text = "word1" + "x" * 100 + "word2" + "y" * 100 + "word3" splitter = RecursiveCharacterTextSplitter(chunk_size=30, chunk_overlap=5) result = splitter.split_text(text) assert len(result) > 3 # Verify all content is present combined = "".join(result) assert "word1" in combined assert "word2" in combined assert "word3" in combined # ============================================================================ # Test Advanced Splitting Scenarios # ============================================================================ class TestAdvancedSplittingScenarios: """ Test advanced and complex splitting scenarios. This test class covers edge cases and advanced use cases that may occur in production environments, including structured documents, special formatting, and boundary conditions. """ def test_markdown_document_splitting(self, markdown_text): """ Test splitting of markdown formatted documents. Markdown documents have hierarchical structure with headers and sections. This test verifies that the splitter respects document structure while maintaining readability of chunks. """ splitter = RecursiveCharacterTextSplitter(chunk_size=150, chunk_overlap=20, keep_separator=True) result = splitter.split_text(markdown_text) # Should create multiple chunks assert len(result) > 0 # Verify markdown structure is somewhat preserved combined = "\n".join(result) assert "#" in combined # Headers should be present assert "Section" in combined # Each chunk should be within size limits assert all(len(chunk) <= 200 for chunk in result) def test_html_content_splitting(self, html_text): """ Test splitting of HTML formatted content. HTML has nested tags and structure. This test ensures that splitting doesn't break the content in ways that would make it unusable. """ splitter = RecursiveCharacterTextSplitter(chunk_size=100, chunk_overlap=15) result = splitter.split_text(html_text) assert len(result) > 0 # Verify HTML content is preserved combined = "".join(result) assert "paragraph" in combined.lower() or "para" in combined.lower() def test_json_structure_splitting(self, json_text): """ Test splitting of JSON formatted data. JSON has specific structure with braces, brackets, and quotes. While the splitter doesn't parse JSON, it should handle it without losing critical content. """ splitter = RecursiveCharacterTextSplitter(chunk_size=80, chunk_overlap=10) result = splitter.split_text(json_text) assert len(result) > 0 # Verify key JSON elements are preserved combined = "".join(result) assert "name" in combined or "content" in combined def test_technical_documentation_splitting(self, technical_text): """ Test splitting of technical documentation. Technical docs often have specific formatting with sections, code examples, and structured information. This test ensures such content is split appropriately. """ splitter = RecursiveCharacterTextSplitter(chunk_size=200, chunk_overlap=30, keep_separator=True) result = splitter.split_text(technical_text) assert len(result) > 0 # Verify technical content is preserved combined = "\n".join(result) assert "API" in combined or "api" in combined.lower() assert "Parameters" in combined or "Error" in combined def test_mixed_content_types(self): """ Test splitting document with mixed content types. Real-world documents often mix prose, code, lists, and other content types. This test verifies handling of such mixed content. """ mixed_text = """Introduction to the API Here is some explanatory text about how to use the API. ```python def example(): return {"status": "success"} ``` Key Points: - Point 1: First important point - Point 2: Second important point - Point 3: Third important point Conclusion paragraph with final thoughts.""" splitter = RecursiveCharacterTextSplitter(chunk_size=120, chunk_overlap=20) result = splitter.split_text(mixed_text) assert len(result) > 0 # Verify different content types are preserved combined = "\n".join(result) assert "API" in combined or "api" in combined.lower() assert "Point" in combined or "point" in combined def test_bullet_points_and_lists(self): """ Test splitting of text with bullet points and lists. Lists are common in documents and should be split in a way that maintains their structure and readability. """ list_text = """Main Topic Key Features: - Feature 1: Description of first feature - Feature 2: Description of second feature - Feature 3: Description of third feature - Feature 4: Description of fourth feature - Feature 5: Description of fifth feature Additional Information: 1. First numbered item 2. Second numbered item 3. Third numbered item""" splitter = RecursiveCharacterTextSplitter(chunk_size=100, chunk_overlap=15) result = splitter.split_text(list_text) assert len(result) > 0 # Verify list structure is somewhat maintained combined = "\n".join(result) assert "Feature" in combined or "feature" in combined def test_quoted_text_handling(self): """ Test handling of quoted text and dialogue. Quotes and dialogue have special formatting that should be preserved during splitting. """ quoted_text = """The speaker said, "This is a very important quote that contains multiple sentences. \ It goes on for quite a while and has significant meaning." Another person responded, "I completely agree with that statement. \ We should consider all the implications." A third voice added, "Let's not forget about the other perspective here." The discussion continued with more detailed points.""" splitter = RecursiveCharacterTextSplitter(chunk_size=100, chunk_overlap=20) result = splitter.split_text(quoted_text) assert len(result) > 0 # Verify quotes are preserved combined = " ".join(result) assert "said" in combined or "responded" in combined def test_table_like_content(self): """ Test splitting of table-like formatted content. Tables and structured data layouts should be handled gracefully even though the splitter doesn't understand table semantics. """ table_text = """Product Comparison Table Name | Price | Rating | Stock ------------- | ------ | ------ | ----- Product A | $29.99 | 4.5 | 100 Product B | $39.99 | 4.8 | 50 Product C | $19.99 | 4.2 | 200 Product D | $49.99 | 4.9 | 25 Notes: All prices include tax.""" splitter = RecursiveCharacterTextSplitter(chunk_size=120, chunk_overlap=15) result = splitter.split_text(table_text) assert len(result) > 0 # Verify table content is preserved combined = "\n".join(result) assert "Product" in combined or "Price" in combined def test_urls_and_links_preservation(self): """ Test that URLs and links are preserved during splitting. URLs should not be broken across chunks as that would make them unusable. """ url_text = """For more information, visit https://www.example.com/very/long/path/to/resource You can also check out https://api.example.com/v1/documentation for API details. Additional resources: - https://github.com/example/repo - https://stackoverflow.com/questions/12345/example-question Contact us at support@example.com for help.""" splitter = RecursiveCharacterTextSplitter( chunk_size=100, chunk_overlap=20, separators=["\n\n", "\n", " ", ""], # Space separator helps keep URLs together ) result = splitter.split_text(url_text) assert len(result) > 0 # Verify URLs are present in chunks combined = " ".join(result) assert "http" in combined or "example.com" in combined def test_email_content_splitting(self): """ Test splitting of email-like content. Emails have headers, body, and signatures that should be handled appropriately. """ email_text = """From: sender@example.com To: recipient@example.com Subject: Important Update Dear Team, I wanted to inform you about the recent changes to our project timeline. \ The new deadline is next month, and we need to adjust our priorities accordingly. Please review the attached documents and provide your feedback by end of week. Key action items: 1. Review documentation 2. Update project plan 3. Schedule follow-up meeting Best regards, John Doe Senior Manager""" splitter = RecursiveCharacterTextSplitter(chunk_size=150, chunk_overlap=20) result = splitter.split_text(email_text) assert len(result) > 0 # Verify email structure is preserved combined = "\n".join(result) assert "From" in combined or "Subject" in combined or "Dear" in combined # ============================================================================ # Test Splitter Configuration and Customization # ============================================================================ class TestSplitterConfiguration: """ Test various configuration options for text splitters. This class tests different parameter combinations and configurations to ensure splitters behave correctly under various settings. """ def test_custom_length_function(self): """ Test using a custom length function. The splitter allows custom length functions for specialized counting (e.g., word count instead of character count). """ # Custom length function that counts words def word_count_length(texts: list[str]) -> list[int]: return [len(text.split()) for text in texts] splitter = RecursiveCharacterTextSplitter( chunk_size=10, # 10 words chunk_overlap=2, # 2 words overlap length_function=word_count_length, ) text = " ".join([f"word{i}" for i in range(30)]) result = splitter.split_text(text) # Should create multiple chunks based on word count assert len(result) > 1 # Each chunk should have roughly 10 words or fewer for chunk in result: word_count = len(chunk.split()) assert word_count <= 15 # Allow some tolerance def test_different_separator_orders(self): """ Test different orderings of separators. The order of separators affects how text is split. This test verifies that different orders produce different results. """ text = "Paragraph one.\n\nParagraph two.\nLine break here.\nAnother line." # Try paragraph-first splitting splitter1 = RecursiveCharacterTextSplitter( chunk_size=50, chunk_overlap=5, separators=["\n\n", "\n", ".", " ", ""] ) result1 = splitter1.split_text(text) # Try line-first splitting splitter2 = RecursiveCharacterTextSplitter( chunk_size=50, chunk_overlap=5, separators=["\n", "\n\n", ".", " ", ""] ) result2 = splitter2.split_text(text) # Both should produce valid results assert len(result1) > 0 assert len(result2) > 0 # Results may differ based on separator priority assert isinstance(result1, list) assert isinstance(result2, list) def test_extreme_overlap_ratios(self): """ Test splitters with extreme overlap ratios. Tests edge cases where overlap is very small or very large relative to chunk size. """ text = "A B C D E F G H I J K L M N O P Q R S T U V W X Y Z" # Very small overlap (1% of chunk size) splitter_small = RecursiveCharacterTextSplitter(chunk_size=20, chunk_overlap=1) result_small = splitter_small.split_text(text) # Large overlap (90% of chunk size) splitter_large = RecursiveCharacterTextSplitter(chunk_size=20, chunk_overlap=18) result_large = splitter_large.split_text(text) # Both should work assert len(result_small) > 0 assert len(result_large) > 0 # Large overlap should create more chunks assert len(result_large) >= len(result_small) def test_add_start_index_accuracy(self): """ Test that start_index metadata is accurately calculated. The start_index should point to the actual position of the chunk in the original text. """ text = string.ascii_uppercase splitter = RecursiveCharacterTextSplitter(chunk_size=10, chunk_overlap=2, add_start_index=True) docs = splitter.create_documents([text]) # Verify start indices are correct for doc in docs: start_idx = doc.metadata.get("start_index") if start_idx is not None: # The chunk should actually appear at that index assert text[start_idx : start_idx + len(doc.page_content)] == doc.page_content def test_separator_regex_patterns(self): """ Test using regex patterns as separators. Separators can be regex patterns for more sophisticated splitting. """ # Text with multiple spaces and tabs text = "Word1 Word2\t\tWord3 Word4\tWord5" splitter = RecursiveCharacterTextSplitter( chunk_size=20, chunk_overlap=3, separators=[r"\s+", ""], # Split on any whitespace ) result = splitter.split_text(text) assert len(result) > 0 # Verify words are split combined = " ".join(result) assert "Word" in combined # ============================================================================ # Test Error Handling and Robustness # ============================================================================ class TestErrorHandlingAndRobustness: """ Test error handling and robustness of splitters. This class tests how splitters handle invalid inputs, edge cases, and error conditions. """ def test_none_text_handling(self): """ Test handling of None as input. Splitters should handle None gracefully without crashing. """ splitter = RecursiveCharacterTextSplitter(chunk_size=100, chunk_overlap=10) # Should handle None without crashing try: result = splitter.split_text(None) # If it doesn't raise an error, result should be empty or handle gracefully assert result is not None except (TypeError, AttributeError): # It's acceptable to raise a type error for None input pass def test_very_large_chunk_size(self): """ Test splitter with chunk size larger than any reasonable text. When chunk size is very large, text should remain unsplit. """ text = "This is a short text." splitter = RecursiveCharacterTextSplitter(chunk_size=1000000, chunk_overlap=100) result = splitter.split_text(text) # Should return single chunk assert len(result) == 1 assert result[0] == text def test_chunk_size_one(self): """ Test splitter with minimum chunk size of 1. This extreme case should split text character by character. """ text = "ABC" splitter = RecursiveCharacterTextSplitter(chunk_size=1, chunk_overlap=0) result = splitter.split_text(text) # Should split into individual characters assert len(result) >= 3 # Verify all content is preserved combined = "".join(result) assert "A" in combined assert "B" in combined assert "C" in combined def test_special_unicode_characters(self): """ Test handling of special unicode characters. Splitters should handle emojis, special symbols, and other unicode characters without issues. """ text = "Hello 👋 World 🌍 Test 🚀 Data 📊 End 🎉" splitter = RecursiveCharacterTextSplitter(chunk_size=20, chunk_overlap=5) result = splitter.split_text(text) assert len(result) > 0 # Verify unicode is preserved combined = " ".join(result) assert "Hello" in combined assert "World" in combined def test_control_characters(self): """ Test handling of control characters. Text may contain tabs, carriage returns, and other control characters that should be handled properly. """ text = "Line1\r\nLine2\tTabbed\r\nLine3" splitter = RecursiveCharacterTextSplitter(chunk_size=30, chunk_overlap=5) result = splitter.split_text(text) assert len(result) > 0 # Verify content is preserved combined = "".join(result) assert "Line1" in combined assert "Line2" in combined def test_repeated_separators(self): """ Test text with many repeated separators. Multiple consecutive separators should be handled without creating empty chunks. """ text = "Word1\n\n\n\n\nWord2\n\n\n\nWord3" splitter = RecursiveCharacterTextSplitter(chunk_size=50, chunk_overlap=5) result = splitter.split_text(text) assert len(result) > 0 # Should not have empty chunks assert all(len(chunk.strip()) > 0 for chunk in result) def test_documents_with_empty_metadata(self): """ Test splitting documents with empty metadata. Documents may have empty metadata dict, which should be handled properly and preserved in chunks. """ splitter = RecursiveCharacterTextSplitter(chunk_size=30, chunk_overlap=5) # Create documents with empty metadata docs = [Document(page_content="Content here", metadata={})] result = splitter.split_documents(docs) assert len(result) > 0 # Metadata should be dict (empty dict is valid) for doc in result: assert isinstance(doc.metadata, dict) def test_empty_separator_list(self): """ Test splitter with empty separator list. Edge case where no separators are provided should still work by falling back to default behavior. """ text = "Test text here" try: splitter = RecursiveCharacterTextSplitter(chunk_size=20, chunk_overlap=5, separators=[]) result = splitter.split_text(text) # Should still produce some result assert isinstance(result, list) except (ValueError, IndexError): # It's acceptable to raise an error for empty separators pass # ============================================================================ # Test Performance Characteristics # ============================================================================ class TestPerformanceCharacteristics: """ Test performance-related characteristics of splitters. These tests verify that splitters perform efficiently and handle large-scale operations appropriately. """ def test_consistent_chunk_sizes(self): """ Test that chunk sizes are relatively consistent. While chunks may vary in size, they should generally be close to the target chunk size (except for the last chunk). """ text = " ".join([f"Word{i}" for i in range(200)]) splitter = RecursiveCharacterTextSplitter(chunk_size=100, chunk_overlap=10) result = splitter.split_text(text) # Most chunks should be close to target size sizes = [len(chunk) for chunk in result[:-1]] # Exclude last chunk if sizes: avg_size = sum(sizes) / len(sizes) # Average should be reasonably close to target assert 50 <= avg_size <= 150 def test_minimal_information_loss(self): """ Test that splitting and rejoining preserves information. When chunks are rejoined, the content should be largely preserved (accounting for separator handling). """ text = "The quick brown fox jumps over the lazy dog. " * 10 splitter = RecursiveCharacterTextSplitter(chunk_size=50, chunk_overlap=10, keep_separator=True) result = splitter.split_text(text) combined = "".join(result) # Most of the original text should be preserved # (Some separators might be handled differently) assert "quick" in combined assert "brown" in combined assert "fox" in combined assert "dog" in combined def test_deterministic_splitting(self): """ Test that splitting is deterministic. Running the same splitter on the same text multiple times should produce identical results. """ text = "Consistent text for deterministic testing. " * 5 splitter = RecursiveCharacterTextSplitter(chunk_size=50, chunk_overlap=10) result1 = splitter.split_text(text) result2 = splitter.split_text(text) result3 = splitter.split_text(text) # All results should be identical assert result1 == result2 assert result2 == result3 def test_chunk_count_estimation(self): """ Test that chunk count is reasonable for given text length. The number of chunks should be proportional to text length and inversely proportional to chunk size. """ base_text = "Word " * 100 # Small chunks should create more chunks splitter_small = RecursiveCharacterTextSplitter(chunk_size=20, chunk_overlap=5) result_small = splitter_small.split_text(base_text) # Large chunks should create fewer chunks splitter_large = RecursiveCharacterTextSplitter(chunk_size=100, chunk_overlap=5) result_large = splitter_large.split_text(base_text) # Small chunk size should produce more chunks assert len(result_small) > len(result_large)