Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions lib/crewai-tools/src/crewai_tools/adapters/crewai_rag_adapter.py
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,8 @@ def add(self, *args: ContentItem, **kwargs: Unpack[AddDocumentParams]) -> None:
else:
source_ref = str(arg)

if not data_type:
item_data_type = data_type
if not item_data_type:
ext = os.path.splitext(source_ref)[1].lower()
is_url = source_ref.startswith(("http://", "https://", "file://"))
if (
Expand All @@ -206,9 +207,9 @@ def add(self, *args: ContentItem, **kwargs: Unpack[AddDocumentParams]) -> None:
and not os.path.isfile(source_ref)
):
raise FileNotFoundError(f"File does not exist: {source_ref}")
data_type = DataTypes.from_content(source_ref)
item_data_type = DataTypes.from_content(source_ref)

if data_type == DataType.DIRECTORY:
if item_data_type == DataType.DIRECTORY:
if not os.path.isdir(source_ref):
raise ValueError(f"Directory does not exist: {source_ref}")

Expand Down Expand Up @@ -304,7 +305,7 @@ def add(self, *args: ContentItem, **kwargs: Unpack[AddDocumentParams]) -> None:
metadata: dict[str, Any] = base_metadata.copy()
source_content = SourceContent(source_ref)

if data_type in [
if item_data_type in [
DataType.PDF_FILE,
DataType.TEXT_FILE,
DataType.DOCX,
Expand All @@ -316,8 +317,8 @@ def add(self, *args: ContentItem, **kwargs: Unpack[AddDocumentParams]) -> None:
if not source_content.is_url() and not source_content.path_exists():
raise FileNotFoundError(f"File does not exist: {source_ref}")

loader = data_type.get_loader()
chunker = data_type.get_chunker()
loader = item_data_type.get_loader()
chunker = item_data_type.get_chunker()

loader_result: LoaderResult = loader.load(source_content, **kwargs)

Expand All @@ -326,7 +327,7 @@ def add(self, *args: ContentItem, **kwargs: Unpack[AddDocumentParams]) -> None:
for i, chunk in enumerate(chunks):
chunk_metadata: dict[str, Any] = metadata.copy()
chunk_metadata.update(loader_result.metadata)
chunk_metadata["data_type"] = str(data_type)
chunk_metadata["data_type"] = str(item_data_type)
chunk_metadata["chunk_index"] = i
chunk_metadata["total_chunks"] = len(chunks)
chunk_metadata["source"] = source_ref
Expand Down
3 changes: 2 additions & 1 deletion lib/crewai-tools/src/crewai_tools/rag/data_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -123,8 +123,9 @@ def get_file_type(path: str) -> DataType | None:
".xml": DataType.XML,
".txt": DataType.TEXT_FILE,
}
path_lower = path.lower()
for ext, dtype in mapping.items():
if path.endswith(ext):
if path_lower.endswith(ext):
return dtype
return None

Expand Down
52 changes: 51 additions & 1 deletion lib/crewai-tools/tests/tools/rag/test_rag_tool_add_data_type.py
Original file line number Diff line number Diff line change
Expand Up @@ -475,4 +475,54 @@ def test_metadata_passed_to_documents(
assert any(
doc.get("metadata", {}).get("custom_key") == "custom_value"
for doc in documents
)
)


class TestAutoDetectMultipleItemsAndCaseSensitivity:
"""Tests for case-insensitive extensions and multi-item auto-detection."""

def test_auto_detect_uppercase_file_extension(
self, rag_tool: RagTool, mock_rag_client: MagicMock
) -> None:
"""Test that uppercase file extensions like .JSON and .CSV are correctly detected."""
with TemporaryDirectory() as tmpdir:
json_file = Path(tmpdir) / "DATA.JSON"
json_file.write_text('{"name": "test"}')

rag_tool.add(str(json_file))

assert mock_rag_client.add_documents.called
call_args = mock_rag_client.add_documents.call_args
documents = call_args.kwargs.get("documents", [])
assert any(
doc.get("metadata", {}).get("data_type") == str(DataType.JSON)
for doc in documents
)

def test_auto_detect_uppercase_url_extension(self) -> None:
"""Test that URLs with uppercase extensions resolve to their correct DataType."""
from crewai_tools.rag.data_types import DataTypes

assert DataTypes.from_content("https://example.com/report.PDF") == DataType.PDF_FILE
assert DataTypes.from_content("https://example.com/data.CSV") == DataType.CSV

def test_auto_detect_multiple_items_independent_types(
self, rag_tool: RagTool, mock_rag_client: MagicMock
) -> None:
"""Test that adding a directory followed by a file does not leak DataType.DIRECTORY."""
with TemporaryDirectory() as tmpdir:
subfolder = Path(tmpdir) / "subfolder"
subfolder.mkdir()
(subfolder / "inner.txt").write_text("inner content")

file2 = Path(tmpdir) / "outer.txt"
file2.write_text("outer content")

# Must not raise ValueError: Directory does not exist on outer.txt
rag_tool.add(str(subfolder), str(file2))

assert mock_rag_client.add_documents.called
call_args = mock_rag_client.add_documents.call_args
documents = call_args.kwargs.get("documents", [])
sources = [doc.get("metadata", {}).get("source") for doc in documents]
assert any("outer.txt" in s for s in sources if s)
Loading