diff --git a/tests/workspace/test_parsers.py b/tests/workspace/test_parsers.py new file mode 100644 index 0000000000..0c01a43182 --- /dev/null +++ b/tests/workspace/test_parsers.py @@ -0,0 +1,57 @@ +import pytest + +from workspace.constants import BINARY_SUFFIXES, CODE_SUFFIXES, MARKDOWN_SUFFIXES, PARSEABLE_SUFFIXES + + +class TestParseableSuffixes: + def test_contains_expected_extensions(self): + assert PARSEABLE_SUFFIXES == frozenset({".pdf", ".docx", ".pptx"}) + + def test_is_subset_of_binary_suffixes(self): + assert PARSEABLE_SUFFIXES <= BINARY_SUFFIXES + + def test_no_overlap_with_code_suffixes(self): + assert PARSEABLE_SUFFIXES & CODE_SUFFIXES == frozenset() + + def test_no_overlap_with_markdown_suffixes(self): + assert PARSEABLE_SUFFIXES & MARKDOWN_SUFFIXES == frozenset() + + +from workspace.config import KnowledgebaseConfig, ParsingConfig + + +class TestParsingConfig: + def test_default_values(self): + cfg = ParsingConfig() + assert cfg.default == "markitdown" + assert cfg.overrides == {} + + def test_custom_default(self): + cfg = ParsingConfig(default="pandoc") + assert cfg.default == "pandoc" + + def test_per_extension_overrides(self): + cfg = ParsingConfig(overrides={".docx": "pandoc"}) + assert cfg.overrides[".docx"] == "pandoc" + + def test_frozen(self): + cfg = ParsingConfig() + with pytest.raises(Exception): + cfg.default = "pandoc" + + def test_nested_in_knowledgebase_config(self): + kb = KnowledgebaseConfig() + assert isinstance(kb.parsing, ParsingConfig) + assert kb.parsing.default == "markitdown" + + def test_knowledgebase_config_from_raw(self): + kb = KnowledgebaseConfig.model_validate( + {"parsing": {"default": "pandoc", "overrides": {".docx": "pandoc"}}} + ) + assert kb.parsing.default == "pandoc" + assert kb.parsing.overrides == {".docx": "pandoc"} + + def test_unknown_keys_ignored(self): + cfg = ParsingConfig.model_validate({"default": "markitdown", "future_key": True}) + assert cfg.default == "markitdown" + assert not hasattr(cfg, "future_key") diff --git a/workspace/config.py b/workspace/config.py index d9ae7b8ddb..8bc0025daa 100644 --- a/workspace/config.py +++ b/workspace/config.py @@ -59,12 +59,19 @@ class SearchConfig(BaseModel): return v +class ParsingConfig(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + default: str = "markitdown" + overrides: dict[str, str] = {} + + class KnowledgebaseConfig(BaseModel): model_config = ConfigDict(frozen=True) roots: list[WorkspaceRoot] = [] chunking: ChunkingConfig = ChunkingConfig() indexing: IndexingConfig = IndexingConfig() search: SearchConfig = SearchConfig() + parsing: ParsingConfig = ParsingConfig() class WorkspaceConfig(BaseModel): diff --git a/workspace/constants.py b/workspace/constants.py index c584d60a9c..08972f3ffd 100644 --- a/workspace/constants.py +++ b/workspace/constants.py @@ -63,6 +63,14 @@ BINARY_SUFFIXES: frozenset[str] = frozenset( } ) +PARSEABLE_SUFFIXES: frozenset[str] = frozenset( + { + ".pdf", + ".docx", + ".pptx", + } +) + CODE_SUFFIXES: frozenset[str] = frozenset( { ".py",