feat(workspace): add PARSEABLE_SUFFIXES and ParsingConfig

This commit is contained in:
alt-glitch
2026-04-20 03:35:26 +05:30
parent fe80e118b7
commit ec0fa5a2be
3 changed files with 72 additions and 0 deletions
+57
View File
@@ -0,0 +1,57 @@
import pytest
from workspace.constants import BINARY_SUFFIXES, CODE_SUFFIXES, MARKDOWN_SUFFIXES, PARSEABLE_SUFFIXES
class TestParseableSuffixes:
def test_contains_expected_extensions(self):
assert PARSEABLE_SUFFIXES == frozenset({".pdf", ".docx", ".pptx"})
def test_is_subset_of_binary_suffixes(self):
assert PARSEABLE_SUFFIXES <= BINARY_SUFFIXES
def test_no_overlap_with_code_suffixes(self):
assert PARSEABLE_SUFFIXES & CODE_SUFFIXES == frozenset()
def test_no_overlap_with_markdown_suffixes(self):
assert PARSEABLE_SUFFIXES & MARKDOWN_SUFFIXES == frozenset()
from workspace.config import KnowledgebaseConfig, ParsingConfig
class TestParsingConfig:
def test_default_values(self):
cfg = ParsingConfig()
assert cfg.default == "markitdown"
assert cfg.overrides == {}
def test_custom_default(self):
cfg = ParsingConfig(default="pandoc")
assert cfg.default == "pandoc"
def test_per_extension_overrides(self):
cfg = ParsingConfig(overrides={".docx": "pandoc"})
assert cfg.overrides[".docx"] == "pandoc"
def test_frozen(self):
cfg = ParsingConfig()
with pytest.raises(Exception):
cfg.default = "pandoc"
def test_nested_in_knowledgebase_config(self):
kb = KnowledgebaseConfig()
assert isinstance(kb.parsing, ParsingConfig)
assert kb.parsing.default == "markitdown"
def test_knowledgebase_config_from_raw(self):
kb = KnowledgebaseConfig.model_validate(
{"parsing": {"default": "pandoc", "overrides": {".docx": "pandoc"}}}
)
assert kb.parsing.default == "pandoc"
assert kb.parsing.overrides == {".docx": "pandoc"}
def test_unknown_keys_ignored(self):
cfg = ParsingConfig.model_validate({"default": "markitdown", "future_key": True})
assert cfg.default == "markitdown"
assert not hasattr(cfg, "future_key")
+7
View File
@@ -59,12 +59,19 @@ class SearchConfig(BaseModel):
return v
class ParsingConfig(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
default: str = "markitdown"
overrides: dict[str, str] = {}
class KnowledgebaseConfig(BaseModel):
model_config = ConfigDict(frozen=True)
roots: list[WorkspaceRoot] = []
chunking: ChunkingConfig = ChunkingConfig()
indexing: IndexingConfig = IndexingConfig()
search: SearchConfig = SearchConfig()
parsing: ParsingConfig = ParsingConfig()
class WorkspaceConfig(BaseModel):
+8
View File
@@ -63,6 +63,14 @@ BINARY_SUFFIXES: frozenset[str] = frozenset(
}
)
PARSEABLE_SUFFIXES: frozenset[str] = frozenset(
{
".pdf",
".docx",
".pptx",
}
)
CODE_SUFFIXES: frozenset[str] = frozenset(
{
".py",