From c85ea9d64f199a3fc96776b28cb881a20410d4ad Mon Sep 17 00:00:00 2001 From: Artem Akymenko Date: Sun, 12 Jul 2026 13:02:10 +0000 Subject: [PATCH] refactor(tests): add auto-discovery test system for TTS plugins - Create tests/plugins/ with auto-discovery fixtures and generic tests - Add conftest.py with plugin_ids, loaded_plugin, host_context fixtures - Add test_all_plugins.py with 3 test classes: - TestAllPluginsManifest: validates manifest structure - TestAllPluginsEngine: validates engine lifecycle contract - TestAllPluginsCapabilities: validates capability implementation - Update docs/testing.md with auto-discovery documentation - Plugin-specific tests remain in tests/test_*_plugin.py for integration New plugins in plugins/ are now automatically tested without manual test creation. --- docs/testing.md | 115 +++++++++++---- tests/plugins/__init__.py | 11 ++ tests/plugins/conftest.py | 169 +++++++++++++++++++++ tests/plugins/test_all_plugins.py | 235 ++++++++++++++++++++++++++++++ 4 files changed, 498 insertions(+), 32 deletions(-) create mode 100644 tests/plugins/__init__.py create mode 100644 tests/plugins/conftest.py create mode 100644 tests/plugins/test_all_plugins.py diff --git a/docs/testing.md b/docs/testing.md index c7de656..7ae7863 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -4,6 +4,60 @@ This document describes the testing strategy for Abogen's Plugin Architecture. ## Test Categories +### 0. Auto-Discovery Plugin Tests (`tests/plugins/`) + +**Purpose**: Automatically test every plugin in `plugins/` directory without manual test creation. These tests use discovery to find all plugins and run generic tests against each one. + +**What They Test**: +- **Manifest structure**: Required fields, API version format, voices field +- **Engine lifecycle**: `create_engine`, `dispose` idempotency, post-dispose behavior +- **Capability implementation**: Declared capabilities are implemented (e.g., `voice_list` → `VoiceLister`) + +**How Auto-Discovery Works**: +```python +# tests/plugins/conftest.py +@pytest.fixture(scope="module") +def plugin_ids(plugins_dir: Path) -> list[str]: + """Discovers all plugin directories with __init__.py""" + return [item.name for item in plugins_dir.iterdir() + if item.is_dir() and (item / "__init__.py").exists()] +``` + +**Test Structure**: +``` +tests/plugins/ +├── conftest.py # Fixtures: plugin_ids, loaded_plugin, host_context +└── test_all_plugins.py # Generic tests for every plugin + ├── TestAllPluginsManifest + ├── TestAllPluginsEngine + └── TestAllPluginsCapabilities +``` + +**Running Auto-Discovery Tests**: +```bash +# Test all plugins automatically +pytest tests/plugins/ -v + +# Test specific plugin +pytest tests/plugins/ -v -k "kokoro" + +# See which plugins were discovered +pytest tests/plugins/ --collect-only +``` + +**Adding a New Plugin**: +1. Create plugin directory: `plugins/my_plugin/` +2. Add `__init__.py` with `PLUGIN_MANIFEST`, `MODEL_REQUIREMENTS`, `create_engine` +3. Run `pytest tests/plugins/` — tests automatically discover and test your plugin! + +**When to Add Plugin-Specific Tests**: +Auto-discovery tests cover generic contract validation. Create plugin-specific tests in `tests/test__plugin.py` for: +- Integration with real dependencies (e.g., KPipeline for Kokoro) +- Specific voice IDs and behavior +- Plugin-specific parameters and features + +--- + ### 1. Contract Tests (`tests/contracts/`) **Purpose**: Verify that every plugin satisfies the architectural contract. These tests ensure the Plugin Architecture's invariants are maintained. @@ -82,7 +136,7 @@ pytest tests/test_behavioral_regression.py -v -k "kokoro" --- -### 3. Unit Tests (`tests/`) +### 4. Unit Tests (`tests/`) **Purpose**: Test individual modules in isolation. @@ -94,7 +148,7 @@ pytest tests/test_behavioral_regression.py -v -k "kokoro" --- -### 4. Integration Tests +### 5. Integration Tests **Purpose**: Test cross-component interactions. @@ -132,38 +186,35 @@ tests/ ## Adding Tests for a New Plugin -### Contract Tests (Required) +### Auto-Discovery Tests (Automatic!) -Create `tests/contracts/test_your_plugin.py`: +**No manual test creation required!** When you add a new plugin to `plugins/`: -```python -"""Contract tests for YourPlugin — verifies architectural compliance.""" +1. Create plugin directory: `plugins/my_plugin/` +2. Add `__init__.py` with required exports: + ```python + PLUGIN_MANIFEST = PluginManifest(...) + MODEL_REQUIREMENTS = [...] + def create_engine(...): ... + ``` +3. Run `pytest tests/plugins/` — auto-discovery tests automatically find and test your plugin! -from pathlib import Path -from abogen.tts_plugin.loader import load_plugin_from_dir -from abogen.tts_plugin.manifest import PluginManifest -from abogen.tts_plugin.engine import Engine +**What's Tested Automatically**: +- Manifest structure and required fields +- API version compatibility +- Engine creation and dispose contract +- Capability implementation (if declared) -def test_your_plugin_loads(): - result = load_plugin_from_dir(Path("plugins/your_tts")) - assert result.success - assert isinstance(result.manifest, PluginManifest) - assert result.manifest.id == "your_tts" - assert callable(result.create_engine) +### Plugin-Specific Tests (Optional) -def test_your_plugin_creates_engine(): - result = load_plugin_from_dir(Path("plugins/your_tts")) - # ... create HostContext, EngineConfig - engine = result.create_engine(ctx, None, EngineConfig(device="cpu")) - assert isinstance(engine, Engine) - engine.dispose() +Create `tests/test_my_plugin_plugin.py` for: +- Integration with real backend (e.g., KPipeline for Kokoro) +- Specific voice IDs and behavior +- Plugin-specific parameters and features -def test_your_plugin_capabilities(): - """If plugin declares capabilities, verify they're implemented.""" - result = load_plugin_from_dir(Path("plugins/your_tts")) - # Check VoiceLister, PreviewGenerator, etc. - ... -``` +### Contract Tests (Deprecated for New Plugins) + +**Note**: Auto-discovery tests (`tests/plugins/`) now cover contract validation for all plugins. Manual contract tests in `tests/contracts/` are only needed for testing internal architecture components. ### Behavioral Tests (Recommended) @@ -171,10 +222,10 @@ Add parametrized tests to `tests/test_behavioral_regression.py`: ```python # In _plugin_ids list, add your plugin -_plugin_ids = ["kokoro", "supertonic", "your_tts"] -_plugin_engines["your_tts"] = _YourMockEngine -_plugin_default_voices["your_tts"] = "voice1" -_plugin_all_voices["your_tts"] = ["voice1", "voice2"] +_plugin_ids = ["kokoro", "supertonic", "my_plugin"] +_plugin_engines["my_plugin"] = _YourMockEngine +_plugin_default_voices["my_plugin"] = "voice1" +_plugin_all_voices["my_plugin"] = ["voice1", "voice2"] ``` All existing behavioral tests will automatically run against your plugin. diff --git a/tests/plugins/__init__.py b/tests/plugins/__init__.py new file mode 100644 index 0000000..b48ae16 --- /dev/null +++ b/tests/plugins/__init__.py @@ -0,0 +1,11 @@ +"""Auto-discovery tests for TTS plugins. + +This package contains generic tests that automatically run for every plugin +in the plugins/ directory. Tests verify: +- Manifest structure +- Engine creation and dispose contract +- Capability implementation + +Plugin-specific tests remain in tests/test__plugin.py for +integration with real dependencies (e.g., KPipeline for Kokoro). +""" \ No newline at end of file diff --git a/tests/plugins/conftest.py b/tests/plugins/conftest.py new file mode 100644 index 0000000..5dc5ede --- /dev/null +++ b/tests/plugins/conftest.py @@ -0,0 +1,169 @@ +"""Fixtures for auto-discovery plugin tests. + +This module provides shared fixtures for testing all plugins: +- plugin_ids: List of all plugin IDs from plugins/ directory +- plugin_dir: Path to a specific plugin directory (parametrized) +- loaded_plugin: Loaded plugin data (manifest, model_requirements, create_engine) +- host_context: Test HostContext with fake HTTP client +""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Any + +import pytest + +from abogen.tts_plugin.host_context import HostContext +from abogen.tts_plugin.loader import load_plugin_from_dir + + +def _discover_plugin_ids() -> list[str]: + """Discover all plugin IDs from the plugins/ directory. + + Returns: + List of plugin IDs (directory names with __init__.py). + """ + plugins_dir = Path(__file__).parent.parent.parent / "plugins" + plugin_ids = [] + if plugins_dir.exists(): + for item in sorted(plugins_dir.iterdir()): + if item.is_dir() and not item.name.startswith("."): + init_file = item / "__init__.py" + if init_file.exists(): + plugin_ids.append(item.name) + return plugin_ids + + +@pytest.fixture(scope="module") +def plugins_dir() -> Path: + """Return the path to the plugins directory.""" + return Path(__file__).parent.parent.parent / "plugins" + + +@pytest.fixture(scope="module") +def plugin_ids(plugins_dir: Path) -> list[str]: + """Return a list of all plugin IDs from the plugins/ directory. + + This fixture discovers all subdirectories in plugins/ that contain + an __init__.py file (i.e., valid plugin directories). + + Returns: + List of plugin IDs (directory names). + """ + return _discover_plugin_ids() + + +@pytest.fixture(params=_discover_plugin_ids()) +def plugin_id(request: pytest.FixtureRequest) -> str: + """Parametrized fixture returning each plugin ID. + + This fixture is parametrized to run tests for each plugin. + """ + return request.param + + +@pytest.fixture +def plugin_dir(plugins_dir: Path, plugin_id: str) -> Path: + """Return the path to a specific plugin directory. + + Args: + plugins_dir: Base plugins directory. + plugin_id: The plugin ID (directory name). + + Returns: + Path to the plugin directory. + """ + return plugins_dir / plugin_id + + +@pytest.fixture +def loaded_plugin(plugin_dir: Path): + """Load a plugin and return its load result. + + This fixture loads the plugin using the loader, providing access to: + - manifest: PluginManifest + - model_requirements: tuple[ModelManifest, ...] + - create_engine: Callable + - module: The loaded module + + Args: + plugin_dir: Path to the plugin directory. + + Returns: + PluginLoadResult from load_plugin_from_dir. + + Raises: + pytest.skip: If plugin fails to load (should not happen for valid plugins). + """ + from abogen.tts_plugin.loader import PluginLoadResult + + result = load_plugin_from_dir(plugin_dir) + if not result.success: + pytest.fail( + f"Plugin {plugin_dir.name} failed to load: " + f"{result.error.errors if result.error else 'Unknown error'}" + ) + return result + + +@pytest.fixture +def host_context(tmp_path: Path) -> HostContext: + """Create a test HostContext with fake HTTP client. + + Args: + tmp_path: Pytest tmp_path fixture for config directory. + + Returns: + HostContext with fake HTTP client and test logger. + """ + class FakeHttpClient: + """Fake HTTP client for testing.""" + + def get(self, url: str, **kwargs: object) -> object: + """Fake GET request.""" + return None + + def post(self, url: str, **kwargs: object) -> object: + """Fake POST request.""" + return None + + return HostContext( + config_dir=tmp_path, + logger=logging.getLogger("test"), + http_client=FakeHttpClient(), + ) + + +@pytest.fixture +def engine_config() -> Any: + """Return a default EngineConfig for testing. + + Returns: + EngineConfig with device="cpu" for testing. + """ + from abogen.tts_plugin.types import EngineConfig + return EngineConfig(device="cpu") + + +@pytest.fixture +def create_engine(loaded_plugin, host_context, engine_config): + """Create an engine instance from a loaded plugin. + + This fixture creates an engine and ensures proper cleanup via dispose. + + Args: + loaded_plugin: Loaded plugin result. + host_context: Test HostContext. + engine_config: EngineConfig for engine creation. + + Yields: + Engine instance. + """ + from abogen.tts_plugin.engine import Engine + + engine = loaded_plugin.create_engine(host_context, None, engine_config) + assert isinstance(engine, Engine) + yield engine + engine.dispose() \ No newline at end of file diff --git a/tests/plugins/test_all_plugins.py b/tests/plugins/test_all_plugins.py new file mode 100644 index 0000000..f9b5477 --- /dev/null +++ b/tests/plugins/test_all_plugins.py @@ -0,0 +1,235 @@ +"""Auto-discovery tests for all TTS plugins. + +These tests automatically run for every plugin in the plugins/ directory. +Tests are grouped into three categories: + +1. TestAllPluginsManifest - Validates manifest structure + - Required fields (id, name, version, api_version, etc.) + - API version format (semver MAJOR.MINOR) + - Voices field (optional) + +2. TestAllPluginsEngine - Validates engine lifecycle + - create_engine returns valid Engine + - dispose is idempotent + - dispose → createSession raises EngineError + +3. TestAllPluginsCapabilities - Validates capability implementation + - Declared capabilities are implemented + - voice_list → VoiceLister interface +""" + +from __future__ import annotations + +import re + +import pytest + +from abogen.tts_plugin.engine import Engine, EngineSession +from abogen.tts_plugin.errors import EngineError +from abogen.tts_plugin.manifest import PluginManifest +from abogen.tts_plugin.types import EngineConfig, VoiceSelection, AudioFormat, ParameterValues, SynthesisRequest + + +class TestAllPluginsManifest: + """Test that all plugins have valid manifest structure.""" + + def test_manifest_has_required_fields(self, loaded_plugin): + """Verify manifest has all required fields.""" + manifest = loaded_plugin.manifest + assert manifest is not None + + # Required fields + assert manifest.id, "Plugin id must not be empty" + assert manifest.name, "Plugin name must not be empty" + assert manifest.version, "Plugin version must not be empty" + assert manifest.api_version, "Plugin api_version must not be empty" + assert manifest.description, "Plugin description must not be empty" + assert manifest.author, "Plugin author must not be empty" + + def test_manifest_id_matches_directory(self, loaded_plugin, plugin_dir): + """Verify manifest id matches the plugin directory name.""" + manifest = loaded_plugin.manifest + assert manifest.id == plugin_dir.name, \ + f"Manifest id '{manifest.id}' must match directory name '{plugin_dir.name}'" + + def test_api_version_format(self, loaded_plugin): + """Verify api_version follows semver format (MAJOR.MINOR).""" + manifest = loaded_plugin.manifest + api_version = manifest.api_version + + # Must match MAJOR.MINOR format + pattern = r"^\d+\.\d+$" + assert re.match(pattern, api_version), \ + f"api_version '{api_version}' must be in format MAJOR.MINOR (e.g., 1.0)" + + def test_api_version_compatibility(self, loaded_plugin): + """Verify api_version major version matches host API version.""" + from abogen.tts_plugin.loader import HOST_API_VERSION + + manifest = loaded_plugin.manifest + plugin_ver = manifest.api_version + host_ver = HOST_API_VERSION + + # Extract major versions + plugin_major = int(plugin_ver.split(".")[0]) + host_major = int(host_ver.split(".")[0]) + + assert plugin_major == host_major, \ + f"API version major mismatch: plugin={plugin_major}, host={host_major}" + + def test_voices_field_is_optional(self, loaded_plugin): + """Verify voices field is optional (can be None or tuple).""" + manifest = loaded_plugin.manifest + voices = manifest.voices + + # voices can be None (not declared) or tuple (empty or with voices) + assert voices is None or isinstance(voices, tuple), \ + f"voices must be None or tuple, got {type(voices).__name__}" + + def test_capabilities_is_tuple(self, loaded_plugin): + """Verify capabilities is a tuple.""" + manifest = loaded_plugin.manifest + assert isinstance(manifest.capabilities, tuple), \ + f"capabilities must be a tuple, got {type(manifest.capabilities).__name__}" + + def test_engine_manifest_exists(self, loaded_plugin): + """Verify engine manifest exists and has required fields.""" + manifest = loaded_plugin.manifest + engine_manifest = manifest.engine + + assert engine_manifest is not None + assert isinstance(engine_manifest.voiceSources, tuple) + assert isinstance(engine_manifest.parameters, tuple) + assert isinstance(engine_manifest.audioFormats, tuple) + + +class TestAllPluginsEngine: + """Test that all plugins satisfy the Engine contract.""" + + def test_create_engine_returns_engine(self, loaded_plugin, host_context, engine_config): + """Verify create_engine returns an Engine instance.""" + engine = loaded_plugin.create_engine(host_context, None, engine_config) + assert isinstance(engine, Engine), \ + f"create_engine must return Engine instance, got {type(engine).__name__}" + engine.dispose() + + def test_dispose_is_idempotent(self, loaded_plugin, host_context, engine_config): + """Verify dispose can be called multiple times without error.""" + engine = loaded_plugin.create_engine(host_context, None, engine_config) + + # First dispose + engine.dispose() + + # Second dispose should not raise + engine.dispose() + + def test_create_session_after_dispose_raises(self, loaded_plugin, host_context, engine_config): + """Verify createSession raises EngineError after dispose.""" + engine = loaded_plugin.create_engine(host_context, None, engine_config) + engine.dispose() + + with pytest.raises(EngineError): + engine.createSession() + + def test_dispose_after_dispose_raises(self, loaded_plugin, host_context, engine_config): + """Verify dispose after dispose does not raise (idempotent).""" + engine = loaded_plugin.create_engine(host_context, None, engine_config) + engine.dispose() + + # Should not raise + engine.dispose() + + def test_create_session_returns_valid_session(self, loaded_plugin, host_context, engine_config): + """Verify createSession returns an EngineSession instance.""" + engine = loaded_plugin.create_engine(host_context, None, engine_config) + session = engine.createSession() + + assert isinstance(session, EngineSession), \ + f"createSession must return EngineSession instance, got {type(session).__name__}" + session.dispose() + engine.dispose() + + def test_session_dispose_is_idempotent(self, loaded_plugin, host_context, engine_config): + """Verify session dispose can be called multiple times.""" + engine = loaded_plugin.create_engine(host_context, None, engine_config) + session = engine.createSession() + + # First dispose + session.dispose() + + # Second dispose should not raise + session.dispose() + engine.dispose() + + def test_session_synthesize_after_dispose_raises(self, loaded_plugin, host_context, engine_config): + """Verify session.synthesize raises EngineError after dispose.""" + engine = loaded_plugin.create_engine(host_context, None, engine_config) + session = engine.createSession() + session.dispose() + + # Create a minimal request (won't actually synthesize, just test error) + request = SynthesisRequest( + text="test", + voice=VoiceSelection(source="test", key="test"), + parameters=ParameterValues(values={}), + format=AudioFormat(mime="audio/wav", extension="wav"), + ) + + with pytest.raises(EngineError): + session.synthesize(request) + engine.dispose() + + +class TestAllPluginsCapabilities: + """Test that all plugins implement their declared capabilities.""" + + def test_voice_list_capability(self, loaded_plugin, host_context, engine_config): + """Verify plugins with 'voice_list' capability implement VoiceLister.""" + manifest = loaded_plugin.manifest + + if "voice_list" not in manifest.capabilities: + pytest.skip("Plugin does not declare 'voice_list' capability") + + engine = loaded_plugin.create_engine(host_context, None, engine_config) + + # Check if listVoices method exists + assert hasattr(engine, "listVoices"), \ + "Plugin with 'voice_list' capability must have listVoices method" + + # listVoices should be callable + assert callable(engine.listVoices), "listVoices must be callable" + + # listVoices should return a list/tuple of voices + # Get first voice source from manifest + if manifest.engine.voiceSources: + source_id = manifest.engine.voiceSources[0].id + voices = engine.listVoices(source_id) + + assert isinstance(voices, (list, tuple)), \ + f"listVoices must return list or tuple, got {type(voices).__name__}" + + # Each voice should have id, name, tags + for voice in voices: + assert hasattr(voice, "id"), "Voice must have 'id' attribute" + assert hasattr(voice, "name"), "Voice must have 'name' attribute" + assert hasattr(voice, "tags"), "Voice must have 'tags' attribute" + assert isinstance(voice.tags, tuple), "Voice tags must be a tuple" + + engine.dispose() + + def test_known_capabilities_are_valid(self, loaded_plugin): + """Verify all declared capabilities are known.""" + manifest = loaded_plugin.manifest + + known_capabilities = frozenset({ + "voice_list", + "preview", + "voice_clone", + "voice_blend", + "streaming", + "cancel", + }) + + for cap in manifest.capabilities: + assert cap in known_capabilities, \ + f"Unknown capability: '{cap}'. Known capabilities: {known_capabilities}" \ No newline at end of file