Fixed incorrect sentence segmentation when using spaCy

This commit is contained in:
Deniz Şafak
2025-12-10 03:20:59 +03:00
parent 174ff4232f
commit 968c673e58
2 changed files with 13 additions and 6 deletions
+1
View File
@@ -4,6 +4,7 @@
- Fixed the `No module named pip` error that occurred for users who installed Abogen via the [**uv**](https://github.com/astral-sh/uv) installer.
- Fixed defaults for `replace_single_newlines` not being applied correctly in some cases.
- Fixed `Save chapters separately for queued epubs is ignored`, issue mentioned by @dymas-cz in #109.
- Fixed incorrect sentence segmentation when using spaCy where text would erroneously split after opening parentheses.
- Improvements in code and documentation.
# 1.2.4
+12 -6
View File
@@ -77,13 +77,18 @@ def get_spacy_model(lang_code, log_callback=None):
# Try to load the model
try:
log(f"\nLoading spaCy model '{model_name}'...")
# sentence segmentation involving parentheses, quotes, and complex structure.
# We only disable heavier components we don't need like NER.
nlp = spacy.load(
model_name,
disable=["ner", "parser", "tagger", "lemmatizer", "attribute_ruler"],
disable=["ner", "tagger", "lemmatizer", "attribute_ruler"],
)
# Enable sentence segmentation only
if "sentencizer" not in nlp.pipe_names:
# Ensure a sentence segmentation strategy is in place
# The parser provides sents, but if it's missing (unlikely for core models), fallback to sentencizer
if "parser" not in nlp.pipe_names and "sentencizer" not in nlp.pipe_names:
nlp.add_pipe("sentencizer")
_nlp_cache[lang_code] = nlp
return nlp
except OSError:
@@ -93,13 +98,14 @@ def get_spacy_model(lang_code, log_callback=None):
from spacy.cli import download
download(model_name)
# Retry loading
# Retry loading with the same fix
nlp = spacy.load(
model_name,
disable=["ner", "parser", "tagger", "lemmatizer", "attribute_ruler"],
disable=["ner", "tagger", "lemmatizer", "attribute_ruler"],
)
if "sentencizer" not in nlp.pipe_names:
if "parser" not in nlp.pipe_names and "sentencizer" not in nlp.pipe_names:
nlp.add_pipe("sentencizer")
_nlp_cache[lang_code] = nlp
log(f"spaCy model '{model_name}' downloaded and loaded")
return nlp