mirror of
https://github.com/denizsafak/abogen.git
synced 2026-07-18 13:40:27 +02:00
Fixed incorrect sentence segmentation when using spaCy
This commit is contained in:
@@ -4,6 +4,7 @@
|
|||||||
- Fixed the `No module named pip` error that occurred for users who installed Abogen via the [**uv**](https://github.com/astral-sh/uv) installer.
|
- Fixed the `No module named pip` error that occurred for users who installed Abogen via the [**uv**](https://github.com/astral-sh/uv) installer.
|
||||||
- Fixed defaults for `replace_single_newlines` not being applied correctly in some cases.
|
- Fixed defaults for `replace_single_newlines` not being applied correctly in some cases.
|
||||||
- Fixed `Save chapters separately for queued epubs is ignored`, issue mentioned by @dymas-cz in #109.
|
- Fixed `Save chapters separately for queued epubs is ignored`, issue mentioned by @dymas-cz in #109.
|
||||||
|
- Fixed incorrect sentence segmentation when using spaCy where text would erroneously split after opening parentheses.
|
||||||
- Improvements in code and documentation.
|
- Improvements in code and documentation.
|
||||||
|
|
||||||
# 1.2.4
|
# 1.2.4
|
||||||
|
|||||||
+12
-6
@@ -77,13 +77,18 @@ def get_spacy_model(lang_code, log_callback=None):
|
|||||||
# Try to load the model
|
# Try to load the model
|
||||||
try:
|
try:
|
||||||
log(f"\nLoading spaCy model '{model_name}'...")
|
log(f"\nLoading spaCy model '{model_name}'...")
|
||||||
|
# sentence segmentation involving parentheses, quotes, and complex structure.
|
||||||
|
# We only disable heavier components we don't need like NER.
|
||||||
nlp = spacy.load(
|
nlp = spacy.load(
|
||||||
model_name,
|
model_name,
|
||||||
disable=["ner", "parser", "tagger", "lemmatizer", "attribute_ruler"],
|
disable=["ner", "tagger", "lemmatizer", "attribute_ruler"],
|
||||||
)
|
)
|
||||||
# Enable sentence segmentation only
|
|
||||||
if "sentencizer" not in nlp.pipe_names:
|
# Ensure a sentence segmentation strategy is in place
|
||||||
|
# The parser provides sents, but if it's missing (unlikely for core models), fallback to sentencizer
|
||||||
|
if "parser" not in nlp.pipe_names and "sentencizer" not in nlp.pipe_names:
|
||||||
nlp.add_pipe("sentencizer")
|
nlp.add_pipe("sentencizer")
|
||||||
|
|
||||||
_nlp_cache[lang_code] = nlp
|
_nlp_cache[lang_code] = nlp
|
||||||
return nlp
|
return nlp
|
||||||
except OSError:
|
except OSError:
|
||||||
@@ -93,13 +98,14 @@ def get_spacy_model(lang_code, log_callback=None):
|
|||||||
from spacy.cli import download
|
from spacy.cli import download
|
||||||
|
|
||||||
download(model_name)
|
download(model_name)
|
||||||
# Retry loading
|
# Retry loading with the same fix
|
||||||
nlp = spacy.load(
|
nlp = spacy.load(
|
||||||
model_name,
|
model_name,
|
||||||
disable=["ner", "parser", "tagger", "lemmatizer", "attribute_ruler"],
|
disable=["ner", "tagger", "lemmatizer", "attribute_ruler"],
|
||||||
)
|
)
|
||||||
if "sentencizer" not in nlp.pipe_names:
|
if "parser" not in nlp.pipe_names and "sentencizer" not in nlp.pipe_names:
|
||||||
nlp.add_pipe("sentencizer")
|
nlp.add_pipe("sentencizer")
|
||||||
|
|
||||||
_nlp_cache[lang_code] = nlp
|
_nlp_cache[lang_code] = nlp
|
||||||
log(f"spaCy model '{model_name}' downloaded and loaded")
|
log(f"spaCy model '{model_name}' downloaded and loaded")
|
||||||
return nlp
|
return nlp
|
||||||
|
|||||||
Reference in New Issue
Block a user