mirror of
https://github.com/denizsafak/abogen.git
synced 2026-11-19 01:00:54 +01:00
Merge pull request #225 from denniswebb/publish/fix-alphanumeric-footnotes
fix(normalization): preserve alphanumeric terms during footnote cleanup
This commit is contained in:
2 files changed
+49
-8
No files matched your search
@@ -153,7 +153,6 @@ _CURRENCY_RE = re.compile(
|
||||
_URL_RE = re.compile(
|
||||
r"(https?://)?(www\.)?(?P<domain>[a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)+)(/[^\s]*)?"
|
||||
)
|
||||
_FOOTNOTE_RE = re.compile(r"([a-zA-Z]+)(\d+)")
|
||||
_BRACKET_FOOTNOTE_RE = re.compile(r"\[\d+\]")
|
||||
|
||||
_ISO_DATE_RE = re.compile(
|
||||
@@ -2002,16 +2001,13 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
|
||||
spoken = domain.replace(".", " dot ")
|
||||
return spoken
|
||||
|
||||
def _remove_footnote(match: re.Match[str]) -> str:
|
||||
return match.group(1)
|
||||
|
||||
normalized = text
|
||||
|
||||
# Apply URL replacement first (independent of number conversion).
|
||||
normalized = _URL_RE.sub(_replace_url, normalized)
|
||||
|
||||
if getattr(cfg, "remove_footnotes", False):
|
||||
normalized = _FOOTNOTE_RE.sub(_remove_footnote, normalized)
|
||||
# Unmarked letter-digit text is ambiguous (word1, F1, H2O).
|
||||
normalized = _BRACKET_FOOTNOTE_RE.sub("", normalized)
|
||||
|
||||
if cfg.convert_currency:
|
||||
|
||||
@@ -71,11 +71,10 @@ def test_roman_numerals_world_war():
|
||||
|
||||
|
||||
def test_footnote_removal():
|
||||
# Bob is awesome1. -> Bob is awesome.
|
||||
# A plain digit suffix cannot reliably be identified as a footnote.
|
||||
normalized = normalize("Bob is awesome1.")
|
||||
print(f"Bob is awesome1. -> {normalized}")
|
||||
assert "bob is awesome." in normalized.lower()
|
||||
assert "1" not in normalized
|
||||
assert normalized == "Bob is awesome1."
|
||||
|
||||
# Citation needed[1]. -> Citation needed.
|
||||
normalized = normalize("Citation needed[1].")
|
||||
@@ -84,6 +83,52 @@ def test_footnote_removal():
|
||||
assert "[1]" not in normalized
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["spacy", "off"])
|
||||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
"F1 FP1 Q1 Q2 Q3 CO2 H2O 3D MP3 x86.",
|
||||
"word1 awesome1 version2 item23.",
|
||||
"H2O2 B2B ISO9001 F1-car model2_rev3.",
|
||||
],
|
||||
)
|
||||
def test_footnote_cleanup_preserves_alphanumeric_terms(text, mode):
|
||||
assert normalize(text, {"normalization_apostrophe_mode": mode}) == text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["spacy", "off"])
|
||||
@pytest.mark.parametrize("convert_numbers", [True, False])
|
||||
def test_footnote_cleanup_removes_explicit_markers_only(mode, convert_numbers):
|
||||
assert normalize(
|
||||
"F1[1] and H2O[23][2]. Evidence[123].",
|
||||
{
|
||||
"normalization_apostrophe_mode": mode,
|
||||
"normalization_numbers": convert_numbers,
|
||||
},
|
||||
) == "F1 and H2O. Evidence."
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["spacy", "off"])
|
||||
def test_superscript_digits_are_normalized_without_guessing_footnotes(mode):
|
||||
# NFKC flattens superscripts before footnote cleanup sees the text.
|
||||
assert normalize(
|
||||
"Evidence¹ and Evidence²³.", {"normalization_apostrophe_mode": mode}
|
||||
) == "Evidence1 and Evidence23."
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["spacy", "off"])
|
||||
def test_footnote_cleanup_disabled_preserves_bracket_markers(mode):
|
||||
text = "F1[1] and H2O[23][2]."
|
||||
assert normalize(
|
||||
text,
|
||||
{
|
||||
"normalization_apostrophe_mode": mode,
|
||||
"normalization_footnotes": False,
|
||||
"normalization_numbers": False,
|
||||
},
|
||||
) == "F1[1] and H2O[23] [2]."
|
||||
|
||||
|
||||
def test_manual_override_normalization():
|
||||
from abogen.entity_analysis import normalize_manual_override_token
|
||||
|
||||
|
||||
Reference in new issue
Block a user