Merge pull request #225 from denniswebb/publish/fix-alphanumeric-footnotes

fix(normalization): preserve alphanumeric terms during footnote cleanup
This commit is contained in:
Deniz Şafak authored and GitHub committed 2026-11-01 20:25:07 +03:00
commit 1807822f58
2 files changed
+49 -8

No files matched your search

+1 -5
View File
@@ -153,7 +153,6 @@ _CURRENCY_RE = re.compile(
_URL_RE = re.compile(
r"(https?://)?(www\.)?(?P<domain>[a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)+)(/[^\s]*)?"
)
_FOOTNOTE_RE = re.compile(r"([a-zA-Z]+)(\d+)")
_BRACKET_FOOTNOTE_RE = re.compile(r"\[\d+\]")
_ISO_DATE_RE = re.compile(
@@ -2002,16 +2001,13 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
spoken = domain.replace(".", " dot ")
return spoken
def _remove_footnote(match: re.Match[str]) -> str:
return match.group(1)
normalized = text
# Apply URL replacement first (independent of number conversion).
normalized = _URL_RE.sub(_replace_url, normalized)
if getattr(cfg, "remove_footnotes", False):
normalized = _FOOTNOTE_RE.sub(_remove_footnote, normalized)
# Unmarked letter-digit text is ambiguous (word1, F1, H2O).
normalized = _BRACKET_FOOTNOTE_RE.sub("", normalized)
if cfg.convert_currency:
+48 -3
View File
@@ -71,11 +71,10 @@ def test_roman_numerals_world_war():
def test_footnote_removal():
# Bob is awesome1. -> Bob is awesome.
# A plain digit suffix cannot reliably be identified as a footnote.
normalized = normalize("Bob is awesome1.")
print(f"Bob is awesome1. -> {normalized}")
assert "bob is awesome." in normalized.lower()
assert "1" not in normalized
assert normalized == "Bob is awesome1."
# Citation needed[1]. -> Citation needed.
normalized = normalize("Citation needed[1].")
@@ -84,6 +83,52 @@ def test_footnote_removal():
assert "[1]" not in normalized
@pytest.mark.parametrize("mode", ["spacy", "off"])
@pytest.mark.parametrize(
"text",
[
"F1 FP1 Q1 Q2 Q3 CO2 H2O 3D MP3 x86.",
"word1 awesome1 version2 item23.",
"H2O2 B2B ISO9001 F1-car model2_rev3.",
],
)
def test_footnote_cleanup_preserves_alphanumeric_terms(text, mode):
assert normalize(text, {"normalization_apostrophe_mode": mode}) == text
@pytest.mark.parametrize("mode", ["spacy", "off"])
@pytest.mark.parametrize("convert_numbers", [True, False])
def test_footnote_cleanup_removes_explicit_markers_only(mode, convert_numbers):
assert normalize(
"F1[1] and H2O[23][2]. Evidence[123].",
{
"normalization_apostrophe_mode": mode,
"normalization_numbers": convert_numbers,
},
) == "F1 and H2O. Evidence."
@pytest.mark.parametrize("mode", ["spacy", "off"])
def test_superscript_digits_are_normalized_without_guessing_footnotes(mode):
# NFKC flattens superscripts before footnote cleanup sees the text.
assert normalize(
"Evidence¹ and Evidence²³.", {"normalization_apostrophe_mode": mode}
) == "Evidence1 and Evidence23."
@pytest.mark.parametrize("mode", ["spacy", "off"])
def test_footnote_cleanup_disabled_preserves_bracket_markers(mode):
text = "F1[1] and H2O[23][2]."
assert normalize(
text,
{
"normalization_apostrophe_mode": mode,
"normalization_footnotes": False,
"normalization_numbers": False,
},
) == "F1[1] and H2O[23] [2]."
def test_manual_override_normalization():
from abogen.entity_analysis import normalize_manual_override_token