Add ICU sentence segmentation to OffsetsTextInfo

source/textInfos/offsets.py

@@ -16,8 +16,10 @@
 import locationHelper
 from treeInterceptorHandler import TreeInterceptor
 import textUtils
+from textUtils import icu
 from textUtils.segFlag import CharSegFlag, WordSegFlag
 from textUtils._wordSeg.wordSegmenter import WordSegmenter
+from winBindings.icu import ICU_AVAILABLE
 from dataclasses import dataclass
 from typing import (
 	Any,
@@ -556,13 +558,37 @@ def _getLineOffsets(self, offset):
 		return [start, end]
 
 	def _getSentenceOffsets(self, offset: int) -> tuple[int, int]:
-		"""
-		Gets the start and end offsets of the sentence containing the given offset.
+		"""Gets the start and end offsets of the sentence containing the given offset.
+
+		Sentences are segmented over the containing paragraph (:meth:`_getParagraphOffsets`)
+		using the Windows built-in ICU BreakIterator (UAX#29), so a sentence can span
+		multiple lines.
+
 		:param offset: The offset of the character within the sentence.
 		:return: A tuple of the start and end offsets of the sentence.
-		:raise NotImplementedError: If the method is not implemented.
+		:raises NotImplementedError: If ICU is unavailable (Windows older than version 1703),
+		    the encoding is neither UTF-16 nor one whose offsets are str indices,
+		    or the ICU call fails.
 		"""
-		raise NotImplementedError
+		if not ICU_AVAILABLE:
+			raise NotImplementedError
+		if self.encoding is not None and self.encoding not in (
+			textUtils.WCHAR_ENCODING,
+			textUtils.USER_ANSI_CODE_PAGE,
+			"utf_32_le",
+		):
+			raise NotImplementedError
+		paragraphStart, paragraphEnd = self._getParagraphOffsets(offset)
+		paragraphText = self._getTextRange(paragraphStart, paragraphEnd)
+		result = icu.calculateOffsetsForEncoding(
+			icu.calculateSentenceOffsets,
+			paragraphText,
+			offset - paragraphStart,
+			self.encoding,
+		)
+		if result is None:
+			raise NotImplementedError
+		return (result[0] + paragraphStart, result[1] + paragraphStart)
 
 	def _getParagraphOffsets(self, offset):
 		return self._getLineOffsets(offset)

source/textUtils/icu.py

@@ -17,8 +17,11 @@
 from logHandler import log
 
 _ROOT_LOCALE: bytes = b""
-"""ICU root locale. Word and character segmentation are script-driven, not
-locale-driven (see calculateWordOffsets), so the root locale is always used.
+"""ICU root locale.
+
+Word, character and sentence segmentation are script-driven, not locale-driven
+(see :func:`calculateWordOffsets` and :func:`calculateSentenceOffsets`),
+so the root locale is always used.
 """
 
 
@@ -93,7 +96,7 @@ def calculateWordOffsets(
 	Uniscribe/Notepad behaviour for mixed whitespace runs is itself inconsistent.
 
 	:param text: The line text as a Python str.
-	:param offset: UTF-16 code unit offset within text at which to find the boundary.
+	:param offset: UTF-16 code unit offset within text at which to find the boundaries.
 	:return: (startOffset, endOffset) as UTF-16 code unit indices (endOffset exclusive),
 	    or None if the ICU call failed.
 	"""
@@ -147,6 +150,37 @@ def calculateWordOffsets(
 		return None
 
 
+def calculateSentenceOffsets(
+	text: str,
+	offset: int,
+) -> tuple[int, int] | None:
+	"""Calculate the UTF-16 start and end offsets of the sentence at the given offset.
+
+	Sentence boundaries follow Unicode Standard Annex #29 default rules, driven by the
+	language-neutral Sentence_Break property (STerm/ATerm terminators such as ".", "!",
+	"?" and the ideographic full stop "。"), so the root locale is used.  Abbreviation
+	tailoring, which would keep "Dr." from ending a sentence, is locale-specific and is
+	therefore not applied.  Trailing whitespace and punctuation are attached to the
+	preceding sentence.
+
+	:param text: The paragraph text as a Python str.
+	:param offset: UTF-16 code unit offset within text at which to find the boundaries.
+	:return: (startOffset, endOffset) as UTF-16 code unit indices (endOffset exclusive),
+	    or None if the ICU call failed.
+	"""
+	buf = ctypes.create_unicode_buffer(text)
+	textLength = len(buf) - 1
+	if offset >= textLength:
+		return (offset, offset + 1)
+
+	try:
+		with _breakIterator(icu.UBRK.SENTENCE, _ROOT_LOCALE, buf) as bi:
+			return _containingSegment(bi, offset, textLength)
+	except RuntimeError:
+		log.debugWarning("ICU sentence break iterator failed", exc_info=True)
+		return None
+
+
 def calculateOffsetsForEncoding(
 	calculate: Callable[[str, int], tuple[int, int] | None],
 	text: str,

source/winBindings/icu.py

@@ -52,6 +52,9 @@ class UBRK(IntEnum):
 	WORD = 1
 	"""Word breaks."""
 
+	SENTENCE = 3
+	"""Sentence breaks."""
+
 
 def U_FAILURE(code: int) -> bool:
 	"""Return True if the given UErrorCode indicates an error."""

tests/unit/test_textUtils/test_sentenceSegIcu.py

@@ -0,0 +1,166 @@
+# A part of NonVisual Desktop Access (NVDA)
+# Copyright (C) 2026 NV Access Limited, Leonard de Ruijter
+# This file may be used under the terms of the GNU General Public License, version 2 or later, as modified by the NVDA license.
+# For full terms and any additional permissions, see the NVDA license file: https://github.com/nvaccess/nvda/blob/master/copying.txt
+
+"""Unit tests for ICU sentence segmentation.
+
+Covers the low-level ``textUtils.icu.calculateSentenceOffsets`` primitive and the
+``OffsetsTextInfo._getSentenceOffsets`` integration, including the iteration/tiling
+invariant that ``move``/``expand`` rely on.  Tests that require ICU are skipped when
+the ICU library is not present on the system.
+"""
+
+import unittest
+from unittest.mock import patch
+
+import textInfos
+from textInfos import offsets as offsetsModule
+from textInfos.offsets import Offsets
+from textUtils import icu
+
+from ..textProvider import BasicTextInfo, BasicTextProvider
+from . import skipIfNoICU
+
+
+class _BlockParagraphTextInfo(BasicTextInfo):
+	"""A TextInfo whose paragraphs are blocks separated by a blank line ("\\n\\n").
+
+	Mimics the block-level bounds VirtualBufferTextInfo._getParagraphOffsets returns,
+	rather than the line splitting BasicTextInfo does.  Test text is ASCII, so str
+	offsets equal UTF-16 offsets.
+	"""
+
+	def _getParagraphOffsets(self, offset):
+		text = self._getStoryText()
+		start = text.rfind("\n\n", 0, offset)
+		start = 0 if start < 0 else start + 2
+		end = text.find("\n\n", offset)
+		end = len(text) if end < 0 else end + 2
+		return (start, end)
+
+
+class _BlockParagraphProvider(BasicTextProvider):
+	TextInfo = _BlockParagraphTextInfo
+
+
+@skipIfNoICU
+class TestCalculateSentenceOffsets(unittest.TestCase):
+	"""Low-level UAX#29 sentence boundary tests (UTF-16 code-unit offsets, root locale)."""
+
+	def test_english_two_sentences(self):
+		"""Trailing space after the terminator is attached to the preceding sentence (UAX#29)."""
+		text = "Hello world. Goodbye now."
+		self.assertEqual(icu.calculateSentenceOffsets(text, 0), (0, 13))
+		# Mid-sentence offsets resolve to the same sentence.
+		self.assertEqual(icu.calculateSentenceOffsets(text, 5), (0, 13))
+		self.assertEqual(icu.calculateSentenceOffsets(text, 12), (0, 13))
+		self.assertEqual(icu.calculateSentenceOffsets(text, 13), (13, 25))
+
+	def test_japanese_ideographic_full_stop(self):
+		"""U+3002 (。) is Sentence_Break=STerm, so Japanese segments without a locale."""
+		text = "これは日本語です。次の文です。"
+		self.assertEqual(icu.calculateSentenceOffsets(text, 0), (0, 9))
+		self.assertEqual(icu.calculateSentenceOffsets(text, 9), (9, 15))
+
+	def test_abbreviation_splits_under_root_locale(self):
+		"""Under the root locale, which has no abbreviation tailoring, "Dr." ends a sentence."""
+		text = "Dr. Smith went home."
+		self.assertEqual(icu.calculateSentenceOffsets(text, 0), (0, 4))
+		self.assertEqual(icu.calculateSentenceOffsets(text, 4), (4, 20))
+
+	def test_surrogate_pair_offsets(self):
+		"""Offsets are UTF-16 code-unit indexed; a surrogate pair counts as two units."""
+		# U+1F926 (🤦) is greater than 0xFFFF, so is encoded as the surrogate pair 0xd83e,dd26.
+		text = "Hi \U0001f926 there. Bye now."
+		# First sentence spans the surrogate pair and ends after the space at UTF-16 offset 13.
+		self.assertEqual(icu.calculateSentenceOffsets(text, 0), (0, 13))
+		# An offset inside the surrogate pair still resolves to the containing sentence.
+		self.assertEqual(icu.calculateSentenceOffsets(text, 4), (0, 13))
+		self.assertEqual(icu.calculateSentenceOffsets(text, 13), (13, 21))
+
+	def test_offset_past_end_fast_path(self):
+		"""An offset at/past the end returns a single-unit span (matches word behaviour)."""
+		self.assertEqual(icu.calculateSentenceOffsets("abc", 5), (5, 6))
+
+	def test_offset_containment(self):
+		"""For every in-range offset, start <= offset < end (the tiling precondition)."""
+		text = "One. Two! Three? Four."
+		length = len(text.encode("utf-16-le")) // 2
+		for offset in range(length):
+			start, end = icu.calculateSentenceOffsets(text, offset)
+			self.assertTrue(
+				start <= offset < end,
+				f"offset {offset} not contained in ({start}, {end}) for {text!r}",
+			)
+
+
+@skipIfNoICU
+class TestSentenceIterationTiling(unittest.TestCase):
+	"""The iteration invariant: move(UNIT_SENTENCE) walks sentences gap-free without stalling."""
+
+	def _collectSentences(self, obj, length: int, direction: int) -> list[tuple[int, int]]:
+		"""Enumerate sentence spans by expanding then moving in ``direction`` until the walk stops.
+
+		:param obj: A text provider to navigate.
+		:param length: UTF-16 length of the story text (bounds the loop and seeds the reverse walk).
+		:param direction: 1 to walk forward from the start, -1 to walk backward from the end.
+		:return: Sentence spans in document order (the backward walk is reversed before returning).
+		"""
+		startPos = 0 if direction > 0 else max(0, length - 1)
+		info = obj.makeTextInfo(Offsets(startPos, startPos))
+		info.expand(textInfos.UNIT_SENTENCE)
+		spans = []
+		# The loop is bounded so that a stalled walk fails rather than hangs.
+		for _ in range(length + 2):
+			spans.append((info._startOffset, info._endOffset))
+			if info.move(textInfos.UNIT_SENTENCE, direction) == 0:
+				break
+			info.expand(textInfos.UNIT_SENTENCE)
+		if direction < 0:
+			spans.reverse()
+		return spans
+
+	def _assertTiles(self, spans: list[tuple[int, int]], length: int):
+		"""Assert the spans tile [0, length) gap-free, in order, with no overlaps."""
+		self.assertEqual(spans[0][0], 0, f"first sentence does not start at 0: {spans}")
+		self.assertEqual(spans[-1][1], length, f"last sentence does not reach {length}: {spans}")
+		for (_, prevEnd), (nextStart, _) in zip(spans, spans[1:]):
+			self.assertEqual(prevEnd, nextStart, f"gap/overlap between sentences: {spans}")
+
+	def test_single_paragraph_tiles_both_directions(self):
+		text = "Hello world. Goodbye now. The third one."
+		length = len(text)  # all-ASCII, so str length == UTF-16 length
+		obj = BasicTextProvider(text=text)
+		forward = self._collectSentences(obj, length, 1)
+		self._assertTiles(forward, length)
+		# Three terminators => three sentences.
+		self.assertEqual(len(forward), 3)
+		# Backward navigation (alt+upArrow) visits the same sentences in the same order.
+		backward = self._collectSentences(obj, length, -1)
+		self.assertEqual(backward, forward)
+
+	def test_crosses_block_paragraph_boundary_both_directions(self):
+		"""With block-paragraph semantics, the walk crosses the blank-line boundary."""
+		text = "One. Two.\n\nThree. Four."
+		length = len(text)
+		blockStart = text.index("\n\n") + 2  # start of the second block paragraph
+		obj = _BlockParagraphProvider(text=text)
+		forward = self._collectSentences(obj, length, 1)
+		self._assertTiles(forward, length)
+		# A sentence boundary lands exactly on the second paragraph's start (the walk crossed).
+		self.assertIn(blockStart, [start for start, _ in forward])
+		backward = self._collectSentences(obj, length, -1)
+		self.assertEqual(backward, forward)
+		self.assertIn(blockStart, [start for start, _ in backward])
+
+
+class TestSentenceOffsetsWithoutIcu(unittest.TestCase):
+	"""When ICU is unavailable, _getSentenceOffsets degrades to NotImplementedError."""
+
+	def test_not_implemented_when_icu_unavailable(self):
+		obj = BasicTextProvider(text="Hello world. Goodbye now.")
+		info = obj.makeTextInfo(Offsets(0, 0))
+		with patch.object(offsetsModule, "ICU_AVAILABLE", False):
+			with self.assertRaises(NotImplementedError):
+				info._getSentenceOffsets(0)

user_docs/en/changes.md

@@ -139,6 +139,9 @@ Locale names are now taken from `winKernel.LCIDToLocaleName`, apart from a small
   * As a result, some locale identifiers now resolve to a different name, such as `zh_CN` rather than `zh_CHS`, `km_KH` rather than `kh_KH` and `en_JM` rather than `en_JA`.
   * Locale names now carry a script subtag where Windows reports one, such as `sr_LATN_CS` rather than `sr_SP` for LCID 2074.
   * The SAPI 4 and SAPI 5 synthesizers report voice languages through this function as well, so the language of a voice can now be reported for locale identifiers that `locale.windows_locale` did not cover.
+* `OffsetsTextInfo` now implements `_getSentenceOffsets` using the Windows built-in ICU library.
+This adds support for `textInfos.UNIT_SENTENCE` to all `TextInfo` implementations based on `OffsetsTextInfo`. (#20603, @LeonarddeR)
+  * For unsupported encodings, or when ICU is unavailable (on Windows versions older than 1703), `_getSentenceOffsets` continues to raise `NotImplementedError`.
 * `louisHelper` is now the only module that performs braille translation. (#20600, @LeonarddeR)
   * Added `louisHelper.TranslationMode` and `louisHelper.Typeform`, holding the translation modes and typeforms NVDA uses.
   `braille.Region.rawTextTypeforms` is now annotated as `list[louisHelper.Typeform]`.