spaCy/spacy/training/align.pyx

import re
from itertools import chain
from typing import List, Tuple

from ..errors import Errors


def get_alignments(A: List[str], B: List[str]) -> Tuple[List[List[int]], List[List[int]]]:
    # Create character-to-token mappings
    char_to_token_a = tuple(chain(*((i,) * len(x.lower()) for i, x in enumerate(A))))
    char_to_token_b = tuple(chain(*((i,) * len(x.lower()) for i, x in enumerate(B))))
    str_a = "".join(A).lower()
    str_b = "".join(B).lower()
    cdef int len_str_a = len(str_a)
    cdef int len_str_b = len(str_b)
    # Check that the two texts only differ in whitespace and capitalization
    if re.sub(r"\s+", "", str_a) != re.sub(r"\s+", "", str_b) or \
            len_str_a != len(char_to_token_a) or \
            len_str_b != len(char_to_token_b):
        raise ValueError(Errors.E949.format(x=str(A[:10]), y=str(B[:10])))
    cdef int char_idx_a = 0
    cdef int char_idx_b = 0
    cdef int token_idx_a = 0
    cdef int token_idx_b = 0
    cdef int prev_token_idx_a = -1
    cdef int prev_token_idx_b = -1
    a2b = []
    b2a = []
    while char_idx_a < len_str_a and char_idx_b < len_str_b:
        # Find the current token position from the character position
        token_idx_a = char_to_token_a[char_idx_a]
        token_idx_b = char_to_token_b[char_idx_b]
        # Add a set for the next token if a token boundary has been crossed
        if prev_token_idx_a != token_idx_a:
            a2b.append(set())
        if prev_token_idx_b != token_idx_b:
            b2a.append(set())
        # Process the alignment at the current position
        if A[token_idx_a] == B[token_idx_b] and \
                (
                    char_idx_a == 0 or
                    char_to_token_a[char_idx_a - 1] < token_idx_a
                ) and \
                (
                    char_idx_b == 0 or
                    char_to_token_b[char_idx_b - 1] < token_idx_b
                ):
            # Current tokens are identical and both character offsets are the
            # start of a token (either at the beginning of the document or the
            # previous character belongs to a different token)
            a2b[-1].add(token_idx_b)
            b2a[-1].add(token_idx_a)
            char_idx_a += len(A[token_idx_a])
            char_idx_b += len(B[token_idx_b])
        elif str_a[char_idx_a] == str_b[char_idx_b]:
            # Current chars are identical
            a2b[-1].add(token_idx_b)
            b2a[-1].add(token_idx_a)
            char_idx_a += 1
            char_idx_b += 1
        elif str_a[char_idx_a].isspace():
            # Skip unaligned whitespace char in A
            char_idx_a += 1
        elif str_b[char_idx_b].isspace():
            # Skip unaligned whitespace char in B
            char_idx_b += 1
        else:
            # This should never happen
            raise ValueError(Errors.E949.format(x=str(A[:10]), y=str(B[:10])))
        prev_token_idx_a = token_idx_a
        prev_token_idx_b = token_idx_b
    # Process unaligned trailing whitespace
    a2b.extend([set()] * len(set(char_to_token_a[char_idx_a:])))
    b2a.extend([set()] * len(set(char_to_token_b[char_idx_b:])))
    # Return values as sorted lists per token position
    return [sorted(x) for x in a2b], [sorted(x) for x in b2a]