wuenlp_tools.models.segmentation 

 1from wuenlp_tools.models.segmentation.equal_word import (
 2    DEFAULT_NUM_EQUAL_WORD_SEGMENTS,
 3    EQUAL_SIZE_CHUNK_TYPE,
 4    EqualWordSegmentAnnotator,
 5    annotate_equal_word_segments,
 6    is_equal_size_chunk,
 7)
 8
 9__all__ = [
10    "DEFAULT_NUM_EQUAL_WORD_SEGMENTS",
11    "EQUAL_SIZE_CHUNK_TYPE",
12    "EqualWordSegmentAnnotator",
13    "annotate_equal_word_segments",
14    "is_equal_size_chunk",
15]
DEFAULT_NUM_EQUAL_WORD_SEGMENTS = 75

int([x]) -> integer int(x, base=10) -> integer

Convert a number or string to an integer, or return 0 if no arguments are given. If x is a number, return x.__int__(). For floating-point numbers, this truncates towards zero.

If x is not a number or if base is given, then x must be a string, bytes, or bytearray instance representing an integer literal in the given base. The literal can be preceded by '+' or '-' and be surrounded by whitespace. The base defaults to 10. Valid bases are 0 and 2-36. Base 0 means to interpret the base from the string as an integer literal.

>>> int('0b100', base=0)
4
EQUAL_SIZE_CHUNK_TYPE = 'equal_size_chunk'

str(object='') -> str str(bytes_or_buffer[, encoding[, errors]]) -> str

Create a new string object from the given object. If encoding or errors is specified, then the object must expose a data buffer that will be decoded using the given encoding and error handler. Otherwise, returns the result of object.__str__() (if defined) or repr(object). encoding defaults to sys.getdefaultencoding(). errors defaults to 'strict'.

EqualWordSegmentAnnotator = PipelineStep('Equal Word Segment Annotator', processor=EqualWordSegmentAnnotatorProcessor)

Pipeline step Equal Word Segment Annotator (EqualWordSegmentAnnotatorProcessor).

unit type UIMAChunk.

def annotate_equal_word_segments( doc: wuenlp.impl.uima.UIMANLPStructs.UIMADocument, *, num_segments: int = 75, overwrite: bool = False) -> wuenlp.impl.uima.UIMANLPStructs.UIMADocument: 
46def annotate_equal_word_segments(
47    doc: UIMADocument,
48    *,
49    num_segments: int = DEFAULT_NUM_EQUAL_WORD_SEGMENTS,
50    overwrite: bool = False,
51) -> UIMADocument:
52    """Create ``num_segments`` equal word-count bins as ``UIMAChunk`` annotations.
53
54    Uses ``chunk_type=EQUAL_SIZE_CHUNK_TYPE``. Existing chunks of other types are
55    left untouched. On overwrite, only equal-size chunks are removed and recreated.
56    """
57    if num_segments < 1:
58        raise ValueError(f"num_segments must be >= 1, got {num_segments}")
59
60    existing = _equal_size_chunks(doc)
61    if existing and not overwrite:
62        logger.info(f"Equal-size word chunks already present for {doc.path}. Skipping.")
63        return doc
64
65    for chunk in existing:
66        doc.remove_annotation(chunk)
67
68    tokens = _sorted_tokens(doc)
69    if not tokens:
70        raise ValueError(f"Cannot create equal word segments: document has no tokens ({doc.path})")
71
72    buckets = _bucket_tokens(tokens, num_segments)
73    anchor = tokens[0].begin
74    for segment_index, bucket in enumerate(buckets):
75        begin, end = _span_offsets(bucket, anchor)
76        anchor = end
77        chunk = doc.create_chunk(begin, end, chunk_type=EQUAL_SIZE_CHUNK_TYPE, add_to_document=True)
78        chunk.additional_features[SEGMENT_INDEX_FEATURE] = segment_index
79
80    return doc

Create num_segments equal word-count bins as UIMAChunk annotations.

Uses chunk_type=EQUAL_SIZE_CHUNK_TYPE. Existing chunks of other types are left untouched. On overwrite, only equal-size chunks are removed and recreated.

def is_equal_size_chunk(unit: wuenlp.impl.uima.UIMANLPStructs.UIMASpan) -> bool: 
19def is_equal_size_chunk(unit: UIMASpan) -> bool:
20    chunk_type = getattr(unit, "chunk_type", None)
21    return chunk_type == EQUAL_SIZE_CHUNK_TYPE