wuenlp_tools.models.sentiment.SentimentLexiconBaseline 

  1from typing import Iterable, Type
  2
  3import numpy as np
  4from loguru import logger
  5
  6from wuenlp.impl.UIMANLPStructs import UIMASpan, UIMAToken, UIMADocument, UIMASystemScene
  7from wuenlp.impl.uima import UIMAChunk
  8
  9from wuenlp_tools.models.segmentation.equal_word import is_equal_size_chunk
 10from wuenlp_tools.pipeline import PipelineStep, PipelineProcessor, UnitFilter, get_units
 11from wuenlp_tools.utils.nrc import german_nrc
 12from wuenlp_tools.utils.sentiment import SentimentLexicon
 13
 14negation_words = {"nicht", "kein", "ohne", "nie", "nirgends", "niemand", "niemals",
 15                  "keiner"}  # Loosely based on http: // sentiment.christopherpotts.net / lingstruc.html
 16
 17punctuation = (",", ".", ":", ";", "!", "?")
 18
 19LEXICON_BASELINE_SENTIMENT = "lexicon_baseline_sentiment"
 20
 21# NRC basic emotions + positive/negative (2016 happy-end emotion trajectories).
 22BASIC_EMOTION_FEATURES: tuple[tuple[str, str], ...] = (
 23    ("anticipation", "lexicon_baseline_anticipation"),
 24    ("sadness", "lexicon_baseline_sadness"),
 25    ("fear", "lexicon_baseline_fear"),
 26    ("anger", "lexicon_baseline_anger"),
 27    ("disgust", "lexicon_baseline_disgust"),
 28    ("trust", "lexicon_baseline_trust"),
 29    ("surprise", "lexicon_baseline_surprise"),
 30    ("joy", "lexicon_baseline_joy"),
 31    ("positive", "lexicon_baseline_positive"),
 32    ("negative", "lexicon_baseline_negative"),
 33)
 34
 35
 36def lexicon_baseline_feature_names(*, include_basic_emotions: bool = False) -> list[str]:
 37    features = [LEXICON_BASELINE_SENTIMENT]
 38    if include_basic_emotions:
 39        features.extend(feature for _, feature in BASIC_EMOTION_FEATURES)
 40    return features
 41
 42
 43def get_novel_sentiment(
 44    doc: UIMADocument,
 45    lemma: bool,
 46    lexicon: SentimentLexicon,
 47    segments: Iterable[UIMASpan],
 48    use_negation: bool = True,
 49    cumsum: bool = False,
 50    include_basic_emotions: bool = False,
 51):
 52    """
 53    Annotate lexicon sentiment on the given spans and return the document.
 54
 55    :param doc: the novel to process
 56    :param lemma: does the lexicon use lemmatization?
 57    :param lexicon: the lexicon to use
 58    :param segments: spans to score, in CAS annotation-index order
 59    :param use_negation: if True, apply negation detection
 60    :param cumsum: if True, build features based on the cumulative of sentiments over the novel
 61    :param include_basic_emotions: if True, also write 10 NRC basic-emotion channels per span
 62    :return: the document with lexicon baseline features on each span
 63    """
 64    segments = list(segments)
 65    feature_names = lexicon_baseline_feature_names(include_basic_emotions=include_basic_emotions)
 66    for segment in segments:
 67        annotate_span_lexicon_features(
 68            segment,
 69            lexicon,
 70            lemma,
 71            include_basic_emotions=include_basic_emotions,
 72            use_negation=use_negation,
 73        )
 74    if cumsum:
 75        for feature_name in feature_names:
 76            for i in range(2, len(segments)):
 77                segments[i].additional_features[feature_name] += segments[i - 1].additional_features[feature_name]
 78
 79    return doc
 80
 81
 82def apply_negation_to_span(span: UIMASpan):
 83    negated = False
 84    words = list(span.tokens)
 85    for i, word in enumerate(words):
 86        if word.lemma in negation_words:
 87            negated = True
 88        elif word.text in punctuation:
 89            negated = False
 90        else:
 91            word.additional_features["sentiment_baseline:negated"] = negated
 92    return words
 93
 94
 95def _lexicon_word(token: UIMAToken, lemma: bool) -> str:
 96    return token.lemma if lemma else token.text
 97
 98
 99def _apply_negation_to_score(score: float) -> float:
100    """2016 happy-end rule: flip sign and halve."""
101    return -score / 2
102
103
104def get_sentiment_for_span(span: UIMASpan, lexicon: SentimentLexicon, lemma: bool) -> float:
105    scores = []
106    for token in span.tokens:
107        word = _lexicon_word(token, lemma)
108        row = lexicon.get_word_scores(word)
109        if row is None:
110            continue
111        score = row["sentiment"]
112        if token.additional_features.get("sentiment_baseline:negated", False):
113            score = _apply_negation_to_score(score)
114        scores.append(score)
115    return float(np.mean(scores)) if scores else 0.0
116
117
118def annotate_span_lexicon_features(
119    span: UIMASpan,
120    lexicon: SentimentLexicon,
121    lemma: bool,
122    *,
123    include_basic_emotions: bool = False,
124    use_negation: bool = True,
125) -> None:
126    sentiment_scores: list[float] = []
127    emotion_scores = (
128        {feature_name: [] for _, feature_name in BASIC_EMOTION_FEATURES}
129        if include_basic_emotions
130        else None
131    )
132
133    negated = False
134    for token in span.tokens:
135        in_negated_scope = False
136        if use_negation:
137            if token.lemma in negation_words:
138                negated = True
139            elif token.text in punctuation:
140                negated = False
141            else:
142                in_negated_scope = negated
143                token.additional_features["sentiment_baseline:negated"] = negated
144
145        word = _lexicon_word(token, lemma)
146        row = lexicon.get_word_scores(word)
147        if row is None:
148            continue
149
150        if use_negation and in_negated_scope:
151            sentiment = _apply_negation_to_score(row["sentiment"])
152        else:
153            sentiment = row["sentiment"]
154        sentiment_scores.append(sentiment)
155
156        if emotion_scores is not None:
157            for column, feature_name in BASIC_EMOTION_FEATURES:
158                value = row[column]
159                if use_negation and in_negated_scope:
160                    value = _apply_negation_to_score(value)
161                emotion_scores[feature_name].append(value)
162
163    span.additional_features[LEXICON_BASELINE_SENTIMENT] = (
164        float(np.mean(sentiment_scores)) if sentiment_scores else 0.0
165    )
166    if emotion_scores is not None:
167        for feature_name, values in emotion_scores.items():
168            span.additional_features[feature_name] = float(np.mean(values)) if values else 0.0
169
170
171def get_nrc_sentiment_for_span(
172    doc: UIMADocument,
173    unit_type: Type[UIMASpan],
174    unit_filter: UnitFilter | None = None,
175    include_basic_emotions: bool = False,
176):
177    segments = get_units(doc, unit_type, unit_filter)
178    return get_novel_sentiment(
179        doc,
180        lemma=True,
181        lexicon=german_nrc,
182        segments=segments,
183        use_negation=True,
184        include_basic_emotions=include_basic_emotions,
185    )
186
187
188class NRCSentimentProcessor(PipelineProcessor):
189    def __init__(self, *, include_basic_emotions: bool = False):
190        self.include_basic_emotions = include_basic_emotions
191
192    def __call__(
193        self,
194        doc: UIMADocument,
195        unit_type: Type[UIMASpan],
196        overwrite: bool = False,
197        unit_filter: UnitFilter | None = None,
198        **kwargs,
199    ):
200        segments = get_units(doc, unit_type, unit_filter)
201        expected = lexicon_baseline_feature_names(include_basic_emotions=self.include_basic_emotions)
202        if not overwrite and segments and all(
203            all(feature in segment.additional_features for feature in expected) for segment in segments
204        ):
205            logger.warning(f"Sentiment already calculated for {doc.path}. Skipping...")
206            return doc
207        return get_nrc_sentiment_for_span(
208            doc,
209            unit_type=unit_type,
210            unit_filter=unit_filter,
211            include_basic_emotions=self.include_basic_emotions,
212        )
213
214
215SentimentLexiconBaseline = PipelineStep(
216    "SentimentLexiconBaseline",
217    NRCSentimentProcessor(include_basic_emotions=False),
218    unit_type=UIMASystemScene,
219    added_additional_features=[LEXICON_BASELINE_SENTIMENT],
220)
221
222SentimentLexiconBaselineWithBasicEmotions = PipelineStep(
223    "SentimentLexiconBaseline (basic emotions)",
224    NRCSentimentProcessor(include_basic_emotions=True),
225    unit_type=UIMASystemScene,
226    added_additional_features=lexicon_baseline_feature_names(include_basic_emotions=True),
227)
228
229SentimentLexiconBaselineOnEqualWordChunks = PipelineStep(
230    "SentimentLexiconBaseline (equal word chunks)",
231    NRCSentimentProcessor(include_basic_emotions=False),
232    unit_type=UIMAChunk,
233    unit_filter=is_equal_size_chunk,
234    added_additional_features=[LEXICON_BASELINE_SENTIMENT],
235)
236
237SentimentLexiconBaselineOnEqualWordChunksWithBasicEmotions = PipelineStep(
238    "SentimentLexiconBaseline (equal word chunks, basic emotions)",
239    NRCSentimentProcessor(include_basic_emotions=True),
240    unit_type=UIMAChunk,
241    unit_filter=is_equal_size_chunk,
242    added_additional_features=lexicon_baseline_feature_names(include_basic_emotions=True),
243)
negation_words = {'nie', 'niemand', 'nicht', 'niemals', 'nirgends', 'kein', 'keiner', 'ohne'}

set() -> new empty set object set(iterable) -> new set object

Build an unordered collection of unique elements.

punctuation = (',', '.', ':', ';', '!', '?')

Built-in immutable sequence.

If no argument is given, the constructor returns an empty tuple. If iterable is specified the tuple is initialized from iterable's items.

If the argument is a tuple, the return value is the same object.

LEXICON_BASELINE_SENTIMENT = 'lexicon_baseline_sentiment'

str(object='') -> str str(bytes_or_buffer[, encoding[, errors]]) -> str

Create a new string object from the given object. If encoding or errors is specified, then the object must expose a data buffer that will be decoded using the given encoding and error handler. Otherwise, returns the result of object.__str__() (if defined) or repr(object). encoding defaults to sys.getdefaultencoding(). errors defaults to 'strict'.

BASIC_EMOTION_FEATURES: tuple[tuple[str, str], ...] = (('anticipation', 'lexicon_baseline_anticipation'), ('sadness', 'lexicon_baseline_sadness'), ('fear', 'lexicon_baseline_fear'), ('anger', 'lexicon_baseline_anger'), ('disgust', 'lexicon_baseline_disgust'), ('trust', 'lexicon_baseline_trust'), ('surprise', 'lexicon_baseline_surprise'), ('joy', 'lexicon_baseline_joy'), ('positive', 'lexicon_baseline_positive'), ('negative', 'lexicon_baseline_negative'))

Built-in immutable sequence.

If no argument is given, the constructor returns an empty tuple. If iterable is specified the tuple is initialized from iterable's items.

If the argument is a tuple, the return value is the same object.

def lexicon_baseline_feature_names(*, include_basic_emotions: bool = False) -> list[str]: 
37def lexicon_baseline_feature_names(*, include_basic_emotions: bool = False) -> list[str]:
38    features = [LEXICON_BASELINE_SENTIMENT]
39    if include_basic_emotions:
40        features.extend(feature for _, feature in BASIC_EMOTION_FEATURES)
41    return features
def get_novel_sentiment( doc: wuenlp.impl.uima.UIMANLPStructs.UIMADocument, lemma: bool, lexicon: wuenlp_tools.utils.sentiment.SentimentLexicon, segments: Iterable[wuenlp.impl.uima.UIMANLPStructs.UIMASpan], use_negation: bool = True, cumsum: bool = False, include_basic_emotions: bool = False): 
44def get_novel_sentiment(
45    doc: UIMADocument,
46    lemma: bool,
47    lexicon: SentimentLexicon,
48    segments: Iterable[UIMASpan],
49    use_negation: bool = True,
50    cumsum: bool = False,
51    include_basic_emotions: bool = False,
52):
53    """
54    Annotate lexicon sentiment on the given spans and return the document.
55
56    :param doc: the novel to process
57    :param lemma: does the lexicon use lemmatization?
58    :param lexicon: the lexicon to use
59    :param segments: spans to score, in CAS annotation-index order
60    :param use_negation: if True, apply negation detection
61    :param cumsum: if True, build features based on the cumulative of sentiments over the novel
62    :param include_basic_emotions: if True, also write 10 NRC basic-emotion channels per span
63    :return: the document with lexicon baseline features on each span
64    """
65    segments = list(segments)
66    feature_names = lexicon_baseline_feature_names(include_basic_emotions=include_basic_emotions)
67    for segment in segments:
68        annotate_span_lexicon_features(
69            segment,
70            lexicon,
71            lemma,
72            include_basic_emotions=include_basic_emotions,
73            use_negation=use_negation,
74        )
75    if cumsum:
76        for feature_name in feature_names:
77            for i in range(2, len(segments)):
78                segments[i].additional_features[feature_name] += segments[i - 1].additional_features[feature_name]
79
80    return doc

Annotate lexicon sentiment on the given spans and return the document.

Parameters
  • doc: the novel to process
  • lemma: does the lexicon use lemmatization?
  • lexicon: the lexicon to use
  • segments: spans to score, in CAS annotation-index order
  • use_negation: if True, apply negation detection
  • cumsum: if True, build features based on the cumulative of sentiments over the novel
  • include_basic_emotions: if True, also write 10 NRC basic-emotion channels per span
Returns

the document with lexicon baseline features on each span

def apply_negation_to_span(span: wuenlp.impl.uima.UIMANLPStructs.UIMASpan): 
83def apply_negation_to_span(span: UIMASpan):
84    negated = False
85    words = list(span.tokens)
86    for i, word in enumerate(words):
87        if word.lemma in negation_words:
88            negated = True
89        elif word.text in punctuation:
90            negated = False
91        else:
92            word.additional_features["sentiment_baseline:negated"] = negated
93    return words
def get_sentiment_for_span( span: wuenlp.impl.uima.UIMANLPStructs.UIMASpan, lexicon: wuenlp_tools.utils.sentiment.SentimentLexicon, lemma: bool) -> float: 
105def get_sentiment_for_span(span: UIMASpan, lexicon: SentimentLexicon, lemma: bool) -> float:
106    scores = []
107    for token in span.tokens:
108        word = _lexicon_word(token, lemma)
109        row = lexicon.get_word_scores(word)
110        if row is None:
111            continue
112        score = row["sentiment"]
113        if token.additional_features.get("sentiment_baseline:negated", False):
114            score = _apply_negation_to_score(score)
115        scores.append(score)
116    return float(np.mean(scores)) if scores else 0.0
def annotate_span_lexicon_features( span: wuenlp.impl.uima.UIMANLPStructs.UIMASpan, lexicon: wuenlp_tools.utils.sentiment.SentimentLexicon, lemma: bool, *, include_basic_emotions: bool = False, use_negation: bool = True) -> None: 
119def annotate_span_lexicon_features(
120    span: UIMASpan,
121    lexicon: SentimentLexicon,
122    lemma: bool,
123    *,
124    include_basic_emotions: bool = False,
125    use_negation: bool = True,
126) -> None:
127    sentiment_scores: list[float] = []
128    emotion_scores = (
129        {feature_name: [] for _, feature_name in BASIC_EMOTION_FEATURES}
130        if include_basic_emotions
131        else None
132    )
133
134    negated = False
135    for token in span.tokens:
136        in_negated_scope = False
137        if use_negation:
138            if token.lemma in negation_words:
139                negated = True
140            elif token.text in punctuation:
141                negated = False
142            else:
143                in_negated_scope = negated
144                token.additional_features["sentiment_baseline:negated"] = negated
145
146        word = _lexicon_word(token, lemma)
147        row = lexicon.get_word_scores(word)
148        if row is None:
149            continue
150
151        if use_negation and in_negated_scope:
152            sentiment = _apply_negation_to_score(row["sentiment"])
153        else:
154            sentiment = row["sentiment"]
155        sentiment_scores.append(sentiment)
156
157        if emotion_scores is not None:
158            for column, feature_name in BASIC_EMOTION_FEATURES:
159                value = row[column]
160                if use_negation and in_negated_scope:
161                    value = _apply_negation_to_score(value)
162                emotion_scores[feature_name].append(value)
163
164    span.additional_features[LEXICON_BASELINE_SENTIMENT] = (
165        float(np.mean(sentiment_scores)) if sentiment_scores else 0.0
166    )
167    if emotion_scores is not None:
168        for feature_name, values in emotion_scores.items():
169            span.additional_features[feature_name] = float(np.mean(values)) if values else 0.0
def get_nrc_sentiment_for_span( doc: wuenlp.impl.uima.UIMANLPStructs.UIMADocument, unit_type: Type[wuenlp.impl.uima.UIMANLPStructs.UIMASpan], unit_filter: Optional[Callable[[wuenlp.impl.uima.UIMANLPStructs.UIMASpan], bool]] = None, include_basic_emotions: bool = False): 
172def get_nrc_sentiment_for_span(
173    doc: UIMADocument,
174    unit_type: Type[UIMASpan],
175    unit_filter: UnitFilter | None = None,
176    include_basic_emotions: bool = False,
177):
178    segments = get_units(doc, unit_type, unit_filter)
179    return get_novel_sentiment(
180        doc,
181        lemma=True,
182        lexicon=german_nrc,
183        segments=segments,
184        use_negation=True,
185        include_basic_emotions=include_basic_emotions,
186    )
class NRCSentimentProcessor(wuenlp_tools.pipeline.AbstractPipelineProcessor[wuenlp.impl.uima.UIMANLPStructs.UIMADocument]): 
189class NRCSentimentProcessor(PipelineProcessor):
190    def __init__(self, *, include_basic_emotions: bool = False):
191        self.include_basic_emotions = include_basic_emotions
192
193    def __call__(
194        self,
195        doc: UIMADocument,
196        unit_type: Type[UIMASpan],
197        overwrite: bool = False,
198        unit_filter: UnitFilter | None = None,
199        **kwargs,
200    ):
201        segments = get_units(doc, unit_type, unit_filter)
202        expected = lexicon_baseline_feature_names(include_basic_emotions=self.include_basic_emotions)
203        if not overwrite and segments and all(
204            all(feature in segment.additional_features for feature in expected) for segment in segments
205        ):
206            logger.warning(f"Sentiment already calculated for {doc.path}. Skipping...")
207            return doc
208        return get_nrc_sentiment_for_span(
209            doc,
210            unit_type=unit_type,
211            unit_filter=unit_filter,
212            include_basic_emotions=self.include_basic_emotions,
213        )

Base class for protocol classes.

Protocol classes are defined as::

class Proto(Protocol):
    def meth(self) -> int:
        ...

Such classes are primarily used with static type checkers that recognize structural subtyping (static duck-typing).

For example::

class C:
    def meth(self) -> int:
        return 0

def func(x: Proto) -> int:
    return x.meth()

func(C())  # Passes static type check

See PEP 544 for details. Protocol classes decorated with @typing.runtime_checkable act as simple-minded runtime protocols that check only the presence of given attributes, ignoring their type signatures. Protocol classes can be generic, they are defined as::

class GenProto[T](Protocol):
    def meth(self) -> T:
        ...
NRCSentimentProcessor(*, include_basic_emotions: bool = False) 
190    def __init__(self, *, include_basic_emotions: bool = False):
191        self.include_basic_emotions = include_basic_emotions
include_basic_emotions
SentimentLexiconBaseline = PipelineStep('SentimentLexiconBaseline', processor=NRCSentimentProcessor)

Pipeline step SentimentLexiconBaseline (NRCSentimentProcessor).

unit type UIMASystemScene; additional features lexicon_baseline_sentiment.

SentimentLexiconBaselineWithBasicEmotions = PipelineStep('SentimentLexiconBaseline (basic emotions)', processor=NRCSentimentProcessor)

Pipeline step SentimentLexiconBaseline (basic emotions) (NRCSentimentProcessor).

unit type UIMASystemScene; additional features lexicon_baseline_sentiment, lexicon_baseline_anticipation, lexicon_baseline_sadness, lexicon_baseline_fear, lexicon_baseline_anger, lexicon_baseline_disgust, lexicon_baseline_trust, lexicon_baseline_surprise, lexicon_baseline_joy, lexicon_baseline_positive, lexicon_baseline_negative.

SentimentLexiconBaselineOnEqualWordChunks = PipelineStep('SentimentLexiconBaseline (equal word chunks)', processor=NRCSentimentProcessor)

Pipeline step SentimentLexiconBaseline (equal word chunks) (NRCSentimentProcessor).

unit type UIMAChunk; additional features lexicon_baseline_sentiment.

SentimentLexiconBaselineOnEqualWordChunksWithBasicEmotions = PipelineStep('SentimentLexiconBaseline (equal word chunks, basic emotions)', processor=NRCSentimentProcessor)

Pipeline step SentimentLexiconBaseline (equal word chunks, basic emotions) (NRCSentimentProcessor).

unit type UIMAChunk; additional features lexicon_baseline_sentiment, lexicon_baseline_anticipation, lexicon_baseline_sadness, lexicon_baseline_fear, lexicon_baseline_anger, lexicon_baseline_disgust, lexicon_baseline_trust, lexicon_baseline_surprise, lexicon_baseline_joy, lexicon_baseline_positive, lexicon_baseline_negative.