wuenlp_tools.utils.prompting

  1from __future__ import annotations
  2
  3import atexit
  4import json
  5import os
  6import uuid
  7from dataclasses import dataclass
  8from functools import lru_cache
  9from typing import Callable, T, Type
 10
 11import requests
 12import tiktoken
 13from loguru import logger
 14from numba.core.types import Optional
 15from openai import NOT_GIVEN, OpenAI
 16from pydantic import BaseModel
 17from wuenlp import UIMADocument
 18from wuenlp.impl.uima import UIMAAnnotation, UIMASentence
 19
 20from wuenlp_tools.config import get_lightllm_config, get_litellm_metadata
 21from wuenlp_tools.keys import LIGHTLLM_API_KEY, OPENAI_API_KEY, OPENROUTER_API_KEY
 22from wuenlp_tools.llm import get_service_tier
 23from wuenlp_tools.utils.local_ollama import (
 24    ensure_local_ollama,
 25    ensure_model_pulled,
 26    is_model_not_found_error,
 27    restart_local_ollama,
 28)
 29
 30from wuenlp_tools.utils.model_cost import get_cost
 31
 32RUN_COST = 0
 33
 34litellm_session_id = "wuenlp_" + str(uuid.uuid4())
 35
 36
 37@dataclass(frozen=True)
 38class LLMCompletion:
 39    content: str | BaseModel | dict
 40    reasoning: str | None = None
 41
 42
 43@atexit.register
 44def print_run_cost():
 45    logger.info(f"Total cost of the run: {RUN_COST:.2f} €")
 46
 47
 48@dataclass
 49class LLMArchitecture:
 50    name: str
 51    model: str
 52    max_tokens: int
 53    provider: str  # "openai", "openrouter", "ollama", "local"
 54    free: bool = False
 55
 56    @property
 57    def estimated_num_characters(self):
 58        return int(self.max_tokens * 3.5)
 59
 60    @property
 61    def uses_openai_structured_output(self) -> bool:
 62        return self.provider == "openai"
 63
 64
 65def _is_ollama_like_provider(provider: str | None) -> bool:
 66    return provider in ("ollama", "local")
 67
 68
 69def _ollama_generation_format(
 70    output_format: type[BaseModel] | None,
 71    *,
 72    json: bool,
 73) -> str | dict | None:
 74    """Ollama ``format`` payload for schema- or JSON-constrained generation."""
 75    if isinstance(output_format, type) and issubclass(output_format, BaseModel):
 76        return output_format.model_json_schema()
 77    if json:
 78        return "json"
 79    return None
 80
 81
 82_RESPONSE_FORMAT_DEFAULT = object()
 83
 84
 85def _openrouter_structured_enabled() -> bool:
 86    return os.environ.get("WUENLP_OPENROUTER_STRUCTURED", "1").strip().lower() not in (
 87        "0",
 88        "false",
 89        "no",
 90        "off",
 91    )
 92
 93
 94def _local_llm_timeout() -> float:
 95    raw = os.environ.get("WUENLP_LOCAL_LLM_TIMEOUT", "600").strip()
 96    try:
 97        return float(raw)
 98    except ValueError:
 99        logger.warning("Invalid WUENLP_LOCAL_LLM_TIMEOUT={!r}; using 600s", raw)
100        return 600.0
101
102
103def _local_llm_keep_alive() -> str:
104    """Ollama model retention after a request (e.g. ``-1`` = until process exit)."""
105    return os.environ.get("WUENLP_LOCAL_LLM_KEEP_ALIVE", "-1").strip() or "-1"
106
107
108def _local_llm_num_predict() -> int:
109    raw = os.environ.get("WUENLP_LOCAL_LLM_NUM_PREDICT", "512").strip()
110    try:
111        return max(32, int(raw))
112    except ValueError:
113        logger.warning("Invalid WUENLP_LOCAL_LLM_NUM_PREDICT={!r}; using 512", raw)
114        return 512
115
116
117def _is_timeout_error(exc: Exception) -> bool:
118    if type(exc).__name__ in ("TimeoutError", "ReadTimeout", "ConnectTimeout"):
119        return True
120    message = str(exc).lower()
121    return "timeout" in message or "timed out" in message
122
123
124def _openrouter_response_format(
125    output_format: type[BaseModel] | None,
126    *,
127    json: bool,
128) -> dict | None:
129    """``response_format`` for OpenRouter ``chat.completions`` structured output."""
130    if isinstance(output_format, type) and issubclass(output_format, BaseModel):
131        return {
132            "type": "json_schema",
133            "json_schema": {
134                "name": output_format.__name__,
135                "strict": True,
136                "schema": output_format.model_json_schema(),
137            },
138        }
139    if json:
140        return {"type": "json_object"}
141    return None
142
143
144def _is_unsupported_response_format_error(exc: Exception) -> bool:
145    if getattr(exc, "status_code", None) != 400:
146        return False
147    message = str(exc).lower()
148    tokens = (
149        "response_format",
150        "json_schema",
151        "structured output",
152        "structured outputs",
153        "json_object",
154        "invalid schema",
155    )
156    return any(token in message for token in tokens)
157
158
159llama31_70b = LLMArchitecture(
160    name="llama3.1:70b", model="llama3.1:70b", max_tokens=-1, provider="ollama"
161)
162llama31_70b_local = LLMArchitecture(
163    name="llama3.1:70b", model="llama3.1:70b", max_tokens=-1, provider="local"
164)
165llama31_8b = LLMArchitecture(
166    name="llama3.1:8b", model="llama3.1:8b", max_tokens=-1, provider="ollama"
167)
168llama31_8b_local = LLMArchitecture(
169    name="llama3.1:8b", model="llama3.1:8b", max_tokens=-1, provider="local"
170)
171llama32_7b = LLMArchitecture(
172    name="llama3.2:7b", model="llama3.1:8b", max_tokens=-1, provider="ollama"
173)
174llama32_7b_local = LLMArchitecture(
175    name="llama3.2:7b", model="llama3.1:8b", max_tokens=-1, provider="local"
176)
177# gpt4o = LLMArchitecture(name="gpt-4o", model="gpt-4o", max_tokens=128000, provider="openai")
178gpt4mini = LLMArchitecture(
179    name="gpt-4o-mini", model="gpt-4o-mini", max_tokens=128000, provider="openai"
180)
181gpt41mini = LLMArchitecture(
182    name="gpt-4.1-mini", model="gpt-4.1-mini", max_tokens=128000, provider="openai"
183)
184gpt41nano = LLMArchitecture(
185    name="gpt-4.1-nano", model="gpt-4.1-nano", max_tokens=128000, provider="openai"
186)
187
188gpt5mini = LLMArchitecture(
189    name="gpt-5-mini", model="gpt-5-mini", max_tokens=128000, provider="openai"
190)
191gpt5nano = LLMArchitecture(
192    name="gpt-5-nano", model="gpt-5-nano", max_tokens=128000, provider="openai"
193)
194
195gpt52 = LLMArchitecture(
196    name="gpt-5.2", model="gpt-5.2", max_tokens=128000, provider="openai"
197)
198
199o3mini = LLMArchitecture(
200    name="o3-mini", model="o3-mini", max_tokens=128000, provider="openai"
201)
202
203llaemmlein = LLMArchitecture(
204    name="llaemmlein", model="llaemmlein", max_tokens=128000, provider=None
205)
206
207r1 = LLMArchitecture(
208    name="r1",
209    model="deepseek/deepseek-r1",
210    max_tokens=128000,
211    provider="openrouter",
212)
213
214# OpenRouter context windows (see openrouter.ai/google/gemini-*): 1M tokens each.
215_GEMINI_CONTEXT_TOKENS = 1_000_000
216
217gemini3flash = LLMArchitecture(
218    name="gemini3flash",
219    model="google/gemini-3-flash-preview",
220    max_tokens=_GEMINI_CONTEXT_TOKENS,
221    provider="openrouter",
222)
223
224gemini3pro = LLMArchitecture(
225    name="gemini3pro",
226    model="google/gemini-3.1-pro-preview",
227    max_tokens=_GEMINI_CONTEXT_TOKENS,
228    provider="openrouter",
229)
230
231gemini31lite = LLMArchitecture(
232    name="gemini31lite",
233    model="google/gemini-3.1-flash-lite-preview",
234    max_tokens=_GEMINI_CONTEXT_TOKENS,
235    provider="openrouter",
236)
237
238default_llm = gpt5nano
239
240
241class Language(BaseModel):
242    name: str
243
244
245class LLM:
246    def __init__(
247            self,
248            model: LLMArchitecture,
249            system_prompt: str,
250            post_prompt: str = None,
251            cache_maxsize: int = 0,
252            seed=NOT_GIVEN,
253            json: bool = False,
254            output_format: T[BaseModel] = None,
255            base_url: str = "https://ollama.professor-x.de/v1/",
256    ):
257        self.proxy_config = get_lightllm_config()
258        # Merged LiteLLM metadata (system + project level)
259        self.litellm_metadata = get_litellm_metadata()
260        self.local_base_url: str | None = None
261
262        # Local Ollama always bypasses the LightLLM proxy (must hit localhost).
263        use_proxy = self.proxy_config.enabled and model.provider != "local"
264        if use_proxy:
265            self.model_name = self._get_proxied_model_name(model, self.proxy_config)
266            self.openai = OpenAI(
267                api_key=LIGHTLLM_API_KEY, base_url=self.proxy_config.base_url
268            )
269            self.use_proxy = True
270        else:
271            self.model_name = model.model
272            self.use_proxy = False
273
274            if model.provider == "openai":
275                self.openai = OpenAI(api_key=OPENAI_API_KEY)
276            elif model.provider == "openrouter":
277                self.openai = OpenAI(
278                    api_key=OPENROUTER_API_KEY,
279                    base_url="https://openrouter.ai/api/v1/",
280                    timeout=180.0,
281                )
282            elif model.provider == "local":
283                self.local_base_url = ensure_local_ollama()
284                self.openai = OpenAI(
285                    api_key="ollama",
286                    base_url=self.local_base_url,
287                    timeout=_local_llm_timeout(),
288                )
289            else:  # ollama
290                self.openai = OpenAI(api_key=None, base_url=base_url)
291
292        self.model = model
293        self.system_prompt = system_prompt
294        self.post_prompt = ("\n\n" + post_prompt) if post_prompt else ""
295        self.cache_maxsize = cache_maxsize
296        if seed != NOT_GIVEN:
297            logger.warning(f"Seed is currently unsupported and will be ignored!")
298        self.seed = seed
299        self.json = json
300        self.output_format = output_format
301
302        self.with_reasoning = lru_cache(maxsize=self.cache_maxsize)(self.with_reasoning)
303
304    def _get_proxied_model_name(self, model: LLMArchitecture, proxy_config) -> str:
305        """Get model name with appropriate prefix for LightLLM proxy."""
306        prefix = proxy_config.model_prefixes.get(model.provider, "")
307        return f"{prefix}{model.model}"
308
309    def _extract_lightllm_cost(self, response) -> float:
310        """Extract cost from LightLLM response (body usage.cost or response headers)."""
311        usage = getattr(response, "usage", None)
312        if usage is not None:
313            cost = getattr(usage, "cost", None)
314            if cost is not None:
315                return float(cost)
316        raw_response = requests.get(url=self.proxy_config.base_url + f"responses/{response.id}",
317                                    headers={"Authorization": f"Bearer {LIGHTLLM_API_KEY}"})
318        headers = raw_response.headers
319        if headers and "x-litellm-response-cost" in headers:
320            return float(headers["x-litellm-response-cost"])
321        logger.info(f"Could not extract cost from LightLLM response headers: {response}")
322        return 0.0
323
324    def _extract_text(self, response) -> str:
325        """
326        Safely extract plain text from a Responses API response.
327        Avoids relying on response.output_text, which may include None segments.
328        """
329        pieces: list[str] = []
330        try:
331            outputs = getattr(response, "output", None)
332            if outputs is None:
333                # Fallback to output_text if available
334                text = getattr(response, "output_text", None)
335                return text or ""
336
337            for out in outputs:
338                contents = getattr(out, "content", []) or []
339                for content in contents:
340                    txt = None
341                    # New Responses API: content.text.value
342                    try:
343                        text_obj = getattr(content, "text", None)
344                        if text_obj is not None:
345                            txt = getattr(text_obj, "value", None)
346                    except Exception:
347                        txt = None
348
349                    # Older-style: content.text as plain string
350                    if txt is None:
351                        maybe_txt = getattr(content, "text", None)
352                        if isinstance(maybe_txt, str):
353                            txt = maybe_txt
354
355                    if isinstance(txt, str) and txt:
356                        pieces.append(txt)
357        except Exception as e:
358            logger.warning(f"Could not safely extract text from response {response}: {e}")
359            text = getattr(response, "output_text", None)
360            return text or ""
361
362        return "".join(pieces)
363
364    def _uses_responses_parse(self) -> bool:
365        if not self.output_format:
366            return False
367        if self.use_proxy:
368            return True
369        return self.model.provider == "openai"
370
371    @staticmethod
372    def _extract_chat_content(response) -> str:
373        try:
374            message = response.choices[0].message
375            content = getattr(message, "content", None)
376            return content if isinstance(content, str) else ""
377        except (AttributeError, IndexError, TypeError):
378            return ""
379
380    @staticmethod
381    def _extract_reasoning(response) -> str | None:
382        try:
383            message = response.choices[0].message
384        except (AttributeError, IndexError, TypeError):
385            return None
386        for attr in ("reasoning", "thinking"):
387            value = getattr(message, attr, None)
388            if isinstance(value, str) and value.strip():
389                return value
390        return None
391
392    def _parse_chat_output(self, raw: str):
393        if self.output_format is None:
394            return raw
395        try:
396            parsed = json.loads(raw)
397        except json.JSONDecodeError:
398            return raw
399        if isinstance(self.output_format, type) and issubclass(
400            self.output_format, BaseModel
401        ):
402            try:
403                return self.output_format.model_validate(parsed)
404            except Exception:
405                return parsed
406        return parsed
407
408    def _chat_response_format(self) -> dict | None:
409        if self.use_proxy or self.model.provider != "openrouter":
410            return None
411        if not _openrouter_structured_enabled():
412            return None
413        return _openrouter_response_format(self.output_format, json=self.json)
414
415    def _build_extra_body(self, system_role: str) -> dict:
416        extra_body = {}
417        if self.model.provider == "openrouter" and not self.use_proxy:
418            extra_body["include_reasoning"] = True
419            service_tier = get_service_tier()
420            if service_tier is not None:
421                extra_body["service_tier"] = service_tier
422        if not self.use_proxy and _is_ollama_like_provider(self.model.provider):
423            ollama_format = _ollama_generation_format(
424                self.output_format, json=self.json,
425            )
426            if ollama_format is not None:
427                extra_body["format"] = ollama_format
428            extra_body.setdefault("options", {})["num_predict"] = _local_llm_num_predict()
429            if self.model.provider == "local":
430                extra_body["keep_alive"] = _local_llm_keep_alive()
431        if self.use_proxy:
432            extra_body["litellm_session_id"] = litellm_session_id
433            if self.litellm_metadata:
434                metadata = dict(self.litellm_metadata)
435                tags: list[str] = []
436                existing_tags = metadata.get("tags")
437                if isinstance(existing_tags, list):
438                    tags.extend(str(t) for t in existing_tags)
439                for key, value in metadata.items():
440                    if isinstance(value, bool):
441                        tag = f"{key}:{str(value).lower()}"
442                        if tag not in tags:
443                            tags.append(tag)
444                if tags:
445                    metadata["tags"] = tags
446                    extra_body["tags"] = tags
447                extra_body["metadata"] = metadata
448        return extra_body
449
450    def _openrouter_headers(self) -> dict:
451        if not self.use_proxy and self.model.provider == "openrouter":
452            return {"Authorization": f"Bearer {self.openai.api_key}"}
453        return {}
454
455    def _create_completion(
456        self,
457        prompt: str,
458        system_role: str,
459        *,
460        response_format=_RESPONSE_FORMAT_DEFAULT,
461    ):
462        extra_body = self._build_extra_body(system_role)
463        extra_headers = self._openrouter_headers()
464        if self._uses_responses_parse():
465            return self.openai.responses.parse(
466                model=self.model_name,
467                input=[
468                    {"role": system_role, "content": self.system_prompt},
469                    {"role": "user", "content": prompt},
470                ],
471                text_format=self.output_format
472                if self.output_format is not None
473                else {"type": "json_object"}
474                if self.json
475                else {"type": "text"},
476                extra_headers=extra_headers,
477                extra_body=extra_body,
478            )
479        completion_kwargs: dict = {
480            "model": self.model_name,
481            "messages": [
482                {"role": system_role, "content": self.system_prompt},
483                {"role": "user", "content": prompt},
484            ],
485            "extra_headers": extra_headers,
486            "extra_body": extra_body,
487        }
488        if response_format is _RESPONSE_FORMAT_DEFAULT:
489            response_format = self._chat_response_format()
490        if response_format is not None:
491            completion_kwargs["response_format"] = response_format
492        return self.openai.chat.completions.create(**completion_kwargs)
493
494    def _local_ollama_base_url(self) -> str:
495        if self.local_base_url is None:
496            self.local_base_url = ensure_local_ollama()
497        return self.local_base_url
498
499    def _request_with_local_model_retry(self, prompt: str, system_role: str):
500        if self.model.provider == "local":
501            ensure_model_pulled(self._local_ollama_base_url(), self.model_name)
502        try:
503            return self._create_completion(prompt, system_role)
504        except Exception as exc:
505            if self.model.provider == "local" and is_model_not_found_error(exc):
506                logger.info(
507                    "Model {} not found locally; pulling and retrying once",
508                    self.model_name,
509                )
510                ensure_model_pulled(self._local_ollama_base_url(), self.model_name)
511                return self._create_completion(prompt, system_role)
512            if self.model.provider == "local" and _is_timeout_error(exc):
513                logger.warning(
514                    "Local LLM request timed out for {}; restarting ollama and retrying once: {}",
515                    self.model_name,
516                    exc,
517                )
518                new_url = restart_local_ollama()
519                self.local_base_url = new_url
520                self.openai = OpenAI(
521                    api_key="ollama",
522                    base_url=new_url,
523                    timeout=_local_llm_timeout(),
524                )
525                return self._create_completion(prompt, system_role)
526            if (
527                self._chat_response_format() is not None
528                and _is_unsupported_response_format_error(exc)
529            ):
530                logger.warning(
531                    "OpenRouter rejected response_format for {}; "
532                    "retrying once without structured output: {}",
533                    self.model_name,
534                    exc,
535                )
536                return self._create_completion(
537                    prompt, system_role, response_format=None,
538                )
539            raise
540
541    def with_reasoning(self, prompt) -> LLMCompletion:
542        global RUN_COST
543
544        model_max_chars = self.model.estimated_num_characters
545        if model_max_chars > -1 and len(prompt) > model_max_chars:
546            prompt = prompt[:model_max_chars - 3 - len(self.post_prompt)] + "..."
547            logger.warning(f"Truncated prompt to {model_max_chars} characters.")
548        prompt += self.post_prompt
549        logger.debug(f"System prompt is: {self.system_prompt}")
550        shortened_prompt = (
551            prompt if len(prompt) < 1000 else prompt[:1000] + "\n...\n" + prompt[-1000:]
552        )
553        logger.debug(
554            f"Querying {self.model}, seed {self.seed} with prompt: {shortened_prompt}"
555        )
556        system_role = (
557            "system" if not self.model.name[:2] in ("o3", "o4") else "developer"
558        )
559        response = self._request_with_local_model_retry(prompt, system_role)
560
561        if self._uses_responses_parse():
562            if self.output_format is not None:
563                content = response.output_parsed
564                if content is None:
565                    raw_text = self._extract_text(response)
566                    try:
567                        content = json.loads(raw_text)
568                    except json.JSONDecodeError:
569                        content = raw_text
570            else:
571                content = self._extract_text(response)
572            response_reasoning = None
573        else:
574            raw_text = self._extract_chat_content(response)
575            content = self._parse_chat_output(raw_text)
576            response_reasoning = self._extract_reasoning(response)
577        # Cost calculation based on provider/proxy
578        if self.use_proxy:
579            # Extract cost from LightLLM response headers
580            cost = self._extract_lightllm_cost(response)
581        elif self.model.provider == "openai":
582            cost = get_cost(response, self.model.model)
583        elif self.model.provider == "openrouter":
584            url = f"https://openrouter.ai/api/v1/generation?id={response.id}"
585            gen_response = requests.post(
586                url=url,
587                headers={
588                    "Authorization": f"Bearer {OPENROUTER_API_KEY}",
589                    "Content-Type": "application/json",
590                },
591            ).json()
592
593            try:
594                cost = gen_response["total_cost"]
595            except KeyError:
596                cost = 0
597        elif _is_ollama_like_provider(self.model.provider):
598            cost = 0
599        else:
600            cost = 0
601
602        RUN_COST += cost
603
604        logger.debug(f"Model {self.model} says: {content}")
605        if response_reasoning:
606            logger.debug(f"Reasoning: {response_reasoning}")
607        logger.debug(f"Cost: {cost:.2f} €")
608        logger.debug(f"Total cost: {RUN_COST:.2f} €")
609        return LLMCompletion(content=content, reasoning=response_reasoning)
610
611    def __call__(self, prompt) -> str | T:
612        return self.with_reasoning(prompt).content
613
614
615SENTENCE_IN_CONTEXT_PROMPT = "You will be passed a piece of text with one sentence marked in <sentence>...</sentence>. While you should take the context into account, your answer should be specific to the marked sentence. Your task is as follows: "
616
617
618class SentenceInContextLLM(LLM):
619    def __init__(
620            self,
621            model: LLMArchitecture,
622            system_prompt: str,
623            post_prompt: str = None,
624            cache_maxsize: int = 0,
625            seed=NOT_GIVEN,
626            json: bool = False,
627            output_format: T[BaseModel] = None,
628            base_url: str = "https://ollama.professor-x.de/v1/",
629    ):
630        super().__init__(
631            model,
632            SENTENCE_IN_CONTEXT_PROMPT + system_prompt,
633            post_prompt,
634            cache_maxsize,
635            seed,
636            json,
637            output_format,
638            base_url,
639        )
640
641
642class YesNoReason(BaseModel):
643    answer: bool
644    reason: str
645
646
647class YesNo(BaseModel):
648    answer: bool
649
650
651class YesNoLLM(LLM):
652    def __init__(
653            self,
654            model: LLMArchitecture,
655            system_prompt: str,
656            post_prompt: str = None,
657            cache_maxsize: int = 0,
658            seed=NOT_GIVEN,
659            max_tries=3,
660            reason: bool = False,
661            retry_prompt_add: str = "",
662    ):
663        if model.provider == "openai":
664            super().__init__(
665                model,
666                system_prompt,
667                post_prompt,
668                cache_maxsize,
669                seed=seed,
670                output_format=YesNoReason if reason else YesNo,
671            )
672        else:
673            if not post_prompt or "yes" not in post_prompt:
674                logger.warning(
675                    "Post prompt does not contain 'yes'. Adding instruction."
676                )
677                if post_prompt is None:
678                    post_prompt = ""
679                post_prompt = post_prompt + (
680                    "Answer only with yes or no, nothing else."
681                    if not reason
682                    else "Answer with yes or no, followed by a reason"
683                )
684            super(YesNoLLM, self).__init__(
685                model, system_prompt, post_prompt, cache_maxsize, seed=seed, json=False
686            )
687        self.max_tries = max_tries
688        self.retry_prompt_add = (
689            retry_prompt_add
690            if retry_prompt_add.startswith(" ")
691            else " " + retry_prompt_add
692        )
693        self.reason = reason
694
695        if max_tries > 1 and seed != NOT_GIVEN and retry_prompt_add is None:
696            logger.warning(
697                "max_tries > 1 but seed is fixed and no retry prompt is given. This will lead to the same answer every time. Consider adding a retry prompt to avoid this."
698            )
699
700    def __call__(self, prompt) -> YesNo | YesNoReason:
701        if self.model.provider == "openai":
702            response = super().__call__(prompt)
703            return response
704        tries = 0
705        while tries < self.max_tries:
706            if tries > 0 and self.retry_prompt_add:
707                prompt += self.retry_prompt_add
708            response = super().__call__(prompt)
709            response_lower = response.lower().strip("*").strip()
710            if self.is_positive(response_lower) or self.is_negative(response_lower):
711                answer = self.is_positive(response_lower)
712                if self.reason:
713                    return YesNoReason(answer=answer, reason=response)
714                else:
715                    return YesNo(answer=answer)
716            tries += 1
717        raise ValueError(f"Could not get a yes/no answer after {self.max_tries} tries.")
718
719    def is_negative(self, response_lower):
720        return response_lower.startswith("no") or response_lower.startswith("nein")
721
722    def is_positive(self, response_lower):
723        return response_lower.startswith("yes") or response_lower.startswith("ja")
724
725
726# @cache_to_disk(maxsize=-1)
727def get_example_for_schema(schema: BaseModel, query: str):
728    language_llm = LLM(
729        gpt5nano,
730        system_prompt="Return the English name of the language of the provided text.",
731        seed=42,
732        output_format=Language,
733    )
734
735    language = language_llm(query).name
736
737    system_prompt = f"Your job is to guide a language model in generating a response in the provided output format. Your instructions should be in {language}. Address the language model informally."
738    instruction_gpt = LLM(gpt4mini, system_prompt, seed=42)
739    example_gpt = LLM(gpt4mini, system_prompt, seed=42, output_format=schema)
740
741    example: schema = example_gpt(
742        "Give an example of the output format you want the model to generate."
743    )
744    return (
745            instruction_gpt(
746                "Start by telling the model 'Your output should match this format:'."
747            )
748            + "\n"
749            + example.json()
750    )
751
752
753def build_sentence_sample_in_context(
754        sentence: UIMASentence,
755        model: LLMArchitecture,
756        max_context_size: Optional[int] = None,
757) -> str:
758    s = sentence.text
759
760    encoding = tiktoken.get_encoding("cl100k_base")
761
762    context = ["<sentence>" + sentence.text + "</sentence>"]
763    prev_sentence = sentence.previous
764    next_sentence = sentence.next
765    context_size = model.max_tokens if max_context_size is None else max_context_size
766    while (
767            len(
768                encoding.encode(
769                    " ".join(
770                        context
771                        + [
772                            prev_sentence.text if prev_sentence is not None else "",
773                            next_sentence.text if next_sentence is not None else "",
774                        ]
775                    )
776                )
777            )
778            < context_size
779    ):
780        if prev_sentence is not None:
781            context.insert(0, prev_sentence.text)
782            prev_sentence = prev_sentence.previous
783        if next_sentence is not None:
784            context.append(next_sentence.text)
785            next_sentence = next_sentence.next
786
787    context = " ".join(context)
788    return context
789
790
791def build_sentence_samples(
792        doc: UIMADocument,
793        annotation_type: Type[UIMAAnnotation],
794        model: LLMArchitecture,
795        max_context_size: Optional[int] = None,
796        include_labels: bool = True,
797        sentence_to_label: Optional[Callable] = None,
798):
799    samples = []
800
801    if include_labels and sentence_to_label is None:
802        logger.info(
803            "sentence_to_label is not specified. Extracting sentence labels by checking for overlap with given annotation_type."
804        )
805
806    for sentence in doc.sentences:
807        sample_text = build_sentence_sample_in_context(
808            sentence, model, max_context_size=max_context_size
809        )
810        if include_labels:
811            if sentence_to_label is None:
812                sample_label = bool(sentence.overlapping(annotation_type))
813            else:
814                sample_label = sentence_to_label(sentence)
815        else:
816            sample_label = None
817
818        samples.append((sample_text, sample_label))
819
820    return samples
821
822
823if __name__ == "__main__":
824    gpt = YesNoLLM(llaemmlein, "This is a test prompt.", seed=42, reason=True)
825    response = gpt("What is the meaning of life?")
826    print(response)