"""Generic low-information filler/hedge/politeness stripping.

This is the borrowed MECHANISM layer. Proven prompt-compression tools
(LLMLingua, less-tokens, defluffer) get most of their savings from the same
generic move: deleting low-information padding that carries no instruction
content. That mechanism is separable from those tools' content, so it is
reimplemented here from scratch — stdlib ``re`` only, no external NLP
dependency — as the first pass. The AAVE grammar layer (the differentiating
second layer) is applied on top by :mod:`aave_compress.engine`.

Design rules (conservative by construction):

* Only clearly-low-information items are ever touched: hedge/filler adverbs,
  padding phrases, redundant politeness, sentence-initial openers, and
  comma-delimited discourse markers.
* Content words, negations, question words, and technical terms are never
  removed. The discourse-marker "like" is only stripped in the unambiguous
  ``, like,`` form — bare "like" (verb/preposition) is never touched.
  "kind of"/"sort of" are kept whenever the preceding word is a determiner
  or wh-word ("what kind of database…"), because there they are content.
* Backtick-delimited spans (inline `` `code` `` and fenced ``` blocks)
  pass through byte-for-byte untouched.
* The whole rule set lives in the module-level :data:`FILLER_PATTERNS`
  table — one (compiled regex, replacement) entry per rule — so it is easy
  to audit and extend. Replacements may be strings or ``re.sub`` callables.
"""

from __future__ import annotations

import re
from collections.abc import Callable

__all__ = ["FILLER_PATTERNS", "imperative_ize", "strip_fillers"]

# Matches the start of a sentence: start of text, or after ., !, ? + space.
_SENT_START = r"(^|[.!?]\s+)"

# Words after which "kind of"/"sort of" is CONTENT ("what kind of index"),
# not hedging ("it kind of works") — never strip in that position.
_KIND_SORT_PREV_BLOCK = frozenset({
    "what", "which", "the", "a", "an", "this", "that", "these", "those",
    "some", "any", "every", "each", "no", "one", "first", "other", "another",
    "same", "right", "wrong", "best", "worst", "different", "certain", "of",
    "her", "his", "my", "your", "our", "their", "its",
})

_KIND_SORT_RE = re.compile(r"\b(?:kind|sort)\s+of\b[ \t]*", re.IGNORECASE)


def _kind_sort_repl(m: re.Match[str]) -> str:
    """Strip hedging "kind of"/"sort of" but keep it after a determiner/wh-word."""
    prev = re.search(r"([\w'’]+)\W*$", m.string[: m.start()])
    if prev and prev.group(1).lower() in _KIND_SORT_PREV_BLOCK:
        return m.group(0)
    return ""


# Each entry: (compiled regex, replacement). Applied IN ORDER to every
# non-code segment. Every entry is deliberately conservative — when in doubt
# a pattern requires disambiguating context (commas, sentence position,
# following punctuation) rather than matching bare words.
FILLER_PATTERNS: list[tuple[re.Pattern[str], str | Callable[[re.Match[str]], str]]] = [
    # --- redundant politeness openers (sentence-initial only) ---
    (re.compile(rf"{_SENT_START}(?:hey|hi)\b[,!.]?\s+", re.IGNORECASE), r"\1"),
    (re.compile(rf"{_SENT_START}(?:okay|ok|alright)[,]?\s+so\b[,]?\s+", re.IGNORECASE), r"\1"),
    # bare sentence-initial "so" — but never "so far/long/much/many/that",
    # where "so" is doing real work.
    (
        re.compile(
            rf"{_SENT_START}so\b[,]?\s+(?!(?:far|long|much|many|that)\b)",
            re.IGNORECASE,
        ),
        r"\1",
    ),
    # --- comma-delimited discourse markers (unambiguous positions only) ---
    (re.compile(r",\s*like\s*,", re.IGNORECASE), ","),  # never bare "like"
    (re.compile(r",\s*you know\s*,", re.IGNORECASE), ","),
    (re.compile(r",\s*i mean\s*,", re.IGNORECASE), ","),
    (re.compile(rf"{_SENT_START}you know,\s+", re.IGNORECASE), r"\1"),
    (re.compile(rf"{_SENT_START}i mean,\s+", re.IGNORECASE), r"\1"),
    # --- politeness reductions ---
    (re.compile(r"\b(can|could) you please\b", re.IGNORECASE), r"\1 you"),
    # --- hedge/filler adverbs (word-boundary, hyphen-guarded) ---
    (
        re.compile(
            r"(?<![\w-])(?:really|just|actually|basically|simply|literally"
            r"|honestly)(?![\w-])[ \t]*",
            re.IGNORECASE,
        ),
        "",
    ),
    (re.compile(r"(?<![\w-])pretty much(?![\w-])[ \t]*", re.IGNORECASE), ""),
    (_KIND_SORT_RE, _kind_sort_repl),
    # --- padding phrases (run after hedges so "really just want to know if"
    #     has already been cleaned down to the canonical form) ---
    (re.compile(r"\bi want(?:ed)? to know if\b\s*", re.IGNORECASE), ""),
    (re.compile(r"\bi was wondering if\b\s*", re.IGNORECASE), ""),
    (
        re.compile(
            r"\bi['’]?m wondering\b(?:\s+(?:if|whether)\b)?\s*", re.IGNORECASE
        ),
        "",
    ),
    (re.compile(r"(?:,\s*)?\bif you don['’]?t mind\b,?", re.IGNORECASE), ""),
    (re.compile(r"(?:,\s*)?\bif that makes sense\b,?", re.IGNORECASE), ""),
    # only at a clause/sentence end — "or something else" is content
    (
        re.compile(
            r"(?:,\s*)?\bor something(?:\s+like that)?(?=\s*[,.;:!?]|\s*$)",
            re.IGNORECASE,
        ),
        "",
    ),
    (re.compile(r"(?:,\s*)?\bat the end of the day\b,?\s*", re.IGNORECASE), ""),
    (re.compile(r"(?:,\s*)?\bfor what it['’]?s worth\b,?\s*", re.IGNORECASE), ""),
]

# --- softened requests -> bare imperatives --------------------------------
#
# "you could maybe look at X" is a REQUEST wearing politeness clothing.
# :func:`imperative_ize` strips the softening frame down to the bare
# imperative ("look at X") — the same instinct as the rest of this module:
# pleasantries that add tokens without adding meaning go, exactly like
# "really"/"just"/"I was wondering if" already do. English modals take the
# bare infinitive, so no conjugation work is needed — dropping the frame
# leaves a well-formed imperative.
#
# Conservative by construction (false negatives fine, false positives not):
#
# * A frame only fires at a sentence/clause boundary (start of text, after
#   sentence punctuation, or after a comma for question-order frames).
# * The word after the frame must look like a bare-infinitive action verb.
#   Genuine capability/permission questions ("could you lift 200 lbs?") are
#   inherently ambiguous in isolation, so the check is a denylist heuristic
#   in the style of the verb blocks in :mod:`aave_compress.rules`: if the
#   next word is a determiner, pronoun, auxiliary, adverb, adjective, or a
#   known rhetorical/perception verb, the sentence is left untouched.
#   Concretely: "you could be right" is an assessment, not a request, and
#   must survive — "be" is blocked.
# * Declarative-order frames ("you could VERB", "you might want to VERB")
#   only fire when they OPEN the sentence; after a comma they are often
#   reported speech or a mid-argument possibility, so they never fire there.
# * "you could VERB" is ALSO the standard English WARNING construction
#   ("you could break prod if you deploy without testing") — the opposite
#   of a request. Three guards keep warnings intact (see below): a
#   negative-outcome verb denylist, a conditional/consequence-tail check,
#   and a blanket block on the "could ... use" idiom. Each guard trades
#   acceptable false negatives for zero meaning inversions.
# * ROUND-3 NARROWING — the bare-modal frames ("could you", "can you",
#   "would you", "you could") additionally REQUIRE a softening hedge
#   ("maybe"/"possibly"/"perhaps"; "just" also qualifies after "can you")
#   between the modal and the verb, or they do not fire at all. Two rounds
#   of adversarial testing showed the bare frames cannot be reliably told
#   apart from warnings and rhetorical/guilt questions ("Can you afford to
#   lose this account?", "Could you live with yourself after that?") by
#   any verb denylist — the harm verb can sit one clause deeper ("afford
#   to LOSE"). Real warnings and rhetorical questions essentially never
#   insert "maybe" between the modal and the verb, so the hedge word is
#   the reliable request marker. Plain "Can you check the logs?" staying
#   untouched is an accepted, intentional false-negative cost. The three
#   round-1 guards above stay active on these frames as defense in depth.
# * REMOVED frames (round 2 proved each misfires; do NOT re-add with more
#   guards):
#     "do you think you could VERB"  — rhetorical ("Do you think you could
#                                      live with the consequences?")
#     "would you be able to VERB"    — rhetorical ("Would you be able to
#                                      sleep at night after that?")
#     "if you could VERB ..."        — hypothetical musing imperativizes
#                                      into a grammatically broken fragment
#                                      ("If you could go back in time,
#                                      what would you change?")

# Words that cannot start a bare imperative — if the token right after a
# softening frame is one of these (or is not a lowercase word at all), the
# frame is NOT a request for action and nothing fires.
_IMPERATIVE_VERB_BLOCK = frozenset({
    # determiners / noun-phrase starters
    "the", "a", "an", "this", "that", "these", "those", "some", "any",
    "all", "both", "each", "every", "no", "one", "two",
    "my", "your", "his", "her", "its", "our", "their",
    # pronouns / existentials
    "i", "you", "he", "she", "it", "we", "they",
    "me", "him", "them", "us", "there", "here",
    # copula / auxiliaries / modals — "you could be right",
    # "could you have known" are assessments or real questions
    "be", "been", "being", "have", "has", "had", "do", "does", "did",
    "will", "would", "can", "could", "may", "might", "must", "should",
    "not", "never",
    # adverbs / degree words that never start an imperative
    "also", "always", "still", "even", "ever", "just", "really",
    "actually", "maybe", "perhaps", "possibly", "probably", "definitely",
    "certainly", "totally", "seriously", "honestly", "kindly", "rather",
    "already", "too", "so", "very", "more", "most", "only", "almost",
    "least",
    # adjectives / interjections after genuine capability questions
    "sure", "right", "wrong", "okay", "ok", "fine", "good", "bad",
    "serious", "kidding", "able", "ready", "free", "available",
    # rhetorical / perception verbs — "can you believe it",
    # "can you hear me" are not requests for action
    "believe", "imagine", "hear", "feel", "smell", "taste",
    # conjunctions / prepositions / wh-words
    "and", "or", "but", "because", "if", "when", "while", "then", "than",
    "to", "for", "with", "from", "by", "of", "at", "in", "on", "up",
    "down", "off", "out",
    "what", "which", "who", "whom", "whose", "how", "why", "where",
})

# Verbs that overwhelmingly name a NEGATIVE OUTCOME after a softening frame
# — "you could lose your job", "you could break prod". There the sentence is
# a WARNING (the opposite of a request), so imperativizing would invert the
# meaning into an instruction to cause the harm. Blocked for every frame:
# the collateral false negatives ("could you break the task into steps?")
# are acceptable, a meaning inversion is not.
_WARNING_VERB_BLOCK = frozenset({
    "lose", "break", "regret", "fail", "screw", "mess", "ruin", "damage",
    "hurt", "jeopardize", "compromise", "waste", "blow", "crash", "corrupt",
    "destroy", "forfeit", "void", "forget", "risk",
})

# ROUND-4 FIX: the check above only inspected the word directly after the
# frame. That misses a whole class of rhetorical guilt/capacity questions
# where the real harm verb sits one infinitival clause deeper — "Can you
# maybe AFFORD to LOSE this account?" checked only "afford", never saw
# "lose". Two fixes, defense in depth:
#
# (a) CAPACITY_VERB_BLOCK — verbs that almost always introduce a tolerance/
#     capacity question about a negative situation ("can you handle X",
#     "could you live with X", "can you afford X") rather than a genuine
#     task request. Blocked outright as the checked verb, even when nothing
#     downstream literally matches a warning word ("live with yourself" has
#     no blocked verb in it — "live" itself is the signal).
# (b) A scan of the ENTIRE rest of the clause (not just the next word) for
#     any _WARNING_VERB_BLOCK word — catches "afford to LOSE",
#     "manage to FAIL", etc. for any leading verb, not just the ones in (a).
_CAPACITY_VERB_BLOCK = frozenset({
    "afford", "manage", "handle", "cope", "stand", "bear", "stomach",
    "survive", "live", "dare",
})
_WARNING_VERB_IN_REST_RE = re.compile(
    r"\b(?:" + "|".join(sorted(_WARNING_VERB_BLOCK)) + r")\b", re.IGNORECASE
)

# A conditional/causal tail after the frame ("... if you deploy without
# testing", "... unless you back it up", "... or else", "... otherwise")
# marks the sentence as a warning about a consequence, not a task request —
# warning sentences overwhelmingly attach such a tail, simple requests
# essentially never do. Likewise a trailing consequence idiom right before
# the final punctuation ("over this/it/that/here", "for this/it/that").
_CONDITIONAL_TAIL_RE = re.compile(
    r"\b(?:if|when|unless|otherwise|or|in\s+case)\b", re.IGNORECASE
)
_CONSEQUENCE_TAIL_RE = re.compile(
    r"\b(?:over\s+(?:this|it|that|here)|for\s+(?:this|it|that))\s*$",
    re.IGNORECASE,
)

# Optional extra hedges riding inside the frame after any required one
# ("could you maybe just please look at X").
_OPTIONAL_HEDGES = r"(?:(?:maybe|possibly|perhaps|just|please)\s+)*"

# The MANDATORY hedge for the bare-modal frames (see the round-3 bullet in
# the design notes above): without one of these directly after the modal
# frame, the frame does not fire at all.
_REQUIRED_HEDGE = r"(?:maybe|possibly|perhaps)\s+"
# "can you just VERB" also reads as a request marker (matching the
# pattern's existing behavior) — "just" qualifies for "can you" only.
_REQUIRED_HEDGE_CAN = r"(?:maybe|possibly|perhaps|just)\s+"

_IMPERATIVE_FRAME_RE = re.compile(
    r"(?P<lead>^|[.!?]\s+|,\s+)"
    r"(?P<frame>(?:"
    # hedge-OPTIONAL frames — long, explicit request framings that survived
    # two adversarial rounds without a misfire.
    r"(?:is\s+there\s+any\s+way\s+you\s+could|you\s+might\s+want\s+to)"
    rf"\s+{_OPTIONAL_HEDGES}"
    # hedge-MANDATORY frames — a bare modal is ambiguous between request,
    # warning, and rhetorical question; only the hedge disambiguates.
    rf"|(?:could\s+you|would\s+you|you\s+could)\s+{_REQUIRED_HEDGE}{_OPTIONAL_HEDGES}"
    rf"|can\s+you\s+{_REQUIRED_HEDGE_CAN}{_OPTIONAL_HEDGES}"
    r"))"
    r"(?P<verb>[a-z]+(?:-[a-z]+)*)"
    r"(?P<rest>[^.!?\n]*)"
    r"(?P<end>[.!?]?)",
    re.IGNORECASE,
)

# Declarative word order — only a request when it opens the sentence.
_DECLARATIVE_FRAME_RE = re.compile(r"you\s+(?:could|might)\b", re.IGNORECASE)


def _imperative_repl(m: re.Match[str]) -> str:
    frame = m.group("frame")
    verb = m.group("verb")
    lead = m.group("lead")
    rest = m.group("rest")
    # re.IGNORECASE lets [a-z] match uppercase — require a real lowercase
    # word ("Can you DM me?" stays untouched), and block non-action words.
    if not verb.islower() or verb.lower() in _IMPERATIVE_VERB_BLOCK:
        return m.group(0)
    # WARNING guard: negative-outcome verbs signal risk, not a request —
    # "You could lose your job over this" must never become "Lose your job
    # over this".
    if verb.lower() in _WARNING_VERB_BLOCK:
        return m.group(0)
    # CAPACITY guard (round 4): "afford"/"handle"/"live"/etc. as the checked
    # verb almost always introduce a rhetorical tolerance question, not a
    # task request — "Could you maybe live with yourself after that?" must
    # never become "Live with yourself after that."
    if verb.lower() in _CAPACITY_VERB_BLOCK:
        return m.group(0)
    # BURIED-VERB guard (round 4): the harm verb can sit one infinitival
    # clause past the checked word — "Can you maybe AFFORD to LOSE this
    # account?" only checks "afford"; this scans the whole rest of the
    # clause so "afford to lose", "manage to fail", etc. are caught
    # regardless of which verb introduces the clause.
    if _WARNING_VERB_IN_REST_RE.search(rest):
        return m.group(0)
    # IDIOM guard: "you could use some sleep" (= you need/deserve) collides
    # with the task-suggestion sense ("you could use a helper function") —
    # the whole could+use pattern is blocked in both word orders rather
    # than trying to disambiguate; the lost suggestion case is an
    # acceptable false negative.
    if verb.lower() == "use" and re.search(r"\bcould\b", frame, re.IGNORECASE):
        return m.group(0)
    # CONSEQUENCE guard: a conditional/causal tail after the frame marks a
    # warning ("You could break prod if you deploy without testing"), not
    # a request — leave the whole sentence untouched, frame and all.
    if _CONDITIONAL_TAIL_RE.search(rest) or _CONSEQUENCE_TAIL_RE.search(
        rest.rstrip()
    ):
        return m.group(0)
    if lead.startswith(",") and _DECLARATIVE_FRAME_RE.match(frame):
        return m.group(0)
    if frame[:1].isupper():
        # the frame carried the sentence's capital — hand it to the verb
        verb = verb[0].upper() + verb[1:]
    end = m.group("end")
    if end == "?":
        end = "."  # an imperative is not a question
    return f"{lead}{verb}{m.group('rest')}{end}"


def _imperative_segment(segment: str) -> str:
    return _IMPERATIVE_FRAME_RE.sub(_imperative_repl, segment)


# Inline `code` and fenced ```blocks``` — left untouched by strip_fillers.
_CODE_SPAN_RE = re.compile(r"(```.*?```|`[^`]*`)", re.DOTALL)


def _apply_outside_code(text: str, transform: Callable[[str], str]) -> str:
    """Apply ``transform`` to every non-code segment; backtick-delimited
    spans (inline `` `code` `` and fenced ``` blocks) pass through
    byte-for-byte untouched."""
    parts = _CODE_SPAN_RE.split(text)
    return "".join(
        part if i % 2 == 1 else transform(part)
        for i, part in enumerate(parts)
    )


def imperative_ize(text: str) -> str:
    """Convert clearly-softened requests for action into bare imperatives.

    "you could maybe look at X" -> "look at X"; "Could you maybe restart
    it?" -> "Restart it." Only fires on unambiguous request-softening
    frames at a sentence/clause boundary whose next word looks like a
    bare-infinitive action verb — when in doubt, the sentence is left alone
    ("you could be right" survives untouched). The bare-modal frames
    ("could/can/would you", "you could") additionally REQUIRE a softening
    hedge ("maybe"/"possibly"/"perhaps"; also "just" for "can you") between
    the modal and the verb — a plain "Can you check the logs?" is left
    untouched, an accepted false negative that keeps rhetorical questions
    ("Can you afford to lose this account?") from ever converting.
    Warnings never convert: a negative-outcome verb ("you could lose your
    job"), a conditional/consequence tail ("you could break prod if you
    deploy without testing", "... over this"), or the "could use" idiom
    ("you could use some sleep") each block the frame entirely. Rhetorical
    capacity/guilt questions never convert either, even with the hedge word
    present: a capacity verb as the checked word ("Could you maybe live
    with yourself after that?") or a negative-outcome verb one clause
    deeper than the checked word ("Can you maybe afford to lose this
    account?" — "afford" alone looks harmless, but "lose" is scanned for
    across the whole rest of the clause) both block the frame.
    Backtick-delimited code spans pass through byte-for-byte.
    """
    return _apply_outside_code(text, _imperative_segment)


def _normalize(segment: str) -> str:
    """Clean up artifacts of removal: doubled spaces, orphaned commas,
    space before punctuation. Never touches words."""
    segment = re.sub(r"[ \t]{2,}", " ", segment)
    segment = re.sub(r"[ \t]+([,.;:!?])", r"\1", segment)
    segment = re.sub(r",\s*,", ",", segment)
    segment = re.sub(r",([.;:!?])", r"\1", segment)
    segment = re.sub(rf"{_SENT_START},\s*", r"\1", segment)
    return segment


def _strip_segment(segment: str) -> str:
    for pattern, replacement in FILLER_PATTERNS:
        segment = pattern.sub(replacement, segment)
    # AFTER the filler removals, so an opener like "Hey, so I really just
    # want to know if you could maybe look at X" has already been cleaned
    # down to "you could maybe look at X" before the frame check runs.
    segment = _imperative_segment(segment)
    return _normalize(segment)


def strip_fillers(text: str) -> str:
    """Remove low-information filler/hedge/politeness padding.

    Conservative — content words are never removed and instruction meaning is
    preserved. Mechanism borrowed from LLMLingua/less-tokens/defluffer-style
    compression (generic low-information-word stripping); grammar and content
    are preserved, and backtick-delimited code spans pass through untouched.

    Softened requests for action are part of the same politeness pass: after
    the filler removals, :func:`imperative_ize` reduces unambiguous request
    frames to bare imperatives ("could you maybe look at X?" -> "Look at X.").
    """
    return _apply_outside_code(text, _strip_segment).strip(" \t")