j/ddlZddlmZddlmZGddZGddeZdZd Z d Z ed d d gZ GddZ dS)N) namedtuple) find_binaryceZdZdZd dZedZedZdZdZ eee Z dZ d Z d Z d Zd ZdS) AlignedSenta# Return an aligned sentence object, which encapsulates two sentences along with an ``Alignment`` between them. Typically used in machine translation to represent a sentence and its translation. >>> from nltk.translate import AlignedSent, Alignment >>> algnsent = AlignedSent(['klein', 'ist', 'das', 'Haus'], ... ['the', 'house', 'is', 'small'], Alignment.fromstring('0-3 1-2 2-0 3-1')) >>> algnsent.words ['klein', 'ist', 'das', 'Haus'] >>> algnsent.mots ['the', 'house', 'is', 'small'] >>> algnsent.alignment Alignment([(0, 3), (1, 2), (2, 0), (3, 1)]) >>> from nltk.corpus import comtrans >>> print(comtrans.aligned_sents()[54]) 'So why should EU arm...'> >>> print(comtrans.aligned_sents()[54].alignment) 0-0 0-1 1-0 2-2 3-4 3-5 4-7 5-8 6-3 7-9 8-9 9-10 9-11 10-12 11-6 12-6 13-13 :param words: Words in the target language sentence :type words: list(str) :param mots: Words in the source language sentence :type mots: list(str) :param alignment: Word-level alignments between ``words`` and ``mots``. Each alignment is represented as a 2-tuple (words_index, mots_index). :type alignment: Alignment Nc||_||_|tg|_dSt |tusJ||_dSN)_words_mots Alignment alignmenttype)selfwordsmotsr s u/home/lighthouse/badge-speaker-benchmark/.benchmark-venv-community/lib/python3.11/site-packages/nltk/translate/api.py__init__zAlignedSent.__init__1sI   &r]]DNNN ??i////&DNNNc|jSr)r rs rrzAlignedSent.words:s {rc|jSr)r rs rrzAlignedSent.mots>s zrc|jSr) _alignmentrs r_get_alignmentzAlignedSent._get_alignmentBs rc~tt|jt|j|||_dSr)_check_alignmentlenrrr)rr s r_set_alignmentzAlignedSent._set_alignmentEs/TZ#di..)DDD#rcddd|jDz}ddd|jDz}d|d|d|jdS)z_ Return a string representation for this ``AlignedSent``. :rtype: str z[%s]z, c3 K|] }d|zV dSz'%s'N.0ws r z'AlignedSent.__repr__..Qs&#D#D1FQJ#D#D#D#D#D#Drc3 K|] }d|zV dSr r!r"s rr%z'AlignedSent.__repr__..Rs&"B"B!6A:"B"B"B"B"B"Brz AlignedSent())joinr r r)rrrs r__repr__zAlignedSent.__repr__Ksu $))#D#D #D#D#DDDE"B"Btz"B"B"BBBCCeCCtCCtCCCCrcd}|dz }|ddjDz }|ddjDz }|dfdjDz }t t jdz D]4}|dj|j|dzz }5t t jdz D]4}|d j|j|dzz }5|d d d jDzz }|d d d jDzz }|dz }|S)z< Dot representation of the aligned sentence zgraph align { znode[shape=plaintext] c g|] }d|d|d S)"z_source" [label=""] r!r"s r z'AlignedSent._to_dot..^s,LLL16!66a666LLLrc g|] }d|d|d S)r-z_target" [label="r.r!r"s rr/z'AlignedSent._to_dot.._s,KKK16!66a666KKKrcTg|]$\}}dj|dj|d%S)r-z _source" -- "z _target" )r r )r#uvrs rr/z'AlignedSent._to_dot..csM   AqLDKNKKAKKK   rz)"{}_source" -- "{}_source" [style=invis] z)"{}_target" -- "{}_target" [style=invis] z{rank = same; %s}  c3 K|] }d|zV dS)z "%s_source"Nr!r"s rr%z&AlignedSent._to_dot..xs'.V.VQ}q/@.V.V.V.V.V.Vrc3 K|] }d|zV dS)z "%s_target"Nr!r"s rr%z&AlignedSent._to_dot..ys'.U.UQ}q/@.U.U.U.U.U.Ur})r(r r rrangerformat)rsis` r_to_dotzAlignedSent._to_dotVs  && RWWLL LLL M MM RWWKK KKK L LL RWW     O      s4;''!+,,  A =DD A AE" AA s4:*++  A =DD 1  1q5! AA "chh.V.V$+.V.V.V&V&V WW "chh.U.U$*.U.U.U&U&U VV Src|d}d} td}n"#t$r}t d|d}~wwxYw t j|d|zgt jt jt j}n"#t$r}t d|d}~wwxYw| |\}}| dS)zR Ipython magic : show SVG representation of this ``AlignedSent``. utf8svgdotz0Cannot find the dot binary from Graphviz packageNz-T%s)stdinstdoutstderr) r=encoder LookupError Exception subprocessPopenPIPEOSError communicatedecode)r dot_string output_format dot_binaryeprocessouterrs r _repr_svg_zAlignedSent._repr_svg_s \\^^**622   W%U++JJ W W WNOOUV V W W &Vm34 o!! GG  W W WNOOUV V W&&z22Szz&!!!s-; AAA;B B9$B44B9cd|jdddz}d|jdddz}d|d|dS)zn Return a human-readable string representation for this ``AlignedSent``. :rtype: str r5Nz...z 'z'>)r(r r )rsourcetargets r__str__zAlignedSent.__str__s_ $+&&ss+e3$*%%crc*U2999v9999rcft|j|j|jS)zm Return the aligned sentence pair, reversing the directionality :rtype: AlignedSent )rr r rinvertrs rr\zAlignedSent.inverts( 4:t{DO4J4J4L4LMMMrr)__name__ __module__ __qualname____doc__rpropertyrrrrr r)r=rUrZr\r!rrrrs>''''XX$$$88I D D D'''R"""4:::NNNNNrrcTeZdZdZdZedZdZdZd dZ dZ d Z d Z dS) r ac A storage class for representing alignment between two sequences, s1, s2. In general, an alignment is a set of tuples of the form (i, j, ...) representing an alignment between the i-th element of s1 and the j-th element of s2. Tuples are extensible (they might contain additional data, such as a boolean to indicate sure vs possible alignments). >>> from nltk.translate import Alignment >>> a = Alignment([(0, 0), (0, 1), (1, 2), (2, 2)]) >>> a.invert() Alignment([(0, 0), (1, 0), (2, 1), (2, 2)]) >>> print(a.invert()) 0-0 1-0 2-1 2-2 >>> a[0] [(0, 1), (0, 0)] >>> a.invert()[2] [(2, 1), (2, 2)] >>> b = Alignment([(0, 0), (0, 1)]) >>> b.issubset(a) True >>> c = Alignment.fromstring('0-0 0-1') >>> b == c True ct||}|tgkrtd|Dnd|_d|_|S)Nc3&K|] }|dV dSrNr!r#ps rr%z$Alignment.__new__..s&++!++++++rr) frozenset__new__max_len_index)clspairsrs rrizAlignment.__new__sV  e,,/3y}}/D/DC++d++++++!   rcXtd|DS)a Read a giza-formatted string and return an Alignment object. >>> Alignment.fromstring('0-0 2-1 9-2 21-3 10-4 7-5') Alignment([(0, 0), (2, 1), (7, 5), (9, 2), (10, 4), (21, 3)]) :type s: str :param s: the positional alignments in giza format :rtype: Alignment :return: An Alignment object corresponding to the string representation ``s``. c,g|]}t|Sr!) _giza2pair)r#as rr/z(Alignment.fromstring..s;;;A*Q--;;;r)r split)rmr;s r fromstringzAlignment.fromstrings);;;;;<<.s;>>A1Q41,122.>>>>>>r)r rs rr\zAlignment.inverts!>>>>>>>>rNc"t}|j||s|j}|D]:}|d|j|gD;t |S)z Work out the range of the mapping from the given positions. If no positions are specified, compute the range of the entire mapping. Nc3 K|] \}}|V dSrr!)r#_fs rr%z"Alignment.range..s&>>tq!>>>>>>r)setrlrykeysupdaterzsorted)r positionsimagergs rr9zAlignment.ranges  ;        + ((**I ? ?A LL>>t{q"'='=>>> > > > >e}}rc&dt|zS)M Produce a Giza-formatted string representing the alignment. z Alignment(%r))rrs rr)zAlignment.__repr__s--rcZddt|DS)rr5c30K|]}d|ddzVdS)z%d-%dNrr!rfs rr%z$Alignment.__str__.. s.>>A!BQB%>>>>>>r)r(rrs rrZzAlignment.__str__s+xx>>>>>>>>rci|_|D]6}|j|dg|7dS)a Build a sparse index mapping each left index ``i`` to the list of alignments originating from word ``i``. The index is keyed only by the left indices that actually occur, so its size is bounded by the number of pairs rather than by the largest left index (which is attacker-controlled in giza-format input). A dense ``[[] for _ in range(self._len + 1)]`` list would instead let a single tiny pair with a huge left index (e.g. ``"0-0 100000000-1"``) allocate ~100M empty lists -- gigabytes of memory -- an unbounded-allocation DoS (CWE-770; CVE-2026-12837). rN)rl setdefaultappend)rrgs rryzAlignment._build_indexsQ  7 7A K " "1Q4 , , 3 3A 6 6 6 6 7 7rr) r]r^r_r`ri classmethodrtr|r\r9r)rZryr!rrr r s2  = =[ =((($???    ... ??? 77777rr cn|d\}}t|t|fSN-rsrw) pair_stringr<js rrqrq s/   S ! !DAq q663q66>rcp|d\}}}t|t|fSrr)rr<rrgs r _naacl2pairr%s1$$GAq! q663q66>rct|tusJtfd|Dstdtfd|DstddS)ab Check whether the alignments are legal. :param num_words: the number of source language words :type num_words: int :param num_mots: the number of target language words :type num_mots: int :param alignment: alignment to be checked :type alignment: Alignment :raise IndexError: if alignment falls outside the sentence c3DK|]}d|dcxkokncVdSrer!)r#pair num_wordss rr%z#_check_alignment..9sC>>DqDG''''i''''>>>>>>rz&Alignment is outside boundary of wordsc3XK|]$}|ddupd|dcxkokncV%dS)r4Nrr!)r#rnum_motss rr%z#_check_alignment..;sPPPdtAw$9!tAw"9"9"9"9"9"9"9"9PPPPPPrz%Alignment is outside boundary of motsN)r r all IndexError)rrr s`` rrr*s  ??i ' ' ' ' >>>>I>>> > >CABBB PPPPiPPP P PB@AAABBrPhraseTableEntry trg_phraselog_probc*eZdZdZdZdZdZdZdS) PhraseTablezs In-memory store of translations for a given phrase, and the log probability of the those translations c,t|_dSr)dict src_phrasesrs rrzPhraseTable.__init__Hs66rc|j|S)a Get the translations for a source language phrase :param src_phrase: Source language phrase of interest :type src_phrase: tuple(str) :return: A list of target language phrases that are translations of ``src_phrase``, ordered in decreasing order of likelihood. Each list element is a tuple of the target phrase and its log probability. :rtype: list(PhraseTableEntry) rr src_phrases rtranslations_forzPhraseTable.translations_forKs ++rct||}||jvr g|j|<|j|||j|dddS)z :type src_phrase: tuple(str) :type trg_phrase: tuple(str) :param log_prob: Log probability that given ``src_phrase``, ``trg_phrase`` is its translation :type log_prob: float )rrc|jSr)r)rQs rz!PhraseTable.add..gs rT)r{reverseN)rrrsort)rrrrentrys raddzPhraseTable.addZsx!JJJJ T- - -+-D Z ( $++E222 $)).B.BD)QQQQQrc||jvSrrrs r __contains__zPhraseTable.__contains__isT---rN)r]r^r_r`rrrrr!rrrrBs] """ , , , R R R.....rr) rH collectionsrnltk.internalsrrrhr rqrrrrr!rrrs""""""&&&&&&XNXNXNXNXNXNXNXNvq7q7q7q7q7 q7q7q7h  BBB*:0<2LMM(.(.(.(.(.(.(.(.(.(.r