jcQndZddlZddlmZddlmZejdZdMdZdNd Z dOd Z d Z dOd Z dMdZ dOdZiddddddddddddddddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1id2d3d4d5d6d7d8d9d:d;dd?d@dAdBdCdDdEdFdGdHdIdJdKdLdMdNdOdPdQdRdSidTdUdVdWdXdYdZd[d\d]d^d_d`dadbdcdddedfdgdhdidjdkdldmdndodpdqdrdsdtduidvdwdxdydzd{d|d}d~dddddddddddddddddddddddddiddddddddddddddddddddddddddddddddddiddddddddddÓddœddǓddɓdd˓dd͓ddϓddѓddӓddՓddדddٓddۓiddݓddߓddddddddddddddddddddddddddddddidddddddddddd d d d d ddddddddddddddddddid d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4d5d6d7d8d9d:d;d<d=d>d?d@dAidBdCdDdEdFdGdHdIdJdKdLdMdNdOdPdQdRdSdTdUdVdWdXdYdZd[d\d]d^d_d`dadbdcdddedfdgdhdidjdkdldmdndodpdqdrZidsdtdudvdwdxdydzd{d|d}d~ddddddddddddddddddddddiddddddddddddddddddddddddddddddddddidddddddddddd“dÐdēdŐdƓdǐdȓdɐdʓdːd̓d͐dΓdϐdГdѐdғdӐdԓdՐd֓dאdؓidِdړdېdܓdݐdޓdߐdddddddddddddddddddddddddddiddddddddddddddd d d d d dddddddddddddddidddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4d5d6d7d8d9d:d;d<d=d>id?d@dAdBdCdDdEdFdGdHdIdJdKdLdMdNdOdPdQdRdSdTdUdVdWdXdYdZd[d\d]d^d_d`idadbdcdddedfdgdhdidjdkdldmdndodpdqdrdsdtdudvdwdxdydzd{d|d}d~ddddiddddddddddddddddddddddddddddddddddidddddddddddddddddddddddddddddd“dÐdēdŐdƓidǐdȓdɐdʓdːd̓d͐dΓdϐdГdѐdғdӐdԓdՐd֓dאdؓdِdړdېdܓdݐdޓdߐdddddddddiddddddddddddddddddddddddddddddddd d id d d ddddddddddddddddddd d!d"d#d$d%d&d'd(d)d*d+d,id-d.d/d0d1d2d3d4d5d6d7d8d9d:d;d<d=d>d?d@dAdBdCdDdEdFdGdHdIdJdKdLdMdNidOdPdQdRdSdTdUdVdWdXdYdZd[d\d]d^d_d`dadbdcdddedfdgdhdidjdkdldmdndodpidqdrdsdtdudvdwdxdydzd{d|d}d~ddddddddddddddddddddiddddddddddddddddddddddddddddddddddidddddddddddddd“dÐdēdŐdƓdǐdȓdɐdʓdːd̓d͐dΓdϐdГdѐdғdӐdԓdՐd֓idאdؓdِdړdېdܓdݐdޓdߐdddddddddddddddddddddddddiddddddddddddddddd d d d d dddddddddddddidddddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4d5d6d7d8d9d:d;d<d=d>d?d@dAdBdCdDdEdFdGdHdIdJdKdLZe eZe eZdS(Pa Translate between language names and language codes. The iso639-3 language codes were downloaded from the registration authority at https://iso639-3.sil.org/ The iso639-3 codeset is evolving, so retired language codes are kept in the "iso639retired" dictionary, which is used as fallback by the wrapper functions "langname" and "langcode", in order to support the lookup of retired codes. The "langcode" function returns the current iso639-3 code if there is one, and falls back to the retired code otherwise. As specified by BCP-47, it returns the shortest (2-letter) code by default, but 3-letter codes are also available: >>> import nltk.langnames as lgn >>> lgn.langname('fri') #'fri' is a retired code 'Western Frisian' The current code is different from the retired one: >>> lgn.langcode('Western Frisian') 'fy' >>> lgn.langcode('Western Frisian', typ = 3) 'fry' N)warn)bcp47z[a-z][a-z][a-z]?fullFcJ||rtddS|d}|d}t|r|t vr t |S|t vrEt |}td|d|dd|g|d dz}tj |}|#|d kr|S|d dSd |}|rt|t|ddS) a$ Convert a composite BCP-47 tag to a language name. Returns None if the tag is not found, unless strict=True, in which case a LookupError is raised. >>> from nltk.langnames import langname >>> langname('ca-Latn-ES-valencia') 'Catalan: Latin: Spain: Valencian' >>> langname('ca-Latn-ES-valencia', typ="short") 'Catalan' >>> print(langname('zzz')) None >>> langname('zzz', strict=True) Traceback (most recent call last): ... LookupError: Could not find language name for tag 'zzz' Nz)Could not find language name for tag None-rz Shortening z to  stacklevelr:z%Could not find language name for tag ) LookupErrorsplitlower codepattern fullmatch iso639retired iso639shortrjoinrname)tagtypstricttagscodecode2rfaileds q/home/lighthouse/badge-speaker-benchmark/.benchmark-venv-community/lib/python3.11/site-packages/nltk/langnames.pylangnamer.s=, {  KIJJ Jt 99S>>D 7==??DT"" * =  & & [ %E 4t44544 C C C C((E7T!""X-..Cz#  f}} zz#q)) >> from nltk.langnames import langcode >>> langcode('Modern Greek (1453-)') 'el' Specify 'typ=3' to get the 3-letter code: >>> langcode('Modern Greek (1453-)', typ=3) 'ell' >>> print(langcode('NotALanguage')) None >>> langcode('NotALanguage', strict=True) Traceback (most recent call last): ... LookupError: Could not find language code for name 'NotALanguage' z&Could not find language code for name zCould not find language in rr N)rlangcode iso639longiso639code_retiredr r)rrrrs rr"r"]s2 u~~d# !88 **d#D # # #!$'' CK4KKLLL 34 3 3BBBBBBrcttdr tjtjtj|}||rt d||S)a Convert BCP-47 tag to Wikidata Q-code. Returns the Wikidata Q-code for the given BCP-47 tag, or None if the tag is not found. If strict=True, raises a LookupError instead of returning None. >>> tag2q('nds-u-sd-demv') 'Q4289225' >>> print(tag2q('unknown-tag')) None >>> tag2q('unknown-tag', strict=True) Traceback (most recent call last): ... LookupError: Could not find Wikidata Q-code for BCP-47 tag 'unknown-tag' wiki_qNz.Could not find Wikidata Q-code for BCP-47 tag )hasattrrr& load_wiki_qgetr )rrresults rtag2qr+sf" 5( # #u|';  \  c " "F ~&~R3RRSSS Mrct|tt|krd|DSt ddS)z Return the inverse mapping of a dictionary if it is bijective. If the input dictionary is bijective (i.e., all values are unique), returns a new dictionary mapping values to keys. Otherwise, returns None and emits a warning. ci|]\}}|| Sr.).0keyvals r z inverse_dict..s777Zc3S777rz1This dictionary has no bijective inverse mapping.N)lenkeyssetvaluesitemsr)dics r inverse_dictr9sf 388::#c#**,,//000077399;;7777 @AAAAArcttdr tjtjttdr tj6t tj}|t d|t_tj|}||rtd||S)a Convert Wikidata Q-code to BCP-47 tag. Returns the BCP-47 tag for the given Wikidata Q-code, or None if the Q-code is not found. If strict=True, raises a LookupError instead of returning None. >>> q2tag('Q4289225') 'nds-u-sd-demv' >>> print(q2tag('Q0000000')) None >>> q2tag('Q0000000', strict=True) Traceback (most recent call last): ... LookupError: Could not find BCP-47 tag for Wikidata Q-code 'Q0000000' r&N wiki_bcp47zTThe Wikidata mapping (wiki_q) is not bijective. Cannot safely build inverse mapping.z.Could not find BCP-47 tag for Wikidata Q-code ) r'rr&r(r;r9 ValueErrorr)r )qcoderinverser*s rq2tagr?s" 5( # #u|';  5, ' '#5+;+Cu|,, ?f #   ! !% ( (F ~&~T5TTUUU MrcPt||}|dSt|||S)a Convert Wikidata Q-code to BCP-47 (full or short) language name. Returns None if the Q-code is not found, unless strict=True, in which case a LookupError is raised. >>> q2name('Q4289225') 'Low German: Mecklenburg-Vorpommern' >>> q2name('Q4289225', "short") 'Low German' >>> print(q2name('Q0000000')) None >>> q2name('Q0000000', strict=True) Traceback (most recent call last): ... LookupError: Could not find BCP-47 tag for Wikidata Q-code 'Q0000000' rN)r?r)r=rrrs rq2namerBs5* f % % %C {t CV , , ,,rcNt||}|dSt||S)a Convert simple language name to Wikidata Q-code. Returns None if the name is not found, unless strict=True, in which case a LookupError is raised. >>> lang2q('Low German') 'Q25433' >>> print(lang2q('NonexistentLanguage')) None >>> lang2q('NonexistentLanguage', strict=True) Traceback (most recent call last): ... LookupError: Could not find language code for name 'NonexistentLanguage' rAN)r"r+)rrrs rlang2qrDs3$ D ( ( (D |t f % % %%raaraaabkabafrafakaakamhamaraararganasmasavaavaveaeaymayazeazbakbabambmbelbebenbnbisbibodbobosbsbrebrbulbgcatcacescschachchecechucuchvcvcorkwcoscocrecrcymcydandadeudedivdvdzodzellelengenepoeoesteteuseueweeefaofofasfafijfjfinfifrafrfryfyfulffglagdglegaglgglglvgvgrngngujguhaththauhahbsshhebheherhzhinhihmohohrvhrhunhuhyehyiboigidoioiiiiiikuiuileieinaiaindidipkikislisitaitjavjvjpnjakalklkanknkaskskatkakaukrkazkkkhmkmkikkikinrwkirkykomkvkonkgkorkokuakjkurkulaololatlalavlvlimlilinlnlitltltzlblubluluglgmahmhmalmlmarmrmkdmkmlgmgmltmtmonmnmrimimsamsmyamynaunanavnvnblnrndendndongnepnenldnlnnonnnobnbnornonyanyociocojiojoriorormomossospanpaplipipolplporptpuspsquequrohrmronrorunrnrusrusagsgsansasinsislkskslvslsmesesmosmsnasnsndsdsomsosotstspaessqisqsrdscsrpsrsswsssunsuswaswswesvtahtytamtatatttteltetgktgtgltlthathtirtitontotsntntsotstuktkturtrtwitwuigugukuruzvevivowawoxhyiyozazhzu)ukrurduzbvenvievolwlnwolxhoyidyorzhazhozulfrizWestern Frisianauv AuvergnatgscGasconlmsLimousinlnc Languedocienprvu Provençalamdu Amapá CreolebghBoganbnhuBanawábvszBelgian Sign LanguageccyzSouthern Zhuangcit Chittagonianflmz Falam ChinjapuJaruárakob KohoroxitarimobMoinbamzfAikunhjzTlalitzlipa NahuatlnhszSoutheastern Puebla Nahuatlocc OccidentaltmxTomyangtotzPatla-Chicontla TotonacxmiuMiarrãyibYinglishztczLachirioag ZapotecatfAtuencebqezNavarro-Labourdin BasquebszzSouletin BasqueaexAmeraxaheAheaizAariaknAmikoanaarfArafundiazrAdzerabcxPamonabiiBisubkeBengkulubluz Hmong Njuabocz Bakung KenyahbsdzSarawak BisayabwvzBahau River KenyahbxtBuxinhuabyuBuyangccxzNorthern Zhuangcruu Carútanadatz Darang Dengdykz Land DayakeniEnimfizIzeregenz Geman Denggghz Garreh-AjuranituItutangkdszLahu ShiknhzKayan River Kenyahkrgz North KorowaikrqKruikxgKatinganlmtLematanglntLintanglodBerawanmbguNorthern NambikuáramdozSouthwest Gbayamhv ArakanesemivMimimqdMadangnkyzKhiamniungan NaganxjNyaduognOganorkOrokaivapajz Ipeka-TapuiapeczSouthern PesisirpenPenesakplm Palembangpojz Lower PokomopunPubianraeRanaurjb RajbanshirwsRawassddSemendosdizSindang KelingisklSelakoslbzKahumamahon SaluansrjSerawaisufTarpiasuhSubasuuSungkaiszkSizakitlezSouthern MarakwettnjTanjongttxzTutong 1ubmzUpper Baram Kenyahvkyz Kayu Agungvmoz Muko-MukowreWarexahKahayanxkmzMahakam KenyahxufKunfalyiozDayao YiymjzMuji YiyplzPula YiypwzPuwa Yiywmz Wumeng YiyymzYuanjiang-Mojiang YimlyzMalay (individual language)muwMundarixstzSilt'eopez Old PersiansccSerbianscrCroatianxskSakanmol MoldavianaayAariyaaccu Cubulco Achícbmz Yepocapa Southwestern CakchiquelchsChumashckczNorthern CakchiquelckdzSouth Central CakchiquelckezEastern CakchiquelckfzSouthern Cakchiquelckiu!Santa María De Jesús Cakchiquelckjz Santo Domingo Xenacoj Cakchiquelckkz"Acatenango Southwestern CakchiquelckwzWestern Cakchiquelcnmu Ixtatán Chujctiz Tila CholcunuCunén QuichéemlzEmiliano-Romagnoloeur EuropantogmozGamo-Gofa-DawrohsfzSoutheastern HuastechvauSan Luís Potosí Huastecixiz Nebaj Ixilixjz Chajul IxiljaizWestern Jacaltecommsz Southern Mammpfz Tajumulco MammtzTacanecmvcz Central MammvjuTodos Santos Cuchumatán MampoazEastern PokomampobuWestern PokomchípouzSouthern PokomamppvuPapavôqujuJoyabaj QuichéqutuWest Central QuichéquuuEastern QuichéqxiuSan Andrés Quichésic Malinguatstcz Santa CruztlzzToala'tzbuBachajón TzeltaltzczChamula TzotziltzeuChenalhó TzotziltzsuSan Andrés Larrainzar TzotziltztzWestern TzutujiltzuuHuixtán TzotziltzzuZinacantán TzotzilvlrVatratayuszChan Santa Cruz MayanfgNyengnfkShakaraagpParananbhkzAlbay BicolanobkbFinalligbtbzBeti (Cameroon)cjr ChorotegacmkChimakumdrhDarkhatdrwDarwazigav GabutamonmofzMohegan-Montauk-NarragansettmstzCataelano MandayamytzSangab MandayarmruCalósglzSanglechi-Ishkashimisul Surigaononsumz Sumo-Mayangnatnf TangshewiwgwWagawagaayxz Ayi (China)bjqzSouthern Betsimisaraka MalagasydhazDhanwar (India)dklzKolum So DogonmjaMaheinbfNaxinooNootkatieTingaltkkTakpabazTunenbjd Bandjigaliccq ChaungthackazKhumi Awa Chindapz Nisi (India)dwlzWalo Kumbe Dogonelp ElpaputihgbcGarawagioGelaohrrHoruruibiIbilojarzJarawa (Nigeria)kdvKadokghzUpper Tanudan Kalingakppz Paku KarenkzhzKenuzi-DongolalcqLuhumgxOmatinlnzDurango NahuatlpbzPalupgyPongyongscaSansutlwz South WemaleunpWororawiwWiranguybdYangbyeyenYendangymaYamphedafDandjlDjiwarliggrzAghu TharnggaluilwTalurizizIzi-Ezaa-Ikwo-MgbomegMeamldMalakhelmntMaykulanmwdMudburamyqzForest ManinkanbxNguranlrNgarlapcrPanangpprPirutggTanggawitWintuxiaXiandaoyiyz Yir YorontyosYosemoEmokggmz Gugu MinilegLengualmmLamammhhz Maskoy Pidginpuzz Purum Nagasapu SanapanáyuuYughaamAramanikadpAdapaueuǂKxʼauǁʼeinbmyz$Bemba (Democratic Republic of Congo)bxxz$Borna (Democratic Republic of Congo)byyBuyadzdDazagfxuMangetti Dune ǃXunggtizGbati-riime ImeraguenkbfKakauhuakojz Sara DunjokwqKwakkxeKakihumliiLingkhimmwjMaligonnxNgongounuǃOǃungpmuzMirpur PanjabisgoSongathxThetsfzSouthwestern TamanguokUokhaxsjSubiydszYiddish Sign LanguageymtzMator-Taygi-KaragasynhYanghobgmz Baga MbotenibtlBhatolacbe ChipiajescbhCaguacoyCoyaimacquzChilean QuechuacumCumeraldujDhuwalggnzEastern GurungggozSouthern GondiguvGeyiapIapamaillIranunkgcKassengkoxCoximaktrzKota Marudu TinagaskvsKunggarakzjzCoastal KadazankztzTambunan DusunnadNijadalints NatagaimasomeOmejespmcPalumatapodPonaresppaPaopryzPray 3rnaRunasvrSavaratduzTempasuk Dusunthcz Tai Hang TongtidTidongtmpu Tai MènetnezTinoc KallahantoeTomedesxbazKamba (Brazil)xbxuKabixíxipu Xipináwaxkh KarahawyanayriuYaríjegJengkgdKataangkrmKrimprbzLua'pukzPu KorieRienrsizRennellese Sign LanguageskkSoksnhShinabolsgzLyons Sign LanguagemwxMediakmwyMosironcpNdaktupaisz Nataoran AmisasdAsasditDiraridudz Hun-SaarelbaLuilloKhlormydMarambamyiz Mina (India)nnsNingyeaohArmaayyz Tayabas AytabbzzBabalia Creole Arabicbpb BarbacoasccaCaucacdgChamaridguDegarudrrDororoekczEastern KarnicgliGuligulikjfKhalajkxlz Nepali Kuruxkxuz Kui (India)lmzLumbeenxuNarauplpPalpasdm SemandangtbbTapebaxrqKarrangaxtz TasmanianzirZiriyathwThudambicBikarubijzVaghat-Ya-Bijim-LegeriblgBalaugjiGejiMuyaNgoni PapitalaizIja-ZubaWarapuzJudeo-Tunisian Arabic ChungmbokozLaka (Nigeria)zLango (South Sudan)PiniSamaSebuyauz Kulon-PazehWardujiWyandot)mvmngopatvkiwraajtcuglaklnopiismdsnbuunwrdwya)rF)rF)F)__doc__rewarningsr nltk.corpusrcompilerrr"r+r9r?rBrDrrr#r$r.rrrs!8 bj+,, ,,,,^#C#C#C#CV2 B B B@----6&&&&:y 4y 4y 4y 4 y  4 y  4 y 4y 4y 4y 4y 4y 4y 4y 4y 4y  4!y" 4#yy$ 4%y& 4'y( 4)y* 4+y, 4-y. 4/y0 41y2 43y4 45y6 47y8 49y: 4;y< 4=y> 4?y@ 4AyB 4CyD 4EyyyF 4GyH 4IyJ 4KyL 4MyN 4OyP 4QyR 4SyT 4UyV 4WyX 4YyZ 4[y\ 4]y^ 4_y` 4ayb 4cyd 4eyf 4gyyyh 4iyj 4kyl 4myn 4oyp 4qyr 4syt 4uyv 4wyx 4yyz 4{y| 4}y~ 4y@ 4AyB 4CyD 4EyF 4GyH 4IyyyJ 4KyL 4MyN 4OyP 4QyR 4SyT 4UyV 4WyX 4YyZ 4[y\ 4]y^ 4_y` 4ayb 4cyd 4eyf 4gyh 4iyj 4kyyyl 4myn 4oyp 4qyr 4syt 4uyv 4wyx 4yyz 4{y| 4}y~ 4y@ 4AyB 4CyD 4EyF 4GyH 4IyJ 4KyL 4MyyyN 4OyP 4QyR 4SyT 4UyV 4WyX 4YyZ 4[y\ 4]y^ 4_y` 4ayb 4cyd 4eyf 4gyh 4iyj 4kyl 4myn 4oyyyp 4qyr 4syt 4uyv 4wyx 4yyz 4{y| 4}y~ 4y@ 4AyB 4CyD 4EyF 4GyH 4IyJ 4KyL 4MyN 4OyP 4QyyyR 4SyT 4UyV 4WyX 4YyZ 4[y\ 4]y^ 4_y` 4ayb 4cyd 4eyf 4gyh 4iyj 4kyl 4myn 4oyp 4qyr 4syyyt 4uyv 4wyx 4yyz 4{y| 4}y~ 4y@ 4AyB 4CyD 4EyF 4GyH 4IyJ 4KyL 4MyN 4OyP 4QyR 4SyT 4UyyV              qyyy xu u ;u 8u : u  > u  < u ?u 7u 9u "u u >u <u :u >u  8!u" 6#uu$ %u& ('u( <)u* 9+u, $-u. 9/u0 :1u2 3u4 95u6 %7u8 9u: 8;u< 5=u> 6?u@ :AuB :CuD 8EuuuF 8GuH 6IuJ :KuL MuN OuP 9QuR ;SuT >UuV 8WuX 7YuZ ;[u\ 7]u^ 9_u` aub 8cud euf 9guh 8iuj 6kuuul 9mun 8oup qur 9sut :uuv wux OuP ?QuuuR 9SuT =UuV )WuX YuZ [u\ ]u^ 9_u` aub !cud euf guh ;iuj CuD EuF ;GuH 8IuJ 7KuL 8MuN 7OuP QuR 6SuT "UuV guh 8iuj 9kul 9mun 9oup 8qur 5sut :uuv wux 7yuz {uuu| 5}u~ :u@ :AuB 9CuD EuF 7GuH 8IuJ 8KuL 6MuN 8OuP 7QuR 9SuT _u` 9auuub ;cud 7euf 9guh iuj 9kul 8mun oup qur 5sut 8uuv 8wux 9yuz 8{u| }u~ :u@  A uB  C uuuD  :E uF  U uV  8W uX  6Y uZ  >[ u\  "] u^  ;_ u`  7a ub  9c ud  8e uf  8g uh  i uuuj  :k ul  8m un  >o up  =q ur  8s ut  7u uv  7w ux  ;y uz  8{ u|  :} u~  ; u@  8A uB  8C uD  8E uF  #G uH  7I uJ  6K uuL      "        i uuu p \+ & & !\-00r