104class Cutlet:
105    def __init__(
106        self,
107        system="hepburn",
108        use_foreign_spelling=True,
109        ensure_ascii=True,
110        mecab_args="",
111    ):
112        """Create a Cutlet object, which holds configuration as well as
113        tokenizer state.
114
115        `system` is `hepburn` by default, and may also be `kunrei` or
116        `nihon`. `nippon` is permitted as a synonym for `nihon`.
117
118        If `use_foreign_spelling` is true, output will use the foreign spelling
119        provided in a UniDic lemma when available. For example, "カツ" will
120        become "cutlet" instead of "katsu".
121
122        If `ensure_ascii` is true, any non-ASCII characters that can't be
123        romanized will be replaced with `?`. If false, they will be passed
124        through.
125
126        Typical usage:
127
128        ```python
129        katsu = Cutlet()
130        roma = katsu.romaji("カツカレーを食べた")
131        # "Cutlet curry wo tabeta"
132        ```
133        """
134        # allow 'nippon' for 'nihon'
135        if system == "nippon":
136            system = "nihon"
137        self.system = system
138        try:
139            # make a copy so we can modify it
140            self.table = dict(SYSTEMS[system])
141        except KeyError:
142            print("unknown system: {}".format(system))
143            raise
144
145        self.tagger = fugashi.Tagger(mecab_args)
146        self.exceptions = load_exceptions()
147
148        # these are too minor to be worth exposing as arguments
149        self.use_tch = self.system in ("hepburn",)
150        self.use_wa = self.system in ("hepburn", "kunrei")
151        self.use_he = self.system in ("nihon",)
152        self.use_wo = self.system in ("hepburn", "nihon")
153
154        self.use_foreign_spelling = use_foreign_spelling
155        self.ensure_ascii = ensure_ascii
156
157    def add_exception(self, key, val):
158        """Add an exception to the internal list.
159
160        An exception overrides a whole token, for example to replace "Toukyou"
161        with "Tokyo". Note that it must match the tokenizer output and be a
162        single token to work. To replace longer phrases, you'll need to use a
163        different strategy, like string replacement.
164        """
165        self.exceptions[key] = val
166
167    def update_mapping(self, key, val):
168        """Update mapping table for a single kana.
169
170        This can be used to mix common systems, or to modify particular
171        details. For example, you can use `update_mapping("ぢ", "di")` to
172        differentiate ぢ and じ in Hepburn.
173
174        Example usage:
175
176        ```
177        cut = Cutlet()
178        cut.romaji("お茶漬け") # Ochazuke
179        cut.update_mapping("づ", "du")
180        cut.romaji("お茶漬け") # Ochaduke
181        ```
182        """
183        self.table[key] = val
184
185    def slug(self, text):
186        """Generate a URL-friendly slug.
187
188        After converting the input to romaji using `Cutlet.romaji` and making
189        the result lower-case, any runs of non alpha-numeric characters are
190        replaced with a single hyphen. Any leading or trailing hyphens are
191        stripped.
192        """
193        roma = self.romaji(text).lower()
194        slug = re.sub(r"[^a-z0-9]+", "-", roma).strip("-")
195        return slug
196
197    def romaji_tokens(self, words, capitalize=True, title=False):
198        """Build a list of tokens from input nodes.
199
200        If `capitalize` is true, then the first letter of the first token will be
201        capitalized. This is typically the desired behavior if the input is a
202        complete sentence.
203
204        If `title` is true, then words will be capitalized as in a book title.
205        This means most words will be capitalized, but some parts of speech
206        (particles, endings) will not.
207
208        If the text was not normalized before being tokenized, the output is
209        undefined. For details of normalization, see `normalize_text`.
210
211        The number of output tokens will equal the number of input nodes.
212        """
213
214        out = []
215
216        for wi, word in enumerate(words):
217            po = out[-1] if out else None
218            pw = words[wi - 1] if wi > 0 else None
219            nw = words[wi + 1] if wi < len(words) - 1 else None
220
221            # handle possessive apostrophe as a special case
222            if (
223                word.surface == "'"
224                and (nw and nw.char_type == CHAR_ALPHA and not nw.white_space)
225                and not word.white_space
226            ):
227                # remove preceeding space
228                if po:
229                    po.space = False
230                out.append(Token(word.surface, False))
231                continue
232
233            # resolve split verbs / adjectives
234            roma = self.romaji_word(word)
235            if roma and po and po.surface and po.surface[-1] == "っ":
236                po.surface = po.surface[:-1] + roma[0]
237            if word.feature.pos2 == "固有名詞":
238                roma = roma.title()
239            if (
240                title
241                and word.feature.pos1 not in ("助詞", "助動詞", "接尾辞")
242                and not (pw and pw.feature.pos1 == "接頭辞")
243            ):
244                roma = roma.title()
245
246            foreign = self.use_foreign_spelling and has_foreign_lemma(word)
247            tok = Token(roma, False, foreign)
248            # handle punctuation with atypical spacing
249            if word.surface in "「『":
250                if po:
251                    po.space = True
252                out.append(tok)
253                continue
254            if roma in "([":
255                if po:
256                    po.space = True
257                out.append(tok)
258                continue
259            if roma == "/":
260                out.append(tok)
261                continue
262
263            # preserve spaces between ascii tokens
264            if word.surface.isascii() and nw and nw.surface.isascii():
265                use_space = bool(nw.white_space)
266                out.append(Token(word.surface, use_space))
267                continue
268
269            out.append(tok)
270
271            # no space sometimes
272            # お酒 -> osake
273            if word.feature.pos1 == "接頭辞":
274                continue
275            # 今日、 -> kyou, ; 図書館 -> toshokan
276            if nw and nw.feature.pos1 in ("補助記号", "接尾辞"):
277                continue
278            # special case for half-width commas
279            if nw and nw.surface == ",":
280                continue
281            # special case for prefixes
282            if foreign and roma[-1] == "-":
283                continue
284            # 思えば -> omoeba
285            if nw and nw.feature.pos2 in ("接続助詞"):
286                continue
287            # 333 -> 333 ; this should probably be handled in mecab
288            if word.surface.isdigit() and nw and nw.surface.isdigit():
289                continue
290            # そうでした -> sou deshita
291            if (
292                nw
293                and word.feature.pos1 in ("動詞", "助動詞", "形容詞")
294                and nw.feature.pos1 == "助動詞"
295                and nw.surface != "です"
296            ):
297                continue
298
299            # if we get here, it does need a space
300            tok.space = True
301
302        # remove any leftover っ
303        for tok in out:
304            tok.surface = tok.surface.replace("っ", "")
305
306        # capitalize the first letter
307        if capitalize and out and out[0].surface:
308            ss = out[0].surface
309            out[0].surface = ss[0].capitalize() + ss[1:]
310        return out
311
312    def romaji(self, text, capitalize=True, title=False):
313        """Build a complete string from input text.
314
315        If `capitalize` is true, then the first letter of the text will be
316        capitalized. This is typically the desired behavior if the input is a
317        complete sentence.
318
319        If `title` is true, then words will be capitalized as in a book title.
320        This means most words will be capitalized, but some parts of speech
321        (particles, endings) will not.
322        """
323        if not text:
324            return ""
325
326        text = normalize_text(text)
327        words = self.tagger(text)
328
329        tokens = self.romaji_tokens(words, capitalize, title)
330        out = "".join([str(tok) for tok in tokens]).strip()
331        return out
332
333    def romaji_word(self, word):
334        """Return the romaji for a single word (node)."""
335
336        if word.surface in self.exceptions:
337            return self.exceptions[word.surface]
338
339        if word.surface.isdigit():
340            return word.surface
341
342        if word.surface.isascii():
343            return word.surface
344
345        # deal with exceptional non-ascii punctuation
346        # may be better to have a more general strategy for this
347        # see #70
348        if word.surface in ("€", "¥"):
349            if self.ensure_ascii:
350                return ""
351            else:
352                return word.surface
353
354        # deal with unks first
355        if word.is_unk:
356            # at this point is is presumably an unk
357            # Check character type using the values defined in char.def.
358            # This is constant across unidic versions so far but not guaranteed.
359            if word.char_type in (CHAR_HIRAGANA, CHAR_KATAKANA):
360                kana = jaconv.kata2hira(word.surface)
361                return self.map_kana(kana)
362
363            # At this point this is an unknown word and not kana. Could be
364            # unknown kanji, could be hangul, cyrillic, something else.
365            # By default ensure ascii by replacing with ?, but allow pass-through.
366            if self.ensure_ascii:
367                out = "?" * len(word.surface)
368                return out
369            else:
370                return word.surface
371
372        if word.feature.pos1 == "補助記号":
373            # If it's punctuation we don't recognize, just discard it
374            return self.table.get(word.surface, "")
375        elif self.use_wa and word.feature.pos1 == "助詞" and word.feature.pron == "ワ":
376            return "wa"
377        elif (
378            not self.use_he
379            and word.feature.pos1 == "助詞"
380            and word.feature.pron == "エ"
381        ):
382            return "e"
383        elif (
384            not self.use_wo
385            and word.feature.pos1 == "助詞"
386            and word.feature.pron == "オ"
387        ):
388            return "o"
389        elif self.use_foreign_spelling and has_foreign_lemma(word):
390            # this is a foreign word with known spelling
391            return word.feature.lemma.split("-", 1)[-1]
392        elif word.feature.kana:
393            # for known words
394            kana = jaconv.kata2hira(word.feature.kana)
395            return self.map_kana(kana)
396        else:
397            # unclear when we would actually get here
398            return word.surface
399
400    def map_kana(self, kana):
401        """Given a list of kana, convert them to romaji.
402
403        The exact romaji resulting from a kana sequence depend on the preceding
404        or following kana, so this handles that conversion.
405        """
406        out = ""
407        for ki, char in enumerate(kana):
408            nk = kana[ki + 1] if ki < len(kana) - 1 else None
409            pk = kana[ki - 1] if ki > 0 else None
410            out += self.get_single_mapping(pk, char, nk)
411        return out
412
413    def get_single_mapping(self, pk, kk, nk):
414        """Given a single kana and its neighbors, return the mapped romaji."""
415        # handle odoriji
416        # NOTE: This is very rarely useful at present because odoriji are not
417        # left in readings for dictionary words, and we can't follow kana
418        # across word boundaries.
419        if kk in ODORI:
420            if kk in "ゝヽ":
421                if pk:
422                    return pk
423                else:
424                    return ""  # invalid but be nice
425            if kk in "ゞヾ":  # repeat with voicing
426                if not pk:
427                    return ""
428                vv = add_dakuten(pk)
429                if vv:
430                    return self.table[vv]
431                else:
432                    return ""
433            # remaining are 々 for kanji and 〃 for symbols, but we can't
434            # infer their span reliably (or handle rendaku)
435            return ""
436
437        # handle digraphs
438        if pk and (pk + kk) in self.table:
439            return self.table[pk + kk]
440        if nk and (kk + nk) in self.table:
441            return ""
442
443        if nk and nk in SUTEGANA:
444            if kk == "っ":
445                return ""  # never valid, just ignore
446            return self.table[kk][:-1] + self.table[nk]
447        if kk in SUTEGANA:
448            return ""
449
450        if kk == "ー":  # 長音符
451            if pk and pk in self.table:
452                return self.table[pk][-1]
453            else:
454                return "-"
455
456        if kk == "っ":
457            if nk:
458                if self.use_tch and nk == "ち":
459                    return "t"
460                elif nk in "あいうえおっ":
461                    return "-"
462                else:
463                    return self.table[nk][0]  # first character
464            else:
465                # seems like it should never happen, but 乗っ|た is two tokens
466                # so leave this as is and pick it up at the word level
467                return "っ"
468
469        if kk == "ん":
470            if nk and nk in "あいうえおやゆよ":
471                return "n'"
472            else:
473                return "n"
474
475        return self.table[kk]