1 |
|
2 |
|
3 |
|
4 | import { SubSModuleTokenizer } from '../mod';
|
5 | import { IWord } from '../Segment';
|
6 | import { IWordDebug } from '../util';
|
7 | export declare const enum EnumJpSimpleTokenizerType {
|
8 | |
9 |
|
10 |
|
11 |
|
12 | HIRAGANA = 1,
|
13 | |
14 |
|
15 |
|
16 |
|
17 | KATAKANA = 2
|
18 | }
|
19 | export declare class JpSimpleTokenizer extends SubSModuleTokenizer {
|
20 | static NAME: "JpSimpleTokenizer";
|
21 | name: "JpSimpleTokenizer";
|
22 | split(words: IWord[], ...argv: any[]): IWord[];
|
23 | protected createJpSimpleToken<T extends IWordDebug>(data: T, type: EnumJpSimpleTokenizerType): T;
|
24 | protected _splitText(text: string): IWord[];
|
25 | }
|
26 | export declare const init: typeof SubSModuleTokenizer.init;
|
27 | export default JpSimpleTokenizer;
|