| 1 | 'use strict';
|
| 2 | Object.defineProperty(exports, "__esModule", { value: true });
|
| 3 | exports.init = exports.ZhuyinTokenizer = void 0;
|
| 4 | const mod_1 = require("../mod");
|
| 5 | |
| 6 | |
| 7 |
|
| 8 | class ZhuyinTokenizer extends mod_1.SubSModuleTokenizer {
|
| 9 | constructor() {
|
| 10 | super(...arguments);
|
| 11 | this.name = 'ZhuyinTokenizer';
|
| 12 | }
|
| 13 | _cache(...argv) {
|
| 14 | super._cache(...argv);
|
| 15 | }
|
| 16 | split(words) {
|
| 17 | return this._splitUnset(words, this.splitZhuyin);
|
| 18 | }
|
| 19 | splitZhuyin(text, cur) {
|
| 20 | let ret = [];
|
| 21 | let self = this;
|
| 22 | let _r = /[\u31A0-\u31BA\u3105-\u312E]/u;
|
| 23 | if (!_r.test(text)) {
|
| 24 | return null;
|
| 25 | }
|
| 26 | text
|
| 27 | .split(/([\u31A0-\u31BA\u3105-\u312E]+)/u)
|
| 28 | .forEach(function (w, i) {
|
| 29 | if (w !== '') {
|
| 30 | if (_r.test(w)) {
|
| 31 | ret.push(self.debugToken({
|
| 32 | w,
|
| 33 | }, {
|
| 34 | [self.name]: true,
|
| 35 | }, true));
|
| 36 | }
|
| 37 | else {
|
| 38 | ret.push({
|
| 39 | w,
|
| 40 | });
|
| 41 | }
|
| 42 | }
|
| 43 | });
|
| 44 | return ret.length ? ret : null;
|
| 45 | }
|
| 46 | }
|
| 47 | exports.ZhuyinTokenizer = ZhuyinTokenizer;
|
| 48 | exports.init = ZhuyinTokenizer.init.bind(ZhuyinTokenizer);
|
| 49 | exports.default = ZhuyinTokenizer;
|
| 50 | |
| \ | No newline at end of file |