| 1 | {"version":3,"file":"SingleTokenizer.js","sourceRoot":"","sources":["SingleTokenizer.ts"],"names":[],"mappings":"AAAA,YAAY,CAAC;;;AAEb,gCAAyD;AAEzD,2CAAiC;AAEjC;;;;;GAKG;AACH,MAAa,eAAgB,SAAQ,yBAAmB;IAGvD;;;;;OAKG;IACH,KAAK,CAAC,KAAc;QAEnB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;QAEnC,IAAI,GAAG,GAAG,EAAE,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAC1C;YACC,IAAI,OAAO,IAAI,CAAC,CAAC,IAAI,WAAW,IAAI,IAAI,CAAC,CAAC,EAC1C;gBACC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aACf;iBAED;gBACC,cAAc;gBACd,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;aAC3C;SACD;QACD,OAAO,GAAG,CAAC;IACZ,CAAC;IAED;;;;;;OAMG;IACH,WAAW,CAAC,IAAI,EAAE,GAAY;QAE7B,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;QAEnC,IAAI,KAAK,CAAC,GAAG,CAAC;YAAE,GAAG,GAAG,CAAC,CAAC;QAExB,IAAI,GAAG,GAAG,CAAC,EACX;YACC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;SACvB;QAED,IAAI,GAAG,GAAY,EAAE,CAAC;QAEtB,oBAAO;aACL,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;aACf,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;YAEtB,GAAG,CAAC,IAAI,CAAC;gBACR,CAAC;gBACD,CAAC,EAAE,MAAM,CAAC,GAAG;aACb,CAAC,CAAC;QACJ,CAAC,CAAC,CACF;QAED,OAAO,GAAG,CAAC;IACZ,CAAC;CACD;AA9DD,0CA8DC;AAEY,QAAA,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAgC,CAAC;AAE9F,kBAAe,eAAe,CAAC","sourcesContent":["'use strict';\n\nimport { SubSModule, SubSModuleTokenizer } from '../mod';\nimport { Segment, IWord } from '../Segment';\nimport UString from 'uni-string';\n\n/**\n * 单字切分模块\n * 此模組不包含模組列表內 需要手動指定\n *\n * @author 老雷<leizongmin@gmail.com>\n */\nexport class SingleTokenizer extends SubSModuleTokenizer\n{\n\n\t/**\n\t * 对未识别的单词进行分词\n\t *\n\t * @param {array} words 单词数组\n\t * @return {array}\n\t */\n\tsplit(words: IWord[]): IWord[]\n\t{\n\t\tconst POSTAG = this.segment.POSTAG;\n\n\t\tlet ret = [];\n\t\tfor (let i = 0, word; word = words[i]; i++)\n\t\t{\n\t\t\tif (typeof word.p == 'undefined' || word.p)\n\t\t\t{\n\t\t\t\tret.push(word);\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\t// 仅对未识别的词进行匹配\n\t\t\t\tret = ret.concat(this.splitSingle(word.w));\n\t\t\t}\n\t\t}\n\t\treturn ret;\n\t}\n\n\t/**\n\t * 单字切分\n\t *\n\t * @param {string} text 要切分的文本\n\t * @param {int} cur 开始位置\n\t * @return {array}\n\t */\n\tsplitSingle(text, cur?: number): IWord[]\n\t{\n\t\tconst POSTAG = this.segment.POSTAG;\n\n\t\tif (isNaN(cur)) cur = 0;\n\n\t\tif (cur > 0)\n\t\t{\n\t\t\ttext = text.slice(cur);\n\t\t}\n\n\t\tlet ret: IWord[] = [];\n\n\t\tUString\n\t\t\t.split(text, '')\n\t\t\t.forEach(function (w, i)\n\t\t\t{\n\t\t\t\tret.push({\n\t\t\t\t\tw,\n\t\t\t\t\tp: POSTAG.UNK,\n\t\t\t\t});\n\t\t\t})\n\t\t;\n\n\t\treturn ret;\n\t}\n}\n\nexport const init = SingleTokenizer.init.bind(SingleTokenizer) as typeof SingleTokenizer.init;\n\nexport default SingleTokenizer;\n"]} |