enum TokenTypeEnum {
  SPACE_CHAR = 'spaceChar',
  CHINESE_CHAR = 'chineseChar',
  CHINESE_PUNCTUATION = 'chinesePunctuation',
  ENGLISH_WORD = 'englishWord',
  ENGLISH_PUNCTUATION = 'englishPunctuation',
  UNKNOWN_CHAR = 'unknownChar',
}

interface Token {
  type: TokenTypeEnum
  value: string
}

export type {
  Token,
}

export {
  TokenTypeEnum,
}