| 1 | {"version":3,"file":"CharStreams.js","sourceRoot":"","sources":["../../src/CharStreams.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,uDAAoD;AACpD,+DAA4D;AAC5D,2CAAwC;AAExC,4CAA4C;AAE5C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AACH,IAAiB,WAAW,CA+Q3B;AA/QD,WAAiB,WAAW;IAC3B,MAAM;IACN,0DAA0D;IAC1D,2BAA2B;IAC3B,KAAK;IACL,6EAA6E;IAC7E,MAAM;IACN,oDAAoD;IACpD,sEAAsE;IACtE,wEAAwE;IACxE,gCAAgC;IAChC,wCAAwC;IACxC,KAAK;IAuLL,SAAgB,UAAU,CAAC,CAAS,EAAE,UAAmB;QACxD,IAAI,UAAU,KAAK,SAAS,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE;YACxD,UAAU,GAAG,qBAAS,CAAC,mBAAmB,CAAC;SAC3C;QAED,0DAA0D;QAC1D,yCAAyC;QACzC,IAAI,sBAAsB,GAA4B,iCAAe,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAExF,sEAAsE;QACtE,qDAAqD;QACrD,IAAI,EAAE,GAAgB,IAAI,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAChD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAClC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;SACxB;QAED,sBAAsB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAClC,OAAO,yCAAmB,CAAC,UAAU,CAAC,sBAAsB,CAAC,KAAK,EAAE,EAAE,UAAU,CAAC,CAAC;IACnF,CAAC;IAlBe,sBAAU,aAkBzB,CAAA;IAED,qCAAqC;IACrC,iCAAiC;IACjC,qBAAqB;IACrB,uBAAuB;IACvB,2CAA2C;IAC3C,yCAAyC;IACzC,SAAS;IACT,8DAA8D;IAC9D,sEAAsE;IACtE,4BAA4B;IAC5B,6BAA6B;IAC7B,gDAAgD;IAChD,kDAAkD;IAClD,0FAA0F;IAC1F,MAAM;IAEN,8FAA8F;IAC9F,0CAA0C;IAC1C,oBAAoB;IACpB,6CAA6C;IAC7C,mDAAmD;IAEnD,qCAAqC;IACrC,0BAA0B;IAC1B,wDAAwD;IACxD,sCAAsC;IACtC,yBAAyB;IACzB,+CAA+C;IAC/C,mBAAmB;IACnB,yBAAyB;IACzB,mBAAmB;IACnB,iFAAiF;IACjF,+BAA+B;IAC/B,OAAO;IAEP,+BAA+B;IAC/B,uDAAuD;IACvD,4BAA4B;IAC5B,kCAAkC;IAClC,MAAM;IACN,6DAA6D;IAC7D,4DAA4D;IAC5D,qEAAqE;IACrE,qFAAqF;IACrF,mCAAmC;IACnC,MAAM;IAEN,8BAA8B;IAC9B,sDAAsD;IAEtD,2CAA2C;IAC3C,KAAK;IACL,aAAa;IACb,qBAAqB;IACrB,KAAK;IACL,IAAI;AACL,CAAC,EA/QgB,WAAW,GAAX,mBAAW,KAAX,mBAAW,QA+Q3B","sourcesContent":["/*!\r\n * Copyright 2016 The ANTLR Project. All rights reserved.\r\n * Licensed under the BSD-3-Clause license. See LICENSE file in the project root for license information.\r\n */\r\n\r\nimport { CodePointBuffer } from \"./CodePointBuffer\";\r\nimport { CodePointCharStream } from \"./CodePointCharStream\";\r\nimport { IntStream } from \"./IntStream\";\r\n\r\n// const DEFAULT_BUFFER_SIZE: number = 4096;\r\n\r\n/** This class represents the primary interface for creating {@link CharStream}s\r\n * from a variety of sources as of 4.7. The motivation was to support\r\n * Unicode code points > U+FFFF. {@link ANTLRInputStream} and\r\n * {@link ANTLRFileStream} are now deprecated in favor of the streams created\r\n * by this interface.\r\n *\r\n * DEPRECATED: {@code new ANTLRFileStream(\"myinputfile\")}\r\n * NEW: {@code CharStreams.fromFileName(\"myinputfile\")}\r\n *\r\n * WARNING: If you use both the deprecated and the new streams, you will see\r\n * a nontrivial performance degradation. This speed hit is because the\r\n * {@link Lexer}'s internal code goes from a monomorphic to megamorphic\r\n * dynamic dispatch to get characters from the input stream. Java's\r\n * on-the-fly compiler (JIT) is unable to perform the same optimizations\r\n * so stick with either the old or the new streams, if performance is\r\n * a primary concern. See the extreme debugging and spelunking\r\n * needed to identify this issue in our timing rig:\r\n *\r\n * https://github.com/antlr/antlr4/pull/1781\r\n *\r\n * The ANTLR character streams still buffer all the input when you create\r\n * the stream, as they have done for ~20 years. If you need unbuffered\r\n * access, please note that it becomes challenging to create\r\n * parse trees. The parse tree has to point to tokens which will either\r\n * point into a stale location in an unbuffered stream or you have to copy\r\n * the characters out of the buffer into the token. That defeats the purpose\r\n * of unbuffered input. Per the ANTLR book, unbuffered streams are primarily\r\n * useful for processing infinite streams *during the parse.*\r\n *\r\n * The new streams also use 8-bit buffers when possible so this new\r\n * interface supports character streams that use half as much memory\r\n * as the old {@link ANTLRFileStream}, which assumed 16-bit characters.\r\n *\r\n * A big shout out to Ben Hamilton (github bhamiltoncx) for his superhuman\r\n * efforts across all targets to get true Unicode 3.1 support for U+10FFFF.\r\n *\r\n * @since 4.7\r\n */\r\nexport namespace CharStreams {\r\n\t// /**\r\n\t// * Creates a {@link CharStream} given a path to a UTF-8\r\n\t// * encoded file on disk.\r\n\t// *\r\n\t// * Reads the entire contents of the file into the result before returning.\r\n\t// */\r\n\t// export function fromFile(file: File): CharStream;\r\n\t// export function fromFile(file: File, charset: Charset): CharStream;\r\n\t// export function fromFile(file: File, charset?: Charset): CharStream {\r\n\t// \tif (charset === undefined) {\r\n\t// \t\tcharset = Charset.forName(\"UTF-8\");\r\n\t// \t}\r\n\r\n\t// \tlet size: number = file.length();\r\n\t// \treturn fromStream(new FileInputStream(file), charset, file.toString(), size);\r\n\t// }\r\n\r\n\t// /**\r\n\t// * Creates a {@link CharStream} given a string containing a\r\n\t// * path to a UTF-8 file on disk.\r\n\t// *\r\n\t// * Reads the entire contents of the file into the result before returning.\r\n\t// */\r\n\t// export function fromFileName(fileName: string): CharStream;\r\n\r\n\t// /**\r\n\t// * Creates a {@link CharStream} given a string containing a\r\n\t// * path to a file on disk and the charset of the bytes\r\n\t// * contained in the file.\r\n\t// *\r\n\t// * Reads the entire contents of the file into the result before returning.\r\n\t// */\r\n\t// export function fromFileName(fileName: string, charset: Charset): CharStream;\r\n\t// export function fromFileName(fileName: string, charset?: Charset): CharStream {\r\n\t// \tif (charset === undefined) {\r\n\t// \t\tcharset = Charset.forName(\"UTF-8\");\r\n\t// \t}\r\n\r\n\t// \treturn fromFile(new File(fileName), charset);\r\n\t// }\r\n\r\n\t// /**\r\n\t// * Creates a {@link CharStream} given an opened {@link InputStream}\r\n\t// * containing UTF-8 bytes.\r\n\t// *\r\n\t// * Reads the entire contents of the {@code InputStream} into\r\n\t// * the result before returning, then closes the {@code InputStream}.\r\n\t// */\r\n\t// export function fromStream(is: InputStream): CharStream;\r\n\r\n\t// /**\r\n\t// * Creates a {@link CharStream} given an opened {@link InputStream} and the\r\n\t// * charset of the bytes contained in the stream.\r\n\t// *\r\n\t// * Reads the entire contents of the {@code InputStream} into\r\n\t// * the result before returning, then closes the {@code InputStream}.\r\n\t// */\r\n\t// export function fromStream(is: InputStream, charset: Charset): CharStream;\r\n\r\n\t// export function fromStream(is: InputStream, charset: Charset, sourceName: string, inputSize: number): CharStream;\r\n\t// export function fromStream(is: InputStream, charset?: Charset, sourceName?: string, inputSize?: number): CharStream {\r\n\t// \tif (charset === undefined) {\r\n\t// \t\tcharset = Charset.forName(\"UTF-8\");\r\n\t// \t}\r\n\r\n\t// \tif (sourceName === undefined) {\r\n\t// \t\tsourceName = IntStream.UNKNOWN_SOURCE_NAME;\r\n\t// \t}\r\n\r\n\t// \tif (inputSize === undefined) {\r\n\t// \t\tinputSize = -1;\r\n\t// \t}\r\n\r\n\t// \treturn fromChannel(\r\n\t// \t\tChannels.newChannel(is),\r\n\t// \t\tcharset,\r\n\t// \t\tDEFAULT_BUFFER_SIZE,\r\n\t// \t\tCodingErrorAction.REPLACE,\r\n\t// \t\tsourceName,\r\n\t// \t\tinputSize);\r\n\t// }\r\n\r\n\t// /**\r\n\t// * Creates a {@link CharStream} given an opened {@link ReadableByteChannel}\r\n\t// * containing UTF-8 bytes.\r\n\t// *\r\n\t// * Reads the entire contents of the {@code channel} into\r\n\t// * the result before returning, then closes the {@code channel}.\r\n\t// */\r\n\t// export function fromChannel(channel: ReadableByteChannel): CharStream;\r\n\r\n\t// /**\r\n\t// * Creates a {@link CharStream} given an opened {@link ReadableByteChannel} and the\r\n\t// * charset of the bytes contained in the channel.\r\n\t// *\r\n\t// * Reads the entire contents of the {@code channel} into\r\n\t// * the result before returning, then closes the {@code channel}.\r\n\t// */\r\n\t// export function fromChannel(channel: ReadableByteChannel, charset: Charset): CharStream;\r\n\r\n\t// /**\r\n\t// * Creates a {@link CharStream} given an opened {@link ReadableByteChannel}\r\n\t// * containing UTF-8 bytes.\r\n\t// *\r\n\t// * Reads the entire contents of the {@code channel} into\r\n\t// * the result before returning, then closes the {@code channel}.\r\n\t// */\r\n\t// export function fromChannel(\r\n\t// \tchannel: ReadableByteChannel,\r\n\t// \tcharset: Charset,\r\n\t// \tbufferSize: number,\r\n\t// \tdecodingErrorAction: CodingErrorAction,\r\n\t// \tsourceName: string): CodePointCharStream;\r\n\r\n\t// export function fromChannel(\r\n\t// \tchannel: ReadableByteChannel,\r\n\t// \tcharset: Charset,\r\n\t// \tbufferSize: number,\r\n\t// \tdecodingErrorAction: CodingErrorAction,\r\n\t// \tsourceName: string,\r\n\t// \tinputSize: number): CodePointCharStream;\r\n\t// export function fromChannel(\r\n\t// \tchannel: ReadableByteChannel,\r\n\t// \tcharset?: Charset,\r\n\t// \tbufferSize?: number,\r\n\t// \tdecodingErrorAction?: CodingErrorAction,\r\n\t// \tsourceName?: string,\r\n\t// \tinputSize?: number): CodePointCharStream\r\n\t// {\r\n\t// \tif (charset === undefined) {\r\n\t// \t\tcharset = Charset.forName(\"UTF-8\");\r\n\t// \t}\r\n\r\n\t// \tif (bufferSize === undefined) {\r\n\t// \t\tbufferSize = DEFAULT_BUFFER_SIZE;\r\n\t// \t}\r\n\r\n\t// \tif (decodingErrorAction === undefined) {\r\n\t// \t\tdecodingErrorAction = CodingErrorAction.REPLACE;\r\n\t// \t}\r\n\r\n\t// \tif (sourceName === undefined || sourceName.length === 0) {\r\n\t// \t\tsourceName = IntStream.UNKNOWN_SOURCE_NAME;\r\n\t// \t}\r\n\r\n\t// \tif (inputSize === undefined) {\r\n\t// \t\tinputSize = -1;\r\n\t// \t}\r\n\r\n\t// \tlet codePointBuffer: CodePointBuffer = bufferFromChannel(channel, charset, bufferSize, decodingErrorAction, inputSize);\r\n\t// \treturn CodePointCharStream.fromBuffer(codePointBuffer, sourceName);\r\n\t// }\r\n\r\n\t// /**\r\n\t// * Creates a {@link CharStream} given a {@link Reader}. Closes\r\n\t// * the reader before returning.\r\n\t// */\r\n\t// export function fromReader(r: Reader): CodePointCharStream;\r\n\r\n\t// /**\r\n\t// * Creates a {@link CharStream} given a {@link Reader} and its\r\n\t// * source name. Closes the reader before returning.\r\n\t// */\r\n\t// export function fromReader(r: Reader, sourceName: string): CodePointCharStream;\r\n\t// export function fromReader(r: Reader, sourceName?: string): CodePointCharStream {\r\n\t// \tif (sourceName === undefined) {\r\n\t// \t\tsourceName = IntStream.UNKNOWN_SOURCE_NAME;\r\n\t// \t}\r\n\r\n\t// \ttry {\r\n\t// \t\tlet codePointBufferBuilder: CodePointBuffer.Builder = CodePointBuffer.builder(DEFAULT_BUFFER_SIZE);\r\n\t// \t\tlet charBuffer: CharBuffer = CharBuffer.allocate(DEFAULT_BUFFER_SIZE);\r\n\t// \t\twhile ((r.read(charBuffer)) !== -1) {\r\n\t// \t\t\tcharBuffer.flip();\r\n\t// \t\t\tcodePointBufferBuilder.append(charBuffer);\r\n\t// \t\t\tcharBuffer.compact();\r\n\t// \t\t}\r\n\r\n\t// \t\treturn CodePointCharStream.fromBuffer(codePointBufferBuilder.build(), sourceName);\r\n\t// \t} finally {\r\n\t// \t\tr.close();\r\n\t// \t}\r\n\t// }\r\n\r\n\t/**\r\n\t * Creates a {@link CharStream} given a {@link String}.\r\n\t */\r\n\texport function fromString(s: string): CodePointCharStream;\r\n\r\n\t/**\r\n\t * Creates a {@link CharStream} given a {@link String} and the {@code sourceName}\r\n\t * from which it came.\r\n\t */\r\n\texport function fromString(s: string, sourceName: string): CodePointCharStream;\r\n\texport function fromString(s: string, sourceName?: string): CodePointCharStream {\r\n\t\tif (sourceName === undefined || sourceName.length === 0) {\r\n\t\t\tsourceName = IntStream.UNKNOWN_SOURCE_NAME;\r\n\t\t}\r\n\r\n\t\t// Initial guess assumes no code points > U+FFFF: one code\r\n\t\t// point for each code unit in the string\r\n\t\tlet codePointBufferBuilder: CodePointBuffer.Builder = CodePointBuffer.builder(s.length);\r\n\r\n\t\t// TODO: CharBuffer.wrap(String) rightfully returns a read-only buffer\r\n\t\t// which doesn't expose its array, so we make a copy.\r\n\t\tlet cb: Uint16Array = new Uint16Array(s.length);\r\n\t\tfor (let i = 0; i < s.length; i++) {\r\n\t\t\tcb[i] = s.charCodeAt(i);\r\n\t\t}\r\n\r\n\t\tcodePointBufferBuilder.append(cb);\r\n\t\treturn CodePointCharStream.fromBuffer(codePointBufferBuilder.build(), sourceName);\r\n\t}\r\n\r\n\t// export function bufferFromChannel(\r\n\t// \tchannel: ReadableByteChannel,\r\n\t// \tcharset: Charset,\r\n\t// \tbufferSize: number,\r\n\t// \tdecodingErrorAction: CodingErrorAction,\r\n\t// \tinputSize: number): CodePointBuffer {\r\n\t// \ttry {\r\n\t// \t\tlet utf8BytesIn: Uint8Array = new Uint8Array(bufferSize);\r\n\t// \t\tlet utf16CodeUnitsOut: Uint16Array = new Uint16Array(bufferSize);\r\n\t// \t\tif (inputSize === -1) {\r\n\t// \t\t\tinputSize = bufferSize;\r\n\t// \t\t} else if (inputSize > Integer.MAX_VALUE) {\r\n\t// \t\t\t// ByteBuffer et al don't support long sizes\r\n\t// \t\t\tthrow new RangeError(`inputSize ${inputSize} larger than max ${Integer.MAX_VALUE}`);\r\n\t// \t\t}\r\n\r\n\t// \t\tlet codePointBufferBuilder: CodePointBuffer.Builder = CodePointBuffer.builder(inputSize);\r\n\t// \t\tlet decoder: CharsetDecoder = charset\r\n\t// \t\t\t\t.newDecoder()\r\n\t// \t\t\t\t.onMalformedInput(decodingErrorAction)\r\n\t// \t\t\t\t.onUnmappableCharacter(decodingErrorAction);\r\n\r\n\t// \t\tlet endOfInput: boolean = false;\r\n\t// \t\twhile (!endOfInput) {\r\n\t// \t\t\tlet bytesRead: number = channel.read(utf8BytesIn);\r\n\t// \t\t\tendOfInput = (bytesRead === -1);\r\n\t// \t\t\tutf8BytesIn.flip();\r\n\t// \t\t\tlet result: CoderResult = decoder.decode(\r\n\t// \t\t\t\tutf8BytesIn,\r\n\t// \t\t\t\tutf16CodeUnitsOut,\r\n\t// \t\t\t\tendOfInput);\r\n\t// \t\t\tif (result.isError() && decodingErrorAction === CodingErrorAction.REPORT) {\r\n\t// \t\t\t\tresult.throwException();\r\n\t// \t\t\t}\r\n\r\n\t// \t\t\tutf16CodeUnitsOut.flip();\r\n\t// \t\t\tcodePointBufferBuilder.append(utf16CodeUnitsOut);\r\n\t// \t\t\tutf8BytesIn.compact();\r\n\t// \t\t\tutf16CodeUnitsOut.compact();\r\n\t// \t\t}\r\n\t// \t\t// Handle any bytes at the end of the file which need to\r\n\t// \t\t// be represented as errors or substitution characters.\r\n\t// \t\tlet flushResult: CoderResult = decoder.flush(utf16CodeUnitsOut);\r\n\t// \t\tif (flushResult.isError() && decodingErrorAction === CodingErrorAction.REPORT) {\r\n\t// \t\t\tflushResult.throwException();\r\n\t// \t\t}\r\n\r\n\t// \t\tutf16CodeUnitsOut.flip();\r\n\t// \t\tcodePointBufferBuilder.append(utf16CodeUnitsOut);\r\n\r\n\t// \t\treturn codePointBufferBuilder.build();\r\n\t// \t}\r\n\t// \tfinally {\r\n\t// \t\tchannel.close();\r\n\t// \t}\r\n\t// }\r\n}\r\n"]} |