1 line
11 KiB
Plaintext
1 line
11 KiB
Plaintext
{"version":3,"file":"TokenReader.js","sourceRoot":"","sources":["../../src/parser/TokenReader.ts"],"names":[],"mappings":";;;AAAA,iCAA2C;AAC3C,iDAAgD;AAGhD;;;;;;;;;GASG;AACH;IAgBE,qBAAmB,aAA4B,EAAE,qBAAqC;QACpF,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,IAAI,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;QAEnC,IAAI,qBAAqB,EAAE;YACzB,IAAI,qBAAqB,CAAC,aAAa,KAAK,IAAI,CAAC,cAAc,EAAE;gBAC/D,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;aAC/E;YACD,IAAI,CAAC,iBAAiB,GAAG,qBAAqB,CAAC,UAAU,CAAC;YAC1D,IAAI,CAAC,eAAe,GAAG,qBAAqB,CAAC,QAAQ,CAAC;SACvD;aAAM;YACL,IAAI,CAAC,iBAAiB,GAAG,CAAC,CAAC;YAC3B,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;SAC3C;QAED,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,iBAAiB,CAAC;QAC5C,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC,iBAAiB,CAAC;IACvD,CAAC;IAED;;;OAGG;IACI,gDAA0B,GAAjC;QACE,IAAI,IAAI,CAAC,sBAAsB,KAAK,IAAI,CAAC,aAAa,EAAE;YACtD,8CAA8C;YAC9C,MAAM,IAAI,KAAK,CACb,6DAA6D;gBAC3D,yCAAyC,CAC5C,CAAC;SACH;QAED,IAAM,QAAQ,GAAkB,IAAI,6BAAa,CAAC;YAChD,aAAa,EAAE,IAAI,CAAC,cAAc;YAClC,UAAU,EAAE,IAAI,CAAC,sBAAsB;YACvC,QAAQ,EAAE,IAAI,CAAC,aAAa;SAC7B,CAAC,CAAC;QAEH,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC,aAAa,CAAC;QAEjD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;;;;OAKG;IACI,gDAA0B,GAAjC;QACE,OAAO,IAAI,CAAC,sBAAsB,KAAK,IAAI,CAAC,aAAa,CAAC;IAC5D,CAAC;IAED;;;OAGG;IACI,mDAA6B,GAApC;QACE,IAAI,IAAI,CAAC,0BAA0B,EAAE,EAAE;YACrC,OAAO,SAAS,CAAC;SAClB;QACD,OAAO,IAAI,CAAC,0BAA0B,EAAE,CAAC;IAC3C,CAAC;IAED;;;OAGG;IACI,sDAAgC,GAAvC;QACE,IAAI,CAAC,IAAI,CAAC,0BAA0B,EAAE,EAAE;YACtC,8CAA8C;YAC9C,IAAM,QAAQ,GAAkB,IAAI,6BAAa,CAAC;gBAChD,aAAa,EAAE,IAAI,CAAC,cAAc;gBAClC,UAAU,EAAE,IAAI,CAAC,sBAAsB;gBACvC,QAAQ,EAAE,IAAI,CAAC,aAAa;aAC7B,CAAC,CAAC;YACH,IAAM,YAAY,GAAa,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,UAAC,CAAC,IAAK,OAAA,CAAC,CAAC,QAAQ,EAAE,EAAZ,CAAY,CAAC,CAAC;YACxE,MAAM,IAAI,KAAK,CACb,wEAAwE;gBACtE,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,CAC/B,CAAC;SACH;IACH,CAAC;IAED;;;OAGG;IACI,+BAAS,GAAhB;QACE,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IACzC,CAAC;IAED;;;OAGG;IACI,mCAAa,GAApB;QACE,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,eAAe,EAAE;YAC9C,OAAO,iBAAS,CAAC,UAAU,CAAC;SAC7B;QACD,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC;IAC9C,CAAC;IAED;;OAEG;IACI,wCAAkB,GAAzB;QACE,IAAI,IAAI,CAAC,aAAa,GAAG,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE;YAClD,OAAO,iBAAS,CAAC,UAAU,CAAC;SAC7B;QACD,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;IAClD,CAAC;IAED;;OAEG;IACI,6CAAuB,GAA9B;QACE,IAAI,IAAI,CAAC,aAAa,GAAG,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE;YAClD,OAAO,iBAAS,CAAC,UAAU,CAAC;SAC7B;QACD,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;IAClD,CAAC;IAED;;;;OAIG;IACI,+BAAS,GAAhB;QACE,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,eAAe,EAAE;YAC9C,qCAAqC;YACrC,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;SACnD;QACD,IAAM,KAAK,GAAU,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACrD,IAAI,KAAK,CAAC,IAAI,KAAK,iBAAS,CAAC,UAAU,EAAE;YACvC,4EAA4E;YAC5E,oDAAoD;YAEpD,qCAAqC;YACrC,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;SACxD;QACD,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACI,2CAAqB,GAA5B;QACE,IAAI,IAAI,CAAC,aAAa,KAAK,CAAC,EAAE;YAC5B,OAAO,iBAAS,CAAC,UAAU,CAAC;SAC7B;QACD,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;IAClD,CAAC;IAED;;OAEG;IACI,kCAAY,GAAnB;QACE,OAAO,IAAI,CAAC,aAAa,CAAC;IAC5B,CAAC;IAED;;OAEG;IACI,uCAAiB,GAAxB,UAAyB,MAAc;QACrC,IAAI,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE;YAC/B,qCAAqC;YACrC,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;SAC3C;QAED,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC;QAC5B,IAAI,MAAM,GAAG,IAAI,CAAC,sBAAsB,EAAE;YACxC,IAAI,CAAC,sBAAsB,GAAG,MAAM,CAAC;SACtC;IACH,CAAC;IACH,kBAAC;AAAD,CAAC,AAhMD,IAgMC;AAhMY,kCAAW","sourcesContent":["import { Token, TokenKind } from './Token';\r\nimport { TokenSequence } from './TokenSequence';\r\nimport { ParserContext } from './ParserContext';\r\n\r\n/**\r\n * Manages a stream of tokens that are read by the parser.\r\n *\r\n * @remarks\r\n * Use TokenReader.readToken() to read a token and advance the stream pointer.\r\n * Use TokenReader.peekToken() to preview the next token.\r\n * Use TokenReader.createMarker() and backtrackToMarker() to rewind to an earlier point.\r\n * Whenever readToken() is called, the token is added to an accumulated TokenSequence\r\n * that can be extracted by calling extractAccumulatedSequence().\r\n */\r\nexport class TokenReader {\r\n public readonly tokens: ReadonlyArray<Token>;\r\n\r\n private readonly _parserContext: ParserContext;\r\n\r\n // The subrange of tokens to be processed by the TokenReader. By default this is\r\n // start=0 and end=tokens.length, unless an embeddedTokenSequence is specified.\r\n private _readerStartIndex: number;\r\n private _readerEndIndex: number;\r\n\r\n // The index of the next token to be read\r\n private _currentIndex: number;\r\n\r\n // The start of the range to be returned by extractAccumulatedSequence()\r\n private _accumulatedStartIndex: number;\r\n\r\n public constructor(parserContext: ParserContext, embeddedTokenSequence?: TokenSequence) {\r\n this._parserContext = parserContext;\r\n this.tokens = parserContext.tokens;\r\n\r\n if (embeddedTokenSequence) {\r\n if (embeddedTokenSequence.parserContext !== this._parserContext) {\r\n throw new Error('The embeddedTokenSequence must use the same parser context');\r\n }\r\n this._readerStartIndex = embeddedTokenSequence.startIndex;\r\n this._readerEndIndex = embeddedTokenSequence.endIndex;\r\n } else {\r\n this._readerStartIndex = 0;\r\n this._readerEndIndex = this.tokens.length;\r\n }\r\n\r\n this._currentIndex = this._readerStartIndex;\r\n this._accumulatedStartIndex = this._readerStartIndex;\r\n }\r\n\r\n /**\r\n * Extracts and returns the TokenSequence that was accumulated so far by calls to readToken().\r\n * The next call to readToken() will start a new accumulated sequence.\r\n */\r\n public extractAccumulatedSequence(): TokenSequence {\r\n if (this._accumulatedStartIndex === this._currentIndex) {\r\n // If this happens, it indicates a parser bug:\r\n throw new Error(\r\n 'Parser assertion failed: The queue should not be empty when' +\r\n ' extractAccumulatedSequence() is called'\r\n );\r\n }\r\n\r\n const sequence: TokenSequence = new TokenSequence({\r\n parserContext: this._parserContext,\r\n startIndex: this._accumulatedStartIndex,\r\n endIndex: this._currentIndex\r\n });\r\n\r\n this._accumulatedStartIndex = this._currentIndex;\r\n\r\n return sequence;\r\n }\r\n\r\n /**\r\n * Returns true if the accumulated sequence has any tokens yet. This will be false\r\n * when the TokenReader starts, and it will be false immediately after a call\r\n * to extractAccumulatedSequence(). Otherwise, it will become true whenever readToken()\r\n * is called.\r\n */\r\n public isAccumulatedSequenceEmpty(): boolean {\r\n return this._accumulatedStartIndex === this._currentIndex;\r\n }\r\n\r\n /**\r\n * Like extractAccumulatedSequence(), but returns undefined if nothing has been\r\n * accumulated yet.\r\n */\r\n public tryExtractAccumulatedSequence(): TokenSequence | undefined {\r\n if (this.isAccumulatedSequenceEmpty()) {\r\n return undefined;\r\n }\r\n return this.extractAccumulatedSequence();\r\n }\r\n\r\n /**\r\n * Asserts that isAccumulatedSequenceEmpty() should return false. If not, an exception\r\n * is throw indicating a parser bug.\r\n */\r\n public assertAccumulatedSequenceIsEmpty(): void {\r\n if (!this.isAccumulatedSequenceEmpty()) {\r\n // If this happens, it indicates a parser bug:\r\n const sequence: TokenSequence = new TokenSequence({\r\n parserContext: this._parserContext,\r\n startIndex: this._accumulatedStartIndex,\r\n endIndex: this._currentIndex\r\n });\r\n const tokenStrings: string[] = sequence.tokens.map((x) => x.toString());\r\n throw new Error(\r\n 'Parser assertion failed: The queue should be empty, but it contains:\\n' +\r\n JSON.stringify(tokenStrings)\r\n );\r\n }\r\n }\r\n\r\n /**\r\n * Returns the next token that would be returned by _readToken(), without\r\n * consuming anything.\r\n */\r\n public peekToken(): Token {\r\n return this.tokens[this._currentIndex];\r\n }\r\n\r\n /**\r\n * Returns the TokenKind for the next token that would be returned by _readToken(), without\r\n * consuming anything.\r\n */\r\n public peekTokenKind(): TokenKind {\r\n if (this._currentIndex >= this._readerEndIndex) {\r\n return TokenKind.EndOfInput;\r\n }\r\n return this.tokens[this._currentIndex].kind;\r\n }\r\n\r\n /**\r\n * Like peekTokenKind(), but looks ahead two tokens.\r\n */\r\n public peekTokenAfterKind(): TokenKind {\r\n if (this._currentIndex + 1 >= this._readerEndIndex) {\r\n return TokenKind.EndOfInput;\r\n }\r\n return this.tokens[this._currentIndex + 1].kind;\r\n }\r\n\r\n /**\r\n * Like peekTokenKind(), but looks ahead three tokens.\r\n */\r\n public peekTokenAfterAfterKind(): TokenKind {\r\n if (this._currentIndex + 2 >= this._readerEndIndex) {\r\n return TokenKind.EndOfInput;\r\n }\r\n return this.tokens[this._currentIndex + 2].kind;\r\n }\r\n\r\n /**\r\n * Extract the next token from the input stream and return it.\r\n * The token will also be appended to the accumulated sequence, which can\r\n * later be accessed via extractAccumulatedSequence().\r\n */\r\n public readToken(): Token {\r\n if (this._currentIndex >= this._readerEndIndex) {\r\n // If this happens, it's a parser bug\r\n throw new Error('Cannot read past end of stream');\r\n }\r\n const token: Token = this.tokens[this._currentIndex];\r\n if (token.kind === TokenKind.EndOfInput) {\r\n // We don't allow reading the EndOfInput token, because we want _peekToken()\r\n // to be always guaranteed to return a valid result.\r\n\r\n // If this happens, it's a parser bug\r\n throw new Error('The EndOfInput token cannot be read');\r\n }\r\n this._currentIndex++;\r\n return token;\r\n }\r\n\r\n /**\r\n * Returns the kind of the token immediately before the current token.\r\n */\r\n public peekPreviousTokenKind(): TokenKind {\r\n if (this._currentIndex === 0) {\r\n return TokenKind.EndOfInput;\r\n }\r\n return this.tokens[this._currentIndex - 1].kind;\r\n }\r\n\r\n /**\r\n * Remembers the current position in the stream.\r\n */\r\n public createMarker(): number {\r\n return this._currentIndex;\r\n }\r\n\r\n /**\r\n * Rewinds the stream pointer to a previous position in the stream.\r\n */\r\n public backtrackToMarker(marker: number): void {\r\n if (marker > this._currentIndex) {\r\n // If this happens, it's a parser bug\r\n throw new Error('The marker has expired');\r\n }\r\n\r\n this._currentIndex = marker;\r\n if (marker < this._accumulatedStartIndex) {\r\n this._accumulatedStartIndex = marker;\r\n }\r\n }\r\n}\r\n"]} |