TokenProcessor.js 9.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357
  1. import { TokenType as tt} from "./parser/tokenizer/types";
  2. import isAsyncOperation from "./util/isAsyncOperation";
  3. export default class TokenProcessor {
  4. __init() {this.resultCode = ""}
  5. // Array mapping input token index to optional string index position in the
  6. // output code.
  7. __init2() {this.resultMappings = new Array(this.tokens.length)}
  8. __init3() {this.tokenIndex = 0}
  9. constructor(
  10. code,
  11. tokens,
  12. isFlowEnabled,
  13. disableESTransforms,
  14. helperManager,
  15. ) {;this.code = code;this.tokens = tokens;this.isFlowEnabled = isFlowEnabled;this.disableESTransforms = disableESTransforms;this.helperManager = helperManager;TokenProcessor.prototype.__init.call(this);TokenProcessor.prototype.__init2.call(this);TokenProcessor.prototype.__init3.call(this);}
  16. /**
  17. * Snapshot the token state in a way that can be restored later, useful for
  18. * things like lookahead.
  19. *
  20. * resultMappings do not need to be copied since in all use cases, they will
  21. * be overwritten anyway after restore.
  22. */
  23. snapshot() {
  24. return {
  25. resultCode: this.resultCode,
  26. tokenIndex: this.tokenIndex,
  27. };
  28. }
  29. restoreToSnapshot(snapshot) {
  30. this.resultCode = snapshot.resultCode;
  31. this.tokenIndex = snapshot.tokenIndex;
  32. }
  33. /**
  34. * Remove and return the code generated since the snapshot, leaving the
  35. * current token position in-place. Unlike most TokenProcessor operations,
  36. * this operation can result in input/output line number mismatches because
  37. * the removed code may contain newlines, so this operation should be used
  38. * sparingly.
  39. */
  40. dangerouslyGetAndRemoveCodeSinceSnapshot(snapshot) {
  41. const result = this.resultCode.slice(snapshot.resultCode.length);
  42. this.resultCode = snapshot.resultCode;
  43. return result;
  44. }
  45. reset() {
  46. this.resultCode = "";
  47. this.resultMappings = new Array(this.tokens.length);
  48. this.tokenIndex = 0;
  49. }
  50. matchesContextualAtIndex(index, contextualKeyword) {
  51. return (
  52. this.matches1AtIndex(index, tt.name) &&
  53. this.tokens[index].contextualKeyword === contextualKeyword
  54. );
  55. }
  56. identifierNameAtIndex(index) {
  57. // TODO: We need to process escapes since technically you can have unicode escapes in variable
  58. // names.
  59. return this.identifierNameForToken(this.tokens[index]);
  60. }
  61. identifierNameAtRelativeIndex(relativeIndex) {
  62. return this.identifierNameForToken(this.tokenAtRelativeIndex(relativeIndex));
  63. }
  64. identifierName() {
  65. return this.identifierNameForToken(this.currentToken());
  66. }
  67. identifierNameForToken(token) {
  68. return this.code.slice(token.start, token.end);
  69. }
  70. rawCodeForToken(token) {
  71. return this.code.slice(token.start, token.end);
  72. }
  73. stringValueAtIndex(index) {
  74. return this.stringValueForToken(this.tokens[index]);
  75. }
  76. stringValue() {
  77. return this.stringValueForToken(this.currentToken());
  78. }
  79. stringValueForToken(token) {
  80. // This is used to identify when two imports are the same and to resolve TypeScript enum keys.
  81. // Ideally we'd process escapes within the strings, but for now we pretty much take the raw
  82. // code.
  83. return this.code.slice(token.start + 1, token.end - 1);
  84. }
  85. matches1AtIndex(index, t1) {
  86. return this.tokens[index].type === t1;
  87. }
  88. matches2AtIndex(index, t1, t2) {
  89. return this.tokens[index].type === t1 && this.tokens[index + 1].type === t2;
  90. }
  91. matches3AtIndex(index, t1, t2, t3) {
  92. return (
  93. this.tokens[index].type === t1 &&
  94. this.tokens[index + 1].type === t2 &&
  95. this.tokens[index + 2].type === t3
  96. );
  97. }
  98. matches1(t1) {
  99. return this.tokens[this.tokenIndex].type === t1;
  100. }
  101. matches2(t1, t2) {
  102. return this.tokens[this.tokenIndex].type === t1 && this.tokens[this.tokenIndex + 1].type === t2;
  103. }
  104. matches3(t1, t2, t3) {
  105. return (
  106. this.tokens[this.tokenIndex].type === t1 &&
  107. this.tokens[this.tokenIndex + 1].type === t2 &&
  108. this.tokens[this.tokenIndex + 2].type === t3
  109. );
  110. }
  111. matches4(t1, t2, t3, t4) {
  112. return (
  113. this.tokens[this.tokenIndex].type === t1 &&
  114. this.tokens[this.tokenIndex + 1].type === t2 &&
  115. this.tokens[this.tokenIndex + 2].type === t3 &&
  116. this.tokens[this.tokenIndex + 3].type === t4
  117. );
  118. }
  119. matches5(t1, t2, t3, t4, t5) {
  120. return (
  121. this.tokens[this.tokenIndex].type === t1 &&
  122. this.tokens[this.tokenIndex + 1].type === t2 &&
  123. this.tokens[this.tokenIndex + 2].type === t3 &&
  124. this.tokens[this.tokenIndex + 3].type === t4 &&
  125. this.tokens[this.tokenIndex + 4].type === t5
  126. );
  127. }
  128. matchesContextual(contextualKeyword) {
  129. return this.matchesContextualAtIndex(this.tokenIndex, contextualKeyword);
  130. }
  131. matchesContextIdAndLabel(type, contextId) {
  132. return this.matches1(type) && this.currentToken().contextId === contextId;
  133. }
  134. previousWhitespaceAndComments() {
  135. let whitespaceAndComments = this.code.slice(
  136. this.tokenIndex > 0 ? this.tokens[this.tokenIndex - 1].end : 0,
  137. this.tokenIndex < this.tokens.length ? this.tokens[this.tokenIndex].start : this.code.length,
  138. );
  139. if (this.isFlowEnabled) {
  140. whitespaceAndComments = whitespaceAndComments.replace(/@flow/g, "");
  141. }
  142. return whitespaceAndComments;
  143. }
  144. replaceToken(newCode) {
  145. this.resultCode += this.previousWhitespaceAndComments();
  146. this.appendTokenPrefix();
  147. this.resultMappings[this.tokenIndex] = this.resultCode.length;
  148. this.resultCode += newCode;
  149. this.appendTokenSuffix();
  150. this.tokenIndex++;
  151. }
  152. replaceTokenTrimmingLeftWhitespace(newCode) {
  153. this.resultCode += this.previousWhitespaceAndComments().replace(/[^\r\n]/g, "");
  154. this.appendTokenPrefix();
  155. this.resultMappings[this.tokenIndex] = this.resultCode.length;
  156. this.resultCode += newCode;
  157. this.appendTokenSuffix();
  158. this.tokenIndex++;
  159. }
  160. removeInitialToken() {
  161. this.replaceToken("");
  162. }
  163. removeToken() {
  164. this.replaceTokenTrimmingLeftWhitespace("");
  165. }
  166. /**
  167. * Remove all code until the next }, accounting for balanced braces.
  168. */
  169. removeBalancedCode() {
  170. let braceDepth = 0;
  171. while (!this.isAtEnd()) {
  172. if (this.matches1(tt.braceL)) {
  173. braceDepth++;
  174. } else if (this.matches1(tt.braceR)) {
  175. if (braceDepth === 0) {
  176. return;
  177. }
  178. braceDepth--;
  179. }
  180. this.removeToken();
  181. }
  182. }
  183. copyExpectedToken(tokenType) {
  184. if (this.tokens[this.tokenIndex].type !== tokenType) {
  185. throw new Error(`Expected token ${tokenType}`);
  186. }
  187. this.copyToken();
  188. }
  189. copyToken() {
  190. this.resultCode += this.previousWhitespaceAndComments();
  191. this.appendTokenPrefix();
  192. this.resultMappings[this.tokenIndex] = this.resultCode.length;
  193. this.resultCode += this.code.slice(
  194. this.tokens[this.tokenIndex].start,
  195. this.tokens[this.tokenIndex].end,
  196. );
  197. this.appendTokenSuffix();
  198. this.tokenIndex++;
  199. }
  200. copyTokenWithPrefix(prefix) {
  201. this.resultCode += this.previousWhitespaceAndComments();
  202. this.appendTokenPrefix();
  203. this.resultCode += prefix;
  204. this.resultMappings[this.tokenIndex] = this.resultCode.length;
  205. this.resultCode += this.code.slice(
  206. this.tokens[this.tokenIndex].start,
  207. this.tokens[this.tokenIndex].end,
  208. );
  209. this.appendTokenSuffix();
  210. this.tokenIndex++;
  211. }
  212. appendTokenPrefix() {
  213. const token = this.currentToken();
  214. if (token.numNullishCoalesceStarts || token.isOptionalChainStart) {
  215. token.isAsyncOperation = isAsyncOperation(this);
  216. }
  217. if (this.disableESTransforms) {
  218. return;
  219. }
  220. if (token.numNullishCoalesceStarts) {
  221. for (let i = 0; i < token.numNullishCoalesceStarts; i++) {
  222. if (token.isAsyncOperation) {
  223. this.resultCode += "await ";
  224. this.resultCode += this.helperManager.getHelperName("asyncNullishCoalesce");
  225. } else {
  226. this.resultCode += this.helperManager.getHelperName("nullishCoalesce");
  227. }
  228. this.resultCode += "(";
  229. }
  230. }
  231. if (token.isOptionalChainStart) {
  232. if (token.isAsyncOperation) {
  233. this.resultCode += "await ";
  234. }
  235. if (this.tokenIndex > 0 && this.tokenAtRelativeIndex(-1).type === tt._delete) {
  236. if (token.isAsyncOperation) {
  237. this.resultCode += this.helperManager.getHelperName("asyncOptionalChainDelete");
  238. } else {
  239. this.resultCode += this.helperManager.getHelperName("optionalChainDelete");
  240. }
  241. } else if (token.isAsyncOperation) {
  242. this.resultCode += this.helperManager.getHelperName("asyncOptionalChain");
  243. } else {
  244. this.resultCode += this.helperManager.getHelperName("optionalChain");
  245. }
  246. this.resultCode += "([";
  247. }
  248. }
  249. appendTokenSuffix() {
  250. const token = this.currentToken();
  251. if (token.isOptionalChainEnd && !this.disableESTransforms) {
  252. this.resultCode += "])";
  253. }
  254. if (token.numNullishCoalesceEnds && !this.disableESTransforms) {
  255. for (let i = 0; i < token.numNullishCoalesceEnds; i++) {
  256. this.resultCode += "))";
  257. }
  258. }
  259. }
  260. appendCode(code) {
  261. this.resultCode += code;
  262. }
  263. currentToken() {
  264. return this.tokens[this.tokenIndex];
  265. }
  266. currentTokenCode() {
  267. const token = this.currentToken();
  268. return this.code.slice(token.start, token.end);
  269. }
  270. tokenAtRelativeIndex(relativeIndex) {
  271. return this.tokens[this.tokenIndex + relativeIndex];
  272. }
  273. currentIndex() {
  274. return this.tokenIndex;
  275. }
  276. /**
  277. * Move to the next token. Only suitable in preprocessing steps. When
  278. * generating new code, you should use copyToken or removeToken.
  279. */
  280. nextToken() {
  281. if (this.tokenIndex === this.tokens.length) {
  282. throw new Error("Unexpectedly reached end of input.");
  283. }
  284. this.tokenIndex++;
  285. }
  286. previousToken() {
  287. this.tokenIndex--;
  288. }
  289. finish() {
  290. if (this.tokenIndex !== this.tokens.length) {
  291. throw new Error("Tried to finish processing tokens before reaching the end.");
  292. }
  293. this.resultCode += this.previousWhitespaceAndComments();
  294. return {code: this.resultCode, mappings: this.resultMappings};
  295. }
  296. isAtEnd() {
  297. return this.tokenIndex === this.tokens.length;
  298. }
  299. }