tokens_javascript.json 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592
  1. [[
  2. "start",
  3. ["comment","//test: tokenize 'standard' functions"]
  4. ],[
  5. "no_regex",
  6. ["identifier","string"],
  7. ["punctuation.operator","."],
  8. ["support.function","charCodeAt"],
  9. ["paren.lparen","("],
  10. ["constant.numeric","23"],
  11. ["paren.rparen",")"],
  12. ["punctuation.operator",";"],
  13. ["text"," "],
  14. ["variable.language","document"],
  15. ["punctuation.operator","."],
  16. ["support.function.dom","getElementById"],
  17. ["paren.lparen","("],
  18. ["string","'test'"],
  19. ["paren.rparen",")"],
  20. ["punctuation.operator",";"],
  21. ["text"," "],
  22. ["storage.type","console"],
  23. ["punctuation.operator","."],
  24. ["support.function.firebug","log"],
  25. ["paren.lparen","("],
  26. ["string","'Here it is'"],
  27. ["paren.rparen",")"],
  28. ["punctuation.operator",";"],
  29. ["string","\";"]
  30. ],[
  31. "no_regex",
  32. ["identifier","test"],
  33. ["punctuation.operator",":"],
  34. ["text"," "],
  35. ["comment.doc","/**tokenize doc*/"],
  36. ["text"," "],
  37. ["identifier","comment"]
  38. ],[
  39. "no_regex",
  40. ["comment.doc","/**tokenize doc comment with "],
  41. ["comment.doc.tag","@tag"],
  42. ["comment.doc"," {}*/"]
  43. ],[
  44. "no_regex",
  45. ["comment","//test: tokenize parens"]
  46. ],[
  47. "start",
  48. ["text"," "],
  49. ["storage.type","var"],
  50. ["text"," "],
  51. ["identifier","line"],
  52. ["text"," "],
  53. ["keyword.operator","="],
  54. ["text"," "],
  55. ["string","\"[{( )}]\""],
  56. ["punctuation.operator",";"]
  57. ],[
  58. "start",
  59. ["comment","//test tokenize arithmetic expression which looks like a regexp"]
  60. ],[
  61. "no_regex",
  62. ["identifier","a"],
  63. ["keyword.operator","/"],
  64. ["identifier","b"],
  65. ["keyword.operator","/"],
  66. ["identifier","c"]
  67. ],[
  68. "no_regex",
  69. ["identifier","a"],
  70. ["keyword.operator","/="],
  71. ["identifier","b"],
  72. ["keyword.operator","/"],
  73. ["identifier","c"]
  74. ],[
  75. "no_regex",
  76. ["comment","//test tokenize reg exps"]
  77. ],[
  78. "no_regex",
  79. ["identifier","a"],
  80. ["keyword.operator","="],
  81. ["string.regexp","/b/g"]
  82. ],[
  83. "no_regex",
  84. ["identifier","a"],
  85. ["keyword.operator","+"],
  86. ["string.regexp","/b/g"]
  87. ],[
  88. "no_regex",
  89. ["identifier","a"],
  90. ["text"," "],
  91. ["keyword.operator","="],
  92. ["text"," "],
  93. ["constant.numeric","1"],
  94. ["text"," "],
  95. ["keyword.operator","+"],
  96. ["text"," "],
  97. ["string.regexp","/2 "],
  98. ["constant.language.escape","+"],
  99. ["string.regexp"," 1/b"]
  100. ],[
  101. "no_regex",
  102. ["identifier","a"],
  103. ["keyword.operator","="],
  104. ["string.regexp","/a/"],
  105. ["text"," "],
  106. ["keyword.operator","/"],
  107. ["text"," "],
  108. ["string.regexp","/a/"]
  109. ],[
  110. "no_regex",
  111. ["keyword","case"],
  112. ["text"," "],
  113. ["string.regexp","/a/"],
  114. ["punctuation.operator","."],
  115. ["support.function","test"],
  116. ["paren.lparen","("],
  117. ["identifier","c"],
  118. ["paren.rparen",")"]
  119. ],[
  120. "no_regex",
  121. ["comment","//test tokenize multi-line comment containing a single line comment"]
  122. ],[
  123. "no_regex",
  124. ["identifier","noRegex"]
  125. ],[
  126. "no_regex",
  127. ["comment","/* foo // bar */"]
  128. ],[
  129. "start",
  130. ["identifier","canBeRegex"],
  131. ["punctuation.operator",";"]
  132. ],[
  133. "start",
  134. ["comment","/* foo // bar */"]
  135. ],[
  136. "start",
  137. ["comment","// test tokenize identifier with umlauts"]
  138. ],[
  139. "no_regex",
  140. ["identifier","fu"],
  141. ["punctuation.operator","?"],
  142. ["identifier","e"]
  143. ],[
  144. "no_regex",
  145. ["comment","// test // is not a regexp"]
  146. ],[
  147. "start",
  148. ["paren.lparen","{"],
  149. ["text"," "],
  150. ["comment","// 123"]
  151. ],[
  152. "start",
  153. ["comment","//test skipping escaped chars"]
  154. ],[
  155. "no_regex",
  156. ["string","'Meh"],
  157. ["constant.language.escape","\\\\"],
  158. ["string","nNeh'"]
  159. ],[
  160. "no_regex",
  161. ["storage.type","console"],
  162. ["punctuation.operator","."],
  163. ["support.function.firebug","log"],
  164. ["paren.lparen","("],
  165. ["string","'"],
  166. ["constant.language.escape","\\\\"],
  167. ["string","u1232Feh'"]
  168. ],[
  169. "qqstring",
  170. ["string","\"test multiline\\"]
  171. ],[
  172. "no_regex",
  173. ["string"," strings\""]
  174. ],[
  175. "no_regex",
  176. ["identifier","a"],
  177. ["keyword.operator","="],
  178. ["text","'"]
  179. ],[
  180. "qqstring",
  181. ["identifier","b"],
  182. ["keyword.operator","="],
  183. ["string","\"\\"]
  184. ],[
  185. "no_regex",
  186. ["string","still a string"]
  187. ],[
  188. "no_regex",
  189. ["text"," "]
  190. ],[
  191. "no_regex",
  192. ["text"," "]
  193. ],[
  194. "start",
  195. ["storage.type","function"],
  196. ["text"," "],
  197. ["entity.name.function","foo"],
  198. ["paren.lparen","("],
  199. ["variable.parameter","items"],
  200. ["punctuation.operator",", "],
  201. ["variable.parameter","nada"],
  202. ["paren.rparen",")"],
  203. ["text"," "],
  204. ["paren.lparen","{"]
  205. ],[
  206. "start",
  207. ["text"," "],
  208. ["keyword","for"],
  209. ["text"," "],
  210. ["paren.lparen","("],
  211. ["storage.type","var"],
  212. ["text"," "],
  213. ["identifier","i"],
  214. ["keyword.operator","="],
  215. ["constant.numeric","0"],
  216. ["punctuation.operator",";"],
  217. ["text"," "],
  218. ["identifier","i"],
  219. ["keyword.operator","<"],
  220. ["identifier","items"],
  221. ["punctuation.operator","."],
  222. ["support.constant","length"],
  223. ["punctuation.operator",";"],
  224. ["text"," "],
  225. ["identifier","i"],
  226. ["keyword.operator","++"],
  227. ["paren.rparen",")"],
  228. ["text"," "],
  229. ["paren.lparen","{"]
  230. ],[
  231. "start",
  232. ["text"," "],
  233. ["support.function","alert"],
  234. ["paren.lparen","("],
  235. ["identifier","items"],
  236. ["paren.lparen","["],
  237. ["identifier","i"],
  238. ["paren.rparen","]"],
  239. ["text"," "],
  240. ["keyword.operator","+"],
  241. ["text"," "],
  242. ["string","\"juhu"],
  243. ["constant.language.escape","\\n"],
  244. ["string","\""],
  245. ["paren.rparen",")"],
  246. ["punctuation.operator",";"]
  247. ],[
  248. "no_regex",
  249. ["text"," "],
  250. ["paren.rparen","}"],
  251. ["text","\t"],
  252. ["comment","// Real Tab."]
  253. ],[
  254. "no_regex",
  255. ["paren.rparen","}"]
  256. ],[
  257. "no_regex"
  258. ],[
  259. "no_regex",
  260. ["identifier","regexp"],
  261. ["text"," "],
  262. ["keyword.operator","="],
  263. ["text"," "],
  264. ["string.regexp","/p"],
  265. ["constant.language.delimiter","|"],
  266. ["string.regexp","p/"],
  267. ["text"," "],
  268. ["comment","// ends here"]
  269. ],[
  270. "no_regex"
  271. ],[
  272. "no_regex",
  273. ["identifier","r"],
  274. ["text"," "],
  275. ["keyword.operator","="],
  276. ["text"," "],
  277. ["string.regexp","/d"],
  278. ["constant.language.escape","{1,2}?"],
  279. ["string.regexp","f{e}"],
  280. ["invalid","++"],
  281. ["string.regexp","r"],
  282. ["constant.language.escape","*?"],
  283. ["regexp.keyword.operator","\\d"],
  284. ["constant.language.escape","+?[]"],
  285. ["string.regexp","r"],
  286. ["constant.language.escape","[^"],
  287. ["string.regexp.charachterclass","r"],
  288. ["constant.language.escape","-"],
  289. ["string.regexp.charachterclass","o"],
  290. ["regexp.keyword.operator","\\f\\f"],
  291. ["string.regexp.charachterclass","["],
  292. ["regexp.keyword.operator","\\f"],
  293. ["constant.language.escape","]?"],
  294. ["string.regexp","r"],
  295. ["invalid","{7}+"],
  296. ["string.regexp","r"],
  297. ["regexp.keyword.operator","\\{"],
  298. ["string.regexp","7}"],
  299. ["constant.language.escape","+"],
  300. ["string.regexp","rr--rr"],
  301. ["constant.language.escape","$^(?:"],
  302. ["string.regexp","d"],
  303. ["constant.language.delimiter","|"],
  304. ["string.regexp","s"],
  305. ["constant.language.escape",")(?="],
  306. ["string.regexp","a"],
  307. ["constant.language.delimiter","|"],
  308. ["constant.language.escape",")(?!"],
  309. ["string.regexp","y"],
  310. ["constant.language.escape",")[]"],
  311. ["constant.language.delimiter","|"],
  312. ["invalid","$?"],
  313. ["constant.language.delimiter","|"],
  314. ["invalid","^*"],
  315. ["string.regexp","/"],
  316. ["text"," "],
  317. ["identifier","o"]
  318. ],[
  319. "no_regex",
  320. ["identifier","a"],
  321. ["keyword.operator","="],
  322. ["string.regexp","/a/"],
  323. ["text"," "],
  324. ["identifier","jk"],
  325. ["text"," "],
  326. ["keyword.operator","="],
  327. ["text"," "],
  328. ["string.regexp","/ /"],
  329. ["text"," "],
  330. ["keyword.operator","/"],
  331. ["text"," "],
  332. ["string.regexp","/ /"]
  333. ],[
  334. "no_regex",
  335. ["text"," "],
  336. ["comment.doc","/************************************/"]
  337. ],[
  338. "no_regex",
  339. ["comment.doc","/** total mess, tricky to highlight**/"]
  340. ],[
  341. "no_regex"
  342. ],[
  343. "start",
  344. ["storage.type","function"],
  345. ["text"," "],
  346. ["paren.lparen","("],
  347. ["paren.rparen",")"],
  348. ["text"," "],
  349. ["paren.lparen","{"]
  350. ],[
  351. "doc-start",
  352. ["text","\t"],
  353. ["comment.doc","/**"]
  354. ],[
  355. "doc-start",
  356. ["comment.doc","\t * docComment"]
  357. ],[
  358. "no_regex",
  359. ["comment.doc","\t **/"]
  360. ],[
  361. "no_regex",
  362. ["text","\t"],
  363. ["identifier","r"],
  364. ["text"," "],
  365. ["keyword.operator","="],
  366. ["text"," "],
  367. ["string.regexp","/u"],
  368. ["regexp.keyword.operator","\\t"],
  369. ["constant.language.escape","*"],
  370. ["string.regexp","/"]
  371. ],[
  372. "no_regex",
  373. ["text","\t"],
  374. ["identifier","g"],
  375. ["text"," "],
  376. ["keyword.operator","="],
  377. ["text"," "],
  378. ["constant.numeric","1."],
  379. ["text","00"],
  380. ["identifier","E"],
  381. ["text","^"],
  382. ["constant.numeric","1"],
  383. ["punctuation.operator",","],
  384. ["text"," "],
  385. ["identifier","y"],
  386. ["text"," "],
  387. ["keyword.operator","="],
  388. ["text"," "],
  389. ["constant.numeric","1.2"],
  390. ["text"," "],
  391. ["keyword.operator","+"],
  392. ["text"," "],
  393. ["punctuation.operator","."],
  394. ["constant.numeric","2"],
  395. ["text"," "],
  396. ["keyword.operator","+"],
  397. ["text"," "],
  398. ["constant.numeric","052"],
  399. ["text"," "],
  400. ["keyword.operator","+"],
  401. ["text"," "],
  402. ["constant.numeric","0x25"]
  403. ],[
  404. "no_regex",
  405. ["text","\t"],
  406. ["identifier","t"],
  407. ["text"," "],
  408. ["keyword.operator","="],
  409. ["text"," "],
  410. ["paren.lparen","["],
  411. ["string","'d'"],
  412. ["punctuation.operator",","],
  413. ["text"," "],
  414. ["string","''"],
  415. ["paren.rparen","]"]
  416. ],[
  417. "no_regex",
  418. ["paren.rparen","}"]
  419. ],[
  420. "start",
  421. ["storage.type","function"],
  422. ["text"," "],
  423. ["paren.lparen","("],
  424. ["paren.rparen",")"],
  425. ["text"," "],
  426. ["paren.lparen","{"]
  427. ],[
  428. "start",
  429. ["text","\t"],
  430. ["comment","/* eee */"]
  431. ],[
  432. "no_regex",
  433. ["paren.rparen","}"]
  434. ],[
  435. "no_regex"
  436. ],[
  437. "qqstring",
  438. ["string","\"s\\"]
  439. ],[
  440. "no_regex",
  441. ["string","s"],
  442. ["constant.language.escape","\\u7824"],
  443. ["string","sss"],
  444. ["constant.language.escape","\\u"],
  445. ["string","1\""]
  446. ],[
  447. "no_regex"
  448. ],[
  449. "qstring",
  450. ["string","'\\"]
  451. ],[
  452. "no_regex",
  453. ["string","string'"]
  454. ],[
  455. "no_regex"
  456. ],[
  457. "no_regex",
  458. ["text","'"]
  459. ],[
  460. "no_regex",
  461. ["identifier","string"],
  462. ["text","'"]
  463. ],[
  464. "no_regex"
  465. ],[
  466. "no_regex",
  467. ["string","\"trailing space"],
  468. ["constant.language.escape","\\ "],
  469. ["string"," "]
  470. ],[
  471. "no_regex",
  472. ["string","\" \""],
  473. ["text"," "],
  474. ["keyword.operator","/"],
  475. ["identifier","not"],
  476. ["text"," "],
  477. ["identifier","a"],
  478. ["text"," "],
  479. ["identifier","regexp"],
  480. ["keyword.operator","/"],
  481. ["identifier","g"]
  482. ],[
  483. "no_regex"
  484. ],[
  485. "doc-start",
  486. ["comment.doc","/**"]
  487. ],[
  488. "doc-start",
  489. ["comment.doc"," *doc"]
  490. ],[
  491. "no_regex",
  492. ["comment.doc"," */"]
  493. ],[
  494. "no_regex"
  495. ],[
  496. "start",
  497. ["identifier","a"],
  498. ["text"," "],
  499. ["keyword.operator","="],
  500. ["text"," "],
  501. ["paren.lparen","{"]
  502. ],[
  503. "start",
  504. ["text","\t"],
  505. ["string","'a'"],
  506. ["punctuation.operator",":"],
  507. ["text"," "],
  508. ["identifier","b"],
  509. ["punctuation.operator",","]
  510. ],[
  511. "no_regex",
  512. ["text","\t"],
  513. ["string","'g'"],
  514. ["text",":"],
  515. ["text"," "],
  516. ["storage.type","function"],
  517. ["paren.lparen","("],
  518. ["variable.parameter","t"],
  519. ["paren.rparen",")"]
  520. ],[
  521. "no_regex",
  522. ["text","\t"],
  523. ["entity.name.function","gta"],
  524. ["punctuation.operator",":"],
  525. ["storage.type","function"],
  526. ["paren.lparen","("],
  527. ["variable.parameter","a"],
  528. ["punctuation.operator",","],
  529. ["variable.parameter","b"],
  530. ["paren.rparen",")"]
  531. ],[
  532. "no_regex",
  533. ["paren.rparen","}"]
  534. ],[
  535. "no_regex"
  536. ],[
  537. "no_regex"
  538. ],[
  539. "function_arguments",
  540. ["identifier","foo"],
  541. ["punctuation.operator","."],
  542. ["storage.type","protoype"],
  543. ["punctuation.operator","."],
  544. ["entity.name.function","d"],
  545. ["text"," "],
  546. ["keyword.operator","="],
  547. ["text"," "],
  548. ["storage.type","function"],
  549. ["paren.lparen","("],
  550. ["variable.parameter","a"],
  551. ["punctuation.operator",", "],
  552. ["variable.parameter","b"],
  553. ["punctuation.operator",","]
  554. ],[
  555. "no_regex",
  556. ["punctuation.operator"," "],
  557. ["variable.parameter","c"],
  558. ["punctuation.operator",", "],
  559. ["variable.parameter","d"],
  560. ["paren.rparen",")"]
  561. ],[
  562. "no_regex",
  563. ["storage.type","foo"],
  564. ["punctuation.operator","."],
  565. ["entity.name.function","d"],
  566. ["text"," "],
  567. ["keyword.operator","="],
  568. ["storage.type","function"],
  569. ["paren.lparen","("],
  570. ["variable.parameter","a"],
  571. ["punctuation.operator",", "],
  572. ["variable.parameter","b"],
  573. ["paren.rparen",")"]
  574. ],[
  575. "no_regex",
  576. ["storage.type","foo"],
  577. ["punctuation.operator","."],
  578. ["entity.name.function","d"],
  579. ["text"," "],
  580. ["keyword.operator","="],
  581. ["storage.type","function"],
  582. ["paren.lparen","("],
  583. ["variable.parameter","a"],
  584. ["punctuation.operator",", "],
  585. ["comment.doc","/*****/"],
  586. ["text"," "],
  587. ["identifier","d"],
  588. ["string","\"string\""],
  589. ["text"," "]
  590. ],[
  591. "no_regex"
  592. ]]