spellchecker.ts 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439
  1. import { addMisspelledWords, misspelledWordsField } from './misspelled-words'
  2. import { ignoredWordsField, resetSpellChecker } from './ignored-words'
  3. import { cacheField, addWordToCache, WordCacheValue } from './cache'
  4. import { WORD_REGEX } from './helpers'
  5. import OError from '@overleaf/o-error'
  6. import { spellCheckRequest } from './backend'
  7. import { EditorView, ViewUpdate } from '@codemirror/view'
  8. import { ChangeSet, Line, Range, RangeValue } from '@codemirror/state'
  9. import { IgnoredWords } from '../../../dictionary/ignored-words'
  10. import { getNormalTextSpansFromLine } from '../../utils/tree-query'
  11. import { waitForParser } from '../wait-for-parser'
  12. import { debugConsole } from '@/utils/debugging'
  13. import type { HunspellManager } from '../../hunspell/HunspellManager'
  14. import { captureException } from '@/infrastructure/error-reporter'
  15. /*
  16. * Spellchecker, handles updates, schedules spelling checks
  17. */
  18. export class SpellChecker {
  19. private abortController?: AbortController | null = null
  20. private timeout: number | null = null
  21. private firstCheck = true
  22. private waitingForParser = false
  23. private firstCheckPending = false
  24. private trackedChanges: ChangeSet
  25. private readonly segmenter?: Intl.Segmenter
  26. // eslint-disable-next-line no-useless-constructor
  27. constructor(
  28. private readonly language: string,
  29. private hunspellManager?: HunspellManager
  30. ) {
  31. debugConsole.log('SpellChecker', language, hunspellManager)
  32. this.trackedChanges = ChangeSet.empty(0)
  33. const locale = language.replace(/_/, '-')
  34. try {
  35. if (Intl.Segmenter) {
  36. const supportedLocales = Intl.Segmenter.supportedLocalesOf([locale], {
  37. localeMatcher: 'lookup',
  38. })
  39. if (supportedLocales.includes(locale)) {
  40. this.segmenter = new Intl.Segmenter(locale, {
  41. localeMatcher: 'lookup',
  42. granularity: 'word',
  43. })
  44. }
  45. }
  46. } catch (error) {
  47. // ignore, not supported for some reason
  48. debugConsole.error(error)
  49. }
  50. if (this.segmenter) {
  51. debugConsole.log(`Using Intl.Segmenter for ${locale}`)
  52. } else {
  53. debugConsole.warn(`Not using Intl.Segmenter for ${locale}`)
  54. }
  55. }
  56. destroy() {
  57. this._clearPendingSpellCheck()
  58. }
  59. _abortRequest() {
  60. if (this.abortController) {
  61. this.abortController.abort()
  62. this.abortController = null
  63. }
  64. }
  65. handleUpdate(update: ViewUpdate) {
  66. if (update.docChanged) {
  67. this.trackedChanges = this.trackedChanges.compose(update.changes)
  68. this.scheduleSpellCheck(update.view)
  69. } else if (update.viewportChanged) {
  70. this.trackedChanges = ChangeSet.empty(0)
  71. this.scheduleSpellCheck(update.view)
  72. } else if (
  73. update.transactions.some(tr => {
  74. return tr.effects.some(effect => effect.is(resetSpellChecker))
  75. })
  76. ) {
  77. // for tests
  78. this.trackedChanges = ChangeSet.empty(0)
  79. this.spellCheckAsap(update.view)
  80. }
  81. // At the point that the spellchecker is initialized, the editor may not
  82. // yet be editable, and the parser may not be ready. Therefore, to do the
  83. // initial spellcheck, watch for changes in the editability of the editor
  84. // and kick off the process that performs a spellcheck once the parser is
  85. // ready. CM6 dispatches a transaction after every chunk of parser work and
  86. // when the editability changes, which means the spell checker is
  87. // initialized as soon as possible.
  88. else if (
  89. this.firstCheck &&
  90. !this.firstCheckPending &&
  91. update.state.facet(EditorView.editable)
  92. ) {
  93. this.firstCheckPending = true
  94. this.spellCheckAsap(update.view)
  95. }
  96. }
  97. _performSpellCheck(view: EditorView) {
  98. const wordsToCheck = this.getWordsToCheck(view)
  99. if (wordsToCheck.length === 0) {
  100. return
  101. }
  102. const cache = view.state.field(cacheField)
  103. const { knownMisspelledWords, unknownWords } = cache.checkWords(
  104. this.language,
  105. wordsToCheck
  106. )
  107. const processResult = (
  108. misspellings: { index: number; suggestions?: string[] }[]
  109. ) => {
  110. this.trackedChanges = ChangeSet.empty(0)
  111. if (this.firstCheck) {
  112. this.firstCheck = false
  113. this.firstCheckPending = false
  114. }
  115. const result = buildSpellCheckResult(
  116. knownMisspelledWords,
  117. unknownWords,
  118. misspellings
  119. )
  120. view.dispatch({
  121. effects: compileEffects(result),
  122. })
  123. }
  124. if (unknownWords.length === 0) {
  125. processResult([])
  126. } else {
  127. this._abortRequest()
  128. this.abortController = new AbortController()
  129. if (this.hunspellManager) {
  130. const signal = this.abortController.signal
  131. this.hunspellManager.send(
  132. {
  133. type: 'spell',
  134. words: unknownWords.map(word => word.text),
  135. },
  136. result => {
  137. if (!signal.aborted) {
  138. if ('error' in result) {
  139. debugConsole.error(result.error)
  140. captureException(
  141. new Error('Error running spellcheck for word'),
  142. { tags: { ol_spell_check_language: this.language } }
  143. )
  144. } else {
  145. processResult(result.misspellings)
  146. }
  147. }
  148. }
  149. )
  150. } else {
  151. spellCheckRequest(this.language, unknownWords, this.abortController)
  152. .then(result => {
  153. this.abortController = null
  154. return processResult(result.misspellings)
  155. })
  156. .catch(error => {
  157. this.abortController = null
  158. debugConsole.error(error)
  159. })
  160. }
  161. }
  162. }
  163. suggest(word: string) {
  164. return new Promise<{ suggestions: string[] }>((resolve, reject) => {
  165. if (this.hunspellManager) {
  166. this.hunspellManager.send({ type: 'suggest', word }, result => {
  167. if ('error' in result) {
  168. reject(new Error('Error finding spelling suggestions for word'))
  169. } else {
  170. resolve(result)
  171. }
  172. })
  173. }
  174. })
  175. }
  176. addWord(word: string) {
  177. return new Promise<void>((resolve, reject) => {
  178. if (this.hunspellManager) {
  179. this.hunspellManager.send({ type: 'add_word', word }, result => {
  180. if ('error' in result) {
  181. reject(new Error('Error adding word to spellcheck'))
  182. } else {
  183. resolve()
  184. }
  185. })
  186. }
  187. })
  188. }
  189. removeWord(word: string) {
  190. return new Promise<void>((resolve, reject) => {
  191. if (this.hunspellManager) {
  192. this.hunspellManager.send({ type: 'remove_word', word }, result => {
  193. if ('error' in result) {
  194. reject(new Error('Error removing word from spellcheck'))
  195. } else {
  196. resolve()
  197. }
  198. })
  199. }
  200. })
  201. }
  202. _spellCheckWhenParserReady(view: EditorView) {
  203. if (this.waitingForParser) {
  204. return
  205. }
  206. this.waitingForParser = true
  207. waitForParser(view, view => view.viewport.to).then(() => {
  208. this.waitingForParser = false
  209. this._performSpellCheck(view)
  210. })
  211. }
  212. _clearPendingSpellCheck() {
  213. if (this.timeout) {
  214. window.clearTimeout(this.timeout)
  215. this.timeout = null
  216. }
  217. this._abortRequest()
  218. }
  219. _asyncSpellCheck(view: EditorView, delay: number) {
  220. this._clearPendingSpellCheck()
  221. this.timeout = window.setTimeout(() => {
  222. this._spellCheckWhenParserReady(view)
  223. this.timeout = null
  224. }, delay)
  225. }
  226. spellCheckAsap(view: EditorView) {
  227. this._asyncSpellCheck(view, 0)
  228. }
  229. scheduleSpellCheck(view: EditorView) {
  230. this._asyncSpellCheck(view, 1000)
  231. }
  232. getWordsToCheck(view: EditorView) {
  233. const wordsToCheck: Word[] = []
  234. const { from, to } = view.viewport
  235. const changedLineNumbers = new Set<number>()
  236. if (this.trackedChanges.length > 0) {
  237. this.trackedChanges.iterChangedRanges((_fromA, _toA, fromB, toB) => {
  238. if (fromB <= to && toB >= from) {
  239. const fromLine = view.state.doc.lineAt(fromB).number
  240. const toLine = view.state.doc.lineAt(toB).number
  241. for (let i = fromLine; i <= toLine; i++) {
  242. changedLineNumbers.add(i)
  243. }
  244. }
  245. })
  246. } else {
  247. const fromLine = view.state.doc.lineAt(from).number
  248. const toLine = view.state.doc.lineAt(to).number
  249. for (let i = fromLine; i <= toLine; i++) {
  250. changedLineNumbers.add(i)
  251. }
  252. }
  253. const ignoredWords = this.hunspellManager
  254. ? null
  255. : view.state.field(ignoredWordsField)
  256. for (const i of changedLineNumbers) {
  257. const line = view.state.doc.line(i)
  258. wordsToCheck.push(
  259. ...getWordsFromLine(
  260. view,
  261. line,
  262. ignoredWords,
  263. this.language,
  264. this.segmenter
  265. )
  266. )
  267. }
  268. return wordsToCheck
  269. }
  270. }
  271. export class Word {
  272. public text: string
  273. public from: number
  274. public to: number
  275. public lineNumber: number
  276. public lang: string
  277. public suggestions?: WordCacheValue
  278. constructor(options: {
  279. text: string
  280. from: number
  281. to: number
  282. lineNumber: number
  283. lang: string
  284. }) {
  285. const { text, from, to, lineNumber, lang } = options
  286. if (
  287. text == null ||
  288. from == null ||
  289. to == null ||
  290. lineNumber == null ||
  291. lang == null
  292. ) {
  293. throw new OError('Spellcheck: invalid word').withInfo({ options })
  294. }
  295. this.text = text
  296. this.from = from
  297. this.to = to
  298. this.lineNumber = lineNumber
  299. this.lang = lang
  300. }
  301. }
  302. export const buildSpellCheckResult = (
  303. knownMisspelledWords: Word[],
  304. unknownWords: Word[],
  305. misspellings: { index: number; suggestions?: string[] }[]
  306. ) => {
  307. const cacheAdditions: [Word, string[] | boolean][] = []
  308. // Put known misspellings into cache
  309. const misspelledWords = misspellings.map(item => {
  310. const word = {
  311. ...unknownWords[item.index],
  312. }
  313. word.suggestions = item.suggestions
  314. if (word.suggestions) {
  315. cacheAdditions.push([word, word.suggestions])
  316. }
  317. return word
  318. })
  319. // if word was not misspelled, put it in the cache
  320. for (const word of unknownWords) {
  321. if (!misspelledWords.find(mw => mw.text === word.text)) {
  322. cacheAdditions.push([word, true])
  323. }
  324. }
  325. return {
  326. cacheAdditions,
  327. misspelledWords: misspelledWords.concat(knownMisspelledWords),
  328. }
  329. }
  330. export const compileEffects = (results: {
  331. cacheAdditions: [Word, string[] | boolean][]
  332. misspelledWords: Word[]
  333. }) => {
  334. const { cacheAdditions, misspelledWords } = results
  335. return [
  336. addMisspelledWords.of(misspelledWords),
  337. ...cacheAdditions.map(([word, value]) => {
  338. return addWordToCache.of({
  339. lang: word.lang,
  340. wordText: word.text,
  341. value,
  342. })
  343. }),
  344. ]
  345. }
  346. export function* getWordsFromLine(
  347. view: EditorView,
  348. line: Line,
  349. ignoredWords: IgnoredWords | null,
  350. lang: string,
  351. segmenter?: Intl.Segmenter
  352. ) {
  353. for (const span of getNormalTextSpansFromLine(view, line)) {
  354. if (segmenter) {
  355. for (const value of segmenter.segment(span.text)) {
  356. if (value.isWordLike && !ignoredWords?.has(value.segment)) {
  357. const word = value.segment
  358. const from = span.from + value.index
  359. yield new Word({
  360. text: word,
  361. from,
  362. to: from + word.length,
  363. lineNumber: line.number,
  364. lang,
  365. })
  366. }
  367. }
  368. } else {
  369. for (const match of span.text.matchAll(WORD_REGEX)) {
  370. let word = match[0].replace(/'+$/, '')
  371. let from = span.from + match.index
  372. while (word.startsWith("'")) {
  373. word = word.slice(1)
  374. from++
  375. }
  376. if (!ignoredWords?.has(word)) {
  377. yield new Word({
  378. text: word,
  379. from,
  380. to: from + word.length,
  381. lineNumber: line.number,
  382. lang,
  383. })
  384. }
  385. }
  386. }
  387. }
  388. }
  389. export type Mark = Range<RangeValue & { spec: { word: Word } }>
  390. export const getMarkAtPosition = (
  391. view: EditorView,
  392. position: number
  393. ): Mark | null => {
  394. const marks = view.state.field(misspelledWordsField)
  395. let targetMark: Mark | null = null
  396. marks.between(view.viewport.from, view.viewport.to, (from, to, value) => {
  397. if (position >= from && position <= to) {
  398. targetMark = { from, to, value }
  399. return false
  400. }
  401. })
  402. return targetMark
  403. }