spellchecker.ts 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429
  1. import { addMisspelledWords, misspelledWordsField } from './misspelled-words'
  2. import { ignoredWordsField, resetSpellChecker } from './ignored-words'
  3. import { cacheField, addWordToCache, WordCacheValue } from './cache'
  4. import { WORD_REGEX } from './helpers'
  5. import OError from '@overleaf/o-error'
  6. import { EditorView, ViewUpdate } from '@codemirror/view'
  7. import { ChangeSet, Line, Range, RangeValue } from '@codemirror/state'
  8. import { IgnoredWords } from '../../../dictionary/ignored-words'
  9. import { getNormalTextSpansFromLine } from '../../utils/tree-query'
  10. import { waitForParser } from '../wait-for-parser'
  11. import { debugConsole } from '@/utils/debugging'
  12. import type { HunspellManager } from '../../hunspell/HunspellManager'
  13. import { captureException } from '@/infrastructure/error-reporter'
  14. /*
  15. * Spellchecker, handles updates, schedules spelling checks
  16. */
  17. export class SpellChecker {
  18. private abortController?: AbortController | null = null
  19. private timeout: number | null = null
  20. private firstCheck = true
  21. private waitingForParser = false
  22. private firstCheckPending = false
  23. private trackedChanges: ChangeSet
  24. private readonly segmenter?: Intl.Segmenter
  25. // eslint-disable-next-line no-useless-constructor
  26. constructor(
  27. private readonly language: string,
  28. private hunspellManager?: HunspellManager
  29. ) {
  30. debugConsole.log('SpellChecker', language, hunspellManager)
  31. this.trackedChanges = ChangeSet.empty(0)
  32. const locale = language.replace(/_/, '-')
  33. try {
  34. if (Intl.Segmenter) {
  35. const supportedLocales = Intl.Segmenter.supportedLocalesOf([locale], {
  36. localeMatcher: 'lookup',
  37. })
  38. if (supportedLocales.includes(locale)) {
  39. this.segmenter = new Intl.Segmenter(locale, {
  40. localeMatcher: 'lookup',
  41. granularity: 'word',
  42. })
  43. }
  44. }
  45. } catch (error) {
  46. // ignore, not supported for some reason
  47. debugConsole.error(error)
  48. }
  49. if (this.segmenter) {
  50. debugConsole.log(`Using Intl.Segmenter for ${locale}`)
  51. } else {
  52. debugConsole.warn(`Not using Intl.Segmenter for ${locale}`)
  53. }
  54. }
  55. destroy() {
  56. this._clearPendingSpellCheck()
  57. }
  58. _abortRequest() {
  59. if (this.abortController) {
  60. this.abortController.abort()
  61. this.abortController = null
  62. }
  63. }
  64. handleUpdate(update: ViewUpdate) {
  65. if (update.docChanged) {
  66. this.trackedChanges = this.trackedChanges.compose(update.changes)
  67. this.scheduleSpellCheck(update.view)
  68. } else if (update.viewportChanged) {
  69. this.trackedChanges = ChangeSet.empty(0)
  70. this.scheduleSpellCheck(update.view)
  71. } else if (
  72. update.transactions.some(tr => {
  73. return tr.effects.some(effect => effect.is(resetSpellChecker))
  74. })
  75. ) {
  76. // for tests
  77. this.trackedChanges = ChangeSet.empty(0)
  78. this.spellCheckAsap(update.view)
  79. }
  80. // At the point that the spellchecker is initialized, the editor may not
  81. // yet be editable, and the parser may not be ready. Therefore, to do the
  82. // initial spellcheck, watch for changes in the editability of the editor
  83. // and kick off the process that performs a spellcheck once the parser is
  84. // ready. CM6 dispatches a transaction after every chunk of parser work and
  85. // when the editability changes, which means the spell checker is
  86. // initialized as soon as possible.
  87. else if (
  88. this.firstCheck &&
  89. !this.firstCheckPending &&
  90. update.state.facet(EditorView.editable)
  91. ) {
  92. this.firstCheckPending = true
  93. this.spellCheckAsap(update.view)
  94. }
  95. }
  96. _performSpellCheck(view: EditorView) {
  97. const wordsToCheck = this.getWordsToCheck(view)
  98. if (wordsToCheck.length === 0) {
  99. this.trackedChanges = ChangeSet.empty(0)
  100. return
  101. }
  102. const cache = view.state.field(cacheField)
  103. const { knownMisspelledWords, unknownWords } = cache.checkWords(
  104. this.language,
  105. wordsToCheck
  106. )
  107. const processResult = (
  108. misspellings: { index: number; suggestions?: string[] }[]
  109. ) => {
  110. this.trackedChanges = ChangeSet.empty(0)
  111. if (this.firstCheck) {
  112. this.firstCheck = false
  113. this.firstCheckPending = false
  114. }
  115. const result = buildSpellCheckResult(
  116. knownMisspelledWords,
  117. unknownWords,
  118. misspellings
  119. )
  120. view.dispatch({
  121. effects: compileEffects(result),
  122. })
  123. }
  124. if (unknownWords.length === 0) {
  125. processResult([])
  126. } else {
  127. this._abortRequest()
  128. this.abortController = new AbortController()
  129. if (this.hunspellManager) {
  130. const signal = this.abortController.signal
  131. this.hunspellManager.send(
  132. {
  133. type: 'spell',
  134. words: unknownWords.map(word => word.text),
  135. },
  136. result => {
  137. if (!signal.aborted) {
  138. if ('error' in result) {
  139. debugConsole.error(result.error)
  140. captureException(
  141. new Error('Error running spellcheck for word'),
  142. { tags: { ol_spell_check_language: this.language } }
  143. )
  144. } else {
  145. processResult(result.misspellings)
  146. }
  147. }
  148. }
  149. )
  150. }
  151. }
  152. }
  153. suggest(word: string) {
  154. return new Promise<{ suggestions: string[] }>((resolve, reject) => {
  155. if (this.hunspellManager) {
  156. this.hunspellManager.send({ type: 'suggest', word }, result => {
  157. if ('error' in result) {
  158. reject(new Error('Error finding spelling suggestions for word'))
  159. } else {
  160. resolve(result)
  161. }
  162. })
  163. }
  164. })
  165. }
  166. addWord(word: string) {
  167. return new Promise<void>((resolve, reject) => {
  168. if (this.hunspellManager) {
  169. this.hunspellManager.send({ type: 'add_word', word }, result => {
  170. if ('error' in result) {
  171. reject(new Error('Error adding word to spellcheck'))
  172. } else {
  173. resolve()
  174. }
  175. })
  176. }
  177. })
  178. }
  179. removeWord(word: string) {
  180. return new Promise<void>((resolve, reject) => {
  181. if (this.hunspellManager) {
  182. this.hunspellManager.send({ type: 'remove_word', word }, result => {
  183. if ('error' in result) {
  184. reject(new Error('Error removing word from spellcheck'))
  185. } else {
  186. resolve()
  187. }
  188. })
  189. }
  190. })
  191. }
  192. _spellCheckWhenParserReady(view: EditorView) {
  193. if (this.waitingForParser) {
  194. return
  195. }
  196. this.waitingForParser = true
  197. waitForParser(view, view => view.viewport.to).then(() => {
  198. this.waitingForParser = false
  199. this._performSpellCheck(view)
  200. })
  201. }
  202. _clearPendingSpellCheck() {
  203. if (this.timeout) {
  204. window.clearTimeout(this.timeout)
  205. this.timeout = null
  206. }
  207. this._abortRequest()
  208. }
  209. _asyncSpellCheck(view: EditorView, delay: number) {
  210. this._clearPendingSpellCheck()
  211. this.timeout = window.setTimeout(() => {
  212. this._spellCheckWhenParserReady(view)
  213. this.timeout = null
  214. }, delay)
  215. }
  216. spellCheckAsap(view: EditorView) {
  217. this._asyncSpellCheck(view, 0)
  218. }
  219. scheduleSpellCheck(view: EditorView) {
  220. this._asyncSpellCheck(view, 1000)
  221. }
  222. getWordsToCheck(view: EditorView) {
  223. const wordsToCheck: Word[] = []
  224. const { from, to } = view.viewport
  225. const changedLineNumbers = new Set<number>()
  226. if (!this.trackedChanges.empty) {
  227. this.trackedChanges.iterChangedRanges((_fromA, _toA, fromB, toB) => {
  228. if (fromB <= to && toB >= from) {
  229. const fromLine = view.state.doc.lineAt(fromB).number
  230. const toLine = view.state.doc.lineAt(toB).number
  231. for (let i = fromLine; i <= toLine; i++) {
  232. changedLineNumbers.add(i)
  233. }
  234. }
  235. })
  236. } else {
  237. const fromLine = view.state.doc.lineAt(from).number
  238. const toLine = view.state.doc.lineAt(to).number
  239. for (let i = fromLine; i <= toLine; i++) {
  240. changedLineNumbers.add(i)
  241. }
  242. }
  243. const ignoredWords = this.hunspellManager
  244. ? null
  245. : view.state.field(ignoredWordsField)
  246. for (const i of changedLineNumbers) {
  247. const line = view.state.doc.line(i)
  248. wordsToCheck.push(
  249. ...getWordsFromLine(
  250. view,
  251. line,
  252. ignoredWords,
  253. this.language,
  254. this.segmenter
  255. )
  256. )
  257. }
  258. return wordsToCheck
  259. }
  260. }
  261. export class Word {
  262. public text: string
  263. public from: number
  264. public to: number
  265. public lineNumber: number
  266. public lang: string
  267. public suggestions?: WordCacheValue
  268. constructor(options: {
  269. text: string
  270. from: number
  271. to: number
  272. lineNumber: number
  273. lang: string
  274. }) {
  275. const { text, from, to, lineNumber, lang } = options
  276. if (
  277. text == null ||
  278. from == null ||
  279. to == null ||
  280. lineNumber == null ||
  281. lang == null
  282. ) {
  283. throw new OError('Spellcheck: invalid word').withInfo({ options })
  284. }
  285. this.text = text
  286. this.from = from
  287. this.to = to
  288. this.lineNumber = lineNumber
  289. this.lang = lang
  290. }
  291. }
  292. export const buildSpellCheckResult = (
  293. knownMisspelledWords: Word[],
  294. unknownWords: Word[],
  295. misspellings: { index: number; suggestions?: string[] }[]
  296. ) => {
  297. const cacheAdditions: [Word, string[] | boolean][] = []
  298. // Put known misspellings into cache
  299. const misspelledWords = misspellings.map(item => {
  300. const word = {
  301. ...unknownWords[item.index],
  302. }
  303. word.suggestions = item.suggestions
  304. if (word.suggestions) {
  305. cacheAdditions.push([word, word.suggestions])
  306. }
  307. return word
  308. })
  309. // if word was not misspelled, put it in the cache
  310. for (const word of unknownWords) {
  311. if (!misspelledWords.find(mw => mw.text === word.text)) {
  312. cacheAdditions.push([word, true])
  313. }
  314. }
  315. return {
  316. cacheAdditions,
  317. misspelledWords: misspelledWords.concat(knownMisspelledWords),
  318. }
  319. }
  320. export const compileEffects = (results: {
  321. cacheAdditions: [Word, string[] | boolean][]
  322. misspelledWords: Word[]
  323. }) => {
  324. const { cacheAdditions, misspelledWords } = results
  325. return [
  326. addMisspelledWords.of(misspelledWords),
  327. ...cacheAdditions.map(([word, value]) => {
  328. return addWordToCache.of({
  329. lang: word.lang,
  330. wordText: word.text,
  331. value,
  332. })
  333. }),
  334. ]
  335. }
  336. export function* getWordsFromLine(
  337. view: EditorView,
  338. line: Line,
  339. ignoredWords: IgnoredWords | null,
  340. lang: string,
  341. segmenter?: Intl.Segmenter
  342. ) {
  343. for (const span of getNormalTextSpansFromLine(view, line)) {
  344. if (segmenter) {
  345. for (const value of segmenter.segment(span.text)) {
  346. if (value.isWordLike && !ignoredWords?.has(value.segment)) {
  347. const word = value.segment
  348. const from = span.from + value.index
  349. yield new Word({
  350. text: word,
  351. from,
  352. to: from + word.length,
  353. lineNumber: line.number,
  354. lang,
  355. })
  356. }
  357. }
  358. } else {
  359. for (const match of span.text.matchAll(WORD_REGEX)) {
  360. let word = match[0].replace(/'+$/, '')
  361. let from = span.from + match.index
  362. while (word.startsWith("'")) {
  363. word = word.slice(1)
  364. from++
  365. }
  366. if (!ignoredWords?.has(word)) {
  367. yield new Word({
  368. text: word,
  369. from,
  370. to: from + word.length,
  371. lineNumber: line.number,
  372. lang,
  373. })
  374. }
  375. }
  376. }
  377. }
  378. }
  379. export type Mark = Range<RangeValue & { spec: { word: Word } }>
  380. export const getMarkAtPosition = (
  381. view: EditorView,
  382. position: number
  383. ): Mark | null => {
  384. const marks = view.state.field(misspelledWordsField)
  385. let targetMark: Mark | null = null
  386. marks.between(view.viewport.from, view.viewport.to, (from, to, value) => {
  387. if (position >= from && position <= to) {
  388. targetMark = { from, to, value }
  389. return false
  390. }
  391. })
  392. return targetMark
  393. }