Initial decaffeination (#24)

services/spelling/.dockerignore

@@ -5,5 +5,3 @@ gitrev
 .npm
 .nvmrc
 nodemon.json
-app.js
-**/js/*

services/spelling/.eslintrc

@@ -0,0 +1,36 @@
+// this file was auto-generated, do not edit it directly.
+// instead run bin/update_build_scripts from
+// https://github.com/sharelatex/sharelatex-dev-environment
+// Version: 1.1.21
+{
+  "extends": [
+    "standard",
+    "prettier",
+    "prettier/standard",
+  ],
+  "plugins": [
+    "mocha",
+    "chai-expect",
+    "chai-friendly"
+  ],
+  "env": {
+    "node": true
+  },
+  "rules": {
+    // Add some mocha specific rules
+    "mocha/handle-done-callback": "error",
+    "mocha/no-exclusive-tests": "error",
+    "mocha/no-global-tests": "error",
+    "mocha/no-identical-title": "error",
+    "mocha/no-nested-tests": "error",
+    "mocha/no-pending-tests": "error",
+    "mocha/no-skipped-tests": "error",
+
+    // Add some chai specific rules
+    "chai-expect/missing-assertion": "error",
+    "chai-expect/terminating-properties": "error",
+    // Swap the no-unused-expressions rule with a more chai-friendly one
+    "no-unused-expressions": 0,
+    "chai-friendly/no-unused-expressions": "error"
+  }
+}

services/spelling/.gitignore

@@ -1,9 +1,5 @@
 **.swp
 **.swo
-app/js/*
-app.js
-test/UnitTests/js/*
 node_modules/*
-test/unit/js/
 cache/spell.cache
 **/*.map

services/spelling/.prettierrc

@@ -0,0 +1,8 @@
+# This file was auto-generated, do not edit it directly.
+# Instead run bin/update_build_scripts from
+# https://github.com/sharelatex/sharelatex-dev-environment
+# Version: 1.1.21
+{
+  "semi": false,
+  "singleQuote": true
+}

services/spelling/Dockerfile

@@ -10,7 +10,6 @@ RUN npm install --quiet
 COPY . /app
 
 
-RUN npm run compile:all
 
 FROM node:6.16.0
 

services/spelling/Jenkinsfile

@@ -36,6 +36,14 @@ pipeline {
       }
     }
 
+    // should be enabled once Node version is updated to >=8
+    // stage('Linting') {
+    //   steps {
+    //     sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make format'
+    //     sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make lint'
+    //   }
+    // }
+
     stage('Unit Tests') {
       steps {
         sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_unit'

services/spelling/Makefile

@@ -16,12 +16,17 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
 clean:
 	docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
 	docker rmi gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
-	rm -f app.js
-	rm -rf app/js
-	rm -rf test/unit/js
-	rm -rf test/acceptance/js
 
-test: test_unit test_acceptance
+format:
+	$(DOCKER_COMPOSE) run --rm test_unit npm run format
+
+format_fix:
+	$(DOCKER_COMPOSE) run --rm test_unit npm run format:fix
+
+lint:
+	$(DOCKER_COMPOSE) run --rm test_unit npm run lint
+
+test: format lint test_unit test_acceptance
 
 test_unit:
 	@[ ! -d test/unit ] && echo "spelling has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit

services/spelling/app.coffee

@@ -1,47 +0,0 @@
-metrics = require("metrics-sharelatex")
-metrics.initialize("spelling")
-
-
-Settings = require 'settings-sharelatex'
-logger = require 'logger-sharelatex'
-logger.initialize("spelling")
-if Settings.sentry?.dsn?
-	logger.initializeErrorReporting(Settings.sentry.dsn)
-metrics.memory.monitor(logger)
-
-SpellingAPIController = require './app/js/SpellingAPIController'
-express = require('express')
-Path = require("path")
-server = express()
-metrics.injectMetricsRoute(server)
-bodyParser = require('body-parser')
-HealthCheckController = require("./app/js/HealthCheckController")
-
-
-
-server.use bodyParser.json(limit: "2mb")
-server.use metrics.http.monitor(logger)
-
-server.del "/user/:user_id", SpellingAPIController.deleteDic
-server.get "/user/:user_id", SpellingAPIController.getDic
-server.post "/user/:user_id/check", SpellingAPIController.check
-server.post "/user/:user_id/learn", SpellingAPIController.learn
-server.get "/status", (req, res)->
-	res.send(status:'spelling api is up')
-
-server.get "/health_check", HealthCheckController.healthCheck
-
-profiler = require "v8-profiler"
-server.get "/profile", (req, res) ->
-	time = parseInt(req.query.time || "1000")
-	profiler.startProfiling("test")
-	setTimeout () ->
-		profile = profiler.stopProfiling("test")
-		res.json(profile)
-	, time
-
-host = Settings.internal?.spelling?.host || "localhost"
-port = Settings.internal?.spelling?.port || 3005
-server.listen port, host, (error) ->
-	throw error if error?
-	logger.info "spelling starting up, listening on #{host}:#{port}"

services/spelling/app.js

@@ -0,0 +1,68 @@
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS103: Rewrite code to no longer use __guard__
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const metrics = require('metrics-sharelatex')
+metrics.initialize('spelling')
+
+const Settings = require('settings-sharelatex')
+const logger = require('logger-sharelatex')
+logger.initialize('spelling')
+if ((Settings.sentry != null ? Settings.sentry.dsn : undefined) != null) {
+  logger.initializeErrorReporting(Settings.sentry.dsn)
+}
+metrics.memory.monitor(logger)
+
+const SpellingAPIController = require('./app/js/SpellingAPIController')
+const express = require('express')
+const server = express()
+metrics.injectMetricsRoute(server)
+const bodyParser = require('body-parser')
+const HealthCheckController = require('./app/js/HealthCheckController')
+
+server.use(bodyParser.json({ limit: '2mb' }))
+server.use(metrics.http.monitor(logger))
+
+server.del('/user/:user_id', SpellingAPIController.deleteDic)
+server.get('/user/:user_id', SpellingAPIController.getDic)
+server.post('/user/:user_id/check', SpellingAPIController.check)
+server.post('/user/:user_id/learn', SpellingAPIController.learn)
+server.get('/status', (req, res) => res.send({ status: 'spelling api is up' }))
+
+server.get('/health_check', HealthCheckController.healthCheck)
+
+const profiler = require('v8-profiler')
+server.get('/profile', function(req, res) {
+  const time = parseInt(req.query.time || '1000')
+  profiler.startProfiling('test')
+  return setTimeout(function() {
+    const profile = profiler.stopProfiling('test')
+    return res.json(profile)
+  }, time)
+})
+
+const host =
+  __guard__(
+    Settings.internal != null ? Settings.internal.spelling : undefined,
+    x => x.host
+  ) || 'localhost'
+const port =
+  __guard__(
+    Settings.internal != null ? Settings.internal.spelling : undefined,
+    x1 => x1.port
+  ) || 3005
+server.listen(port, host, function(error) {
+  if (error != null) {
+    throw error
+  }
+  return logger.info(`spelling starting up, listening on ${host}:${port}`)
+})
+
+function __guard__(value, transform) {
+  return typeof value !== 'undefined' && value !== null
+    ? transform(value)
+    : undefined
+}

services/spelling/app/coffee/ASpell.coffee

@@ -1,119 +0,0 @@
-async = require "async"
-ASpellWorkerPool = require "./ASpellWorkerPool"
-LRU = require "lru-cache"
-logger = require 'logger-sharelatex'
-fs = require 'fs'
-settings = require("settings-sharelatex")
-Path = require("path")
-
-OneMinute = 60 * 1000
-opts = {max:10000, maxAge: OneMinute * 60 * 10}
-cache = LRU(opts)
-
-cacheFsPath = Path.resolve(settings.cacheDir, "spell.cache")
-cacheFsPathTmp = cacheFsPath + ".tmp"
-
-# load any existing cache
-try
-	oldCache = fs.readFileSync cacheFsPath
-	cache.load JSON.parse(oldCache)
-catch err
-	logger.log err:err, cacheFsPath:cacheFsPath, "could not load the cache file"
-
-# write the cache every 30 minutes
-setInterval () ->
-	dump = JSON.stringify cache.dump()
-	fs.writeFile cacheFsPathTmp, dump, (err) ->
-		if err?
-			logger.log {err}, "error writing cache file"
-			fs.unlink cacheFsPathTmp
-		else
-			fs.rename cacheFsPathTmp, cacheFsPath
-			logger.log {len: dump.length, cacheFsPath:cacheFsPath}, "wrote cache file"
-, 30 * OneMinute
-
-class ASpellRunner
-	checkWords: (language, words, callback = (error, result) ->) ->
-		@runAspellOnWords language, words, (error, output) =>
-			return callback(error) if error?
-			#output = @removeAspellHeader(output)
-			suggestions = @getSuggestions(language, output)
-			results = []
-			hits = 0
-			addToCache = {}
-			for word, i in words
-				key = language + ':' + word
-				cached = cache.get(key)
-				if cached?
-					hits++
-					if	cached == true
-						# valid word, no need to do anything
-						continue
-					else
-						results.push index: i, suggestions: cached
-				else
-					if suggestions[key]?
-						addToCache[key] = suggestions[key]
-						results.push index: i, suggestions: suggestions[key]
-					else
-						# a valid word, but uncached
-						addToCache[key] = true
-
-			# update the cache after processing all words, to avoid cache
-			# changing while we use it
-			for k, v of addToCache
-				cache.set(k, v)
-
-			logger.info hits: hits, total: words.length, hitrate: (hits/words.length).toFixed(2), "cache hit rate"
-			callback null, results
-
-	getSuggestions: (language, output) ->
-		lines = output.split("\n")
-		suggestions = {}
-		for line in lines
-			if line[0] == "&" # Suggestions found
-				parts = line.split(" ")
-				if parts.length > 1
-					word = parts[1]
-					suggestionsString = line.slice(line.indexOf(":") + 2)
-					suggestions[language + ":" + word] = suggestionsString.split(", ")
-			else if line[0] == "#" # No suggestions
-				parts = line.split(" ")
-				if parts.length > 1
-					word = parts[1]
-					suggestions[language + ":" + word] = []
-		return suggestions
-
-	#removeAspellHeader: (output) -> output.slice(1)
-
-	runAspellOnWords: (language, words, callback = (error, output) ->) ->
-		# send words to aspell, get back string output for those words
-		# find a free pipe for the language (or start one)
-		# send the words down the pipe
-		# send an END marker that will generate a "*" line in the output
-		# when the output pipe receives the "*" return the data sofar and reset the pipe to be available
-		#
-		# @open(language)
-		# @captureOutput(callback)
-		# @setTerseMode()
-		# start = new Date()
-
-		newWord = {}
-		for word in words
-			newWord[word] = true if !newWord[word] && !cache.has(language + ':' + word)
-		words = Object.keys(newWord)
-
-		if words.length
-			WorkerPool.check(language, words, ASpell.ASPELL_TIMEOUT, callback)
-		else
-			callback null, ""
-
-module.exports = ASpell =
-	# The description of how to call aspell from another program can be found here:
-	# http://aspell.net/man-html/Through-A-Pipe.html
-	checkWords: (language, words, callback = (error, result) ->) ->
-		runner = new ASpellRunner()
-		runner.checkWords language, words, callback
-	ASPELL_TIMEOUT : 4000
-
-WorkerPool = new ASpellWorkerPool()

services/spelling/app/coffee/ASpellWorker.coffee

@@ -1,117 +0,0 @@
-child_process = require("child_process")
-logger = require 'logger-sharelatex'
-metrics = require('metrics-sharelatex')
-_ = require "underscore"
-
-BATCH_SIZE = 100
-
-class ASpellWorker
-	constructor: (language) ->
-		@language = language
-		@count = 0
-		@pipe = child_process.spawn("aspell", ["pipe", "-t", "--encoding=utf-8", "-d", language])
-		logger.info process: @pipe.pid, lang: @language, "starting new aspell worker"
-		metrics.inc "aspellWorker", 1, {status: "start", method: @language}
-		@pipe.on 'exit', () =>
-			@state = 'killed'
-			logger.info process: @pipe.pid, lang: @language, "aspell worker has exited"
-			metrics.inc "aspellWorker" , 1, {status: "exit", method: @language}
-		@pipe.on 'close', () =>
-			@state = 'closed' unless @state == 'killed'
-			if @callback?
-				logger.err process: @pipe.pid, lang: @language, "aspell worker closed output streams with uncalled callback"
-				@callback new Error("aspell worker closed output streams with uncalled callback"), []
-				@callback = null
-		@pipe.on 'error', (err) =>
-			@state = 'error' unless @state == 'killed'
-			logger.log process: @pipe.pid, error: err, stdout: output.slice(-1024), stderr: error.slice(-1024), lang: @language, "aspell worker error"
-			if @callback?
-				@callback err, []
-				@callback = null
-		@pipe.stdin.on 'error', (err) =>
-			@state = 'error' unless @state == 'killed'
-			logger.info process: @pipe.pid, error: err, stdout: output.slice(-1024), stderr: error.slice(-1024), lang: @language, "aspell worker error on stdin"
-			if @callback?
-				@callback err, []
-				@callback = null
-
-		output = ""
-		endMarker = new RegExp("^[a-z][a-z]", "m")
-		@pipe.stdout.on "data", (chunk) =>
-			output = output + chunk
-			# We receive the language code from Aspell as the end of data marker
-			if chunk.toString().match(endMarker)
-				if @callback?
-					@callback(null, output.slice())
-					@callback = null # only allow one callback in use
-				else
-					logger.err process: @pipe.pid, lang: @language, "end of data marker received when callback already used"
-				@state = 'ready'
-				output = ""
-				error = ""
-
-		error = ""
-		@pipe.stderr.on "data", (chunk) =>
-			error = error + chunk
-
-		@pipe.stdout.on "end", () =>
-			# process has ended
-			@state = "end"
-
-	isReady: () ->
-		return @state == 'ready'
-
-	check: (words, callback) ->
-		# we will now send data to aspell, and be ready again when we
-		# receive the end of data marker
-		@state = 'busy'
-		if @callback? # only allow one callback in use
-			logger.err process: @pipe.pid, lang: @language, "CALLBACK ALREADY IN USE"
-			return @callback new Error("Aspell callback already in use - SHOULD NOT HAPPEN")
-		@callback = _.once callback # extra defence against double callback
-		@setTerseMode()
-		@write(words)
-		@flush()
-
-	write: (words) ->
-		i = 0
-		while i < words.length
-			# batch up the words to check for efficiency
-			batch = words.slice(i, i + BATCH_SIZE)
-			@sendWords batch
-			i += BATCH_SIZE
-
-	flush: () ->
-		# get aspell to send an end of data marker "*" when ready
-		#@sendCommand("%")		# take the aspell pipe out of terse mode so we can look for a '*'
-		#@sendCommand("^ENDOFSTREAMMARKER") # send our marker which will generate a '*'
-		#@sendCommand("!")		# go back into terse mode
-		@sendCommand("$$l")
-
-	shutdown: (reason) ->
-		logger.info process: @pipe.pid, reason: reason, 'shutting down'
-		@state = "closing"
-		@pipe.stdin.end()
-
-	kill: (reason) ->
-		logger.info process: @pipe.pid, reason: reason, 'killing'
-		return if @state == 'killed'
-		@pipe.kill('SIGKILL')
-
-	setTerseMode: () ->
-		@sendCommand("!")
-
-	sendWord: (word) ->
-		@sendCommand("^" + word)
-
-	sendWords: (words) ->
-		# Aspell accepts multiple words to check on the same line
-		# ^word1 word2 word3 ...
-		# See aspell.info, writing programs to use Aspell Through A Pipe
-		@sendCommand("^" + words.join(" "))
-		@count++
-
-	sendCommand: (command) ->
-		@pipe.stdin.write(command + "\n")
-
-module.exports = ASpellWorker

services/spelling/app/coffee/ASpellWorkerPool.coffee

@@ -1,76 +0,0 @@
-ASpellWorker = require "./ASpellWorker"
-_ = require "underscore"
-logger = require 'logger-sharelatex'
-metrics = require('metrics-sharelatex')
-
-class ASpellWorkerPool
-	MAX_REQUESTS: 100*1024
-	MAX_WORKERS: 32
-	MAX_IDLE_TIME: 1000
-	MAX_REQUEST_TIME: 60*1000
-
-	constructor: (@options) ->
-		@PROCESS_POOL = []
-
-	create: (language) ->
-		if @PROCESS_POOL.length >= @MAX_WORKERS
-			logger.log maxworkers: @MAX_WORKERS, "maximum number of workers already running"
-			return null
-		worker = new ASpellWorker(language, @options)
-		worker.pipe.on 'exit', () =>
-			if worker.killTimer?
-				clearTimeout worker.killTimer
-				worker.killTimer = null
-			if worker.idleTimer?
-				clearTimeout worker.idleTimer
-				worker.idleTimer = null
-			logger.info process: worker.pipe.pid, lang: language, "removing aspell worker from pool"
-			@cleanup()
-		@PROCESS_POOL.push(worker)
-		metrics.gauge 'aspellWorkerPool-size', @PROCESS_POOL.length
-		return worker
-
-	cleanup: () ->
-		active = @PROCESS_POOL.filter (worker) ->
-			worker.state != 'killed'
-		@PROCESS_POOL = active
-		metrics.gauge 'aspellWorkerPool-size', @PROCESS_POOL.length
-
-	check: (language, words, timeout, callback) ->
-		# look for an existing process in the pool
-		availableWorker = _.find @PROCESS_POOL, (cached) ->
-			cached.language == language && cached.isReady()
-		if not availableWorker?
-			worker = @create(language)
-		else
-			worker = availableWorker
-
-		if not worker?
-			# return error if too many workers
-			callback(new Error("no worker available"))
-			return
-
-		if worker.idleTimer?
-			clearTimeout worker.idleTimer
-			worker.idleTimer = null
-
-		worker.killTimer = setTimeout () ->
-			worker.kill("spell check timed out")
-		, timeout || @MAX_REQUEST_TIME
-
-		worker.check words, (err, output) =>
-			if worker.killTimer?
-				clearTimeout worker.killTimer
-				worker.killTimer = null
-			callback(err, output)
-			return if err? # process has shut down
-			if worker.count > @MAX_REQUESTS
-				worker.shutdown("reached limit of " + @MAX_REQUESTS + " requests")
-			else
-				# queue a shutdown if worker is idle
-				worker.idleTimer = setTimeout () ->
-					worker.shutdown("idle worker")
-					worker.idleTimer = null
-				, @MAX_IDLE_TIME
-
-module.exports = ASpellWorkerPool

services/spelling/app/coffee/DB.coffee

@@ -1,4 +0,0 @@
-MongoJS = require "mongojs"
-Settings = require "settings-sharelatex"
-module.exports = MongoJS(Settings.mongo.url, ["spellingPreferences"])
-

services/spelling/app/coffee/HealthCheckController.coffee

@@ -1,23 +0,0 @@
-request = require("request")
-logger = require 'logger-sharelatex'
-settings = require 'settings-sharelatex'
-
-module.exports =
-
-	healthCheck: (req, res)->
-		opts =
-			url: "http://localhost:3005/user/#{settings.healthCheckUserId}/check"
-			json:
-				words:["helllo"]
-				language: "en"
-			timeout: 1000 * 20
-		request.post opts, (err, response, body)->
-			if err?
-				return res.sendStatus 500
-			numberOfSuggestions = body?.misspellings?[0]?.suggestions?.length
-			if numberOfSuggestions > 10
-				logger.log "health check passed"
-				res.sendStatus 200
-			else
-				logger.err body:body, numberOfSuggestions:numberOfSuggestions, "health check failed"
-				res.sendStatus 500

services/spelling/app/coffee/LearnedWordsManager.coffee

@@ -1,40 +0,0 @@
-db = require("./DB")
-mongoCache = require("./MongoCache")
-logger = require 'logger-sharelatex'
-metrics = require('metrics-sharelatex')
-
-module.exports = LearnedWordsManager =
-	learnWord: (user_token, word, callback = (error)->) ->
-		mongoCache.del(user_token)
-		db.spellingPreferences.update {
-			token: user_token
-		}, {
-			$push: learnedWords: word
-		}, {
-			upsert: true
-		}, callback
-
-	getLearnedWords: (user_token, callback = (error, words)->) ->
-		mongoCachedWords = mongoCache.get(user_token)
-		if mongoCachedWords?
-			metrics.inc "mongoCache", 0.1, {status: "hit"}
-			return callback(null, mongoCachedWords)
-
-		metrics.inc "mongoCache", 0.1, {status: "miss"}
-		logger.info user_token:user_token, "mongoCache miss"
-		
-		db.spellingPreferences.findOne token: user_token, (error, preferences) ->
-			return callback error if error?
-			words = preferences?.learnedWords || []
-			mongoCache.set(user_token, words)
-			callback null, words
-
-	deleteUsersLearnedWords: (user_token, callback =(error)->)->
-		db.spellingPreferences.remove token: user_token, callback
-
-
-[
-	'learnWord',
-	'getLearnedWords'
-].map (method) ->
-	metrics.timeAsyncMethod(LearnedWordsManager, method, 'mongo.LearnedWordsManager', logger)

services/spelling/app/coffee/MongoCache.coffee

@@ -1,8 +0,0 @@
-LRU = require("lru-cache")
-cacheOpts = 
-	max: 15000
-	maxAge: 1000 * 60 * 60 * 10
-
-cache = LRU(cacheOpts)
-
-module.exports = cache

services/spelling/app/coffee/SpellingAPIController.coffee

@@ -1,35 +0,0 @@
-SpellingAPIManager = require './SpellingAPIManager'
-logger = require 'logger-sharelatex'
-metrics = require('metrics-sharelatex')
-
-module.exports = SpellingAPIController =
-	check: (req, res, next) ->
-		metrics.inc "spelling-check", 0.1
-		logger.info token: req?.params?.user_id, word_count: req?.body?.words?.length, "running check"
-		SpellingAPIManager.runRequest req.params.user_id, req.body, (error, result) ->
-			if error?
-				logger.err err:error, user_id:req?.params?.user_id, word_count: req?.body?.words?.length, "error processing spelling request"
-				return res.sendStatus(500)
-			res.send(result)
-
-	learn: (req, res, next) ->
-		metrics.inc "spelling-learn", 0.1
-		logger.info token: req?.params?.user_id, word: req?.body?.word, "learning word"
-		SpellingAPIManager.learnWord req.params.user_id, req.body, (error, result) ->
-			return next(error) if error?
-			res.sendStatus(200)
-			next()
-
-
-	deleteDic: (req, res, next)->
-		logger.log token: req?.params?.user_id, word: req?.body?.word, "deleting user dictionary"
-		SpellingAPIManager.deleteDic req.params.user_id, (error) ->
-			return next(error) if error?
-			res.sendStatus(204)
-
-
-	getDic: (req, res, next)->
-		logger.info token: req?.params?.user_id, "getting user dictionary"
-		SpellingAPIManager.getDic req.params.user_id, (error, words)->
-			return next(error) if error?
-			res.send(words)

services/spelling/app/coffee/SpellingAPIManager.coffee

@@ -1,54 +0,0 @@
-ASpell = require './ASpell'
-LearnedWordsManager = require './LearnedWordsManager'
-async = require 'async'
-
-module.exports = SpellingAPIManager =
-
-	whitelist: [
-		'ShareLaTeX',
-		'sharelatex',
-		'LaTeX',
-		'http',
-		'https',
-		'www'
-	]
-
-	runRequest: (token, request, callback = (error, result) ->) ->
-		if !request.words?
-			return callback(new Error("malformed JSON"))
-
-		lang = request.language || "en"
-
-		check = (words, callback) ->
-			ASpell.checkWords lang, words, (error, misspellings) ->
-				callback error, misspellings: misspellings
-
-		wordsToCheck = request.words || []
-
-		if token?
-			LearnedWordsManager.getLearnedWords token, (error, learnedWords) ->
-				return callback(error) if error?
-				words = (wordsToCheck).slice(0,10000)
-				check words, (error, result) ->
-					return callback error if error?
-					result.misspellings = result.misspellings.filter (m) ->
-						word = words[m.index]
-						learnedWords.indexOf(word) == -1 and SpellingAPIManager.whitelist.indexOf(word) == -1
-					callback error, result
-		else
-			check(wordsToCheck, callback)
-
-	learnWord: (token, request, callback = (error) ->) ->
-		if !request.word?
-			return callback(new Error("malformed JSON"))
-		if !token?
-			return callback(new Error("no token provided"))
-
-		LearnedWordsManager.learnWord token, request.word, callback
-
-	deleteDic: (token, callback)->
-		LearnedWordsManager.deleteUsersLearnedWords token, callback
-
-	getDic: (token, callback)->
-		LearnedWordsManager.getLearnedWords token, callback
-

services/spelling/app/js/ASpell.js

@@ -0,0 +1,175 @@
+// TODO: This file was created by bulk-decaffeinate.
+// Sanity-check the conversion and remove this comment.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let ASpell
+const ASpellWorkerPool = require('./ASpellWorkerPool')
+const LRU = require('lru-cache')
+const logger = require('logger-sharelatex')
+const fs = require('fs')
+const settings = require('settings-sharelatex')
+const Path = require('path')
+
+const OneMinute = 60 * 1000
+const opts = { max: 10000, maxAge: OneMinute * 60 * 10 }
+const cache = LRU(opts)
+
+const cacheFsPath = Path.resolve(settings.cacheDir, 'spell.cache')
+const cacheFsPathTmp = cacheFsPath + '.tmp'
+
+// load any existing cache
+try {
+  const oldCache = fs.readFileSync(cacheFsPath)
+  cache.load(JSON.parse(oldCache))
+} catch (error) {
+  const err = error
+  logger.log({ err, cacheFsPath }, 'could not load the cache file')
+}
+
+// write the cache every 30 minutes
+setInterval(function() {
+  const dump = JSON.stringify(cache.dump())
+  return fs.writeFile(cacheFsPathTmp, dump, function(err) {
+    if (err != null) {
+      logger.log({ err }, 'error writing cache file')
+      return fs.unlink(cacheFsPathTmp)
+    } else {
+      fs.rename(cacheFsPathTmp, cacheFsPath)
+      return logger.log({ len: dump.length, cacheFsPath }, 'wrote cache file')
+    }
+  })
+}, 30 * OneMinute)
+
+class ASpellRunner {
+  checkWords(language, words, callback) {
+    if (callback == null) {
+      callback = () => {}
+    }
+    return this.runAspellOnWords(language, words, (error, output) => {
+      if (error != null) {
+        return callback(error)
+      }
+      // output = @removeAspellHeader(output)
+      const suggestions = this.getSuggestions(language, output)
+      const results = []
+      let hits = 0
+      const addToCache = {}
+      for (let i = 0; i < words.length; i++) {
+        const word = words[i]
+        const key = language + ':' + word
+        const cached = cache.get(key)
+        if (cached != null) {
+          hits++
+          if (cached === true) {
+            // valid word, no need to do anything
+            continue
+          } else {
+            results.push({ index: i, suggestions: cached })
+          }
+        } else {
+          if (suggestions[key] != null) {
+            addToCache[key] = suggestions[key]
+            results.push({ index: i, suggestions: suggestions[key] })
+          } else {
+            // a valid word, but uncached
+            addToCache[key] = true
+          }
+        }
+      }
+
+      // update the cache after processing all words, to avoid cache
+      // changing while we use it
+      for (let k in addToCache) {
+        const v = addToCache[k]
+        cache.set(k, v)
+      }
+
+      logger.info(
+        {
+          hits,
+          total: words.length,
+          hitrate: (hits / words.length).toFixed(2)
+        },
+        'cache hit rate'
+      )
+      return callback(null, results)
+    })
+  }
+
+  getSuggestions(language, output) {
+    const lines = output.split('\n')
+    const suggestions = {}
+    for (let line of Array.from(lines)) {
+      var parts, word
+      if (line[0] === '&') {
+        // Suggestions found
+        parts = line.split(' ')
+        if (parts.length > 1) {
+          word = parts[1]
+          const suggestionsString = line.slice(line.indexOf(':') + 2)
+          suggestions[language + ':' + word] = suggestionsString.split(', ')
+        }
+      } else if (line[0] === '#') {
+        // No suggestions
+        parts = line.split(' ')
+        if (parts.length > 1) {
+          word = parts[1]
+          suggestions[language + ':' + word] = []
+        }
+      }
+    }
+    return suggestions
+  }
+
+  // removeAspellHeader: (output) -> output.slice(1)
+
+  runAspellOnWords(language, words, callback) {
+    // send words to aspell, get back string output for those words
+    // find a free pipe for the language (or start one)
+    // send the words down the pipe
+    // send an END marker that will generate a "*" line in the output
+    // when the output pipe receives the "*" return the data sofar and reset the pipe to be available
+    //
+    // @open(language)
+    // @captureOutput(callback)
+    // @setTerseMode()
+    // start = new Date()
+
+    if (callback == null) {
+      callback = () => {}
+    }
+    const newWord = {}
+    for (let word of Array.from(words)) {
+      if (!newWord[word] && !cache.has(language + ':' + word)) {
+        newWord[word] = true
+      }
+    }
+    words = Object.keys(newWord)
+
+    if (words.length) {
+      return WorkerPool.check(language, words, ASpell.ASPELL_TIMEOUT, callback)
+    } else {
+      return callback(null, '')
+    }
+  }
+}
+
+module.exports = ASpell = {
+  // The description of how to call aspell from another program can be found here:
+  // http://aspell.net/man-html/Through-A-Pipe.html
+  checkWords(language, words, callback) {
+    if (callback == null) {
+      callback = () => {}
+    }
+    const runner = new ASpellRunner()
+    return runner.checkWords(language, words, callback)
+  },
+  ASPELL_TIMEOUT: 4000
+}
+
+var WorkerPool = new ASpellWorkerPool()

services/spelling/app/js/ASpellWorker.js

@@ -0,0 +1,213 @@
+// TODO: This file was created by bulk-decaffeinate.
+// Sanity-check the conversion and remove this comment.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS205: Consider reworking code to avoid use of IIFEs
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const childProcess = require('child_process')
+const logger = require('logger-sharelatex')
+const metrics = require('metrics-sharelatex')
+const _ = require('underscore')
+
+const BATCH_SIZE = 100
+
+class ASpellWorker {
+  constructor(language) {
+    this.language = language
+    this.count = 0
+    this.pipe = childProcess.spawn('aspell', [
+      'pipe',
+      '-t',
+      '--encoding=utf-8',
+      '-d',
+      language
+    ])
+    logger.info(
+      { process: this.pipe.pid, lang: this.language },
+      'starting new aspell worker'
+    )
+    metrics.inc('aspellWorker', 1, { status: 'start', method: this.language })
+    this.pipe.on('exit', () => {
+      this.state = 'killed'
+      logger.info(
+        { process: this.pipe.pid, lang: this.language },
+        'aspell worker has exited'
+      )
+      return metrics.inc('aspellWorker', 1, {
+        status: 'exit',
+        method: this.language
+      })
+    })
+    this.pipe.on('close', () => {
+      if (this.state !== 'killed') {
+        this.state = 'closed'
+      }
+      if (this.callback != null) {
+        logger.err(
+          { process: this.pipe.pid, lang: this.language },
+          'aspell worker closed output streams with uncalled callback'
+        )
+        this.callback(
+          new Error(
+            'aspell worker closed output streams with uncalled callback'
+          ),
+          []
+        )
+        return (this.callback = null)
+      }
+    })
+    this.pipe.on('error', err => {
+      if (this.state !== 'killed') {
+        this.state = 'error'
+      }
+      logger.log(
+        {
+          process: this.pipe.pid,
+          error: err,
+          stdout: output.slice(-1024),
+          stderr: error.slice(-1024),
+          lang: this.language
+        },
+        'aspell worker error'
+      )
+      if (this.callback != null) {
+        this.callback(err, [])
+        return (this.callback = null)
+      }
+    })
+    this.pipe.stdin.on('error', err => {
+      if (this.state !== 'killed') {
+        this.state = 'error'
+      }
+      logger.info(
+        {
+          process: this.pipe.pid,
+          error: err,
+          stdout: output.slice(-1024),
+          stderr: error.slice(-1024),
+          lang: this.language
+        },
+        'aspell worker error on stdin'
+      )
+      if (this.callback != null) {
+        this.callback(err, [])
+        return (this.callback = null)
+      }
+    })
+
+    var output = ''
+    const endMarker = new RegExp('^[a-z][a-z]', 'm')
+    this.pipe.stdout.on('data', chunk => {
+      output = output + chunk
+      // We receive the language code from Aspell as the end of data marker
+      if (chunk.toString().match(endMarker)) {
+        if (this.callback != null) {
+          this.callback(null, output.slice())
+          this.callback = null // only allow one callback in use
+        } else {
+          logger.err(
+            { process: this.pipe.pid, lang: this.language },
+            'end of data marker received when callback already used'
+          )
+        }
+        this.state = 'ready'
+        output = ''
+      }
+    })
+
+    var error = ''
+    this.pipe.stderr.on('data', chunk => {
+      return (error = error + chunk)
+    })
+
+    this.pipe.stdout.on('end', () => {
+      // process has ended
+      return (this.state = 'end')
+    })
+  }
+
+  isReady() {
+    return this.state === 'ready'
+  }
+
+  check(words, callback) {
+    // we will now send data to aspell, and be ready again when we
+    // receive the end of data marker
+    this.state = 'busy'
+    if (this.callback != null) {
+      // only allow one callback in use
+      logger.err(
+        { process: this.pipe.pid, lang: this.language },
+        'CALLBACK ALREADY IN USE'
+      )
+      return this.callback(
+        new Error('Aspell callback already in use - SHOULD NOT HAPPEN')
+      )
+    }
+    this.callback = _.once(callback) // extra defence against double callback
+    this.setTerseMode()
+    this.write(words)
+    return this.flush()
+  }
+
+  write(words) {
+    let i = 0
+    return (() => {
+      const result = []
+      while (i < words.length) {
+        // batch up the words to check for efficiency
+        const batch = words.slice(i, i + BATCH_SIZE)
+        this.sendWords(batch)
+        result.push((i += BATCH_SIZE))
+      }
+      return result
+    })()
+  }
+
+  flush() {
+    // get aspell to send an end of data marker "*" when ready
+    // @sendCommand("%")		# take the aspell pipe out of terse mode so we can look for a '*'
+    // @sendCommand("^ENDOFSTREAMMARKER") # send our marker which will generate a '*'
+    // @sendCommand("!")		# go back into terse mode
+    return this.sendCommand('$$l')
+  }
+
+  shutdown(reason) {
+    logger.info({ process: this.pipe.pid, reason }, 'shutting down')
+    this.state = 'closing'
+    return this.pipe.stdin.end()
+  }
+
+  kill(reason) {
+    logger.info({ process: this.pipe.pid, reason }, 'killing')
+    if (this.state === 'killed') {
+      return
+    }
+    return this.pipe.kill('SIGKILL')
+  }
+
+  setTerseMode() {
+    return this.sendCommand('!')
+  }
+
+  sendWord(word) {
+    return this.sendCommand(`^${word}`)
+  }
+
+  sendWords(words) {
+    // Aspell accepts multiple words to check on the same line
+    // ^word1 word2 word3 ...
+    // See aspell.info, writing programs to use Aspell Through A Pipe
+    this.sendCommand(`^${words.join(' ')}`)
+    return this.count++
+  }
+
+  sendCommand(command) {
+    return this.pipe.stdin.write(command + '\n')
+  }
+}
+
+module.exports = ASpellWorker

services/spelling/app/js/ASpellWorkerPool.js

@@ -0,0 +1,115 @@
+// TODO: This file was created by bulk-decaffeinate.
+// Sanity-check the conversion and remove this comment.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS206: Consider reworking classes to avoid initClass
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const ASpellWorker = require('./ASpellWorker')
+const _ = require('underscore')
+const logger = require('logger-sharelatex')
+const metrics = require('metrics-sharelatex')
+
+class ASpellWorkerPool {
+  static initClass() {
+    this.prototype.MAX_REQUESTS = 100 * 1024
+    this.prototype.MAX_WORKERS = 32
+    this.prototype.MAX_IDLE_TIME = 1000
+    this.prototype.MAX_REQUEST_TIME = 60 * 1000
+  }
+
+  constructor(options) {
+    this.options = options
+    this.PROCESS_POOL = []
+  }
+
+  create(language) {
+    if (this.PROCESS_POOL.length >= this.MAX_WORKERS) {
+      logger.log(
+        { maxworkers: this.MAX_WORKERS },
+        'maximum number of workers already running'
+      )
+      return null
+    }
+    const worker = new ASpellWorker(language, this.options)
+    worker.pipe.on('exit', () => {
+      if (worker.killTimer != null) {
+        clearTimeout(worker.killTimer)
+        worker.killTimer = null
+      }
+      if (worker.idleTimer != null) {
+        clearTimeout(worker.idleTimer)
+        worker.idleTimer = null
+      }
+      logger.info(
+        { process: worker.pipe.pid, lang: language },
+        'removing aspell worker from pool'
+      )
+      return this.cleanup()
+    })
+    this.PROCESS_POOL.push(worker)
+    metrics.gauge('aspellWorkerPool-size', this.PROCESS_POOL.length)
+    return worker
+  }
+
+  cleanup() {
+    const active = this.PROCESS_POOL.filter(worker => worker.state !== 'killed')
+    this.PROCESS_POOL = active
+    return metrics.gauge('aspellWorkerPool-size', this.PROCESS_POOL.length)
+  }
+
+  check(language, words, timeout, callback) {
+    // look for an existing process in the pool
+    let worker
+    const availableWorker = _.find(
+      this.PROCESS_POOL,
+      cached => cached.language === language && cached.isReady()
+    )
+    if (availableWorker == null) {
+      worker = this.create(language)
+    } else {
+      worker = availableWorker
+    }
+
+    if (worker == null) {
+      // return error if too many workers
+      callback(new Error('no worker available'))
+      return
+    }
+
+    if (worker.idleTimer != null) {
+      clearTimeout(worker.idleTimer)
+      worker.idleTimer = null
+    }
+
+    worker.killTimer = setTimeout(
+      () => worker.kill('spell check timed out'),
+      timeout || this.MAX_REQUEST_TIME
+    )
+
+    return worker.check(words, (err, output) => {
+      if (worker.killTimer != null) {
+        clearTimeout(worker.killTimer)
+        worker.killTimer = null
+      }
+      callback(err, output)
+      if (err != null) {
+        return
+      } // process has shut down
+      if (worker.count > this.MAX_REQUESTS) {
+        return worker.shutdown(`reached limit of ${this.MAX_REQUESTS} requests`)
+      } else {
+        // queue a shutdown if worker is idle
+        return (worker.idleTimer = setTimeout(function() {
+          worker.shutdown('idle worker')
+          return (worker.idleTimer = null)
+        }, this.MAX_IDLE_TIME))
+      }
+    })
+  }
+}
+ASpellWorkerPool.initClass()
+
+module.exports = ASpellWorkerPool

services/spelling/app/js/DB.js

@@ -0,0 +1,5 @@
+// TODO: This file was created by bulk-decaffeinate.
+// Sanity-check the conversion and remove this comment.
+const MongoJS = require('mongojs')
+const Settings = require('settings-sharelatex')
+module.exports = MongoJS(Settings.mongo.url, ['spellingPreferences'])

services/spelling/app/js/HealthCheckController.js

@@ -0,0 +1,50 @@
+// TODO: This file was created by bulk-decaffeinate.
+// Sanity-check the conversion and remove this comment.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS103: Rewrite code to no longer use __guard__
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const request = require('request')
+const logger = require('logger-sharelatex')
+const settings = require('settings-sharelatex')
+
+module.exports = {
+  healthCheck(req, res) {
+    const opts = {
+      url: `http://localhost:3005/user/${settings.healthCheckUserId}/check`,
+      json: {
+        words: ['helllo'],
+        language: 'en'
+      },
+      timeout: 1000 * 20
+    }
+    return request.post(opts, function(err, response, body) {
+      if (err != null) {
+        return res.sendStatus(500)
+      }
+      const numberOfSuggestions = __guard__(
+        __guard__(
+          __guard__(body != null ? body.misspellings : undefined, x2 => x2[0]),
+          x1 => x1.suggestions
+        ),
+        x => x.length
+      )
+      if (numberOfSuggestions > 10) {
+        logger.log('health check passed')
+        return res.sendStatus(200)
+      } else {
+        logger.err({ body, numberOfSuggestions }, 'health check failed')
+        return res.sendStatus(500)
+      }
+    })
+  }
+}
+
+function __guard__(value, transform) {
+  return typeof value !== 'undefined' && value !== null
+    ? transform(value)
+    : undefined
+}

services/spelling/app/js/LearnedWordsManager.js

@@ -0,0 +1,76 @@
+// TODO: This file was created by bulk-decaffeinate.
+// Sanity-check the conversion and remove this comment.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let LearnedWordsManager
+const db = require('./DB')
+const mongoCache = require('./MongoCache')
+const logger = require('logger-sharelatex')
+const metrics = require('metrics-sharelatex')
+
+module.exports = LearnedWordsManager = {
+  learnWord(userToken, word, callback) {
+    if (callback == null) {
+      callback = () => {}
+    }
+    mongoCache.del(userToken)
+    return db.spellingPreferences.update(
+      {
+        token: userToken
+      },
+      {
+        $push: { learnedWords: word }
+      },
+      {
+        upsert: true
+      },
+      callback
+    )
+  },
+
+  getLearnedWords(userToken, callback) {
+    if (callback == null) {
+      callback = () => {}
+    }
+    const mongoCachedWords = mongoCache.get(userToken)
+    if (mongoCachedWords != null) {
+      metrics.inc('mongoCache', 0.1, { status: 'hit' })
+      return callback(null, mongoCachedWords)
+    }
+
+    metrics.inc('mongoCache', 0.1, { status: 'miss' })
+    logger.info({ userToken }, 'mongoCache miss')
+
+    return db.spellingPreferences.findOne({ token: userToken }, function(
+      error,
+      preferences
+    ) {
+      if (error != null) {
+        return callback(error)
+      }
+      const words =
+        (preferences != null ? preferences.learnedWords : undefined) || []
+      mongoCache.set(userToken, words)
+      return callback(null, words)
+    })
+  },
+
+  deleteUsersLearnedWords(userToken, callback) {
+    if (callback == null) {
+      callback = () => {}
+    }
+    return db.spellingPreferences.remove({ token: userToken }, callback)
+  }
+}
+;['learnWord', 'getLearnedWords'].map(method =>
+  metrics.timeAsyncMethod(
+    LearnedWordsManager,
+    method,
+    'mongo.LearnedWordsManager',
+    logger
+  )
+)

services/spelling/app/js/MongoCache.js

@@ -0,0 +1,11 @@
+// TODO: This file was created by bulk-decaffeinate.
+// Sanity-check the conversion and remove this comment.
+const LRU = require('lru-cache')
+const cacheOpts = {
+  max: 15000,
+  maxAge: 1000 * 60 * 60 * 10
+}
+
+const cache = LRU(cacheOpts)
+
+module.exports = cache

services/spelling/app/js/SpellingAPIController.js

@@ -0,0 +1,111 @@
+// TODO: This file was created by bulk-decaffeinate.
+// Sanity-check the conversion and remove this comment.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS103: Rewrite code to no longer use __guard__
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const SpellingAPIManager = require('./SpellingAPIManager')
+const logger = require('logger-sharelatex')
+const metrics = require('metrics-sharelatex')
+
+module.exports = {
+  check(req, res, next) {
+    metrics.inc('spelling-check', 0.1)
+    logger.info(
+      {
+        token: __guard__(req != null ? req.params : undefined, x => x.user_id),
+        word_count: __guard__(
+          __guard__(req != null ? req.body : undefined, x2 => x2.words),
+          x1 => x1.length
+        )
+      },
+      'running check'
+    )
+    return SpellingAPIManager.runRequest(req.params.user_id, req.body, function(
+      error,
+      result
+    ) {
+      if (error != null) {
+        logger.err(
+          {
+            err: error,
+            user_id: __guard__(
+              req != null ? req.params : undefined,
+              x3 => x3.user_id
+            ),
+            word_count: __guard__(
+              __guard__(req != null ? req.body : undefined, x5 => x5.words),
+              x4 => x4.length
+            )
+          },
+          'error processing spelling request'
+        )
+        return res.sendStatus(500)
+      }
+      return res.send(result)
+    })
+  },
+
+  learn(req, res, next) {
+    metrics.inc('spelling-learn', 0.1)
+    logger.info(
+      {
+        token: __guard__(req != null ? req.params : undefined, x => x.user_id),
+        word: __guard__(req != null ? req.body : undefined, x1 => x1.word)
+      },
+      'learning word'
+    )
+    return SpellingAPIManager.learnWord(req.params.user_id, req.body, function(
+      error,
+      result
+    ) {
+      if (error != null) {
+        return next(error)
+      }
+      res.sendStatus(200)
+      return next()
+    })
+  },
+
+  deleteDic(req, res, next) {
+    logger.log(
+      {
+        token: __guard__(req != null ? req.params : undefined, x => x.user_id),
+        word: __guard__(req != null ? req.body : undefined, x1 => x1.word)
+      },
+      'deleting user dictionary'
+    )
+    return SpellingAPIManager.deleteDic(req.params.user_id, function(error) {
+      if (error != null) {
+        return next(error)
+      }
+      return res.sendStatus(204)
+    })
+  },
+
+  getDic(req, res, next) {
+    logger.info(
+      {
+        token: __guard__(req != null ? req.params : undefined, x => x.user_id)
+      },
+      'getting user dictionary'
+    )
+    return SpellingAPIManager.getDic(req.params.user_id, function(
+      error,
+      words
+    ) {
+      if (error != null) {
+        return next(error)
+      }
+      return res.send(words)
+    })
+  }
+}
+function __guard__(value, transform) {
+  return typeof value !== 'undefined' && value !== null
+    ? transform(value)
+    : undefined
+}

services/spelling/app/js/SpellingAPIManager.js

@@ -0,0 +1,81 @@
+// TODO: This file was created by bulk-decaffeinate.
+// Sanity-check the conversion and remove this comment.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let SpellingAPIManager
+const ASpell = require('./ASpell')
+const LearnedWordsManager = require('./LearnedWordsManager')
+
+module.exports = SpellingAPIManager = {
+  whitelist: ['ShareLaTeX', 'sharelatex', 'LaTeX', 'http', 'https', 'www'],
+
+  runRequest(token, request, callback) {
+    if (callback == null) {
+      callback = () => {}
+    }
+    if (request.words == null) {
+      return callback(new Error('malformed JSON'))
+    }
+
+    const lang = request.language || 'en'
+
+    const check = (words, callback) =>
+      ASpell.checkWords(lang, words, (error, misspellings) =>
+        callback(error, { misspellings })
+      )
+    const wordsToCheck = request.words || []
+
+    if (token != null) {
+      return LearnedWordsManager.getLearnedWords(token, function(
+        error,
+        learnedWords
+      ) {
+        if (error != null) {
+          return callback(error)
+        }
+        const words = wordsToCheck.slice(0, 10000)
+        return check(words, function(error, result) {
+          if (error != null) {
+            return callback(error)
+          }
+          result.misspellings = result.misspellings.filter(function(m) {
+            const word = words[m.index]
+            return (
+              learnedWords.indexOf(word) === -1 &&
+              SpellingAPIManager.whitelist.indexOf(word) === -1
+            )
+          })
+          return callback(error, result)
+        })
+      })
+    } else {
+      return check(wordsToCheck, callback)
+    }
+  },
+
+  learnWord(token, request, callback) {
+    if (callback == null) {
+      callback = () => {}
+    }
+    if (request.word == null) {
+      return callback(new Error('malformed JSON'))
+    }
+    if (token == null) {
+      return callback(new Error('no token provided'))
+    }
+
+    return LearnedWordsManager.learnWord(token, request.word, callback)
+  },
+
+  deleteDic(token, callback) {
+    return LearnedWordsManager.deleteUsersLearnedWords(token, callback)
+  },
+
+  getDic(token, callback) {
+    return LearnedWordsManager.getLearnedWords(token, callback)
+  }
+}

services/spelling/buildscript.txt

@@ -1,8 +1,8 @@
 spelling
---language=coffeescript
 --node-version=6.16.0
---acceptance-creds=None
+--script-version=1.1.21
+--build-target=docker
 --dependencies=mongo,redis
+--language=es
 --docker-repos=gcr.io/overleaf-ops
---build-target=docker
---script-version=1.1.21
+--acceptance-creds=None

services/spelling/config/settings.defaults.coffee

@@ -1,18 +0,0 @@
-Path = require('path')
-
-module.exports = Settings =
-	internal:
-		spelling:
-			port: 3005
-			host: process.env["LISTEN_ADDRESS"] or "localhost"
-		
-	mongo:
-		url: process.env['MONGO_CONNECTION_STRING'] or "mongodb://#{process.env["MONGO_HOST"] or "localhost"}/sharelatex"
-
-	cacheDir: Path.resolve "cache"
-
-
-	healthCheckUserId: "53c64d2fd68c8d000010bb5f"
-
-	sentry:
-		dsn: process.env.SENTRY_DSN

services/spelling/config/settings.defaults.js

@@ -0,0 +1,24 @@
+const Path = require('path')
+
+module.exports = {
+  internal: {
+    spelling: {
+      port: 3005,
+      host: process.env['LISTEN_ADDRESS'] || 'localhost'
+    }
+  },
+
+  mongo: {
+    url:
+      process.env['MONGO_CONNECTION_STRING'] ||
+      `mongodb://${process.env['MONGO_HOST'] || 'localhost'}/sharelatex`
+  },
+
+  cacheDir: Path.resolve('cache'),
+
+  healthCheckUserId: '53c64d2fd68c8d000010bb5f',
+
+  sentry: {
+    dsn: process.env.SENTRY_DSN
+  }
+}

services/spelling/decaffeinate.sh

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+set -ex
+
+# Check .eslintrc and .prettierc are present
+
+npm install --save-dev eslint eslint-config-prettier eslint-config-standard \
+    eslint-plugin-chai-expect eslint-plugin-chai-friendly eslint-plugin-import \
+    eslint-plugin-mocha eslint-plugin-node eslint-plugin-prettier \
+    eslint-plugin-promise eslint-plugin-standard prettier-eslint-cli
+
+git add .
+git commit -m "Decaffeinate: add eslint and prettier rc files"
+
+
+echo "------------------------"
+echo "----------APP-----------"
+echo "------------------------"
+
+# bulk-decaffeinate will commit for you
+npx bulk-decaffeinate convert --dir app/coffee
+npx bulk-decaffeinate clean
+
+git mv app/coffee app/js
+git commit -m "Rename app/coffee dir to app/js"
+
+npx prettier-eslint 'app/js/**/*.js' --write
+git add .
+git commit -m "Prettier: convert app/js decaffeinated files to Prettier format"
+
+
+echo "-------------------------"
+echo "--------UNIT TESTS-------"
+echo "-------------------------"
+npx bulk-decaffeinate convert --dir test/unit/coffee
+npx bulk-decaffeinate clean
+
+git mv test/unit/coffee test/unit/js
+git commit -m "Rename test/unit/coffee to test/unit/js"
+
+npx prettier-eslint 'test/unit/js/**/*.js' --write
+git add .
+git commit -m "Prettier: convert test/unit decaffeinated files to Prettier format"
+
+
+echo "-------------------------"
+echo "-------STRESS TESTS------"
+echo "-------------------------"
+
+npx bulk-decaffeinate convert --dir test/stress/coffee
+npx bulk-decaffeinate clean
+
+git mv test/stress/coffee test/stress/js
+git commit -m "Rename test/stress/coffee to test/stress/js"
+
+npx prettier-eslint 'test/stress/js/**/*.js' --write
+git add .
+git commit -m "Prettier: convert test/stress decaffeinated files to Prettier format"
+
+
+echo "--------------------------"
+echo "-----INDIVIDUAL FILES-----"
+echo "--------------------------"
+
+rm -f app.js config/settings.defaults.js
+git mv app.coffee app.js
+git mv config/settings.defaults.coffee config/settings.defaults.js
+git commit -m "Rename individual coffee files to js files"
+
+decaffeinate app.js
+decaffeinate config/settings.defaults.js
+git add .
+git commit -m "Decaffeinate: convert individual files to js"
+
+npx prettier-eslint 'app.js' 'config/settings.defaults.js' --write
+
+git add .
+git commit -m "Prettier: convert individual decaffeinated files to Prettier format"
+
+echo "done."

services/spelling/nodemon.json

@@ -10,10 +10,9 @@
   },
 
   "watch": [
-    "app/coffee/",
-    "app.coffee",
+    "app/js/",
+    "app.js",
     "config/"
   ],
-  "ext": "coffee"
-
+  "ext": "js"
 }

services/spelling/npm-shrinkwrap.json

@@ -1849,4 +1849,4 @@
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz"
     }
   }
-}
+}

services/spelling/package.json

@@ -8,16 +8,19 @@
   },
   "scripts": {
     "compile:app": "([ -e app/coffee ] && coffee -m $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee -m $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')",
-    "start": "npm run compile:app && node $NODE_APP_OPTIONS app.js",
-    "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 30000 --exit $@ test/acceptance/js",
-    "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
+    "start": "node $NODE_APP_OPTIONS app.js",
+    "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
+    "test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
     "test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js",
-    "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- --grep=$MOCHA_GREP",
+    "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
     "compile:unit_tests": "[ ! -e test/unit/coffee ] &&  echo 'No unit tests to compile' || coffee -o test/unit/js -c test/unit/coffee",
     "compile:acceptance_tests": "[ ! -e test/acceptance/coffee ] && echo 'No acceptance tests to compile' || coffee -o test/acceptance/js -c test/acceptance/coffee",
     "compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests && npm run compile:smoke_tests",
     "nodemon": "nodemon --config nodemon.json",
-    "compile:smoke_tests": "[ ! -e test/smoke/coffee ] &&  echo 'No smoke tests to compile' || coffee -o test/smoke/js -c test/smoke/coffee"
+    "compile:smoke_tests": "[ ! -e test/smoke/coffee ] &&  echo 'No smoke tests to compile' || coffee -o test/smoke/js -c test/smoke/coffee",
+    "lint": "node_modules/.bin/eslint .",
+    "format": "node_modules/.bin/prettier-eslint '**/*.js' --list-different",
+    "format:fix": "node_modules/.bin/prettier-eslint '**/*.js' --write"
   },
   "version": "0.1.4",
   "dependencies": {
@@ -39,7 +42,19 @@
   "devDependencies": {
     "bunyan": "^1.0.0",
     "chai": "",
+    "eslint": "^6.0.1",
+    "eslint-config-prettier": "^6.0.0",
+    "eslint-config-standard": "^12.0.0",
+    "eslint-plugin-chai-expect": "^2.0.1",
+    "eslint-plugin-chai-friendly": "^0.4.1",
+    "eslint-plugin-import": "^2.18.0",
+    "eslint-plugin-mocha": "^5.3.0",
+    "eslint-plugin-node": "^9.1.0",
+    "eslint-plugin-prettier": "^3.1.0",
+    "eslint-plugin-promise": "^4.2.1",
+    "eslint-plugin-standard": "^4.0.0",
     "mocha": "^4.1.0",
+    "prettier-eslint-cli": "^5.0.0",
     "sandboxed-module": "",
     "sinon": "^1.17.0"
   }

services/spelling/test/stress/coffee/stressTest.coffee

@@ -1,86 +0,0 @@
-# N requests in parallel
-# send P correct words and Q incorrect words
-# generate incorrect words by qq+random
-
-async = require "async"
-request = require "request"
-fs = require "fs"
-
-# created with
-# aspell -d en dump master | aspell -l en expand | shuf -n 150000 > words.txt
-WORDS = "words.txt"
-wordlist = fs.readFileSync(WORDS).toString().split('\n').filter (w) ->
-	w.match(/^[a-z]+$/)
-
-generateCorrectWords = (n) ->
-	words = []
-	N = if Math.random() > 0.5 then wordlist.length else 10
-	for i in [1 .. n]
-		j = Math.floor(N * Math.random())
-		words.push wordlist[j]
-	return words
-	
-generateIncorrectWords = (n) ->
-	words = []
-	N = wordlist.length
-	for i in [1 .. n]
-		j = Math.floor(N * Math.random())
-		words.push("qzxq" + wordlist[j])
-	return words
-
-make_request = (correctWords, incorrectWords, callback) ->
-	correctSet = generateCorrectWords(correctWords)
-	incorrectSet = generateIncorrectWords(incorrectWords)
-	correctSet.push('constructor')
-	incorrectSet.push('qzxqfoofoofoo')
-	full = correctSet.concat incorrectSet
-	bad = []
-	for w, i in correctSet
-		bad[i] = false
-	for w, i in incorrectSet
-		bad[i+correctSet.length] = true
-	k = full.length
-	full.forEach (e, i) ->
-		j = Math.floor(k * Math.random())
-		[ full[i], full[j] ] = [ full[j], full[i] ]
-		[ bad[i], bad[j] ] = [ bad[j], bad[i] ]
-	expected = []
-	for tf, i in bad
-		if tf
-			expected.push {index: i, word: full[i]}
-	request.post 'http://localhost:3005/user/1/check', json:true, body: {words: full}, (err, req, body) ->
-		misspellings = body.misspellings
-		console.log JSON.stringify({full: full, misspellings: misspellings})
-		if expected.length != misspellings.length
-			console.log "ERROR: length mismatch", expected.length, misspellings.length
-			console.log full, bad
-			console.log 'expected', expected, 'mispellings', misspellings
-			for i in [0 .. Math.max(expected.length, misspellings.length)-1]
-				if expected[i].index !=  misspellings[i].index
-					console.log "ERROR", i, expected[i], misspellings[i], full[misspellings[i].index]
-			for m in misspellings
-				console.log full[m.index], "=>", m
-			process.exit()
-			callback("error")
-		else 
-			for m,i in body.misspellings
-				if m.index != expected[i].index
-					console.log "ERROR AT RESULT", i, m, expected[i]
-					process.exit()
-					callback("error")
-		callback(null, full)
-
-q = async.queue	 (task, callback) ->
-	setTimeout () ->
-		make_request task.correct, task.incorrect, callback
-	, Math.random() * 100
-, 3
-
-q.drain = () ->
-	console.log('all items have been processed');
-
-for i in [0 .. 1000]
-	q.push({correct: Math.floor(30*Math.random()) + 1, incorrect: Math.floor(3*Math.random())})
-	# if Math.random() < 0.1
-	# else
-	# 	q.push({correct: Math.floor(100*Math.random()) + 1, incorrect: Math.floor(3*Math.random())})

services/spelling/test/stress/js/stressTest.js

@@ -0,0 +1,162 @@
+/* eslint-disable */
+// TODO: This file was created by bulk-decaffeinate.
+// Sanity-check the conversion and remove this comment.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS104: Avoid inline assignments
+ * DS202: Simplify dynamic range loops
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+// N requests in parallel
+// send P correct words and Q incorrect words
+// generate incorrect words by qq+random
+
+const async = require('async')
+const request = require('request')
+const fs = require('fs')
+
+// created with
+// aspell -d en dump master | aspell -l en expand | shuf -n 150000 > words.txt
+const WORDS = 'words.txt'
+const wordlist = fs
+  .readFileSync(WORDS)
+  .toString()
+  .split('\n')
+  .filter(w => w.match(/^[a-z]+$/))
+
+const generateCorrectWords = function(n) {
+  const words = []
+  const N = Math.random() > 0.5 ? wordlist.length : 10
+  for (
+    let i = 1, end = n, asc = 1 <= end;
+    asc ? i <= end : i >= end;
+    asc ? i++ : i--
+  ) {
+    const j = Math.floor(N * Math.random())
+    words.push(wordlist[j])
+  }
+  return words
+}
+
+const generateIncorrectWords = function(n) {
+  const words = []
+  const N = wordlist.length
+  for (
+    let i = 1, end = n, asc = 1 <= end;
+    asc ? i <= end : i >= end;
+    asc ? i++ : i--
+  ) {
+    const j = Math.floor(N * Math.random())
+    words.push(`qzxq${wordlist[j]}`)
+  }
+  return words
+}
+
+const make_request = function(correctWords, incorrectWords, callback) {
+  let i, j, w
+  let i1
+  let j1
+  const correctSet = generateCorrectWords(correctWords)
+  const incorrectSet = generateIncorrectWords(incorrectWords)
+  correctSet.push('constructor')
+  incorrectSet.push('qzxqfoofoofoo')
+  const full = correctSet.concat(incorrectSet)
+  const bad = []
+  for (j = 0, i = j; j < correctSet.length; j++, i = j) {
+    w = correctSet[i]
+    bad[i] = false
+  }
+  for (i1 = 0, i = i1; i1 < incorrectSet.length; i1++, i = i1) {
+    w = incorrectSet[i]
+    bad[i + correctSet.length] = true
+  }
+  const k = full.length
+  full.forEach(function(e, i) {
+    let ref
+    j = Math.floor(k * Math.random())
+    ;[full[i], full[j]] = Array.from([full[j], full[i]])
+    return ([bad[i], bad[j]] = Array.from((ref = [bad[j], bad[i]]))), ref
+  })
+  const expected = []
+  for (j1 = 0, i = j1; j1 < bad.length; j1++, i = j1) {
+    const tf = bad[i]
+    if (tf) {
+      expected.push({ index: i, word: full[i] })
+    }
+  }
+  return request.post(
+    'http://localhost:3005/user/1/check',
+    { json: true, body: { words: full } },
+    function(err, req, body) {
+      let m
+      const { misspellings } = body
+      console.log(JSON.stringify({ full, misspellings }))
+      if (expected.length !== misspellings.length) {
+        let asc, end
+        console.log(
+          'ERROR: length mismatch',
+          expected.length,
+          misspellings.length
+        )
+        console.log(full, bad)
+        console.log('expected', expected, 'mispellings', misspellings)
+        for (
+          i = 0,
+            end = Math.max(expected.length, misspellings.length) - 1,
+            asc = 0 <= end;
+          asc ? i <= end : i >= end;
+          asc ? i++ : i--
+        ) {
+          if (expected[i].index !== misspellings[i].index) {
+            console.log(
+              'ERROR',
+              i,
+              expected[i],
+              misspellings[i],
+              full[misspellings[i].index]
+            )
+          }
+        }
+        for (m of Array.from(misspellings)) {
+          console.log(full[m.index], '=>', m)
+        }
+        process.exit()
+        callback('error')
+      } else {
+        for (i = 0; i < body.misspellings.length; i++) {
+          m = body.misspellings[i]
+          if (m.index !== expected[i].index) {
+            console.log('ERROR AT RESULT', i, m, expected[i])
+            process.exit()
+            callback('error')
+          }
+        }
+      }
+      return callback(null, full)
+    }
+  )
+}
+
+const q = async.queue(
+  (task, callback) =>
+    setTimeout(
+      () => make_request(task.correct, task.incorrect, callback),
+      Math.random() * 100
+    ),
+
+  3
+)
+
+q.drain = () => console.log('all items have been processed')
+
+for (let i = 0; i <= 1000; i++) {
+  q.push({
+    correct: Math.floor(30 * Math.random()) + 1,
+    incorrect: Math.floor(3 * Math.random())
+  })
+}
+// if Math.random() < 0.1
+// else
+// 	q.push({correct: Math.floor(100*Math.random()) + 1, incorrect: Math.floor(3*Math.random())})

services/spelling/test/unit/coffee/ASpellTests.coffee

@@ -1,70 +0,0 @@
-sinon = require 'sinon'
-chai = require 'chai'
-should = chai.should()
-SandboxedModule = require('sandboxed-module')
-assert = require("chai").assert
-
-describe "ASpell", ->
-	beforeEach ->
-		@ASpell = SandboxedModule.require "../../../app/js/ASpell", requires:
-			"logger-sharelatex":
-				log:->
-				info:->
-				err:->
-			'metrics-sharelatex':
-				gauge:->
-				inc: ->
-
-	describe "a correctly spelled word", ->
-		beforeEach (done) ->
-			@ASpell.checkWords "en", ["word"], (error, @result) => done()
-
-		it "should not correct the word", ->
-			@result.length.should.equal 0
-
-	describe "a misspelled word", ->
-		beforeEach (done) ->
-			@ASpell.checkWords "en", ["bussines"], (error, @result) => done()
-
-		it "should correct the word", ->
-			@result.length.should.equal 1
-			@result[0].suggestions.indexOf("business").should.not.equal -1
-
-	describe "multiple words", ->
-		beforeEach (done) ->
-			@ASpell.checkWords "en", ["bussines", "word", "neccesary"], (error, @result) => done()
-
-		it "should correct the incorrect words", ->
-			@result[0].index.should.equal 0
-			@result[0].suggestions.indexOf("business").should.not.equal -1
-			@result[1].index.should.equal 2
-			@result[1].suggestions.indexOf("necessary").should.not.equal -1
-
-	describe "without a valid language", ->
-		beforeEach (done) ->
-			@ASpell.checkWords "notALang", ["banana"], (@error, @result) => done()
-
-		it "should return an error", ->
-			should.exist @error
-
-	describe "when there are no suggestions", ->
-		beforeEach (done) ->
-			@ASpell.checkWords "en", ["asdkfjalkdjfadhfkajsdhfashdfjhadflkjadhflajsd"], (@error, @result) => done()
-
-		it "should return a blank array", ->
-			@result.length.should.equal 1
-			assert.deepEqual @result[0].suggestions, []
-
-	describe "when the request times out", ->
-		beforeEach (done) ->
-			words = ("abcdefg" for i in [0..1000])
-			@ASpell.ASPELL_TIMEOUT = 1
-			@start = Date.now()
-			@ASpell.checkWords "en", words, (error, @result) => done()
-
-		# Note that this test fails on OS X, due to differing pipe behaviour
-		# on killing the child process. It can be tested successfully on Travis
-		# or the CI server.
-		it "should return in reasonable time", () ->
-			delta = Date.now()-@start
-			delta.should.be.below(@ASpell.ASPELL_TIMEOUT + 1000)

services/spelling/test/unit/coffee/LearnedWordsManagerTests.coffee

@@ -1,97 +0,0 @@
-sinon = require('sinon')
-chai = require 'chai'
-expect = chai.expect
-SandboxedModule = require('sandboxed-module')
-modulePath = require('path').join __dirname, '../../../app/js/LearnedWordsManager'
-assert = require("chai").assert
-should = require("chai").should()
-describe "LearnedWordsManager", ->
-	beforeEach ->
-		@token = "a6b3cd919ge"
-		@callback = sinon.stub()
-		@db =
-			spellingPreferences:
-				update: sinon.stub().callsArg(3)
-		@cache = 
-			get:sinon.stub()
-			set:sinon.stub()
-			del:sinon.stub()
-		@LearnedWordsManager = SandboxedModule.require modulePath, requires:
-			"./DB" : @db
-			"./MongoCache":@cache
-			"logger-sharelatex":
-				log:->
-				err:->
-				info:->
-			'metrics-sharelatex': {timeAsyncMethod: sinon.stub(), inc: sinon.stub()}
-
-	describe "learnWord", ->
-		beforeEach ->
-			@word = "instanton"
-			@LearnedWordsManager.learnWord @token, @word, @callback
-
-		it "should insert the word in the word list in the database", ->
-			expect(
-				@db.spellingPreferences.update.calledWith({
-					token: @token
-				}, {
-					$push : learnedWords: @word
-				}, {
-					upsert: true
-				})
-			).to.equal true
-
-		it "should call the callback", ->
-			expect(@callback.called).to.equal true
-
-	describe "getLearnedWords", ->
-		beforeEach ->
-			@wordList = ["apples", "bananas", "pears"]
-			@db.spellingPreferences.findOne = (conditions, callback) =>
-				callback null, learnedWords: @wordList
-			sinon.spy @db.spellingPreferences, "findOne"
-			@LearnedWordsManager.getLearnedWords @token, @callback
-
-		it "should get the word list for the given user", ->
-			expect(
-				@db.spellingPreferences.findOne.calledWith token: @token
-			).to.equal true
-
-		it "should return the word list in the callback", ->
-			expect(@callback.calledWith null, @wordList).to.equal true
-
-	
-	describe "caching the result", ->
-		it 'should use the cache first if it is primed', (done)->
-			@wordList = ["apples", "bananas", "pears"]
-			@cache.get.returns(@wordList)
-			@db.spellingPreferences.findOne = sinon.stub()
-			@LearnedWordsManager.getLearnedWords @token, (err, spellings)=>
-				@db.spellingPreferences.findOne.called.should.equal false
-				assert.deepEqual @wordList, spellings
-				done()
-
-		it 'should set the cache after hitting the db', (done)->
-			@wordList = ["apples", "bananas", "pears"]
-			@db.spellingPreferences.findOne = sinon.stub().callsArgWith(1, null, learnedWords: @wordList)
-			@LearnedWordsManager.getLearnedWords @token, (err, spellings)=>
-				@cache.set.calledWith(@token, @wordList).should.equal true
-				done()
-
-		it 'should break cache when update is called', (done)->
-			@word = "instanton"
-			@LearnedWordsManager.learnWord @token, @word, =>
-				@cache.del.calledWith(@token).should.equal true
-				done()
-
-
-	describe "deleteUsersLearnedWords", ->
-		beforeEach ->
-			@db.spellingPreferences.remove = sinon.stub().callsArgWith(1)
-
-
-		it "should get the word list for the given user", (done)->
-			@LearnedWordsManager.deleteUsersLearnedWords @token, =>
-				@db.spellingPreferences.remove.calledWith(token: @token).should.equal true
-				done()
-

services/spelling/test/unit/coffee/SpellingAPIManagerTests.coffee

@@ -1,119 +0,0 @@
-sinon = require('sinon')
-chai = require 'chai'
-expect = chai.expect
-chai.should()
-SandboxedModule = require('sandboxed-module')
-modulePath = require('path').join __dirname, '../../../app/js/SpellingAPIManager'
-
-describe "SpellingAPIManager", ->
-	beforeEach ->
-		@token = "user-id-123"
-		@ASpell = {}
-		@learnedWords = ["lerned"]
-		@LearnedWordsManager =
-			getLearnedWords: sinon.stub().callsArgWith(1, null, @learnedWords)
-			learnWord: sinon.stub().callsArg(2)
-
-		@SpellingAPIManager = SandboxedModule.require modulePath, requires:
-			"./ASpell" : @ASpell
-			"./LearnedWordsManager" : @LearnedWordsManager
-
-	describe "runRequest", ->
-		beforeEach ->
-			@nonLearnedWords = ["some", "words", "htat", "are", "speled", "rong", "lerned"]
-			@allWords = @nonLearnedWords.concat(@learnedWords)
-			@misspellings = [
-				{ index: 2, suggestions: ["that"] }
-				{ index: 4, suggestions: ["spelled"] }
-				{ index: 5, suggestions: ["wrong", "ring"] }
-				{ index: 6, suggestions: ["learned"] }
-			]
-			@misspellingsWithoutLearnedWords = @misspellings.slice(0,3)
-
-			@ASpell.checkWords = (lang, word, callback) =>
-				callback null, @misspellings
-			sinon.spy @ASpell, "checkWords"
-
-		describe "with sensible JSON", ->
-			beforeEach (done) ->
-				@SpellingAPIManager.runRequest @token, words: @allWords, (error, @result) => done()
-
-			it "should return the words that are spelled incorrectly and not learned", ->
-				expect(@result.misspellings).to.deep.equal @misspellingsWithoutLearnedWords
-
-		describe "with a missing words array", ->
-			beforeEach (done) ->
-				@SpellingAPIManager.runRequest @token, {}, (@error, @result) => done()
-
-			it "should return an error", ->
-				expect(@error).to.exist
-				expect(@error).to.be.instanceof Error
-				expect(@error.message).to.equal "malformed JSON"
-
-		describe "with a missing token", ->
-			beforeEach (done) ->
-				@SpellingAPIManager.runRequest null, words: @allWords, (@error, @result) => done()
-
-			it "should spell check without using any learned words", ->
-				@LearnedWordsManager.getLearnedWords.called.should.equal false
-
-		describe "without a language", ->
-			beforeEach (done) ->
-				@SpellingAPIManager.runRequest @token, words: @allWords, (error, @result) => done()
-
-			it "should use en as the default", ->
-				@ASpell.checkWords.calledWith("en").should.equal true
-
-		describe "with a language", ->
-			beforeEach (done) ->
-				@SpellingAPIManager.runRequest @token, {
-					words: @allWords
-					language: @language = "fr"
-				}, (error, @result) => done()
-
-			it "should use the language", ->
-				@ASpell.checkWords.calledWith(@language).should.equal true
-
-		describe "with a very large collection of words", ->
-			beforeEach (done) ->
-				@manyWords = ("word" for i in [1..100000])
-				@SpellingAPIManager.runRequest @token, words: @manyWords, (error, @result) => done()
-
-			it "should truncate to 10,000 words", ->
-				@ASpell.checkWords.calledWith(sinon.match.any, @manyWords.slice(0, 10000)).should.equal true
-
-		describe 'with words from the whitelist', ->
-			beforeEach (done) ->
-				@whitelistWord = @SpellingAPIManager.whitelist[0]
-				@words = ["One",  "Two", @whitelistWord]
-				@SpellingAPIManager.runRequest @token, words: @words, (error, @result) => done()
-
-			it 'should ignore the white-listed word', ->
-				expect(@result.misspellings.length).to.equal @misspellings.length-1
-
-	describe "learnWord", ->
-		describe "without a token", ->
-			beforeEach (done) -> @SpellingAPIManager.learnWord null, word: "banana", (@error) => done()
-
-			it "should return an error", ->
-				expect(@error).to.exist
-				expect(@error).to.be.instanceof Error
-				expect(@error.message).to.equal "no token provided"
-
-		describe "without a word", ->
-			beforeEach (done) -> @SpellingAPIManager.learnWord @token, {}, (@error) => done()
-
-			it "should return an error", ->
-				expect(@error).to.exist
-				expect(@error).to.be.instanceof Error
-				expect(@error.message).to.equal "malformed JSON"
-
-		describe "with a word and a token", ->
-			beforeEach (done) ->
-				@word = "banana"
-				@SpellingAPIManager.learnWord @token, word: @word, (@error) => done()
-
-			it "should call LearnedWordsManager.learnWord", ->
-				@LearnedWordsManager.learnWord.calledWith(@token, @word).should.equal true
-			
-

services/spelling/test/unit/js/ASpellTests.js

@@ -0,0 +1,145 @@
+/* eslint-disable
+    handle-callback-err,
+    no-undef
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Sanity-check the conversion and remove this comment.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const chai = require('chai')
+const should = chai.should()
+const SandboxedModule = require('sandboxed-module')
+const { assert } = require('chai')
+
+describe('ASpell', function() {
+  beforeEach(function() {
+    return (this.ASpell = SandboxedModule.require('../../../app/js/ASpell', {
+      requires: {
+        'logger-sharelatex': {
+          log() {},
+          info() {},
+          err() {}
+        },
+        'metrics-sharelatex': {
+          gauge() {},
+          inc() {}
+        }
+      }
+    }))
+  })
+
+  describe('a correctly spelled word', function() {
+    beforeEach(function(done) {
+      return this.ASpell.checkWords('en', ['word'], (error, result) => {
+        this.result = result
+        return done()
+      })
+    })
+
+    return it('should not correct the word', function() {
+      return this.result.length.should.equal(0)
+    })
+  })
+
+  describe('a misspelled word', function() {
+    beforeEach(function(done) {
+      return this.ASpell.checkWords('en', ['bussines'], (error, result) => {
+        this.result = result
+        return done()
+      })
+    })
+
+    return it('should correct the word', function() {
+      this.result.length.should.equal(1)
+      return this.result[0].suggestions.indexOf('business').should.not.equal(-1)
+    })
+  })
+
+  describe('multiple words', function() {
+    beforeEach(function(done) {
+      return this.ASpell.checkWords(
+        'en',
+        ['bussines', 'word', 'neccesary'],
+        (error, result) => {
+          this.result = result
+          return done()
+        }
+      )
+    })
+
+    return it('should correct the incorrect words', function() {
+      this.result[0].index.should.equal(0)
+      this.result[0].suggestions.indexOf('business').should.not.equal(-1)
+      this.result[1].index.should.equal(2)
+      return this.result[1].suggestions
+        .indexOf('necessary')
+        .should.not.equal(-1)
+    })
+  })
+
+  describe('without a valid language', function() {
+    beforeEach(function(done) {
+      return this.ASpell.checkWords('notALang', ['banana'], (error, result) => {
+        this.error = error
+        this.result = result
+        return done()
+      })
+    })
+
+    return it('should return an error', function() {
+      return should.exist(this.error)
+    })
+  })
+
+  describe('when there are no suggestions', function() {
+    beforeEach(function(done) {
+      return this.ASpell.checkWords(
+        'en',
+        ['asdkfjalkdjfadhfkajsdhfashdfjhadflkjadhflajsd'],
+        (error, result) => {
+          this.error = error
+          this.result = result
+          return done()
+        }
+      )
+    })
+
+    return it('should return a blank array', function() {
+      this.result.length.should.equal(1)
+      return assert.deepEqual(this.result[0].suggestions, [])
+    })
+  })
+
+  return describe('when the request times out', function() {
+    beforeEach(function(done) {
+      const words = __range__(0, 1000, true).map(i => 'abcdefg')
+      this.ASpell.ASPELL_TIMEOUT = 1
+      this.start = Date.now()
+      return this.ASpell.checkWords('en', words, (error, result) => {
+        this.result = result
+        return done()
+      })
+    })
+
+    // Note that this test fails on OS X, due to differing pipe behaviour
+    // on killing the child process. It can be tested successfully on Travis
+    // or the CI server.
+    return it('should return in reasonable time', function() {
+      const delta = Date.now() - this.start
+      return delta.should.be.below(this.ASpell.ASPELL_TIMEOUT + 1000)
+    })
+  })
+})
+
+function __range__(left, right, inclusive) {
+  let range = []
+  let ascending = left < right
+  let end = !inclusive ? right : ascending ? right + 1 : right - 1
+  for (let i = left; ascending ? i < end : i > end; ascending ? i++ : i--) {
+    range.push(i)
+  }
+  return range
+}

services/spelling/test/unit/js/LearnedWordsManagerTests.js

@@ -0,0 +1,163 @@
+/* eslint-disable
+    handle-callback-err,
+    no-undef
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Sanity-check the conversion and remove this comment.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const sinon = require('sinon')
+const chai = require('chai')
+const { expect } = chai
+const SandboxedModule = require('sandboxed-module')
+const modulePath = require('path').join(
+  __dirname,
+  '../../../app/js/LearnedWordsManager'
+)
+const { assert } = require('chai')
+describe('LearnedWordsManager', function() {
+  beforeEach(function() {
+    this.token = 'a6b3cd919ge'
+    this.callback = sinon.stub()
+    this.db = {
+      spellingPreferences: {
+        update: sinon.stub().callsArg(3)
+      }
+    }
+    this.cache = {
+      get: sinon.stub(),
+      set: sinon.stub(),
+      del: sinon.stub()
+    }
+    return (this.LearnedWordsManager = SandboxedModule.require(modulePath, {
+      requires: {
+        './DB': this.db,
+        './MongoCache': this.cache,
+        'logger-sharelatex': {
+          log() {},
+          err() {},
+          info() {}
+        },
+        'metrics-sharelatex': {
+          timeAsyncMethod: sinon.stub(),
+          inc: sinon.stub()
+        }
+      }
+    }))
+  })
+
+  describe('learnWord', function() {
+    beforeEach(function() {
+      this.word = 'instanton'
+      return this.LearnedWordsManager.learnWord(
+        this.token,
+        this.word,
+        this.callback
+      )
+    })
+
+    it('should insert the word in the word list in the database', function() {
+      return expect(
+        this.db.spellingPreferences.update.calledWith(
+          {
+            token: this.token
+          },
+          {
+            $push: { learnedWords: this.word }
+          },
+          {
+            upsert: true
+          }
+        )
+      ).to.equal(true)
+    })
+
+    return it('should call the callback', function() {
+      return expect(this.callback.called).to.equal(true)
+    })
+  })
+
+  describe('getLearnedWords', function() {
+    beforeEach(function() {
+      this.wordList = ['apples', 'bananas', 'pears']
+      this.db.spellingPreferences.findOne = (conditions, callback) => {
+        return callback(null, { learnedWords: this.wordList })
+      }
+      sinon.spy(this.db.spellingPreferences, 'findOne')
+      return this.LearnedWordsManager.getLearnedWords(this.token, this.callback)
+    })
+
+    it('should get the word list for the given user', function() {
+      return expect(
+        this.db.spellingPreferences.findOne.calledWith({ token: this.token })
+      ).to.equal(true)
+    })
+
+    return it('should return the word list in the callback', function() {
+      return expect(this.callback.calledWith(null, this.wordList)).to.equal(
+        true
+      )
+    })
+  })
+
+  describe('caching the result', function() {
+    it('should use the cache first if it is primed', function(done) {
+      this.wordList = ['apples', 'bananas', 'pears']
+      this.cache.get.returns(this.wordList)
+      this.db.spellingPreferences.findOne = sinon.stub()
+      return this.LearnedWordsManager.getLearnedWords(
+        this.token,
+        (err, spellings) => {
+          this.db.spellingPreferences.findOne.called.should.equal(false)
+          assert.deepEqual(this.wordList, spellings)
+          return done()
+        }
+      )
+    })
+
+    it('should set the cache after hitting the db', function(done) {
+      this.wordList = ['apples', 'bananas', 'pears']
+      this.db.spellingPreferences.findOne = sinon
+        .stub()
+        .callsArgWith(1, null, { learnedWords: this.wordList })
+      return this.LearnedWordsManager.getLearnedWords(
+        this.token,
+        (err, spellings) => {
+          this.cache.set
+            .calledWith(this.token, this.wordList)
+            .should.equal(true)
+          return done()
+        }
+      )
+    })
+
+    return it('should break cache when update is called', function(done) {
+      this.word = 'instanton'
+      return this.LearnedWordsManager.learnWord(this.token, this.word, () => {
+        this.cache.del.calledWith(this.token).should.equal(true)
+        return done()
+      })
+    })
+  })
+
+  return describe('deleteUsersLearnedWords', function() {
+    beforeEach(function() {
+      return (this.db.spellingPreferences.remove = sinon.stub().callsArgWith(1))
+    })
+
+    return it('should get the word list for the given user', function(done) {
+      return this.LearnedWordsManager.deleteUsersLearnedWords(
+        this.token,
+        () => {
+          this.db.spellingPreferences.remove
+            .calledWith({ token: this.token })
+            .should.equal(true)
+          return done()
+        }
+      )
+    })
+  })
+})

services/spelling/test/unit/js/SpellingAPIManagerTests.js

@@ -0,0 +1,271 @@
+/* eslint-disable
+    handle-callback-err,
+    no-undef
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Sanity-check the conversion and remove this comment.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const sinon = require('sinon')
+const chai = require('chai')
+const { expect } = chai
+chai.should()
+const SandboxedModule = require('sandboxed-module')
+const modulePath = require('path').join(
+  __dirname,
+  '../../../app/js/SpellingAPIManager'
+)
+
+describe('SpellingAPIManager', function() {
+  beforeEach(function() {
+    this.token = 'user-id-123'
+    this.ASpell = {}
+    this.learnedWords = ['lerned']
+    this.LearnedWordsManager = {
+      getLearnedWords: sinon.stub().callsArgWith(1, null, this.learnedWords),
+      learnWord: sinon.stub().callsArg(2)
+    }
+
+    return (this.SpellingAPIManager = SandboxedModule.require(modulePath, {
+      requires: {
+        './ASpell': this.ASpell,
+        './LearnedWordsManager': this.LearnedWordsManager
+      }
+    }))
+  })
+
+  describe('runRequest', function() {
+    beforeEach(function() {
+      this.nonLearnedWords = [
+        'some',
+        'words',
+        'htat',
+        'are',
+        'speled',
+        'rong',
+        'lerned'
+      ]
+      this.allWords = this.nonLearnedWords.concat(this.learnedWords)
+      this.misspellings = [
+        { index: 2, suggestions: ['that'] },
+        { index: 4, suggestions: ['spelled'] },
+        { index: 5, suggestions: ['wrong', 'ring'] },
+        { index: 6, suggestions: ['learned'] }
+      ]
+      this.misspellingsWithoutLearnedWords = this.misspellings.slice(0, 3)
+
+      this.ASpell.checkWords = (lang, word, callback) => {
+        return callback(null, this.misspellings)
+      }
+      return sinon.spy(this.ASpell, 'checkWords')
+    })
+
+    describe('with sensible JSON', function() {
+      beforeEach(function(done) {
+        return this.SpellingAPIManager.runRequest(
+          this.token,
+          { words: this.allWords },
+          (error, result) => {
+            this.result = result
+            return done()
+          }
+        )
+      })
+
+      return it('should return the words that are spelled incorrectly and not learned', function() {
+        return expect(this.result.misspellings).to.deep.equal(
+          this.misspellingsWithoutLearnedWords
+        )
+      })
+    })
+
+    describe('with a missing words array', function() {
+      beforeEach(function(done) {
+        return this.SpellingAPIManager.runRequest(
+          this.token,
+          {},
+          (error, result) => {
+            this.error = error
+            this.result = result
+            return done()
+          }
+        )
+      })
+
+      return it('should return an error', function() {
+        expect(this.error).to.exist
+        expect(this.error).to.be.instanceof(Error)
+        return expect(this.error.message).to.equal('malformed JSON')
+      })
+    })
+
+    describe('with a missing token', function() {
+      beforeEach(function(done) {
+        return this.SpellingAPIManager.runRequest(
+          null,
+          { words: this.allWords },
+          (error, result) => {
+            this.error = error
+            this.result = result
+            return done()
+          }
+        )
+      })
+
+      return it('should spell check without using any learned words', function() {
+        return this.LearnedWordsManager.getLearnedWords.called.should.equal(
+          false
+        )
+      })
+    })
+
+    describe('without a language', function() {
+      beforeEach(function(done) {
+        return this.SpellingAPIManager.runRequest(
+          this.token,
+          { words: this.allWords },
+          (error, result) => {
+            this.result = result
+            return done()
+          }
+        )
+      })
+
+      return it('should use en as the default', function() {
+        return this.ASpell.checkWords.calledWith('en').should.equal(true)
+      })
+    })
+
+    describe('with a language', function() {
+      beforeEach(function(done) {
+        return this.SpellingAPIManager.runRequest(
+          this.token,
+          {
+            words: this.allWords,
+            language: (this.language = 'fr')
+          },
+          (error, result) => {
+            this.result = result
+            return done()
+          }
+        )
+      })
+
+      return it('should use the language', function() {
+        return this.ASpell.checkWords
+          .calledWith(this.language)
+          .should.equal(true)
+      })
+    })
+
+    describe('with a very large collection of words', function() {
+      beforeEach(function(done) {
+        this.manyWords = __range__(1, 100000, true).map(i => 'word')
+        return this.SpellingAPIManager.runRequest(
+          this.token,
+          { words: this.manyWords },
+          (error, result) => {
+            this.result = result
+            return done()
+          }
+        )
+      })
+
+      return it('should truncate to 10,000 words', function() {
+        return this.ASpell.checkWords
+          .calledWith(sinon.match.any, this.manyWords.slice(0, 10000))
+          .should.equal(true)
+      })
+    })
+
+    return describe('with words from the whitelist', function() {
+      beforeEach(function(done) {
+        this.whitelistWord = this.SpellingAPIManager.whitelist[0]
+        this.words = ['One', 'Two', this.whitelistWord]
+        return this.SpellingAPIManager.runRequest(
+          this.token,
+          { words: this.words },
+          (error, result) => {
+            this.result = result
+            return done()
+          }
+        )
+      })
+
+      return it('should ignore the white-listed word', function() {
+        return expect(this.result.misspellings.length).to.equal(
+          this.misspellings.length - 1
+        )
+      })
+    })
+  })
+
+  return describe('learnWord', function() {
+    describe('without a token', function() {
+      beforeEach(function(done) {
+        return this.SpellingAPIManager.learnWord(
+          null,
+          { word: 'banana' },
+          error => {
+            this.error = error
+            return done()
+          }
+        )
+      })
+
+      return it('should return an error', function() {
+        expect(this.error).to.exist
+        expect(this.error).to.be.instanceof(Error)
+        return expect(this.error.message).to.equal('no token provided')
+      })
+    })
+
+    describe('without a word', function() {
+      beforeEach(function(done) {
+        return this.SpellingAPIManager.learnWord(this.token, {}, error => {
+          this.error = error
+          return done()
+        })
+      })
+
+      return it('should return an error', function() {
+        expect(this.error).to.exist
+        expect(this.error).to.be.instanceof(Error)
+        return expect(this.error.message).to.equal('malformed JSON')
+      })
+    })
+
+    return describe('with a word and a token', function() {
+      beforeEach(function(done) {
+        this.word = 'banana'
+        return this.SpellingAPIManager.learnWord(
+          this.token,
+          { word: this.word },
+          error => {
+            this.error = error
+            return done()
+          }
+        )
+      })
+
+      return it('should call LearnedWordsManager.learnWord', function() {
+        return this.LearnedWordsManager.learnWord
+          .calledWith(this.token, this.word)
+          .should.equal(true)
+      })
+    })
+  })
+})
+
+function __range__(left, right, inclusive) {
+  let range = []
+  let ascending = left < right
+  let end = !inclusive ? right : ascending ? right + 1 : right - 1
+  for (let i = left; ascending ? i < end : i > end; ascending ? i++ : i--) {
+    range.push(i)
+  }
+  return range
+}