Overleaf/services/web/app/coffee/Features/Compile/ClsiManager.coffee

Path  = require "path"
async = require "async"
Settings = require "settings-sharelatex"
request = require('request')
Project = require("../../models/Project").Project
ProjectGetter = require("../Project/ProjectGetter")
ProjectEntityHandler = require("../Project/ProjectEntityHandler")
logger = require "logger-sharelatex"
Url = require("url")
ClsiCookieManager = require("./ClsiCookieManager")(Settings.apis.clsi?.backendGroupName)
NewBackendCloudClsiCookieManager = require("./ClsiCookieManager")(Settings.apis.clsi_new?.backendGroupName)
ClsiStateManager = require("./ClsiStateManager")
_ = require("underscore")
async = require("async")
ClsiFormatChecker = require("./ClsiFormatChecker")
DocumentUpdaterHandler = require "../DocumentUpdater/DocumentUpdaterHandler"
Metrics = require('metrics-sharelatex')
Errors = require ('../Errors/Errors')

module.exports = ClsiManager =

	sendRequest: (project_id, user_id, options = {}, callback) ->
		ClsiManager.sendRequestOnce project_id, user_id, options, (error, status, result...) ->
			return callback(error) if error?
			if status is 'conflict'
				options = _.clone(options)
				options.syncType = "full" #  force full compile
				ClsiManager.sendRequestOnce project_id, user_id, options, callback # try again
			else
				callback(error, status, result...)

	sendRequestOnce: (project_id, user_id, options = {}, callback = (error, status, outputFiles, clsiServerId, validationProblems) ->) ->
		ClsiManager._buildRequest project_id, options, (error, req) ->
			if error?
				if error.message is "no main file specified"
					return callback(null, "validation-problems", null, null, {mainFile:error.message})
				else
					return callback(error)
			logger.log project_id: project_id, "sending compile to CLSI"
			ClsiManager._sendBuiltRequest project_id, user_id, req, options, (error, status, result...) ->
				return callback(error) if error?
				callback(error, status, result...)

	# for public API requests where there is no project id
	sendExternalRequest: (submission_id, clsi_request, options = {}, callback = (error, status, outputFiles, clsiServerId, validationProblems) ->) ->
		logger.log submission_id: submission_id, "sending external compile to CLSI", clsi_request
		ClsiManager._sendBuiltRequest submission_id, null, clsi_request, options, (error, status, result...) ->
			return callback(error) if error?
			callback(error, status, result...)

	stopCompile: (project_id, user_id, options, callback = (error) ->) ->
		compilerUrl = @_getCompilerUrl(options?.compileGroup, project_id, user_id, "compile/stop")
		opts =
			url:compilerUrl
			method:"POST"
		ClsiManager._makeRequest project_id, opts, callback

	deleteAuxFiles: (project_id, user_id, options, callback = (error) ->) ->
		compilerUrl = @_getCompilerUrl(options?.compileGroup, project_id, user_id)
		opts =
			url:compilerUrl
			method:"DELETE"
		ClsiManager._makeRequest project_id, opts, (clsiError) ->
			# always clear the project state from the docupdater, even if there
			# was a problem with the request to the clsi
			DocumentUpdaterHandler.clearProjectState project_id, (docUpdaterError) ->
				error = clsiError or docUpdaterError
				return callback(error) if error?
				callback()

	_sendBuiltRequest: (project_id, user_id, req, options = {}, callback = (error, status, outputFiles, clsiServerId, validationProblems) ->) ->
		ClsiFormatChecker.checkRecoursesForProblems req.compile?.resources, (err, validationProblems)->
			if err?
				logger.err err, project_id, "could not check resources for potential problems before sending to clsi"
				return callback(err)
			if validationProblems?
				logger.log project_id:project_id, validationProblems:validationProblems, "problems with users latex before compile was attempted"
				return callback(null, "validation-problems", null, null, validationProblems)
			ClsiManager._postToClsi project_id, user_id, req, options.compileGroup, (error, response) ->
				if error?
					logger.err err:error, project_id:project_id, "error sending request to clsi"
					return callback(error)
				logger.log project_id: project_id, outputFilesLength: response?.outputFiles?.length, status: response?.status, compile_status: response?.compile?.status, "received compile response from CLSI"
				ClsiCookieManager._getServerId project_id, (err, clsiServerId)->
					if err?
						logger.err err:err, project_id:project_id, "error getting server id"
						return callback(err)
					outputFiles = ClsiManager._parseOutputFiles(project_id, response?.compile?.outputFiles)
					callback(null, response?.compile?.status, outputFiles, clsiServerId)

	_makeRequest: (project_id, opts, callback)->
		async.series {
			currentBackend: (cb)-> 
				startTime = new Date()
				ClsiCookieManager.getCookieJar project_id, (err, jar)->
					if err?
						logger.err err:err, "error getting cookie jar for clsi request"
						return callback(err)
					opts.jar = jar
					timer = new Metrics.Timer("compile.currentBackend")
					request opts, (err, response, body)->
						timer.done()
						Metrics.inc "compile.currentBackend.response.#{response?.statusCode}"
						if err?
							logger.err err:err, project_id:project_id, url:opts?.url, "error making request to clsi"
							return callback(err)
						ClsiCookieManager.setServerId project_id, response, (err)->
							if err?
								logger.warn err:err, project_id:project_id, "error setting server id"
							callback err, response, body #return as soon as the standard compile has returned
							cb(err, {response:response, body:body, finishTime:new Date() - startTime })
			newBackend: (cb)-> 
				startTime = new Date()
				ClsiManager._makeNewBackendRequest project_id, opts, (err, response, body)->
					Metrics.inc "compile.newBackend.response.#{response?.statusCode}"
					cb(err, {response:response, body:body, finishTime:new Date() - startTime})
		}, (err, results)->
			timeDifference = results.newBackend?.finishTime - results.currentBackend?.finishTime 
			statusCodeSame = results.newBackend?.response?.statusCode == results.currentBackend?.response?.statusCode
			currentCompileTime = results.currentBackend?.finishTime
			newBackendCompileTime = results.newBackend?.finishTime
			logger.log {statusCodeSame, timeDifference, currentCompileTime, newBackendCompileTime, project_id}, "both clsi requests returned"



	_makeNewBackendRequest: (project_id, baseOpts, callback)->
		if !Settings.apis.clsi_new?.url?
			return callback()
		opts = _.clone(baseOpts)
		opts.url = opts.url.replace(Settings.apis.clsi.url, Settings.apis.clsi_new?.url)
		NewBackendCloudClsiCookieManager.getCookieJar project_id, (err, jar)->
			if err?
				logger.err err:err, "error getting cookie jar for clsi request"
				return callback(err)
			opts.jar = jar
			timer = new Metrics.Timer("compile.newBackend")
			request opts, (err, response, body)->
				timer.done()
				if err?
					logger.warn err:err, project_id:project_id, url:opts?.url, "error making request to new clsi"
					return callback(err)
				NewBackendCloudClsiCookieManager.setServerId project_id, response, (err)->
					if err?
						logger.warn err:err, project_id:project_id, "error setting server id new backend"
					return callback err, response, body


	_getCompilerUrl: (compileGroup, project_id, user_id, action) ->
		host = Settings.apis.clsi.url
		path = "/project/#{project_id}"
		path += "/user/#{user_id}" if user_id?
		path += "/#{action}" if action?
		return "#{host}#{path}"

	_postToClsi: (project_id, user_id, req, compileGroup, callback = (error, response) ->) ->
		compileUrl = @_getCompilerUrl(compileGroup, project_id, user_id, "compile")
		opts =
			url:  compileUrl
			json: req
			method: "POST"
		ClsiManager._makeRequest project_id, opts, (error, response, body) ->
			return callback(error) if error?
			if 200 <= response.statusCode < 300
				callback null, body
			else if response.statusCode == 413
				callback null, compile:status:"project-too-large"
			else if response.statusCode == 409
				callback null, compile:status:"conflict"
			else if response.statusCode == 423
				callback null, compile:status:"compile-in-progress"
			else
				error = new Error("CLSI returned non-success code: #{response.statusCode}")
				logger.error err: error, project_id: project_id, "CLSI returned failure code"
				callback error, body

	_parseOutputFiles: (project_id, rawOutputFiles = []) ->
		outputFiles = []
		for file in rawOutputFiles
			outputFiles.push
				path: file.path # the clsi is now sending this to web
				url: Url.parse(file.url).path # the location of the file on the clsi, excluding the host part
				type: file.type
				build: file.build
		return outputFiles

	VALID_COMPILERS: ["pdflatex", "latex", "xelatex", "lualatex"]

	_buildRequest: (project_id, options={}, callback = (error, request) ->) ->
		ProjectGetter.getProject project_id, {compiler: 1, rootDoc_id: 1, imageName: 1, rootFolder:1}, (error, project) ->
			return callback(error) if error?
			return callback(new Errors.NotFoundError("project does not exist: #{project_id}")) if !project?
			if project.compiler not in ClsiManager.VALID_COMPILERS
				project.compiler = "pdflatex"

			if options.incrementalCompilesEnabled or options.syncType? # new way, either incremental or full
				timer = new Metrics.Timer("editor.compile-getdocs-redis")
				ClsiManager.getContentFromDocUpdaterIfMatch project_id, project, options, (error, projectStateHash, docUpdaterDocs) ->
					timer.done()
					if error?
						logger.error err: error, project_id: project_id, "error checking project state"
						# note: we don't bail out when there's an error getting
						# incremental files from the docupdater, we just fall back
						# to a normal compile below
					else
						logger.log project_id: project_id, projectStateHash: projectStateHash, docs: docUpdaterDocs?, "checked project state"
					# see if we can send an incremental update to the CLSI
					if docUpdaterDocs? and (options.syncType isnt "full") and not error?
						Metrics.inc "compile-from-redis"
						ClsiManager._buildRequestFromDocupdater project_id, options, project, projectStateHash, docUpdaterDocs, callback
					else
						Metrics.inc "compile-from-mongo"
						ClsiManager._buildRequestFromMongo project_id, options, project, projectStateHash, callback
			else # old way, always from mongo
				timer = new Metrics.Timer("editor.compile-getdocs-mongo")
				ClsiManager._getContentFromMongo project_id, (error, docs, files) ->
					timer.done()
					return callback(error) if error?
					ClsiManager._finaliseRequest project_id, options, project, docs, files, callback

	getContentFromDocUpdaterIfMatch: (project_id, project, options, callback = (error, projectStateHash, docs) ->) ->
		ClsiStateManager.computeHash project, options, (error, projectStateHash) ->
			return callback(error) if error?
			DocumentUpdaterHandler.getProjectDocsIfMatch project_id, projectStateHash, (error, docs) ->
				return callback(error) if error?
				callback(null, projectStateHash, docs)

	getOutputFileStream: (project_id, user_id, build_id, output_file_path, callback=(err, readStream)->) ->
		url = "#{Settings.apis.clsi.url}/project/#{project_id}/user/#{user_id}/build/#{build_id}/output/#{output_file_path}"
		ClsiCookieManager.getCookieJar project_id, (err, jar)->
			return callback(err) if err?
			options = { url: url, method: "GET", timeout: 60 * 1000, jar : jar }
			readStream = request(options)
			callback(null, readStream)

	_buildRequestFromDocupdater: (project_id, options, project, projectStateHash, docUpdaterDocs, callback = (error, request) ->) ->
		ProjectEntityHandler.getAllDocPathsFromProject project, (error, docPath) ->
				return callback(error) if error?
				docs = {}
				for doc in docUpdaterDocs or []
					path = docPath[doc._id]
					docs[path] = doc
				# send new docs but not files as those are already on the clsi
				options = _.clone(options)
				options.syncType = "incremental"
				options.syncState = projectStateHash
				# create stub doc entries for any possible root docs, if not
				# present in the docupdater. This allows finaliseRequest to
				# identify the root doc.
				possibleRootDocIds = [options.rootDoc_id, project.rootDoc_id]
				for rootDoc_id in possibleRootDocIds when rootDoc_id? and rootDoc_id of docPath
					path = docPath[rootDoc_id]
					docs[path] ?= {_id: rootDoc_id, path: path}
				ClsiManager._finaliseRequest project_id, options, project, docs, [], callback

	_buildRequestFromMongo: (project_id, options, project, projectStateHash, callback = (error, request) ->) ->
		ClsiManager._getContentFromMongo project_id, (error, docs, files) ->
			return callback(error) if error?
			options = _.clone(options)
			options.syncType = "full"
			options.syncState = projectStateHash
			ClsiManager._finaliseRequest project_id, options, project, docs, files, callback

	_getContentFromMongo: (project_id, callback = (error, docs, files) ->) ->
		DocumentUpdaterHandler.flushProjectToMongo project_id, (error) ->
			return callback(error) if error?
			ProjectEntityHandler.getAllDocs project_id, (error, docs = {}) ->
				return callback(error) if error?
				ProjectEntityHandler.getAllFiles project_id, (error, files = {}) ->
					return callback(error) if error?
					callback(null, docs, files)

	_finaliseRequest: (project_id, options, project, docs, files, callback = (error, params) -> ) ->
		resources = []
		rootResourcePath = null
		rootResourcePathOverride = null
		hasMainFile = false
		numberOfDocsInProject = 0

		for path, doc of docs
			path = path.replace(/^\//, "") # Remove leading /
			numberOfDocsInProject++
			if doc.lines? # add doc to resources unless it is just a stub entry
				resources.push
					path:    path
					content: doc.lines.join("\n")
			if project.rootDoc_id? and doc._id.toString() == project.rootDoc_id.toString()
				rootResourcePath = path
			if options.rootDoc_id? and doc._id.toString() == options.rootDoc_id.toString()
				rootResourcePathOverride = path
			if path is "main.tex"
				hasMainFile = true

		rootResourcePath = rootResourcePathOverride if rootResourcePathOverride?
		if !rootResourcePath?
			if hasMainFile
				logger.warn {project_id}, "no root document found, setting to main.tex"
				rootResourcePath = "main.tex"
			else if numberOfDocsInProject is 1 # only one file, must be the main document
				for path, doc of docs
					rootResourcePath = path.replace(/^\//, "") # Remove leading /
				logger.warn {project_id, rootResourcePath: rootResourcePath}, "no root document found, single document in project"
			else
				return callback new Error("no main file specified")

		for path, file of files
			path = path.replace(/^\//, "") # Remove leading /
			resources.push
				path:     path
				url:      "#{Settings.apis.filestore.url}/project/#{project._id}/file/#{file._id}"
				modified: file.created?.getTime()

		callback null, {
			compile:
				options:
					compiler: project.compiler
					timeout: options.timeout
					imageName: project.imageName
					draft: !!options.draft
					check: options.check
					syncType: options.syncType
					syncState: options.syncState
				rootResourcePath: rootResourcePath
				resources: resources
		}

	wordCount: (project_id, user_id, file, options, callback = (error, response) ->) ->
		ClsiManager._buildRequest project_id, options, (error, req) ->
			filename = file || req?.compile?.rootResourcePath
			wordcount_url = ClsiManager._getCompilerUrl(options?.compileGroup, project_id, user_id, "wordcount")
			opts =
				url: wordcount_url
				qs:
					file: filename
					image: req.compile.options.imageName
				method: "GET"
			ClsiManager._makeRequest project_id, opts, (error, response, body) ->
				return callback(error) if error?
				if 200 <= response.statusCode < 300
					callback null, body
				else
					error = new Error("CLSI returned non-success code: #{response.statusCode}")
					logger.error err: error, project_id: project_id, "CLSI returned failure code"
					callback error, body