Add in flags to run strace and capture logs

services/clsi/app/coffee/LatexRunner.coffee

@@ -26,6 +26,9 @@ module.exports = LatexRunner =
 			command = LatexRunner._lualatexCommand mainFile
 		else
 			return callback new Error("unknown compiler: #{compiler}")
+		
+		if Settings.clsi?.strace
+			command = ["strace", "-o", "strace-#{Date.now()}", "-ff"].concat(command)
 
 		CommandRunner.run project_id, command, directory, image, timeout, (error, output) ->
 			return callback(error) if error?

services/clsi/app/coffee/OutputCacheManager.coffee

@@ -4,11 +4,13 @@ fse = require "fs-extra"
 Path = require "path"
 logger = require "logger-sharelatex"
 _ = require "underscore"
+Settings = require "settings-sharelatex"
 
 OutputFileOptimiser = require "./OutputFileOptimiser"
 
 module.exports = OutputCacheManager =
 	CACHE_SUBDIR: '.cache/clsi'
+	ARCHIVE_SUBDIR: '.archive/clsi'
 	BUILD_REGEX: /^[0-9a-f]+$/  # build id is Date.now() converted to hex
 	CACHE_LIMIT: 2  # maximum number of cache directories
 	CACHE_AGE: 60*60*1000 # up to one hour old
@@ -28,33 +30,15 @@ module.exports = OutputCacheManager =
 		# Put the files into a new cache subdirectory
 		buildId = Date.now().toString(16)
 		cacheDir = Path.join(compileDir, OutputCacheManager.CACHE_SUBDIR, buildId)
+
 		# let file expiry run in the background
 		OutputCacheManager.expireOutputFiles cacheRoot, {keep: buildId}
 
-		checkFile = (src, callback) ->
-			# check if we have a valid file to copy into the cache
-			fs.stat src, (err, stats) ->
-				if err?
-					# some problem reading the file
-					logger.error err: err, file: src, "stat error for file in cache"
-					callback(err)
-				else if not stats.isFile()
-					# other filetype - reject it
-					logger.error err: err, src: src, stat: stats, "nonfile output - refusing to copy to cache"
-					callback(new Error("output file is not a file"), file)
-				else
-					# it's a plain file, ok to copy
-					callback(null)
-
-		copyFile = (src, dst, callback) ->
-			# copy output file into the cache
-			fse.copy src, dst, (err) ->
+		# Archive logs in background
+		if Settings.clsi?.archive_logs
+			OutputCacheManager.archiveLogs outputFiles, compileDir, (err) ->
 				if err?
-					logger.error err: err, src: src, dst: dst, "copy error for file in cache"
-					callback(err)
-				else
-					# call the optimiser for the file too
-					OutputFileOptimiser.optimiseFile src, dst, callback
+					logger.warn err:err, "erroring archiving log files"
 
 		# make the new cache directory
 		fse.ensureDir cacheDir, (err) ->
@@ -63,21 +47,47 @@ module.exports = OutputCacheManager =
 				callback(err, outputFiles)
 			else
 				# copy all the output files into the new cache directory
+				results = []
 				async.mapSeries outputFiles, (file, cb) ->
 					newFile = _.clone(file)
 					[src, dst] = [Path.join(compileDir, file.path), Path.join(cacheDir, file.path)]
-					checkFile src, (err) ->
-						copyFile src, dst, (err) ->
-							if not err?
+					OutputCacheManager._checkFileIsSafe src, (err, isSafe) ->
+						return cb(err) if err?
+						if !isSafe
+							return cb()
+						OutputCacheManager._checkIfShouldCopy src, (err, shouldCopy) ->
+							return cb(err) if err?
+							if !shouldCopy
+								return cb()
+							OutputCacheManager._copyFile src, dst, (err) ->
+								return cb(err) if err?
 								newFile.build = buildId  # attach a build id if we cached the file
-							cb(err, newFile)
-				, (err, results) ->
+								results.push newFile
+								cb()
+				, (err) ->
 					if err?
 						# pass back the original files if we encountered *any* error
 						callback(err, outputFiles)
 					else
 						# pass back the list of new files in the cache
 						callback(err, results)
+	
+	archiveLogs: (outputFiles, compileDir, callback = (error) ->) ->
+		buildId = Date.now().toString(16)
+		archiveDir = Path.join(compileDir, OutputCacheManager.ARCHIVE_SUBDIR, buildId)
+		logger.log {dir: archiveDir}, "archiving log files for project"
+		fse.ensureDir archiveDir, (err) ->
+			return callback(err) if err?
+			async.mapSeries outputFiles, (file, cb) ->
+				[src, dst] = [Path.join(compileDir, file.path), Path.join(archiveDir, file.path)]
+				OutputCacheManager._checkFileIsSafe src, (err, isSafe) ->
+					return cb(err) if err?
+					return cb() if !isSafe
+					OutputCacheManager._checkIfShouldArchive src, (err, shouldArchive) ->
+						return cb(err) if err?
+						return cb() if !shouldArchive
+						OutputCacheManager._copyFile src, dst, cb
+			, callback
 
 	expireOutputFiles: (cacheRoot, options, callback = (error) ->) ->
 		# look in compileDir for build dirs and delete if > N or age of mod time > T
@@ -111,3 +121,38 @@ module.exports = OutputCacheManager =
 			async.eachSeries toRemove, (dir, cb) ->
 				removeDir dir, cb
 			, callback
+
+	_checkFileIsSafe: (src, callback = (error, isSafe) ->) ->
+		# check if we have a valid file to copy into the cache
+		fs.stat src, (err, stats) ->
+			if err?
+				# some problem reading the file
+				logger.error err: err, file: src, "stat error for file in cache"
+				callback(err, false)
+			else if not stats.isFile()
+				# other filetype - reject it
+				logger.warn src: src, stat: stats, "nonfile output - refusing to copy to cache"
+				callback(null, false)
+			else
+				# it's a plain file, ok to copy
+				callback(null, true)
+
+	_copyFile: (src, dst, callback) ->
+		# copy output file into the cache
+		fse.copy src, dst, (err) ->
+			if err?
+				logger.error err: err, src: src, dst: dst, "copy error for file in cache"
+				callback(err)
+			else
+				# call the optimiser for the file too
+				OutputFileOptimiser.optimiseFile src, dst, callback
+
+	_checkIfShouldCopy: (src, callback = (err, shouldCopy) ->) ->
+		return callback(null, !Path.basename(src).match(/^strace/))
+
+	_checkIfShouldArchive: (src, callback = (err, shouldCopy) ->) ->
+		if Path.basename(src).match(/^strace/)
+			return callback(null, true)
+		if Path.basename(src).match(/^output\.(?!pdf)/)
+			return callback(null, true)
+		return callback(null, false)