Overleaf/services/web/app/coffee/Features/Project/ProjectEntityUpdateHandler.coffee

_ = require 'lodash'
async = require 'async'
logger = require('logger-sharelatex')
path = require('path')
Doc = require('../../models/Doc').Doc
DocstoreManager = require('../Docstore/DocstoreManager')
DocumentUpdaterHandler = require('../../Features/DocumentUpdater/DocumentUpdaterHandler')
Errors = require '../Errors/Errors'
File = require('../../models/File').File
FileStoreHandler = require('../FileStore/FileStoreHandler')
LockManager = require('../../infrastructure/LockManager')
Project = require('../../models/Project').Project
ProjectEntityHandler = require('./ProjectEntityHandler')
ProjectGetter = require('./ProjectGetter')
ProjectLocator = require('./ProjectLocator')
ProjectUpdateHandler = require('./ProjectUpdateHandler')
ProjectEntityMongoUpdateHandler = require('./ProjectEntityMongoUpdateHandler')
SafePath = require './SafePath'
TpdsUpdateSender = require('../ThirdPartyDataStore/TpdsUpdateSender')

LOCK_NAMESPACE = "sequentialProjectStructureUpdateLock"

wrapWithLock = (methodWithoutLock) ->
	# This lock is used to make sure that the project structure updates are made
	# sequentially. In particular the updates must be made in mongo and sent to
	# the doc-updater in the same order.
	if typeof methodWithoutLock is 'function'
		methodWithLock = (project_id, args..., callback) ->
			LockManager.runWithLock LOCK_NAMESPACE, project_id,
				(cb) -> methodWithoutLock project_id, args..., cb
				callback
		methodWithLock.withoutLock = methodWithoutLock
		methodWithLock
	else
		# handle case with separate setup and locked stages
		wrapWithSetup = methodWithoutLock.beforeLock  # a function to set things up before the lock
		mainTask = methodWithoutLock.withLock # function to execute inside the lock
		methodWithLock = wrapWithSetup (project_id, args..., callback) ->
			LockManager.runWithLock(LOCK_NAMESPACE, project_id, (cb) ->
				mainTask(project_id, args..., cb)
			callback)
		methodWithLock.withoutLock = wrapWithSetup mainTask
		methodWithLock.beforeLock = methodWithoutLock.beforeLock
		methodWithLock.mainTask = methodWithoutLock.withLock
		methodWithLock

module.exports = ProjectEntityUpdateHandler = self =
	copyFileFromExistingProjectWithProject: wrapWithLock
		beforeLock: (next) ->
			(project_id, project, folder_id, originalProject_id, origonalFileRef, userId, callback = (error, fileRef, folder_id) ->)->
				logger.log { project_id, folder_id, originalProject_id, origonalFileRef }, "copying file in s3 with project"
				ProjectEntityMongoUpdateHandler._confirmFolder project, folder_id, (folder_id) ->
					if !origonalFileRef?
						logger.err { project_id, folder_id, originalProject_id, origonalFileRef }, "file trying to copy is null"
						return callback()
					# convert any invalid characters in original file to '_'
					fileProperties = name : SafePath.clean(origonalFileRef.name)
					if origonalFileRef.linkedFileData?
						fileProperties.linkedFileData = origonalFileRef.linkedFileData
					if origonalFileRef.hash?
						fileProperties.hash = origonalFileRef.hash
					fileRef = new File(fileProperties)
					FileStoreHandler.copyFile originalProject_id, origonalFileRef._id, project._id, fileRef._id, (err, fileStoreUrl)->
						if err?
							logger.err { err, project_id, folder_id, originalProject_id, origonalFileRef }, "error coping file in s3"
							return callback(err)
						next(project_id, project, folder_id, originalProject_id, origonalFileRef, userId, fileRef, fileStoreUrl, callback)
		withLock: (project_id, project, folder_id, originalProject_id, origonalFileRef, userId, fileRef, fileStoreUrl, callback = (error, fileRef, folder_id) ->)->
			projectHistoryId = project.overleaf?.history?.id
			ProjectEntityMongoUpdateHandler._putElement project, folder_id, fileRef, "file", (err, result, newProject) ->
				if err?
					logger.err { err, project_id, folder_id }, "error putting element as part of copy"
					return callback(err)
				TpdsUpdateSender.addFile { project_id, file_id:fileRef._id, path:result?.path?.fileSystem, rev:fileRef.rev, project_name:project.name}, (err) ->
					if err?
						logger.err { err, project_id, folder_id, originalProject_id, origonalFileRef }, "error sending file to tpds worker"
					newFiles = [
						file: fileRef
						path: result?.path?.fileSystem
						url: fileStoreUrl
					]
					DocumentUpdaterHandler.updateProjectStructure project_id, projectHistoryId, userId, {newFiles, newProject}, (error) ->
						return callback(error) if error?
						callback null, fileRef, folder_id

	updateDocLines: (project_id, doc_id, lines, version, ranges, lastUpdatedAt, lastUpdatedBy, callback = (error) ->)->
		ProjectGetter.getProjectWithoutDocLines project_id, (err, project)->
			return callback(err) if err?
			return callback(new Errors.NotFoundError("project not found")) if !project?
			logger.log project_id: project_id, doc_id: doc_id, "updating doc lines"
			ProjectLocator.findElement {project:project, element_id:doc_id, type:"docs"}, (err, doc, path)->
				isDeletedDoc = false
				if err?
					if err instanceof Errors.NotFoundError
						# We need to be able to update the doclines of deleted docs. This is
						# so the doc-updater can flush a doc's content to the doc-store after
						# the doc is deleted.
						isDeletedDoc = true
						doc = _.find project.deletedDocs, (doc) ->
							doc._id.toString() == doc_id.toString()
					else
						return callback(err)

				if !doc?
					# Do not allow an update to a doc which has never exist on this project
					logger.error {doc_id, project_id, lines}, "doc not found while updating doc lines"
					return callback(new Errors.NotFoundError('doc not found'))

				logger.log {project_id, doc_id}, "telling docstore manager to update doc"
				DocstoreManager.updateDoc project_id, doc_id, lines, version, ranges, (err, modified, rev) ->
					if err?
						logger.error {err, doc_id, project_id, lines}, "error sending doc to docstore"
						return callback(err)
					logger.log {project_id, doc_id, modified}, "finished updating doc lines"
					# path will only be present if the doc is not deleted
					if modified && !isDeletedDoc
						# Don't need to block for marking as updated
						ProjectUpdateHandler.markAsUpdated project_id, lastUpdatedAt, lastUpdatedBy
						TpdsUpdateSender.addDoc {project_id:project_id, path:path.fileSystem, doc_id:doc_id, project_name:project.name, rev:rev}, callback
					else
						callback()

	setRootDoc: (project_id, newRootDocID, callback = (error) ->)->
		logger.log project_id: project_id, rootDocId: newRootDocID, "setting root doc"
		Project.update {_id:project_id}, {rootDoc_id:newRootDocID}, {}, callback

	unsetRootDoc: (project_id, callback = (error) ->) ->
		logger.log project_id: project_id, "removing root doc"
		Project.update {_id:project_id}, {$unset: {rootDoc_id: true}}, {}, callback

	_addDocAndSendToTpds: (project_id, folder_id, doc, callback = (error, result, project) ->)->
		ProjectEntityMongoUpdateHandler.addDoc project_id, folder_id, doc, (err, result, project) ->
			if err?
				logger.err err:err, project_id: project_id, folder_id: folder_id, doc_name: doc?.name, doc_id:doc?._id, "error adding file with project"
				return callback(err)
			TpdsUpdateSender.addDoc {
				project_id:   project_id,
				doc_id:       doc?._id,
				path:         result?.path?.fileSystem,
				project_name: project.name,
				rev:          0
			}, (err) ->
				return callback(err) if err?
				callback(null, result, project)

	addDoc: (project_id, folder_id, docName, docLines, userId, callback) ->
		self.addDocWithRanges(project_id, folder_id, docName, docLines, {}, userId, callback)

	addDocWithRanges: wrapWithLock
		beforeLock: (next) ->
			(project_id, folder_id, docName, docLines, ranges, userId, callback = (error, doc, folder_id) ->) ->
				if not SafePath.isCleanFilename docName
					return callback new Errors.InvalidNameError("invalid element name")
				# Put doc in docstore first, so that if it errors, we don't have a doc_id in the project
				# which hasn't been created in docstore.
				doc = new Doc name: docName
				DocstoreManager.updateDoc project_id.toString(), doc._id.toString(), docLines, 0, ranges, (err, modified, rev) ->
					return callback(err) if err?
					next(project_id, folder_id, doc, docName, docLines, ranges, userId, callback)
		withLock: (project_id, folder_id, doc, docName, docLines, ranges, userId, callback = (error, doc, folder_id) ->) ->
			ProjectEntityUpdateHandler._addDocAndSendToTpds project_id, folder_id, doc, (err, result, project) ->
				return callback(err) if err?
				docPath = result?.path?.fileSystem
				projectHistoryId = project.overleaf?.history?.id
				newDocs = [
					doc: doc
					path: docPath
					docLines: docLines.join('\n')
				]
				DocumentUpdaterHandler.updateProjectStructure project_id, projectHistoryId, userId, {newDocs, newProject: project}, (error) ->
					return callback(error) if error?
					callback null, doc, folder_id

	_uploadFile: (project_id, folder_id, fileName, fsPath, linkedFileData, callback = (error, fileStoreUrl, fileRef) ->)->
		if not SafePath.isCleanFilename fileName
			return callback new Errors.InvalidNameError("invalid element name")
		fileArgs =
			name: fileName
			linkedFileData: linkedFileData
		FileStoreHandler.uploadFileFromDisk project_id, fileArgs, fsPath, (err, fileStoreUrl, fileRef)->
			if err?
				logger.err err:err, project_id: project_id, folder_id: folder_id, file_name: fileName, fileRef:fileRef, "error uploading image to s3"
				return callback(err)
			callback(null, fileStoreUrl, fileRef)

	_addFileAndSendToTpds: (project_id, folder_id, fileRef, callback = (error) ->)->
		ProjectEntityMongoUpdateHandler.addFile project_id, folder_id, fileRef, (err, result, project) ->
			if err?
				logger.err err:err, project_id: project_id, folder_id: folder_id, file_name: fileRef.name, fileRef:fileRef, "error adding file with project"
				return callback(err)
			TpdsUpdateSender.addFile {project_id:project_id, file_id:fileRef._id, path:result?.path?.fileSystem, project_name:project.name, rev:fileRef.rev}, (err) ->
				return callback(err) if err?
				callback(null, result, project)

	addFile: wrapWithLock
		beforeLock: (next) ->
			(project_id, folder_id, fileName, fsPath, linkedFileData, userId, callback) ->
				if not SafePath.isCleanFilename fileName
					return callback new Errors.InvalidNameError("invalid element name")
				ProjectEntityUpdateHandler._uploadFile project_id, folder_id, fileName, fsPath, linkedFileData, (error, fileStoreUrl, fileRef) ->
					return callback(error) if error?
					next(project_id, folder_id, fileName, fsPath, linkedFileData, userId, fileRef, fileStoreUrl, callback)
		withLock: (project_id, folder_id, fileName, fsPath, linkedFileData, userId, fileRef, fileStoreUrl, callback = (error, fileRef, folder_id) ->)->
			ProjectEntityUpdateHandler._addFileAndSendToTpds project_id, folder_id, fileRef, (err, result, project) ->
				return callback(err) if err?
				projectHistoryId = project.overleaf?.history?.id
				newFiles = [
					file: fileRef
					path: result?.path?.fileSystem
					url: fileStoreUrl
				]
				DocumentUpdaterHandler.updateProjectStructure project_id, projectHistoryId, userId, {newFiles, newProject: project}, (error) ->
					return callback(error) if error?
					callback(null, fileRef, folder_id)

	replaceFile: wrapWithLock
		beforeLock: (next) ->
			(project_id, file_id, fsPath, linkedFileData, userId, callback)->
				# create a new file
				fileArgs =
					name: "dummy-upload-filename"
					linkedFileData: linkedFileData
				FileStoreHandler.uploadFileFromDisk project_id, fileArgs, fsPath, (err, fileStoreUrl, fileRef)->
					return callback(err) if err?
					next project_id, file_id, fsPath, linkedFileData, userId, fileRef, fileStoreUrl, callback
		withLock: (project_id, file_id, fsPath, linkedFileData, userId, newFileRef, fileStoreUrl, callback)->
			ProjectEntityMongoUpdateHandler.replaceFileWithNew project_id, file_id, newFileRef, (err, oldFileRef, project, path, newProject) ->
				return callback(err) if err?
				oldFiles = [
					file: oldFileRef
					path: path.fileSystem
				]
				newFiles = [
					file: newFileRef
					path: path.fileSystem
					url: fileStoreUrl
				]
				projectHistoryId = project.overleaf?.history?.id
				# Increment the rev for an in-place update (with the same path) so the third-party-datastore
				# knows this is a new file.
				# Ideally we would get this from ProjectEntityMongoUpdateHandler.replaceFileWithNew
				# but it returns the original oldFileRef (after incrementing the rev value in mongo),
				# so we add 1 to the rev from that. This isn't atomic and relies on the lock 
				# but it is acceptable for now.
				TpdsUpdateSender.addFile {project_id:project._id, file_id:newFileRef._id, path:path.fileSystem, rev:oldFileRef.rev + 1, project_name:project.name}, (err) ->
					return callback(err) if err?
					DocumentUpdaterHandler.updateProjectStructure project_id, projectHistoryId, userId, {oldFiles, newFiles, newProject}, callback

	upsertDoc: wrapWithLock (project_id, folder_id, docName, docLines, source, userId, callback = (err, doc, folder_id, isNewDoc)->)->
		if not SafePath.isCleanFilename docName
			return callback new Errors.InvalidNameError("invalid element name")
		ProjectLocator.findElement project_id: project_id, element_id: folder_id, type: "folder", (error, folder) ->
			return callback(error) if error?
			return callback(new Error("Couldn't find folder")) if !folder?
			existingDoc = null
			for doc in folder.docs
				if doc.name == docName
					existingDoc = doc
					break
			if existingDoc?
				DocumentUpdaterHandler.setDocument project_id, existingDoc._id, userId, docLines, source, (err)=>
					logger.log project_id:project_id, doc_id:existingDoc._id, "notifying users that the document has been updated"
					DocumentUpdaterHandler.flushDocToMongo project_id, existingDoc._id, (err) ->
						return callback(err) if err?
						callback null, existingDoc, !existingDoc?
			else
				self.addDocWithRanges.withoutLock project_id, folder_id, docName, docLines, {}, userId, (err, doc) ->
					return callback(err) if err?
					callback null, doc, !existingDoc?

	upsertFile: wrapWithLock
		beforeLock: (next) ->
			(project_id, folder_id, fileName, fsPath, linkedFileData, userId, callback)->
				if not SafePath.isCleanFilename fileName
					return callback new Errors.InvalidNameError("invalid element name")
				# create a new file
				fileArgs =
					name: fileName
					linkedFileData: linkedFileData
				FileStoreHandler.uploadFileFromDisk project_id, fileArgs, fsPath, (err, fileStoreUrl, fileRef)->
					return callback(err) if err?
					next(project_id, folder_id, fileName, fsPath, linkedFileData, userId, fileRef, fileStoreUrl, callback)
		withLock: (project_id, folder_id, fileName, fsPath, linkedFileData, userId, newFileRef, fileStoreUrl, callback = (err, file, isNewFile, existingFile)->)->
			ProjectLocator.findElement project_id: project_id, element_id: folder_id, type: "folder", (error, folder) ->
				return callback(error) if error?
				return callback(new Error("Couldn't find folder")) if !folder?
				existingFile = null
				for fileRef in folder.fileRefs
					if fileRef.name == fileName
						existingFile = fileRef
						break
				if existingFile?
					# this calls directly into the replaceFile main task (without the beforeLock part)
					self.replaceFile.mainTask project_id, existingFile._id, fsPath, linkedFileData, userId, newFileRef, fileStoreUrl, (err) ->
						return callback(err) if err?
						callback null, newFileRef, !existingFile?, existingFile
				else
					# this calls directly into the addFile main task (without the beforeLock part)
					self.addFile.mainTask project_id, folder_id, fileName, fsPath, linkedFileData, userId, newFileRef, fileStoreUrl, (err) ->
						return callback(err) if err?
						callback null, newFileRef, !existingFile?, existingFile

	upsertDocWithPath: wrapWithLock (project_id, elementPath, docLines, source, userId, callback) ->
		if not SafePath.isCleanPath elementPath
			return callback new Errors.InvalidNameError("invalid element name")
		docName = path.basename(elementPath)
		folderPath = path.dirname(elementPath)
		self.mkdirp.withoutLock project_id, folderPath, (err, newFolders, folder) ->
			return callback(err) if err?
			self.upsertDoc.withoutLock project_id, folder._id, docName, docLines, source, userId, (err, doc, isNewDoc) ->
				return callback(err) if err?
				callback null, doc, isNewDoc, newFolders, folder

	upsertFileWithPath: wrapWithLock
		beforeLock: (next) ->
			(project_id, elementPath, fsPath, linkedFileData, userId, callback)->
				if not SafePath.isCleanPath elementPath
					return callback new Errors.InvalidNameError("invalid element name")
				fileName = path.basename(elementPath)
				folderPath = path.dirname(elementPath)
				# create a new file
				fileArgs =
					name: fileName
					linkedFileData: linkedFileData
				FileStoreHandler.uploadFileFromDisk project_id, fileArgs, fsPath, (err, fileStoreUrl, fileRef)->
					return callback(err) if err?
					next project_id, folderPath, fileName, fsPath, linkedFileData, userId, fileRef, fileStoreUrl, callback
		withLock: (project_id, folderPath, fileName, fsPath, linkedFileData, userId, fileRef, fileStoreUrl, callback) ->
			self.mkdirp.withoutLock project_id, folderPath, (err, newFolders, folder) ->
				return callback(err) if err?
				# this calls directly into the upsertFile main task (without the beforeLock part)
				self.upsertFile.mainTask project_id, folder._id, fileName, fsPath, linkedFileData, userId, fileRef, fileStoreUrl, (err, newFile, isNewFile, existingFile) ->
					return callback(err) if err?
					callback null, newFile, isNewFile, existingFile, newFolders, folder

	deleteEntity: wrapWithLock (project_id, entity_id, entityType, userId, callback = (error) ->)->
		logger.log entity_id:entity_id, entityType:entityType, project_id:project_id, "deleting project entity"
		if !entityType?
			logger.err err: "No entityType set", project_id: project_id, entity_id: entity_id
			return callback("No entityType set")
		entityType = entityType.toLowerCase()
		ProjectEntityMongoUpdateHandler.deleteEntity project_id, entity_id, entityType, (error, entity, path, projectBeforeDeletion, newProject) ->
			return callback(error) if error?
			self._cleanUpEntity projectBeforeDeletion, newProject, entity, entityType, path.fileSystem, userId, (error) ->
				return callback(error) if error?
				TpdsUpdateSender.deleteEntity project_id:project_id, path:path.fileSystem, project_name:projectBeforeDeletion.name, (error) ->
					return callback(error) if error?
					callback null, entity_id

	deleteEntityWithPath: wrapWithLock (project_id, path, userId, callback) ->
		ProjectLocator.findElementByPath project_id: project_id, path: path, (err, element, type)->
			return callback(err) if err?
			return callback(new Errors.NotFoundError("project not found")) if !element?
			self.deleteEntity.withoutLock project_id, element._id, type, userId, callback

	mkdirp: wrapWithLock (project_id, path, callback = (err, newlyCreatedFolders, lastFolderInPath)->)->
		for folder in path.split('/')
			if folder.length > 0 and not SafePath.isCleanFilename folder
				return callback new Errors.InvalidNameError("invalid element name")
		ProjectEntityMongoUpdateHandler.mkdirp project_id, path, {exactCaseMatch: false}, callback

	mkdirpWithExactCase: wrapWithLock (project_id, path, callback = (err, newlyCreatedFolders, lastFolderInPath)->)->
		for folder in path.split('/')
			if folder.length > 0 and not SafePath.isCleanFilename folder
				return callback new Errors.InvalidNameError("invalid element name")
		ProjectEntityMongoUpdateHandler.mkdirp project_id, path, {exactCaseMatch: true}, callback

	addFolder: wrapWithLock (project_id, parentFolder_id, folderName, callback) ->
		if not SafePath.isCleanFilename folderName
			return callback new Errors.InvalidNameError("invalid element name")
		ProjectEntityMongoUpdateHandler.addFolder project_id, parentFolder_id, folderName, callback

	moveEntity: wrapWithLock (project_id, entity_id, destFolderId, entityType, userId, callback = (error) ->)->
		logger.log {entityType, entity_id, project_id, destFolderId}, "moving entity"
		if !entityType?
			logger.err {err: "No entityType set", project_id, entity_id}
			return callback("No entityType set")
		entityType = entityType.toLowerCase()
		ProjectEntityMongoUpdateHandler.moveEntity project_id, entity_id, destFolderId, entityType, (err, project, startPath, endPath, rev, changes) ->
			return callback(err) if err?
			projectHistoryId = project.overleaf?.history?.id
			TpdsUpdateSender.moveEntity { project_id, project_name: project.name, startPath, endPath, rev }
			DocumentUpdaterHandler.updateProjectStructure project_id, projectHistoryId, userId, changes, callback

	renameEntity: wrapWithLock (project_id, entity_id, entityType, newName, userId, callback)->
		if not SafePath.isCleanFilename newName
			return callback new Errors.InvalidNameError("invalid element name")
		logger.log(entity_id: entity_id, project_id: project_id, ('renaming '+entityType))
		if !entityType?
			logger.err err: "No entityType set", project_id: project_id, entity_id: entity_id
			return callback("No entityType set")
		entityType = entityType.toLowerCase()

		ProjectEntityMongoUpdateHandler.renameEntity project_id, entity_id, entityType, newName, (err, project, startPath, endPath, rev, changes) ->
			return callback(err) if err?
			projectHistoryId = project.overleaf?.history?.id
			TpdsUpdateSender.moveEntity { project_id, project_name: project.name, startPath, endPath, rev }
			DocumentUpdaterHandler.updateProjectStructure project_id, projectHistoryId, userId, changes, callback

	# This doesn't directly update project structure but we need to take the lock
	# to prevent anything else being queued before the resync update
	resyncProjectHistory: wrapWithLock (project_id, callback) ->
		ProjectGetter.getProject project_id, rootFolder: true, overleaf: true, (error, project) ->
			return callback(error) if error?

			projectHistoryId = project?.overleaf?.history?.id
			if !projectHistoryId?
				error = new Errors.ProjectHistoryDisabledError("project history not enabled for #{project_id}")
				return callback(error)

			ProjectEntityHandler.getAllEntitiesFromProject project, (error, docs, files) ->
				return callback(error) if error?

				docs = _.map docs, (doc) ->
					doc: doc.doc._id
					path: doc.path

				files = _.map files, (file) ->
					file: file.file._id
					path: file.path
					url: FileStoreHandler._buildUrl(project_id, file.file._id)

				DocumentUpdaterHandler.resyncProjectHistory project_id, projectHistoryId, docs, files, callback

	_cleanUpEntity: (project, newProject, entity, entityType, path, userId, callback = (error) ->) ->
		self._updateProjectStructureWithDeletedEntity project, newProject, entity, entityType, path, userId, (error) ->
			return callback(error) if error?
			if(entityType.indexOf("file") != -1)
				self._cleanUpFile project, entity, path, userId, callback
			else if (entityType.indexOf("doc") != -1)
				self._cleanUpDoc project, entity, path, userId, callback
			else if (entityType.indexOf("folder") != -1)
				self._cleanUpFolder project, entity, path, userId, callback
			else
				callback()

	# Note: the _cleanUpEntity code and _updateProjectStructureWithDeletedEntity
	# methods both need to recursively iterate over the entities in folder.
	# These are currently using separate implementations of the recursion. In
	# future, these could be simplified using a common project entity iterator.
	_updateProjectStructureWithDeletedEntity: (project, newProject, entity, entityType, entityPath, userId, callback = (error) ->) ->
		# compute the changes to the project structure
		if(entityType.indexOf("file") != -1)
			changes = oldFiles: [ {file: entity, path: entityPath} ]
		else if (entityType.indexOf("doc") != -1)
			changes = oldDocs: [ {doc: entity, path: entityPath} ]
		else if (entityType.indexOf("folder") != -1)
			changes = {oldDocs: [], oldFiles: []}
			_recurseFolder = (folder, folderPath) ->
				for doc in folder.docs
					changes.oldDocs.push {doc, path: path.join(folderPath, doc.name)}
				for file in folder.fileRefs
					changes.oldFiles.push {file, path: path.join(folderPath, file.name)}
				for childFolder in folder.folders
					_recurseFolder(childFolder, path.join(folderPath, childFolder.name))
			_recurseFolder entity, entityPath
		# now send the project structure changes to the docupdater
		changes.newProject = newProject
		project_id = project._id.toString()
		projectHistoryId = project.overleaf?.history?.id
		DocumentUpdaterHandler.updateProjectStructure project_id, projectHistoryId, userId, changes, callback

	_cleanUpDoc: (project, doc, path, userId, callback = (error) ->) ->
		project_id = project._id.toString()
		doc_id = doc._id.toString()
		unsetRootDocIfRequired = (callback) =>
			if project.rootDoc_id? and project.rootDoc_id.toString() == doc_id
				@unsetRootDoc project_id, callback
			else
				callback()

		unsetRootDocIfRequired (error) ->
			return callback(error) if error?
			ProjectEntityMongoUpdateHandler._insertDeletedDocReference project._id, doc, (error) ->
				return callback(error) if error?
				DocumentUpdaterHandler.deleteDoc project_id, doc_id, (error) ->
					return callback(error) if error?
					DocstoreManager.deleteDoc project_id, doc_id, callback

	_cleanUpFile: (project, file, path, userId, callback = (error) ->) ->
		ProjectEntityMongoUpdateHandler._insertDeletedFileReference project._id, file, callback

	_cleanUpFolder: (project, folder, folderPath, userId, callback = (error) ->) ->
		jobs = []
		for doc in folder.docs
			do (doc) ->
				docPath = path.join(folderPath, doc.name)
				jobs.push (callback) -> self._cleanUpDoc project, doc, docPath, userId, callback

		for file in folder.fileRefs
			do (file) ->
				filePath = path.join(folderPath, file.name)
				jobs.push (callback) -> self._cleanUpFile project, file, filePath, userId, callback

		for childFolder in folder.folders
			do (childFolder) ->
				folderPath = path.join(folderPath, childFolder.name)
				jobs.push (callback) -> self._cleanUpFolder project, childFolder, folderPath, userId, callback

		async.series jobs, callback