Merge pull request #2328 from overleaf/em-project-imports

Move ProjectEntityMongoUpdateHandler to async/await

GitOrigin-RevId: e5c0d4a7ece34c3ded89b6eae3673135061f375a

services/web/app/src/Features/Project/ProjectEntityHandler.js

@@ -363,5 +363,10 @@ const ProjectEntityHandler = {
   }
 }
 
-ProjectEntityHandler.promises = promisifyAll(ProjectEntityHandler)
 module.exports = ProjectEntityHandler
+module.exports.promises = promisifyAll(ProjectEntityHandler, {
+  multiResult: {
+    getAllEntities: ['docs', 'files'],
+    getAllEntitiesFromProject: ['docs', 'files']
+  }
+})

services/web/app/src/Features/Project/ProjectEntityMongoUpdateHandlerCanary.js

@@ -0,0 +1,608 @@
+/* NOTE: this file is an async/await version of
+ * ProjectEntityMongoUpdateHandler.js. It's temporarily separate from the
+ * callback-style version so that we can test it in production for some code
+ * paths only.
+ */
+const { callbackify } = require('util')
+const { callbackifyMultiResult } = require('../../util/promises')
+const _ = require('underscore')
+const logger = require('logger-sharelatex')
+const path = require('path')
+const { ObjectId } = require('mongodb')
+const Settings = require('settings-sharelatex')
+const CooldownManager = require('../Cooldown/CooldownManager')
+const Errors = require('../Errors/Errors')
+const { Folder } = require('../../models/Folder')
+const LockManager = require('../../infrastructure/LockManager')
+const { Project } = require('../../models/Project')
+const ProjectEntityHandler = require('./ProjectEntityHandler')
+const ProjectGetter = require('./ProjectGetter')
+const ProjectLocator = require('./ProjectLocator')
+const SafePath = require('./SafePath')
+
+const LOCK_NAMESPACE = 'mongoTransaction'
+const ENTITY_TYPE_TO_MONGO_PATH_SEGMENT = {
+  doc: 'docs',
+  docs: 'docs',
+  file: 'fileRefs',
+  files: 'fileRefs',
+  fileRefs: 'fileRefs',
+  folder: 'folders',
+  folders: 'folders'
+}
+
+module.exports = {
+  LOCK_NAMESPACE,
+  addDoc: callbackifyMultiResult(wrapWithLock(addDoc), ['result', 'project']),
+  addFile: callbackifyMultiResult(wrapWithLock(addFile), ['result', 'project']),
+  addFolder: callbackifyMultiResult(wrapWithLock(addFolder), [
+    'folder',
+    'parentFolderId'
+  ]),
+  replaceFileWithNew: callbackifyMultiResult(wrapWithLock(replaceFileWithNew), [
+    'oldFileRef',
+    'project',
+    'path',
+    'newProject'
+  ]),
+  mkdirp: callbackifyMultiResult(wrapWithLock(mkdirp), [
+    'newFolders',
+    'folder'
+  ]),
+  moveEntity: callbackifyMultiResult(wrapWithLock(moveEntity), [
+    'project',
+    'startPath',
+    'endPath',
+    'rev',
+    'changes'
+  ]),
+  deleteEntity: callbackifyMultiResult(wrapWithLock(deleteEntity), [
+    'entity',
+    'path',
+    'projectBeforeDeletion',
+    'newProject'
+  ]),
+  renameEntity: callbackifyMultiResult(wrapWithLock(renameEntity), [
+    'project',
+    'startPath',
+    'endPath',
+    'rev',
+    'changes'
+  ]),
+  _insertDeletedDocReference: callbackify(_insertDeletedDocReference),
+  _insertDeletedFileReference: callbackify(_insertDeletedFileReference),
+  _putElement: callbackifyMultiResult(_putElement, ['result', 'project']),
+  _confirmFolder,
+  promises: {
+    addDoc: wrapWithLock(addDoc),
+    addFile: wrapWithLock(addFile),
+    addFolder: wrapWithLock(addFolder),
+    replaceFileWithNew: wrapWithLock(replaceFileWithNew),
+    mkdirp: wrapWithLock(mkdirp),
+    moveEntity: wrapWithLock(moveEntity),
+    deleteEntity: wrapWithLock(deleteEntity),
+    renameEntity: wrapWithLock(renameEntity),
+    _insertDeletedDocReference,
+    _insertDeletedFileReference,
+    _putElement
+  }
+}
+
+function wrapWithLock(methodWithoutLock) {
+  // This lock is used whenever we read or write to an existing project's
+  // structure. Some operations to project structure cannot be done atomically
+  // in mongo, this lock is used to prevent reading the structure between two
+  // parts of a staged update.
+  async function methodWithLock(projectId, ...rest) {
+    return LockManager.promises.runWithLock(LOCK_NAMESPACE, projectId, () =>
+      methodWithoutLock(projectId, ...rest)
+    )
+  }
+  return methodWithLock
+}
+
+async function addDoc(projectId, folderId, doc) {
+  const project = await ProjectGetter.promises.getProjectWithoutLock(
+    projectId,
+    {
+      rootFolder: true,
+      name: true,
+      overleaf: true
+    }
+  )
+  folderId = _confirmFolder(project, folderId)
+  const { result, project: newProject } = await _putElement(
+    project,
+    folderId,
+    doc,
+    'doc'
+  )
+  return { result, project: newProject }
+}
+
+async function addFile(projectId, folderId, fileRef) {
+  const project = await ProjectGetter.promises.getProjectWithoutLock(
+    projectId,
+    { rootFolder: true, name: true, overleaf: true }
+  )
+  folderId = _confirmFolder(project, folderId)
+  const { result, project: newProject } = await _putElement(
+    project,
+    folderId,
+    fileRef,
+    'file'
+  )
+  return { result, project: newProject }
+}
+
+async function addFolder(projectId, parentFolderId, folderName) {
+  const project = await ProjectGetter.promises.getProjectWithoutLock(
+    projectId,
+    { rootFolder: true, name: true, overleaf: true }
+  )
+  parentFolderId = _confirmFolder(project, parentFolderId)
+  const folder = new Folder({ name: folderName })
+  await _putElement(project, parentFolderId, folder, 'folder')
+  return { folder, parentFolderId }
+}
+
+async function replaceFileWithNew(projectId, fileId, newFileRef) {
+  const project = await ProjectGetter.promises.getProjectWithoutLock(
+    projectId,
+    { rootFolder: true, name: true, overleaf: true }
+  )
+  const { element: fileRef, path } = await ProjectLocator.promises.findElement({
+    project,
+    element_id: fileId,
+    type: 'file'
+  })
+  await _insertDeletedFileReference(projectId, fileRef)
+  const newProject = await Project.findOneAndUpdate(
+    { _id: project._id },
+    {
+      $set: {
+        [`${path.mongo}._id`]: newFileRef._id,
+        [`${path.mongo}.created`]: new Date(),
+        [`${path.mongo}.linkedFileData`]: newFileRef.linkedFileData,
+        [`${path.mongo}.hash`]: newFileRef.hash
+      },
+      $inc: {
+        version: 1,
+        [`${path.mongo}.rev`]: 1
+      }
+    },
+    { new: true }
+  ).exec()
+  // Note: Mongoose uses new:true to return the modified document
+  // https://mongoosejs.com/docs/api.html#model_Model.findOneAndUpdate
+  // but Mongo uses returnNewDocument:true instead
+  // https://docs.mongodb.com/manual/reference/method/db.collection.findOneAndUpdate/
+  // We are using Mongoose here, but if we ever switch to a direct mongo call
+  // the next line will need to be updated.
+  return { oldFileRef: fileRef, project, path, newProject }
+}
+
+async function mkdirp(projectId, path, options = {}) {
+  // defaults to case insensitive paths, use options {exactCaseMatch:true}
+  // to make matching case-sensitive
+  let folders = path.split('/')
+  folders = _.select(folders, folder => folder.length !== 0)
+
+  const project = await ProjectGetter.promises.getProjectWithOnlyFolders(
+    projectId
+  )
+  if (path === '/') {
+    logger.log(
+      { projectId: project._id },
+      'mkdir is only trying to make path of / so sending back root folder'
+    )
+    return { newFolders: [], folder: project.rootFolder[0] }
+  }
+
+  const newFolders = []
+  let builtUpPath = ''
+  let lastFolder = null
+  for (const folderName of folders) {
+    builtUpPath += `/${folderName}`
+    try {
+      const {
+        element: foundFolder
+      } = await ProjectLocator.promises.findElementByPath({
+        project,
+        path: builtUpPath,
+        exactCaseMatch: options.exactCaseMatch
+      })
+      lastFolder = foundFolder
+    } catch (err) {
+      // Folder couldn't be found. Create it.
+      logger.log(
+        { path, projectId: project._id, folderName },
+        'making folder from mkdirp'
+      )
+      const parentFolderId = lastFolder && lastFolder._id
+      const {
+        folder: newFolder,
+        parentFolderId: newParentFolderId
+      } = await addFolder(projectId, parentFolderId, folderName)
+      newFolder.parentFolder_id = newParentFolderId
+      lastFolder = newFolder
+      newFolders.push(newFolder)
+    }
+  }
+  return { folder: lastFolder, newFolders }
+}
+
+async function moveEntity(projectId, entityId, destFolderId, entityType) {
+  const project = await ProjectGetter.promises.getProjectWithoutLock(
+    projectId,
+    { rootFolder: true, name: true, overleaf: true }
+  )
+  const {
+    element: entity,
+    path: entityPath
+  } = await ProjectLocator.promises.findElement({
+    project,
+    element_id: entityId,
+    type: entityType
+  })
+  // Prevent top-level docs/files with reserved names (to match v1 behaviour)
+  if (_blockedFilename(entityPath, entityType)) {
+    throw new Errors.InvalidNameError('blocked element name')
+  }
+  await _checkValidMove(project, entityType, entity, entityPath, destFolderId)
+  const {
+    docs: oldDocs,
+    files: oldFiles
+  } = await ProjectEntityHandler.promises.getAllEntitiesFromProject(project)
+  // For safety, insert the entity in the destination
+  // location first, and then remove the original.  If
+  // there is an error the entity may appear twice. This
+  // will cause some breakage but is better than being
+  // lost, which is what happens if this is done in the
+  // opposite order.
+  const { result } = await _putElement(
+    project,
+    destFolderId,
+    entity,
+    entityType
+  )
+  // Note: putElement always pushes onto the end of an
+  // array so it will never change an existing mongo
+  // path. Therefore it is safe to remove an element
+  // from the project with an existing path after
+  // calling putElement. But we must be sure that we
+  // have not moved a folder subfolder of itself (which
+  // is done by _checkValidMove above) because that
+  // would lead to it being deleted.
+  const newProject = await _removeElementFromMongoArray(
+    Project,
+    projectId,
+    entityPath.mongo,
+    entityId
+  )
+  const {
+    docs: newDocs,
+    files: newFiles
+  } = await ProjectEntityHandler.promises.getAllEntitiesFromProject(newProject)
+  const startPath = entityPath.fileSystem
+  const endPath = result.path.fileSystem
+  const changes = {
+    oldDocs,
+    newDocs,
+    oldFiles,
+    newFiles,
+    newProject
+  }
+  // check that no files have been lost (or duplicated)
+  if (
+    oldFiles.length !== newFiles.length ||
+    oldDocs.length !== newDocs.length
+  ) {
+    logger.warn(
+      {
+        projectId,
+        oldDocs: oldDocs.length,
+        newDocs: newDocs.length,
+        oldFiles: oldFiles.length,
+        newFiles: newFiles.length,
+        origProject: project,
+        newProject
+      },
+      "project corrupted moving files - shouldn't happen"
+    )
+    throw new Error('unexpected change in project structure')
+  }
+  return { project, startPath, endPath, rev: entity.rev, changes }
+}
+
+async function deleteEntity(projectId, entityId, entityType, callback) {
+  const project = await ProjectGetter.promises.getProjectWithoutLock(
+    projectId,
+    { name: true, rootFolder: true, overleaf: true }
+  )
+  const { element: entity, path } = await ProjectLocator.promises.findElement({
+    project,
+    element_id: entityId,
+    type: entityType
+  })
+  const newProject = await _removeElementFromMongoArray(
+    Project,
+    projectId,
+    path.mongo,
+    entityId
+  )
+  return { entity, path, projectBeforeDeletion: project, newProject }
+}
+
+async function renameEntity(
+  projectId,
+  entityId,
+  entityType,
+  newName,
+  callback
+) {
+  const project = await ProjectGetter.promises.getProjectWithoutLock(
+    projectId,
+    { rootFolder: true, name: true, overleaf: true }
+  )
+  const {
+    element: entity,
+    path: entPath,
+    folder: parentFolder
+  } = await ProjectLocator.promises.findElement({
+    project,
+    element_id: entityId,
+    type: entityType
+  })
+  const startPath = entPath.fileSystem
+  const endPath = path.join(path.dirname(entPath.fileSystem), newName)
+
+  // Prevent top-level docs/files with reserved names (to match v1 behaviour)
+  if (_blockedFilename({ fileSystem: endPath }, entityType)) {
+    throw new Errors.InvalidNameError('blocked element name')
+  }
+
+  // check if the new name already exists in the current folder
+  _checkValidElementName(parentFolder, newName)
+
+  const {
+    docs: oldDocs,
+    files: oldFiles
+  } = await ProjectEntityHandler.promises.getAllEntitiesFromProject(project)
+
+  // we need to increment the project version number for any structure change
+  const newProject = await Project.findOneAndUpdate(
+    { _id: projectId },
+    { $set: { [`${entPath.mongo}.name`]: newName }, $inc: { version: 1 } },
+    { new: true }
+  ).exec()
+
+  const {
+    docs: newDocs,
+    files: newFiles
+  } = await ProjectEntityHandler.promises.getAllEntitiesFromProject(newProject)
+  return {
+    project,
+    startPath,
+    endPath,
+    rev: entity.rev,
+    changes: { oldDocs, newDocs, oldFiles, newFiles, newProject }
+  }
+}
+
+async function _insertDeletedDocReference(projectId, doc) {
+  await Project.updateOne(
+    { _id: projectId },
+    {
+      $push: {
+        deletedDocs: { _id: doc._id, name: doc.name, deletedAt: new Date() }
+      }
+    }
+  ).exec()
+}
+
+async function _insertDeletedFileReference(projectId, fileRef) {
+  await Project.updateOne(
+    { _id: projectId },
+    {
+      $push: {
+        deletedFiles: {
+          _id: fileRef._id,
+          name: fileRef.name,
+          linkedFileData: fileRef.linkedFileData,
+          hash: fileRef.hash,
+          deletedAt: new Date()
+        }
+      }
+    }
+  ).exec()
+}
+
+async function _removeElementFromMongoArray(model, modelId, path, elementId) {
+  const nonArrayPath = path.slice(0, path.lastIndexOf('.'))
+  const newDoc = model
+    .findOneAndUpdate(
+      { _id: modelId },
+      {
+        $pull: { [nonArrayPath]: { _id: elementId } },
+        $inc: { version: 1 }
+      },
+      { new: true }
+    )
+    .exec()
+  return newDoc
+}
+
+function _countElements(project) {
+  function countFolder(folder) {
+    if (folder == null) {
+      return 0
+    }
+
+    let total = 0
+    if (folder.folders) {
+      total += folder.folders.length
+      for (const subfolder of folder.folders) {
+        total += countFolder(subfolder)
+      }
+    }
+    if (folder.docs) {
+      total += folder.docs.length
+    }
+    if (folder.fileRefs) {
+      total += folder.fileRefs.length
+    }
+    return total
+  }
+
+  return countFolder(project.rootFolder[0])
+}
+
+async function _putElement(project, folderId, element, type) {
+  if (element == null || element._id == null) {
+    logger.warn(
+      { projectId: project._id, folderId, element, type },
+      'failed trying to insert element as it was null'
+    )
+    throw new Error('no element passed to be inserted')
+  }
+
+  const pathSegment = _getMongoPathSegmentFromType(type)
+
+  // original check path.resolve("/", element.name) isnt "/#{element.name}" or element.name.match("/")
+  // check if name is allowed
+  if (!SafePath.isCleanFilename(element.name)) {
+    logger.warn(
+      { projectId: project._id, folderId, element, type },
+      'failed trying to insert element as name was invalid'
+    )
+    throw new Errors.InvalidNameError('invalid element name')
+  }
+
+  if (folderId == null) {
+    folderId = project.rootFolder[0]._id
+  }
+
+  if (_countElements(project) > Settings.maxEntitiesPerProject) {
+    logger.warn(
+      { projectId: project._id },
+      'project too big, stopping insertions'
+    )
+    CooldownManager.putProjectOnCooldown(project._id)
+    throw new Error('project_has_to_many_files')
+  }
+
+  const { element: folder, path } = await ProjectLocator.promises.findElement({
+    project,
+    element_id: folderId,
+    type: 'folder'
+  })
+  const newPath = {
+    fileSystem: `${path.fileSystem}/${element.name}`,
+    mongo: path.mongo
+  }
+  // check if the path would be too long
+  if (!SafePath.isAllowedLength(newPath.fileSystem)) {
+    throw new Errors.InvalidNameError('path too long')
+  }
+  // Prevent top-level docs/files with reserved names (to match v1 behaviour)
+  if (_blockedFilename(newPath, type)) {
+    throw new Errors.InvalidNameError('blocked element name')
+  }
+  _checkValidElementName(folder, element.name)
+  element._id = ObjectId(element._id.toString())
+  const mongoPath = `${path.mongo}.${pathSegment}`
+  const newProject = await Project.findOneAndUpdate(
+    { _id: project._id },
+    { $push: { [mongoPath]: element }, $inc: { version: 1 } },
+    { new: true }
+  ).exec()
+  return { result: { path: newPath }, project: newProject }
+}
+
+function _blockedFilename(entityPath, entityType) {
+  // check if name would be blocked in v1
+  // javascript reserved names are forbidden for docs and files
+  // at the top-level (but folders with reserved names are allowed).
+  const isFolder = entityType === 'folder'
+  const dir = path.dirname(entityPath.fileSystem)
+  const file = path.basename(entityPath.fileSystem)
+  const isTopLevel = dir === '/'
+  if (isTopLevel && !isFolder && SafePath.isBlockedFilename(file)) {
+    return true
+  } else {
+    return false
+  }
+}
+
+function _getMongoPathSegmentFromType(type) {
+  const pathSegment = ENTITY_TYPE_TO_MONGO_PATH_SEGMENT[type]
+  if (pathSegment == null) {
+    throw new Error(`Unknown entity type: ${type}`)
+  }
+  return pathSegment
+}
+
+/**
+ * Check if the name is already taken by a doc, file or folder. If so, return an
+ * error "file already exists".
+ */
+function _checkValidElementName(folder, name) {
+  if (folder == null) {
+    return
+  }
+  const elements = []
+    .concat(folder.docs || [])
+    .concat(folder.fileRefs || [])
+    .concat(folder.folders || [])
+  for (const element of elements) {
+    if (element.name === name) {
+      throw new Errors.InvalidNameError('file already exists')
+    }
+  }
+}
+
+function _confirmFolder(project, folderId) {
+  if (folderId == null) {
+    return project.rootFolder[0]._id
+  } else {
+    return folderId
+  }
+}
+
+async function _checkValidMove(
+  project,
+  entityType,
+  entity,
+  entityPath,
+  destFolderId
+) {
+  const {
+    element: destEntity,
+    path: destFolderPath
+  } = await ProjectLocator.promises.findElement({
+    project,
+    element_id: destFolderId,
+    type: 'folder'
+  })
+  // check if there is already a doc/file/folder with the same name
+  // in the destination folder
+  _checkValidElementName(destEntity, entity.name)
+  if (/folder/.test(entityType)) {
+    logger.log(
+      {
+        destFolderPath: destFolderPath.fileSystem,
+        folderPath: entityPath.fileSystem
+      },
+      'checking folder is not moving into child folder'
+    )
+    const isNestedFolder =
+      destFolderPath.fileSystem.slice(0, entityPath.fileSystem.length) ===
+      entityPath.fileSystem
+    if (isNestedFolder) {
+      throw new Errors.InvalidNameError(
+        'destination folder is a child folder of me'
+      )
+    }
+  }
+}

services/web/app/src/Features/Project/ProjectEntityUpdateHandlerCanary.js

@@ -0,0 +1,1634 @@
+/* NOTE: this file is an almost exact copy of ProjectEntityUpdateHandler.js.
+ * The only difference is that it imports
+ * ProjectEntityMongoUpdateHandlerCanary.js. It's meant to be a short-lived
+ * module, so that we can test the async/await code in production for some code
+ * paths only.
+ */
+
+/* eslint-disable
+    camelcase,
+    handle-callback-err,
+    max-len,
+    one-var,
+    standard/no-callback-literal,
+*/
+
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS103: Rewrite code to no longer use __guard__
+ * DS201: Simplify complex destructure assignments
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let ProjectEntityUpdateHandler, self
+const _ = require('lodash')
+const async = require('async')
+const logger = require('logger-sharelatex')
+const Settings = require('settings-sharelatex')
+const path = require('path')
+const { Doc } = require('../../models/Doc')
+const DocstoreManager = require('../Docstore/DocstoreManager')
+const DocumentUpdaterHandler = require('../../Features/DocumentUpdater/DocumentUpdaterHandler')
+const Errors = require('../Errors/Errors')
+const { File } = require('../../models/File')
+const FileStoreHandler = require('../FileStore/FileStoreHandler')
+const LockManager = require('../../infrastructure/LockManager')
+const { Project } = require('../../models/Project')
+const ProjectEntityHandler = require('./ProjectEntityHandler')
+const ProjectGetter = require('./ProjectGetter')
+const ProjectLocator = require('./ProjectLocator')
+const ProjectUpdateHandler = require('./ProjectUpdateHandler')
+const ProjectEntityMongoUpdateHandler = require('./ProjectEntityMongoUpdateHandlerCanary')
+const SafePath = require('./SafePath')
+const TpdsUpdateSender = require('../ThirdPartyDataStore/TpdsUpdateSender')
+
+const LOCK_NAMESPACE = 'sequentialProjectStructureUpdateLock'
+
+const validRootDocExtensions = Settings.validRootDocExtensions
+const validRootDocRegExp = new RegExp(
+  `^\\.(${validRootDocExtensions.join('|')})$`,
+  'i'
+)
+
+const wrapWithLock = function(methodWithoutLock) {
+  // This lock is used to make sure that the project structure updates are made
+  // sequentially. In particular the updates must be made in mongo and sent to
+  // the doc-updater in the same order.
+  let methodWithLock
+  if (typeof methodWithoutLock === 'function') {
+    methodWithLock = function(project_id, ...rest) {
+      const adjustedLength = Math.max(rest.length, 1),
+        args = rest.slice(0, adjustedLength - 1),
+        callback = rest[adjustedLength - 1]
+      return LockManager.runWithLock(
+        LOCK_NAMESPACE,
+        project_id,
+        cb => methodWithoutLock(project_id, ...Array.from(args), cb),
+        callback
+      )
+    }
+    methodWithLock.withoutLock = methodWithoutLock
+    return methodWithLock
+  } else {
+    // handle case with separate setup and locked stages
+    const wrapWithSetup = methodWithoutLock.beforeLock // a function to set things up before the lock
+    const mainTask = methodWithoutLock.withLock // function to execute inside the lock
+    methodWithLock = wrapWithSetup(function(project_id, ...rest) {
+      const adjustedLength = Math.max(rest.length, 1),
+        args = rest.slice(0, adjustedLength - 1),
+        callback = rest[adjustedLength - 1]
+      return LockManager.runWithLock(
+        LOCK_NAMESPACE,
+        project_id,
+        cb => mainTask(project_id, ...Array.from(args), cb),
+        callback
+      )
+    })
+    methodWithLock.withoutLock = wrapWithSetup(mainTask)
+    methodWithLock.beforeLock = methodWithoutLock.beforeLock
+    methodWithLock.mainTask = methodWithoutLock.withLock
+    return methodWithLock
+  }
+}
+
+module.exports = ProjectEntityUpdateHandler = self = {
+  copyFileFromExistingProjectWithProject: wrapWithLock({
+    beforeLock(next) {
+      return function(
+        project_id,
+        project,
+        folder_id,
+        originalProject_id,
+        origonalFileRef,
+        userId,
+        callback
+      ) {
+        if (callback == null) {
+          callback = function(error, fileRef, folder_id) {}
+        }
+        logger.log(
+          { project_id, folder_id, originalProject_id, origonalFileRef },
+          'copying file in s3 with project'
+        )
+        folder_id = ProjectEntityMongoUpdateHandler._confirmFolder(
+          project,
+          folder_id
+        )
+        if (origonalFileRef == null) {
+          logger.err(
+            { project_id, folder_id, originalProject_id, origonalFileRef },
+            'file trying to copy is null'
+          )
+          return callback()
+        }
+        // convert any invalid characters in original file to '_'
+        const fileProperties = {
+          name: SafePath.clean(origonalFileRef.name)
+        }
+        if (origonalFileRef.linkedFileData != null) {
+          fileProperties.linkedFileData = origonalFileRef.linkedFileData
+        }
+        if (origonalFileRef.hash != null) {
+          fileProperties.hash = origonalFileRef.hash
+        }
+        const fileRef = new File(fileProperties)
+        return FileStoreHandler.copyFile(
+          originalProject_id,
+          origonalFileRef._id,
+          project._id,
+          fileRef._id,
+          function(err, fileStoreUrl) {
+            if (err != null) {
+              logger.warn(
+                {
+                  err,
+                  project_id,
+                  folder_id,
+                  originalProject_id,
+                  origonalFileRef
+                },
+                'error coping file in s3'
+              )
+              return callback(err)
+            }
+            return next(
+              project_id,
+              project,
+              folder_id,
+              originalProject_id,
+              origonalFileRef,
+              userId,
+              fileRef,
+              fileStoreUrl,
+              callback
+            )
+          }
+        )
+      }
+    },
+    withLock(
+      project_id,
+      project,
+      folder_id,
+      originalProject_id,
+      origonalFileRef,
+      userId,
+      fileRef,
+      fileStoreUrl,
+      callback
+    ) {
+      if (callback == null) {
+        callback = function(error, fileRef, folder_id) {}
+      }
+      const projectHistoryId = __guard__(
+        project.overleaf != null ? project.overleaf.history : undefined,
+        x => x.id
+      )
+      return ProjectEntityMongoUpdateHandler._putElement(
+        project,
+        folder_id,
+        fileRef,
+        'file',
+        function(err, result, newProject) {
+          if (err != null) {
+            logger.warn(
+              { err, project_id, folder_id },
+              'error putting element as part of copy'
+            )
+            return callback(err)
+          }
+          return TpdsUpdateSender.addFile(
+            {
+              project_id,
+              file_id: fileRef._id,
+              path: __guard__(
+                result != null ? result.path : undefined,
+                x1 => x1.fileSystem
+              ),
+              rev: fileRef.rev,
+              project_name: project.name
+            },
+            function(err) {
+              if (err != null) {
+                logger.err(
+                  {
+                    err,
+                    project_id,
+                    folder_id,
+                    originalProject_id,
+                    origonalFileRef
+                  },
+                  'error sending file to tpds worker'
+                )
+              }
+              const newFiles = [
+                {
+                  file: fileRef,
+                  path: __guard__(
+                    result != null ? result.path : undefined,
+                    x2 => x2.fileSystem
+                  ),
+                  url: fileStoreUrl
+                }
+              ]
+              return DocumentUpdaterHandler.updateProjectStructure(
+                project_id,
+                projectHistoryId,
+                userId,
+                { newFiles, newProject },
+                function(error) {
+                  if (error != null) {
+                    return callback(error)
+                  }
+                  return callback(null, fileRef, folder_id)
+                }
+              )
+            }
+          )
+        }
+      )
+    }
+  }),
+
+  updateDocLines(
+    project_id,
+    doc_id,
+    lines,
+    version,
+    ranges,
+    lastUpdatedAt,
+    lastUpdatedBy,
+    callback
+  ) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    return ProjectGetter.getProjectWithoutDocLines(project_id, function(
+      err,
+      project
+    ) {
+      if (err != null) {
+        return callback(err)
+      }
+      if (project == null) {
+        return callback(new Errors.NotFoundError('project not found'))
+      }
+      logger.log({ project_id, doc_id }, 'updating doc lines')
+      return ProjectLocator.findElement(
+        { project, element_id: doc_id, type: 'docs' },
+        function(err, doc, path) {
+          let isDeletedDoc = false
+          if (err != null) {
+            if (err instanceof Errors.NotFoundError) {
+              // We need to be able to update the doclines of deleted docs. This is
+              // so the doc-updater can flush a doc's content to the doc-store after
+              // the doc is deleted.
+              isDeletedDoc = true
+              doc = _.find(
+                project.deletedDocs,
+                doc => doc._id.toString() === doc_id.toString()
+              )
+            } else {
+              return callback(err)
+            }
+          }
+
+          if (doc == null) {
+            // Do not allow an update to a doc which has never exist on this project
+            logger.warn(
+              { doc_id, project_id, lines },
+              'doc not found while updating doc lines'
+            )
+            return callback(new Errors.NotFoundError('doc not found'))
+          }
+
+          logger.log(
+            { project_id, doc_id },
+            'telling docstore manager to update doc'
+          )
+          return DocstoreManager.updateDoc(
+            project_id,
+            doc_id,
+            lines,
+            version,
+            ranges,
+            function(err, modified, rev) {
+              if (err != null) {
+                logger.warn(
+                  { err, doc_id, project_id, lines },
+                  'error sending doc to docstore'
+                )
+                return callback(err)
+              }
+              logger.log(
+                { project_id, doc_id, modified },
+                'finished updating doc lines'
+              )
+              // path will only be present if the doc is not deleted
+              if (modified && !isDeletedDoc) {
+                // Don't need to block for marking as updated
+                ProjectUpdateHandler.markAsUpdated(
+                  project_id,
+                  lastUpdatedAt,
+                  lastUpdatedBy
+                )
+                return TpdsUpdateSender.addDoc(
+                  {
+                    project_id,
+                    path: path.fileSystem,
+                    doc_id,
+                    project_name: project.name,
+                    rev
+                  },
+                  callback
+                )
+              } else {
+                return callback()
+              }
+            }
+          )
+        }
+      )
+    })
+  },
+
+  setRootDoc(project_id, newRootDocID, callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    logger.log({ project_id, rootDocId: newRootDocID }, 'setting root doc')
+    if (project_id == null || newRootDocID == null) {
+      return callback(
+        new Errors.InvalidError('missing arguments (project or doc)')
+      )
+    }
+    ProjectEntityHandler.getDocPathByProjectIdAndDocId(
+      project_id,
+      newRootDocID,
+      function(err, docPath) {
+        if (err != null) {
+          return callback(err)
+        }
+        if (ProjectEntityUpdateHandler.isPathValidForRootDoc(docPath)) {
+          return Project.update(
+            { _id: project_id },
+            { rootDoc_id: newRootDocID },
+            {},
+            callback
+          )
+        } else {
+          return callback(
+            new Errors.UnsupportedFileTypeError(
+              'invalid file extension for root doc'
+            )
+          )
+        }
+      }
+    )
+  },
+
+  unsetRootDoc(project_id, callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    logger.log({ project_id }, 'removing root doc')
+    return Project.update(
+      { _id: project_id },
+      { $unset: { rootDoc_id: true } },
+      {},
+      callback
+    )
+  },
+
+  _addDocAndSendToTpds(project_id, folder_id, doc, callback) {
+    if (callback == null) {
+      callback = function(error, result, project) {}
+    }
+    return ProjectEntityMongoUpdateHandler.addDoc(
+      project_id,
+      folder_id,
+      doc,
+      function(err, result, project) {
+        if (err != null) {
+          logger.warn(
+            {
+              err,
+              project_id,
+              folder_id,
+              doc_name: doc != null ? doc.name : undefined,
+              doc_id: doc != null ? doc._id : undefined
+            },
+            'error adding file with project'
+          )
+          return callback(err)
+        }
+        return TpdsUpdateSender.addDoc(
+          {
+            project_id,
+            doc_id: doc != null ? doc._id : undefined,
+            path: __guard__(
+              result != null ? result.path : undefined,
+              x => x.fileSystem
+            ),
+            project_name: project.name,
+            rev: 0
+          },
+          function(err) {
+            if (err != null) {
+              return callback(err)
+            }
+            return callback(null, result, project)
+          }
+        )
+      }
+    )
+  },
+
+  addDoc(project_id, folder_id, docName, docLines, userId, callback) {
+    return self.addDocWithRanges(
+      project_id,
+      folder_id,
+      docName,
+      docLines,
+      {},
+      userId,
+      callback
+    )
+  },
+
+  addDocWithRanges: wrapWithLock({
+    beforeLock(next) {
+      return function(
+        project_id,
+        folder_id,
+        docName,
+        docLines,
+        ranges,
+        userId,
+        callback
+      ) {
+        if (callback == null) {
+          callback = function(error, doc, folder_id) {}
+        }
+        if (!SafePath.isCleanFilename(docName)) {
+          return callback(new Errors.InvalidNameError('invalid element name'))
+        }
+        // Put doc in docstore first, so that if it errors, we don't have a doc_id in the project
+        // which hasn't been created in docstore.
+        const doc = new Doc({ name: docName })
+        return DocstoreManager.updateDoc(
+          project_id.toString(),
+          doc._id.toString(),
+          docLines,
+          0,
+          ranges,
+          function(err, modified, rev) {
+            if (err != null) {
+              return callback(err)
+            }
+            return next(
+              project_id,
+              folder_id,
+              doc,
+              docName,
+              docLines,
+              ranges,
+              userId,
+              callback
+            )
+          }
+        )
+      }
+    },
+    withLock(
+      project_id,
+      folder_id,
+      doc,
+      docName,
+      docLines,
+      ranges,
+      userId,
+      callback
+    ) {
+      if (callback == null) {
+        callback = function(error, doc, folder_id) {}
+      }
+      return ProjectEntityUpdateHandler._addDocAndSendToTpds(
+        project_id,
+        folder_id,
+        doc,
+        function(err, result, project) {
+          if (err != null) {
+            return callback(err)
+          }
+          const docPath = __guard__(
+            result != null ? result.path : undefined,
+            x => x.fileSystem
+          )
+          const projectHistoryId = __guard__(
+            project.overleaf != null ? project.overleaf.history : undefined,
+            x1 => x1.id
+          )
+          const newDocs = [
+            {
+              doc,
+              path: docPath,
+              docLines: docLines.join('\n')
+            }
+          ]
+          return DocumentUpdaterHandler.updateProjectStructure(
+            project_id,
+            projectHistoryId,
+            userId,
+            { newDocs, newProject: project },
+            function(error) {
+              if (error != null) {
+                return callback(error)
+              }
+              return callback(null, doc, folder_id)
+            }
+          )
+        }
+      )
+    }
+  }),
+
+  _uploadFile(
+    project_id,
+    folder_id,
+    fileName,
+    fsPath,
+    linkedFileData,
+    callback
+  ) {
+    if (callback == null) {
+      callback = function(error, fileStoreUrl, fileRef) {}
+    }
+    if (!SafePath.isCleanFilename(fileName)) {
+      return callback(new Errors.InvalidNameError('invalid element name'))
+    }
+    const fileArgs = {
+      name: fileName,
+      linkedFileData
+    }
+    return FileStoreHandler.uploadFileFromDisk(
+      project_id,
+      fileArgs,
+      fsPath,
+      function(err, fileStoreUrl, fileRef) {
+        if (err != null) {
+          logger.warn(
+            { err, project_id, folder_id, file_name: fileName, fileRef },
+            'error uploading image to s3'
+          )
+          return callback(err)
+        }
+        return callback(null, fileStoreUrl, fileRef)
+      }
+    )
+  },
+
+  _addFileAndSendToTpds(project_id, folder_id, fileRef, callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    return ProjectEntityMongoUpdateHandler.addFile(
+      project_id,
+      folder_id,
+      fileRef,
+      function(err, result, project) {
+        if (err != null) {
+          logger.warn(
+            { err, project_id, folder_id, file_name: fileRef.name, fileRef },
+            'error adding file with project'
+          )
+          return callback(err)
+        }
+        return TpdsUpdateSender.addFile(
+          {
+            project_id,
+            file_id: fileRef._id,
+            path: __guard__(
+              result != null ? result.path : undefined,
+              x => x.fileSystem
+            ),
+            project_name: project.name,
+            rev: fileRef.rev
+          },
+          function(err) {
+            if (err != null) {
+              return callback(err)
+            }
+            return callback(null, result, project)
+          }
+        )
+      }
+    )
+  },
+
+  addFile: wrapWithLock({
+    beforeLock(next) {
+      return function(
+        project_id,
+        folder_id,
+        fileName,
+        fsPath,
+        linkedFileData,
+        userId,
+        callback
+      ) {
+        if (!SafePath.isCleanFilename(fileName)) {
+          return callback(new Errors.InvalidNameError('invalid element name'))
+        }
+        return ProjectEntityUpdateHandler._uploadFile(
+          project_id,
+          folder_id,
+          fileName,
+          fsPath,
+          linkedFileData,
+          function(error, fileStoreUrl, fileRef) {
+            if (error != null) {
+              return callback(error)
+            }
+            return next(
+              project_id,
+              folder_id,
+              fileName,
+              fsPath,
+              linkedFileData,
+              userId,
+              fileRef,
+              fileStoreUrl,
+              callback
+            )
+          }
+        )
+      }
+    },
+    withLock(
+      project_id,
+      folder_id,
+      fileName,
+      fsPath,
+      linkedFileData,
+      userId,
+      fileRef,
+      fileStoreUrl,
+      callback
+    ) {
+      if (callback == null) {
+        callback = function(error, fileRef, folder_id) {}
+      }
+      return ProjectEntityUpdateHandler._addFileAndSendToTpds(
+        project_id,
+        folder_id,
+        fileRef,
+        function(err, result, project) {
+          if (err != null) {
+            return callback(err)
+          }
+          const projectHistoryId = __guard__(
+            project.overleaf != null ? project.overleaf.history : undefined,
+            x => x.id
+          )
+          const newFiles = [
+            {
+              file: fileRef,
+              path: __guard__(
+                result != null ? result.path : undefined,
+                x1 => x1.fileSystem
+              ),
+              url: fileStoreUrl
+            }
+          ]
+          return DocumentUpdaterHandler.updateProjectStructure(
+            project_id,
+            projectHistoryId,
+            userId,
+            { newFiles, newProject: project },
+            function(error) {
+              if (error != null) {
+                return callback(error)
+              }
+              return callback(null, fileRef, folder_id)
+            }
+          )
+        }
+      )
+    }
+  }),
+
+  replaceFile: wrapWithLock({
+    beforeLock(next) {
+      return function(
+        project_id,
+        file_id,
+        fsPath,
+        linkedFileData,
+        userId,
+        callback
+      ) {
+        // create a new file
+        const fileArgs = {
+          name: 'dummy-upload-filename',
+          linkedFileData
+        }
+        return FileStoreHandler.uploadFileFromDisk(
+          project_id,
+          fileArgs,
+          fsPath,
+          function(err, fileStoreUrl, fileRef) {
+            if (err != null) {
+              return callback(err)
+            }
+            return next(
+              project_id,
+              file_id,
+              fsPath,
+              linkedFileData,
+              userId,
+              fileRef,
+              fileStoreUrl,
+              callback
+            )
+          }
+        )
+      }
+    },
+    withLock(
+      project_id,
+      file_id,
+      fsPath,
+      linkedFileData,
+      userId,
+      newFileRef,
+      fileStoreUrl,
+      callback
+    ) {
+      return ProjectEntityMongoUpdateHandler.replaceFileWithNew(
+        project_id,
+        file_id,
+        newFileRef,
+        function(err, oldFileRef, project, path, newProject) {
+          if (err != null) {
+            return callback(err)
+          }
+          const oldFiles = [
+            {
+              file: oldFileRef,
+              path: path.fileSystem
+            }
+          ]
+          const newFiles = [
+            {
+              file: newFileRef,
+              path: path.fileSystem,
+              url: fileStoreUrl
+            }
+          ]
+          const projectHistoryId = __guard__(
+            project.overleaf != null ? project.overleaf.history : undefined,
+            x => x.id
+          )
+          // Increment the rev for an in-place update (with the same path) so the third-party-datastore
+          // knows this is a new file.
+          // Ideally we would get this from ProjectEntityMongoUpdateHandler.replaceFileWithNew
+          // but it returns the original oldFileRef (after incrementing the rev value in mongo),
+          // so we add 1 to the rev from that. This isn't atomic and relies on the lock
+          // but it is acceptable for now.
+          return TpdsUpdateSender.addFile(
+            {
+              project_id: project._id,
+              file_id: newFileRef._id,
+              path: path.fileSystem,
+              rev: oldFileRef.rev + 1,
+              project_name: project.name
+            },
+            function(err) {
+              if (err != null) {
+                return callback(err)
+              }
+              return DocumentUpdaterHandler.updateProjectStructure(
+                project_id,
+                projectHistoryId,
+                userId,
+                { oldFiles, newFiles, newProject },
+                callback
+              )
+            }
+          )
+        }
+      )
+    }
+  }),
+
+  upsertDoc: wrapWithLock(function(
+    project_id,
+    folder_id,
+    docName,
+    docLines,
+    source,
+    userId,
+    callback
+  ) {
+    if (callback == null) {
+      callback = function(err, doc, folder_id, isNewDoc) {}
+    }
+    if (!SafePath.isCleanFilename(docName)) {
+      return callback(new Errors.InvalidNameError('invalid element name'))
+    }
+    return ProjectLocator.findElement(
+      { project_id, element_id: folder_id, type: 'folder' },
+      function(error, folder) {
+        if (error != null) {
+          return callback(error)
+        }
+        if (folder == null) {
+          return callback(new Error("Couldn't find folder"))
+        }
+        let existingDoc = null
+        for (let doc of Array.from(folder.docs)) {
+          if (doc.name === docName) {
+            existingDoc = doc
+            break
+          }
+        }
+        if (existingDoc != null) {
+          return DocumentUpdaterHandler.setDocument(
+            project_id,
+            existingDoc._id,
+            userId,
+            docLines,
+            source,
+            err => {
+              logger.log(
+                { project_id, doc_id: existingDoc._id },
+                'notifying users that the document has been updated'
+              )
+              return DocumentUpdaterHandler.flushDocToMongo(
+                project_id,
+                existingDoc._id,
+                function(err) {
+                  if (err != null) {
+                    return callback(err)
+                  }
+                  return callback(null, existingDoc, existingDoc == null)
+                }
+              )
+            }
+          )
+        } else {
+          return self.addDocWithRanges.withoutLock(
+            project_id,
+            folder_id,
+            docName,
+            docLines,
+            {},
+            userId,
+            function(err, doc) {
+              if (err != null) {
+                return callback(err)
+              }
+              return callback(null, doc, existingDoc == null)
+            }
+          )
+        }
+      }
+    )
+  }),
+
+  upsertFile: wrapWithLock({
+    beforeLock(next) {
+      return function(
+        project_id,
+        folder_id,
+        fileName,
+        fsPath,
+        linkedFileData,
+        userId,
+        callback
+      ) {
+        if (!SafePath.isCleanFilename(fileName)) {
+          return callback(new Errors.InvalidNameError('invalid element name'))
+        }
+        // create a new file
+        const fileArgs = {
+          name: fileName,
+          linkedFileData
+        }
+        return FileStoreHandler.uploadFileFromDisk(
+          project_id,
+          fileArgs,
+          fsPath,
+          function(err, fileStoreUrl, fileRef) {
+            if (err != null) {
+              return callback(err)
+            }
+            return next(
+              project_id,
+              folder_id,
+              fileName,
+              fsPath,
+              linkedFileData,
+              userId,
+              fileRef,
+              fileStoreUrl,
+              callback
+            )
+          }
+        )
+      }
+    },
+    withLock(
+      project_id,
+      folder_id,
+      fileName,
+      fsPath,
+      linkedFileData,
+      userId,
+      newFileRef,
+      fileStoreUrl,
+      callback
+    ) {
+      if (callback == null) {
+        callback = function(err, file, isNewFile, existingFile) {}
+      }
+      return ProjectLocator.findElement(
+        { project_id, element_id: folder_id, type: 'folder' },
+        function(error, folder) {
+          if (error != null) {
+            return callback(error)
+          }
+          if (folder == null) {
+            return callback(new Error("Couldn't find folder"))
+          }
+          let existingFile = null
+          for (let fileRef of Array.from(folder.fileRefs)) {
+            if (fileRef.name === fileName) {
+              existingFile = fileRef
+              break
+            }
+          }
+          if (existingFile != null) {
+            // this calls directly into the replaceFile main task (without the beforeLock part)
+            return self.replaceFile.mainTask(
+              project_id,
+              existingFile._id,
+              fsPath,
+              linkedFileData,
+              userId,
+              newFileRef,
+              fileStoreUrl,
+              function(err) {
+                if (err != null) {
+                  return callback(err)
+                }
+                return callback(
+                  null,
+                  newFileRef,
+                  existingFile == null,
+                  existingFile
+                )
+              }
+            )
+          } else {
+            // this calls directly into the addFile main task (without the beforeLock part)
+            return self.addFile.mainTask(
+              project_id,
+              folder_id,
+              fileName,
+              fsPath,
+              linkedFileData,
+              userId,
+              newFileRef,
+              fileStoreUrl,
+              function(err) {
+                if (err != null) {
+                  return callback(err)
+                }
+                return callback(
+                  null,
+                  newFileRef,
+                  existingFile == null,
+                  existingFile
+                )
+              }
+            )
+          }
+        }
+      )
+    }
+  }),
+
+  upsertDocWithPath: wrapWithLock(function(
+    project_id,
+    elementPath,
+    docLines,
+    source,
+    userId,
+    callback
+  ) {
+    if (!SafePath.isCleanPath(elementPath)) {
+      return callback(new Errors.InvalidNameError('invalid element name'))
+    }
+    const docName = path.basename(elementPath)
+    const folderPath = path.dirname(elementPath)
+    return self.mkdirp.withoutLock(project_id, folderPath, function(
+      err,
+      newFolders,
+      folder
+    ) {
+      if (err != null) {
+        return callback(err)
+      }
+      return self.upsertDoc.withoutLock(
+        project_id,
+        folder._id,
+        docName,
+        docLines,
+        source,
+        userId,
+        function(err, doc, isNewDoc) {
+          if (err != null) {
+            return callback(err)
+          }
+          return callback(null, doc, isNewDoc, newFolders, folder)
+        }
+      )
+    })
+  }),
+
+  upsertFileWithPath: wrapWithLock({
+    beforeLock(next) {
+      return function(
+        project_id,
+        elementPath,
+        fsPath,
+        linkedFileData,
+        userId,
+        callback
+      ) {
+        if (!SafePath.isCleanPath(elementPath)) {
+          return callback(new Errors.InvalidNameError('invalid element name'))
+        }
+        const fileName = path.basename(elementPath)
+        const folderPath = path.dirname(elementPath)
+        // create a new file
+        const fileArgs = {
+          name: fileName,
+          linkedFileData
+        }
+        return FileStoreHandler.uploadFileFromDisk(
+          project_id,
+          fileArgs,
+          fsPath,
+          function(err, fileStoreUrl, fileRef) {
+            if (err != null) {
+              return callback(err)
+            }
+            return next(
+              project_id,
+              folderPath,
+              fileName,
+              fsPath,
+              linkedFileData,
+              userId,
+              fileRef,
+              fileStoreUrl,
+              callback
+            )
+          }
+        )
+      }
+    },
+    withLock(
+      project_id,
+      folderPath,
+      fileName,
+      fsPath,
+      linkedFileData,
+      userId,
+      fileRef,
+      fileStoreUrl,
+      callback
+    ) {
+      return self.mkdirp.withoutLock(project_id, folderPath, function(
+        err,
+        newFolders,
+        folder
+      ) {
+        if (err != null) {
+          return callback(err)
+        }
+        // this calls directly into the upsertFile main task (without the beforeLock part)
+        return self.upsertFile.mainTask(
+          project_id,
+          folder._id,
+          fileName,
+          fsPath,
+          linkedFileData,
+          userId,
+          fileRef,
+          fileStoreUrl,
+          function(err, newFile, isNewFile, existingFile) {
+            if (err != null) {
+              return callback(err)
+            }
+            return callback(
+              null,
+              newFile,
+              isNewFile,
+              existingFile,
+              newFolders,
+              folder
+            )
+          }
+        )
+      })
+    }
+  }),
+
+  deleteEntity: wrapWithLock(function(
+    project_id,
+    entity_id,
+    entityType,
+    userId,
+    callback
+  ) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    logger.log({ entity_id, entityType, project_id }, 'deleting project entity')
+    if (entityType == null) {
+      logger.warn({ err: 'No entityType set', project_id, entity_id })
+      return callback(new Error('No entityType set'))
+    }
+    entityType = entityType.toLowerCase()
+    return ProjectEntityMongoUpdateHandler.deleteEntity(
+      project_id,
+      entity_id,
+      entityType,
+      function(error, entity, path, projectBeforeDeletion, newProject) {
+        if (error != null) {
+          return callback(error)
+        }
+        return self._cleanUpEntity(
+          projectBeforeDeletion,
+          newProject,
+          entity,
+          entityType,
+          path.fileSystem,
+          userId,
+          function(error) {
+            if (error != null) {
+              return callback(error)
+            }
+            return TpdsUpdateSender.deleteEntity(
+              {
+                project_id,
+                path: path.fileSystem,
+                project_name: projectBeforeDeletion.name
+              },
+              function(error) {
+                if (error != null) {
+                  return callback(error)
+                }
+                return callback(null, entity_id)
+              }
+            )
+          }
+        )
+      }
+    )
+  }),
+
+  deleteEntityWithPath: wrapWithLock((project_id, path, userId, callback) =>
+    ProjectLocator.findElementByPath({ project_id, path }, function(
+      err,
+      element,
+      type
+    ) {
+      if (err != null) {
+        return callback(err)
+      }
+      if (element == null) {
+        return callback(new Errors.NotFoundError('project not found'))
+      }
+      return self.deleteEntity.withoutLock(
+        project_id,
+        element._id,
+        type,
+        userId,
+        callback
+      )
+    })
+  ),
+
+  mkdirp: wrapWithLock(function(project_id, path, callback) {
+    if (callback == null) {
+      callback = function(err, newlyCreatedFolders, lastFolderInPath) {}
+    }
+    for (let folder of Array.from(path.split('/'))) {
+      if (folder.length > 0 && !SafePath.isCleanFilename(folder)) {
+        return callback(new Errors.InvalidNameError('invalid element name'))
+      }
+    }
+    return ProjectEntityMongoUpdateHandler.mkdirp(
+      project_id,
+      path,
+      { exactCaseMatch: false },
+      callback
+    )
+  }),
+
+  mkdirpWithExactCase: wrapWithLock(function(project_id, path, callback) {
+    if (callback == null) {
+      callback = function(err, newlyCreatedFolders, lastFolderInPath) {}
+    }
+    for (let folder of Array.from(path.split('/'))) {
+      if (folder.length > 0 && !SafePath.isCleanFilename(folder)) {
+        return callback(new Errors.InvalidNameError('invalid element name'))
+      }
+    }
+    return ProjectEntityMongoUpdateHandler.mkdirp(
+      project_id,
+      path,
+      { exactCaseMatch: true },
+      callback
+    )
+  }),
+
+  addFolder: wrapWithLock(function(
+    project_id,
+    parentFolder_id,
+    folderName,
+    callback
+  ) {
+    if (!SafePath.isCleanFilename(folderName)) {
+      return callback(new Errors.InvalidNameError('invalid element name'))
+    }
+    return ProjectEntityMongoUpdateHandler.addFolder(
+      project_id,
+      parentFolder_id,
+      folderName,
+      callback
+    )
+  }),
+
+  moveEntity: wrapWithLock(function(
+    project_id,
+    entity_id,
+    destFolderId,
+    entityType,
+    userId,
+    callback
+  ) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    logger.log(
+      { entityType, entity_id, project_id, destFolderId },
+      'moving entity'
+    )
+    if (entityType == null) {
+      logger.warn({ err: 'No entityType set', project_id, entity_id })
+      return callback(new Error('No entityType set'))
+    }
+    entityType = entityType.toLowerCase()
+    return ProjectEntityMongoUpdateHandler.moveEntity(
+      project_id,
+      entity_id,
+      destFolderId,
+      entityType,
+      function(err, project, startPath, endPath, rev, changes) {
+        if (err != null) {
+          return callback(err)
+        }
+        const projectHistoryId = __guard__(
+          project.overleaf != null ? project.overleaf.history : undefined,
+          x => x.id
+        )
+        TpdsUpdateSender.moveEntity({
+          project_id,
+          project_name: project.name,
+          startPath,
+          endPath,
+          rev
+        })
+        return DocumentUpdaterHandler.updateProjectStructure(
+          project_id,
+          projectHistoryId,
+          userId,
+          changes,
+          callback
+        )
+      }
+    )
+  }),
+
+  renameEntity: wrapWithLock(function(
+    project_id,
+    entity_id,
+    entityType,
+    newName,
+    userId,
+    callback
+  ) {
+    if (!SafePath.isCleanFilename(newName)) {
+      return callback(new Errors.InvalidNameError('invalid element name'))
+    }
+    logger.log({ entity_id, project_id }, `renaming ${entityType}`)
+    if (entityType == null) {
+      logger.warn({ err: 'No entityType set', project_id, entity_id })
+      return callback(new Error('No entityType set'))
+    }
+    entityType = entityType.toLowerCase()
+
+    return ProjectEntityMongoUpdateHandler.renameEntity(
+      project_id,
+      entity_id,
+      entityType,
+      newName,
+      function(err, project, startPath, endPath, rev, changes) {
+        if (err != null) {
+          return callback(err)
+        }
+        const projectHistoryId = __guard__(
+          project.overleaf != null ? project.overleaf.history : undefined,
+          x => x.id
+        )
+        TpdsUpdateSender.moveEntity({
+          project_id,
+          project_name: project.name,
+          startPath,
+          endPath,
+          rev
+        })
+        return DocumentUpdaterHandler.updateProjectStructure(
+          project_id,
+          projectHistoryId,
+          userId,
+          changes,
+          callback
+        )
+      }
+    )
+  }),
+
+  // This doesn't directly update project structure but we need to take the lock
+  // to prevent anything else being queued before the resync update
+  resyncProjectHistory: wrapWithLock((project_id, callback) =>
+    ProjectGetter.getProject(
+      project_id,
+      { rootFolder: true, overleaf: true },
+      function(error, project) {
+        if (error != null) {
+          return callback(error)
+        }
+
+        const projectHistoryId = __guard__(
+          __guard__(
+            project != null ? project.overleaf : undefined,
+            x1 => x1.history
+          ),
+          x => x.id
+        )
+        if (projectHistoryId == null) {
+          error = new Errors.ProjectHistoryDisabledError(
+            `project history not enabled for ${project_id}`
+          )
+          return callback(error)
+        }
+
+        return ProjectEntityHandler.getAllEntitiesFromProject(project, function(
+          error,
+          docs,
+          files
+        ) {
+          if (error != null) {
+            return callback(error)
+          }
+
+          docs = _.map(docs, doc => ({
+            doc: doc.doc._id,
+            path: doc.path
+          }))
+
+          files = _.map(files, file => ({
+            file: file.file._id,
+            path: file.path,
+            url: FileStoreHandler._buildUrl(project_id, file.file._id)
+          }))
+
+          return DocumentUpdaterHandler.resyncProjectHistory(
+            project_id,
+            projectHistoryId,
+            docs,
+            files,
+            callback
+          )
+        })
+      }
+    )
+  ),
+
+  isPathValidForRootDoc(docPath) {
+    let docExtension = path.extname(docPath)
+    return validRootDocRegExp.test(docExtension)
+  },
+
+  _cleanUpEntity(
+    project,
+    newProject,
+    entity,
+    entityType,
+    path,
+    userId,
+    callback
+  ) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    return self._updateProjectStructureWithDeletedEntity(
+      project,
+      newProject,
+      entity,
+      entityType,
+      path,
+      userId,
+      function(error) {
+        if (error != null) {
+          return callback(error)
+        }
+        if (entityType.indexOf('file') !== -1) {
+          return self._cleanUpFile(project, entity, path, userId, callback)
+        } else if (entityType.indexOf('doc') !== -1) {
+          return self._cleanUpDoc(project, entity, path, userId, callback)
+        } else if (entityType.indexOf('folder') !== -1) {
+          return self._cleanUpFolder(project, entity, path, userId, callback)
+        } else {
+          return callback()
+        }
+      }
+    )
+  },
+
+  // Note: the _cleanUpEntity code and _updateProjectStructureWithDeletedEntity
+  // methods both need to recursively iterate over the entities in folder.
+  // These are currently using separate implementations of the recursion. In
+  // future, these could be simplified using a common project entity iterator.
+  _updateProjectStructureWithDeletedEntity(
+    project,
+    newProject,
+    entity,
+    entityType,
+    entityPath,
+    userId,
+    callback
+  ) {
+    // compute the changes to the project structure
+    let changes
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    if (entityType.indexOf('file') !== -1) {
+      changes = { oldFiles: [{ file: entity, path: entityPath }] }
+    } else if (entityType.indexOf('doc') !== -1) {
+      changes = { oldDocs: [{ doc: entity, path: entityPath }] }
+    } else if (entityType.indexOf('folder') !== -1) {
+      changes = { oldDocs: [], oldFiles: [] }
+      var _recurseFolder = function(folder, folderPath) {
+        for (let doc of Array.from(folder.docs)) {
+          changes.oldDocs.push({ doc, path: path.join(folderPath, doc.name) })
+        }
+        for (let file of Array.from(folder.fileRefs)) {
+          changes.oldFiles.push({
+            file,
+            path: path.join(folderPath, file.name)
+          })
+        }
+        return Array.from(folder.folders).map(childFolder =>
+          _recurseFolder(childFolder, path.join(folderPath, childFolder.name))
+        )
+      }
+      _recurseFolder(entity, entityPath)
+    }
+    // now send the project structure changes to the docupdater
+    changes.newProject = newProject
+    const project_id = project._id.toString()
+    const projectHistoryId = __guard__(
+      project.overleaf != null ? project.overleaf.history : undefined,
+      x => x.id
+    )
+    return DocumentUpdaterHandler.updateProjectStructure(
+      project_id,
+      projectHistoryId,
+      userId,
+      changes,
+      callback
+    )
+  },
+
+  _cleanUpDoc(project, doc, path, userId, callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    const project_id = project._id.toString()
+    const doc_id = doc._id.toString()
+    const unsetRootDocIfRequired = callback => {
+      if (
+        project.rootDoc_id != null &&
+        project.rootDoc_id.toString() === doc_id
+      ) {
+        return this.unsetRootDoc(project_id, callback)
+      } else {
+        return callback()
+      }
+    }
+
+    return unsetRootDocIfRequired(function(error) {
+      if (error != null) {
+        return callback(error)
+      }
+      return ProjectEntityMongoUpdateHandler._insertDeletedDocReference(
+        project._id,
+        doc,
+        function(error) {
+          if (error != null) {
+            return callback(error)
+          }
+          return DocumentUpdaterHandler.deleteDoc(project_id, doc_id, function(
+            error
+          ) {
+            if (error != null) {
+              return callback(error)
+            }
+            return DocstoreManager.deleteDoc(project_id, doc_id, callback)
+          })
+        }
+      )
+    })
+  },
+
+  _cleanUpFile(project, file, path, userId, callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    return ProjectEntityMongoUpdateHandler._insertDeletedFileReference(
+      project._id,
+      file,
+      callback
+    )
+  },
+
+  _cleanUpFolder(project, folder, folderPath, userId, callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    const jobs = []
+    for (let doc of Array.from(folder.docs)) {
+      ;(function(doc) {
+        const docPath = path.join(folderPath, doc.name)
+        return jobs.push(callback =>
+          self._cleanUpDoc(project, doc, docPath, userId, callback)
+        )
+      })(doc)
+    }
+
+    for (let file of Array.from(folder.fileRefs)) {
+      ;(function(file) {
+        const filePath = path.join(folderPath, file.name)
+        return jobs.push(callback =>
+          self._cleanUpFile(project, file, filePath, userId, callback)
+        )
+      })(file)
+    }
+
+    for (let childFolder of Array.from(folder.folders)) {
+      ;(function(childFolder) {
+        folderPath = path.join(folderPath, childFolder.name)
+        return jobs.push(callback =>
+          self._cleanUpFolder(
+            project,
+            childFolder,
+            folderPath,
+            userId,
+            callback
+          )
+        )
+      })(childFolder)
+    }
+
+    return async.series(jobs, callback)
+  }
+}
+
+function __guard__(value, transform) {
+  return typeof value !== 'undefined' && value !== null
+    ? transform(value)
+    : undefined
+}

services/web/app/src/Features/Project/ProjectLocator.js

@@ -4,6 +4,7 @@ const Errors = require('../Errors/Errors')
 const _ = require('underscore')
 const logger = require('logger-sharelatex')
 const async = require('async')
+const { promisifyAll } = require('../../util/promises')
 
 const ProjectLocator = {
   findElement(options, _callback) {
@@ -327,3 +328,9 @@ function getIndexOf(searchEntity, id) {
 }
 
 module.exports = ProjectLocator
+module.exports.promises = promisifyAll(ProjectLocator, {
+  multiResult: {
+    findElement: ['element', 'path', 'folder'],
+    findElementByPath: ['element', 'type']
+  }
+})

services/web/app/src/infrastructure/LockManager.js

@@ -1,3 +1,4 @@
+const { callbackify, promisify } = require('util')
 const metrics = require('metrics-sharelatex')
 const RedisWrapper = require('./RedisWrapper')
 const rclient = RedisWrapper.client('lock')
@@ -182,3 +183,11 @@ const LockManager = {
 }
 
 module.exports = LockManager
+
+const promisifiedRunWithLock = promisify(LockManager.runWithLock)
+LockManager.promises = {
+  runWithLock(namespace, id, runner) {
+    const cbRunner = callbackify(runner)
+    return promisifiedRunWithLock(namespace, id, cbRunner)
+  }
+}

services/web/app/src/util/promises.js

@@ -3,6 +3,8 @@ const pLimit = require('p-limit')
 
 module.exports = {
   promisifyAll,
+  promisifyMultiResult,
+  callbackifyMultiResult,
   expressify,
   promiseMapWithLimit
 }
@@ -19,22 +21,100 @@ module.exports = {
  *
  * This will not magically fix all modules. Special cases should be promisified
  * manually.
+ *
+ * The second argument is a bag of options:
+ *
+ * - without: an array of function names that shouldn't be promisified
+ *
+ * - multiResult: an object whose keys are function names and values are lists
+ *   of parameter names. This is meant for functions that invoke their callbacks
+ *   with more than one result in separate parameters. The promisifed function
+ *   will return these results as a single object, with each result keyed under
+ *   the corresponding parameter name.
  */
 function promisifyAll(module, opts = {}) {
-  const { without = [] } = opts
+  const { without = [], multiResult = {} } = opts
   const promises = {}
   for (const propName of Object.getOwnPropertyNames(module)) {
     if (without.includes(propName)) {
       continue
     }
     const propValue = module[propName]
-    if (typeof propValue === 'function') {
+    if (typeof propValue !== 'function') {
+      continue
+    }
+    if (multiResult[propName] != null) {
+      promises[propName] = promisifyMultiResult(
+        propValue,
+        multiResult[propName]
+      ).bind(module)
+    } else {
       promises[propName] = promisify(propValue).bind(module)
     }
   }
   return promises
 }
 
+/**
+ * Promisify a function that returns multiple results via additional callback
+ * parameters.
+ *
+ * The promisified function returns the results in a single object whose keys
+ * are the names given in the array `resultNames`.
+ *
+ * Example:
+ *
+ *     function f(callback) {
+ *       return callback(null, 1, 2, 3)
+ *     }
+ *
+ *     const g = promisifyMultiResult(f, ['a', 'b', 'c'])
+ *
+ *     const result = await g()  // returns {a: 1, b: 2, c: 3}
+ */
+function promisifyMultiResult(fn, resultNames) {
+  function promisified(...args) {
+    return new Promise((resolve, reject) => {
+      try {
+        fn(...args, (err, ...results) => {
+          if (err != null) {
+            return reject(err)
+          }
+          const promiseResult = {}
+          for (let i = 0; i < resultNames.length; i++) {
+            promiseResult[resultNames[i]] = results[i]
+          }
+          resolve(promiseResult)
+        })
+      } catch (err) {
+        reject(err)
+      }
+    })
+  }
+  return promisified
+}
+
+/**
+ * Reverse the effect of `promisifyMultiResult`.
+ *
+ * This is meant for providing a temporary backward compatible callback
+ * interface while we migrate to promises.
+ */
+function callbackifyMultiResult(fn, resultNames) {
+  function callbackified(...args) {
+    const [callback] = args.splice(-1)
+    fn(...args)
+      .then(result => {
+        const cbResults = resultNames.map(resultName => result[resultName])
+        callback(null, ...cbResults)
+      })
+      .catch(err => {
+        callback(err)
+      })
+  }
+  return callbackified
+}
+
 /**
  * Transform an async function into an Express middleware
  *

services/web/test/unit/src/Project/ProjectEntityMongoUpdateHandlerCanaryTests.js

@@ -0,0 +1,1034 @@
+const { expect } = require('chai')
+const sinon = require('sinon')
+const tk = require('timekeeper')
+const Errors = require('../../../../app/src/Features/Errors/Errors')
+const { ObjectId } = require('mongoose').Types
+const SandboxedModule = require('sandboxed-module')
+const { Project } = require('../helpers/models/Project')
+
+const MODULE_PATH =
+  '../../../../app/src/Features/Project/ProjectEntityMongoUpdateHandlerCanary'
+
+describe('ProjectEntityMongoUpdateHandler', function() {
+  beforeEach(function() {
+    this.doc = {
+      _id: ObjectId(),
+      name: 'test-doc.txt',
+      lines: ['hello', 'world'],
+      rev: 1234
+    }
+    this.docPath = {
+      mongo: 'rootFolder.0.docs.0',
+      fileSystem: '/test-doc.txt'
+    }
+    this.file = {
+      _id: ObjectId(),
+      name: 'something.jpg',
+      linkedFileData: { provider: 'url' },
+      hash: 'some-hash'
+    }
+    this.filePath = {
+      fileSystem: '/something.png',
+      mongo: 'rootFolder.0.fileRefs.0'
+    }
+    this.subfolder = { _id: ObjectId(), name: 'test-subfolder' }
+    this.subfolderPath = {
+      fileSystem: '/test-folder/test-subfolder',
+      mongo: 'rootFolder.0.folders.0.folders.0'
+    }
+    this.folder = {
+      _id: ObjectId(),
+      name: 'test-folder',
+      folders: [this.subfolder]
+    }
+    this.folderPath = {
+      fileSystem: '/test-folder',
+      mongo: 'rootFolder.0.folders.0'
+    }
+    this.rootFolder = {
+      _id: ObjectId(),
+      folders: [this.folder],
+      docs: [this.doc],
+      fileRefs: [this.file]
+    }
+    this.rootFolderPath = {
+      fileSystem: '/',
+      mongo: 'rootFolder.0'
+    }
+    this.project = {
+      _id: ObjectId(),
+      name: 'project name',
+      rootFolder: [this.rootFolder]
+    }
+
+    this.logger = {
+      log: sinon.stub(),
+      warn: sinon.stub(),
+      error: sinon.stub(),
+      err() {}
+    }
+    this.Settings = { maxEntitiesPerProject: 100 }
+    this.CooldownManager = {}
+    this.LockManager = {
+      promises: {
+        runWithLock: sinon.spy((namespace, id, runner) => runner())
+      }
+    }
+
+    this.FolderModel = sinon.stub()
+    this.ProjectMock = sinon.mock(Project)
+    this.ProjectEntityHandler = {
+      promises: {
+        getAllEntitiesFromProject: sinon.stub()
+      }
+    }
+    this.ProjectLocator = {
+      promises: {
+        findElement: sinon.stub().rejects(new Error('not found')),
+        findElementByPath: sinon.stub().rejects(new Error('not found'))
+      }
+    }
+    this.ProjectLocator.promises.findElement
+      .withArgs({
+        project: this.project,
+        element_id: this.rootFolder._id,
+        type: 'folder'
+      })
+      .resolves({
+        element: this.rootFolder,
+        path: this.rootFolderPath
+      })
+    this.ProjectLocator.promises.findElement
+      .withArgs({
+        project: this.project,
+        element_id: this.folder._id,
+        type: 'folder'
+      })
+      .resolves({
+        element: this.folder,
+        path: this.folderPath,
+        folder: this.rootFolder
+      })
+    this.ProjectLocator.promises.findElement
+      .withArgs({
+        project: this.project,
+        element_id: this.subfolder._id,
+        type: 'folder'
+      })
+      .resolves({
+        element: this.subfolder,
+        path: this.subfolderPath,
+        folder: this.folder
+      })
+    this.ProjectLocator.promises.findElement
+      .withArgs({
+        project: this.project,
+        element_id: this.file._id,
+        type: 'file'
+      })
+      .resolves({
+        element: this.file,
+        path: this.filePath,
+        folder: this.rootFolder
+      })
+    this.ProjectLocator.promises.findElement
+      .withArgs({
+        project: this.project,
+        element_id: this.doc._id,
+        type: 'doc'
+      })
+      .resolves({
+        element: this.doc,
+        path: this.docPath,
+        folder: this.rootFolder
+      })
+    this.ProjectLocator.promises.findElementByPath
+      .withArgs(
+        sinon.match({
+          project: this.project,
+          path: '/'
+        })
+      )
+      .resolves({ element: this.rootFolder, type: 'folder' })
+    this.ProjectLocator.promises.findElementByPath
+      .withArgs(
+        sinon.match({
+          project: this.project,
+          path: '/test-folder'
+        })
+      )
+      .resolves({ element: this.folder, type: 'folder' })
+    this.ProjectLocator.promises.findElementByPath
+      .withArgs(
+        sinon.match({
+          project: this.project,
+          path: '/test-folder/test-subfolder'
+        })
+      )
+      .resolves({ element: this.subfolder, type: 'folder' })
+
+    this.ProjectGetter = {
+      promises: {
+        getProjectWithoutLock: sinon
+          .stub()
+          .withArgs(this.project._id)
+          .resolves(this.project),
+        getProjectWithOnlyFolders: sinon.stub().resolves(this.project)
+      }
+    }
+
+    this.subject = SandboxedModule.require(MODULE_PATH, {
+      globals: {
+        console: console
+      },
+      requires: {
+        'logger-sharelatex': this.logger,
+        'settings-sharelatex': this.Settings,
+        '../Cooldown/CooldownManager': this.CooldownManager,
+        '../../models/Folder': { Folder: this.FolderModel },
+        '../../infrastructure/LockManager': this.LockManager,
+        '../../models/Project': { Project },
+        './ProjectEntityHandler': this.ProjectEntityHandler,
+        './ProjectLocator': this.ProjectLocator,
+        './ProjectGetter': this.ProjectGetter,
+        // We need to provide Errors here to make instance check work
+        '../Errors/Errors': Errors
+      }
+    })
+  })
+
+  afterEach(function() {
+    this.ProjectMock.restore()
+  })
+
+  beforeEach(function() {
+    tk.freeze(Date.now())
+  })
+
+  afterEach(function() {
+    tk.reset()
+  })
+
+  describe('addDoc', function() {
+    beforeEach(async function() {
+      const doc = { _id: ObjectId(), name: 'other.txt' }
+      this.ProjectMock.expects('findOneAndUpdate')
+        .withArgs(
+          { _id: this.project._id },
+          {
+            $push: { 'rootFolder.0.folders.0.docs': doc },
+            $inc: { version: 1 }
+          }
+        )
+        .chain('exec')
+        .resolves(this.project)
+      this.result = await this.subject.promises.addDoc(
+        this.project._id,
+        this.folder._id,
+        doc
+      )
+    })
+
+    it('adds the document in Mongo', function() {
+      this.ProjectMock.verify()
+    })
+
+    it('returns path info and the project', function() {
+      expect(this.result).to.deep.equal({
+        result: {
+          path: {
+            mongo: 'rootFolder.0.folders.0',
+            fileSystem: '/test-folder/other.txt'
+          }
+        },
+        project: this.project
+      })
+    })
+  })
+
+  describe('addFile', function() {
+    beforeEach(function() {
+      this.newFile = { _id: ObjectId(), name: 'picture.jpg' }
+      this.ProjectMock.expects('findOneAndUpdate')
+        .withArgs(
+          { _id: this.project._id },
+          {
+            $push: { 'rootFolder.0.folders.0.fileRefs': this.newFile },
+            $inc: { version: 1 }
+          }
+        )
+        .chain('exec')
+        .resolves(this.project)
+    })
+
+    describe('happy path', function() {
+      beforeEach(async function() {
+        this.result = await this.subject.promises.addFile(
+          this.project._id,
+          this.folder._id,
+          this.newFile
+        )
+      })
+
+      it('adds the file in Mongo', function() {
+        this.ProjectMock.verify()
+      })
+
+      it('returns path info and the project', function() {
+        expect(this.result).to.deep.equal({
+          result: {
+            path: {
+              mongo: 'rootFolder.0.folders.0',
+              fileSystem: '/test-folder/picture.jpg'
+            }
+          },
+          project: this.project
+        })
+      })
+    })
+
+    describe('when entity limit is reached', function() {
+      beforeEach(function() {
+        this.savedMaxEntities = this.Settings.maxEntitiesPerProject
+        this.Settings.maxEntitiesPerProject = 3
+      })
+
+      afterEach(function() {
+        this.Settings.maxEntitiesPerProject = this.savedMaxEntities
+      })
+
+      it('should throw an error', async function() {
+        await expect(
+          this.subject.promises.addFile(
+            this.project._id,
+            this.folder._id,
+            this.newFile
+          )
+        ).to.be.rejected
+      })
+    })
+  })
+
+  describe('addFolder', function() {
+    beforeEach(async function() {
+      const folderName = 'New folder'
+      this.FolderModel.withArgs({ name: folderName }).returns({
+        _id: ObjectId(),
+        name: folderName
+      })
+      this.ProjectMock.expects('findOneAndUpdate')
+        .withArgs(
+          { _id: this.project._id },
+          {
+            $push: {
+              'rootFolder.0.folders.0.folders': sinon.match({
+                name: folderName
+              })
+            },
+            $inc: { version: 1 }
+          }
+        )
+        .chain('exec')
+        .resolves(this.project)
+      await this.subject.promises.addFolder(
+        this.project._id,
+        this.folder._id,
+        folderName
+      )
+    })
+
+    it('adds the folder in Mongo', function() {
+      this.ProjectMock.verify()
+    })
+  })
+
+  describe('replaceFileWithNew', function() {
+    beforeEach(async function() {
+      const newFile = {
+        _id: ObjectId(),
+        name: 'some-other-file.png',
+        linkedFileData: { some: 'data' },
+        hash: 'some-hash'
+      }
+      // Add a deleted file record
+      this.ProjectMock.expects('updateOne')
+        .withArgs(
+          { _id: this.project._id },
+          {
+            $push: {
+              deletedFiles: {
+                _id: this.file._id,
+                name: this.file.name,
+                linkedFileData: this.file.linkedFileData,
+                hash: this.file.hash,
+                deletedAt: sinon.match.date
+              }
+            }
+          }
+        )
+        .chain('exec')
+        .resolves()
+      // Update the file in place
+      this.ProjectMock.expects('findOneAndUpdate')
+        .withArgs(
+          { _id: this.project._id },
+          {
+            $set: {
+              'rootFolder.0.fileRefs.0._id': newFile._id,
+              'rootFolder.0.fileRefs.0.created': sinon.match.date,
+              'rootFolder.0.fileRefs.0.linkedFileData': newFile.linkedFileData,
+              'rootFolder.0.fileRefs.0.hash': newFile.hash
+            },
+            $inc: {
+              version: 1,
+              'rootFolder.0.fileRefs.0.rev': 1
+            }
+          }
+        )
+        .chain('exec')
+        .resolves(this.project)
+      await this.subject.promises.replaceFileWithNew(
+        this.project._id,
+        this.file._id,
+        newFile
+      )
+    })
+
+    it('updates the database', function() {
+      this.ProjectMock.verify()
+    })
+  })
+
+  describe('mkdirp', function() {
+    describe('when the path is just a slash', function() {
+      beforeEach(async function() {
+        this.result = await this.subject.promises.mkdirp(this.project._id, '/')
+      })
+
+      it('should return the root folder', function() {
+        expect(this.result.folder).to.deep.equal(this.rootFolder)
+      })
+
+      it('should not report a parent folder', function() {
+        expect(this.result.folder.parentFolder_id).not.to.exist
+      })
+
+      it('should not return new folders', function() {
+        expect(this.result.newFolders).to.have.length(0)
+      })
+    })
+
+    describe('when the folder already exists', function() {
+      beforeEach(async function() {
+        this.result = await this.subject.promises.mkdirp(
+          this.project._id,
+          '/test-folder'
+        )
+      })
+
+      it('should return the existing folder', function() {
+        expect(this.result.folder).to.deep.equal(this.folder)
+      })
+
+      it('should report the parent folder', function() {
+        expect(this.result.folder.parentFolder_id).not.equal(
+          this.rootFolder._id
+        )
+      })
+
+      it('should not return new folders', function() {
+        expect(this.result.newFolders).to.have.length(0)
+      })
+    })
+
+    describe('when the path is a new folder at the top level', function() {
+      beforeEach(async function() {
+        this.newFolder = { _id: ObjectId(), name: 'new-folder' }
+        this.FolderModel.returns(this.newFolder)
+        this.exactCaseMatch = false
+        this.ProjectMock.expects('findOneAndUpdate')
+          .withArgs(
+            { _id: this.project._id },
+            {
+              $push: { 'rootFolder.0.folders': this.newFolder },
+              $inc: { version: 1 }
+            }
+          )
+          .chain('exec')
+          .resolves(this.project)
+        this.result = await this.subject.promises.mkdirp(
+          this.project._id,
+          '/new-folder/',
+          { exactCaseMatch: this.exactCaseMatch }
+        )
+      })
+      it('should update the database', function() {
+        this.ProjectMock.verify()
+      })
+
+      it('should make just one folder', function() {
+        expect(this.result.newFolders).to.have.length(1)
+      })
+
+      it('should return the new folder', function() {
+        expect(this.result.folder.name).to.equal('new-folder')
+      })
+
+      it('should return the parent folder', function() {
+        expect(this.result.folder.parentFolder_id).to.equal(this.rootFolder._id)
+      })
+
+      it('should pass the exactCaseMatch option to ProjectLocator', function() {
+        expect(
+          this.ProjectLocator.promises.findElementByPath
+        ).to.have.been.calledWithMatch({ exactCaseMatch: this.exactCaseMatch })
+      })
+    })
+
+    describe('adding a subfolder', function() {
+      beforeEach(async function() {
+        this.newFolder = { _id: ObjectId(), name: 'new-folder' }
+        this.FolderModel.returns(this.newFolder)
+        this.ProjectMock.expects('findOneAndUpdate')
+          .withArgs(
+            { _id: this.project._id },
+            {
+              $push: {
+                'rootFolder.0.folders.0.folders': sinon.match({
+                  name: 'new-folder'
+                })
+              },
+              $inc: { version: 1 }
+            }
+          )
+          .chain('exec')
+          .resolves(this.project)
+        this.result = await this.subject.promises.mkdirp(
+          this.project._id,
+          '/test-folder/new-folder'
+        )
+      })
+
+      it('should update the database', function() {
+        this.ProjectMock.verify()
+      })
+
+      it('should create one folder', function() {
+        expect(this.result.newFolders).to.have.length(1)
+      })
+
+      it('should return the new folder', function() {
+        expect(this.result.folder.name).to.equal('new-folder')
+      })
+
+      it('should return the parent folder', function() {
+        expect(this.result.folder.parentFolder_id).to.equal(this.folder._id)
+      })
+    })
+
+    describe('when mutliple folders are missing', async function() {
+      beforeEach(function() {
+        this.folder1 = { _id: ObjectId(), name: 'folder1' }
+        this.folder1Path = {
+          fileSystem: '/test-folder/folder1',
+          mongo: 'rootFolder.0.folders.0.folders.0'
+        }
+        this.folder2 = { _id: ObjectId(), name: 'folder2' }
+        this.folder2Path = {
+          fileSystem: '/test-folder/folder1/folder2',
+          mongo: 'rootFolder.0.folders.0.folders.0.folders.0'
+        }
+        this.FolderModel.onFirstCall().returns(this.folder1)
+        this.FolderModel.onSecondCall().returns(this.folder2)
+        this.ProjectLocator.promises.findElement
+          .withArgs({
+            project: this.project,
+            element_id: this.folder1._id,
+            type: 'folder'
+          })
+          .resolves({
+            element: this.folder1,
+            path: this.folder1Path
+          })
+        this.ProjectLocator.promises.findElement
+          .withArgs({
+            project: this.project,
+            element_id: this.folder2._id,
+            type: 'folder'
+          })
+          .resolves({
+            element: this.folder2,
+            path: this.folder2Path
+          })
+        this.ProjectMock.expects('findOneAndUpdate')
+          .withArgs(
+            { _id: this.project._id },
+            {
+              $push: {
+                'rootFolder.0.folders.0.folders': sinon.match({
+                  name: 'folder1'
+                })
+              },
+              $inc: { version: 1 }
+            }
+          )
+          .chain('exec')
+          .resolves(this.project)
+        this.ProjectMock.expects('findOneAndUpdate')
+          .withArgs(
+            { _id: this.project._id },
+            {
+              $push: {
+                'rootFolder.0.folders.0.folders.0.folders': sinon.match({
+                  name: 'folder2'
+                })
+              },
+              $inc: { version: 1 }
+            }
+          )
+          .chain('exec')
+          .resolves(this.project)
+      })
+      ;[
+        {
+          description: 'without a trailing slash',
+          path: '/test-folder/folder1/folder2'
+        },
+        {
+          description: 'with a trailing slash',
+          path: '/test-folder/folder1/folder2/'
+        }
+      ].forEach(({ description, path }) => {
+        describe(description, function() {
+          beforeEach(async function() {
+            this.result = await this.subject.promises.mkdirp(
+              this.project._id,
+              path
+            )
+          })
+
+          it('should update the database', function() {
+            this.ProjectMock.verify()
+          })
+
+          it('should add multiple folders', function() {
+            const newFolders = this.result.newFolders
+            expect(newFolders).to.have.length(2)
+            expect(newFolders[0].name).to.equal('folder1')
+            expect(newFolders[1].name).to.equal('folder2')
+          })
+
+          it('should return the last folder', function() {
+            expect(this.result.folder.name).to.equal('folder2')
+          })
+
+          it('should return the parent folder', function() {
+            expect(this.result.folder.parentFolder_id).to.equal(
+              this.folder1._id
+            )
+          })
+        })
+      })
+    })
+  })
+
+  describe('moveEntity', function() {
+    describe('moving a doc into a different folder', function() {
+      beforeEach(async function() {
+        this.pathAfterMove = {
+          fileSystem: '/somewhere/else.txt'
+        }
+        this.oldDocs = ['old-doc']
+        this.oldFiles = ['old-file']
+        this.newDocs = ['new-doc']
+        this.newFiles = ['new-file']
+
+        this.ProjectEntityHandler.promises.getAllEntitiesFromProject
+          .onFirstCall()
+          .resolves({ docs: this.oldDocs, files: this.oldFiles })
+        this.ProjectEntityHandler.promises.getAllEntitiesFromProject
+          .onSecondCall()
+          .resolves({ docs: this.newDocs, files: this.newFiles })
+
+        this.ProjectMock.expects('findOneAndUpdate')
+          .withArgs(
+            { _id: this.project._id },
+            {
+              $push: { 'rootFolder.0.folders.0.docs': this.doc },
+              $inc: { version: 1 }
+            }
+          )
+          .chain('exec')
+          .resolves(this.project)
+        this.ProjectMock.expects('findOneAndUpdate')
+          .withArgs(
+            { _id: this.project._id },
+            {
+              $pull: { 'rootFolder.0.docs': { _id: this.doc._id } },
+              $inc: { version: 1 }
+            }
+          )
+          .chain('exec')
+          .resolves(this.project)
+        this.result = await this.subject.promises.moveEntity(
+          this.project._id,
+          this.doc._id,
+          this.folder._id,
+          'doc'
+        )
+      })
+
+      it('should update the database', function() {
+        this.ProjectMock.verify()
+      })
+
+      it('should report what changed', function() {
+        expect(this.result).to.deep.equal({
+          project: this.project,
+          startPath: '/test-doc.txt',
+          endPath: '/test-folder/test-doc.txt',
+          rev: this.doc.rev,
+          changes: {
+            oldDocs: this.oldDocs,
+            newDocs: this.newDocs,
+            oldFiles: this.oldFiles,
+            newFiles: this.newFiles,
+            newProject: this.project
+          }
+        })
+      })
+    })
+
+    describe('when moving a folder inside itself', function() {
+      it('throws an error', async function() {
+        await expect(
+          this.subject.promises.moveEntity(
+            this.project._id,
+            this.folder._id,
+            this.folder._id,
+            'folder'
+          )
+        ).to.be.rejectedWith(Errors.InvalidNameError)
+      })
+    })
+
+    describe('when moving a folder to a subfolder of itself', function() {
+      it('throws an error', async function() {
+        await expect(
+          this.subject.promises.moveEntity(
+            this.project._id,
+            this.folder._id,
+            this.subfolder._id,
+            'folder'
+          )
+        ).to.be.rejectedWith(Errors.InvalidNameError)
+      })
+    })
+  })
+
+  describe('deleteEntity', function() {
+    beforeEach(async function() {
+      this.ProjectMock.expects('findOneAndUpdate')
+        .withArgs(
+          { _id: this.project._id },
+          {
+            $pull: { 'rootFolder.0.docs': { _id: this.doc._id } },
+            $inc: { version: 1 }
+          }
+        )
+        .chain('exec')
+        .resolves(this.project)
+      await this.subject.promises.deleteEntity(
+        this.project._id,
+        this.doc._id,
+        'doc'
+      )
+    })
+
+    it('should update the database', function() {
+      this.ProjectMock.verify()
+    })
+  })
+
+  describe('renameEntity', function() {
+    describe('happy path', function() {
+      beforeEach(async function() {
+        this.newName = 'new.tex'
+        this.oldDocs = ['old-doc']
+        this.oldFiles = ['old-file']
+        this.newDocs = ['new-doc']
+        this.newFiles = ['new-file']
+
+        this.ProjectEntityHandler.promises.getAllEntitiesFromProject
+          .onFirstCall()
+          .resolves({ docs: this.oldDocs, files: this.oldFiles })
+        this.ProjectEntityHandler.promises.getAllEntitiesFromProject
+          .onSecondCall()
+          .resolves({ docs: this.newDocs, files: this.newFiles })
+
+        this.ProjectMock.expects('findOneAndUpdate')
+          .withArgs(
+            { _id: this.project._id },
+            {
+              $set: { 'rootFolder.0.docs.0.name': this.newName },
+              $inc: { version: 1 }
+            }
+          )
+          .chain('exec')
+          .resolves(this.project)
+        this.result = await this.subject.promises.renameEntity(
+          this.project._id,
+          this.doc._id,
+          'doc',
+          this.newName
+        )
+      })
+
+      it('should update the database', function() {
+        this.ProjectMock.verify()
+      })
+
+      it('returns info', function() {
+        expect(this.result).to.deep.equal({
+          project: this.project,
+          startPath: '/test-doc.txt',
+          endPath: '/new.tex',
+          rev: this.doc.rev,
+          changes: {
+            oldDocs: this.oldDocs,
+            newDocs: this.newDocs,
+            oldFiles: this.oldFiles,
+            newFiles: this.newFiles,
+            newProject: this.project
+          }
+        })
+      })
+    })
+
+    describe('name already exists', function() {
+      it('should throw an error', async function() {
+        await expect(
+          this.subject.promises.renameEntity(
+            this.project._id,
+            this.doc._id,
+            'doc',
+            this.folder.name
+          )
+        ).to.be.rejectedWith(Errors.InvalidNameError)
+      })
+    })
+  })
+
+  describe('_putElement', function() {
+    describe('updating the project', function() {
+      describe('when the parent folder is given', function() {
+        beforeEach(function() {
+          this.newFile = { _id: ObjectId(), name: 'new file.png' }
+          this.ProjectMock.expects('findOneAndUpdate')
+            .withArgs(
+              { _id: this.project._id },
+              {
+                $push: { 'rootFolder.0.folders.0.fileRefs': this.newFile },
+                $inc: { version: 1 }
+              }
+            )
+            .chain('exec')
+            .resolves(this.project)
+        })
+
+        it('should update the database', async function() {
+          await this.subject.promises._putElement(
+            this.project,
+            this.folder._id,
+            this.newFile,
+            'files'
+          )
+          this.ProjectMock.verify()
+        })
+
+        it('should add an s onto the type if not included', async function() {
+          await this.subject.promises._putElement(
+            this.project,
+            this.folder._id,
+            this.newFile,
+            'file'
+          )
+          this.ProjectMock.verify()
+        })
+      })
+
+      describe('error cases', function() {
+        it('should throw an error if element is null', async function() {
+          await expect(
+            this.subject.promises._putElement(
+              this.project,
+              this.folder._id,
+              null,
+              'file'
+            )
+          ).to.be.rejected
+        })
+
+        it('should error if the element has no _id', async function() {
+          const file = { name: 'something' }
+          await expect(
+            this.subject.promises._putElement(
+              this.project,
+              this.folder._id,
+              file,
+              'file'
+            )
+          ).to.be.rejected
+        })
+
+        it('should error if element name contains invalid characters', async function() {
+          const file = { _id: ObjectId(), name: 'something*bad' }
+          await expect(
+            this.subject.promises._putElement(
+              this.project,
+              this.folder._id,
+              file,
+              'file'
+            )
+          ).to.be.rejected
+        })
+
+        it('should error if element name is too long', async function() {
+          const file = {
+            _id: ObjectId(),
+            name: 'long-'.repeat(1000) + 'something'
+          }
+          await expect(
+            this.subject.promises._putElement(
+              this.project,
+              this.folder._id,
+              file,
+              'file'
+            )
+          ).to.be.rejectedWith(Errors.InvalidNameError)
+        })
+
+        it('should error if the folder name is too long', async function() {
+          const file = {
+            _id: ObjectId(),
+            name: 'something'
+          }
+          this.ProjectLocator.promises.findElement
+            .withArgs({
+              project: this.project,
+              element_id: this.folder._id,
+              type: 'folder'
+            })
+            .resolves({
+              element: this.folder,
+              path: { fileSystem: 'subdir/'.repeat(1000) + 'foo' }
+            })
+          await expect(
+            this.subject.promises._putElement(
+              this.project,
+              this.folder._id,
+              file,
+              'file'
+            )
+          ).to.be.rejectedWith(Errors.InvalidNameError)
+        })
+        ;['file', 'doc', 'folder'].forEach(entityType => {
+          it(`should error if a ${entityType} already exists with the same name`, async function() {
+            const file = {
+              _id: ObjectId(),
+              name: this[entityType].name
+            }
+            await expect(
+              this.subject.promises._putElement(
+                this.project,
+                null,
+                file,
+                'file'
+              )
+            ).to.be.rejectedWith(Errors.InvalidNameError)
+          })
+        })
+      })
+    })
+
+    describe('when the parent folder is not given', function() {
+      it('should default to root folder insert', async function() {
+        this.newFile = { _id: ObjectId(), name: 'new file.png' }
+        this.ProjectMock.expects('findOneAndUpdate')
+          .withArgs(
+            { _id: this.project._id },
+            {
+              $push: { 'rootFolder.0.fileRefs': this.newFile },
+              $inc: { version: 1 }
+            }
+          )
+          .chain('exec')
+          .resolves(this.project)
+        await this.subject.promises._putElement(
+          this.project,
+          this.rootFolder._id,
+          this.newFile,
+          'file'
+        )
+      })
+    })
+  })
+
+  describe('_insertDeletedDocReference', function() {
+    beforeEach(async function() {
+      this.ProjectMock.expects('updateOne')
+        .withArgs(
+          { _id: this.project._id },
+          {
+            $push: {
+              deletedDocs: {
+                _id: this.doc._id,
+                name: this.doc.name,
+                deletedAt: sinon.match.date
+              }
+            }
+          }
+        )
+        .chain('exec')
+        .resolves()
+      await this.subject.promises._insertDeletedDocReference(
+        this.project._id,
+        this.doc
+      )
+    })
+
+    it('should update the database', function() {
+      this.ProjectMock.verify()
+    })
+  })
+
+  describe('_insertDeletedFileReference', function() {
+    beforeEach(async function() {
+      this.ProjectMock.expects('updateOne')
+        .withArgs(
+          { _id: this.project._id },
+          {
+            $push: {
+              deletedFiles: {
+                _id: this.file._id,
+                name: this.file.name,
+                linkedFileData: this.file.linkedFileData,
+                hash: this.file.hash,
+                deletedAt: sinon.match.date
+              }
+            }
+          }
+        )
+        .chain('exec')
+        .resolves()
+      await this.subject.promises._insertDeletedFileReference(
+        this.project._id,
+        this.file
+      )
+    })
+
+    it('should update the database', function() {
+      this.ProjectMock.verify()
+    })
+  })
+})

services/web/test/unit/src/util/promisesTests.js

@@ -1,5 +1,8 @@
 const { expect } = require('chai')
-const { promisifyAll } = require('../../../../app/src/util/promises')
+const {
+  promisifyAll,
+  callbackifyMultiResult
+} = require('../../../../app/src/util/promises')
 
 describe('promisifyAll', function() {
   describe('basic functionality', function() {
@@ -57,4 +60,64 @@ describe('promisifyAll', function() {
       expect(sum).to.equal(101)
     })
   })
+
+  describe('multiResult option', function() {
+    before(function() {
+      this.module = {
+        asyncAdd(a, b, callback) {
+          callback(null, a + b)
+        },
+        asyncArithmetic(a, b, callback) {
+          callback(null, a + b, a * b)
+        }
+      }
+      this.promisified = promisifyAll(this.module, {
+        multiResult: { asyncArithmetic: ['sum', 'product'] }
+      })
+    })
+
+    it('promisifies multi-result functions', async function() {
+      const result = await this.promisified.asyncArithmetic(3, 6)
+      expect(result).to.deep.equal({ sum: 9, product: 18 })
+    })
+
+    it('promisifies other functions normally', async function() {
+      const sum = await this.promisified.asyncAdd(6, 1)
+      expect(sum).to.equal(7)
+    })
+  })
+})
+
+describe('callbackifyMultiResult', function() {
+  it('callbackifies a multi-result function', function(done) {
+    async function asyncArithmetic(a, b) {
+      return { sum: a + b, product: a * b }
+    }
+    const callbackified = callbackifyMultiResult(asyncArithmetic, [
+      'sum',
+      'product'
+    ])
+    callbackified(3, 11, (err, sum, product) => {
+      if (err != null) {
+        return done(err)
+      }
+      expect(sum).to.equal(14)
+      expect(product).to.equal(33)
+      done()
+    })
+  })
+
+  it('propagates errors', function(done) {
+    async function asyncBomb() {
+      throw new Error('BOOM!')
+    }
+    const callbackified = callbackifyMultiResult(asyncBomb, [
+      'explosives',
+      'dynamite'
+    ])
+    callbackified(err => {
+      expect(err).to.exist
+      done()
+    })
+  })
 })