Merge pull request #2361 from overleaf/em-project-imports-2

Import full folder structure in a single Mongo update

GitOrigin-RevId: 623d2a098b2084fdd0193e1593c1c55c08a2d92d

services/web/app/src/Features/Project/FolderStructureBuilder.js

@@ -0,0 +1,73 @@
+const Path = require('path')
+const OError = require('@overleaf/o-error')
+const { ObjectId } = require('mongodb')
+
+module.exports = { buildFolderStructure }
+
+function buildFolderStructure(docUploads, fileUploads) {
+  const builder = new FolderStructureBuilder()
+  for (const docUpload of docUploads) {
+    builder.addDocUpload(docUpload)
+  }
+  for (const fileUpload of fileUploads) {
+    builder.addFileUpload(fileUpload)
+  }
+  return builder.rootFolder
+}
+
+class FolderStructureBuilder {
+  constructor() {
+    this.foldersByPath = new Map()
+    this.entityPaths = new Set()
+    this.rootFolder = this.createFolder('rootFolder')
+    this.foldersByPath.set('/', this.rootFolder)
+    this.entityPaths.add('/')
+  }
+
+  addDocUpload(docUpload) {
+    this.recordEntityPath(Path.join(docUpload.dirname, docUpload.doc.name))
+    const folder = this.mkdirp(docUpload.dirname)
+    folder.docs.push(docUpload.doc)
+  }
+
+  addFileUpload(fileUpload) {
+    this.recordEntityPath(
+      Path.join(fileUpload.dirname, fileUpload.fileRef.name)
+    )
+    const folder = this.mkdirp(fileUpload.dirname)
+    folder.fileRefs.push(fileUpload.fileRef)
+  }
+
+  mkdirp(path) {
+    const existingFolder = this.foldersByPath.get(path)
+    if (existingFolder != null) {
+      return existingFolder
+    }
+    // Folder not found, create it.
+    this.recordEntityPath(path)
+    const dirname = Path.dirname(path)
+    const basename = Path.basename(path)
+    const parentFolder = this.mkdirp(dirname)
+    const newFolder = this.createFolder(basename)
+    parentFolder.folders.push(newFolder)
+    this.foldersByPath.set(path, newFolder)
+    return newFolder
+  }
+
+  recordEntityPath(path) {
+    if (this.entityPaths.has(path)) {
+      throw new OError({ message: 'entity already exists', info: { path } })
+    }
+    this.entityPaths.add(path)
+  }
+
+  createFolder(name) {
+    return {
+      _id: ObjectId(),
+      name,
+      folders: [],
+      docs: [],
+      fileRefs: []
+    }
+  }
+}

services/web/app/src/Features/Project/ProjectDuplicator.js

@@ -35,7 +35,7 @@ module.exports = ProjectDuplicator = {
     callback
   ) {
     const setRootDoc = _.once(doc_id =>
-      ProjectEntityUpdateHandler.setRootDoc(newProject._id, doc_id)
+      ProjectEntityUpdateHandler.setRootDoc(newProject._id, doc_id, () => {})
     )
     const docs = originalFolder.docs || []
     const jobs = docs.map(

services/web/app/src/Features/Project/ProjectEntityMongoUpdateHandlerCanary.js

@@ -10,6 +10,7 @@ const logger = require('logger-sharelatex')
 const path = require('path')
 const { ObjectId } = require('mongodb')
 const Settings = require('settings-sharelatex')
+const OError = require('@overleaf/o-error')
 const CooldownManager = require('../Cooldown/CooldownManager')
 const Errors = require('../Errors/Errors')
 const { Folder } = require('../../models/Folder')
@@ -18,6 +19,7 @@ const { Project } = require('../../models/Project')
 const ProjectEntityHandler = require('./ProjectEntityHandler')
 const ProjectGetter = require('./ProjectGetter')
 const ProjectLocator = require('./ProjectLocator')
+const FolderStructureBuilder = require('./FolderStructureBuilder')
 const SafePath = require('./SafePath')
 
 const LOCK_NAMESPACE = 'mongoTransaction'
@@ -69,6 +71,7 @@ module.exports = {
     'rev',
     'changes'
   ]),
+  createNewFolderStructure: callbackify(wrapWithLock(createNewFolderStructure)),
   _insertDeletedDocReference: callbackify(_insertDeletedDocReference),
   _insertDeletedFileReference: callbackify(_insertDeletedFileReference),
   _putElement: callbackifyMultiResult(_putElement, ['result', 'project']),
@@ -82,6 +85,7 @@ module.exports = {
     moveEntity: wrapWithLock(moveEntity),
     deleteEntity: wrapWithLock(deleteEntity),
     renameEntity: wrapWithLock(renameEntity),
+    createNewFolderStructure: wrapWithLock(createNewFolderStructure),
     _insertDeletedDocReference,
     _insertDeletedFileReference,
     _putElement
@@ -606,3 +610,35 @@ async function _checkValidMove(
     }
   }
 }
+
+async function createNewFolderStructure(projectId, docUploads, fileUploads) {
+  try {
+    const rootFolder = FolderStructureBuilder.buildFolderStructure(
+      docUploads,
+      fileUploads
+    )
+    const result = await Project.updateOne(
+      {
+        _id: projectId,
+        'rootFolder.0.folders.0': { $exists: false },
+        'rootFolder.0.docs.0': { $exists: false },
+        'rootFolder.0.files.0': { $exists: false }
+      },
+      {
+        $set: { rootFolder: [rootFolder] },
+        $inc: { version: 1 }
+      }
+    ).exec()
+    if (result.n !== 1) {
+      throw new OError({
+        message: 'project not found or folder structure already exists',
+        info: { projectId }
+      })
+    }
+  } catch (err) {
+    throw new OError({
+      message: 'failed to create folder structure',
+      info: { projectId }
+    }).withCause(err)
+  }
+}

services/web/app/src/Features/Project/ProjectEntityUpdateHandlerCanary.js

@@ -1,1634 +0,0 @@
-/* NOTE: this file is an almost exact copy of ProjectEntityUpdateHandler.js.
- * The only difference is that it imports
- * ProjectEntityMongoUpdateHandlerCanary.js. It's meant to be a short-lived
- * module, so that we can test the async/await code in production for some code
- * paths only.
- */
-
-/* eslint-disable
-    camelcase,
-    handle-callback-err,
-    max-len,
-    one-var,
-    standard/no-callback-literal,
-*/
-
-// TODO: This file was created by bulk-decaffeinate.
-// Fix any style issues and re-enable lint.
-/*
- * decaffeinate suggestions:
- * DS101: Remove unnecessary use of Array.from
- * DS102: Remove unnecessary code created because of implicit returns
- * DS103: Rewrite code to no longer use __guard__
- * DS201: Simplify complex destructure assignments
- * DS207: Consider shorter variations of null checks
- * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
- */
-let ProjectEntityUpdateHandler, self
-const _ = require('lodash')
-const async = require('async')
-const logger = require('logger-sharelatex')
-const Settings = require('settings-sharelatex')
-const path = require('path')
-const { Doc } = require('../../models/Doc')
-const DocstoreManager = require('../Docstore/DocstoreManager')
-const DocumentUpdaterHandler = require('../../Features/DocumentUpdater/DocumentUpdaterHandler')
-const Errors = require('../Errors/Errors')
-const { File } = require('../../models/File')
-const FileStoreHandler = require('../FileStore/FileStoreHandler')
-const LockManager = require('../../infrastructure/LockManager')
-const { Project } = require('../../models/Project')
-const ProjectEntityHandler = require('./ProjectEntityHandler')
-const ProjectGetter = require('./ProjectGetter')
-const ProjectLocator = require('./ProjectLocator')
-const ProjectUpdateHandler = require('./ProjectUpdateHandler')
-const ProjectEntityMongoUpdateHandler = require('./ProjectEntityMongoUpdateHandlerCanary')
-const SafePath = require('./SafePath')
-const TpdsUpdateSender = require('../ThirdPartyDataStore/TpdsUpdateSender')
-
-const LOCK_NAMESPACE = 'sequentialProjectStructureUpdateLock'
-
-const validRootDocExtensions = Settings.validRootDocExtensions
-const validRootDocRegExp = new RegExp(
-  `^\\.(${validRootDocExtensions.join('|')})$`,
-  'i'
-)
-
-const wrapWithLock = function(methodWithoutLock) {
-  // This lock is used to make sure that the project structure updates are made
-  // sequentially. In particular the updates must be made in mongo and sent to
-  // the doc-updater in the same order.
-  let methodWithLock
-  if (typeof methodWithoutLock === 'function') {
-    methodWithLock = function(project_id, ...rest) {
-      const adjustedLength = Math.max(rest.length, 1),
-        args = rest.slice(0, adjustedLength - 1),
-        callback = rest[adjustedLength - 1]
-      return LockManager.runWithLock(
-        LOCK_NAMESPACE,
-        project_id,
-        cb => methodWithoutLock(project_id, ...Array.from(args), cb),
-        callback
-      )
-    }
-    methodWithLock.withoutLock = methodWithoutLock
-    return methodWithLock
-  } else {
-    // handle case with separate setup and locked stages
-    const wrapWithSetup = methodWithoutLock.beforeLock // a function to set things up before the lock
-    const mainTask = methodWithoutLock.withLock // function to execute inside the lock
-    methodWithLock = wrapWithSetup(function(project_id, ...rest) {
-      const adjustedLength = Math.max(rest.length, 1),
-        args = rest.slice(0, adjustedLength - 1),
-        callback = rest[adjustedLength - 1]
-      return LockManager.runWithLock(
-        LOCK_NAMESPACE,
-        project_id,
-        cb => mainTask(project_id, ...Array.from(args), cb),
-        callback
-      )
-    })
-    methodWithLock.withoutLock = wrapWithSetup(mainTask)
-    methodWithLock.beforeLock = methodWithoutLock.beforeLock
-    methodWithLock.mainTask = methodWithoutLock.withLock
-    return methodWithLock
-  }
-}
-
-module.exports = ProjectEntityUpdateHandler = self = {
-  copyFileFromExistingProjectWithProject: wrapWithLock({
-    beforeLock(next) {
-      return function(
-        project_id,
-        project,
-        folder_id,
-        originalProject_id,
-        origonalFileRef,
-        userId,
-        callback
-      ) {
-        if (callback == null) {
-          callback = function(error, fileRef, folder_id) {}
-        }
-        logger.log(
-          { project_id, folder_id, originalProject_id, origonalFileRef },
-          'copying file in s3 with project'
-        )
-        folder_id = ProjectEntityMongoUpdateHandler._confirmFolder(
-          project,
-          folder_id
-        )
-        if (origonalFileRef == null) {
-          logger.err(
-            { project_id, folder_id, originalProject_id, origonalFileRef },
-            'file trying to copy is null'
-          )
-          return callback()
-        }
-        // convert any invalid characters in original file to '_'
-        const fileProperties = {
-          name: SafePath.clean(origonalFileRef.name)
-        }
-        if (origonalFileRef.linkedFileData != null) {
-          fileProperties.linkedFileData = origonalFileRef.linkedFileData
-        }
-        if (origonalFileRef.hash != null) {
-          fileProperties.hash = origonalFileRef.hash
-        }
-        const fileRef = new File(fileProperties)
-        return FileStoreHandler.copyFile(
-          originalProject_id,
-          origonalFileRef._id,
-          project._id,
-          fileRef._id,
-          function(err, fileStoreUrl) {
-            if (err != null) {
-              logger.warn(
-                {
-                  err,
-                  project_id,
-                  folder_id,
-                  originalProject_id,
-                  origonalFileRef
-                },
-                'error coping file in s3'
-              )
-              return callback(err)
-            }
-            return next(
-              project_id,
-              project,
-              folder_id,
-              originalProject_id,
-              origonalFileRef,
-              userId,
-              fileRef,
-              fileStoreUrl,
-              callback
-            )
-          }
-        )
-      }
-    },
-    withLock(
-      project_id,
-      project,
-      folder_id,
-      originalProject_id,
-      origonalFileRef,
-      userId,
-      fileRef,
-      fileStoreUrl,
-      callback
-    ) {
-      if (callback == null) {
-        callback = function(error, fileRef, folder_id) {}
-      }
-      const projectHistoryId = __guard__(
-        project.overleaf != null ? project.overleaf.history : undefined,
-        x => x.id
-      )
-      return ProjectEntityMongoUpdateHandler._putElement(
-        project,
-        folder_id,
-        fileRef,
-        'file',
-        function(err, result, newProject) {
-          if (err != null) {
-            logger.warn(
-              { err, project_id, folder_id },
-              'error putting element as part of copy'
-            )
-            return callback(err)
-          }
-          return TpdsUpdateSender.addFile(
-            {
-              project_id,
-              file_id: fileRef._id,
-              path: __guard__(
-                result != null ? result.path : undefined,
-                x1 => x1.fileSystem
-              ),
-              rev: fileRef.rev,
-              project_name: project.name
-            },
-            function(err) {
-              if (err != null) {
-                logger.err(
-                  {
-                    err,
-                    project_id,
-                    folder_id,
-                    originalProject_id,
-                    origonalFileRef
-                  },
-                  'error sending file to tpds worker'
-                )
-              }
-              const newFiles = [
-                {
-                  file: fileRef,
-                  path: __guard__(
-                    result != null ? result.path : undefined,
-                    x2 => x2.fileSystem
-                  ),
-                  url: fileStoreUrl
-                }
-              ]
-              return DocumentUpdaterHandler.updateProjectStructure(
-                project_id,
-                projectHistoryId,
-                userId,
-                { newFiles, newProject },
-                function(error) {
-                  if (error != null) {
-                    return callback(error)
-                  }
-                  return callback(null, fileRef, folder_id)
-                }
-              )
-            }
-          )
-        }
-      )
-    }
-  }),
-
-  updateDocLines(
-    project_id,
-    doc_id,
-    lines,
-    version,
-    ranges,
-    lastUpdatedAt,
-    lastUpdatedBy,
-    callback
-  ) {
-    if (callback == null) {
-      callback = function(error) {}
-    }
-    return ProjectGetter.getProjectWithoutDocLines(project_id, function(
-      err,
-      project
-    ) {
-      if (err != null) {
-        return callback(err)
-      }
-      if (project == null) {
-        return callback(new Errors.NotFoundError('project not found'))
-      }
-      logger.log({ project_id, doc_id }, 'updating doc lines')
-      return ProjectLocator.findElement(
-        { project, element_id: doc_id, type: 'docs' },
-        function(err, doc, path) {
-          let isDeletedDoc = false
-          if (err != null) {
-            if (err instanceof Errors.NotFoundError) {
-              // We need to be able to update the doclines of deleted docs. This is
-              // so the doc-updater can flush a doc's content to the doc-store after
-              // the doc is deleted.
-              isDeletedDoc = true
-              doc = _.find(
-                project.deletedDocs,
-                doc => doc._id.toString() === doc_id.toString()
-              )
-            } else {
-              return callback(err)
-            }
-          }
-
-          if (doc == null) {
-            // Do not allow an update to a doc which has never exist on this project
-            logger.warn(
-              { doc_id, project_id, lines },
-              'doc not found while updating doc lines'
-            )
-            return callback(new Errors.NotFoundError('doc not found'))
-          }
-
-          logger.log(
-            { project_id, doc_id },
-            'telling docstore manager to update doc'
-          )
-          return DocstoreManager.updateDoc(
-            project_id,
-            doc_id,
-            lines,
-            version,
-            ranges,
-            function(err, modified, rev) {
-              if (err != null) {
-                logger.warn(
-                  { err, doc_id, project_id, lines },
-                  'error sending doc to docstore'
-                )
-                return callback(err)
-              }
-              logger.log(
-                { project_id, doc_id, modified },
-                'finished updating doc lines'
-              )
-              // path will only be present if the doc is not deleted
-              if (modified && !isDeletedDoc) {
-                // Don't need to block for marking as updated
-                ProjectUpdateHandler.markAsUpdated(
-                  project_id,
-                  lastUpdatedAt,
-                  lastUpdatedBy
-                )
-                return TpdsUpdateSender.addDoc(
-                  {
-                    project_id,
-                    path: path.fileSystem,
-                    doc_id,
-                    project_name: project.name,
-                    rev
-                  },
-                  callback
-                )
-              } else {
-                return callback()
-              }
-            }
-          )
-        }
-      )
-    })
-  },
-
-  setRootDoc(project_id, newRootDocID, callback) {
-    if (callback == null) {
-      callback = function(error) {}
-    }
-    logger.log({ project_id, rootDocId: newRootDocID }, 'setting root doc')
-    if (project_id == null || newRootDocID == null) {
-      return callback(
-        new Errors.InvalidError('missing arguments (project or doc)')
-      )
-    }
-    ProjectEntityHandler.getDocPathByProjectIdAndDocId(
-      project_id,
-      newRootDocID,
-      function(err, docPath) {
-        if (err != null) {
-          return callback(err)
-        }
-        if (ProjectEntityUpdateHandler.isPathValidForRootDoc(docPath)) {
-          return Project.update(
-            { _id: project_id },
-            { rootDoc_id: newRootDocID },
-            {},
-            callback
-          )
-        } else {
-          return callback(
-            new Errors.UnsupportedFileTypeError(
-              'invalid file extension for root doc'
-            )
-          )
-        }
-      }
-    )
-  },
-
-  unsetRootDoc(project_id, callback) {
-    if (callback == null) {
-      callback = function(error) {}
-    }
-    logger.log({ project_id }, 'removing root doc')
-    return Project.update(
-      { _id: project_id },
-      { $unset: { rootDoc_id: true } },
-      {},
-      callback
-    )
-  },
-
-  _addDocAndSendToTpds(project_id, folder_id, doc, callback) {
-    if (callback == null) {
-      callback = function(error, result, project) {}
-    }
-    return ProjectEntityMongoUpdateHandler.addDoc(
-      project_id,
-      folder_id,
-      doc,
-      function(err, result, project) {
-        if (err != null) {
-          logger.warn(
-            {
-              err,
-              project_id,
-              folder_id,
-              doc_name: doc != null ? doc.name : undefined,
-              doc_id: doc != null ? doc._id : undefined
-            },
-            'error adding file with project'
-          )
-          return callback(err)
-        }
-        return TpdsUpdateSender.addDoc(
-          {
-            project_id,
-            doc_id: doc != null ? doc._id : undefined,
-            path: __guard__(
-              result != null ? result.path : undefined,
-              x => x.fileSystem
-            ),
-            project_name: project.name,
-            rev: 0
-          },
-          function(err) {
-            if (err != null) {
-              return callback(err)
-            }
-            return callback(null, result, project)
-          }
-        )
-      }
-    )
-  },
-
-  addDoc(project_id, folder_id, docName, docLines, userId, callback) {
-    return self.addDocWithRanges(
-      project_id,
-      folder_id,
-      docName,
-      docLines,
-      {},
-      userId,
-      callback
-    )
-  },
-
-  addDocWithRanges: wrapWithLock({
-    beforeLock(next) {
-      return function(
-        project_id,
-        folder_id,
-        docName,
-        docLines,
-        ranges,
-        userId,
-        callback
-      ) {
-        if (callback == null) {
-          callback = function(error, doc, folder_id) {}
-        }
-        if (!SafePath.isCleanFilename(docName)) {
-          return callback(new Errors.InvalidNameError('invalid element name'))
-        }
-        // Put doc in docstore first, so that if it errors, we don't have a doc_id in the project
-        // which hasn't been created in docstore.
-        const doc = new Doc({ name: docName })
-        return DocstoreManager.updateDoc(
-          project_id.toString(),
-          doc._id.toString(),
-          docLines,
-          0,
-          ranges,
-          function(err, modified, rev) {
-            if (err != null) {
-              return callback(err)
-            }
-            return next(
-              project_id,
-              folder_id,
-              doc,
-              docName,
-              docLines,
-              ranges,
-              userId,
-              callback
-            )
-          }
-        )
-      }
-    },
-    withLock(
-      project_id,
-      folder_id,
-      doc,
-      docName,
-      docLines,
-      ranges,
-      userId,
-      callback
-    ) {
-      if (callback == null) {
-        callback = function(error, doc, folder_id) {}
-      }
-      return ProjectEntityUpdateHandler._addDocAndSendToTpds(
-        project_id,
-        folder_id,
-        doc,
-        function(err, result, project) {
-          if (err != null) {
-            return callback(err)
-          }
-          const docPath = __guard__(
-            result != null ? result.path : undefined,
-            x => x.fileSystem
-          )
-          const projectHistoryId = __guard__(
-            project.overleaf != null ? project.overleaf.history : undefined,
-            x1 => x1.id
-          )
-          const newDocs = [
-            {
-              doc,
-              path: docPath,
-              docLines: docLines.join('\n')
-            }
-          ]
-          return DocumentUpdaterHandler.updateProjectStructure(
-            project_id,
-            projectHistoryId,
-            userId,
-            { newDocs, newProject: project },
-            function(error) {
-              if (error != null) {
-                return callback(error)
-              }
-              return callback(null, doc, folder_id)
-            }
-          )
-        }
-      )
-    }
-  }),
-
-  _uploadFile(
-    project_id,
-    folder_id,
-    fileName,
-    fsPath,
-    linkedFileData,
-    callback
-  ) {
-    if (callback == null) {
-      callback = function(error, fileStoreUrl, fileRef) {}
-    }
-    if (!SafePath.isCleanFilename(fileName)) {
-      return callback(new Errors.InvalidNameError('invalid element name'))
-    }
-    const fileArgs = {
-      name: fileName,
-      linkedFileData
-    }
-    return FileStoreHandler.uploadFileFromDisk(
-      project_id,
-      fileArgs,
-      fsPath,
-      function(err, fileStoreUrl, fileRef) {
-        if (err != null) {
-          logger.warn(
-            { err, project_id, folder_id, file_name: fileName, fileRef },
-            'error uploading image to s3'
-          )
-          return callback(err)
-        }
-        return callback(null, fileStoreUrl, fileRef)
-      }
-    )
-  },
-
-  _addFileAndSendToTpds(project_id, folder_id, fileRef, callback) {
-    if (callback == null) {
-      callback = function(error) {}
-    }
-    return ProjectEntityMongoUpdateHandler.addFile(
-      project_id,
-      folder_id,
-      fileRef,
-      function(err, result, project) {
-        if (err != null) {
-          logger.warn(
-            { err, project_id, folder_id, file_name: fileRef.name, fileRef },
-            'error adding file with project'
-          )
-          return callback(err)
-        }
-        return TpdsUpdateSender.addFile(
-          {
-            project_id,
-            file_id: fileRef._id,
-            path: __guard__(
-              result != null ? result.path : undefined,
-              x => x.fileSystem
-            ),
-            project_name: project.name,
-            rev: fileRef.rev
-          },
-          function(err) {
-            if (err != null) {
-              return callback(err)
-            }
-            return callback(null, result, project)
-          }
-        )
-      }
-    )
-  },
-
-  addFile: wrapWithLock({
-    beforeLock(next) {
-      return function(
-        project_id,
-        folder_id,
-        fileName,
-        fsPath,
-        linkedFileData,
-        userId,
-        callback
-      ) {
-        if (!SafePath.isCleanFilename(fileName)) {
-          return callback(new Errors.InvalidNameError('invalid element name'))
-        }
-        return ProjectEntityUpdateHandler._uploadFile(
-          project_id,
-          folder_id,
-          fileName,
-          fsPath,
-          linkedFileData,
-          function(error, fileStoreUrl, fileRef) {
-            if (error != null) {
-              return callback(error)
-            }
-            return next(
-              project_id,
-              folder_id,
-              fileName,
-              fsPath,
-              linkedFileData,
-              userId,
-              fileRef,
-              fileStoreUrl,
-              callback
-            )
-          }
-        )
-      }
-    },
-    withLock(
-      project_id,
-      folder_id,
-      fileName,
-      fsPath,
-      linkedFileData,
-      userId,
-      fileRef,
-      fileStoreUrl,
-      callback
-    ) {
-      if (callback == null) {
-        callback = function(error, fileRef, folder_id) {}
-      }
-      return ProjectEntityUpdateHandler._addFileAndSendToTpds(
-        project_id,
-        folder_id,
-        fileRef,
-        function(err, result, project) {
-          if (err != null) {
-            return callback(err)
-          }
-          const projectHistoryId = __guard__(
-            project.overleaf != null ? project.overleaf.history : undefined,
-            x => x.id
-          )
-          const newFiles = [
-            {
-              file: fileRef,
-              path: __guard__(
-                result != null ? result.path : undefined,
-                x1 => x1.fileSystem
-              ),
-              url: fileStoreUrl
-            }
-          ]
-          return DocumentUpdaterHandler.updateProjectStructure(
-            project_id,
-            projectHistoryId,
-            userId,
-            { newFiles, newProject: project },
-            function(error) {
-              if (error != null) {
-                return callback(error)
-              }
-              return callback(null, fileRef, folder_id)
-            }
-          )
-        }
-      )
-    }
-  }),
-
-  replaceFile: wrapWithLock({
-    beforeLock(next) {
-      return function(
-        project_id,
-        file_id,
-        fsPath,
-        linkedFileData,
-        userId,
-        callback
-      ) {
-        // create a new file
-        const fileArgs = {
-          name: 'dummy-upload-filename',
-          linkedFileData
-        }
-        return FileStoreHandler.uploadFileFromDisk(
-          project_id,
-          fileArgs,
-          fsPath,
-          function(err, fileStoreUrl, fileRef) {
-            if (err != null) {
-              return callback(err)
-            }
-            return next(
-              project_id,
-              file_id,
-              fsPath,
-              linkedFileData,
-              userId,
-              fileRef,
-              fileStoreUrl,
-              callback
-            )
-          }
-        )
-      }
-    },
-    withLock(
-      project_id,
-      file_id,
-      fsPath,
-      linkedFileData,
-      userId,
-      newFileRef,
-      fileStoreUrl,
-      callback
-    ) {
-      return ProjectEntityMongoUpdateHandler.replaceFileWithNew(
-        project_id,
-        file_id,
-        newFileRef,
-        function(err, oldFileRef, project, path, newProject) {
-          if (err != null) {
-            return callback(err)
-          }
-          const oldFiles = [
-            {
-              file: oldFileRef,
-              path: path.fileSystem
-            }
-          ]
-          const newFiles = [
-            {
-              file: newFileRef,
-              path: path.fileSystem,
-              url: fileStoreUrl
-            }
-          ]
-          const projectHistoryId = __guard__(
-            project.overleaf != null ? project.overleaf.history : undefined,
-            x => x.id
-          )
-          // Increment the rev for an in-place update (with the same path) so the third-party-datastore
-          // knows this is a new file.
-          // Ideally we would get this from ProjectEntityMongoUpdateHandler.replaceFileWithNew
-          // but it returns the original oldFileRef (after incrementing the rev value in mongo),
-          // so we add 1 to the rev from that. This isn't atomic and relies on the lock
-          // but it is acceptable for now.
-          return TpdsUpdateSender.addFile(
-            {
-              project_id: project._id,
-              file_id: newFileRef._id,
-              path: path.fileSystem,
-              rev: oldFileRef.rev + 1,
-              project_name: project.name
-            },
-            function(err) {
-              if (err != null) {
-                return callback(err)
-              }
-              return DocumentUpdaterHandler.updateProjectStructure(
-                project_id,
-                projectHistoryId,
-                userId,
-                { oldFiles, newFiles, newProject },
-                callback
-              )
-            }
-          )
-        }
-      )
-    }
-  }),
-
-  upsertDoc: wrapWithLock(function(
-    project_id,
-    folder_id,
-    docName,
-    docLines,
-    source,
-    userId,
-    callback
-  ) {
-    if (callback == null) {
-      callback = function(err, doc, folder_id, isNewDoc) {}
-    }
-    if (!SafePath.isCleanFilename(docName)) {
-      return callback(new Errors.InvalidNameError('invalid element name'))
-    }
-    return ProjectLocator.findElement(
-      { project_id, element_id: folder_id, type: 'folder' },
-      function(error, folder) {
-        if (error != null) {
-          return callback(error)
-        }
-        if (folder == null) {
-          return callback(new Error("Couldn't find folder"))
-        }
-        let existingDoc = null
-        for (let doc of Array.from(folder.docs)) {
-          if (doc.name === docName) {
-            existingDoc = doc
-            break
-          }
-        }
-        if (existingDoc != null) {
-          return DocumentUpdaterHandler.setDocument(
-            project_id,
-            existingDoc._id,
-            userId,
-            docLines,
-            source,
-            err => {
-              logger.log(
-                { project_id, doc_id: existingDoc._id },
-                'notifying users that the document has been updated'
-              )
-              return DocumentUpdaterHandler.flushDocToMongo(
-                project_id,
-                existingDoc._id,
-                function(err) {
-                  if (err != null) {
-                    return callback(err)
-                  }
-                  return callback(null, existingDoc, existingDoc == null)
-                }
-              )
-            }
-          )
-        } else {
-          return self.addDocWithRanges.withoutLock(
-            project_id,
-            folder_id,
-            docName,
-            docLines,
-            {},
-            userId,
-            function(err, doc) {
-              if (err != null) {
-                return callback(err)
-              }
-              return callback(null, doc, existingDoc == null)
-            }
-          )
-        }
-      }
-    )
-  }),
-
-  upsertFile: wrapWithLock({
-    beforeLock(next) {
-      return function(
-        project_id,
-        folder_id,
-        fileName,
-        fsPath,
-        linkedFileData,
-        userId,
-        callback
-      ) {
-        if (!SafePath.isCleanFilename(fileName)) {
-          return callback(new Errors.InvalidNameError('invalid element name'))
-        }
-        // create a new file
-        const fileArgs = {
-          name: fileName,
-          linkedFileData
-        }
-        return FileStoreHandler.uploadFileFromDisk(
-          project_id,
-          fileArgs,
-          fsPath,
-          function(err, fileStoreUrl, fileRef) {
-            if (err != null) {
-              return callback(err)
-            }
-            return next(
-              project_id,
-              folder_id,
-              fileName,
-              fsPath,
-              linkedFileData,
-              userId,
-              fileRef,
-              fileStoreUrl,
-              callback
-            )
-          }
-        )
-      }
-    },
-    withLock(
-      project_id,
-      folder_id,
-      fileName,
-      fsPath,
-      linkedFileData,
-      userId,
-      newFileRef,
-      fileStoreUrl,
-      callback
-    ) {
-      if (callback == null) {
-        callback = function(err, file, isNewFile, existingFile) {}
-      }
-      return ProjectLocator.findElement(
-        { project_id, element_id: folder_id, type: 'folder' },
-        function(error, folder) {
-          if (error != null) {
-            return callback(error)
-          }
-          if (folder == null) {
-            return callback(new Error("Couldn't find folder"))
-          }
-          let existingFile = null
-          for (let fileRef of Array.from(folder.fileRefs)) {
-            if (fileRef.name === fileName) {
-              existingFile = fileRef
-              break
-            }
-          }
-          if (existingFile != null) {
-            // this calls directly into the replaceFile main task (without the beforeLock part)
-            return self.replaceFile.mainTask(
-              project_id,
-              existingFile._id,
-              fsPath,
-              linkedFileData,
-              userId,
-              newFileRef,
-              fileStoreUrl,
-              function(err) {
-                if (err != null) {
-                  return callback(err)
-                }
-                return callback(
-                  null,
-                  newFileRef,
-                  existingFile == null,
-                  existingFile
-                )
-              }
-            )
-          } else {
-            // this calls directly into the addFile main task (without the beforeLock part)
-            return self.addFile.mainTask(
-              project_id,
-              folder_id,
-              fileName,
-              fsPath,
-              linkedFileData,
-              userId,
-              newFileRef,
-              fileStoreUrl,
-              function(err) {
-                if (err != null) {
-                  return callback(err)
-                }
-                return callback(
-                  null,
-                  newFileRef,
-                  existingFile == null,
-                  existingFile
-                )
-              }
-            )
-          }
-        }
-      )
-    }
-  }),
-
-  upsertDocWithPath: wrapWithLock(function(
-    project_id,
-    elementPath,
-    docLines,
-    source,
-    userId,
-    callback
-  ) {
-    if (!SafePath.isCleanPath(elementPath)) {
-      return callback(new Errors.InvalidNameError('invalid element name'))
-    }
-    const docName = path.basename(elementPath)
-    const folderPath = path.dirname(elementPath)
-    return self.mkdirp.withoutLock(project_id, folderPath, function(
-      err,
-      newFolders,
-      folder
-    ) {
-      if (err != null) {
-        return callback(err)
-      }
-      return self.upsertDoc.withoutLock(
-        project_id,
-        folder._id,
-        docName,
-        docLines,
-        source,
-        userId,
-        function(err, doc, isNewDoc) {
-          if (err != null) {
-            return callback(err)
-          }
-          return callback(null, doc, isNewDoc, newFolders, folder)
-        }
-      )
-    })
-  }),
-
-  upsertFileWithPath: wrapWithLock({
-    beforeLock(next) {
-      return function(
-        project_id,
-        elementPath,
-        fsPath,
-        linkedFileData,
-        userId,
-        callback
-      ) {
-        if (!SafePath.isCleanPath(elementPath)) {
-          return callback(new Errors.InvalidNameError('invalid element name'))
-        }
-        const fileName = path.basename(elementPath)
-        const folderPath = path.dirname(elementPath)
-        // create a new file
-        const fileArgs = {
-          name: fileName,
-          linkedFileData
-        }
-        return FileStoreHandler.uploadFileFromDisk(
-          project_id,
-          fileArgs,
-          fsPath,
-          function(err, fileStoreUrl, fileRef) {
-            if (err != null) {
-              return callback(err)
-            }
-            return next(
-              project_id,
-              folderPath,
-              fileName,
-              fsPath,
-              linkedFileData,
-              userId,
-              fileRef,
-              fileStoreUrl,
-              callback
-            )
-          }
-        )
-      }
-    },
-    withLock(
-      project_id,
-      folderPath,
-      fileName,
-      fsPath,
-      linkedFileData,
-      userId,
-      fileRef,
-      fileStoreUrl,
-      callback
-    ) {
-      return self.mkdirp.withoutLock(project_id, folderPath, function(
-        err,
-        newFolders,
-        folder
-      ) {
-        if (err != null) {
-          return callback(err)
-        }
-        // this calls directly into the upsertFile main task (without the beforeLock part)
-        return self.upsertFile.mainTask(
-          project_id,
-          folder._id,
-          fileName,
-          fsPath,
-          linkedFileData,
-          userId,
-          fileRef,
-          fileStoreUrl,
-          function(err, newFile, isNewFile, existingFile) {
-            if (err != null) {
-              return callback(err)
-            }
-            return callback(
-              null,
-              newFile,
-              isNewFile,
-              existingFile,
-              newFolders,
-              folder
-            )
-          }
-        )
-      })
-    }
-  }),
-
-  deleteEntity: wrapWithLock(function(
-    project_id,
-    entity_id,
-    entityType,
-    userId,
-    callback
-  ) {
-    if (callback == null) {
-      callback = function(error) {}
-    }
-    logger.log({ entity_id, entityType, project_id }, 'deleting project entity')
-    if (entityType == null) {
-      logger.warn({ err: 'No entityType set', project_id, entity_id })
-      return callback(new Error('No entityType set'))
-    }
-    entityType = entityType.toLowerCase()
-    return ProjectEntityMongoUpdateHandler.deleteEntity(
-      project_id,
-      entity_id,
-      entityType,
-      function(error, entity, path, projectBeforeDeletion, newProject) {
-        if (error != null) {
-          return callback(error)
-        }
-        return self._cleanUpEntity(
-          projectBeforeDeletion,
-          newProject,
-          entity,
-          entityType,
-          path.fileSystem,
-          userId,
-          function(error) {
-            if (error != null) {
-              return callback(error)
-            }
-            return TpdsUpdateSender.deleteEntity(
-              {
-                project_id,
-                path: path.fileSystem,
-                project_name: projectBeforeDeletion.name
-              },
-              function(error) {
-                if (error != null) {
-                  return callback(error)
-                }
-                return callback(null, entity_id)
-              }
-            )
-          }
-        )
-      }
-    )
-  }),
-
-  deleteEntityWithPath: wrapWithLock((project_id, path, userId, callback) =>
-    ProjectLocator.findElementByPath({ project_id, path }, function(
-      err,
-      element,
-      type
-    ) {
-      if (err != null) {
-        return callback(err)
-      }
-      if (element == null) {
-        return callback(new Errors.NotFoundError('project not found'))
-      }
-      return self.deleteEntity.withoutLock(
-        project_id,
-        element._id,
-        type,
-        userId,
-        callback
-      )
-    })
-  ),
-
-  mkdirp: wrapWithLock(function(project_id, path, callback) {
-    if (callback == null) {
-      callback = function(err, newlyCreatedFolders, lastFolderInPath) {}
-    }
-    for (let folder of Array.from(path.split('/'))) {
-      if (folder.length > 0 && !SafePath.isCleanFilename(folder)) {
-        return callback(new Errors.InvalidNameError('invalid element name'))
-      }
-    }
-    return ProjectEntityMongoUpdateHandler.mkdirp(
-      project_id,
-      path,
-      { exactCaseMatch: false },
-      callback
-    )
-  }),
-
-  mkdirpWithExactCase: wrapWithLock(function(project_id, path, callback) {
-    if (callback == null) {
-      callback = function(err, newlyCreatedFolders, lastFolderInPath) {}
-    }
-    for (let folder of Array.from(path.split('/'))) {
-      if (folder.length > 0 && !SafePath.isCleanFilename(folder)) {
-        return callback(new Errors.InvalidNameError('invalid element name'))
-      }
-    }
-    return ProjectEntityMongoUpdateHandler.mkdirp(
-      project_id,
-      path,
-      { exactCaseMatch: true },
-      callback
-    )
-  }),
-
-  addFolder: wrapWithLock(function(
-    project_id,
-    parentFolder_id,
-    folderName,
-    callback
-  ) {
-    if (!SafePath.isCleanFilename(folderName)) {
-      return callback(new Errors.InvalidNameError('invalid element name'))
-    }
-    return ProjectEntityMongoUpdateHandler.addFolder(
-      project_id,
-      parentFolder_id,
-      folderName,
-      callback
-    )
-  }),
-
-  moveEntity: wrapWithLock(function(
-    project_id,
-    entity_id,
-    destFolderId,
-    entityType,
-    userId,
-    callback
-  ) {
-    if (callback == null) {
-      callback = function(error) {}
-    }
-    logger.log(
-      { entityType, entity_id, project_id, destFolderId },
-      'moving entity'
-    )
-    if (entityType == null) {
-      logger.warn({ err: 'No entityType set', project_id, entity_id })
-      return callback(new Error('No entityType set'))
-    }
-    entityType = entityType.toLowerCase()
-    return ProjectEntityMongoUpdateHandler.moveEntity(
-      project_id,
-      entity_id,
-      destFolderId,
-      entityType,
-      function(err, project, startPath, endPath, rev, changes) {
-        if (err != null) {
-          return callback(err)
-        }
-        const projectHistoryId = __guard__(
-          project.overleaf != null ? project.overleaf.history : undefined,
-          x => x.id
-        )
-        TpdsUpdateSender.moveEntity({
-          project_id,
-          project_name: project.name,
-          startPath,
-          endPath,
-          rev
-        })
-        return DocumentUpdaterHandler.updateProjectStructure(
-          project_id,
-          projectHistoryId,
-          userId,
-          changes,
-          callback
-        )
-      }
-    )
-  }),
-
-  renameEntity: wrapWithLock(function(
-    project_id,
-    entity_id,
-    entityType,
-    newName,
-    userId,
-    callback
-  ) {
-    if (!SafePath.isCleanFilename(newName)) {
-      return callback(new Errors.InvalidNameError('invalid element name'))
-    }
-    logger.log({ entity_id, project_id }, `renaming ${entityType}`)
-    if (entityType == null) {
-      logger.warn({ err: 'No entityType set', project_id, entity_id })
-      return callback(new Error('No entityType set'))
-    }
-    entityType = entityType.toLowerCase()
-
-    return ProjectEntityMongoUpdateHandler.renameEntity(
-      project_id,
-      entity_id,
-      entityType,
-      newName,
-      function(err, project, startPath, endPath, rev, changes) {
-        if (err != null) {
-          return callback(err)
-        }
-        const projectHistoryId = __guard__(
-          project.overleaf != null ? project.overleaf.history : undefined,
-          x => x.id
-        )
-        TpdsUpdateSender.moveEntity({
-          project_id,
-          project_name: project.name,
-          startPath,
-          endPath,
-          rev
-        })
-        return DocumentUpdaterHandler.updateProjectStructure(
-          project_id,
-          projectHistoryId,
-          userId,
-          changes,
-          callback
-        )
-      }
-    )
-  }),
-
-  // This doesn't directly update project structure but we need to take the lock
-  // to prevent anything else being queued before the resync update
-  resyncProjectHistory: wrapWithLock((project_id, callback) =>
-    ProjectGetter.getProject(
-      project_id,
-      { rootFolder: true, overleaf: true },
-      function(error, project) {
-        if (error != null) {
-          return callback(error)
-        }
-
-        const projectHistoryId = __guard__(
-          __guard__(
-            project != null ? project.overleaf : undefined,
-            x1 => x1.history
-          ),
-          x => x.id
-        )
-        if (projectHistoryId == null) {
-          error = new Errors.ProjectHistoryDisabledError(
-            `project history not enabled for ${project_id}`
-          )
-          return callback(error)
-        }
-
-        return ProjectEntityHandler.getAllEntitiesFromProject(project, function(
-          error,
-          docs,
-          files
-        ) {
-          if (error != null) {
-            return callback(error)
-          }
-
-          docs = _.map(docs, doc => ({
-            doc: doc.doc._id,
-            path: doc.path
-          }))
-
-          files = _.map(files, file => ({
-            file: file.file._id,
-            path: file.path,
-            url: FileStoreHandler._buildUrl(project_id, file.file._id)
-          }))
-
-          return DocumentUpdaterHandler.resyncProjectHistory(
-            project_id,
-            projectHistoryId,
-            docs,
-            files,
-            callback
-          )
-        })
-      }
-    )
-  ),
-
-  isPathValidForRootDoc(docPath) {
-    let docExtension = path.extname(docPath)
-    return validRootDocRegExp.test(docExtension)
-  },
-
-  _cleanUpEntity(
-    project,
-    newProject,
-    entity,
-    entityType,
-    path,
-    userId,
-    callback
-  ) {
-    if (callback == null) {
-      callback = function(error) {}
-    }
-    return self._updateProjectStructureWithDeletedEntity(
-      project,
-      newProject,
-      entity,
-      entityType,
-      path,
-      userId,
-      function(error) {
-        if (error != null) {
-          return callback(error)
-        }
-        if (entityType.indexOf('file') !== -1) {
-          return self._cleanUpFile(project, entity, path, userId, callback)
-        } else if (entityType.indexOf('doc') !== -1) {
-          return self._cleanUpDoc(project, entity, path, userId, callback)
-        } else if (entityType.indexOf('folder') !== -1) {
-          return self._cleanUpFolder(project, entity, path, userId, callback)
-        } else {
-          return callback()
-        }
-      }
-    )
-  },
-
-  // Note: the _cleanUpEntity code and _updateProjectStructureWithDeletedEntity
-  // methods both need to recursively iterate over the entities in folder.
-  // These are currently using separate implementations of the recursion. In
-  // future, these could be simplified using a common project entity iterator.
-  _updateProjectStructureWithDeletedEntity(
-    project,
-    newProject,
-    entity,
-    entityType,
-    entityPath,
-    userId,
-    callback
-  ) {
-    // compute the changes to the project structure
-    let changes
-    if (callback == null) {
-      callback = function(error) {}
-    }
-    if (entityType.indexOf('file') !== -1) {
-      changes = { oldFiles: [{ file: entity, path: entityPath }] }
-    } else if (entityType.indexOf('doc') !== -1) {
-      changes = { oldDocs: [{ doc: entity, path: entityPath }] }
-    } else if (entityType.indexOf('folder') !== -1) {
-      changes = { oldDocs: [], oldFiles: [] }
-      var _recurseFolder = function(folder, folderPath) {
-        for (let doc of Array.from(folder.docs)) {
-          changes.oldDocs.push({ doc, path: path.join(folderPath, doc.name) })
-        }
-        for (let file of Array.from(folder.fileRefs)) {
-          changes.oldFiles.push({
-            file,
-            path: path.join(folderPath, file.name)
-          })
-        }
-        return Array.from(folder.folders).map(childFolder =>
-          _recurseFolder(childFolder, path.join(folderPath, childFolder.name))
-        )
-      }
-      _recurseFolder(entity, entityPath)
-    }
-    // now send the project structure changes to the docupdater
-    changes.newProject = newProject
-    const project_id = project._id.toString()
-    const projectHistoryId = __guard__(
-      project.overleaf != null ? project.overleaf.history : undefined,
-      x => x.id
-    )
-    return DocumentUpdaterHandler.updateProjectStructure(
-      project_id,
-      projectHistoryId,
-      userId,
-      changes,
-      callback
-    )
-  },
-
-  _cleanUpDoc(project, doc, path, userId, callback) {
-    if (callback == null) {
-      callback = function(error) {}
-    }
-    const project_id = project._id.toString()
-    const doc_id = doc._id.toString()
-    const unsetRootDocIfRequired = callback => {
-      if (
-        project.rootDoc_id != null &&
-        project.rootDoc_id.toString() === doc_id
-      ) {
-        return this.unsetRootDoc(project_id, callback)
-      } else {
-        return callback()
-      }
-    }
-
-    return unsetRootDocIfRequired(function(error) {
-      if (error != null) {
-        return callback(error)
-      }
-      return ProjectEntityMongoUpdateHandler._insertDeletedDocReference(
-        project._id,
-        doc,
-        function(error) {
-          if (error != null) {
-            return callback(error)
-          }
-          return DocumentUpdaterHandler.deleteDoc(project_id, doc_id, function(
-            error
-          ) {
-            if (error != null) {
-              return callback(error)
-            }
-            return DocstoreManager.deleteDoc(project_id, doc_id, callback)
-          })
-        }
-      )
-    })
-  },
-
-  _cleanUpFile(project, file, path, userId, callback) {
-    if (callback == null) {
-      callback = function(error) {}
-    }
-    return ProjectEntityMongoUpdateHandler._insertDeletedFileReference(
-      project._id,
-      file,
-      callback
-    )
-  },
-
-  _cleanUpFolder(project, folder, folderPath, userId, callback) {
-    if (callback == null) {
-      callback = function(error) {}
-    }
-    const jobs = []
-    for (let doc of Array.from(folder.docs)) {
-      ;(function(doc) {
-        const docPath = path.join(folderPath, doc.name)
-        return jobs.push(callback =>
-          self._cleanUpDoc(project, doc, docPath, userId, callback)
-        )
-      })(doc)
-    }
-
-    for (let file of Array.from(folder.fileRefs)) {
-      ;(function(file) {
-        const filePath = path.join(folderPath, file.name)
-        return jobs.push(callback =>
-          self._cleanUpFile(project, file, filePath, userId, callback)
-        )
-      })(file)
-    }
-
-    for (let childFolder of Array.from(folder.folders)) {
-      ;(function(childFolder) {
-        folderPath = path.join(folderPath, childFolder.name)
-        return jobs.push(callback =>
-          self._cleanUpFolder(
-            project,
-            childFolder,
-            folderPath,
-            userId,
-            callback
-          )
-        )
-      })(childFolder)
-    }
-
-    return async.series(jobs, callback)
-  }
-}
-
-function __guard__(value, transform) {
-  return typeof value !== 'undefined' && value !== null
-    ? transform(value)
-    : undefined
-}

services/web/test/unit/src/Project/FolderStructureBuilderTests.js

@@ -0,0 +1,115 @@
+const { expect } = require('chai')
+const sinon = require('sinon')
+const SandboxedModule = require('sandboxed-module')
+
+const MODULE_PATH =
+  '../../../../app/src/Features/Project/FolderStructureBuilder'
+const MOCK_OBJECT_ID = 'MOCK_OBJECT_ID'
+
+describe('FolderStructureBuilder', function() {
+  beforeEach(function() {
+    this.ObjectId = sinon.stub().returns(MOCK_OBJECT_ID)
+    this.FolderStructureBuilder = SandboxedModule.require(MODULE_PATH, {
+      requires: {
+        mongodb: { ObjectId: this.ObjectId }
+      }
+    })
+  })
+
+  describe('buildFolderStructure', function() {
+    describe('when given no documents at all', function() {
+      beforeEach(function() {
+        this.result = this.FolderStructureBuilder.buildFolderStructure([], [])
+      })
+
+      it('returns an empty root folder', function() {
+        expect(this.result).to.deep.equal({
+          _id: MOCK_OBJECT_ID,
+          name: 'rootFolder',
+          folders: [],
+          docs: [],
+          fileRefs: []
+        })
+      })
+    })
+
+    describe('when given documents and files', function() {
+      beforeEach(function() {
+        const docUploads = [
+          { dirname: '/', doc: { _id: 'doc-1', name: 'main.tex' } },
+          { dirname: '/foo', doc: { _id: 'doc-2', name: 'other.tex' } },
+          { dirname: '/foo', doc: { _id: 'doc-3', name: 'other.bib' } },
+          {
+            dirname: '/foo/foo1/foo2',
+            doc: { _id: 'doc-4', name: 'another.tex' }
+          }
+        ]
+        const fileUploads = [
+          { dirname: '/', fileRef: { _id: 'file-1', name: 'aaa.jpg' } },
+          { dirname: '/foo', fileRef: { _id: 'file-2', name: 'bbb.jpg' } },
+          { dirname: '/bar', fileRef: { _id: 'file-3', name: 'ccc.jpg' } }
+        ]
+        this.result = this.FolderStructureBuilder.buildFolderStructure(
+          docUploads,
+          fileUploads
+        )
+      })
+
+      it('returns a full folder structure', function() {
+        expect(this.result).to.deep.equal({
+          _id: MOCK_OBJECT_ID,
+          name: 'rootFolder',
+          docs: [{ _id: 'doc-1', name: 'main.tex' }],
+          fileRefs: [{ _id: 'file-1', name: 'aaa.jpg' }],
+          folders: [
+            {
+              _id: MOCK_OBJECT_ID,
+              name: 'foo',
+              docs: [
+                { _id: 'doc-2', name: 'other.tex' },
+                { _id: 'doc-3', name: 'other.bib' }
+              ],
+              fileRefs: [{ _id: 'file-2', name: 'bbb.jpg' }],
+              folders: [
+                {
+                  _id: MOCK_OBJECT_ID,
+                  name: 'foo1',
+                  docs: [],
+                  fileRefs: [],
+                  folders: [
+                    {
+                      _id: MOCK_OBJECT_ID,
+                      name: 'foo2',
+                      docs: [{ _id: 'doc-4', name: 'another.tex' }],
+                      fileRefs: [],
+                      folders: []
+                    }
+                  ]
+                }
+              ]
+            },
+            {
+              _id: MOCK_OBJECT_ID,
+              name: 'bar',
+              docs: [],
+              fileRefs: [{ _id: 'file-3', name: 'ccc.jpg' }],
+              folders: []
+            }
+          ]
+        })
+      })
+    })
+
+    describe('when given duplicate files', function() {
+      it('throws an error', function() {
+        const docUploads = [
+          { dirname: '/foo', doc: { _id: 'doc-1', name: 'doc.tex' } },
+          { dirname: '/foo', doc: { _id: 'doc-2', name: 'doc.tex' } }
+        ]
+        expect(() =>
+          this.FolderStructureBuilder.buildFolderStructure(docUploads, [])
+        ).to.throw()
+      })
+    })
+  })
+})

services/web/test/unit/src/Project/ProjectEntityMongoUpdateHandlerCanaryTests.js

@@ -177,6 +177,10 @@ describe('ProjectEntityMongoUpdateHandler', function() {
       }
     }
 
+    this.FolderStructureBuilder = {
+      buildFolderStructure: sinon.stub()
+    }
+
     this.subject = SandboxedModule.require(MODULE_PATH, {
       globals: {
         console: console
@@ -191,6 +195,7 @@ describe('ProjectEntityMongoUpdateHandler', function() {
         './ProjectEntityHandler': this.ProjectEntityHandler,
         './ProjectLocator': this.ProjectLocator,
         './ProjectGetter': this.ProjectGetter,
+        './FolderStructureBuilder': this.FolderStructureBuilder,
         // We need to provide Errors here to make instance check work
         '../Errors/Errors': Errors
       }
@@ -1031,4 +1036,57 @@ describe('ProjectEntityMongoUpdateHandler', function() {
       this.ProjectMock.verify()
     })
   })
+
+  describe('createNewFolderStructure', function() {
+    beforeEach(function() {
+      this.mockRootFolder = 'MOCK_ROOT_FOLDER'
+      this.docUploads = ['MOCK_DOC_UPLOAD']
+      this.fileUploads = ['MOCK_FILE_UPLOAD']
+      this.FolderStructureBuilder.buildFolderStructure
+        .withArgs(this.docUploads, this.fileUploads)
+        .returns(this.mockRootFolder)
+      this.updateExpectation = this.ProjectMock.expects('updateOne')
+        .withArgs(
+          {
+            _id: this.project._id,
+            'rootFolder.0.folders.0': { $exists: false },
+            'rootFolder.0.docs.0': { $exists: false },
+            'rootFolder.0.files.0': { $exists: false }
+          },
+          { $set: { rootFolder: [this.mockRootFolder] }, $inc: { version: 1 } }
+        )
+        .chain('exec')
+    })
+
+    describe('happy path', function() {
+      beforeEach(async function() {
+        this.updateExpectation.resolves({ n: 1 })
+        await this.subject.promises.createNewFolderStructure(
+          this.project._id,
+          this.docUploads,
+          this.fileUploads
+        )
+      })
+
+      it('updates the database', function() {
+        this.ProjectMock.verify()
+      })
+    })
+
+    describe("when the update doesn't find a matching document", function() {
+      beforeEach(async function() {
+        this.updateExpectation.resolves({ n: 0 })
+      })
+
+      it('throws an error', async function() {
+        await expect(
+          this.subject.promises.createNewFolderStructure(
+            this.project._id,
+            this.docUploads,
+            this.fileUploads
+          )
+        ).to.be.rejected
+      })
+    })
+  })
 })

services/web/test/unit/src/Project/ProjectEntityUpdateHandlerTests.js

@@ -453,8 +453,8 @@ describe('ProjectEntityUpdateHandler', function() {
         return this.logger.warn
           .calledWith(
             {
-              project_id,
-              doc_id,
+              projectId: project_id,
+              docId: doc_id,
               lines: this.docLines
             },
             'doc not found while updating doc lines'
@@ -504,7 +504,11 @@ describe('ProjectEntityUpdateHandler', function() {
         .stub()
         .yields(null, `/main.tex`)
 
-      this.ProjectEntityUpdateHandler.setRootDoc(project_id, this.rootDoc_id)
+      this.ProjectEntityUpdateHandler.setRootDoc(
+        project_id,
+        this.rootDoc_id,
+        () => {}
+      )
       return this.ProjectModel.update
         .calledWith({ _id: project_id }, { rootDoc_id: this.rootDoc_id })
         .should.equal(true)
@@ -516,7 +520,11 @@ describe('ProjectEntityUpdateHandler', function() {
         .stub()
         .yields(Errors.NotFoundError)
 
-      this.ProjectEntityUpdateHandler.setRootDoc(project_id, this.rootDoc_id)
+      this.ProjectEntityUpdateHandler.setRootDoc(
+        project_id,
+        this.rootDoc_id,
+        () => {}
+      )
       return this.ProjectModel.update
         .calledWith({ _id: project_id }, { rootDoc_id: this.rootDoc_id })
         .should.equal(false)