RestoreManager.js 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438
  1. const Settings = require('@overleaf/settings')
  2. const Path = require('path')
  3. const FileWriter = require('../../infrastructure/FileWriter')
  4. const FileSystemImportManager = require('../Uploads/FileSystemImportManager')
  5. const EditorController = require('../Editor/EditorController')
  6. const Errors = require('../Errors/Errors')
  7. const moment = require('moment')
  8. const { callbackifyAll } = require('@overleaf/promise-utils')
  9. const { fetchJson } = require('@overleaf/fetch-utils')
  10. const ProjectLocator = require('../Project/ProjectLocator')
  11. const DocumentUpdaterHandler = require('../DocumentUpdater/DocumentUpdaterHandler')
  12. const ChatApiHandler = require('../Chat/ChatApiHandler')
  13. const DocstoreManager = require('../Docstore/DocstoreManager')
  14. const logger = require('@overleaf/logger')
  15. const EditorRealTimeController = require('../Editor/EditorRealTimeController')
  16. const ChatManager = require('../Chat/ChatManager')
  17. const OError = require('@overleaf/o-error')
  18. const ProjectGetter = require('../Project/ProjectGetter')
  19. const ProjectEntityHandler = require('../Project/ProjectEntityHandler')
  20. async function getCommentThreadIds(projectId) {
  21. await DocumentUpdaterHandler.promises.flushProjectToMongo(projectId)
  22. const raw = await DocstoreManager.promises.getCommentThreadIds(projectId)
  23. return new Map(Object.entries(raw).map(([doc, ids]) => [doc, new Set(ids)]))
  24. }
  25. const RestoreManager = {
  26. async restoreFileFromV2(userId, projectId, version, pathname) {
  27. const fsPath = await RestoreManager._writeFileVersionToDisk(
  28. projectId,
  29. version,
  30. pathname
  31. )
  32. const basename = Path.basename(pathname)
  33. let dirname = Path.dirname(pathname)
  34. if (dirname === '.') {
  35. // no directory
  36. dirname = ''
  37. }
  38. const parentFolderId = await RestoreManager._findOrCreateFolder(
  39. projectId,
  40. dirname,
  41. userId
  42. )
  43. const addEntityWithName = async name =>
  44. await FileSystemImportManager.promises.addEntity(
  45. userId,
  46. projectId,
  47. parentFolderId,
  48. name,
  49. fsPath,
  50. false
  51. )
  52. return await RestoreManager._addEntityWithUniqueName(
  53. addEntityWithName,
  54. basename
  55. )
  56. },
  57. async revertFile(userId, projectId, version, pathname, options = {}) {
  58. const threadIds = await getCommentThreadIds(projectId)
  59. return await RestoreManager._revertSingleFile(
  60. userId,
  61. projectId,
  62. version,
  63. pathname,
  64. threadIds,
  65. options
  66. )
  67. },
  68. async _revertSingleFile(
  69. userId,
  70. projectId,
  71. version,
  72. pathname,
  73. threadIds,
  74. options = {}
  75. ) {
  76. const project = await ProjectGetter.promises.getProject(projectId, {
  77. overleaf: true,
  78. rootDoc_id: true,
  79. })
  80. if (!project?.overleaf?.history?.rangesSupportEnabled) {
  81. throw new OError('project does not have ranges support', { projectId })
  82. }
  83. const fsPath = await RestoreManager._writeFileVersionToDisk(
  84. projectId,
  85. version,
  86. pathname
  87. )
  88. const basename = Path.basename(pathname)
  89. let dirname = Path.dirname(pathname)
  90. if (dirname === '.') {
  91. // root directory
  92. dirname = '/'
  93. }
  94. const parentFolderId = await RestoreManager._findOrCreateFolder(
  95. projectId,
  96. dirname,
  97. userId
  98. )
  99. const file = await ProjectLocator.promises
  100. .findElementByPath({
  101. project_id: projectId,
  102. path: pathname,
  103. })
  104. .catch(() => null)
  105. const updates = await RestoreManager._getUpdatesFromHistory(
  106. projectId,
  107. version
  108. )
  109. const updateAtVersion = updates.find(update => update.toV === version)
  110. const origin = options.origin || {
  111. kind: 'file-restore',
  112. path: pathname,
  113. version,
  114. timestamp: new Date(updateAtVersion.meta.end_ts).toISOString(),
  115. }
  116. const importInfo = await FileSystemImportManager.promises.importFile(
  117. fsPath,
  118. pathname
  119. )
  120. let hadDeletedRootFile = false
  121. if (file) {
  122. if (file.type !== 'doc' && file.type !== 'file') {
  123. throw new OError('unexpected file type', { type: file.type })
  124. }
  125. logger.debug(
  126. { projectId, fileId: file.element._id, type: importInfo.type },
  127. 'deleting entity before reverting it'
  128. )
  129. await EditorController.promises.deleteEntity(
  130. projectId,
  131. file.element._id,
  132. file.type,
  133. origin,
  134. userId
  135. )
  136. if (
  137. file.element._id &&
  138. project.rootDoc_id &&
  139. file.element._id.toString() === project.rootDoc_id.toString()
  140. ) {
  141. hadDeletedRootFile = true
  142. }
  143. threadIds.delete(file.element._id.toString())
  144. }
  145. const { metadata } = await RestoreManager._getMetadataFromHistory(
  146. projectId,
  147. version,
  148. pathname
  149. )
  150. // Look for metadata indicating a linked file.
  151. const isFileMetadata = metadata && 'provider' in metadata
  152. logger.debug({ metadata }, 'metadata from history')
  153. if (importInfo.type === 'file' || isFileMetadata) {
  154. const newFile = await EditorController.promises.upsertFile(
  155. projectId,
  156. parentFolderId,
  157. basename,
  158. fsPath,
  159. metadata,
  160. origin,
  161. userId
  162. )
  163. return {
  164. _id: newFile._id,
  165. type: 'file',
  166. }
  167. }
  168. const ranges = await RestoreManager._getRangesFromHistory(
  169. projectId,
  170. version,
  171. pathname
  172. )
  173. const documentCommentIds = new Set(
  174. ranges.comments?.map(({ op: { t } }) => t)
  175. )
  176. const commentIdsToDuplicate = Array.from(documentCommentIds).filter(id => {
  177. for (const ids of threadIds.values()) {
  178. if (ids.has(id)) return true
  179. }
  180. return false
  181. })
  182. const newRanges = { changes: ranges.changes, comments: [] }
  183. if (commentIdsToDuplicate.length > 0) {
  184. const { newThreads: newCommentIds } =
  185. await ChatApiHandler.promises.duplicateCommentThreads(
  186. projectId,
  187. commentIdsToDuplicate
  188. )
  189. logger.debug({ mapping: newCommentIds }, 'replacing comment threads')
  190. for (const comment of ranges.comments ?? []) {
  191. if (Object.prototype.hasOwnProperty.call(newCommentIds, comment.op.t)) {
  192. const result = newCommentIds[comment.op.t]
  193. if (result.error) {
  194. // We couldn't duplicate the thread, so we need to delete it from
  195. // the resulting ranges.
  196. continue
  197. }
  198. // We have a new id for this comment thread
  199. comment.id = result.duplicateId
  200. comment.op.t = result.duplicateId
  201. }
  202. newRanges.comments.push(comment)
  203. }
  204. } else {
  205. newRanges.comments = ranges.comments
  206. }
  207. const newCommentThreadData =
  208. await ChatApiHandler.promises.generateThreadData(
  209. projectId,
  210. newRanges.comments.map(({ op: { t } }) => t)
  211. )
  212. // Resolve/reopen threads in chat service to match what is in history
  213. for (const commentRange of newRanges.comments) {
  214. const threadData = newCommentThreadData[commentRange.op.t]
  215. if (!threadData) {
  216. // comment thread was deleted
  217. continue
  218. }
  219. if (commentRange.op.resolved && threadData.resolved == null) {
  220. // The history snapshot stores the comment's resolved property as a boolean,
  221. // but it does not include information about who resolved the comment or the timestamp.
  222. // Until this is fixed, we will resolve the thread with the current user and the current timestamp.
  223. await ChatApiHandler.promises.resolveThread(
  224. projectId,
  225. commentRange.op.t,
  226. userId
  227. )
  228. threadData.resolved = true
  229. threadData.resolved_by_user_id = userId
  230. threadData.resolved_at = new Date().toISOString()
  231. } else if (!commentRange.op.resolved && threadData.resolved != null) {
  232. await ChatApiHandler.promises.reopenThread(projectId, commentRange.op.t)
  233. delete threadData.resolved
  234. delete threadData.resolved_by_user_id
  235. delete threadData.resolved_at
  236. }
  237. }
  238. await ChatManager.promises.injectUserInfoIntoThreads(newCommentThreadData)
  239. // Only keep restored comment ranges that point to a valid thread.
  240. // The chat service won't have generated thread data for deleted threads.
  241. newRanges.comments = newRanges.comments.filter(
  242. comment => newCommentThreadData[comment.op.t] != null
  243. )
  244. logger.debug({ newCommentThreadData }, 'emitting new comment threads')
  245. EditorRealTimeController.emitToRoom(
  246. projectId,
  247. 'new-comment-threads',
  248. newCommentThreadData
  249. )
  250. const { _id } = await EditorController.promises.addDocWithRanges(
  251. projectId,
  252. parentFolderId,
  253. basename,
  254. importInfo.lines,
  255. newRanges,
  256. origin,
  257. userId
  258. )
  259. if (hadDeletedRootFile) {
  260. await EditorController.promises.setRootDoc(projectId, _id)
  261. }
  262. // For revertProject: The next doc that gets reverted will need to duplicate all the threads seen here.
  263. threadIds.set(
  264. _id.toString(),
  265. new Set(newRanges.comments.map(({ op: { t } }) => t))
  266. )
  267. return {
  268. _id,
  269. type: importInfo.type,
  270. }
  271. },
  272. async _findOrCreateFolder(projectId, dirname, userId) {
  273. const { lastFolder } = await EditorController.promises.mkdirp(
  274. projectId,
  275. dirname,
  276. userId
  277. )
  278. return lastFolder?._id
  279. },
  280. async _addEntityWithUniqueName(addEntityWithName, basename) {
  281. try {
  282. return await addEntityWithName(basename)
  283. } catch (error) {
  284. if (error instanceof Errors.DuplicateNameError) {
  285. // Duplicate name, so try with a prefix
  286. const date = moment(new Date()).format('Do MMM YY H:mm:ss')
  287. // Move extension to the end so the file type is preserved
  288. const extension = Path.extname(basename)
  289. basename = Path.basename(basename, extension)
  290. basename = `${basename} (Restored on ${date})`
  291. if (extension !== '') {
  292. basename = `${basename}${extension}`
  293. }
  294. return await addEntityWithName(basename)
  295. } else {
  296. throw error
  297. }
  298. }
  299. },
  300. async revertProject(userId, projectId, version) {
  301. const project = await ProjectGetter.promises.getProject(projectId, {
  302. overleaf: true,
  303. })
  304. if (!project?.overleaf?.history?.rangesSupportEnabled) {
  305. throw new OError('project does not have ranges support', { projectId })
  306. }
  307. // Get project paths at version
  308. const pathsAtPastVersion = await RestoreManager._getProjectPathsAtVersion(
  309. projectId,
  310. version
  311. )
  312. const updates = await RestoreManager._getUpdatesFromHistory(
  313. projectId,
  314. version
  315. )
  316. const updateAtVersion = updates.find(update => update.toV === version)
  317. const origin = {
  318. kind: 'project-restore',
  319. version,
  320. timestamp: new Date(updateAtVersion.meta.end_ts).toISOString(),
  321. }
  322. const threadIds = await getCommentThreadIds(projectId)
  323. const reverted = []
  324. for (const pathname of pathsAtPastVersion) {
  325. const res = await RestoreManager._revertSingleFile(
  326. userId,
  327. projectId,
  328. version,
  329. pathname,
  330. threadIds,
  331. { origin }
  332. )
  333. reverted.push({
  334. id: res._id,
  335. type: res.type,
  336. path: pathname,
  337. })
  338. }
  339. const entitiesAtLiveVersion =
  340. await ProjectEntityHandler.promises.getAllEntities(projectId)
  341. const trimLeadingSlash = path => path.replace(/^\//, '')
  342. const pathsAtLiveVersion = entitiesAtLiveVersion.docs
  343. .map(doc => doc.path)
  344. .concat(entitiesAtLiveVersion.files.map(file => file.path))
  345. .map(trimLeadingSlash)
  346. // Delete files that were not present at the reverted version
  347. for (const path of pathsAtLiveVersion) {
  348. if (!pathsAtPastVersion.includes(path)) {
  349. await EditorController.promises.deleteEntityWithPath(
  350. projectId,
  351. path,
  352. origin,
  353. userId
  354. )
  355. }
  356. }
  357. return reverted
  358. },
  359. async _writeFileVersionToDisk(projectId, version, pathname) {
  360. const url = `${
  361. Settings.apis.project_history.url
  362. }/project/${projectId}/version/${version}/${encodeURIComponent(pathname)}`
  363. return await FileWriter.promises.writeUrlToDisk(projectId, url)
  364. },
  365. async _getRangesFromHistory(projectId, version, pathname) {
  366. const url = `${
  367. Settings.apis.project_history.url
  368. }/project/${projectId}/ranges/version/${version}/${encodeURIComponent(pathname)}`
  369. return await fetchJson(url)
  370. },
  371. async _getMetadataFromHistory(projectId, version, pathname) {
  372. const url = `${
  373. Settings.apis.project_history.url
  374. }/project/${projectId}/metadata/version/${version}/${encodeURIComponent(pathname)}`
  375. return await fetchJson(url)
  376. },
  377. async _getUpdatesFromHistory(projectId, version) {
  378. const url = `${Settings.apis.project_history.url}/project/${projectId}/updates?before=${version}&min_count=1`
  379. const res = await fetchJson(url)
  380. return res.updates
  381. },
  382. async _getProjectPathsAtVersion(projectId, version) {
  383. const url = `${Settings.apis.project_history.url}/project/${projectId}/paths/version/${version}`
  384. const res = await fetchJson(url)
  385. return res.paths
  386. },
  387. }
  388. module.exports = { ...callbackifyAll(RestoreManager), promises: RestoreManager }