HistoryManager.mjs 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456
  1. import { callbackify } from 'node:util'
  2. import {
  3. fetchJson,
  4. fetchNothing,
  5. fetchStreamWithResponse,
  6. RequestFailedError,
  7. } from '@overleaf/fetch-utils'
  8. import fs from 'node:fs'
  9. import settings from '@overleaf/settings'
  10. import OError from '@overleaf/o-error'
  11. import UserGetter from '../User/UserGetter.mjs'
  12. import ProjectGetter from '../Project/ProjectGetter.mjs'
  13. import HistoryBackupDeletionHandler from './HistoryBackupDeletionHandler.mjs'
  14. import { db, waitForDb } from '../../infrastructure/mongodb.mjs'
  15. import Metrics from '@overleaf/metrics'
  16. import { NotFoundError } from '../Errors/Errors.js'
  17. const HISTORY_V1_URL = settings.apis.v1_history.url
  18. const HISTORY_V1_BASIC_AUTH = {
  19. user: settings.apis.v1_history.user,
  20. password: settings.apis.v1_history.pass,
  21. }
  22. // BEGIN copy from services/history-v1/storage/lib/blob_store/index.js
  23. const GLOBAL_BLOBS = new Set() // CHANGE FROM SOURCE: only store hashes.
  24. async function loadGlobalBlobs() {
  25. await waitForDb() // CHANGE FROM SOURCE: wait for db before running query.
  26. const blobs = db.projectHistoryGlobalBlobs.find()
  27. for await (const blob of blobs) {
  28. GLOBAL_BLOBS.add(blob._id) // CHANGE FROM SOURCE: only store hashes.
  29. }
  30. }
  31. // END copy from services/history-v1/storage/lib/blob_store/index.js
  32. function isGlobalBlob(hash) {
  33. return GLOBAL_BLOBS.has(hash)
  34. }
  35. function getFilestoreBlobURL(historyId, hash) {
  36. if (GLOBAL_BLOBS.has(hash)) {
  37. return `${settings.apis.filestore.url}/history/global/hash/${hash}`
  38. } else {
  39. return `${settings.apis.filestore.url}/history/project/${historyId}/hash/${hash}`
  40. }
  41. }
  42. async function initializeProject(projectId) {
  43. const body = await fetchJson(`${settings.apis.project_history.url}/project`, {
  44. method: 'POST',
  45. json: { historyId: projectId.toString() },
  46. })
  47. const historyId = body && body.project && body.project.id
  48. if (!historyId) {
  49. throw new OError('project-history did not provide an id', { body })
  50. }
  51. return historyId
  52. }
  53. async function flushProject(projectId) {
  54. try {
  55. await fetchNothing(
  56. `${settings.apis.project_history.url}/project/${projectId}/flush`,
  57. { method: 'POST' }
  58. )
  59. } catch (err) {
  60. throw OError.tag(err, 'failed to flush project to project history', {
  61. projectId,
  62. })
  63. }
  64. }
  65. async function deleteProjectHistory(projectId) {
  66. try {
  67. await fetchNothing(
  68. `${settings.apis.project_history.url}/project/${projectId}`,
  69. { method: 'DELETE' }
  70. )
  71. } catch (err) {
  72. throw OError.tag(err, 'failed to delete project history', {
  73. projectId,
  74. })
  75. }
  76. }
  77. async function resyncProject(projectId, options = {}) {
  78. const body = {}
  79. if (options.force) {
  80. body.force = options.force
  81. }
  82. if (options.origin) {
  83. body.origin = options.origin
  84. }
  85. if (options.historyRangesMigration) {
  86. body.historyRangesMigration = options.historyRangesMigration
  87. }
  88. try {
  89. await fetchNothing(
  90. `${settings.apis.project_history.url}/project/${projectId}/resync`,
  91. {
  92. method: 'POST',
  93. json: body,
  94. signal: AbortSignal.timeout(6 * 60 * 1000),
  95. }
  96. )
  97. } catch (err) {
  98. throw OError.tag(err, 'failed to resync project history', {
  99. projectId,
  100. })
  101. }
  102. }
  103. async function deleteProject(projectId, historyId) {
  104. const tasks = []
  105. tasks.push(_deleteProjectInProjectHistory(projectId))
  106. if (historyId != null) {
  107. tasks.push(_deleteProjectInFullProjectHistory(historyId))
  108. }
  109. await Promise.all(tasks)
  110. await HistoryBackupDeletionHandler.deleteProject(projectId)
  111. }
  112. async function _deleteProjectInProjectHistory(projectId) {
  113. try {
  114. await fetchNothing(
  115. `${settings.apis.project_history.url}/project/${projectId}`,
  116. { method: 'DELETE' }
  117. )
  118. } catch (err) {
  119. throw OError.tag(
  120. err,
  121. 'failed to clear project history in project-history',
  122. { projectId }
  123. )
  124. }
  125. }
  126. async function _deleteProjectInFullProjectHistory(historyId) {
  127. try {
  128. await fetchNothing(`${HISTORY_V1_URL}/projects/${historyId}`, {
  129. method: 'DELETE',
  130. basicAuth: HISTORY_V1_BASIC_AUTH,
  131. })
  132. } catch (err) {
  133. throw OError.tag(err, 'failed to clear project history', { historyId })
  134. }
  135. }
  136. async function uploadBlobFromDisk(historyId, hash, byteLength, fsPath) {
  137. const outStream = fs.createReadStream(fsPath)
  138. const url = `${HISTORY_V1_URL}/projects/${historyId}/blobs/${hash}`
  139. await fetchNothing(url, {
  140. method: 'PUT',
  141. body: outStream,
  142. headers: { 'Content-Length': byteLength }, // add the content length to work around problems with chunked encoding in node 18
  143. signal: AbortSignal.timeout(60 * 1000),
  144. basicAuth: HISTORY_V1_BASIC_AUTH,
  145. })
  146. }
  147. async function copyBlob(sourceHistoryId, targetHistoryId, hash) {
  148. const url = `${HISTORY_V1_URL}/projects/${targetHistoryId}/blobs/${hash}`
  149. await fetchNothing(
  150. `${url}?${new URLSearchParams({ copyFrom: sourceHistoryId })}`,
  151. {
  152. method: 'POST',
  153. basicAuth: HISTORY_V1_BASIC_AUTH,
  154. }
  155. )
  156. }
  157. async function requestBlobWithProjectId(
  158. projectId,
  159. hash,
  160. method = 'GET',
  161. range = ''
  162. ) {
  163. const project = await ProjectGetter.promises.getProject(projectId, {
  164. 'overleaf.history.id': true,
  165. })
  166. return requestBlob(project.overleaf.history.id, hash, method, range)
  167. }
  168. async function requestBlob(historyId, hash, method = 'GET', range = '') {
  169. // Talk to history-v1 directly to avoid streaming via project-history.
  170. const url = new URL(HISTORY_V1_URL)
  171. url.pathname += `/projects/${historyId}/blobs/${hash}`
  172. const opts = { method, headers: { Range: range } }
  173. let stream, response
  174. try {
  175. ;({ stream, response } = await fetchStreamWithResponse(url, {
  176. ...opts,
  177. basicAuth: {
  178. user: settings.apis.v1_history.user,
  179. password: settings.apis.v1_history.pass,
  180. },
  181. }))
  182. } catch (err) {
  183. if (err instanceof RequestFailedError && err.response.status === 404) {
  184. throw new NotFoundError()
  185. } else {
  186. throw err
  187. }
  188. }
  189. Metrics.inc('request_blob', 1, { path: 'history-v1' })
  190. return {
  191. url,
  192. stream,
  193. contentLength: parseInt(response.headers.get('Content-Length'), 10),
  194. contentRange: response.headers.get('Content-Range'),
  195. }
  196. }
  197. /**
  198. * Warning: Don't use this method for large projects. It will eagerly load all
  199. * the history data and apply all operations.
  200. * @param {string} projectId
  201. * @returns Promise<object>
  202. */
  203. async function getCurrentContent(projectId) {
  204. const historyId = await getHistoryId(projectId)
  205. try {
  206. return await fetchJson(
  207. `${HISTORY_V1_URL}/projects/${historyId}/latest/content`,
  208. {
  209. method: 'GET',
  210. basicAuth: HISTORY_V1_BASIC_AUTH,
  211. }
  212. )
  213. } catch (err) {
  214. throw OError.tag(err, 'failed to load project history', { historyId })
  215. }
  216. }
  217. /**
  218. * Warning: Don't use this method for large projects. It will eagerly load all
  219. * the history data and apply all operations.
  220. * @param {string} projectId
  221. * @param {number} version
  222. *
  223. * @returns Promise<object>
  224. */
  225. async function getContentAtVersion(projectId, version) {
  226. const historyId = await getHistoryId(projectId)
  227. try {
  228. return await fetchJson(
  229. `${HISTORY_V1_URL}/projects/${historyId}/versions/${version}/content`,
  230. {
  231. method: 'GET',
  232. basicAuth: HISTORY_V1_BASIC_AUTH,
  233. }
  234. )
  235. } catch (err) {
  236. throw OError.tag(
  237. err,
  238. 'failed to load project history snapshot at version',
  239. { historyId, version }
  240. )
  241. }
  242. }
  243. /**
  244. * Get the latest chunk from history
  245. *
  246. * @param {string} projectId
  247. */
  248. async function getLatestHistory(projectId) {
  249. const historyId = await getHistoryId(projectId)
  250. return await getLatestHistoryWithHistoryId(historyId)
  251. }
  252. /**
  253. * Get the latest chunk from history using already resolved historyId
  254. *
  255. * @param {string} historyId
  256. */
  257. async function getLatestHistoryWithHistoryId(historyId) {
  258. return await fetchJson(
  259. `${HISTORY_V1_URL}/projects/${historyId}/latest/history`,
  260. {
  261. basicAuth: HISTORY_V1_BASIC_AUTH,
  262. }
  263. )
  264. }
  265. /**
  266. * Get history changes since a given version
  267. *
  268. * @param {string} projectId
  269. * @param {object} [opts]
  270. * @param {number} [opts.since] - The start version of changes to get
  271. */
  272. async function getChanges(projectId, opts = {}) {
  273. const historyId = await getHistoryId(projectId)
  274. const url = new URL(`${HISTORY_V1_URL}/projects/${historyId}/changes`)
  275. if (opts.since) {
  276. url.searchParams.set('since', opts.since)
  277. }
  278. return await fetchJson(url, {
  279. basicAuth: HISTORY_V1_BASIC_AUTH,
  280. })
  281. }
  282. async function getHistoryId(projectId) {
  283. const project = await ProjectGetter.promises.getProject(projectId, {
  284. overleaf: true,
  285. })
  286. const historyId = project?.overleaf?.history?.id
  287. if (!historyId) {
  288. throw new OError('project does not have a history id', { projectId })
  289. }
  290. return historyId
  291. }
  292. async function getBlobStats(historyId, blobHashes) {
  293. return await fetchJson(`${HISTORY_V1_URL}/projects/${historyId}/blob-stats`, {
  294. method: 'POST',
  295. basicAuth: HISTORY_V1_BASIC_AUTH,
  296. json: { blobHashes: blobHashes.map(id => id.toString()) },
  297. })
  298. }
  299. async function getProjectBlobStats(historyIds) {
  300. return await fetchJson(`${HISTORY_V1_URL}/projects/blob-stats`, {
  301. method: 'POST',
  302. basicAuth: HISTORY_V1_BASIC_AUTH,
  303. json: { projectIds: historyIds.map(id => id.toString()) },
  304. })
  305. }
  306. async function injectUserDetails(data) {
  307. // data can be either:
  308. // {
  309. // diff: [{
  310. // i: "foo",
  311. // meta: {
  312. // users: ["user_id", v1_user_id, ...]
  313. // ...
  314. // }
  315. // }, ...]
  316. // }
  317. // or
  318. // {
  319. // updates: [{
  320. // pathnames: ["main.tex"]
  321. // meta: {
  322. // users: ["user_id", v1_user_id, ...]
  323. // ...
  324. // },
  325. // ...
  326. // }, ...]
  327. // }
  328. // Either way, the top level key points to an array of objects with a meta.users property
  329. // that we need to replace user_ids with populated user objects.
  330. // Note that some entries in the users arrays may be v1 ids returned by the v1 history
  331. // service. v1 ids will be `numbers`
  332. let userIds = new Set()
  333. let v1UserIds = new Set()
  334. const entries = Array.isArray(data.diff)
  335. ? data.diff
  336. : Array.isArray(data.updates)
  337. ? data.updates
  338. : []
  339. for (const entry of entries) {
  340. for (const user of (entry.meta && entry.meta.users) || []) {
  341. if (typeof user === 'string') {
  342. userIds.add(user)
  343. } else if (typeof user === 'number') {
  344. v1UserIds.add(user)
  345. }
  346. }
  347. }
  348. userIds = Array.from(userIds)
  349. v1UserIds = Array.from(v1UserIds)
  350. const projection = { first_name: 1, last_name: 1, email: 1 }
  351. const usersArray = await UserGetter.promises.getUsers(userIds, projection)
  352. const users = {}
  353. for (const user of usersArray) {
  354. users[user._id.toString()] = _userView(user)
  355. }
  356. projection.overleaf = 1
  357. const v1IdentifiedUsersArray = await UserGetter.promises.getUsersByV1Ids(
  358. v1UserIds,
  359. projection
  360. )
  361. for (const user of v1IdentifiedUsersArray) {
  362. users[user.overleaf.id] = _userView(user)
  363. }
  364. for (const entry of entries) {
  365. if (entry.meta != null) {
  366. entry.meta.users = ((entry.meta && entry.meta.users) || []).map(user => {
  367. if (typeof user === 'string' || typeof user === 'number') {
  368. return users[user]
  369. } else {
  370. return user
  371. }
  372. })
  373. }
  374. }
  375. return data
  376. }
  377. function _userView(user) {
  378. const { _id, first_name: firstName, last_name: lastName, email } = user
  379. return { first_name: firstName, last_name: lastName, email, id: _id }
  380. }
  381. const loadGlobalBlobsPromise = loadGlobalBlobs()
  382. export default {
  383. isGlobalBlob,
  384. getFilestoreBlobURL,
  385. loadGlobalBlobsPromise,
  386. initializeProject: callbackify(initializeProject),
  387. flushProject: callbackify(flushProject),
  388. resyncProject: callbackify(resyncProject),
  389. deleteProject: callbackify(deleteProject),
  390. deleteProjectHistory: callbackify(deleteProjectHistory),
  391. injectUserDetails: callbackify(injectUserDetails),
  392. getCurrentContent: callbackify(getCurrentContent),
  393. uploadBlobFromDisk: callbackify(uploadBlobFromDisk),
  394. copyBlob: callbackify(copyBlob),
  395. requestBlob: callbackify(requestBlob),
  396. requestBlobWithProjectId: callbackify(requestBlobWithProjectId),
  397. getLatestHistory: callbackify(getLatestHistory),
  398. getChanges: callbackify(getChanges),
  399. promises: {
  400. initializeProject,
  401. flushProject,
  402. resyncProject,
  403. deleteProject,
  404. injectUserDetails,
  405. deleteProjectHistory,
  406. getCurrentContent,
  407. getContentAtVersion,
  408. uploadBlobFromDisk,
  409. copyBlob,
  410. requestBlob,
  411. requestBlobWithProjectId,
  412. getLatestHistory,
  413. getChanges,
  414. getProjectBlobStats,
  415. getBlobStats,
  416. getLatestHistoryWithHistoryId,
  417. },
  418. }