HttpController.js 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973
  1. import logger from '@overleaf/logger'
  2. import OError from '@overleaf/o-error'
  3. import * as UpdatesProcessor from './UpdatesProcessor.js'
  4. import * as SummarizedUpdatesManager from './SummarizedUpdatesManager.js'
  5. import * as DiffManager from './DiffManager.js'
  6. import * as HistoryStoreManager from './HistoryStoreManager.js'
  7. import * as WebApiManager from './WebApiManager.js'
  8. import * as SnapshotManager from './SnapshotManager.js'
  9. import * as HealthChecker from './HealthChecker.js'
  10. import * as SyncManager from './SyncManager.js'
  11. import * as ErrorRecorder from './ErrorRecorder.js'
  12. import * as RedisManager from './RedisManager.js'
  13. import * as LabelsManager from './LabelsManager.js'
  14. import * as HistoryApiManager from './HistoryApiManager.js'
  15. import * as RetryManager from './RetryManager.js'
  16. import * as FlushManager from './FlushManager.js'
  17. import Stream, { pipeline } from 'node:stream'
  18. import { fetchNothing, RequestFailedError } from '@overleaf/fetch-utils'
  19. import { z, zz, parseReq } from '@overleaf/validation-tools'
  20. import { IncrementalResponse } from '@overleaf/stream-utils'
  21. const ONE_DAY_IN_SECONDS = 24 * 60 * 60
  22. const cloneProjectSchema = z.object({
  23. body: z.object({
  24. targetProjectId: z.string(),
  25. }),
  26. params: z.object({
  27. project_id: z.string(),
  28. }),
  29. })
  30. export function cloneProject(req, res) {
  31. const {
  32. params: { project_id: sourceProjectId },
  33. body: { targetProjectId },
  34. } = parseReq(req, cloneProjectSchema)
  35. const incrResp = new IncrementalResponse({
  36. res,
  37. timeout: 10 * 60_000 - 5_000,
  38. logger,
  39. label: 'clone history in project-history',
  40. info: { targetProjectId, sourceProjectId },
  41. })
  42. WebApiManager.getHistoryId(targetProjectId, (err, targetHistoryId) => {
  43. if (err) return incrResp.fail(OError.tag(err, 'get target historyId'))
  44. WebApiManager.getHistoryId(sourceProjectId, (err, sourceHistoryId) => {
  45. if (err) return incrResp.fail(OError.tag(err, 'get source historyId'))
  46. incrResp.sendUpdate('cloning full project history data: pending')
  47. HistoryStoreManager.cloneProject(
  48. sourceHistoryId.toString(),
  49. targetHistoryId.toString(),
  50. incrResp.signal(),
  51. (err, stream) => {
  52. if (err) {
  53. incrResp.fail(OError.tag(err, 'clone history-v1 data'))
  54. return
  55. }
  56. // aborted. pipeline() would throw.
  57. if (res.destroyed) {
  58. stream.destroy()
  59. incrResp.fail(new Error('request aborted'))
  60. return
  61. }
  62. // The stream.pipeline callback API does not support options.
  63. Stream.promises.pipeline(stream, res, { end: false }).then(
  64. () => {
  65. incrResp.sendUpdate('clone labels: pending')
  66. LabelsManager.cloneLabels(
  67. sourceProjectId,
  68. targetProjectId,
  69. err => {
  70. if (err) {
  71. incrResp.fail(OError.tag(err, 'clone labels'))
  72. return
  73. }
  74. incrResp.sendUpdate('clone labels: done')
  75. incrResp.sendUpdate('clone resync state: pending')
  76. SyncManager.cloneResyncState(
  77. sourceProjectId,
  78. targetProjectId,
  79. err => {
  80. if (err) {
  81. incrResp.fail(OError.tag(err, 'clone resync state'))
  82. return
  83. }
  84. incrResp.sendUpdate('clone resync state: done')
  85. incrResp.sendUpdate('clone failure record: pending')
  86. ErrorRecorder.cloneFailure(
  87. sourceProjectId,
  88. targetProjectId,
  89. err => {
  90. if (err) {
  91. incrResp.fail(OError.tag(err, 'clone failure'))
  92. return
  93. }
  94. incrResp.sendUpdate('clone failure record: done')
  95. incrResp.sendUpdate('done')
  96. incrResp.end()
  97. }
  98. )
  99. }
  100. )
  101. }
  102. )
  103. },
  104. err => {
  105. incrResp.fail(OError.tag(err, 'stream history-v1 response'))
  106. }
  107. )
  108. }
  109. )
  110. })
  111. })
  112. }
  113. const getProjectBlobSchema = z.object({
  114. params: z.object({
  115. history_id: zz.objectId().or(z.coerce.number()),
  116. hash: z.string(),
  117. }),
  118. })
  119. export function getProjectBlob(req, res, next) {
  120. const { params } = parseReq(req, getProjectBlobSchema)
  121. const historyId = params.history_id
  122. const blobHash = params.hash
  123. HistoryStoreManager.getProjectBlobStream(
  124. historyId,
  125. blobHash,
  126. (err, stream) => {
  127. if (err != null) {
  128. if (err instanceof RequestFailedError && err.response.status === 404) {
  129. return res.status(404).end()
  130. }
  131. return next(OError.tag(err))
  132. }
  133. res.setHeader('Cache-Control', `private, max-age=${ONE_DAY_IN_SECONDS}`)
  134. pipeline(stream, res, err => {
  135. if (err) next(err)
  136. // res.end() is already called via 'end' event by pipeline.
  137. })
  138. }
  139. )
  140. }
  141. export function initializeProject(req, res, next) {
  142. const { historyId } = req.body
  143. HistoryStoreManager.initializeProject(historyId, (error, id) => {
  144. if (error != null) {
  145. return next(OError.tag(error))
  146. }
  147. res.json({ project: { id } })
  148. })
  149. }
  150. const flushProjectSchema = z.object({
  151. params: z.object({
  152. project_id: zz.objectId().or(z.coerce.number()),
  153. }),
  154. query: z.object({
  155. debug: z.stringbool().default(false),
  156. bisect: z.stringbool().default(false),
  157. }),
  158. })
  159. export function flushProject(req, res, next) {
  160. const { query, params } = parseReq(req, flushProjectSchema)
  161. const projectId = params.project_id
  162. if (query.debug) {
  163. logger.debug(
  164. { projectId },
  165. 'compressing project history in single-step mode'
  166. )
  167. UpdatesProcessor.processSingleUpdateForProject(projectId, error => {
  168. if (error != null) {
  169. return next(OError.tag(error))
  170. }
  171. res.sendStatus(204)
  172. })
  173. } else if (query.bisect) {
  174. logger.debug({ projectId }, 'compressing project history in bisect mode')
  175. UpdatesProcessor.processUpdatesForProjectUsingBisect(
  176. projectId,
  177. UpdatesProcessor.REDIS_READ_BATCH_SIZE,
  178. error => {
  179. if (error != null) {
  180. return next(OError.tag(error))
  181. }
  182. res.sendStatus(204)
  183. }
  184. )
  185. } else {
  186. logger.debug({ projectId }, 'compressing project history')
  187. UpdatesProcessor.processUpdatesForProject(projectId, error => {
  188. if (error != null) {
  189. return next(OError.tag(error))
  190. }
  191. res.sendStatus(204)
  192. })
  193. }
  194. }
  195. const dumpProjectSchema = z.object({
  196. params: z.object({
  197. project_id: zz.objectId().or(z.coerce.number()),
  198. }),
  199. query: z.object({
  200. count: z.coerce.number().int().optional(),
  201. }),
  202. })
  203. export function dumpProject(req, res, next) {
  204. const { query, params } = parseReq(req, dumpProjectSchema)
  205. const projectId = params.project_id
  206. const batchSize = query.count || UpdatesProcessor.REDIS_READ_BATCH_SIZE
  207. logger.debug({ projectId }, 'retrieving raw updates')
  208. UpdatesProcessor.getRawUpdates(projectId, batchSize, (error, rawUpdates) => {
  209. if (error != null) {
  210. return next(OError.tag(error))
  211. }
  212. res.json(rawUpdates)
  213. })
  214. }
  215. const flushOldSchema = z.object({
  216. query: z.object({
  217. // flush projects with queued ops older than this
  218. maxAge: z.coerce
  219. .number()
  220. .int()
  221. .default(6 * 3600),
  222. // pause this amount of time between checking queues
  223. queueDelay: z.coerce.number().int().default(100),
  224. // maximum number of queues to check
  225. limit: z.coerce.number().int().default(1000),
  226. // maximum amount of time allowed
  227. timeout: z.coerce
  228. .number()
  229. .int()
  230. .default(60 * 1000),
  231. // whether to run in the background
  232. background: z.stringbool().default(false),
  233. }),
  234. })
  235. export function flushOld(req, res, next) {
  236. const { query } = parseReq(req, flushOldSchema)
  237. const { maxAge, queueDelay, limit, timeout, background } = query
  238. const options = { maxAge, queueDelay, limit, timeout, background }
  239. FlushManager.flushOldOps(options, (error, results) => {
  240. if (error != null) {
  241. return next(OError.tag(error))
  242. }
  243. res.send(results)
  244. })
  245. }
  246. const getDiffSchema = z.object({
  247. params: z.object({
  248. project_id: zz.objectId().or(z.coerce.number()),
  249. }),
  250. query: z.object({
  251. pathname: z.string(),
  252. from: z.coerce.number().int(),
  253. to: z.coerce.number().int(),
  254. }),
  255. })
  256. export function getDiff(req, res, next) {
  257. const { query, params } = parseReq(req, getDiffSchema)
  258. const { pathname, from, to } = query
  259. const projectId = params.project_id
  260. logger.debug({ projectId, pathname, from, to }, 'getting diff')
  261. DiffManager.getDiff(projectId, pathname, from, to, (error, diff) => {
  262. if (error != null) {
  263. return next(OError.tag(error))
  264. }
  265. res.json({ diff })
  266. })
  267. }
  268. const getFileTreeDiffSchema = z.object({
  269. params: z.object({
  270. project_id: zz.objectId().or(z.coerce.number()),
  271. }),
  272. query: z.object({
  273. from: z.coerce.number().int(),
  274. to: z.coerce.number().int(),
  275. }),
  276. })
  277. export function getFileTreeDiff(req, res, next) {
  278. const { query, params } = parseReq(req, getFileTreeDiffSchema)
  279. const { from, to } = query
  280. const projectId = params.project_id
  281. DiffManager.getFileTreeDiff(projectId, from, to, (error, diff) => {
  282. if (error != null) {
  283. return next(OError.tag(error))
  284. }
  285. res.json({ diff })
  286. })
  287. }
  288. const getUpdatesSchema = z.object({
  289. params: z.object({
  290. project_id: zz.objectId().or(z.coerce.number()),
  291. }),
  292. query: z.object({
  293. before: z.coerce.number().int().optional(),
  294. min_count: z.coerce.number().int().optional(),
  295. }),
  296. })
  297. export function getUpdates(req, res, next) {
  298. const { query, params } = parseReq(req, getUpdatesSchema)
  299. const projectId = params.project_id
  300. const { before, min_count: minCount } = query
  301. SummarizedUpdatesManager.getSummarizedProjectUpdates(
  302. projectId,
  303. { before, min_count: minCount },
  304. (error, updates, nextBeforeTimestamp) => {
  305. if (error != null) {
  306. return next(OError.tag(error))
  307. }
  308. for (const update of updates) {
  309. // Sets don't JSONify, so convert to arrays
  310. update.pathnames = Array.from(update.pathnames || []).sort()
  311. }
  312. res.json({
  313. updates,
  314. nextBeforeTimestamp,
  315. })
  316. }
  317. )
  318. }
  319. const getResyncPendingSchema = z.object({
  320. params: z.object({
  321. project_id: zz.objectId(),
  322. }),
  323. })
  324. export function getResyncPending(req, res, next) {
  325. const {
  326. params: { project_id: projectId },
  327. } = parseReq(req, getResyncPendingSchema)
  328. SyncManager.getResyncState(projectId, (err, state) => {
  329. if (err) return next(err)
  330. res.json({
  331. resyncPending: state.isSyncOngoing(),
  332. })
  333. })
  334. }
  335. const getDebugInfoSchema = z.object({
  336. params: z.object({
  337. project_id: zz.objectId(),
  338. }),
  339. })
  340. export function getDebugInfo(req, res, next) {
  341. const {
  342. params: { project_id: projectId },
  343. } = parseReq(req, getDebugInfoSchema)
  344. SyncManager.getResyncState(projectId, (err, state) => {
  345. if (err) return next(err)
  346. ErrorRecorder.getFailureRecord(projectId, (err, failureRecord) => {
  347. if (err) return next(err)
  348. res.json({
  349. failureRecord,
  350. syncState: {
  351. resyncPending: state.isSyncOngoing(),
  352. resyncCount: state.resyncCount,
  353. resyncPendingSince: state.resyncPendingSince,
  354. lastUpdated: state.lastUpdated,
  355. history: state.history,
  356. ...state.toRaw(),
  357. },
  358. })
  359. })
  360. })
  361. }
  362. const latestVersionSchema = z.object({
  363. params: z.object({
  364. project_id: zz.objectId().or(z.coerce.number()),
  365. }),
  366. })
  367. export function latestVersion(req, res, next) {
  368. const { params } = parseReq(req, latestVersionSchema)
  369. const projectId = params.project_id
  370. logger.debug({ projectId }, 'compressing project history and getting version')
  371. UpdatesProcessor.processUpdatesForProject(projectId, error => {
  372. if (error != null) {
  373. return next(OError.tag(error))
  374. }
  375. WebApiManager.getHistoryId(projectId, (error, historyId) => {
  376. if (error != null) {
  377. return next(OError.tag(error))
  378. }
  379. HistoryStoreManager.getMostRecentVersion(
  380. projectId,
  381. historyId,
  382. (error, version, projectStructureAndDocVersions, lastChange) => {
  383. if (error != null) {
  384. return next(OError.tag(error))
  385. }
  386. res.json({
  387. version,
  388. timestamp: lastChange != null ? lastChange.timestamp : undefined,
  389. v2Authors: lastChange != null ? lastChange.v2Authors : undefined,
  390. })
  391. }
  392. )
  393. })
  394. })
  395. }
  396. const getFileSnapshotSchema = z.object({
  397. params: z.object({
  398. project_id: zz.objectId().or(z.coerce.number()),
  399. version: z.coerce.number().int(),
  400. pathname: z.string(),
  401. }),
  402. })
  403. export function getFileSnapshot(req, res, next) {
  404. const { params } = parseReq(req, getFileSnapshotSchema)
  405. const { project_id: projectId, version, pathname } = params
  406. SnapshotManager.getFileSnapshotStream(
  407. projectId,
  408. version,
  409. pathname,
  410. (error, stream) => {
  411. if (error != null) {
  412. return next(OError.tag(error))
  413. }
  414. pipeline(stream, res, err => {
  415. if (err) next(err)
  416. // res.end() is already called via 'end' event by pipeline.
  417. })
  418. }
  419. )
  420. }
  421. const getRangesSnapshotSchema = z.object({
  422. params: z.object({
  423. project_id: zz.objectId().or(z.coerce.number()),
  424. version: z.coerce.number().int(),
  425. pathname: z.string(),
  426. }),
  427. })
  428. export function getRangesSnapshot(req, res, next) {
  429. const { params } = parseReq(req, getRangesSnapshotSchema)
  430. const { project_id: projectId, version, pathname } = params
  431. SnapshotManager.getRangesSnapshot(
  432. projectId,
  433. version,
  434. pathname,
  435. (err, ranges) => {
  436. if (err) {
  437. return next(OError.tag(err))
  438. }
  439. res.json(ranges)
  440. }
  441. )
  442. }
  443. const getFileMetadataSnapshotSchema = z.object({
  444. params: z.object({
  445. project_id: zz.objectId().or(z.coerce.number()),
  446. version: z.coerce.number().int(),
  447. pathname: z.string(),
  448. }),
  449. })
  450. export function getFileMetadataSnapshot(req, res, next) {
  451. const { params } = parseReq(req, getFileMetadataSnapshotSchema)
  452. const { project_id: projectId, version, pathname } = params
  453. SnapshotManager.getFileMetadataSnapshot(
  454. projectId,
  455. version,
  456. pathname,
  457. (err, data) => {
  458. if (err) {
  459. return next(OError.tag(err))
  460. }
  461. res.json(data)
  462. }
  463. )
  464. }
  465. const getLatestSnapshotSchema = z.object({
  466. params: z.object({
  467. project_id: zz.objectId().or(z.coerce.number()),
  468. }),
  469. })
  470. export function getLatestSnapshot(req, res, next) {
  471. const { params } = parseReq(req, getLatestSnapshotSchema)
  472. const { project_id: projectId } = params
  473. WebApiManager.getHistoryId(projectId, (error, historyId) => {
  474. if (error) return next(OError.tag(error))
  475. SnapshotManager.getLatestSnapshot(
  476. projectId,
  477. historyId,
  478. (error, details) => {
  479. if (error != null) {
  480. return next(error)
  481. }
  482. const { snapshot, version } = details
  483. res.json({ snapshot: snapshot.toRaw(), version })
  484. }
  485. )
  486. })
  487. }
  488. const getChangesInChunkSinceSchema = z.object({
  489. params: z.object({
  490. project_id: zz.objectId().or(z.coerce.number()),
  491. }),
  492. query: z.object({
  493. since: z.coerce.number().int().min(0),
  494. }),
  495. })
  496. export function getChangesInChunkSince(req, res, next) {
  497. const { query, params } = parseReq(req, getChangesInChunkSinceSchema)
  498. const { project_id: projectId } = params
  499. const { since } = query
  500. WebApiManager.getHistoryId(projectId, (error, historyId) => {
  501. if (error) return next(OError.tag(error))
  502. SnapshotManager.getChangesInChunkSince(
  503. projectId,
  504. historyId,
  505. since,
  506. (error, details) => {
  507. if (error != null) {
  508. return next(error)
  509. }
  510. const { latestStartVersion, changes } = details
  511. res.json({
  512. latestStartVersion,
  513. changes: changes.map(c => c.toRaw()),
  514. })
  515. }
  516. )
  517. })
  518. }
  519. const getProjectSnapshotSchema = z.object({
  520. params: z.object({
  521. project_id: zz.objectId().or(z.coerce.number()),
  522. version: z.coerce.number().int(),
  523. }),
  524. })
  525. export function getProjectSnapshot(req, res, next) {
  526. const { params } = parseReq(req, getProjectSnapshotSchema)
  527. const { project_id: projectId, version } = params
  528. SnapshotManager.getProjectSnapshot(
  529. projectId,
  530. version,
  531. (error, snapshotData) => {
  532. if (error != null) {
  533. return next(error)
  534. }
  535. res.json(snapshotData)
  536. }
  537. )
  538. }
  539. const getPathsAtVersionSchema = z.object({
  540. params: z.object({
  541. project_id: zz.objectId().or(z.coerce.number()),
  542. version: z.coerce.number().int(),
  543. }),
  544. })
  545. export function getPathsAtVersion(req, res, next) {
  546. const { params } = parseReq(req, getPathsAtVersionSchema)
  547. const { project_id: projectId, version } = params
  548. SnapshotManager.getPathsAtVersion(projectId, version, (error, result) => {
  549. if (error != null) {
  550. return next(error)
  551. }
  552. res.json(result)
  553. })
  554. }
  555. export function healthCheck(req, res) {
  556. HealthChecker.check(err => {
  557. if (err != null) {
  558. logger.err({ err }, 'error performing health check')
  559. res.sendStatus(500)
  560. } else {
  561. res.sendStatus(200)
  562. }
  563. })
  564. }
  565. export function checkLock(req, res) {
  566. HealthChecker.checkLock(err => {
  567. if (err != null) {
  568. logger.err({ err }, 'error performing lock check')
  569. res.sendStatus(500)
  570. } else {
  571. res.sendStatus(200)
  572. }
  573. })
  574. }
  575. const resyncProjectSchema = z.object({
  576. params: z.object({
  577. project_id: zz.objectId().or(z.coerce.number()),
  578. }),
  579. query: z.object({
  580. force: z.stringbool().default(false),
  581. }),
  582. body: z.object({
  583. force: z.boolean().default(false),
  584. origin: z
  585. .object({
  586. kind: z.string(),
  587. })
  588. .optional(),
  589. historyRangesMigration: z.enum(['forwards', 'backwards']).optional(),
  590. }),
  591. })
  592. export function resyncProject(req, res, next) {
  593. const { query, params, body } = parseReq(req, resyncProjectSchema)
  594. const projectId = params.project_id
  595. const options = {}
  596. if (body.origin) {
  597. options.origin = body.origin
  598. }
  599. if (body.historyRangesMigration) {
  600. options.historyRangesMigration = body.historyRangesMigration
  601. }
  602. if (query.force || body.force) {
  603. // this will delete the queue and clear the sync state
  604. // use if the project is completely broken
  605. SyncManager.startHardResync(projectId, options, error => {
  606. if (error != null) {
  607. return next(error)
  608. }
  609. // flush the sync operations
  610. UpdatesProcessor.processUpdatesForProject(projectId, error => {
  611. if (error != null) {
  612. return next(error)
  613. }
  614. res.sendStatus(204)
  615. })
  616. })
  617. } else {
  618. SyncManager.startResync(projectId, options, error => {
  619. if (error != null) {
  620. return next(error)
  621. }
  622. // flush the sync operations
  623. UpdatesProcessor.processUpdatesForProject(projectId, error => {
  624. if (error != null) {
  625. return next(error)
  626. }
  627. res.sendStatus(204)
  628. })
  629. })
  630. }
  631. }
  632. const forceDebugProjectSchema = z.object({
  633. params: z.object({
  634. project_id: zz.objectId().or(z.coerce.number()),
  635. }),
  636. query: z.object({
  637. clear: z.stringbool().default(false),
  638. }),
  639. })
  640. export function forceDebugProject(req, res, next) {
  641. const { query, params } = parseReq(req, forceDebugProjectSchema)
  642. const projectId = params.project_id
  643. // set the debug flag to true unless we see ?clear=true
  644. const state = !query.clear
  645. ErrorRecorder.setForceDebug(projectId, state, error => {
  646. if (error != null) {
  647. return next(error)
  648. }
  649. // display the failure record to help debugging
  650. ErrorRecorder.getFailureRecord(projectId, (error, result) => {
  651. if (error != null) {
  652. return next(error)
  653. }
  654. res.send(result)
  655. })
  656. })
  657. }
  658. export function getFailures(req, res, next) {
  659. ErrorRecorder.getFailures((error, result) => {
  660. if (error != null) {
  661. return next(error)
  662. }
  663. res.send({ failures: result })
  664. })
  665. }
  666. export function getQueueCounts(req, res, next) {
  667. RedisManager.getProjectIdsWithHistoryOpsCount((err, queuedProjectsCount) => {
  668. if (err != null) {
  669. return next(err)
  670. }
  671. res.send({ queuedProjects: queuedProjectsCount })
  672. })
  673. }
  674. const getLabelsSchema = z.object({
  675. params: z.object({
  676. project_id: zz.objectId().or(z.coerce.number()),
  677. }),
  678. })
  679. export function getLabels(req, res, next) {
  680. const { params } = parseReq(req, getLabelsSchema)
  681. const projectId = params.project_id
  682. HistoryApiManager.shouldUseProjectHistory(
  683. projectId,
  684. (error, shouldUseProjectHistory) => {
  685. if (error != null) {
  686. return next(error)
  687. }
  688. if (shouldUseProjectHistory) {
  689. LabelsManager.getLabels(projectId, (error, labels) => {
  690. if (error != null) {
  691. return next(error)
  692. }
  693. res.json(labels)
  694. })
  695. } else {
  696. res.sendStatus(409)
  697. }
  698. }
  699. )
  700. }
  701. const createLabelSchema = z.object({
  702. params: z.object({
  703. project_id: zz.objectId().or(z.coerce.number()),
  704. user_id: zz.objectId().optional(),
  705. }),
  706. body: z.object({
  707. version: z.number().int(),
  708. comment: z.string(),
  709. created_at: z.string().optional(),
  710. validate_exists: z.boolean().default(true),
  711. user_id: zz.objectId().nullable().optional(),
  712. }),
  713. })
  714. export function createLabel(req, res, next) {
  715. const { params, body } = parseReq(req, createLabelSchema)
  716. const { project_id: projectId, user_id: userIdParam } = params
  717. const {
  718. version,
  719. comment,
  720. user_id: userIdBody,
  721. created_at: createdAt,
  722. validate_exists: validateExists,
  723. } = body
  724. // Temporarily looking up both params and body while rolling out changes
  725. // in the router path - https://github.com/overleaf/internal/pull/20200
  726. const userId = userIdParam || userIdBody
  727. HistoryApiManager.shouldUseProjectHistory(
  728. projectId,
  729. (error, shouldUseProjectHistory) => {
  730. if (error != null) {
  731. return next(error)
  732. }
  733. if (shouldUseProjectHistory) {
  734. LabelsManager.createLabel(
  735. projectId,
  736. userId,
  737. version,
  738. comment,
  739. createdAt,
  740. validateExists,
  741. (error, label) => {
  742. if (error != null) {
  743. return next(error)
  744. }
  745. res.json(label)
  746. }
  747. )
  748. } else {
  749. logger.error(
  750. {
  751. projectId,
  752. userId,
  753. version,
  754. comment,
  755. createdAt,
  756. validateExists,
  757. },
  758. 'not using v2 history'
  759. )
  760. res.sendStatus(409)
  761. }
  762. }
  763. )
  764. }
  765. /**
  766. * This will delete a label if it is owned by the current user. If you wish to
  767. * delete a label regardless of the current user, then use `deleteLabel` instead.
  768. */
  769. const deleteLabelForUserSchema = z.object({
  770. params: z.object({
  771. project_id: zz.objectId().or(z.coerce.number()),
  772. user_id: zz.objectId(),
  773. label_id: zz.objectId(),
  774. }),
  775. })
  776. export function deleteLabelForUser(req, res, next) {
  777. const { params } = parseReq(req, deleteLabelForUserSchema)
  778. const { project_id: projectId, user_id: userId, label_id: labelId } = params
  779. LabelsManager.deleteLabelForUser(projectId, userId, labelId, error => {
  780. if (error != null) {
  781. return next(error)
  782. }
  783. res.sendStatus(204)
  784. })
  785. }
  786. const deleteLabelSchema = z.object({
  787. params: z.object({
  788. project_id: zz.objectId().or(z.coerce.number()),
  789. label_id: zz.objectId(),
  790. }),
  791. })
  792. export function deleteLabel(req, res, next) {
  793. const { params } = parseReq(req, deleteLabelSchema)
  794. const { project_id: projectId, label_id: labelId } = params
  795. LabelsManager.deleteLabel(projectId, labelId, error => {
  796. if (error != null) {
  797. return next(error)
  798. }
  799. res.sendStatus(204)
  800. })
  801. }
  802. const retryFailuresSchema = z.object({
  803. query: z.object({
  804. failureType: z.enum(['soft', 'hard']).optional(),
  805. // bail out after this time limit
  806. timeout: z.coerce.number().int().default(300),
  807. // maximum number of projects to check
  808. limit: z.coerce.number().int().default(100),
  809. callbackUrl: z.string().optional(),
  810. }),
  811. })
  812. export function retryFailures(req, res, next) {
  813. const { query } = parseReq(req, retryFailuresSchema)
  814. const { failureType, timeout, limit, callbackUrl } = query
  815. if (callbackUrl) {
  816. // send response but run in background when callbackUrl provided
  817. res.send({ retryStatus: 'running retryFailures in background' })
  818. }
  819. RetryManager.retryFailures(
  820. { failureType, timeout, limit },
  821. (error, result) => {
  822. if (callbackUrl) {
  823. // if present, notify the callbackUrl on success
  824. if (!error) {
  825. // Needs Node 12
  826. // const callbackHeaders = Object.fromEntries(Object.entries(req.headers || {}).filter(([k,v]) => k.match(/^X-CALLBACK-/i)))
  827. const callbackHeaders = {}
  828. for (const key of Object.getOwnPropertyNames(
  829. req.headers || {}
  830. ).filter(key => key.match(/^X-CALLBACK-/i))) {
  831. const found = key.match(/^X-CALLBACK-(.*)/i)
  832. callbackHeaders[found[1]] = req.headers[key]
  833. }
  834. fetchNothing(callbackUrl, { headers: callbackHeaders }).catch(err => {
  835. logger.warn({ err }, 'failed to ping callback url')
  836. })
  837. }
  838. } else {
  839. if (error != null) {
  840. return next(error)
  841. }
  842. res.send({ retryStatus: result })
  843. }
  844. }
  845. )
  846. }
  847. const transferLabelsSchema = z.object({
  848. params: z.object({
  849. from_user: zz.objectId(),
  850. to_user: zz.objectId(),
  851. }),
  852. })
  853. export function transferLabels(req, res, next) {
  854. const { params } = parseReq(req, transferLabelsSchema)
  855. const { from_user: fromUser, to_user: toUser } = params
  856. LabelsManager.transferLabels(fromUser, toUser, error => {
  857. if (error != null) {
  858. return next(error)
  859. }
  860. res.sendStatus(204)
  861. })
  862. }
  863. const deleteProjectSchema = z.object({
  864. params: z.object({
  865. project_id: zz.objectId().or(z.coerce.number()),
  866. }),
  867. })
  868. export function deleteProject(req, res, next) {
  869. const { params } = parseReq(req, deleteProjectSchema)
  870. const { project_id: projectId } = params
  871. // clear the timestamp before clearing the queue,
  872. // because the queue location is used in the migration
  873. RedisManager.clearFirstOpTimestamp(projectId, err => {
  874. if (err) {
  875. return next(err)
  876. }
  877. RedisManager.clearCachedHistoryId(projectId, err => {
  878. if (err) {
  879. return next(err)
  880. }
  881. RedisManager.destroyDocUpdatesQueue(projectId, err => {
  882. if (err) {
  883. return next(err)
  884. }
  885. SyncManager.clearResyncState(projectId, err => {
  886. if (err) {
  887. return next(err)
  888. }
  889. ErrorRecorder.clearError(projectId, err => {
  890. if (err) {
  891. return next(err)
  892. }
  893. res.sendStatus(204)
  894. })
  895. })
  896. })
  897. })
  898. })
  899. }