HttpController.js 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995
  1. import logger from '@overleaf/logger'
  2. import OError from '@overleaf/o-error'
  3. import * as UpdatesProcessor from './UpdatesProcessor.js'
  4. import * as SummarizedUpdatesManager from './SummarizedUpdatesManager.js'
  5. import * as DiffManager from './DiffManager.js'
  6. import * as HistoryStoreManager from './HistoryStoreManager.js'
  7. import * as WebApiManager from './WebApiManager.js'
  8. import * as SnapshotManager from './SnapshotManager.js'
  9. import * as HealthChecker from './HealthChecker.js'
  10. import * as SyncManager from './SyncManager.js'
  11. import * as ErrorRecorder from './ErrorRecorder.js'
  12. import * as RedisManager from './RedisManager.js'
  13. import * as LabelsManager from './LabelsManager.js'
  14. import * as HistoryApiManager from './HistoryApiManager.js'
  15. import * as RetryManager from './RetryManager.js'
  16. import * as FlushManager from './FlushManager.js'
  17. import Stream, { pipeline } from 'node:stream'
  18. import { fetchNothing, RequestFailedError } from '@overleaf/fetch-utils'
  19. import { z, zz, parseReq } from '@overleaf/validation-tools'
  20. import { IncrementalResponse } from '@overleaf/stream-utils'
  21. const ONE_DAY_IN_SECONDS = 24 * 60 * 60
  22. const cloneProjectSchema = z.object({
  23. body: z.object({
  24. targetProjectId: z.string(),
  25. }),
  26. params: z.object({
  27. project_id: z.string(),
  28. }),
  29. })
  30. export function cloneProject(req, res) {
  31. const {
  32. params: { project_id: sourceProjectId },
  33. body: { targetProjectId },
  34. } = parseReq(req, cloneProjectSchema)
  35. const incrResp = new IncrementalResponse({
  36. res,
  37. timeout: 10 * 60_000 - 5_000,
  38. logger,
  39. label: 'clone history in project-history',
  40. info: { targetProjectId, sourceProjectId },
  41. })
  42. incrResp.sendUpdate('best effort history flush: pending')
  43. UpdatesProcessor.processUpdatesForProject(sourceProjectId, err => {
  44. if (err) {
  45. logger.warn(
  46. { err, sourceProjectId },
  47. 'failed to flush during history clone'
  48. )
  49. incrResp.sendUpdate(
  50. 'best effort history flush: failed, a resync will be required'
  51. )
  52. } else {
  53. incrResp.sendUpdate('best effort history flush: done')
  54. }
  55. WebApiManager.getHistoryId(targetProjectId, (err, targetHistoryId) => {
  56. if (err) return incrResp.fail(OError.tag(err, 'get target historyId'))
  57. WebApiManager.getHistoryId(sourceProjectId, (err, sourceHistoryId) => {
  58. if (err) return incrResp.fail(OError.tag(err, 'get source historyId'))
  59. incrResp.sendUpdate('cloning full project history data: pending')
  60. HistoryStoreManager.cloneProject(
  61. sourceHistoryId.toString(),
  62. targetHistoryId.toString(),
  63. incrResp.signal(),
  64. (err, stream) => {
  65. if (err) {
  66. incrResp.fail(OError.tag(err, 'clone history-v1 data'))
  67. return
  68. }
  69. // aborted. pipeline() would throw.
  70. if (res.destroyed) {
  71. stream.destroy()
  72. incrResp.fail(new Error('request aborted'))
  73. return
  74. }
  75. // The stream.pipeline callback API does not support options.
  76. Stream.promises.pipeline(stream, res, { end: false }).then(
  77. () => {
  78. incrResp.sendUpdate('clone labels: pending')
  79. LabelsManager.cloneLabels(
  80. sourceProjectId,
  81. targetProjectId,
  82. err => {
  83. if (err) {
  84. incrResp.fail(OError.tag(err, 'clone labels'))
  85. return
  86. }
  87. incrResp.sendUpdate('clone labels: done')
  88. incrResp.sendUpdate('clone resync state: pending')
  89. SyncManager.cloneResyncState(
  90. sourceProjectId,
  91. targetProjectId,
  92. err => {
  93. if (err) {
  94. incrResp.fail(OError.tag(err, 'clone resync state'))
  95. return
  96. }
  97. incrResp.sendUpdate('clone resync state: done')
  98. incrResp.sendUpdate('clone failure record: pending')
  99. ErrorRecorder.cloneFailure(
  100. sourceProjectId,
  101. targetProjectId,
  102. err => {
  103. if (err) {
  104. incrResp.fail(OError.tag(err, 'clone failure'))
  105. return
  106. }
  107. incrResp.sendUpdate('clone failure record: done')
  108. incrResp.sendUpdate('done')
  109. incrResp.end()
  110. }
  111. )
  112. }
  113. )
  114. }
  115. )
  116. },
  117. err => {
  118. incrResp.fail(OError.tag(err, 'stream history-v1 response'))
  119. }
  120. )
  121. }
  122. )
  123. })
  124. })
  125. })
  126. }
  127. const getProjectBlobSchema = z.object({
  128. params: z.object({
  129. history_id: zz.objectId().or(z.coerce.number()),
  130. hash: z.string(),
  131. }),
  132. })
  133. export function getProjectBlob(req, res, next) {
  134. const { params } = parseReq(req, getProjectBlobSchema)
  135. const historyId = params.history_id
  136. const blobHash = params.hash
  137. HistoryStoreManager.getProjectBlobStream(
  138. historyId,
  139. blobHash,
  140. (err, stream) => {
  141. if (err != null) {
  142. if (err instanceof RequestFailedError && err.response.status === 404) {
  143. return res.status(404).end()
  144. }
  145. return next(OError.tag(err))
  146. }
  147. res.setHeader('Cache-Control', `private, max-age=${ONE_DAY_IN_SECONDS}`)
  148. pipeline(stream, res, err => {
  149. if (err) next(err)
  150. // res.end() is already called via 'end' event by pipeline.
  151. })
  152. }
  153. )
  154. }
  155. export function initializeProject(req, res, next) {
  156. const { historyId } = req.body
  157. HistoryStoreManager.initializeProject(historyId, (error, id) => {
  158. if (error != null) {
  159. return next(OError.tag(error))
  160. }
  161. res.json({ project: { id } })
  162. })
  163. }
  164. const flushProjectSchema = z.object({
  165. params: z.object({
  166. project_id: zz.objectId().or(z.coerce.number()),
  167. }),
  168. query: z.object({
  169. debug: z.stringbool().default(false),
  170. bisect: z.stringbool().default(false),
  171. }),
  172. })
  173. export function flushProject(req, res, next) {
  174. const { query, params } = parseReq(req, flushProjectSchema)
  175. const projectId = params.project_id
  176. if (query.debug) {
  177. logger.debug(
  178. { projectId },
  179. 'compressing project history in single-step mode'
  180. )
  181. UpdatesProcessor.processSingleUpdateForProject(projectId, error => {
  182. if (error != null) {
  183. return next(OError.tag(error))
  184. }
  185. res.sendStatus(204)
  186. })
  187. } else if (query.bisect) {
  188. logger.debug({ projectId }, 'compressing project history in bisect mode')
  189. UpdatesProcessor.processUpdatesForProjectUsingBisect(
  190. projectId,
  191. UpdatesProcessor.REDIS_READ_BATCH_SIZE,
  192. error => {
  193. if (error != null) {
  194. return next(OError.tag(error))
  195. }
  196. res.sendStatus(204)
  197. }
  198. )
  199. } else {
  200. logger.debug({ projectId }, 'compressing project history')
  201. UpdatesProcessor.processUpdatesForProject(projectId, error => {
  202. if (error != null) {
  203. return next(OError.tag(error))
  204. }
  205. res.sendStatus(204)
  206. })
  207. }
  208. }
  209. const dumpProjectSchema = z.object({
  210. params: z.object({
  211. project_id: zz.objectId().or(z.coerce.number()),
  212. }),
  213. query: z.object({
  214. count: z.coerce.number().int().optional(),
  215. }),
  216. })
  217. export function dumpProject(req, res, next) {
  218. const { query, params } = parseReq(req, dumpProjectSchema)
  219. const projectId = params.project_id
  220. const batchSize = query.count || UpdatesProcessor.REDIS_READ_BATCH_SIZE
  221. logger.debug({ projectId }, 'retrieving raw updates')
  222. UpdatesProcessor.getRawUpdates(projectId, batchSize, (error, rawUpdates) => {
  223. if (error != null) {
  224. return next(OError.tag(error))
  225. }
  226. res.json(rawUpdates)
  227. })
  228. }
  229. const flushOldSchema = z.object({
  230. query: z.object({
  231. // flush projects with queued ops older than this
  232. maxAge: z.coerce
  233. .number()
  234. .int()
  235. .default(6 * 3600),
  236. // pause this amount of time between checking queues
  237. queueDelay: z.coerce.number().int().default(100),
  238. // maximum number of queues to check
  239. limit: z.coerce.number().int().default(1000),
  240. // maximum amount of time allowed
  241. timeout: z.coerce
  242. .number()
  243. .int()
  244. .default(60 * 1000),
  245. // whether to run in the background
  246. background: z.stringbool().default(false),
  247. }),
  248. })
  249. export function flushOld(req, res, next) {
  250. const { query } = parseReq(req, flushOldSchema)
  251. const { maxAge, queueDelay, limit, timeout, background } = query
  252. const options = { maxAge, queueDelay, limit, timeout, background }
  253. FlushManager.flushOldOps(options, (error, results) => {
  254. if (error != null) {
  255. return next(OError.tag(error))
  256. }
  257. res.send(results)
  258. })
  259. }
  260. const getDiffSchema = z.object({
  261. params: z.object({
  262. project_id: zz.objectId().or(z.coerce.number()),
  263. }),
  264. query: z.object({
  265. pathname: z.string(),
  266. from: z.coerce.number().int(),
  267. to: z.coerce.number().int(),
  268. }),
  269. })
  270. export function getDiff(req, res, next) {
  271. const { query, params } = parseReq(req, getDiffSchema)
  272. const { pathname, from, to } = query
  273. const projectId = params.project_id
  274. logger.debug({ projectId, pathname, from, to }, 'getting diff')
  275. DiffManager.getDiff(projectId, pathname, from, to, (error, diff) => {
  276. if (error != null) {
  277. return next(OError.tag(error))
  278. }
  279. res.json({ diff })
  280. })
  281. }
  282. const getFileTreeDiffSchema = z.object({
  283. params: z.object({
  284. project_id: zz.objectId().or(z.coerce.number()),
  285. }),
  286. query: z.object({
  287. from: z.coerce.number().int(),
  288. to: z.coerce.number().int(),
  289. }),
  290. })
  291. export function getFileTreeDiff(req, res, next) {
  292. const { query, params } = parseReq(req, getFileTreeDiffSchema)
  293. const { from, to } = query
  294. const projectId = params.project_id
  295. DiffManager.getFileTreeDiff(projectId, from, to, (error, diff) => {
  296. if (error != null) {
  297. return next(OError.tag(error))
  298. }
  299. res.json({ diff })
  300. })
  301. }
  302. const getUpdatesSchema = z.object({
  303. params: z.object({
  304. project_id: zz.objectId().or(z.coerce.number()),
  305. }),
  306. query: z.object({
  307. before: z.coerce.number().int().optional(),
  308. min_count: z.coerce.number().int().optional(),
  309. }),
  310. })
  311. export function getUpdates(req, res, next) {
  312. const { query, params } = parseReq(req, getUpdatesSchema)
  313. const projectId = params.project_id
  314. const { before, min_count: minCount } = query
  315. SummarizedUpdatesManager.getSummarizedProjectUpdates(
  316. projectId,
  317. { before, min_count: minCount },
  318. (error, updates, nextBeforeTimestamp) => {
  319. if (error != null) {
  320. return next(OError.tag(error))
  321. }
  322. for (const update of updates) {
  323. // Sets don't JSONify, so convert to arrays
  324. update.pathnames = Array.from(update.pathnames || []).sort()
  325. }
  326. res.json({
  327. updates,
  328. nextBeforeTimestamp,
  329. })
  330. }
  331. )
  332. }
  333. const getResyncPendingSchema = z.object({
  334. params: z.object({
  335. project_id: zz.objectId(),
  336. }),
  337. })
  338. export function getResyncPending(req, res, next) {
  339. const {
  340. params: { project_id: projectId },
  341. } = parseReq(req, getResyncPendingSchema)
  342. SyncManager.getResyncState(projectId, (err, state) => {
  343. if (err) return next(err)
  344. res.json({
  345. resyncPending: state.isSyncOngoing(),
  346. })
  347. })
  348. }
  349. const getDebugInfoSchema = z.object({
  350. params: z.object({
  351. project_id: zz.objectId(),
  352. }),
  353. })
  354. export function getDebugInfo(req, res, next) {
  355. const {
  356. params: { project_id: projectId },
  357. } = parseReq(req, getDebugInfoSchema)
  358. SyncManager.getResyncState(projectId, (err, state) => {
  359. if (err) return next(err)
  360. ErrorRecorder.getFailureRecord(projectId, (err, failureRecord) => {
  361. if (err) return next(err)
  362. res.json({
  363. failureRecord,
  364. syncState: {
  365. resyncPending: state.isSyncOngoing(),
  366. resyncCount: state.resyncCount,
  367. resyncPendingSince: state.resyncPendingSince,
  368. lastUpdated: state.lastUpdated,
  369. history: state.history,
  370. ...state.toRaw(),
  371. },
  372. })
  373. })
  374. })
  375. }
  376. const latestVersionSchema = z.object({
  377. params: z.object({
  378. project_id: zz.objectId().or(z.coerce.number()),
  379. }),
  380. })
  381. export function latestVersion(req, res, next) {
  382. const { params } = parseReq(req, latestVersionSchema)
  383. const projectId = params.project_id
  384. logger.debug({ projectId }, 'compressing project history and getting version')
  385. UpdatesProcessor.processUpdatesForProject(projectId, error => {
  386. if (error != null) {
  387. return next(OError.tag(error))
  388. }
  389. WebApiManager.getHistoryId(projectId, (error, historyId) => {
  390. if (error != null) {
  391. return next(OError.tag(error))
  392. }
  393. HistoryStoreManager.getMostRecentVersion(
  394. projectId,
  395. historyId,
  396. (error, version, projectStructureAndDocVersions, lastChange) => {
  397. if (error != null) {
  398. return next(OError.tag(error))
  399. }
  400. res.json({
  401. version,
  402. timestamp: lastChange != null ? lastChange.timestamp : undefined,
  403. v2Authors: lastChange != null ? lastChange.v2Authors : undefined,
  404. })
  405. }
  406. )
  407. })
  408. })
  409. }
  410. const getFileSnapshotSchema = z.object({
  411. params: z.object({
  412. project_id: zz.objectId().or(z.coerce.number()),
  413. version: z.coerce.number().int(),
  414. pathname: z.string(),
  415. }),
  416. })
  417. export function getFileSnapshot(req, res, next) {
  418. const { params } = parseReq(req, getFileSnapshotSchema)
  419. const { project_id: projectId, version, pathname } = params
  420. SnapshotManager.getFileSnapshotStream(
  421. projectId,
  422. version,
  423. pathname,
  424. (error, stream) => {
  425. if (error != null) {
  426. return next(OError.tag(error))
  427. }
  428. pipeline(stream, res, err => {
  429. if (err) next(err)
  430. // res.end() is already called via 'end' event by pipeline.
  431. })
  432. }
  433. )
  434. }
  435. const getRangesSnapshotSchema = z.object({
  436. params: z.object({
  437. project_id: zz.objectId().or(z.coerce.number()),
  438. version: z.coerce.number().int(),
  439. pathname: z.string(),
  440. }),
  441. })
  442. export function getRangesSnapshot(req, res, next) {
  443. const { params } = parseReq(req, getRangesSnapshotSchema)
  444. const { project_id: projectId, version, pathname } = params
  445. SnapshotManager.getRangesSnapshot(
  446. projectId,
  447. version,
  448. pathname,
  449. (err, ranges) => {
  450. if (err) {
  451. return next(OError.tag(err))
  452. }
  453. res.json(ranges)
  454. }
  455. )
  456. }
  457. const getFileMetadataSnapshotSchema = z.object({
  458. params: z.object({
  459. project_id: zz.objectId().or(z.coerce.number()),
  460. version: z.coerce.number().int(),
  461. pathname: z.string(),
  462. }),
  463. })
  464. export function getFileMetadataSnapshot(req, res, next) {
  465. const { params } = parseReq(req, getFileMetadataSnapshotSchema)
  466. const { project_id: projectId, version, pathname } = params
  467. SnapshotManager.getFileMetadataSnapshot(
  468. projectId,
  469. version,
  470. pathname,
  471. (err, data) => {
  472. if (err) {
  473. return next(OError.tag(err))
  474. }
  475. res.json(data)
  476. }
  477. )
  478. }
  479. const getLatestSnapshotSchema = z.object({
  480. params: z.object({
  481. project_id: zz.objectId().or(z.coerce.number()),
  482. }),
  483. })
  484. export function getLatestSnapshot(req, res, next) {
  485. const { params } = parseReq(req, getLatestSnapshotSchema)
  486. const { project_id: projectId } = params
  487. WebApiManager.getHistoryId(projectId, (error, historyId) => {
  488. if (error) return next(OError.tag(error))
  489. SnapshotManager.getLatestSnapshot(
  490. projectId,
  491. historyId,
  492. (error, details) => {
  493. if (error != null) {
  494. return next(error)
  495. }
  496. const { snapshot, version } = details
  497. res.json({ snapshot: snapshot.toRaw(), version })
  498. }
  499. )
  500. })
  501. }
  502. const getChangesInChunkSinceSchema = z.object({
  503. params: z.object({
  504. project_id: zz.objectId().or(z.coerce.number()),
  505. }),
  506. query: z.object({
  507. since: z.coerce.number().int().min(0),
  508. }),
  509. })
  510. export function getChangesInChunkSince(req, res, next) {
  511. const { query, params } = parseReq(req, getChangesInChunkSinceSchema)
  512. const { project_id: projectId } = params
  513. const { since } = query
  514. WebApiManager.getHistoryId(projectId, (error, historyId) => {
  515. if (error) return next(OError.tag(error))
  516. SnapshotManager.getChangesInChunkSince(
  517. projectId,
  518. historyId,
  519. since,
  520. (error, details) => {
  521. if (error != null) {
  522. return next(error)
  523. }
  524. const { latestStartVersion, changes } = details
  525. res.json({
  526. latestStartVersion,
  527. changes: changes.map(c => c.toRaw()),
  528. })
  529. }
  530. )
  531. })
  532. }
  533. const getProjectSnapshotSchema = z.object({
  534. params: z.object({
  535. project_id: zz.objectId().or(z.coerce.number()),
  536. version: z.coerce.number().int(),
  537. }),
  538. })
  539. export function getProjectSnapshot(req, res, next) {
  540. const { params } = parseReq(req, getProjectSnapshotSchema)
  541. const { project_id: projectId, version } = params
  542. SnapshotManager.getProjectSnapshot(
  543. projectId,
  544. version,
  545. (error, snapshotData) => {
  546. if (error != null) {
  547. return next(error)
  548. }
  549. res.json(snapshotData)
  550. }
  551. )
  552. }
  553. const getPathsAtVersionSchema = z.object({
  554. params: z.object({
  555. project_id: zz.objectId().or(z.coerce.number()),
  556. version: z.coerce.number().int(),
  557. }),
  558. })
  559. export function getPathsAtVersion(req, res, next) {
  560. const { params } = parseReq(req, getPathsAtVersionSchema)
  561. const { project_id: projectId, version } = params
  562. SnapshotManager.getPathsAtVersion(projectId, version, (error, result) => {
  563. if (error != null) {
  564. return next(error)
  565. }
  566. res.json(result)
  567. })
  568. }
  569. export function healthCheck(req, res) {
  570. HealthChecker.check(err => {
  571. if (err != null) {
  572. logger.err({ err }, 'error performing health check')
  573. res.sendStatus(500)
  574. } else {
  575. res.sendStatus(200)
  576. }
  577. })
  578. }
  579. export function checkLock(req, res) {
  580. HealthChecker.checkLock(err => {
  581. if (err != null) {
  582. logger.err({ err }, 'error performing lock check')
  583. res.sendStatus(500)
  584. } else {
  585. res.sendStatus(200)
  586. }
  587. })
  588. }
  589. const resyncProjectSchema = z.object({
  590. params: z.object({
  591. project_id: zz.objectId().or(z.coerce.number()),
  592. }),
  593. query: z.object({
  594. force: z.stringbool().default(false),
  595. }),
  596. body: z.object({
  597. force: z.boolean().default(false),
  598. origin: z
  599. .object({
  600. kind: z.string(),
  601. })
  602. .optional(),
  603. historyRangesMigration: z.enum(['forwards', 'backwards']).optional(),
  604. }),
  605. })
  606. export function resyncProject(req, res, next) {
  607. const { query, params, body } = parseReq(req, resyncProjectSchema)
  608. const projectId = params.project_id
  609. const options = {}
  610. if (body.origin) {
  611. options.origin = body.origin
  612. }
  613. if (body.historyRangesMigration) {
  614. options.historyRangesMigration = body.historyRangesMigration
  615. }
  616. if (query.force || body.force) {
  617. // this will delete the queue and clear the sync state
  618. // use if the project is completely broken
  619. SyncManager.startHardResync(projectId, options, error => {
  620. if (error != null) {
  621. return next(error)
  622. }
  623. // flush the sync operations
  624. UpdatesProcessor.processUpdatesForProject(projectId, error => {
  625. if (error != null) {
  626. return next(error)
  627. }
  628. res.sendStatus(204)
  629. })
  630. })
  631. } else {
  632. SyncManager.startResync(projectId, options, error => {
  633. if (error != null) {
  634. return next(error)
  635. }
  636. // flush the sync operations
  637. UpdatesProcessor.processUpdatesForProject(projectId, error => {
  638. if (error != null) {
  639. return next(error)
  640. }
  641. res.sendStatus(204)
  642. })
  643. })
  644. }
  645. }
  646. const forceDebugProjectSchema = z.object({
  647. params: z.object({
  648. project_id: zz.objectId().or(z.coerce.number()),
  649. }),
  650. query: z.object({
  651. clear: z.stringbool().default(false),
  652. }),
  653. })
  654. export function forceDebugProject(req, res, next) {
  655. const { query, params } = parseReq(req, forceDebugProjectSchema)
  656. const projectId = params.project_id
  657. // set the debug flag to true unless we see ?clear=true
  658. const state = !query.clear
  659. ErrorRecorder.setForceDebug(projectId, state, error => {
  660. if (error != null) {
  661. return next(error)
  662. }
  663. // display the failure record to help debugging
  664. ErrorRecorder.getFailureRecord(projectId, (error, result) => {
  665. if (error != null) {
  666. return next(error)
  667. }
  668. res.send(result)
  669. })
  670. })
  671. }
  672. export function getFailures(req, res, next) {
  673. ErrorRecorder.getFailures((error, result) => {
  674. if (error != null) {
  675. return next(error)
  676. }
  677. res.send({ failures: result })
  678. })
  679. }
  680. export function getFailuresFull(req, res, next) {
  681. ErrorRecorder.getFailuresFull((error, result) => {
  682. if (error) return next(error)
  683. res.send(result)
  684. })
  685. }
  686. export function getQueueCounts(req, res, next) {
  687. RedisManager.getProjectIdsWithHistoryOpsCount((err, queuedProjectsCount) => {
  688. if (err != null) {
  689. return next(err)
  690. }
  691. res.send({ queuedProjects: queuedProjectsCount })
  692. })
  693. }
  694. const getLabelsSchema = z.object({
  695. params: z.object({
  696. project_id: zz.objectId().or(z.coerce.number()),
  697. }),
  698. })
  699. export function getLabels(req, res, next) {
  700. const { params } = parseReq(req, getLabelsSchema)
  701. const projectId = params.project_id
  702. HistoryApiManager.shouldUseProjectHistory(
  703. projectId,
  704. (error, shouldUseProjectHistory) => {
  705. if (error != null) {
  706. return next(error)
  707. }
  708. if (shouldUseProjectHistory) {
  709. LabelsManager.getLabels(projectId, (error, labels) => {
  710. if (error != null) {
  711. return next(error)
  712. }
  713. res.json(labels)
  714. })
  715. } else {
  716. res.sendStatus(409)
  717. }
  718. }
  719. )
  720. }
  721. const createLabelSchema = z.object({
  722. params: z.object({
  723. project_id: zz.objectId().or(z.coerce.number()),
  724. user_id: zz.objectId().optional(),
  725. }),
  726. body: z.object({
  727. version: z.number().int(),
  728. comment: z.string(),
  729. created_at: z.string().optional(),
  730. validate_exists: z.boolean().default(true),
  731. user_id: zz.objectId().nullable().optional(),
  732. }),
  733. })
  734. export function createLabel(req, res, next) {
  735. const { params, body } = parseReq(req, createLabelSchema)
  736. const { project_id: projectId, user_id: userIdParam } = params
  737. const {
  738. version,
  739. comment,
  740. user_id: userIdBody,
  741. created_at: createdAt,
  742. validate_exists: validateExists,
  743. } = body
  744. // Temporarily looking up both params and body while rolling out changes
  745. // in the router path - https://github.com/overleaf/internal/pull/20200
  746. const userId = userIdParam || userIdBody
  747. HistoryApiManager.shouldUseProjectHistory(
  748. projectId,
  749. (error, shouldUseProjectHistory) => {
  750. if (error != null) {
  751. return next(error)
  752. }
  753. if (shouldUseProjectHistory) {
  754. LabelsManager.createLabel(
  755. projectId,
  756. userId,
  757. version,
  758. comment,
  759. createdAt,
  760. validateExists,
  761. (error, label) => {
  762. if (error != null) {
  763. return next(error)
  764. }
  765. res.json(label)
  766. }
  767. )
  768. } else {
  769. logger.error(
  770. {
  771. projectId,
  772. userId,
  773. version,
  774. comment,
  775. createdAt,
  776. validateExists,
  777. },
  778. 'not using v2 history'
  779. )
  780. res.sendStatus(409)
  781. }
  782. }
  783. )
  784. }
  785. /**
  786. * This will delete a label if it is owned by the current user. If you wish to
  787. * delete a label regardless of the current user, then use `deleteLabel` instead.
  788. */
  789. const deleteLabelForUserSchema = z.object({
  790. params: z.object({
  791. project_id: zz.objectId().or(z.coerce.number()),
  792. user_id: zz.objectId(),
  793. label_id: zz.objectId(),
  794. }),
  795. })
  796. export function deleteLabelForUser(req, res, next) {
  797. const { params } = parseReq(req, deleteLabelForUserSchema)
  798. const { project_id: projectId, user_id: userId, label_id: labelId } = params
  799. LabelsManager.deleteLabelForUser(projectId, userId, labelId, error => {
  800. if (error != null) {
  801. return next(error)
  802. }
  803. res.sendStatus(204)
  804. })
  805. }
  806. const deleteLabelSchema = z.object({
  807. params: z.object({
  808. project_id: zz.objectId().or(z.coerce.number()),
  809. label_id: zz.objectId(),
  810. }),
  811. })
  812. export function deleteLabel(req, res, next) {
  813. const { params } = parseReq(req, deleteLabelSchema)
  814. const { project_id: projectId, label_id: labelId } = params
  815. LabelsManager.deleteLabel(projectId, labelId, error => {
  816. if (error != null) {
  817. return next(error)
  818. }
  819. res.sendStatus(204)
  820. })
  821. }
  822. const retryFailuresSchema = z.object({
  823. query: z.object({
  824. failureType: z.enum(['soft', 'hard']).optional(),
  825. // bail out after this time limit
  826. timeout: z.coerce.number().int().default(300),
  827. // maximum number of projects to check
  828. limit: z.coerce.number().int().default(100),
  829. callbackUrl: z.string().optional(),
  830. }),
  831. })
  832. export function retryFailures(req, res, next) {
  833. const { query } = parseReq(req, retryFailuresSchema)
  834. const { failureType, timeout, limit, callbackUrl } = query
  835. if (callbackUrl) {
  836. // send response but run in background when callbackUrl provided
  837. res.send({ retryStatus: 'running retryFailures in background' })
  838. }
  839. RetryManager.retryFailures(
  840. { failureType, timeout, limit },
  841. (error, result) => {
  842. if (callbackUrl) {
  843. // if present, notify the callbackUrl on success
  844. if (!error) {
  845. // Needs Node 12
  846. // const callbackHeaders = Object.fromEntries(Object.entries(req.headers || {}).filter(([k,v]) => k.match(/^X-CALLBACK-/i)))
  847. const callbackHeaders = {}
  848. for (const key of Object.getOwnPropertyNames(
  849. req.headers || {}
  850. ).filter(key => key.match(/^X-CALLBACK-/i))) {
  851. const found = key.match(/^X-CALLBACK-(.*)/i)
  852. callbackHeaders[found[1]] = req.headers[key]
  853. }
  854. fetchNothing(callbackUrl, { headers: callbackHeaders }).catch(err => {
  855. logger.warn({ err }, 'failed to ping callback url')
  856. })
  857. }
  858. } else {
  859. if (error != null) {
  860. return next(error)
  861. }
  862. res.send({ retryStatus: result })
  863. }
  864. }
  865. )
  866. }
  867. const transferLabelsSchema = z.object({
  868. params: z.object({
  869. from_user: zz.objectId(),
  870. to_user: zz.objectId(),
  871. }),
  872. })
  873. export function transferLabels(req, res, next) {
  874. const { params } = parseReq(req, transferLabelsSchema)
  875. const { from_user: fromUser, to_user: toUser } = params
  876. LabelsManager.transferLabels(fromUser, toUser, error => {
  877. if (error != null) {
  878. return next(error)
  879. }
  880. res.sendStatus(204)
  881. })
  882. }
  883. const deleteProjectSchema = z.object({
  884. params: z.object({
  885. project_id: zz.objectId().or(z.coerce.number()),
  886. }),
  887. })
  888. export function deleteProject(req, res, next) {
  889. const { params } = parseReq(req, deleteProjectSchema)
  890. const { project_id: projectId } = params
  891. // clear the timestamp before clearing the queue,
  892. // because the queue location is used in the migration
  893. RedisManager.clearFirstOpTimestamp(projectId, err => {
  894. if (err) {
  895. return next(err)
  896. }
  897. RedisManager.clearCachedHistoryId(projectId, err => {
  898. if (err) {
  899. return next(err)
  900. }
  901. RedisManager.destroyDocUpdatesQueue(projectId, err => {
  902. if (err) {
  903. return next(err)
  904. }
  905. SyncManager.clearResyncState(projectId, err => {
  906. if (err) {
  907. return next(err)
  908. }
  909. ErrorRecorder.clearError(projectId, err => {
  910. if (err) {
  911. return next(err)
  912. }
  913. res.sendStatus(204)
  914. })
  915. })
  916. })
  917. })
  918. })
  919. }