ProjectManager.js 9.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332
  1. const RedisManager = require('./RedisManager')
  2. const ProjectHistoryRedisManager = require('./ProjectHistoryRedisManager')
  3. const DocumentManager = require('./DocumentManager')
  4. const HistoryManager = require('./HistoryManager')
  5. const async = require('async')
  6. const logger = require('logger-sharelatex')
  7. const Metrics = require('./Metrics')
  8. const Errors = require('./Errors')
  9. module.exports = {
  10. flushProjectWithLocks(projectId, _callback) {
  11. const timer = new Metrics.Timer('projectManager.flushProjectWithLocks')
  12. const callback = function (...args) {
  13. timer.done()
  14. _callback(...args)
  15. }
  16. RedisManager.getDocIdsInProject(projectId, function (error, docIds) {
  17. if (error) {
  18. return callback(error)
  19. }
  20. const jobs = []
  21. const errors = []
  22. for (const docId of docIds) {
  23. ;((docId) =>
  24. jobs.push((callback) =>
  25. DocumentManager.flushDocIfLoadedWithLock(
  26. projectId,
  27. docId,
  28. function (error) {
  29. if (error instanceof Errors.NotFoundError) {
  30. logger.warn(
  31. { err: error, projectId, docId },
  32. 'found deleted doc when flushing'
  33. )
  34. callback()
  35. } else if (error) {
  36. logger.error(
  37. { err: error, projectId, docId },
  38. 'error flushing doc'
  39. )
  40. errors.push(error)
  41. callback()
  42. } else {
  43. callback()
  44. }
  45. }
  46. )
  47. ))(docId)
  48. }
  49. logger.log({ projectId, docIds }, 'flushing docs')
  50. async.series(jobs, function () {
  51. if (errors.length > 0) {
  52. callback(new Error('Errors flushing docs. See log for details'))
  53. } else {
  54. callback(null)
  55. }
  56. })
  57. })
  58. },
  59. flushAndDeleteProjectWithLocks(projectId, options, _callback) {
  60. const timer = new Metrics.Timer(
  61. 'projectManager.flushAndDeleteProjectWithLocks'
  62. )
  63. const callback = function (...args) {
  64. timer.done()
  65. _callback(...args)
  66. }
  67. RedisManager.getDocIdsInProject(projectId, function (error, docIds) {
  68. if (error) {
  69. return callback(error)
  70. }
  71. const jobs = []
  72. const errors = []
  73. for (const docId of docIds) {
  74. ;((docId) =>
  75. jobs.push((callback) =>
  76. DocumentManager.flushAndDeleteDocWithLock(
  77. projectId,
  78. docId,
  79. {},
  80. function (error) {
  81. if (error) {
  82. logger.error(
  83. { err: error, projectId, docId },
  84. 'error deleting doc'
  85. )
  86. errors.push(error)
  87. }
  88. callback()
  89. }
  90. )
  91. ))(docId)
  92. }
  93. logger.log({ projectId, docIds }, 'deleting docs')
  94. async.series(jobs, () =>
  95. // When deleting the project here we want to ensure that project
  96. // history is completely flushed because the project may be
  97. // deleted in web after this call completes, and so further
  98. // attempts to flush would fail after that.
  99. HistoryManager.flushProjectChanges(projectId, options, function (
  100. error
  101. ) {
  102. if (errors.length > 0) {
  103. callback(new Error('Errors deleting docs. See log for details'))
  104. } else if (error) {
  105. callback(error)
  106. } else {
  107. callback(null)
  108. }
  109. })
  110. )
  111. })
  112. },
  113. queueFlushAndDeleteProject(projectId, callback) {
  114. RedisManager.queueFlushAndDeleteProject(projectId, function (error) {
  115. if (error) {
  116. logger.error(
  117. { projectId, error },
  118. 'error adding project to flush and delete queue'
  119. )
  120. return callback(error)
  121. }
  122. Metrics.inc('queued-delete')
  123. callback()
  124. })
  125. },
  126. getProjectDocsTimestamps(projectId, callback) {
  127. RedisManager.getDocIdsInProject(projectId, function (error, docIds) {
  128. if (error) {
  129. return callback(error)
  130. }
  131. if (docIds.length === 0) {
  132. return callback(null, [])
  133. }
  134. RedisManager.getDocTimestamps(docIds, function (error, timestamps) {
  135. if (error) {
  136. return callback(error)
  137. }
  138. callback(null, timestamps)
  139. })
  140. })
  141. },
  142. getProjectDocsAndFlushIfOld(
  143. projectId,
  144. projectStateHash,
  145. excludeVersions,
  146. _callback
  147. ) {
  148. const timer = new Metrics.Timer(
  149. 'projectManager.getProjectDocsAndFlushIfOld'
  150. )
  151. const callback = function (...args) {
  152. timer.done()
  153. _callback(...args)
  154. }
  155. RedisManager.checkOrSetProjectState(projectId, projectStateHash, function (
  156. error,
  157. projectStateChanged
  158. ) {
  159. if (error) {
  160. logger.error(
  161. { err: error, projectId },
  162. 'error getting/setting project state in getProjectDocsAndFlushIfOld'
  163. )
  164. return callback(error)
  165. }
  166. // we can't return docs if project structure has changed
  167. if (projectStateChanged) {
  168. return callback(
  169. Errors.ProjectStateChangedError('project state changed')
  170. )
  171. }
  172. // project structure hasn't changed, return doc content from redis
  173. RedisManager.getDocIdsInProject(projectId, function (error, docIds) {
  174. if (error) {
  175. logger.error(
  176. { err: error, projectId },
  177. 'error getting doc ids in getProjectDocs'
  178. )
  179. return callback(error)
  180. }
  181. const jobs = []
  182. for (const docId of docIds) {
  183. ;((docId) =>
  184. jobs.push((
  185. cb // get the doc lines from redis
  186. ) =>
  187. DocumentManager.getDocAndFlushIfOldWithLock(
  188. projectId,
  189. docId,
  190. function (err, lines, version) {
  191. if (err) {
  192. logger.error(
  193. { err, projectId, docId },
  194. 'error getting project doc lines in getProjectDocsAndFlushIfOld'
  195. )
  196. return cb(err)
  197. }
  198. const doc = { _id: docId, lines, v: version } // create a doc object to return
  199. cb(null, doc)
  200. }
  201. )
  202. ))(docId)
  203. }
  204. async.series(jobs, function (error, docs) {
  205. if (error) {
  206. return callback(error)
  207. }
  208. callback(null, docs)
  209. })
  210. })
  211. })
  212. },
  213. clearProjectState(projectId, callback) {
  214. RedisManager.clearProjectState(projectId, callback)
  215. },
  216. updateProjectWithLocks(
  217. projectId,
  218. projectHistoryId,
  219. userId,
  220. docUpdates,
  221. fileUpdates,
  222. version,
  223. _callback
  224. ) {
  225. const timer = new Metrics.Timer('projectManager.updateProject')
  226. const callback = function (...args) {
  227. timer.done()
  228. _callback(...args)
  229. }
  230. const projectVersion = version
  231. let projectSubversion = 0 // project versions can have multiple operations
  232. let projectOpsLength = 0
  233. const handleDocUpdate = function (projectUpdate, cb) {
  234. const docId = projectUpdate.id
  235. projectUpdate.version = `${projectVersion}.${projectSubversion++}`
  236. if (projectUpdate.docLines != null) {
  237. ProjectHistoryRedisManager.queueAddEntity(
  238. projectId,
  239. projectHistoryId,
  240. 'doc',
  241. docId,
  242. userId,
  243. projectUpdate,
  244. function (error, count) {
  245. projectOpsLength = count
  246. cb(error)
  247. }
  248. )
  249. } else {
  250. DocumentManager.renameDocWithLock(
  251. projectId,
  252. docId,
  253. userId,
  254. projectUpdate,
  255. projectHistoryId,
  256. function (error, count) {
  257. projectOpsLength = count
  258. cb(error)
  259. }
  260. )
  261. }
  262. }
  263. const handleFileUpdate = function (projectUpdate, cb) {
  264. const fileId = projectUpdate.id
  265. projectUpdate.version = `${projectVersion}.${projectSubversion++}`
  266. if (projectUpdate.url != null) {
  267. ProjectHistoryRedisManager.queueAddEntity(
  268. projectId,
  269. projectHistoryId,
  270. 'file',
  271. fileId,
  272. userId,
  273. projectUpdate,
  274. function (error, count) {
  275. projectOpsLength = count
  276. cb(error)
  277. }
  278. )
  279. } else {
  280. ProjectHistoryRedisManager.queueRenameEntity(
  281. projectId,
  282. projectHistoryId,
  283. 'file',
  284. fileId,
  285. userId,
  286. projectUpdate,
  287. function (error, count) {
  288. projectOpsLength = count
  289. cb(error)
  290. }
  291. )
  292. }
  293. }
  294. async.eachSeries(docUpdates, handleDocUpdate, function (error) {
  295. if (error) {
  296. return callback(error)
  297. }
  298. async.eachSeries(fileUpdates, handleFileUpdate, function (error) {
  299. if (error) {
  300. return callback(error)
  301. }
  302. if (
  303. HistoryManager.shouldFlushHistoryOps(
  304. projectOpsLength,
  305. docUpdates.length + fileUpdates.length,
  306. HistoryManager.FLUSH_PROJECT_EVERY_N_OPS
  307. )
  308. ) {
  309. HistoryManager.flushProjectChangesAsync(projectId)
  310. }
  311. callback()
  312. })
  313. })
  314. },
  315. }