|
@@ -194,11 +194,11 @@ describe('SyncManager', function () {
|
|
|
project_id: new ObjectId(this.projectId),
|
|
project_id: new ObjectId(this.projectId),
|
|
|
},
|
|
},
|
|
|
sinon.match({
|
|
sinon.match({
|
|
|
- $set: {
|
|
|
|
|
|
|
+ $set: sinon.match({
|
|
|
resyncProjectStructure: true,
|
|
resyncProjectStructure: true,
|
|
|
resyncDocContents: [],
|
|
resyncDocContents: [],
|
|
|
origin: { kind: 'history-resync' },
|
|
origin: { kind: 'history-resync' },
|
|
|
- },
|
|
|
|
|
|
|
+ }),
|
|
|
$currentDate: { lastUpdated: true },
|
|
$currentDate: { lastUpdated: true },
|
|
|
$inc: { resyncCount: 1 },
|
|
$inc: { resyncCount: 1 },
|
|
|
$unset: { expiresAt: true },
|
|
$unset: { expiresAt: true },
|
|
@@ -212,7 +212,10 @@ describe('SyncManager', function () {
|
|
|
|
|
|
|
|
describe('if project structure sync is in progress', function () {
|
|
describe('if project structure sync is in progress', function () {
|
|
|
beforeEach(function () {
|
|
beforeEach(function () {
|
|
|
- const syncState = { resyncProjectStructure: true }
|
|
|
|
|
|
|
+ const syncState = {
|
|
|
|
|
+ resyncProjectStructure: true,
|
|
|
|
|
+ resyncPendingSince: new Date(),
|
|
|
|
|
+ }
|
|
|
this.db.projectHistorySyncState.findOne.resolves(syncState)
|
|
this.db.projectHistorySyncState.findOne.resolves(syncState)
|
|
|
})
|
|
})
|
|
|
|
|
|
|
@@ -225,7 +228,10 @@ describe('SyncManager', function () {
|
|
|
|
|
|
|
|
describe('if doc content sync in is progress', function () {
|
|
describe('if doc content sync in is progress', function () {
|
|
|
beforeEach(async function () {
|
|
beforeEach(async function () {
|
|
|
- const syncState = { resyncDocContents: ['/foo.tex'] }
|
|
|
|
|
|
|
+ const syncState = {
|
|
|
|
|
+ resyncDocContents: ['/foo.tex'],
|
|
|
|
|
+ resyncPendingSince: new Date(),
|
|
|
|
|
+ }
|
|
|
this.db.projectHistorySyncState.findOne.resolves(syncState)
|
|
this.db.projectHistorySyncState.findOne.resolves(syncState)
|
|
|
})
|
|
})
|
|
|
|
|
|
|
@@ -235,6 +241,112 @@ describe('SyncManager', function () {
|
|
|
).to.be.rejectedWith('sync ongoing')
|
|
).to.be.rejectedWith('sync ongoing')
|
|
|
})
|
|
})
|
|
|
})
|
|
})
|
|
|
|
|
+
|
|
|
|
|
+ describe('if sync is stuck (no resyncPendingSince — legacy state)', function () {
|
|
|
|
|
+ beforeEach(function () {
|
|
|
|
|
+ const syncState = { resyncDocContents: ['/foo.tex'] }
|
|
|
|
|
+ this.db.projectHistorySyncState.findOne.resolves(syncState)
|
|
|
|
|
+ })
|
|
|
|
|
+
|
|
|
|
|
+ it('records the stuck clear, does not delete state, and starts a new resync', async function () {
|
|
|
|
|
+ await this.SyncManager.promises.startResync(this.projectId)
|
|
|
|
|
+ expect(
|
|
|
|
|
+ this.db.projectHistorySyncState.updateOne
|
|
|
|
|
+ ).to.have.been.calledWith(
|
|
|
|
|
+ { project_id: new ObjectId(this.projectId) },
|
|
|
|
|
+ sinon.match({
|
|
|
|
|
+ $inc: { stuckClearCount: 1 },
|
|
|
|
|
+ $unset: { resyncPendingSince: 1 },
|
|
|
|
|
+ })
|
|
|
|
|
+ )
|
|
|
|
|
+ expect(this.WebApiManager.promises.requestResync.calledOnce).to.be.true
|
|
|
|
|
+ expect(
|
|
|
|
|
+ this.Metrics.inc.calledWith('project_history_sync_stuck_cleared')
|
|
|
|
|
+ ).to.be.true
|
|
|
|
|
+ })
|
|
|
|
|
+ })
|
|
|
|
|
+
|
|
|
|
|
+ describe('if sync is stuck (resyncPendingSince older than timeout)', function () {
|
|
|
|
|
+ beforeEach(function () {
|
|
|
|
|
+ const fiveHoursAgo = new Date(Date.now() - 5 * 60 * 60 * 1000)
|
|
|
|
|
+ const syncState = {
|
|
|
|
|
+ resyncDocContents: ['/foo.tex'],
|
|
|
|
|
+ resyncPendingSince: fiveHoursAgo,
|
|
|
|
|
+ }
|
|
|
|
|
+ this.db.projectHistorySyncState.findOne.resolves(syncState)
|
|
|
|
|
+ })
|
|
|
|
|
+
|
|
|
|
|
+ it('records the stuck clear and starts a new resync', async function () {
|
|
|
|
|
+ await this.SyncManager.promises.startResync(this.projectId)
|
|
|
|
|
+ expect(
|
|
|
|
|
+ this.db.projectHistorySyncState.updateOne
|
|
|
|
|
+ ).to.have.been.calledWith(
|
|
|
|
|
+ { project_id: new ObjectId(this.projectId) },
|
|
|
|
|
+ sinon.match({
|
|
|
|
|
+ $inc: { stuckClearCount: 1 },
|
|
|
|
|
+ $unset: { resyncPendingSince: 1 },
|
|
|
|
|
+ })
|
|
|
|
|
+ )
|
|
|
|
|
+ expect(this.WebApiManager.promises.requestResync.calledOnce).to.be.true
|
|
|
|
|
+ })
|
|
|
|
|
+ })
|
|
|
|
|
+
|
|
|
|
|
+ describe('if sync is not stuck (resyncPendingSince within timeout)', function () {
|
|
|
|
|
+ beforeEach(function () {
|
|
|
|
|
+ const threeHoursAgo = new Date(Date.now() - 3 * 60 * 60 * 1000)
|
|
|
|
|
+ const syncState = {
|
|
|
|
|
+ resyncDocContents: ['/foo.tex'],
|
|
|
|
|
+ resyncPendingSince: threeHoursAgo,
|
|
|
|
|
+ }
|
|
|
|
|
+ this.db.projectHistorySyncState.findOne.resolves(syncState)
|
|
|
|
|
+ })
|
|
|
|
|
+
|
|
|
|
|
+ it('throws sync ongoing and does not clear', async function () {
|
|
|
|
|
+ await expect(
|
|
|
|
|
+ this.SyncManager.promises.startResync(this.projectId)
|
|
|
|
|
+ ).to.be.rejectedWith('sync ongoing')
|
|
|
|
|
+ expect(
|
|
|
|
|
+ this.Metrics.inc.calledWith('project_history_sync_stuck_cleared')
|
|
|
|
|
+ ).to.be.false
|
|
|
|
|
+ expect(
|
|
|
|
|
+ this.Metrics.inc.calledWith('project_history_sync_stuck_permanent')
|
|
|
|
|
+ ).to.be.false
|
|
|
|
|
+ })
|
|
|
|
|
+ })
|
|
|
|
|
+
|
|
|
|
|
+ describe('if sync is permanently stuck (exceeded retry limit)', function () {
|
|
|
|
|
+ beforeEach(function () {
|
|
|
|
|
+ const syncState = {
|
|
|
|
|
+ resyncDocContents: ['/foo.tex'],
|
|
|
|
|
+ stuckClearCount: 5,
|
|
|
|
|
+ }
|
|
|
|
|
+ this.db.projectHistorySyncState.findOne.resolves(syncState)
|
|
|
|
|
+ })
|
|
|
|
|
+
|
|
|
|
|
+ it('throws permanently stuck, emits metric, and records the attempt', async function () {
|
|
|
|
|
+ await expect(
|
|
|
|
|
+ this.SyncManager.promises.startResync(this.projectId)
|
|
|
|
|
+ ).to.be.rejectedWith('sync permanently stuck')
|
|
|
|
|
+ expect(
|
|
|
|
|
+ this.Metrics.inc.calledWith('project_history_sync_stuck_permanent')
|
|
|
|
|
+ ).to.be.true
|
|
|
|
|
+ expect(
|
|
|
|
|
+ this.db.projectHistorySyncState.updateOne
|
|
|
|
|
+ ).to.have.been.calledWith(
|
|
|
|
|
+ { project_id: new ObjectId(this.projectId) },
|
|
|
|
|
+ sinon.match({ $inc: { stuckClearCount: 1 } })
|
|
|
|
|
+ )
|
|
|
|
|
+ })
|
|
|
|
|
+
|
|
|
|
|
+ it('does not attempt a resync', async function () {
|
|
|
|
|
+ try {
|
|
|
|
|
+ await this.SyncManager.promises.startResync(this.projectId)
|
|
|
|
|
+ } catch (e) {
|
|
|
|
|
+ // expected
|
|
|
|
|
+ }
|
|
|
|
|
+ expect(this.WebApiManager.promises.requestResync.called).to.be.false
|
|
|
|
|
+ })
|
|
|
|
|
+ })
|
|
|
})
|
|
})
|
|
|
|
|
|
|
|
describe('setResyncState', function () {
|
|
describe('setResyncState', function () {
|
|
@@ -265,7 +377,11 @@ describe('SyncManager', function () {
|
|
|
).to.have.been.calledWith(
|
|
).to.have.been.calledWith(
|
|
|
{ project_id: new ObjectId(this.projectId) },
|
|
{ project_id: new ObjectId(this.projectId) },
|
|
|
sinon.match({
|
|
sinon.match({
|
|
|
- $set: this.syncState.toRaw(),
|
|
|
|
|
|
|
+ $set: sinon.match({
|
|
|
|
|
+ resyncProjectStructure: true,
|
|
|
|
|
+ resyncDocContents: [],
|
|
|
|
|
+ origin: { kind: 'history-resync' },
|
|
|
|
|
+ }),
|
|
|
$currentDate: { lastUpdated: true },
|
|
$currentDate: { lastUpdated: true },
|
|
|
$inc: { resyncCount: 1 },
|
|
$inc: { resyncCount: 1 },
|
|
|
$unset: { expiresAt: true },
|
|
$unset: { expiresAt: true },
|
|
@@ -289,7 +405,7 @@ describe('SyncManager', function () {
|
|
|
}
|
|
}
|
|
|
})
|
|
})
|
|
|
|
|
|
|
|
- it('sets the sync state entry in mongo to expire', async function () {
|
|
|
|
|
|
|
+ it('sets the sync state entry in mongo to expire and clears stuck fields', async function () {
|
|
|
await this.SyncManager.promises.setResyncState(
|
|
await this.SyncManager.promises.setResyncState(
|
|
|
this.projectId,
|
|
this.projectId,
|
|
|
this.syncState
|
|
this.syncState
|
|
@@ -307,6 +423,12 @@ describe('SyncManager', function () {
|
|
|
expiresAt: new Date(this.now.getTime() + 90 * 24 * 3600 * 1000),
|
|
expiresAt: new Date(this.now.getTime() + 90 * 24 * 3600 * 1000),
|
|
|
},
|
|
},
|
|
|
$currentDate: { lastUpdated: true },
|
|
$currentDate: { lastUpdated: true },
|
|
|
|
|
+ $unset: {
|
|
|
|
|
+ resyncPendingSince: 1,
|
|
|
|
|
+ stuckClearCount: 1,
|
|
|
|
|
+ lastStuckClearAt: 1,
|
|
|
|
|
+ lastStuckDocPaths: 1,
|
|
|
|
|
+ },
|
|
|
}),
|
|
}),
|
|
|
{ upsert: true }
|
|
{ upsert: true }
|
|
|
)
|
|
)
|
|
@@ -2005,4 +2127,47 @@ describe('SyncManager', function () {
|
|
|
})
|
|
})
|
|
|
})
|
|
})
|
|
|
})
|
|
})
|
|
|
|
|
+
|
|
|
|
|
+ describe('stuck sync state reproduction', function () {
|
|
|
|
|
+ describe('when resyncDocContents has entries but no resyncDocContent update arrives', function () {
|
|
|
|
|
+ beforeEach(function () {
|
|
|
|
|
+ // Simulate the stuck state: project structure sync is done,
|
|
|
|
|
+ // but doc content sync never completed for /main.tex
|
|
|
|
|
+ const stuckSyncState = {
|
|
|
|
|
+ resyncProjectStructure: false,
|
|
|
|
|
+ resyncDocContents: ['/main.tex'],
|
|
|
|
|
+ origin: { kind: 'history-resync' },
|
|
|
|
|
+ }
|
|
|
|
|
+ this.db.projectHistorySyncState.findOne.resolves(stuckSyncState)
|
|
|
|
|
+ })
|
|
|
|
|
+
|
|
|
|
|
+ it('should skip text updates for a doc stuck in resyncDocContents', async function () {
|
|
|
|
|
+ const textUpdate = {
|
|
|
|
|
+ doc: new ObjectId(),
|
|
|
|
|
+ op: [{ i: 'hello', p: 0 }],
|
|
|
|
|
+ meta: {
|
|
|
|
|
+ pathname: '/main.tex',
|
|
|
|
|
+ doc_length: 0,
|
|
|
|
|
+ },
|
|
|
|
|
+ }
|
|
|
|
|
+ this.UpdateTranslator.isTextUpdate.returns(false)
|
|
|
|
|
+ this.UpdateTranslator.isTextUpdate.withArgs(textUpdate).returns(true)
|
|
|
|
|
+
|
|
|
|
|
+ const { updates: filteredUpdates, syncState } =
|
|
|
|
|
+ await this.SyncManager.promises.skipUpdatesDuringSync(
|
|
|
|
|
+ this.projectId,
|
|
|
|
|
+ [textUpdate]
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ // The text update is silently dropped — this is the stuck state bug
|
|
|
|
|
+ expect(filteredUpdates).to.deep.equal([])
|
|
|
|
|
+ // Sync state still shows /main.tex as syncing
|
|
|
|
|
+ expect(syncState.toRaw().resyncDocContents).to.deep.equal(['/main.tex'])
|
|
|
|
|
+ // Metric is emitted for the skipped update
|
|
|
|
|
+ expect(
|
|
|
|
|
+ this.Metrics.inc.calledWith('project_history_sync_update_skipped')
|
|
|
|
|
+ ).to.be.true
|
|
|
|
|
+ })
|
|
|
|
|
+ })
|
|
|
|
|
+ })
|
|
|
})
|
|
})
|