|
|
@@ -0,0 +1,79 @@
|
|
|
+
|
|
|
+
|
|
|
+diff --git a/services/web/scripts/find_malformed_filetrees.mjs b/services/web/scripts/find_malformed_filetrees.mjs
|
|
|
+index 2614c7d6223c..9f0f6323ab8d 100644
|
|
|
+--- a/services/web/scripts/find_malformed_filetrees.mjs
|
|
|
++++ b/services/web/scripts/find_malformed_filetrees.mjs
|
|
|
+@@ -49,7 +49,7 @@ async function main(trackProgress) {
|
|
|
+ for (const project of projects) {
|
|
|
+ projectsProcessed += 1
|
|
|
+ if (projectsProcessed % 100000 === 0) {
|
|
|
+- console.log(projectsProcessed, 'projects processed')
|
|
|
++ console.warn(projectsProcessed, 'projects processed')
|
|
|
+ }
|
|
|
+ const projectId = project._id.toString()
|
|
|
+ for (const { reason, path, _id } of processProject(project)) {
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs
|
|
|
+index 9c2a9818680a..33962c5da7d4 100644
|
|
|
+--- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs
|
|
|
++++ b/services/history-v1/storage/scripts/back_fill_file_hash.mjs
|
|
|
+@@ -399,7 +399,8 @@ async function getStatsForCollection(
|
|
|
+ const fractionSampled = stats.projectCount / collectionCount
|
|
|
+ const percentageSampled = (fractionSampled * 100).toFixed(0)
|
|
|
+ const fractionConverted = stats.projectsWithAllHashes / stats.projectCount
|
|
|
+- const percentageConverted = (fractionConverted * 100).toFixed(0)
|
|
|
++ const fractionToBackFill = 1 - fractionConverted
|
|
|
++ const percentageToBackFill = (fractionToBackFill * 100).toFixed(0)
|
|
|
+ const fractionMissing = stats.fileMissingInHistoryCount / stats.fileCount
|
|
|
+ const percentageMissing = (fractionMissing * 100).toFixed(0)
|
|
|
+ console.log(
|
|
|
+@@ -409,7 +410,7 @@ async function getStatsForCollection(
|
|
|
+ `- Sampled ${name} with all hashes present: ${stats.projectsWithAllHashes}`
|
|
|
+ )
|
|
|
+ console.log(
|
|
|
+- `- Percentage of ${name} that need back-filling hashes: ${percentageConverted}% (estimated)`
|
|
|
++ `- Percentage of ${name} that need back-filling hashes: ${percentageToBackFill}% (estimated)`
|
|
|
+ )
|
|
|
+ console.log(
|
|
|
+ `- Sampled ${name} have ${stats.fileCount} files that need to be checked against the full project history system.`
|
|
|
+diff --git a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs
|
|
|
+index 601cea13b6a6..bfcf93228d9e 100644
|
|
|
+--- a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs
|
|
|
++++ b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs
|
|
|
+@@ -61,7 +61,7 @@ function objectIdFromTime(timestamp) {
|
|
|
+
|
|
|
+ const PRINT_IDS_AND_HASHES_FOR_DEBUGGING = false
|
|
|
+
|
|
|
+-describe.only('back_fill_file_hash script', function () {
|
|
|
++describe('back_fill_file_hash script', function () {
|
|
|
+ this.timeout(TIMEOUT)
|
|
|
+
|
|
|
+ const projectId0 = objectIdFromTime('2017-01-01T00:00:00Z')
|
|
|
+@@ -1091,7 +1091,6 @@ describe.only('back_fill_file_hash script', function () {
|
|
|
+ })
|
|
|
+ it('should print the report', () => {
|
|
|
+ expect(output.status).to.equal(0)
|
|
|
+- console.log(output.stdout)
|
|
|
+ expect(output.stdout).to.equal(`\
|
|
|
+ Current status:
|
|
|
+ - Total number of projects: 10
|
|
|
+@@ -1100,13 +1099,13 @@ Sampling 1000 projects to estimate progress...
|
|
|
+ Sampled stats for projects:
|
|
|
+ - Sampled projects: 9 (90% of all projects)
|
|
|
+ - Sampled projects with all hashes present: 5
|
|
|
+-- Percentage of projects that need back-filling hashes: 56% (estimated)
|
|
|
++- Percentage of projects that need back-filling hashes: 44% (estimated)
|
|
|
+ - Sampled projects have 11 files that need to be checked against the full project history system.
|
|
|
+ - Sampled projects have 3 files that need to be uploaded to the full project history system (estimating 27% of all files).
|
|
|
+ Sampled stats for deleted projects:
|
|
|
+ - Sampled deleted projects: 4 (80% of all deleted projects)
|
|
|
+ - Sampled deleted projects with all hashes present: 3
|
|
|
+-- Percentage of deleted projects that need back-filling hashes: 75% (estimated)
|
|
|
++- Percentage of deleted projects that need back-filling hashes: 25% (estimated)
|
|
|
+ - Sampled deleted projects have 2 files that need to be checked against the full project history system.
|
|
|
+ - Sampled deleted projects have 1 files that need to be uploaded to the full project history system (estimating 50% of all files).
|
|
|
+ `)
|
|
|
+
|