Re-fetch file to calculate md5 if etag is not in correct format

services/filestore/app/js/S3Persistor.js

@@ -98,7 +98,7 @@ async function sendStream(bucketName, key, readStream, sourceMd5) {
     const response = await _getClientForBucket(bucketName)
       .upload(uploadOptions, { partSize: 100 * 1024 * 1024 })
       .promise()
-    const destMd5 = _md5FromResponse(response)
+    const destMd5 = await _md5FromResponse(response)
 
     // if we didn't have an md5 hash, we should compare our computed one with S3's
     // as we couldn't tell S3 about it beforehand
@@ -219,7 +219,7 @@ async function getFileMd5Hash(bucketName, key) {
     const response = await _getClientForBucket(bucketName)
       .headObject({ Bucket: bucketName, Key: key })
       .promise()
-    const md5 = _md5FromResponse(response)
+    const md5 = await _md5FromResponse(response)
     return md5
   } catch (err) {
     throw PersistorHelper.wrapError(
@@ -364,16 +364,12 @@ function _buildClientOptions(bucketCredentials) {
   return options
 }
 
-function _md5FromResponse(response) {
-  const md5 = (response.ETag || '').replace(/[ "]/g, '')
+async function _md5FromResponse(response) {
+  let md5 = (response.ETag || '').replace(/[ "]/g, '')
   if (!md5.match(/^[a-f0-9]{32}$/)) {
-    throw new ReadError({
-      message: 's3 etag not in md5-hash format',
-      info: {
-        md5,
-        eTag: response.ETag
-      }
-    })
+    // the eTag isn't in md5 format so we need to calculate it ourselves
+    const stream = await getFileStream(response.Bucket, response.Key, {})
+    md5 = await PersistorHelper.calculateStreamMd5(stream)
   }
 
   return md5

services/filestore/test/unit/js/S3PersistorTests.js

@@ -530,6 +530,38 @@ describe('S3PersistorTests', function() {
         expect(error).to.be.an.instanceOf(Errors.WriteError)
       })
     })
+
+    describe("when the etag isn't a valid md5 hash", function() {
+      beforeEach(async function() {
+        S3Client.upload = sinon.stub().returns({
+          promise: sinon.stub().resolves({
+            ETag: 'somethingthatisntanmd5',
+            Bucket: bucket,
+            Key: key
+          })
+        })
+
+        await S3Persistor.promises.sendStream(bucket, key, ReadStream)
+      })
+
+      it('should re-fetch the file to verify it', function() {
+        expect(S3Client.getObject).to.have.been.calledWith({
+          Bucket: bucket,
+          Key: key
+        })
+      })
+
+      it('should meter the download', function() {
+        expect(Stream.pipeline).to.have.been.calledWith(
+          S3ReadStream,
+          MeteredStream
+        )
+      })
+
+      it('should calculate the md5 hash from the file', function() {
+        expect(Stream.pipeline).to.have.been.calledWith(MeteredStream, Hash)
+      })
+    })
   })
 
   describe('sendFile', function() {