Merge pull request #67 from overleaf/jpa-offline-s3

Add support for offline s3 tests

services/track-changes/Dockerfile

@@ -1,3 +1,8 @@
+# This file was auto-generated, do not edit it directly.
+# Instead run bin/update_build_scripts from
+# https://github.com/sharelatex/sharelatex-dev-environment
+# Version: 1.3.4
+
 FROM node:6.11.2 as app
 
 WORKDIR /app

services/track-changes/Jenkinsfile

@@ -16,6 +16,7 @@ pipeline {
   }
 
   stages {
+
     stage('Install') {
       steps {
         withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) {
@@ -44,9 +45,7 @@ pipeline {
 
     stage('Acceptance Tests') {
       steps {
-        withCredentials([usernamePassword(credentialsId: 'S3_DOCSTORE_TEST_AWS_KEYS', passwordVariable: 'AWS_SECRET_ACCESS_KEY', usernameVariable: 'AWS_ACCESS_KEY_ID')]) {
-          sh 'AWS_BUCKET="sl-acceptance-tests" AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_acceptance'
-        }
+        sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_acceptance'
       }
     }
 

services/track-changes/Makefile

@@ -1,7 +1,7 @@
 # This file was auto-generated, do not edit it directly.
 # Instead run bin/update_build_scripts from
 # https://github.com/sharelatex/sharelatex-dev-environment
-# Version: 1.1.24
+# Version: 1.3.4
 
 BUILD_NUMBER ?= local
 BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
@@ -11,9 +11,6 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
 	BRANCH_NAME=$(BRANCH_NAME) \
 	PROJECT_NAME=$(PROJECT_NAME) \
 	MOCHA_GREP=${MOCHA_GREP} \
-	AWS_BUCKET=${AWS_BUCKET} \
-	AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \
-	AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \
 	docker-compose ${DOCKER_COMPOSE_FLAGS}
 
 clean:
@@ -31,14 +28,20 @@ test_unit:
 
 test_acceptance: test_clean test_acceptance_pre_run test_acceptance_run
 
+test_acceptance_debug: test_clean test_acceptance_pre_run test_acceptance_run_debug
+
 test_acceptance_run:
 	@[ ! -d test/acceptance ] && echo "track-changes has no acceptance tests" || $(DOCKER_COMPOSE) run --rm test_acceptance
 
+test_acceptance_run_debug:
+	@[ ! -d test/acceptance ] && echo "track-changes has no acceptance tests" || $(DOCKER_COMPOSE) run -p 127.0.0.9:19999:19999 --rm test_acceptance npm run test:acceptance -- --inspect=0.0.0.0:19999 --inspect-brk
+
 test_clean:
 	$(DOCKER_COMPOSE) down -v -t 0
 
 test_acceptance_pre_run:
 	@[ ! -f test/acceptance/js/scripts/pre-run ] && echo "track-changes has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
+
 build:
 	docker build --pull --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
 		--tag gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
@@ -51,4 +54,5 @@ publish:
 
 	docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
 
+
 .PHONY: clean test test_unit test_acceptance test_clean build publish

services/track-changes/app/coffee/MongoAWS.coffee

@@ -14,6 +14,8 @@ createStream = (streamConstructor, project_id, doc_id, pack_id) ->
 	AWS_CONFIG =
 		accessKeyId: settings.trackchanges.s3.key
 		secretAccessKey: settings.trackchanges.s3.secret
+		endpoint: settings.trackchanges.s3.endpoint
+		s3ForcePathStyle: settings.trackchanges.s3.pathStyle
 
 	return streamConstructor new AWS.S3(AWS_CONFIG), {
 		"Bucket": settings.trackchanges.stores.doc_history,

services/track-changes/buildscript.txt

@@ -1,10 +1,10 @@
 track-changes
+--public-repo=True
 --language=coffeescript
+--env-add=AWS_BUCKET=bucket
 --node-version=6.11.2
---acceptance-creds=aws
---dependencies=mongo,redis
+--acceptance-creds=None
+--dependencies=mongo,redis,s3
 --docker-repos=gcr.io/overleaf-ops
---build-target=docker
---script-version=1.1.24
 --env-pass-through=
---public-repo=True
+--script-version=1.3.4

services/track-changes/config/settings.defaults.coffee

@@ -38,6 +38,8 @@ module.exports =
 		s3:
 			key: process.env['AWS_ACCESS_KEY_ID']
 			secret: process.env['AWS_SECRET_ACCESS_KEY']
+			endpoint: process.env['AWS_S3_ENDPOINT']
+			pathStyle: process.env['AWS_S3_PATH_STYLE'] == 'true'
 		stores:
 			doc_history: process.env['AWS_BUCKET']
 		continueOnError: process.env['TRACK_CHANGES_CONTINUE_ON_ERROR'] or false

services/track-changes/docker-compose.ci.yml

@@ -1,9 +1,9 @@
 # This file was auto-generated, do not edit it directly.
 # Instead run bin/update_build_scripts from
 # https://github.com/sharelatex/sharelatex-dev-environment
-# Version: 1.1.24
+# Version: 1.3.4
 
-version: "2"
+version: "2.1"
 
 services:
   test_unit:
@@ -22,19 +22,24 @@ services:
       REDIS_HOST: redis
       MONGO_HOST: mongo
       POSTGRES_HOST: postgres
-      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
-      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
-      AWS_BUCKET: ${AWS_BUCKET}
+      AWS_S3_ENDPOINT: http://s3:9090
+      AWS_S3_PATH_STYLE: 'true'
+      AWS_ACCESS_KEY_ID: fake
+      AWS_SECRET_ACCESS_KEY: fake
       MOCHA_GREP: ${MOCHA_GREP}
       NODE_ENV: test
+      AWS_BUCKET: bucket
     depends_on:
-      - mongo
-      - redis
+      mongo:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      s3:
+        condition: service_healthy
     user: node
     command: npm run test:acceptance:_run
 
 
-
   tar:
     build: .
     image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
@@ -42,9 +47,14 @@ services:
       - ./:/tmp/build/
     command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
     user: root
-
   redis:
     image: redis
 
   mongo:
-    image: mongo:3.4
+    image: mongo:3.6
+  s3:
+    image: adobe/s3mock
+    environment:
+      - initialBuckets=fake_user_files,fake_template_files,fake_public_files,bucket
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9090"]

services/track-changes/docker-compose.yml

@@ -1,9 +1,9 @@
 # This file was auto-generated, do not edit it directly.
 # Instead run bin/update_build_scripts from
 # https://github.com/sharelatex/sharelatex-dev-environment
-# Version: 1.1.24
+# Version: 1.3.4
 
-version: "2"
+version: "2.1"
 
 services:
   test_unit:
@@ -18,7 +18,7 @@ services:
     user: node
 
   test_acceptance:
-    build: .
+    image: node:6.11.2
     volumes:
       - .:/app
     working_dir: /app
@@ -27,32 +27,33 @@ services:
       REDIS_HOST: redis
       MONGO_HOST: mongo
       POSTGRES_HOST: postgres
-      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
-      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
-      AWS_BUCKET: ${AWS_BUCKET}
+      AWS_S3_ENDPOINT: http://s3:9090
+      AWS_S3_PATH_STYLE: 'true'
+      AWS_ACCESS_KEY_ID: fake
+      AWS_SECRET_ACCESS_KEY: fake
       MOCHA_GREP: ${MOCHA_GREP}
       LOG_LEVEL: ERROR
       NODE_ENV: test
+      AWS_BUCKET: bucket
     user: node
     depends_on:
-      - mongo
-      - redis
+      mongo:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      s3:
+        condition: service_healthy
     command: npm run test:acceptance
 
-
-
-  tar:
-    build: .
-    image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
-    volumes:
-      - ./:/tmp/build/
-    command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
-    user: root
-
   redis:
     image: redis
 
   mongo:
-    image: mongo:3.4
-
+    image: mongo:3.6
 
+  s3:
+    image: adobe/s3mock
+    environment:
+      - initialBuckets=fake_user_files,fake_template_files,fake_public_files,bucket
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9090"]

services/track-changes/package.json

@@ -9,7 +9,7 @@
   "scripts": {
     "compile:app": "([ -e app/coffee ] && coffee -m $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee -m $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')",
     "start": "npm run compile:app && node $NODE_APP_OPTIONS app.js",
-    "test:acceptance:_run": "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY AWS_BUCKET=$AWS_BUCKET AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID mocha --recursive --reporter spec --timeout 30000 --exit $@ test/acceptance/js",
+    "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 30000 --exit $@ test/acceptance/js",
     "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
     "test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js",
     "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- --grep=$MOCHA_GREP",

services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee

@@ -17,9 +17,12 @@ MockWebApi = require "./helpers/MockWebApi"
 describe "Archiving updates", ->
 	before (done) ->
 		if Settings?.trackchanges?.s3?.key.length < 1
-			message = "s3 keys not setup, this test setup will fail"
+			message = new Error("s3 keys not setup, this test setup will fail")
 			return done(message)
 
+		TrackChangesClient.waitForS3 done
+
+	before (done) ->
 		@now = Date.now()
 		@to = @now
 		@user_id = ObjectId().toString()

services/track-changes/test/acceptance/coffee/helpers/TrackChangesApp.coffee

@@ -1,5 +1,5 @@
 app = require('../../../../app')
-require("logger-sharelatex").logger.level("info")
+require("logger-sharelatex")
 logger = require("logger-sharelatex")
 Settings = require("settings-sharelatex")
 

services/track-changes/test/acceptance/coffee/helpers/TrackChangesClient.coffee

@@ -6,6 +6,15 @@ rclient = require("redis-sharelatex").createClient(Settings.redis.history) # Onl
 Keys = Settings.redis.history.key_schema
 {db, ObjectId} = require "../../../../app/js/mongojs"
 
+aws = require "aws-sdk"
+s3 = new aws.S3(
+	accessKeyId: Settings.trackchanges.s3.key
+	secretAccessKey: Settings.trackchanges.s3.secret
+	endpoint: Settings.trackchanges.s3.endpoint
+	s3ForcePathStyle: Settings.trackchanges.s3.pathStyle
+)
+S3_BUCKET = Settings.trackchanges.stores.doc_history
+
 module.exports = TrackChangesClient =
 	flushAndGetCompressedUpdates: (project_id, doc_id, callback = (error, updates) ->) ->
 		TrackChangesClient.flushDoc project_id, doc_id, (error) ->
@@ -91,32 +100,45 @@ module.exports = TrackChangesClient =
 			response.statusCode.should.equal 204
 			callback(error)
 
-	buildS3Options: (content, key)->
-		return {
-				aws:
-					key: Settings.trackchanges.s3.key
-					secret: Settings.trackchanges.s3.secret
-					bucket: Settings.trackchanges.stores.doc_history
-				timeout: 30 * 1000
-				json: content
-				uri:"https://#{Settings.trackchanges.stores.doc_history}.s3.amazonaws.com/#{key}"
-		}
+	waitForS3: (done, retries=42) ->
+		if !Settings.trackchanges.s3.endpoint
+			return done()
+
+		request.get "#{Settings.trackchanges.s3.endpoint}/", (err, res) ->
+			if res && res.statusCode < 500
+				return done()
+
+			if retries == 0
+				return done(err or new Error("s3 returned #{res.statusCode}"))
+
+			setTimeout () ->
+				TrackChangesClient.waitForS3(done, --retries)
+			, 1000
 
 	getS3Doc: (project_id, doc_id, pack_id, callback = (error, body) ->) ->
-		options = TrackChangesClient.buildS3Options(true, project_id+"/changes-"+doc_id+"/pack-"+pack_id)
-		options.encoding = null
-		request.get options, (err, res, body) ->
+		params =
+			Bucket: S3_BUCKET
+			Key: "#{project_id}/changes-#{doc_id}/pack-#{pack_id}"
+
+		s3.getObject params, (error, data) ->
 			return callback(error) if error?
+			body = data.Body
 			return callback(new Error("empty response from s3")) if not body?
 			zlib.gunzip body, (err, result) ->
 				return callback(err) if err?
 				callback(null, JSON.parse(result.toString()))
 
 	removeS3Doc: (project_id, doc_id, callback = (error, res, body) ->) ->
-		options = TrackChangesClient.buildS3Options(true, "?prefix=" + project_id + "/changes-" +doc_id)
-		request.get options, (error, res, body) ->
-			keys = body.match /[0-9a-f]{24}\/changes-[0-9a-f]{24}\/pack-[0-9a-f]{24}/g
-			async.eachSeries keys, (key, cb) ->
-				options = TrackChangesClient.buildS3Options(true, key)
-				request.del options, cb
-			, callback
+		params =
+			Bucket: S3_BUCKET
+			Prefix: "#{project_id}/changes-#{doc_id}"
+
+		s3.listObjects params, (error, data) ->
+			return callback(error) if error?
+
+			params =
+				Bucket: S3_BUCKET
+				Delete:
+					Objects: data.Contents.map((s3object) -> {Key: s3object.Key})
+
+			s3.deleteObjects params, callback