Merge branch 'master' into pr-email-tokens

services/web/Gruntfile.coffee

@@ -199,7 +199,7 @@ module.exports = (grunt) ->
 			acceptance:
 				src: ["test/acceptance/js/#{grunt.option('feature') or '**'}/*.js"]
 				options:
-					timeout: 10000
+					timeout: 40000
 					reporter: grunt.option('reporter') or 'spec'
 					grep: grunt.option("grep")
 

services/web/app/coffee/Features/Blog/BlogController.coffee

@@ -27,6 +27,8 @@ module.exports = BlogController =
 			data = data.trim()
 			try
 				data = JSON.parse(data)
+				if settings.cdn?.web?.host?
+					data?.content = data?.content?.replace(/src="([^"]+)"/g, "src='#{settings.cdn?.web?.host}$1'");
 			catch err
 				logger.err err:err, data:data, "error parsing data from data"
 			res.render "blog/blog_holder", data

services/web/app/coffee/Features/Project/ProjectController.coffee

@@ -245,6 +245,7 @@ module.exports = ProjectController =
 						first_name : user.first_name
 						last_name  : user.last_name
 						referal_id : user.referal_id
+						signUpDate : user.signUpDate
 						subscription :
 							freeTrial: {allowed: allowedFreeTrial}
 						featureSwitches: user.featureSwitches

services/web/app/coffee/Features/Project/ProjectEntityHandler.coffee

@@ -512,6 +512,11 @@ module.exports = ProjectEntityHandler =
 			return callback(e)
 		type = sanitizeTypeOfElement type
 
+		if path.resolve("/", element.name) isnt "/#{element.name}" or element.name.match("/")
+			e = new Error("invalid element name")
+			logger.err project_id:project._id, folder_id:folder_id, element:element, type:type, "failed trying to insert element as name was invalid"
+			return callback(e)
+
 		if !folder_id?
 			folder_id = project.rootFolder[0]._id
 		ProjectEntityHandler._countElements project, (err, count)->

services/web/app/coffee/Features/References/ReferencesController.coffee

@@ -32,8 +32,8 @@ module.exports = ReferencesController =
 
 	_handleIndexResponse: (req, res, projectId, shouldBroadcast, data) ->
 		if !data? or !data.keys?
-			return res.send()
-		else if shouldBroadcast
+			return res.json({projectId, keys: []})
+		if shouldBroadcast
 			logger.log {projectId}, "emitting new references keys to connected clients"
 			EditorRealTimeController.emitToRoom projectId, 'references:keys:updated', data.keys
 		return res.json data

services/web/app/coffee/infrastructure/ExpressLocals.coffee

@@ -7,7 +7,7 @@ querystring = require('querystring')
 SystemMessageManager = require("../Features/SystemMessages/SystemMessageManager")
 _ = require("underscore")
 Modules = require "./Modules"
-url = require "url"
+Url = require "url"
 
 fingerprints = {}
 Path = require 'path'
@@ -51,11 +51,8 @@ getFingerprint = (path) ->
 
 logger.log "Finished generating file fingerprints"
 
-
-staticFilesBase = ""
-if Settings.cdn?.web?.host?
-	staticFilesBase = Settings.cdn?.web?.host
-
+cdnAvailable = Settings.cdn?.web?.host?
+darkCdnAvailable = Settings.cdn?.web?.darkHost?
 
 module.exports = (app, webRouter, apiRouter)->
 	webRouter.use (req, res, next)->
@@ -63,14 +60,24 @@ module.exports = (app, webRouter, apiRouter)->
 		next()
 
 	webRouter.use (req, res, next)-> 
+
+		isDark = req.headers?.host?.slice(0,4)?.toLowerCase() == "dark"
+		isSmoke = req.headers?.host?.slice(0,5)?.toLowerCase() == "smoke"
+		isLive = !isDark and !isSmoke
+
+		if cdnAvailable and isLive
+			staticFilesBase = Settings.cdn?.web?.host
+		else if darkCdnAvailable and isDark
+			staticFilesBase = Settings.cdn?.web?.darkHost
+		else
+			staticFilesBase = ""
+		
 		res.locals.jsPath = jsPath
-		res.locals.fullJsPath = url.resolve(staticFilesBase, jsPath)
+		res.locals.fullJsPath = Url.resolve(staticFilesBase, jsPath)
 
-		imgPath = "/img/"
-		cssPath = "/stylesheets/"
 
 		res.locals.buildJsPath = (jsFile, opts = {})->
-			p = Path.join(jsPath, jsFile)
+			path = Path.join(jsPath, jsFile)
 
 			doFingerPrint = opts.fingerprint != false
 			
@@ -78,23 +85,25 @@ module.exports = (app, webRouter, apiRouter)->
 				opts.qs = {}
 
 			if !opts.qs?.fingerprint? and doFingerPrint
-				opts.qs.fingerprint = getFingerprint(p)
+				opts.qs.fingerprint = getFingerprint(path)
 
-			p = url.resolve(staticFilesBase, p)
+			if opts.cdn != false
+				path = Url.resolve(staticFilesBase, path)
+				
 			qs = querystring.stringify(opts.qs)
 
 			if qs? and qs.length > 0
-				p = p + "?" + qs
-			return p
+				path = path + "?" + qs
+			return path
 
 
 		res.locals.buildCssPath = (cssFile)->
-			p = Path.join(cssPath, cssFile)
-			return url.resolve(staticFilesBase, p) + "?fingerprint=" + getFingerprint(p)
+			path = Path.join("/stylesheets/", cssFile)
+			return Url.resolve(staticFilesBase, path) + "?fingerprint=" + getFingerprint(path)
 
 		res.locals.buildImgPath = (imgFile)->
-			p = Path.join(imgPath, imgFile)
-			return url.resolve(staticFilesBase, p)
+			path = Path.join("/img/", imgFile)
+			return Url.resolve(staticFilesBase, path)
 
 		next()
 

services/web/app/coffee/infrastructure/Server.coffee

@@ -101,19 +101,19 @@ if app.get('env') == 'production'
 	logger.info "Production Enviroment"
 	app.enable('view cache')
 
-
-
 app.use (req, res, next)->
 	metrics.inc "http-request"
 	crawlerLogger.log(req)
 	next()
 
-app.use (req, res, next) ->
-	if !Settings.editorIsOpen
+webRouter.use (req, res, next) ->
+	if Settings.editorIsOpen
+		next()
+	else if req.url.indexOf("/admin") == 0
+		next()
+	else
 		res.status(503)
 		res.render("general/closed", {title:"maintenance"})
-	else
-		next()
 
 apiRouter.get "/status", (req, res)->
 	res.send("web sharelatex is alive")

services/web/app/views/layout.jade

@@ -56,9 +56,6 @@ html(itemscope, itemtype='http://schema.org/Product')
 				Countly.url = '#{settings.analytics.countly.server}';
 				!{ session.user ? 'Countly.device_id = "' + session.user._id + '";' : '' } 
 
-				Countly.q.push(['track_sessions']);
-				Countly.q.push(['track_pageview']);
-
 				(function() {
 					var cly = document.createElement('script'); cly.type = 'text/javascript';
 					cly.async = true;

services/web/app/views/project/editor.jade

@@ -129,7 +129,7 @@ block content
 
 	- var pdfPath = 'libs/pdfjs-1.3.91/pdf.worker.js'
 	- var fingerprintedPath = fingerprint(jsPath+pdfPath)
-	- var pdfJsWorkerPath = buildJsPath(pdfPath, {qs:{fingerprint:fingerprintedPath}})
+	- var pdfJsWorkerPath = buildJsPath(pdfPath, {cdn:false,qs:{fingerprint:fingerprintedPath}}) // don't use worker for cdn
 
 
 	script(type='text/javascript').

services/web/app/views/project/editor/pdf.jade

@@ -108,7 +108,7 @@ div.full-size.pdf(ng-controller="PdfController")
 					'alert-info':    entry.level == 'typesetting'\
 				}"
 				ng-click="openInEditor(entry)"
-				ng-init="feedbackSent = false;"
+				ng-init="feedbackSent = false; showNegFeedbackUI = false; negFeedbackReason = ''; negFeedbackReasonFreeText = ''"
 			)
 				span.line-no
 					i.fa.fa-link(aria-hidden="true")
@@ -135,7 +135,7 @@ div.full-size.pdf(ng-controller="PdfController")
 								i.fa.fa-external-link
 								| #{translate("log_hint_extra_info")}
 						.card-hint-feedback(
-							ng-show="!feedbackSent"
+							ng-hide="feedbackSent || showNegFeedbackUI"
 							ng-class="entry.ruleId"
 						)
 							label.card-hint-feedback-label #{translate("log_hint_feedback_label")}
@@ -145,11 +145,57 @@ div.full-size.pdf(ng-controller="PdfController")
 							) #{translate("answer_yes")}
 							span  / 
 							a.card-hint-feedback-negative(
-								ng-click="trackLogHintsNegativeFeedback(entry.ruleId); feedbackSent = true;"
+								ng-click="trackLogHintsNegativeFeedback(entry.ruleId); showNegFeedbackUI = true;"
 								href
 							) #{translate("answer_no")}
+						.card-hint-extra-feedback(ng-hide="!showNegFeedbackUI || feedbackSent")
+							p.card-hint-extra-feedback-label #{translate("log_hint_ask_extra_feedback")}
+							.radio: label
+								input(
+									type="radio"
+									name="{{ 'neg-feedback-reason-' + $index }}"
+									ng-model="negFeedbackReason"
+									value="{{ logHintsNegFeedbackValues.DIDNT_UNDERSTAND }}"
+								)
+								| #{translate("log_hint_extra_feedback_didnt_understand")}
+							.radio: label
+								input(
+									type="radio"
+									name="{{ 'neg-feedback-reason-' + $index }}"
+									ng-model="negFeedbackReason"
+									value="{{ logHintsNegFeedbackValues.NOT_APPLICABLE }}"
+								)
+								| #{translate("log_hint_extra_feedback_not_applicable")}
+							.radio: label
+								input(
+									type="radio"
+									name="{{ 'neg-feedback-reason-' + $index }}"
+									ng-model="negFeedbackReason"
+									value="{{ logHintsNegFeedbackValues.INCORRECT }}"
+								)
+								| #{translate("log_hint_extra_feedback_incorrect")}
+							.radio: label
+								input(
+									type="radio"
+									name="{{ 'neg-feedback-reason-' + $index }}"
+									ng-model="negFeedbackReason"
+									value="{{ logHintsNegFeedbackValues.OTHER }}"
+								)
+								| #{translate("log_hint_extra_feedback_other")}
+							textarea.form-control(
+								ng-show="negFeedbackReason === logHintsNegFeedbackValues.OTHER"
+								ng-model="negFeedbackReasonFreeText"
+								rows="2"
+							)
+							.clearfix
+								button.btn.btn-default.btn-sm.pull-right(
+									ng-disabled="!negFeedbackReason"
+									ng-click="trackLogHintsNegFeedbackDetails(entry.ruleId, negFeedbackReason, negFeedbackReasonFreeText); feedbackSent = true;"
+								) #{translate("log_hint_extra_feedback_submit")}
+
 						.card-hint-feedback(ng-show="feedbackSent")
 							label.card-hint-feedback-label #{translate("log_hint_feedback_gratitude")}
+
 				p.entry-content(ng-show="entry.content") {{ entry.content.trim() }}
 
 		p

services/web/app/views/sentry.jade

@@ -2,7 +2,7 @@
 	- if (sentrySrc.match(/^([a-z]+:)?\/\//i))
 		script(src="#{sentrySrc}")
 	- else
-		script(src=buildJsPath("libs/#{sentrySrc}"))
+		script(src=buildJsPath("libs/#{sentrySrc}", {fingerprint:false}))
 - if (typeof(sentrySrc) != "undefined")
 	script(type="text/javascript").
 		if (typeof(Raven) != "undefined" && Raven.config) {

services/web/config/settings.defaults.coffee

@@ -116,7 +116,8 @@ module.exports = settings =
 
 	# cdn:
 	# 	web:
-	# 		host:"http://www.sharelatex.dev:3000"
+	# 		host:"http://cdn.sharelatex.dev:3000"
+	#		darkHost:"http://cdn.sharelatex.dev:3000"
 
 	# Where your instance of ShareLaTeX can be found publically. Used in emails
 	# that are sent out, generated links, etc.

services/web/public/coffee/ide.coffee

@@ -71,13 +71,14 @@ define [
 
 		# Tracking code.
 		$scope.$watch "ui.view", (newView, oldView) ->
-			event_tracking.sendCountly "ide-open-view-#{ newView }" if newView?
+			if newView? and newView != "editor" and newView != "pdf"
+				event_tracking.sendCountlyOnce "ide-open-view-#{ newView }-once" 
 
 		$scope.$watch "ui.chatOpen", (isOpen) ->
-			event_tracking.sendCountly "ide-open-chat" if isOpen
+			event_tracking.sendCountlyOnce "ide-open-chat-once" if isOpen
 
 		$scope.$watch "ui.leftMenuShown", (isOpen) ->
-			event_tracking.sendCountly "ide-open-left-menu" if isOpen
+			event_tracking.sendCountlyOnce "ide-open-left-menu-once" if isOpen
 		# End of tracking code.
 
 		window._ide = ide

services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/AutoCompleteManager.coffee

@@ -134,6 +134,8 @@ define [
 						editor.completer.showPopup(editor)
 						editor.completer.cancelContextMenu()
 						$(editor.completer.popup?.container).css({'font-size': @$scope.fontSize + 'px'})
+						if editor.completer?.completions?.filtered?.length == 0
+							editor.completer.detach()
 					bindKey: "Ctrl-Space|Ctrl-Shift-Space|Alt-Space"
 				}
 

services/web/public/coffee/ide/pdf/controllers/PdfController.coffee

@@ -3,8 +3,9 @@ define [
 	"ace/ace"
 	"ide/human-readable-logs/HumanReadableLogs"
 	"libs/bib-log-parser"
+	"services/log-hints-feedback"
 ], (App, Ace, HumanReadableLogs, BibLogParser) ->
-	App.controller "PdfController", ($scope, $http, ide, $modal, synctex, event_tracking, localStorage) ->
+	App.controller "PdfController", ($scope, $http, ide, $modal, synctex, event_tracking, logHintsFeedback, localStorage) ->
 
 		# enable per-user containers by default
 		perUserCompile = true
@@ -32,6 +33,8 @@ define [
 					$scope.shouldDropUp = getFilesDropdownTopCoordAsRatio() > 0.65
 
 		# log hints tracking
+		$scope.logHintsNegFeedbackValues = logHintsFeedback.feedbackOpts
+		
 		$scope.trackLogHintsLearnMore = () ->
 			event_tracking.sendCountly "logs-hints-learn-more"
 
@@ -39,6 +42,9 @@ define [
 			event_tracking.send "log-hints", (if isPositive then "feedback-positive" else "feedback-negative"), hintId
 			event_tracking.sendCountly (if isPositive then "log-hints-feedback-positive" else "log-hints-feedback-negative"), { hintId }
 
+		$scope.trackLogHintsNegFeedbackDetails = (hintId, feedbackOpt, feedbackOtherVal) ->
+			logHintsFeedback.submitFeedback hintId, feedbackOpt, feedbackOtherVal
+
 		$scope.trackLogHintsPositiveFeedback = (hintId) -> trackLogHintsFeedback true, hintId
 		$scope.trackLogHintsNegativeFeedback = (hintId) -> trackLogHintsFeedback false, hintId
 
@@ -327,7 +333,7 @@ define [
 
 		$scope.toggleLogs = () ->
 			$scope.shouldShowLogs = !$scope.shouldShowLogs
-			event_tracking.sendCountly "ide-open-logs" if $scope.shouldShowLogs
+			event_tracking.sendCountlyOnce "ide-open-logs-once" if $scope.shouldShowLogs
 
 		$scope.showPdf = () ->
 			$scope.pdf.view = "pdf"
@@ -493,7 +499,7 @@ define [
 
 	App.controller "PdfLogEntryController", ["$scope", "ide", "event_tracking", ($scope, ide, event_tracking) ->
 		$scope.openInEditor = (entry) ->
-			event_tracking.sendCountly 'logs-jump-to-location'
+			event_tracking.sendCountlyOnce "logs-jump-to-location-once"
 			entity = ide.fileTreeManager.findEntityByPath(entry.file)
 			return if !entity? or entity.type != "doc"
 			if entry.line?

services/web/public/coffee/ide/share/controllers/ShareController.coffee

@@ -3,7 +3,7 @@ define [
 ], (App) ->
 	App.controller "ShareController", ["$scope", "$modal", "event_tracking", ($scope, $modal, event_tracking) ->
 		$scope.openShareProjectModal = () ->
-			event_tracking.sendCountly "ide-open-share-modal"
+			event_tracking.sendCountlyOnce "ide-open-share-modal-once"
 
 			$modal.open(
 				templateUrl: "shareProjectModalTemplate"

services/web/public/coffee/main/event.coffee

@@ -1,8 +1,30 @@
 define [
 	"base"
+	"modules/localStorage"
 ], (App) ->
+	CACHE_KEY = "countlyEvents"
+
+	App.factory "event_tracking", (localStorage) ->
+		_getEventCache = () -> 
+			eventCache = localStorage CACHE_KEY
+
+			# Initialize as an empy object if the event cache is still empty.
+			if !eventCache?
+				eventCache = {}
+				localStorage CACHE_KEY, eventCache 
+
+			return eventCache
+
+		_eventInCache = (key) ->
+			curCache = _getEventCache()
+			curCache[key] || false
+
+		_addEventToCache = (key) ->
+			curCache = _getEventCache()
+			curCache[key] = true
+
+			localStorage CACHE_KEY, curCache
 
-	App.factory "event_tracking", ->
 		return {
 			send: (category, action, label, value)->
 				ga('send', 'event', category, action, label, value)
@@ -10,10 +32,15 @@ define [
 			sendCountly: (key, segmentation) ->
 				eventData = { key }
 				eventData.segmentation = segmentation if segmentation?				
-				Countly?.q.push([ "add_event", eventData ]);
+				Countly?.q.push([ "add_event", eventData ])
 
 			sendCountlySampled: (key, segmentation) ->
 				@sendCountly key, segmentation if Math.random() < .01 
+
+			sendCountlyOnce: (key, segmentation) ->
+				if ! _eventInCache(key)
+					_addEventToCache(key)
+					@sendCountly key, segmentation
 		}
 
 	# App.directive "countlyTrack", () ->

services/web/public/coffee/services/log-hints-feedback.coffee

@@ -0,0 +1,58 @@
+define [
+	"base"
+], (App) ->
+	App.factory "logHintsFeedback", ($http, $q) ->
+		hintsFeedbackFormAPIHash = "rl4xgvr1v5t64a"
+		idStampVal = "OPkEWEFHUFAm7hKlraQMhiOXQabafWo8NipRvLT397w="
+		hintFieldAPIId = "3"
+		reasonFieldAPIId = "1"
+		reasonOtherFieldAPIId = "1_other_other"
+		submitEndpoint = "https://sharelatex.wufoo.eu/forms/#{ hintsFeedbackFormAPIHash }/#public"
+
+		feedbackOpts =
+			DIDNT_UNDERSTAND: "didnt_understand"
+			NOT_APPLICABLE: "not_applicable"
+			INCORRECT: "incorrect"
+			OTHER: "other"
+
+		createRequest = (hintId, feedbackOpt, feedbackOtherVal = "") ->
+			formData = new FormData()
+
+			formData.append "Field#{ hintFieldAPIId }", hintId
+			formData.append "Field#{ reasonFieldAPIId }", feedbackOpt
+			formData.append "idstamp", idStampVal
+
+			# Allow user to specify "other" without any extra details; an empty string 
+			# would trigger an error submitting.
+			if feedbackOpt == feedbackOpts.OTHER and feedbackOtherVal == ""
+				formData.append "Field#{ reasonOtherFieldAPIId }", "#{ feedbackOpts.OTHER } empty"
+			else
+				formData.append "Field#{ reasonOtherFieldAPIId }", feedbackOtherVal
+
+			req =
+				method: 'POST'
+				url: submitEndpoint
+				# This will effectively disable Angular's default serialization mechanisms,
+				# forcing the XHR to be done with whatever data we provide (in this case,
+				# form data). Without this, Angular will forcefully try to serialize data
+				# to JSON.
+				transformRequest: angular.identity
+				data: formData
+				headers : 
+					# This will tell Angular to use the browser-provided value, which is
+					# computed according to the data being sent (in this case, multipart
+					# form + browser-specific multipart boundary). Without this, Angular
+					# will set JSON.
+					"Content-Type": undefined
+
+			return req
+
+		submitFeedback = (hintId, feedbackOpt, feedbackOtherVal = "") ->
+			submitRequest = createRequest hintId, feedbackOpt, feedbackOtherVal
+			$http(submitRequest)
+
+		service =
+			feedbackOpts: feedbackOpts
+			submitFeedback: submitFeedback
+		
+		return service

services/web/public/stylesheets/app/editor.less

@@ -252,7 +252,7 @@
 	margin-bottom:0px;
 }
 
-.sl_references_search_hint {
+.sl_references_search_hint-varDefault {
 	position: absolute;
 	bottom: -22px;
 	left: -1px;
@@ -262,10 +262,49 @@
 	background: rgb(202, 214, 250);
 	border: 1px solid lightgray;
 	box-shadow: 3px 3px 5px rgba(0,0,0,.2);
+
 	span {
 		color: black;
 	}
 }
+
+.sl_references_search_hint-varButton {
+	position: absolute;
+	bottom: -65px;
+	left: -1px;
+	right: 0px;
+	padding: 0 6px 6px;
+	text-align: center;
+	background: #fbfbfb;
+	color: #FFF;
+	box-shadow: 3px 3px 5px rgba(0,0,0,.2);
+	border-left: 1px solid lightgray;
+	font-family: @font-family-sans-serif;
+	font-size: 13px;
+	font-weight: 600;
+
+	hr {
+		margin: 6px -6px;
+	}
+
+	button {
+		width: 100%;
+		font-size: inherit;
+		line-height: 1.4;
+	}
+
+	span {
+		padding-left: 10px;
+	}
+
+	kbd {
+		display: block;
+		font-family: inherit;
+		font-size: 12px;
+		font-weight: normal;
+	}
+}
+
 // -- References Search Modal --
 .references-search-modal-backdrop {
 	// don't grey out the editor when the

services/web/public/stylesheets/app/editor/pdf.less

@@ -203,6 +203,17 @@
 	white-space: nowrap;
 }
 
+
+@keyframes expand-feedback-area {
+	from {
+		max-height: 0;
+	}
+
+	to {
+		max-height: 500px;
+	}
+}
+
 .card-hint:extend(.card-thin) {
 	margin-top: 10px;
 	padding-bottom: 7px;
@@ -280,9 +291,39 @@
 		float: right;
 	}
 
+	&-extra-feedback {
+		color: @gray-dark;
+		font-size: 0.8rem;
+		margin-top: 10px;
+		padding-bottom: 5px;
+		animation: 0.5s ease-out expand-feedback-area;
+		overflow: hidden;
+
+		&-label {
+			margin: 5px 0 10px;
+			padding-top: 5px;
+			border-top: solid 1px @gray-lighter;
+		}
+
+		.radio {
+			margin: 5px;
+		}
+
+		textarea {
+			font-size: 0.8rem;
+			margin-bottom: 10px;
+			padding: 5px;
+		}
+
+		input[type="radio"] {
+			margin-top: 2px;
+		}
+	}
+
 	& + p {
 		margin-top: 20px;
 	}
+
 }
 
 .files-dropdown-container {

services/web/test/UnitTests/coffee/References/ReferencesControllerTests.coffee

@@ -134,9 +134,10 @@ describe "ReferencesController", ->
 					@res.sendStatus.calledWith(400).should.equal false
 					done()
 
-			it 'should close the response', (done) ->
+			it 'should send a response with an empty keys list', (done) ->
 				@call () =>
-					@res.send.called.should.equal true
+					@res.json.called.should.equal true
+					@res.json.calledWith({projectId: @projectId, keys: []}).should.equal true
 					done()
 
 	describe 'index', ->