Merge pull request #295 from sharelatex/ja-postgres-analytics

Postgres analytics

services/web/app/coffee/Features/Analytics/AnalyticsController.coffee

@@ -0,0 +1,7 @@
+AnalyticsManager = require "./AnalyticsManager"
+
+module.exports = AnalyticsController =
+	recordEvent: (req, res, next) ->
+		AnalyticsManager.recordEvent req.session?.user?._id, req.params.event, req.body, (error) ->
+			return next(error) if error?
+			res.send 204

services/web/app/coffee/Features/Analytics/AnalyticsManager.coffee

@@ -0,0 +1,43 @@
+Settings = require "settings-sharelatex"
+logger = require "logger-sharelatex"
+_ = require "underscore"
+
+if !Settings.analytics?.postgres?
+	module.exports =
+		recordEvent: (user_id, event, segmentation, callback = () ->) ->
+			logger.log {user_id, event, segmentation}, "no event tracking configured, logging event"
+			callback()
+else
+	Sequelize = require "sequelize"
+	options = _.extend {logging:false}, Settings.analytics.postgres
+
+	sequelize = new Sequelize(
+		Settings.analytics.postgres.database,
+		Settings.analytics.postgres.username,
+		Settings.analytics.postgres.password,
+		options
+	)
+	
+	Event = sequelize.define("Event", {
+		user_id: Sequelize.STRING,
+		event: Sequelize.STRING,
+		segmentation: Sequelize.JSONB
+	})
+
+	module.exports =
+		recordEvent: (user_id, event, segmentation = {}, callback = (error) ->) ->
+			if user_id? and typeof(user_id) != "string"
+				user_id = user_id.toString()
+			if user_id == Settings.smokeTest?.userId
+				# Don't record smoke tests analytics
+				return callback()
+			Event
+				.create({ user_id, event, segmentation })
+				.then(
+					(result) -> callback(),
+					(error) ->
+						logger.err {err: error, user_id, event, segmentation}, "error recording analytics event"
+						callback(error)
+				)
+			
+		sync: () -> sequelize.sync()

services/web/app/coffee/router.coffee

@@ -38,6 +38,7 @@ ContactRouter = require("./Features/Contacts/ContactRouter")
 ReferencesController = require('./Features/References/ReferencesController')
 AuthorizationMiddlewear = require('./Features/Authorization/AuthorizationMiddlewear')
 BetaProgramController = require('./Features/BetaProgram/BetaProgramController')
+AnalyticsRouter = require('./Features/Analytics/AnalyticsRouter')
 
 logger = require("logger-sharelatex")
 _ = require("underscore")
@@ -68,7 +69,8 @@ module.exports = class Router
 		StaticPagesRouter.apply(webRouter, apiRouter)
 		RealTimeProxyRouter.apply(webRouter, apiRouter)
 		ContactRouter.apply(webRouter, apiRouter)
-
+		AnalyticsRouter.apply(webRouter, apiRouter)
+		
 		Modules.applyRouter(webRouter, apiRouter)
 
 
@@ -280,4 +282,4 @@ module.exports = class Router
 			metrics.inc("client-side-error")
 			res.sendStatus(204)
 
-		webRouter.get '*', ErrorController.notFound
+		webRouter.get '*', ErrorController.notFound

services/web/app/views/layout.jade

@@ -46,76 +46,6 @@ html(itemscope, itemtype='http://schema.org/Product')
 		- else
 			script(type='text/javascript').
 				window.ga = function() { console.log("would send to GA", arguments) };
-
-		// Countly Analytics
-		if (settings.analytics && settings.analytics.countly && settings.analytics.countly.token)
-			script(type="text/javascript").
-				var Countly = Countly || {};
-				Countly.q = Countly.q || [];
-				Countly.app_key = '#{settings.analytics.countly.token}';
-				Countly.url = '#{settings.analytics.countly.server}';
-				!{ session.user ? 'Countly.device_id = "' + session.user._id + '";' : '' } 
-
-				(function() {
-					var cly = document.createElement('script'); cly.type = 'text/javascript';
-					cly.async = true;
-					//enter url of script here
-					cly.src = 'https://cdnjs.cloudflare.com/ajax/libs/countly-sdk-web/16.6.0/countly.min.js';
-					cly.onload = function(){Countly.init()};
-					var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(cly, s);
-				})();
-
-			script(type="text/javascript")
-				if (session && session.user)
-					- var name = session.user.first_name + (session.user.last_name ? ' ' + session.user.last_name : '');
-					| Countly.q.push(['user_details', { email: '#{session.user.email}', name: '#{name}' }]);
-
-				if (justRegistered)
-					| Countly.q.push(['add_event',{ key: 'user-registered' }]);
-
-				if (justLoggedIn)
-					| Countly.q.push(['add_event',{ key: 'user-logged-in' }]);
-
-				if (user && user.features) 
-					- featureFlagSet = false;
-
-					if user.features.hasOwnProperty('collaborators')
-						| Countly.q.push([ 'userData.set', 'features-collaborators', #{ user.features.collaborators } ]);
-						- featureFlagSet = true;
-
-					if user.features.hasOwnProperty('compileGroup')
-						| Countly.q.push([ 'userData.set', 'features-compileGroup', '#{ user.features.compileGroup }' ]);
-						- featureFlagSet = true;
-
-					if user.features.hasOwnProperty('compileTimeout')
-						| Countly.q.push([ 'userData.set', 'features-compileTimeout', #{ user.features.compileTimeout } ]);
-						- featureFlagSet = true;
-
-					if user.features.hasOwnProperty('dropbox')
-						| Countly.q.push([ 'userData.set', 'features-dropbox', #{ user.features.dropbox } ]);
-						- featureFlagSet = true;
-
-					if user.features.hasOwnProperty('github')
-						| Countly.q.push([ 'userData.set', 'features-github', #{ user.features.github } ]);
-						- featureFlagSet = true;
-
-					if user.features.hasOwnProperty('references')
-						| Countly.q.push([ 'userData.set', 'features-references', #{ user.features.references } ]);
-						- featureFlagSet = true;
-
-					if user.features.hasOwnProperty('templates')
-						| Countly.q.push([ 'userData.set', 'features-templates', #{ user.features.templates } ]);
-						- featureFlagSet = true;
-
-					if user.features.hasOwnProperty('versioning')
-						| Countly.q.push([ 'userData.set', 'features-versioning', #{ user.features.versioning } ]);
-						- featureFlagSet = true;
-
-
-					if featureFlagSet
-						| Countly.q.push(['userData.save'])
-
-		// End countly Analytics
 				
 		script(type="text/javascript").
 			window.csrfToken = "#{csrfToken}";

services/web/package.json

@@ -46,6 +46,8 @@
     "nodemailer-sendgrid-transport": "^0.2.0",
     "nodemailer-ses-transport": "^1.3.0",
     "optimist": "0.6.1",
+    "pg": "^6.0.3",
+    "pg-hstore": "^2.3.2",
     "redback": "0.4.0",
     "redis": "0.10.1",
     "redis-sharelatex": "0.0.9",
@@ -53,6 +55,7 @@
     "requests": "^0.1.7",
     "rimraf": "2.2.6",
     "sanitizer": "0.1.1",
+    "sequelize": "^3.2.0",
     "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.0.0",
     "sixpack-client": "^1.0.0",
     "temp": "^0.8.3",

services/web/public/coffee/ide.coffee

@@ -74,16 +74,16 @@ define [
 		# Tracking code.
 		$scope.$watch "ui.view", (newView, oldView) ->
 			if newView? and newView != "editor" and newView != "pdf"
-				event_tracking.sendCountlyOnce "ide-open-view-#{ newView }-once" 
+				event_tracking.sendMBOnce "ide-open-view-#{ newView }-once" 
 
 		$scope.$watch "ui.chatOpen", (isOpen) ->
-			event_tracking.sendCountlyOnce "ide-open-chat-once" if isOpen
+			event_tracking.sendMBOnce "ide-open-chat-once" if isOpen
 
 		$scope.$watch "ui.leftMenuShown", (isOpen) ->
-			event_tracking.sendCountlyOnce "ide-open-left-menu-once" if isOpen
+			event_tracking.sendMBOnce "ide-open-left-menu-once" if isOpen
 
 		$scope.trackHover = (feature) ->
-			event_tracking.sendCountlyOnce "ide-hover-#{feature}-once"
+			event_tracking.sendMBOnce "ide-hover-#{feature}-once"
 		# End of tracking code.
 
 		window._ide = ide

services/web/public/coffee/ide/hotkeys/controllers/HotkeysController.coffee

@@ -4,7 +4,7 @@ define [
 ], (App) ->
 	App.controller "HotkeysController", ($scope, $modal, event_tracking) ->
 		$scope.openHotkeysModal = ->
-			event_tracking.sendCountly "ide-open-hotkeys-modal"
+			event_tracking.sendMB "ide-open-hotkeys-modal"
 
 			$modal.open {
 				templateUrl: "hotkeysModalTemplate"

services/web/public/coffee/ide/pdf/controllers/PdfController.coffee

@@ -36,11 +36,11 @@ define [
 		$scope.logHintsNegFeedbackValues = logHintsFeedback.feedbackOpts
 		
 		$scope.trackLogHintsLearnMore = () ->
-			event_tracking.sendCountly "logs-hints-learn-more"
+			event_tracking.sendMB "logs-hints-learn-more"
 
 		trackLogHintsFeedback = (isPositive, hintId) ->
 			event_tracking.send "log-hints", (if isPositive then "feedback-positive" else "feedback-negative"), hintId
-			event_tracking.sendCountly (if isPositive then "log-hints-feedback-positive" else "log-hints-feedback-negative"), { hintId }
+			event_tracking.sendMB (if isPositive then "log-hints-feedback-positive" else "log-hints-feedback-negative"), { hintId }
 
 		$scope.trackLogHintsNegFeedbackDetails = (hintId, feedbackOpt, feedbackOtherVal) ->
 			logHintsFeedback.submitFeedback hintId, feedbackOpt, feedbackOtherVal
@@ -338,7 +338,7 @@ define [
 		$scope.recompile = (options = {}) ->
 			return if $scope.pdf.compiling
 
-			event_tracking.sendCountlySampled "editor-recompile-sampled", options
+			event_tracking.sendMBSampled "editor-recompile-sampled", options
 
 			$scope.pdf.compiling = true
 
@@ -387,7 +387,7 @@ define [
 
 		$scope.toggleLogs = () ->
 			$scope.shouldShowLogs = !$scope.shouldShowLogs
-			event_tracking.sendCountlyOnce "ide-open-logs-once" if $scope.shouldShowLogs
+			event_tracking.sendMBOnce "ide-open-logs-once" if $scope.shouldShowLogs
 
 		$scope.showPdf = () ->
 			$scope.pdf.view = "pdf"
@@ -395,7 +395,7 @@ define [
 
 		$scope.toggleRawLog = () ->
 			$scope.pdf.showRawLog = !$scope.pdf.showRawLog
-			event_tracking.sendCountly "logs-view-raw" if $scope.pdf.showRawLog
+			event_tracking.sendMB "logs-view-raw" if $scope.pdf.showRawLog
 
 		$scope.openClearCacheModal = () ->
 			modalInstance = $modal.open(
@@ -430,7 +430,7 @@ define [
 		$scope.startFreeTrial = (source) ->
 			ga?('send', 'event', 'subscription-funnel', 'compile-timeout', source)
 
-			event_tracking.sendCountly "subscription-start-trial", { source }
+			event_tracking.sendMB "subscription-start-trial", { source }
 
 			window.open("/user/subscription/new?planCode=student_free_trial_7_days")
 			$scope.startedFreeTrial = true
@@ -553,7 +553,7 @@ define [
 
 	App.controller "PdfLogEntryController", ["$scope", "ide", "event_tracking", ($scope, ide, event_tracking) ->
 		$scope.openInEditor = (entry) ->
-			event_tracking.sendCountlyOnce "logs-jump-to-location-once"
+			event_tracking.sendMBOnce "logs-jump-to-location-once"
 			entity = ide.fileTreeManager.findEntityByPath(entry.file)
 			return if !entity? or entity.type != "doc"
 			if entry.line?

services/web/public/coffee/ide/settings/services/settings.coffee

@@ -8,7 +8,7 @@ define [
 				for key in Object.keys(data)
 					changedSetting = key
 					changedSettingVal = data[key]
-					event_tracking.sendCountly "setting-changed", { changedSetting, changedSettingVal }
+					event_tracking.sendMB "setting-changed", { changedSetting, changedSettingVal }
 				# End of tracking code.
 				
 				data._csrf = window.csrfToken
@@ -20,7 +20,7 @@ define [
 				for key in Object.keys(data)
 					changedSetting = key
 					changedSettingVal = data[key]
-					event_tracking.sendCountly "project-setting-changed", { changedSetting, changedSettingVal}
+					event_tracking.sendMB "project-setting-changed", { changedSetting, changedSettingVal}
 				# End of tracking code.
 				
 				data._csrf = window.csrfToken
@@ -32,7 +32,7 @@ define [
 				for key in Object.keys(data)
 					changedSetting = key
 					changedSettingVal = data[key]
-					event_tracking.sendCountly "project-admin-setting-changed", { changedSetting, changedSettingVal }
+					event_tracking.sendMB "project-admin-setting-changed", { changedSetting, changedSettingVal }
 				# End of tracking code.
 				
 				data._csrf = window.csrfToken

services/web/public/coffee/ide/share/controllers/ShareController.coffee

@@ -3,7 +3,7 @@ define [
 ], (App) ->
 	App.controller "ShareController", ["$scope", "$modal", "event_tracking", ($scope, $modal, event_tracking) ->
 		$scope.openShareProjectModal = () ->
-			event_tracking.sendCountlyOnce "ide-open-share-modal-once"
+			event_tracking.sendMBOnce "ide-open-share-modal-once"
 
 			$modal.open(
 				templateUrl: "shareProjectModalTemplate"

services/web/public/coffee/ide/track-changes/controllers/TrackChangesDiffController.coffee

@@ -3,7 +3,7 @@ define [
 ], (App) ->
 	App.controller "TrackChangesDiffController", ($scope, $modal, ide, event_tracking) ->
 		$scope.restoreDeletedDoc = () ->
-			event_tracking.sendCountly "track-changes-restore-deleted"
+			event_tracking.sendMB "track-changes-restore-deleted"
 			$scope.trackChanges.diff.restoreInProgress = true
 			ide.trackChangesManager
 				.restoreDeletedDoc(
@@ -15,7 +15,7 @@ define [
 					$scope.trackChanges.diff.restoreDeletedSuccess = true
 
 		$scope.openRestoreDiffModal = () ->
-			event_tracking.sendCountly "track-changes-restore-modal"
+			event_tracking.sendMB "track-changes-restore-modal"
 			$modal.open {
 				templateUrl: "trackChangesRestoreDiffModalTemplate"
 				controller: "TrackChangesRestoreDiffModalController"
@@ -33,7 +33,7 @@ define [
 		$scope.diff = diff
 
 		$scope.restore = () ->
-			event_tracking.sendCountly "track-changes-restored"
+			event_tracking.sendMB "track-changes-restored"
 			$scope.state.inflight = true
 			ide.trackChangesManager
 				.restoreDiff(diff)

services/web/public/coffee/ide/track-changes/controllers/TrackChangesListController.coffee

@@ -92,7 +92,7 @@ define [
 				$scope.recalculateSelectedUpdates()
 
 		$scope.select = () ->
-			event_tracking.sendCountly "track-changes-view-change"
+			event_tracking.sendMB "track-changes-view-change"
 			$scope.update.selectedTo = true
 			$scope.update.selectedFrom = true
 

services/web/public/coffee/main/account-upgrade.coffee

@@ -6,7 +6,7 @@ define [
 		$scope.buttonClass = "btn-primary"
 
 		$scope.startFreeTrial = (source, couponCode) ->
-			event_tracking.sendCountly "subscription-start-trial", { source }
+			event_tracking.sendMB "subscription-start-trial", { source }
 
 			w = window.open()
 			sixpack.convert "track-changes-discount", ->

services/web/public/coffee/main/contact-us.coffee

@@ -24,7 +24,7 @@ define [
 					url :"/learn/kb/#{page_underscored}"
 					name : hit._highlightResult.pageName.value
 
-			event_tracking.sendCountly "contact-form-suggestions-shown" if results.hits.length
+			event_tracking.sendMB "contact-form-suggestions-shown" if results.hits.length
 
 			$scope.$applyAsync () -> 
 				$scope.suggestions = suggestions
@@ -60,7 +60,7 @@ define [
 				$scope.suggestions = [];
 
 		$scope.clickSuggestionLink = (url) ->
-			event_tracking.sendCountly "contact-form-suggestions-clicked", { url }
+			event_tracking.sendMB "contact-form-suggestions-clicked", { url }
 
 		$scope.close = () ->
 			$modalInstance.close()

services/web/public/coffee/main/event.coffee

@@ -2,9 +2,14 @@ define [
 	"base"
 	"modules/localStorage"
 ], (App) ->
-	CACHE_KEY = "countlyEvents"
+	CACHE_KEY = "mbEvents"
 
-	App.factory "event_tracking", (localStorage) ->
+	send = (category, action, attributes = {})->
+		ga('send', 'event', category, action)
+		event_name = "#{action}-#{category}"
+		Intercom?("trackEvent", event_name, attributes)
+		
+	App.factory "event_tracking", ($http, localStorage) ->
 		_getEventCache = () -> 
 			eventCache = localStorage CACHE_KEY
 
@@ -29,18 +34,23 @@ define [
 			send: (category, action, label, value)->
 				ga('send', 'event', category, action, label, value)
 
-			sendCountly: (key, segmentation) ->
-				eventData = { key }
-				eventData.segmentation = segmentation if segmentation?				
-				Countly?.q.push([ "add_event", eventData ])
+			sendMB: (key, segmentation = {}) ->
+				$http {
+					url: "/event/#{key}",
+					method: "POST",
+					data: segmentation
+					headers: {
+						"X-CSRF-Token": window.csrfToken
+					}
+				}
 
-			sendCountlySampled: (key, segmentation) ->
-				@sendCountly key, segmentation if Math.random() < .01 
+			sendMBSampled: (key, segmentation) ->
+				@sendMB key, segmentation if Math.random() < .01 
 
-			sendCountlyOnce: (key, segmentation) ->
+			sendMBOnce: (key, segmentation) ->
 				if ! _eventInCache(key)
 					_addEventToCache(key)
-					@sendCountly key, segmentation
+					@sendMB key, segmentation
 		}
 
 	# App.directive "countlyTrack", () ->

services/web/public/coffee/main/new-subscription.coffee

@@ -51,7 +51,7 @@ define [
 			.done()
 
 		pricing.on "change", =>
-			event_tracking.sendCountly "subscription-form", { plan : pricing.items.plan.code }
+			event_tracking.sendMB "subscription-form", { plan : pricing.items.plan.code }
 
 			$scope.planName = pricing.items.plan.name
 			$scope.price = pricing.price
@@ -125,7 +125,7 @@ define [
 							state:       $scope.data.state
 							postal_code: $scope.data.postal_code
 				
-				event_tracking.sendCountly "subscription-form-submitted", { 
+				event_tracking.sendMB "subscription-form-submitted", { 
 					currencyCode	: postData.subscriptionDetails.currencyCode,
 					plan_code		: postData.subscriptionDetails.plan_code,
 					coupon_code		: postData.subscriptionDetails.coupon_code,
@@ -135,7 +135,7 @@ define [
 				$http.post("/user/subscription/create", postData)
 					.success (data, status, headers)->
 						sixpack.convert "in-editor-free-trial-plan", pricing.items.plan.code, (err)->
-							event_tracking.sendCountly "subscription-submission-success"
+							event_tracking.sendMB "subscription-submission-success"
 							window.location.href = "/user/subscription/thank-you"
 					.error (data, status, headers)->
 						$scope.processing = false