Migrating reference settings to mongo and exposing them through code … (#31726)

* Migrating reference settings to mongo and exposing them through code mirror context

* removing undefined optional for reference manager type settings

* allowing partial updates to user settings objects, and remove repeated cypress intercepts

* Zod schema parsing for user reference manager settings

* Splitting ref provider mongoose schema into const

* Persisting local storage reference settings to mongo and deleting from local

* Enforcing ref provider group id to always be type string

* Fixing test and format errors

* Migrated flag for settings to only migrate once

* fixing cypress tests adding migrated flag

* persisting local storage to allow for easy rollback

GitOrigin-RevId: f59522bdee6f0e56efb7d98b9d9373a743619ec8

services/web/app/src/Features/Project/UserSettingsHelper.mjs

@@ -45,6 +45,9 @@ async function buildUserSettings(req, res, user) {
     breadcrumbs: user.ace.breadcrumbs,
     referencesSearchMode: user.ace.referencesSearchMode,
     darkModePdf: user.ace.darkModePdf ?? false,
+    zotero: user.ace.zotero,
+    mendeley: user.ace.mendeley,
+    papers: user.ace.papers,
   }
 }
 

services/web/app/src/Features/User/UserController.mjs

@@ -329,11 +329,23 @@ async function unsubscribe(req, res, next) {
   })
 }
 
+const refProviderSettingsSchema = z
+  .object({
+    enabled: z.boolean().optional(),
+    groups: z.array(z.object({ id: z.string() })).optional(),
+    disablePersonalLibrary: z.boolean().optional(),
+    migrated: z.boolean().optional(),
+  })
+  .optional()
+
 const updateUserSettingsSchema = z.object({
   body: z
     .object({
       first_name: z.string().max(255).nullish(),
       last_name: z.string().max(255).nullish(),
+      zotero: refProviderSettingsSchema,
+      mendeley: refProviderSettingsSchema,
+      papers: refProviderSettingsSchema,
     })
     .passthrough(),
   // TODO: complete the schema and remove the passthrough
@@ -413,6 +425,15 @@ async function updateUserSettings(req, res, next) {
   if (body.darkModePdf != null) {
     user.ace.darkModePdf = Boolean(body.darkModePdf)
   }
+  if (body.zotero != null) {
+    user.ace.zotero = { ...user.ace.zotero, ...body.zotero }
+  }
+  if (body.mendeley != null) {
+    user.ace.mendeley = { ...user.ace.mendeley, ...body.mendeley }
+  }
+  if (body.papers != null) {
+    user.ace.papers = { ...user.ace.papers, ...body.papers }
+  }
   await user.save()
 
   const newEmail = body.email?.trim().toLowerCase()

services/web/app/src/models/User.mjs

@@ -7,6 +7,19 @@ const { ObjectId } = Schema
 // See https://stackoverflow.com/questions/386294/what-is-the-maximum-length-of-a-valid-email-address/574698#574698
 const MAX_EMAIL_LENGTH = 254
 const MAX_NAME_LENGTH = 255
+const refProviderSettingsSchema = {
+  enabled: { type: Boolean, default: true },
+  groups: {
+    type: [
+      {
+        id: { type: String },
+      },
+    ],
+    default: [],
+  },
+  disablePersonalLibrary: { type: Boolean, default: false },
+  migrated: { type: Boolean, default: false },
+}
 
 export const UserSchema = new Schema(
   {
@@ -94,6 +107,9 @@ export const UserSchema = new Schema(
       breadcrumbs: { type: Boolean, default: true },
       referencesSearchMode: { type: String, default: 'advanced' }, // 'advanced' or 'simple'
       darkModePdf: { type: Boolean, default: false },
+      zotero: refProviderSettingsSchema,
+      mendeley: refProviderSettingsSchema,
+      papers: refProviderSettingsSchema,
     },
     features: {
       collaborators: {

services/web/frontend/js/features/editor-left-menu/context/project-settings-context.tsx

@@ -33,6 +33,9 @@ type ProjectSettingsSetterContextValue = {
   setMathPreview: (mathPreview: UserSettings['mathPreview']) => void
   setBreadcrumbs: (breadcrumbs: UserSettings['breadcrumbs']) => void
   setDarkModePdf: (darkModePdf: UserSettings['darkModePdf']) => void
+  setZotero: (zotero: UserSettings['zotero']) => void
+  setMendeley: (mendeley: UserSettings['mendeley']) => void
+  setPapers: (papers: UserSettings['papers']) => void
 }
 
 type ProjectSettingsContextValue = Partial<ProjectSettings> &
@@ -88,6 +91,12 @@ export const ProjectSettingsProvider: FC<React.PropsWithChildren> = ({
     setBreadcrumbs,
     darkModePdf,
     setDarkModePdf,
+    zotero,
+    setZotero,
+    mendeley,
+    setMendeley,
+    papers,
+    setPapers,
   } = useUserWideSettings()
 
   useProjectWideSettingsSocketListener()
@@ -132,6 +141,12 @@ export const ProjectSettingsProvider: FC<React.PropsWithChildren> = ({
       setBreadcrumbs,
       darkModePdf,
       setDarkModePdf,
+      zotero,
+      setZotero,
+      mendeley,
+      setMendeley,
+      papers,
+      setPapers,
     }),
     [
       compiler,
@@ -172,6 +187,12 @@ export const ProjectSettingsProvider: FC<React.PropsWithChildren> = ({
       setBreadcrumbs,
       darkModePdf,
       setDarkModePdf,
+      zotero,
+      setZotero,
+      mendeley,
+      setMendeley,
+      papers,
+      setPapers,
     ]
   )
 

services/web/frontend/js/features/editor-left-menu/hooks/use-user-wide-settings.tsx

@@ -24,6 +24,9 @@ export default function useUserWideSettings() {
     mathPreview,
     breadcrumbs,
     darkModePdf,
+    zotero,
+    mendeley,
+    papers,
   } = userSettings
 
   const setOverallTheme = useSetOverallTheme()
@@ -125,6 +128,27 @@ export default function useUserWideSettings() {
     [saveUserSettings]
   )
 
+  const setZotero = useCallback(
+    (zotero: UserSettings['zotero']) => {
+      saveUserSettings('zotero', { ...zotero, migrated: true })
+    },
+    [saveUserSettings]
+  )
+
+  const setMendeley = useCallback(
+    (mendeley: UserSettings['mendeley']) => {
+      saveUserSettings('mendeley', { ...mendeley, migrated: true })
+    },
+    [saveUserSettings]
+  )
+
+  const setPapers = useCallback(
+    (papers: UserSettings['papers']) => {
+      saveUserSettings('papers', { ...papers, migrated: true })
+    },
+    [saveUserSettings]
+  )
+
   return {
     autoComplete,
     setAutoComplete,
@@ -156,5 +180,11 @@ export default function useUserWideSettings() {
     setBreadcrumbs,
     darkModePdf,
     setDarkModePdf,
+    zotero,
+    setZotero,
+    mendeley,
+    setMendeley,
+    papers,
+    setPapers,
   }
 }

services/web/frontend/js/shared/context/user-settings-context.tsx

@@ -6,9 +6,13 @@ import {
   SetStateAction,
   FC,
   useState,
+  useEffect,
 } from 'react'
 import { UserSettings } from '../../../../types/user-settings'
 import getMeta from '@/utils/meta'
+import customLocalStorage from '@/infrastructure/local-storage'
+import { getLegacyWriteAndCiteMigration } from '../utils/write-and-cite-settings-migration'
+import { saveUserSettings } from '@/features/editor-left-menu/utils/api'
 
 export const defaultSettings: UserSettings = {
   pdfViewer: 'pdfjs',
@@ -27,6 +31,21 @@ export const defaultSettings: UserSettings = {
   referencesSearchMode: 'advanced',
   breadcrumbs: true,
   darkModePdf: false,
+  zotero: {
+    enabled: true,
+    groups: [],
+    disablePersonalLibrary: false,
+  },
+  mendeley: {
+    enabled: true,
+    groups: [],
+    disablePersonalLibrary: false,
+  },
+  papers: {
+    enabled: true,
+    groups: [],
+    disablePersonalLibrary: false,
+  },
 }
 
 type UserSettingsContextValue = {
@@ -47,6 +66,35 @@ export const UserSettingsProvider: FC<React.PropsWithChildren> = ({
     () => getMeta('ol-userSettings') || defaultSettings
   )
 
+  useEffect(() => {
+    const { patch, keysToRemove } = getLegacyWriteAndCiteMigration(userSettings)
+    if (keysToRemove.length === 0) {
+      return
+    }
+
+    if (Object.keys(patch).length === 0) {
+      keysToRemove.forEach(customLocalStorage.removeItem)
+      return
+    }
+
+    Promise.all(
+      Object.entries(patch).map(([key, value]) =>
+        saveUserSettings(
+          key as keyof Pick<UserSettings, 'mendeley' | 'zotero' | 'papers'>,
+          value
+        )
+      )
+    ).then(() => {
+      setUserSettings(currentSettings => ({
+        ...currentSettings,
+        ...patch,
+      }))
+      keysToRemove.forEach(customLocalStorage.removeItem)
+    })
+    // Only run once when the provider mounts
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [])
+
   const value = useMemo<UserSettingsContextValue>(
     () => ({
       userSettings,

services/web/frontend/js/shared/utils/write-and-cite-settings-migration.ts

@@ -0,0 +1,99 @@
+import customLocalStorage from '@/infrastructure/local-storage'
+import getMeta from '@/utils/meta'
+import {
+  RefProviderSettings,
+  UserSettings,
+} from '../../../../types/user-settings'
+
+type RefProvider = 'mendeley' | 'zotero' | 'papers'
+
+const providers: RefProvider[] = ['mendeley', 'zotero', 'papers']
+
+const buildLegacyKey = (userId: string, provider: RefProvider, key: string) =>
+  `user.${userId}.write-and-cite.${provider}.${key}`
+
+const isBoolean = (value: unknown): value is boolean =>
+  typeof value === 'boolean'
+
+const parseGroups = (value: unknown): { id: string }[] | undefined => {
+  return Array.isArray(value)
+    ? (value as { id: string | number }[]).map(group => ({
+        id: typeof group.id === 'number' ? String(group.id) : group.id,
+      }))
+    : undefined
+}
+
+export type LegacyWriteAndCiteMigration = {
+  patch: Partial<Pick<UserSettings, 'mendeley' | 'zotero' | 'papers'>>
+  keysToRemove: string[]
+}
+
+export const getLegacyWriteAndCiteMigration = (
+  userSettings: UserSettings
+): LegacyWriteAndCiteMigration => {
+  const userId = getMeta('ol-user_id')
+  if (!userId) {
+    return {
+      patch: {},
+      keysToRemove: [],
+    }
+  }
+
+  const patch: LegacyWriteAndCiteMigration['patch'] = {}
+  const keysToRemove: string[] = []
+
+  for (const provider of providers) {
+    const currentProviderSettings = userSettings[provider]
+
+    if (currentProviderSettings.migrated) {
+      continue
+    }
+
+    const enabledKey = buildLegacyKey(userId, provider, 'enabled')
+    const groupsKey = buildLegacyKey(userId, provider, 'groups')
+    const disablePersonalLibraryKey = buildLegacyKey(
+      userId,
+      provider,
+      'disablePersonalLibrary'
+    )
+
+    const enabledValue = customLocalStorage.getItem(enabledKey)
+    const groupsValue = customLocalStorage.getItem(groupsKey)
+    const disablePersonalLibraryValue = customLocalStorage.getItem(
+      disablePersonalLibraryKey
+    )
+
+    // Storage.getItem returns null if the key does not exist
+    const hasEnabledValue = enabledValue !== null
+    const hasGroupsValue = groupsValue !== null
+    const hasDisablePersonalLibraryValue = disablePersonalLibraryValue !== null
+
+    if (
+      !hasEnabledValue &&
+      !hasGroupsValue &&
+      !hasDisablePersonalLibraryValue
+    ) {
+      continue
+    }
+
+    const nextProviderSettings: RefProviderSettings = {
+      enabled: isBoolean(enabledValue)
+        ? enabledValue
+        : currentProviderSettings.enabled,
+      groups: parseGroups(groupsValue) ?? currentProviderSettings.groups,
+      disablePersonalLibrary: isBoolean(disablePersonalLibraryValue)
+        ? disablePersonalLibraryValue
+        : currentProviderSettings.disablePersonalLibrary,
+      migrated: true,
+    }
+
+    patch[provider] = nextProviderSettings
+
+    keysToRemove.push(enabledKey, groupsKey, disablePersonalLibraryKey)
+  }
+
+  return {
+    patch,
+    keysToRemove: [],
+  }
+}

services/web/test/unit/src/User/UserController.test.mjs

@@ -565,6 +565,126 @@ describe('UserController', function () {
       })
     })
 
+    it('should set zotero settings object', function (ctx) {
+      return new Promise(resolve => {
+        ctx.req.body = {
+          zotero: {
+            enabled: false,
+            groups: [{ id: '123' }],
+            disablePersonalLibrary: true,
+          },
+        }
+        ctx.res.sendStatus = code => {
+          ctx.user.ace.zotero.enabled.should.equal(false)
+          ctx.user.ace.zotero.groups.should.deep.equal([{ id: '123' }])
+          ctx.user.ace.zotero.disablePersonalLibrary.should.equal(true)
+          resolve()
+        }
+        ctx.UserController.updateUserSettings(ctx.req, ctx.res)
+      })
+    })
+
+    it('should set zotero settings with partial update', function (ctx) {
+      return new Promise(resolve => {
+        ctx.user.ace.zotero = {
+          enabled: true,
+          groups: [{ id: 'existing' }],
+          disablePersonalLibrary: false,
+        }
+        ctx.req.body = {
+          zotero: { enabled: false },
+        }
+        ctx.res.sendStatus = code => {
+          ctx.user.ace.zotero.enabled.should.equal(false)
+          resolve()
+        }
+        ctx.UserController.updateUserSettings(ctx.req, ctx.res)
+      })
+    })
+
+    it('should set mendeley settings object', function (ctx) {
+      return new Promise(resolve => {
+        ctx.req.body = {
+          mendeley: {
+            enabled: false,
+            groups: [{ id: 'group-456' }],
+            disablePersonalLibrary: true,
+          },
+        }
+        ctx.res.sendStatus = code => {
+          ctx.user.ace.mendeley.enabled.should.equal(false)
+          ctx.user.ace.mendeley.groups.should.deep.equal([{ id: 'group-456' }])
+          ctx.user.ace.mendeley.disablePersonalLibrary.should.equal(true)
+          resolve()
+        }
+        ctx.UserController.updateUserSettings(ctx.req, ctx.res)
+      })
+    })
+
+    it('should set mendeley with multiple groups', function (ctx) {
+      return new Promise(resolve => {
+        ctx.req.body = {
+          mendeley: {
+            enabled: true,
+            groups: [{ id: 'group-1' }, { id: 'group-2' }, { id: 'group-3' }],
+            disablePersonalLibrary: false,
+          },
+        }
+        ctx.res.sendStatus = code => {
+          ctx.user.ace.mendeley.groups.should.have.length(3)
+          ctx.user.ace.mendeley.groups[0].id.should.equal('group-1')
+          ctx.user.ace.mendeley.groups[1].id.should.equal('group-2')
+          ctx.user.ace.mendeley.groups[2].id.should.equal('group-3')
+          resolve()
+        }
+        ctx.UserController.updateUserSettings(ctx.req, ctx.res)
+      })
+    })
+
+    it('should set papers settings object', function (ctx) {
+      return new Promise(resolve => {
+        ctx.req.body = {
+          papers: {
+            enabled: true,
+            groups: [],
+            disablePersonalLibrary: false,
+          },
+        }
+        ctx.res.sendStatus = code => {
+          ctx.user.ace.papers.enabled.should.equal(true)
+          ctx.user.ace.papers.groups.should.deep.equal([])
+          ctx.user.ace.papers.disablePersonalLibrary.should.equal(false)
+          resolve()
+        }
+        ctx.UserController.updateUserSettings(ctx.req, ctx.res)
+      })
+    })
+
+    it('should allow setting only papers disablePersonalLibrary', function (ctx) {
+      return new Promise(resolve => {
+        ctx.req.body = {
+          papers: { disablePersonalLibrary: true },
+        }
+        ctx.res.sendStatus = code => {
+          ctx.user.ace.papers.disablePersonalLibrary.should.equal(true)
+          resolve()
+        }
+        ctx.UserController.updateUserSettings(ctx.req, ctx.res)
+      })
+    })
+
+    it('should handle undefined mendeley by not setting it', function (ctx) {
+      return new Promise(resolve => {
+        ctx.user.ace.mendeley = { enabled: true, groups: [] }
+        ctx.req.body = { mendeley: undefined }
+        ctx.res.sendStatus = code => {
+          ctx.user.ace.mendeley.enabled.should.equal(true)
+          resolve()
+        }
+        ctx.UserController.updateUserSettings(ctx.req, ctx.res)
+      })
+    })
+
     it('should send an error if the email is 0 len', function (ctx) {
       return new Promise(resolve => {
         ctx.req.body.email = ''

services/web/types/user-settings.ts

@@ -3,6 +3,13 @@ import { FontFamily, LineHeight, OverallTheme } from '@/shared/utils/styles'
 export type Keybindings = 'none' | 'default' | 'vim' | 'emacs'
 export type PdfViewer = 'pdfjs' | 'native'
 
+export type RefProviderSettings = {
+  enabled: boolean
+  disablePersonalLibrary: boolean
+  groups: { id: string }[]
+  migrated?: boolean
+}
+
 export type UserSettings = {
   pdfViewer: PdfViewer
   autoComplete: boolean
@@ -20,4 +27,7 @@ export type UserSettings = {
   referencesSearchMode: 'advanced' | 'simple'
   breadcrumbs: boolean
   darkModePdf: boolean
+  zotero: RefProviderSettings
+  mendeley: RefProviderSettings
+  papers: RefProviderSettings
 }