Overleaf/services/web/app/src/Features/SplitTests/SplitTestManager.mjs

import { SplitTest } from '../../models/SplitTest.mjs'
import SplitTestUtils from './SplitTestUtils.mjs'
import OError from '@overleaf/o-error'
import _ from 'lodash'
import { CacheFlow } from 'cache-flow'

const ALPHA_PHASE = 'alpha'
const BETA_PHASE = 'beta'
const RELEASE_PHASE = 'release'

async function getSplitTests({ name, phase, type, active, archived }) {
  const filters = {}
  if (name && name !== '') {
    filters.name = { $regex: _.escapeRegExp(name) }
  }
  if (active) {
    filters.$where = 'this.versions[this.versions.length - 1].active === true'
  }
  if (type === 'split-test') {
    const query =
      'this.versions[this.versions.length - 1].analyticsEnabled === true'
    if (filters.$where) {
      filters.$where += `&& ${query}`
    } else {
      filters.$where = query
    }
  }
  if (type === 'gradual-rollout') {
    const query =
      'this.versions[this.versions.length - 1].analyticsEnabled === false'
    if (filters.$where) {
      filters.$where += `&& ${query}`
    } else {
      filters.$where = query
    }
  }
  if (['alpha', 'beta', 'release'].includes(phase)) {
    const query = `this.versions[this.versions.length - 1].phase === "${phase}"`
    if (filters.$where) {
      filters.$where += `&& ${query}`
    } else {
      filters.$where = query
    }
  }
  if (archived === true) {
    filters.archived = true
  } else if (archived === false) {
    filters.archived = { $ne: true }
  }
  try {
    return await SplitTest.find(filters)
      .populate('archivedBy', ['email', 'first_name', 'last_name'])
      .populate('versions.author', ['email', 'first_name', 'last_name'])
      .limit(300)
      .exec()
  } catch (error) {
    throw OError.tag(error, 'Failed to get split tests list')
  }
}

async function getRuntimeTests() {
  try {
    return SplitTest.find({}).lean().exec()
  } catch (error) {
    throw OError.tag(error, 'Failed to get active split tests list')
  }
}

async function getSplitTest(query) {
  try {
    return await SplitTest.findOne(query)
      .populate('archivedBy', ['email', 'first_name', 'last_name'])
      .populate('versions.author', ['email', 'first_name', 'last_name'])
      .exec()
  } catch (error) {
    throw OError.tag(error, 'Failed to get split test', { query })
  }
}

async function createSplitTest(
  { name, configuration, badgeInfo = {}, info = {} },
  userId
) {
  const stripedVariants = []
  let stripeStart = 0

  _checkNewVariantsConfiguration(
    [],
    configuration.variants,
    configuration.analyticsEnabled
  )
  for (const variant of configuration.variants) {
    const variantData = {
      name: (variant.name || '').trim(),
      rolloutPercent: variant.rolloutPercent,
      userLimit: variant.userLimit,
      rolloutStripes:
        variant.rolloutPercent > 0
          ? [
              {
                start: stripeStart,
                end: stripeStart + variant.rolloutPercent,
              },
            ]
          : [],
    }
    if (variant.userLimit && typeof variant.userLimit === 'number') {
      variantData.userCount = 0
    }
    stripedVariants.push(variantData)
    stripeStart += variant.rolloutPercent
  }
  const splitTest = new SplitTest({
    name: (name || '').trim(),
    description: info.description,
    ticketUrl: info.ticketUrl,
    reportsUrls: info.reportsUrls,
    winningVariant: info.winningVariant,
    badgeInfo,
    versions: [
      {
        versionNumber: 1,
        phase: configuration.phase,
        active: configuration.active,
        analyticsEnabled:
          configuration.active && configuration.analyticsEnabled,
        variants: stripedVariants,
        author: userId,
      },
    ],
    expectedEndDate: info.expectedEndDate,
    expectedUplift: info.expectedUplift,
    requiredCohortSize: info.requiredCohortSize,
  })
  return _saveSplitTest(splitTest)
}

async function updateSplitTestConfig({ name, configuration, comment }, userId) {
  const splitTest = await getSplitTest({ name })
  if (!splitTest) {
    throw new OError(`Cannot update split test '${name}': not found`)
  }
  if (splitTest.archived) {
    throw new OError('Cannot update an archived split test', { name })
  }
  const lastVersion = SplitTestUtils.getCurrentVersion(splitTest).toObject()
  if (configuration.phase !== lastVersion.phase) {
    throw new OError(
      `Cannot update with different phase - use /switch-to-next-phase endpoint instead`
    )
  }
  _checkNewVariantsConfiguration(
    lastVersion.variants,
    configuration.variants,
    configuration.analyticsEnabled
  )
  const updatedVariants = _updateVariantsWithNewConfiguration(
    lastVersion.variants,
    configuration.variants
  )

  splitTest.versions.push({
    versionNumber: lastVersion.versionNumber + 1,
    phase: configuration.phase,
    active: configuration.active,
    analyticsEnabled: configuration.analyticsEnabled,
    variants: updatedVariants,
    author: userId,
    comment,
  })
  return _saveSplitTest(splitTest)
}

async function updateSplitTestInfo(name, info) {
  const splitTest = await getSplitTest({ name })
  if (!splitTest) {
    throw new OError(`Cannot update split test '${name}': not found`)
  }
  splitTest.description = info.description
  splitTest.expectedEndDate = info.expectedEndDate
  splitTest.ticketUrl = info.ticketUrl
  splitTest.reportsUrls = info.reportsUrls
  splitTest.winningVariant = info.winningVariant
  return _saveSplitTest(splitTest)
}

async function updateSplitTestBadgeInfo(name, badgeInfo) {
  const splitTest = await getSplitTest({ name })
  if (!splitTest) {
    throw new OError(`Cannot update split test '${name}': not found`)
  }
  splitTest.badgeInfo = badgeInfo
  return _saveSplitTest(splitTest)
}

async function replaceSplitTests(tests) {
  _checkEnvIsSafe('replace')

  try {
    await _deleteSplitTests()
    return await SplitTest.create(tests)
  } catch (error) {
    throw OError.tag(error, 'Failed to replace all split tests', { tests })
  }
}

async function mergeSplitTests(incomingTests, overWriteLocal) {
  _checkEnvIsSafe('merge')

  // this is required as the query returns models, and we need all the items to be objects,
  //   similar to the ones we recieve as incomingTests
  const localTests = await SplitTest.find({}).lean().exec()

  let merged
  // we preserve the state of the local tests (baseTests)
  // eg: if inTest is in phase 1, and basetest is in phase 2, the merged will be in state 2
  // therefore, we can have the opposite effect by swapping the order of args (overwrite locals with sent tests)
  if (overWriteLocal) {
    merged = _mergeFlags(localTests, incomingTests)
  } else {
    merged = _mergeFlags(incomingTests, localTests)
  }

  try {
    await _deleteSplitTests()
    const success = await SplitTest.create(merged)
    return success
  } catch (error) {
    throw OError.tag(error, 'Failed to merge all split tests, merged set was', {
      merged,
    })
  }
}

async function switchToNextPhase({ name, comment }, userId) {
  const splitTest = await getSplitTest({ name })
  if (!splitTest) {
    throw new OError(
      `Cannot switch split test with ID '${name}' to next phase: not found`
    )
  }
  if (splitTest.archived) {
    throw new OError('Cannot switch an archived split test to next phase', {
      name,
    })
  }
  const lastVersionCopy = SplitTestUtils.getCurrentVersion(splitTest).toObject()
  lastVersionCopy.versionNumber++
  if (lastVersionCopy.phase === ALPHA_PHASE) {
    lastVersionCopy.phase = BETA_PHASE
  } else if (lastVersionCopy.phase === BETA_PHASE) {
    if (splitTest.forbidReleasePhase) {
      throw new OError('Switch to release phase is disabled for this test', {
        name,
      })
    }
    lastVersionCopy.phase = RELEASE_PHASE
  } else if (splitTest.phase === RELEASE_PHASE) {
    throw new OError(
      `Split test with ID '${name}' is already in the release phase`
    )
  }
  for (const variant of lastVersionCopy.variants) {
    variant.rolloutPercent = 0
    variant.rolloutStripes = []
    if (variant.userCount) {
      variant.userCount = 0
    }
  }
  lastVersionCopy.author = userId
  lastVersionCopy.comment = comment
  lastVersionCopy.createdAt = new Date()
  splitTest.versions.push(lastVersionCopy)
  return _saveSplitTest(splitTest)
}

async function revertToPreviousVersion(
  { name, versionNumber, comment },
  userId
) {
  const splitTest = await getSplitTest({ name })
  if (!splitTest) {
    throw new OError(
      `Cannot revert split test with ID '${name}' to previous version: not found`
    )
  }
  if (splitTest.archived) {
    throw new OError(
      'Cannot revert an archived split test to previous version',
      {
        name,
      }
    )
  }
  if (splitTest.versions.length <= 1) {
    throw new OError(
      `Cannot revert split test with ID '${name}' to previous version: split test must have at least 2 versions`
    )
  }
  const previousVersion = SplitTestUtils.getVersion(splitTest, versionNumber)
  if (!previousVersion) {
    throw new OError(
      `Cannot revert split test with ID '${name}' to version number ${versionNumber}: version not found`
    )
  }
  const lastVersion = SplitTestUtils.getCurrentVersion(splitTest)
  if (
    lastVersion.phase === RELEASE_PHASE &&
    previousVersion.phase !== RELEASE_PHASE
  ) {
    splitTest.forbidReleasePhase = true
  }
  const previousVersionCopy = previousVersion.toObject()
  previousVersionCopy.versionNumber = lastVersion.versionNumber + 1
  previousVersionCopy.createdAt = new Date()
  previousVersionCopy.author = userId
  previousVersionCopy.comment = comment

  // restore user count from most recent version of this phase
  const mostRecentVersionOfTargetPhase = splitTest.versions.findLast(
    v => v.phase === previousVersion.phase
  )

  if (mostRecentVersionOfTargetPhase) {
    for (const variant of previousVersionCopy.variants) {
      const correspondingVariant = mostRecentVersionOfTargetPhase.variants.find(
        v => v.name === variant.name
      )
      if (correspondingVariant?.userCount) {
        variant.userCount = correspondingVariant.userCount
      }
    }
  } else {
    for (const variant of previousVersionCopy.variants) {
      if (variant.userCount) {
        variant.userCount = 0
      }
    }
  }

  splitTest.versions.push(previousVersionCopy)
  return _saveSplitTest(splitTest)
}

async function archive(name, userId) {
  const splitTest = await getSplitTest({ name })
  if (!splitTest) {
    throw new OError(`Cannot archive split test with ID '${name}': not found`)
  }
  if (splitTest.archived) {
    throw new OError(`Split test with ID '${name}' is already archived`)
  }
  splitTest.archived = true
  splitTest.archivedAt = new Date()
  splitTest.archivedBy = userId
  return _saveSplitTest(splitTest)
}

async function clearCache() {
  await CacheFlow.reset('split-test')
}

function _checkNewVariantsConfiguration(
  variants,
  newVariantsConfiguration,
  analyticsEnabled
) {
  if (newVariantsConfiguration?.length > 1 && !analyticsEnabled) {
    throw new OError(`Gradual rollouts can only have a single variant`)
  }

  const totalRolloutPercentage = _getTotalRolloutPercentage(
    newVariantsConfiguration
  )
  if (totalRolloutPercentage > 100) {
    throw new OError(`Total variants rollout percentage cannot exceed 100`)
  }
  for (const variant of variants) {
    const newVariantConfiguration = _.find(newVariantsConfiguration, {
      name: variant.name,
    })
    if (!newVariantConfiguration) {
      throw new OError(
        `Variant defined in previous version as ${JSON.stringify(
          variant
        )} cannot be removed in new configuration: either set it inactive or create a new split test`
      )
    }
    if (newVariantConfiguration.rolloutPercent < variant.rolloutPercent) {
      throw new OError(
        `Rollout percentage for variant defined in previous version as ${JSON.stringify(
          variant
        )} cannot be decreased: revert to a previous configuration instead`
      )
    }
    if (variant.userLimit !== undefined) {
      // Existing variant has a user limit - can only increase it
      if (
        newVariantConfiguration.userLimit !== undefined &&
        newVariantConfiguration.userLimit < variant.userLimit
      ) {
        throw new OError(
          `User limit for variant '${variant.name}' cannot be decreased: revert to a previous configuration instead`
        )
      }
    } else {
      // Existing variant has no user limit - cannot add one
      if (newVariantConfiguration.userLimit !== undefined) {
        throw new OError(
          `User limit cannot be added to variant '${variant.name}' after creation: user limits can only be set when the split test is created`
        )
      }
    }
  }
}

function _updateVariantsWithNewConfiguration(
  variants,
  newVariantsConfiguration
) {
  let totalRolloutPercentage = _getTotalRolloutPercentage(variants)
  const variantsCopy = _.clone(variants)
  for (const newVariantConfig of newVariantsConfiguration) {
    if (newVariantConfig.rolloutPercent === 0) {
      continue
    }
    const variant = _.find(variantsCopy, { name: newVariantConfig.name })
    if (!variant) {
      const newVariant = {
        name: newVariantConfig.name,
        rolloutPercent: newVariantConfig.rolloutPercent,
        rolloutStripes: [
          {
            start: totalRolloutPercentage,
            end: totalRolloutPercentage + newVariantConfig.rolloutPercent,
          },
        ],
      }
      if (
        newVariantConfig.userLimit &&
        typeof newVariantConfig.userLimit === 'number'
      ) {
        newVariant.userLimit = newVariantConfig.userLimit
        newVariant.userCount = 0
      }
      variantsCopy.push(newVariant)
      totalRolloutPercentage += newVariantConfig.rolloutPercent
    } else if (variant.rolloutPercent < newVariantConfig.rolloutPercent) {
      const newStripeSize =
        newVariantConfig.rolloutPercent - variant.rolloutPercent
      variant.rolloutPercent = newVariantConfig.rolloutPercent
      variant.rolloutStripes.push({
        start: totalRolloutPercentage,
        end: totalRolloutPercentage + newStripeSize,
      })
      totalRolloutPercentage += newStripeSize
    }
    if (newVariantConfig.userLimit >= variant?.userLimit) {
      variant.userLimit = newVariantConfig.userLimit
    }
    if (variant?.userLimit && !variant.userCount) {
      variant.userCount = 0
    }
  }
  return variantsCopy
}

function _getTotalRolloutPercentage(variants) {
  return _.sumBy(variants, 'rolloutPercent')
}

async function _saveSplitTest(splitTest) {
  try {
    const savedSplitTest = await splitTest.save()
    await savedSplitTest.populate('archivedBy', [
      'email',
      'first_name',
      'last_name',
    ])
    await savedSplitTest.populate('versions.author', [
      'email',
      'first_name',
      'last_name',
    ])
    return savedSplitTest.toObject()
  } catch (error) {
    throw OError.tag(error, 'Failed to save split test', {
      splitTest: JSON.stringify(splitTest),
    })
  }
}

/*
 * As this is only used for utility in local dev environment, we should make sure this isn't run in
 * any other deployment environment.
 */
function _checkEnvIsSafe(operation) {
  if (process.env.NODE_ENV !== 'development') {
    throw new OError(
      `Attempted to ${operation} all feature flags outside of local env`
    )
  }
}

async function _deleteSplitTests() {
  _checkEnvIsSafe('delete')
  let deleted

  try {
    deleted = await SplitTest.deleteMany({}).exec()
  } catch (error) {
    throw new OError('Failed to delete all split tests')
  }

  if (!deleted.acknowledged) {
    throw new OError('Error deleting split tests, split tests have not updated')
  }
}

function _mergeFlags(incomingTests, baseTests) {
  // copy all base versions
  const mergedSet = baseTests.map(test => test)
  for (const inTest of incomingTests) {
    // since name is a unique key, we can use it to compare
    const newFeatureFlag = !mergedSet.some(bTest => bTest.name === inTest.name)
    // only add new feature flags, instead of overwriting ones in baseTests, meaning baseTests take precendence
    if (newFeatureFlag) {
      mergedSet.push(inTest)
    }
  }
  return mergedSet
}

export default {
  getSplitTest,
  getSplitTests,
  getRuntimeTests,
  createSplitTest,
  updateSplitTestConfig,
  updateSplitTestInfo,
  updateSplitTestBadgeInfo,
  switchToNextPhase,
  revertToPreviousVersion,
  archive,
  replaceSplitTests,
  mergeSplitTests,
  clearCache,
}