Merge pull request #9116 from overleaf/ab-script-group-membership-sync

[web] Script & cron job to synchronize group subscriptions memberships in BQ

GitOrigin-RevId: 0180f7586eb520f37d4d600bcb8d3eeea36a538a

package-lock.json

@@ -25881,9 +25881,9 @@
       }
     },
     "node_modules/node-forge": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.2.1.tgz",
-      "integrity": "sha512-Fcvtbb+zBcZXbTTVwqGA5W+MKBj56UjVRevvchv5XrcyXbmNdesfZL37nlcWOfpgHhgmxApw3tQbTr4CqNmX4w==",
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.1.tgz",
+      "integrity": "sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==",
       "engines": {
         "node": ">= 6.13.0"
       }
@@ -38918,6 +38918,7 @@
         "@codemirror/view": "^6.7.1",
         "@contentful/rich-text-html-renderer": "^16.0.2",
         "@contentful/rich-text-types": "^16.0.2",
+        "@google-cloud/bigquery": "^6.0.1",
         "@lezer/common": "^1.0.2",
         "@lezer/highlight": "^1.1.3",
         "@lezer/lr": "^1.2.5",
@@ -39170,6 +39171,97 @@
         "worker-loader": "^3.0.8"
       }
     },
+    "services/web/node_modules/@google-cloud/bigquery": {
+      "version": "6.0.3",
+      "resolved": "https://registry.npmjs.org/@google-cloud/bigquery/-/bigquery-6.0.3.tgz",
+      "integrity": "sha512-BP464228S9dqDCb4dR99h9D8+N498YZi/AZvoOJUaieg2H6qbiYBE1xlYuaMvyV1WEQT/2/yZTCJnCo5WiaY0Q==",
+      "dependencies": {
+        "@google-cloud/common": "^4.0.0",
+        "@google-cloud/paginator": "^4.0.0",
+        "@google-cloud/promisify": "^3.0.0",
+        "arrify": "^2.0.1",
+        "big.js": "^6.0.0",
+        "duplexify": "^4.0.0",
+        "extend": "^3.0.2",
+        "is": "^3.3.0",
+        "p-event": "^4.1.0",
+        "readable-stream": "^4.0.0",
+        "stream-events": "^1.0.5",
+        "uuid": "^8.0.0"
+      },
+      "engines": {
+        "node": ">=12.0.0"
+      }
+    },
+    "services/web/node_modules/@google-cloud/bigquery/node_modules/readable-stream": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.2.0.tgz",
+      "integrity": "sha512-gJrBHsaI3lgBoGMW/jHZsQ/o/TIWiu5ENCJG1BB7fuCKzpFM8GaS2UoBVt9NO+oI+3FcrBNbUkl3ilDe09aY4A==",
+      "dependencies": {
+        "abort-controller": "^3.0.0",
+        "buffer": "^6.0.3",
+        "events": "^3.3.0",
+        "process": "^0.11.10"
+      },
+      "engines": {
+        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+      }
+    },
+    "services/web/node_modules/@google-cloud/bigquery/node_modules/uuid": {
+      "version": "8.3.2",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
+      "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
+      "bin": {
+        "uuid": "dist/bin/uuid"
+      }
+    },
+    "services/web/node_modules/@google-cloud/common": {
+      "version": "4.0.3",
+      "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-4.0.3.tgz",
+      "integrity": "sha512-fUoMo5b8iAKbrYpneIRV3z95AlxVJPrjpevxs4SKoclngWZvTXBSGpNisF5+x5m+oNGve7jfB1e6vNBZBUs7Fw==",
+      "dependencies": {
+        "@google-cloud/projectify": "^3.0.0",
+        "@google-cloud/promisify": "^3.0.0",
+        "arrify": "^2.0.1",
+        "duplexify": "^4.1.1",
+        "ent": "^2.2.0",
+        "extend": "^3.0.2",
+        "google-auth-library": "^8.0.2",
+        "retry-request": "^5.0.0",
+        "teeny-request": "^8.0.0"
+      },
+      "engines": {
+        "node": ">=12.0.0"
+      }
+    },
+    "services/web/node_modules/@google-cloud/paginator": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-4.0.1.tgz",
+      "integrity": "sha512-6G1ui6bWhNyHjmbYwavdN7mpVPRBtyDg/bfqBTAlwr413On2TnFNfDxc9UhTJctkgoCDgQXEKiRPLPR9USlkbQ==",
+      "dependencies": {
+        "arrify": "^2.0.0",
+        "extend": "^3.0.2"
+      },
+      "engines": {
+        "node": ">=12.0.0"
+      }
+    },
+    "services/web/node_modules/@google-cloud/projectify": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-3.0.0.tgz",
+      "integrity": "sha512-HRkZsNmjScY6Li8/kb70wjGlDDyLkVk3KvoEo9uIoxSjYLJasGiCch9+PqRVDOCGUFvEIqyogl+BeqILL4OJHA==",
+      "engines": {
+        "node": ">=12.0.0"
+      }
+    },
+    "services/web/node_modules/@google-cloud/promisify": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-3.0.1.tgz",
+      "integrity": "sha512-z1CjRjtQyBOYL+5Qr9DdYIfrdLBe746jRTYfaYU6MeXkqp7UfYs/jX16lFFVzZ7PGEJvqZNqYUEtb1mvDww4pA==",
+      "engines": {
+        "node": ">=12"
+      }
+    },
     "services/web/node_modules/@hapi/address": {
       "version": "2.1.4",
       "resolved": "https://registry.npmjs.org/@hapi/address/-/address-2.1.4.tgz",
@@ -39263,6 +39355,14 @@
         "lodash": "^4.17.15"
       }
     },
+    "services/web/node_modules/@tootallnate/once": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz",
+      "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==",
+      "engines": {
+        "node": ">= 10"
+      }
+    },
     "services/web/node_modules/@uppy/utils": {
       "version": "4.0.7",
       "resolved": "https://registry.npmjs.org/@uppy/utils/-/utils-4.0.7.tgz",
@@ -39331,6 +39431,29 @@
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
       "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
     },
+    "services/web/node_modules/buffer": {
+      "version": "6.0.3",
+      "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz",
+      "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ],
+      "dependencies": {
+        "base64-js": "^1.3.1",
+        "ieee754": "^1.2.1"
+      }
+    },
     "services/web/node_modules/busboy": {
       "version": "1.6.0",
       "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz",
@@ -39803,6 +39926,17 @@
         "node": ">=0.3.1"
       }
     },
+    "services/web/node_modules/duplexify": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.2.tgz",
+      "integrity": "sha512-fz3OjcNCHmRP12MJoZMPglx8m4rrFP8rovnk4vT8Fs+aonZoCwGg10dSsQsfP/E62eZcPTMSMP6686fu9Qlqtw==",
+      "dependencies": {
+        "end-of-stream": "^1.4.1",
+        "inherits": "^2.0.3",
+        "readable-stream": "^3.1.1",
+        "stream-shift": "^1.0.0"
+      }
+    },
     "services/web/node_modules/emoji-regex": {
       "version": "9.2.2",
       "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz",
@@ -40006,6 +40140,32 @@
         "universalify": "^0.1.0"
       }
     },
+    "services/web/node_modules/gaxios": {
+      "version": "5.0.2",
+      "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-5.0.2.tgz",
+      "integrity": "sha512-TjtV2AJOZoMQqRYoy5eM8cCQogYwazWNYLQ72QB0kwa6vHHruYkGmhhyrlzbmgNHK1dNnuP2WSH81urfzyN2Og==",
+      "dependencies": {
+        "extend": "^3.0.2",
+        "https-proxy-agent": "^5.0.0",
+        "is-stream": "^2.0.0",
+        "node-fetch": "^2.6.7"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "services/web/node_modules/gcp-metadata": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-5.0.1.tgz",
+      "integrity": "sha512-jiRJ+Fk7e8FH68Z6TLaqwea307OktJpDjmYnU7/li6ziwvVvU2RlrCyQo5vkdeP94chm0kcSCOOszvmuaioq3g==",
+      "dependencies": {
+        "gaxios": "^5.0.0",
+        "json-bigint": "^1.0.0"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
     "services/web/node_modules/glob-parent": {
       "version": "6.0.2",
       "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz",
@@ -40018,6 +40178,63 @@
         "node": ">=10.13.0"
       }
     },
+    "services/web/node_modules/google-auth-library": {
+      "version": "8.7.0",
+      "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-8.7.0.tgz",
+      "integrity": "sha512-1M0NG5VDIvJZEnstHbRdckLZESoJwguinwN8Dhae0j2ZKIQFIV63zxm6Fo6nM4xkgqUr2bbMtV5Dgo+Hy6oo0Q==",
+      "dependencies": {
+        "arrify": "^2.0.0",
+        "base64-js": "^1.3.0",
+        "ecdsa-sig-formatter": "^1.0.11",
+        "fast-text-encoding": "^1.0.0",
+        "gaxios": "^5.0.0",
+        "gcp-metadata": "^5.0.0",
+        "gtoken": "^6.1.0",
+        "jws": "^4.0.0",
+        "lru-cache": "^6.0.0"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "services/web/node_modules/google-auth-library/node_modules/lru-cache": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+      "dependencies": {
+        "yallist": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "services/web/node_modules/google-p12-pem": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-4.0.1.tgz",
+      "integrity": "sha512-WPkN4yGtz05WZ5EhtlxNDWPhC4JIic6G8ePitwUWy4l+XPVYec+a0j0Ts47PDtW59y3RwAhUd9/h9ZZ63px6RQ==",
+      "dependencies": {
+        "node-forge": "^1.3.1"
+      },
+      "bin": {
+        "gp12-pem": "build/src/bin/gp12-pem.js"
+      },
+      "engines": {
+        "node": ">=12.0.0"
+      }
+    },
+    "services/web/node_modules/gtoken": {
+      "version": "6.1.2",
+      "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-6.1.2.tgz",
+      "integrity": "sha512-4ccGpzz7YAr7lxrT2neugmXQ3hP9ho2gcaityLVkiUecAiwiy60Ii8gRbZeOsXV19fYaRjgBSshs8kXw+NKCPQ==",
+      "dependencies": {
+        "gaxios": "^5.0.1",
+        "google-p12-pem": "^4.0.0",
+        "jws": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=12.0.0"
+      }
+    },
     "services/web/node_modules/http-errors": {
       "version": "1.7.2",
       "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz",
@@ -40033,6 +40250,19 @@
         "node": ">= 0.6"
       }
     },
+    "services/web/node_modules/http-proxy-agent": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz",
+      "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==",
+      "dependencies": {
+        "@tootallnate/once": "2",
+        "agent-base": "6",
+        "debug": "4"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
     "services/web/node_modules/icss-utils": {
       "version": "5.1.0",
       "resolved": "https://registry.npmjs.org/icss-utils/-/icss-utils-5.1.0.tgz",
@@ -40045,6 +40275,25 @@
         "postcss": "^8.1.0"
       }
     },
+    "services/web/node_modules/ieee754": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
+      "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ]
+    },
     "services/web/node_modules/inherits": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
@@ -41009,6 +41258,18 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "services/web/node_modules/retry-request": {
+      "version": "5.0.2",
+      "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-5.0.2.tgz",
+      "integrity": "sha512-wfI3pk7EE80lCIXprqh7ym48IHYdwmAAzESdbU8Q9l7pnRCk9LEhpbOTNKjz6FARLm/Bl5m+4F0ABxOkYUujSQ==",
+      "dependencies": {
+        "debug": "^4.1.1",
+        "extend": "^3.0.2"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
     "services/web/node_modules/sandboxed-module": {
       "version": "2.3.0",
       "resolved": "https://github.com/overleaf/node-sandboxed-module/archive/cafa2d60f17ce75cc023e6f296eb8de79d92d35d.tar.gz",
@@ -41215,6 +41476,29 @@
         "node": ">=10.13.0"
       }
     },
+    "services/web/node_modules/teeny-request": {
+      "version": "8.0.2",
+      "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-8.0.2.tgz",
+      "integrity": "sha512-34pe0a4zASseXZCKdeTiIZqSKA8ETHb1EwItZr01PAR3CLPojeAKgSjzeNS4373gi59hNulyDrPKEbh2zO9sCg==",
+      "dependencies": {
+        "http-proxy-agent": "^5.0.0",
+        "https-proxy-agent": "^5.0.0",
+        "node-fetch": "^2.6.1",
+        "stream-events": "^1.0.5",
+        "uuid": "^9.0.0"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "services/web/node_modules/teeny-request/node_modules/uuid": {
+      "version": "9.0.0",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz",
+      "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==",
+      "bin": {
+        "uuid": "dist/bin/uuid"
+      }
+    },
     "services/web/node_modules/toidentifier": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz",
@@ -41271,8 +41555,7 @@
     "services/web/node_modules/yallist": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
-      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
-      "dev": true
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
     }
   },
   "dependencies": {
@@ -49185,6 +49468,7 @@
         "@contentful/rich-text-html-renderer": "^16.0.2",
         "@contentful/rich-text-types": "^16.0.2",
         "@cypress/react": "^6.2.0",
+        "@google-cloud/bigquery": "^6.0.1",
         "@juggle/resize-observer": "^3.3.1",
         "@lezer/common": "^1.0.2",
         "@lezer/generator": "^1.1.3",
@@ -49431,6 +49715,78 @@
         "yup": "^0.32.11"
       },
       "dependencies": {
+        "@google-cloud/bigquery": {
+          "version": "6.0.3",
+          "resolved": "https://registry.npmjs.org/@google-cloud/bigquery/-/bigquery-6.0.3.tgz",
+          "integrity": "sha512-BP464228S9dqDCb4dR99h9D8+N498YZi/AZvoOJUaieg2H6qbiYBE1xlYuaMvyV1WEQT/2/yZTCJnCo5WiaY0Q==",
+          "requires": {
+            "@google-cloud/common": "^4.0.0",
+            "@google-cloud/paginator": "^4.0.0",
+            "@google-cloud/promisify": "^3.0.0",
+            "arrify": "^2.0.1",
+            "big.js": "^6.0.0",
+            "duplexify": "^4.0.0",
+            "extend": "^3.0.2",
+            "is": "^3.3.0",
+            "p-event": "^4.1.0",
+            "readable-stream": "^4.0.0",
+            "stream-events": "^1.0.5",
+            "uuid": "^8.0.0"
+          },
+          "dependencies": {
+            "readable-stream": {
+              "version": "4.2.0",
+              "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.2.0.tgz",
+              "integrity": "sha512-gJrBHsaI3lgBoGMW/jHZsQ/o/TIWiu5ENCJG1BB7fuCKzpFM8GaS2UoBVt9NO+oI+3FcrBNbUkl3ilDe09aY4A==",
+              "requires": {
+                "abort-controller": "^3.0.0",
+                "buffer": "^6.0.3",
+                "events": "^3.3.0",
+                "process": "^0.11.10"
+              }
+            },
+            "uuid": {
+              "version": "8.3.2",
+              "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
+              "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg=="
+            }
+          }
+        },
+        "@google-cloud/common": {
+          "version": "4.0.3",
+          "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-4.0.3.tgz",
+          "integrity": "sha512-fUoMo5b8iAKbrYpneIRV3z95AlxVJPrjpevxs4SKoclngWZvTXBSGpNisF5+x5m+oNGve7jfB1e6vNBZBUs7Fw==",
+          "requires": {
+            "@google-cloud/projectify": "^3.0.0",
+            "@google-cloud/promisify": "^3.0.0",
+            "arrify": "^2.0.1",
+            "duplexify": "^4.1.1",
+            "ent": "^2.2.0",
+            "extend": "^3.0.2",
+            "google-auth-library": "^8.0.2",
+            "retry-request": "^5.0.0",
+            "teeny-request": "^8.0.0"
+          }
+        },
+        "@google-cloud/paginator": {
+          "version": "4.0.1",
+          "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-4.0.1.tgz",
+          "integrity": "sha512-6G1ui6bWhNyHjmbYwavdN7mpVPRBtyDg/bfqBTAlwr413On2TnFNfDxc9UhTJctkgoCDgQXEKiRPLPR9USlkbQ==",
+          "requires": {
+            "arrify": "^2.0.0",
+            "extend": "^3.0.2"
+          }
+        },
+        "@google-cloud/projectify": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-3.0.0.tgz",
+          "integrity": "sha512-HRkZsNmjScY6Li8/kb70wjGlDDyLkVk3KvoEo9uIoxSjYLJasGiCch9+PqRVDOCGUFvEIqyogl+BeqILL4OJHA=="
+        },
+        "@google-cloud/promisify": {
+          "version": "3.0.1",
+          "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-3.0.1.tgz",
+          "integrity": "sha512-z1CjRjtQyBOYL+5Qr9DdYIfrdLBe746jRTYfaYU6MeXkqp7UfYs/jX16lFFVzZ7PGEJvqZNqYUEtb1mvDww4pA=="
+        },
         "@hapi/address": {
           "version": "2.1.4",
           "resolved": "https://registry.npmjs.org/@hapi/address/-/address-2.1.4.tgz",
@@ -49503,6 +49859,11 @@
             "lodash": "^4.17.15"
           }
         },
+        "@tootallnate/once": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz",
+          "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A=="
+        },
         "@uppy/utils": {
           "version": "4.0.7",
           "resolved": "https://registry.npmjs.org/@uppy/utils/-/utils-4.0.7.tgz",
@@ -49558,6 +49919,15 @@
             }
           }
         },
+        "buffer": {
+          "version": "6.0.3",
+          "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz",
+          "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==",
+          "requires": {
+            "base64-js": "^1.3.1",
+            "ieee754": "^1.2.1"
+          }
+        },
         "busboy": {
           "version": "1.6.0",
           "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz",
@@ -49888,6 +50258,17 @@
           "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==",
           "dev": true
         },
+        "duplexify": {
+          "version": "4.1.2",
+          "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.2.tgz",
+          "integrity": "sha512-fz3OjcNCHmRP12MJoZMPglx8m4rrFP8rovnk4vT8Fs+aonZoCwGg10dSsQsfP/E62eZcPTMSMP6686fu9Qlqtw==",
+          "requires": {
+            "end-of-stream": "^1.4.1",
+            "inherits": "^2.0.3",
+            "readable-stream": "^3.1.1",
+            "stream-shift": "^1.0.0"
+          }
+        },
         "emoji-regex": {
           "version": "9.2.2",
           "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz",
@@ -50040,6 +50421,26 @@
             "universalify": "^0.1.0"
           }
         },
+        "gaxios": {
+          "version": "5.0.2",
+          "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-5.0.2.tgz",
+          "integrity": "sha512-TjtV2AJOZoMQqRYoy5eM8cCQogYwazWNYLQ72QB0kwa6vHHruYkGmhhyrlzbmgNHK1dNnuP2WSH81urfzyN2Og==",
+          "requires": {
+            "extend": "^3.0.2",
+            "https-proxy-agent": "^5.0.0",
+            "is-stream": "^2.0.0",
+            "node-fetch": "^2.6.7"
+          }
+        },
+        "gcp-metadata": {
+          "version": "5.0.1",
+          "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-5.0.1.tgz",
+          "integrity": "sha512-jiRJ+Fk7e8FH68Z6TLaqwea307OktJpDjmYnU7/li6ziwvVvU2RlrCyQo5vkdeP94chm0kcSCOOszvmuaioq3g==",
+          "requires": {
+            "gaxios": "^5.0.0",
+            "json-bigint": "^1.0.0"
+          }
+        },
         "glob-parent": {
           "version": "6.0.2",
           "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz",
@@ -50049,6 +50450,50 @@
             "is-glob": "^4.0.3"
           }
         },
+        "google-auth-library": {
+          "version": "8.7.0",
+          "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-8.7.0.tgz",
+          "integrity": "sha512-1M0NG5VDIvJZEnstHbRdckLZESoJwguinwN8Dhae0j2ZKIQFIV63zxm6Fo6nM4xkgqUr2bbMtV5Dgo+Hy6oo0Q==",
+          "requires": {
+            "arrify": "^2.0.0",
+            "base64-js": "^1.3.0",
+            "ecdsa-sig-formatter": "^1.0.11",
+            "fast-text-encoding": "^1.0.0",
+            "gaxios": "^5.0.0",
+            "gcp-metadata": "^5.0.0",
+            "gtoken": "^6.1.0",
+            "jws": "^4.0.0",
+            "lru-cache": "^6.0.0"
+          },
+          "dependencies": {
+            "lru-cache": {
+              "version": "6.0.0",
+              "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+              "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+              "requires": {
+                "yallist": "^4.0.0"
+              }
+            }
+          }
+        },
+        "google-p12-pem": {
+          "version": "4.0.1",
+          "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-4.0.1.tgz",
+          "integrity": "sha512-WPkN4yGtz05WZ5EhtlxNDWPhC4JIic6G8ePitwUWy4l+XPVYec+a0j0Ts47PDtW59y3RwAhUd9/h9ZZ63px6RQ==",
+          "requires": {
+            "node-forge": "^1.3.1"
+          }
+        },
+        "gtoken": {
+          "version": "6.1.2",
+          "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-6.1.2.tgz",
+          "integrity": "sha512-4ccGpzz7YAr7lxrT2neugmXQ3hP9ho2gcaityLVkiUecAiwiy60Ii8gRbZeOsXV19fYaRjgBSshs8kXw+NKCPQ==",
+          "requires": {
+            "gaxios": "^5.0.1",
+            "google-p12-pem": "^4.0.0",
+            "jws": "^4.0.0"
+          }
+        },
         "http-errors": {
           "version": "1.7.2",
           "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz",
@@ -50061,6 +50506,16 @@
             "toidentifier": "1.0.0"
           }
         },
+        "http-proxy-agent": {
+          "version": "5.0.0",
+          "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz",
+          "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==",
+          "requires": {
+            "@tootallnate/once": "2",
+            "agent-base": "6",
+            "debug": "4"
+          }
+        },
         "icss-utils": {
           "version": "5.1.0",
           "resolved": "https://registry.npmjs.org/icss-utils/-/icss-utils-5.1.0.tgz",
@@ -50068,6 +50523,11 @@
           "dev": true,
           "requires": {}
         },
+        "ieee754": {
+          "version": "1.2.1",
+          "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
+          "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="
+        },
         "inherits": {
           "version": "2.0.3",
           "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
@@ -50707,6 +51167,15 @@
             "path-parse": "^1.0.6"
           }
         },
+        "retry-request": {
+          "version": "5.0.2",
+          "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-5.0.2.tgz",
+          "integrity": "sha512-wfI3pk7EE80lCIXprqh7ym48IHYdwmAAzESdbU8Q9l7pnRCk9LEhpbOTNKjz6FARLm/Bl5m+4F0ABxOkYUujSQ==",
+          "requires": {
+            "debug": "^4.1.1",
+            "extend": "^3.0.2"
+          }
+        },
         "sandboxed-module": {
           "version": "https://github.com/overleaf/node-sandboxed-module/archive/cafa2d60f17ce75cc023e6f296eb8de79d92d35d.tar.gz",
           "integrity": "sha512-8h+3BGQj0Xou2mRYrRpFvDQ5Ne0/obZH9X27m/5zVMrvckNQazn1YPab7OE45VMzOI25iE9e0602QY2QmEnANQ==",
@@ -50873,6 +51342,25 @@
             "stable": "^0.1.8"
           }
         },
+        "teeny-request": {
+          "version": "8.0.2",
+          "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-8.0.2.tgz",
+          "integrity": "sha512-34pe0a4zASseXZCKdeTiIZqSKA8ETHb1EwItZr01PAR3CLPojeAKgSjzeNS4373gi59hNulyDrPKEbh2zO9sCg==",
+          "requires": {
+            "http-proxy-agent": "^5.0.0",
+            "https-proxy-agent": "^5.0.0",
+            "node-fetch": "^2.6.1",
+            "stream-events": "^1.0.5",
+            "uuid": "^9.0.0"
+          },
+          "dependencies": {
+            "uuid": {
+              "version": "9.0.0",
+              "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz",
+              "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg=="
+            }
+          }
+        },
         "toidentifier": {
           "version": "1.0.0",
           "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz",
@@ -50908,8 +51396,7 @@
         "yallist": {
           "version": "4.0.0",
           "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
-          "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
-          "dev": true
+          "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
         }
       }
     },
@@ -65239,9 +65726,9 @@
       }
     },
     "node-forge": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.2.1.tgz",
-      "integrity": "sha512-Fcvtbb+zBcZXbTTVwqGA5W+MKBj56UjVRevvchv5XrcyXbmNdesfZL37nlcWOfpgHhgmxApw3tQbTr4CqNmX4w=="
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.1.tgz",
+      "integrity": "sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA=="
     },
     "node-gyp-build": {
       "version": "3.9.0",

services/web/package.json

@@ -81,6 +81,7 @@
     "@codemirror/view": "^6.7.1",
     "@contentful/rich-text-html-renderer": "^16.0.2",
     "@contentful/rich-text-types": "^16.0.2",
+    "@google-cloud/bigquery": "^6.0.1",
     "@lezer/common": "^1.0.2",
     "@lezer/highlight": "^1.1.3",
     "@lezer/lr": "^1.2.5",

services/web/scripts/analytics/helpers/GoogleBigQueryHelper.js

@@ -0,0 +1,30 @@
+const GoogleBigQuery = require('@google-cloud/bigquery').BigQuery
+
+let dataset = null
+
+function getDataset() {
+  if (!dataset) {
+    console.log(
+      'Connecting to BigQuery dataset: ',
+      process.env.BQ_PROJECT_ID,
+      process.env.BQ_DATASET
+    )
+
+    dataset = new GoogleBigQuery({
+      projectId: process.env.BQ_PROJECT_ID,
+      keyFilename: process.env.GCS_KEY_FILE,
+    }).dataset(process.env.BQ_DATASET)
+  }
+
+  return dataset
+}
+
+async function query(query) {
+  const [job] = await getDataset().createQueryJob({ query })
+  const [rows] = await job.getQueryResults()
+  return rows
+}
+
+module.exports = {
+  query,
+}

services/web/scripts/analytics/sync_group_subscription_memberships.js

@@ -0,0 +1,241 @@
+const GoogleBigQueryHelper = require('./helpers/GoogleBigQueryHelper')
+const { Subscription } = require('../../app/src/models/Subscription')
+const { waitForDb } = require('../../app/src/infrastructure/mongodb')
+const AnalyticsManager = require('../../app/src/Features/Analytics/AnalyticsManager')
+const {
+  DeletedSubscription,
+} = require('../../app/src/models/DeletedSubscription')
+const minimist = require('minimist')
+const _ = require('lodash')
+
+let FETCH_LIMIT, COMMIT, VERBOSE
+
+async function main() {
+  await waitForDb()
+
+  console.log('## Syncing group subscription memberships...')
+
+  const subscriptionsCount = await Subscription.count({ groupPlan: true })
+  const deletedSubscriptionsCount = await DeletedSubscription.count({
+    'subscription.groupPlan': true,
+  })
+
+  console.log(
+    `## Going to synchronize ${subscriptionsCount} subscriptions and ${deletedSubscriptionsCount} deleted subscriptions`
+  )
+
+  await checkActiveSubscriptions()
+  await checkDeletedSubscriptions()
+}
+
+async function checkActiveSubscriptions() {
+  let totalSubscriptionsChecked = 0
+  let subscriptions
+  do {
+    subscriptions = await Subscription.find(
+      { groupPlan: true },
+      { recurlySubscription_id: 1, member_ids: 1 }
+    )
+      .sort('_id')
+      .skip(totalSubscriptionsChecked)
+      .limit(FETCH_LIMIT)
+      .lean()
+
+    if (subscriptions.length) {
+      const groupIds = subscriptions.map(sub => sub._id)
+      const bigQueryGroupMemberships = await fetchBigQueryMembershipStatuses(
+        groupIds
+      )
+      const membershipsByGroupId = _.groupBy(
+        bigQueryGroupMemberships,
+        'group_id'
+      )
+
+      for (const subscription of subscriptions) {
+        checkSubscriptionMemberships(
+          subscription,
+          membershipsByGroupId[subscription._id.toString()]
+        )
+      }
+      totalSubscriptionsChecked += subscriptions.length
+    }
+  } while (subscriptions.length > 0)
+}
+
+async function checkDeletedSubscriptions() {
+  let totalDeletedSubscriptionsChecked = 0
+  let deletedSubscriptions
+  do {
+    deletedSubscriptions = (
+      await DeletedSubscription.find(
+        { 'subscription.groupPlan': true },
+        { subscription: 1 }
+      )
+        .sort('deletedAt')
+        .skip(totalDeletedSubscriptionsChecked)
+        .limit(FETCH_LIMIT)
+    ).map(sub => sub.toObject().subscription)
+
+    if (deletedSubscriptions.length) {
+      const groupIds = deletedSubscriptions.map(sub => sub._id.toString())
+      const bigQueryGroupMemberships = await fetchBigQueryMembershipStatuses(
+        groupIds
+      )
+
+      for (const deletedSubscription of deletedSubscriptions) {
+        checkDeletedSubscriptionMemberships(
+          deletedSubscription,
+          _.filter(bigQueryGroupMemberships, {
+            group_id: deletedSubscription._id.toString(),
+          })
+        )
+      }
+      totalDeletedSubscriptionsChecked += deletedSubscriptions.length
+    }
+  } while (deletedSubscriptions.length > 0)
+}
+
+function checkSubscriptionMemberships(subscription, membershipStatuses) {
+  if (VERBOSE) {
+    console.log(
+      '\n###########################################################################################',
+      '\n# Subscription (mongo): ',
+      '\n# _id: \t\t\t\t',
+      subscription._id.toString(),
+      '\n# member_ids: \t\t\t',
+      subscription.member_ids,
+      '\n# recurlySubscription_id: \t',
+      subscription.recurlySubscription_id
+    )
+    console.log('#\n# Membership statuses found in BigQuery: ')
+    console.table(membershipStatuses)
+  }
+  // create missing `joined` events when membership status is missing
+  for (const memberId of subscription.member_ids) {
+    if (
+      !_.find(membershipStatuses, {
+        user_id: memberId.toString(),
+        is_member: true,
+      })
+    ) {
+      sendCorrectiveEvent(memberId, 'group-subscription-joined', subscription)
+    }
+  }
+  // create missing `left` events if user is not a member of the group anymore
+  for (const { user_id: userId, is_member: isMember } of membershipStatuses) {
+    if (
+      isMember &&
+      !subscription.member_ids.some(sub => sub.id.toString() === userId)
+    ) {
+      sendCorrectiveEvent(userId, 'group-subscription-left', subscription)
+    }
+  }
+}
+
+function checkDeletedSubscriptionMemberships(subscription, membershipStatuses) {
+  if (VERBOSE) {
+    console.log(
+      '\n###########################################################################################',
+      '\n# Deleted subscription (mongo): ',
+      '\n# _id: \t\t\t\t',
+      subscription._id.toString(),
+      '\n# member_ids: \t\t\t',
+      subscription.member_ids,
+      '\n# recurlySubscription_id: \t',
+      subscription.recurlySubscription_id
+    )
+    console.log('#\n# Membership statuses found in BigQuery: ')
+    console.table(membershipStatuses)
+  }
+
+  const updatedUserIds = new Set()
+  // create missing `left` events if user was a member of the group in BQ and status is not up-to-date
+  for (const memberId of subscription.member_ids.map(id => id.toString())) {
+    if (
+      _.find(membershipStatuses, {
+        user_id: memberId,
+        is_member: true,
+      })
+    ) {
+      sendCorrectiveEvent(memberId, 'group-subscription-left', subscription)
+      updatedUserIds.add(memberId)
+    }
+  }
+  // for cases where the user has been removed from the subscription before it was deleted and status is not up-to-date
+  for (const { user_id: userId, is_member: isMember } of membershipStatuses) {
+    if (isMember && !updatedUserIds.has(userId)) {
+      sendCorrectiveEvent(userId, 'group-subscription-left', subscription)
+      updatedUserIds.add(userId)
+    }
+  }
+}
+
+function sendCorrectiveEvent(userId, event, subscription) {
+  const segmentation = {
+    groupId: subscription._id.toString(),
+    subscriptionId: subscription.recurlySubscription_id,
+    source: 'sync',
+  }
+  if (COMMIT) {
+    console.log(
+      `Sending event '${event}' for user ${userId} with segmentation: ${JSON.stringify(
+        segmentation
+      )}`
+    )
+    AnalyticsManager.recordEventForUser(userId, event, segmentation)
+  } else {
+    console.log(
+      `Dry run - would send event '${event}' for user ${userId} with segmentation: ${JSON.stringify(
+        segmentation
+      )}`
+    )
+  }
+}
+
+async function fetchBigQueryMembershipStatuses(groupIds) {
+  const joinedGroupIds = groupIds.map(id => `"${id}"`).join(',')
+  const query = `\
+    WITH memberships AS (
+      SELECT
+        user_id, group_id, is_member, created_at,
+        ROW_NUMBER() OVER(PARTITION BY group_id, user_id ORDER BY created_at DESC) AS row_number
+      FROM analytics.user_group_memberships
+      WHERE group_id IN (${joinedGroupIds})
+    )
+
+    SELECT
+      group_id,
+      COALESCE(user_aliases.user_id, memberships.user_id) AS user_id,
+      is_member,
+      memberships.created_at
+    FROM memberships
+    LEFT JOIN analytics.user_aliases ON memberships.user_id = user_aliases.analytics_id
+    WHERE row_number = 1;
+  `
+
+  return GoogleBigQueryHelper.query(query)
+}
+
+const setup = () => {
+  const argv = minimist(process.argv.slice(2))
+  FETCH_LIMIT = argv.fetch ? argv.fetch : 100
+  COMMIT = argv.commit !== undefined
+  VERBOSE = argv.debug !== undefined
+  if (!COMMIT) {
+    console.warn('Doing dry run without --commit')
+  }
+  if (VERBOSE) {
+    console.log('Running in verbose mode')
+  }
+}
+
+setup()
+main()
+  .then(() => {
+    console.error('Done.')
+    process.exit(0)
+  })
+  .catch(error => {
+    console.error({ error })
+    process.exit(1)
+  })