Merge pull request #21589 from overleaf/jpa-s3-ssec-hkdf

[object-persistor] s3SSEC: add HKDF layer to KEK

GitOrigin-RevId: 1def9e378e1dfd90f449ad392b0db2101584e17f

libraries/object-persistor/src/PerProjectEncryptedS3Persistor.js

@@ -15,6 +15,9 @@ const {
 const logger = require('@overleaf/logger')
 
 const generateKey = promisify(Crypto.generateKey)
+const hkdf = promisify(Crypto.hkdf)
+
+const AES256_KEY_LENGTH = 32
 
 /**
  * @typedef {import('aws-sdk').AWSError} AWSError
@@ -25,8 +28,8 @@ const generateKey = promisify(Crypto.generateKey)
  * @property {boolean} automaticallyRotateDEKEncryption
  * @property {boolean} ignoreErrorsFromDEKReEncryption
  * @property {(bucketName: string, path: string) => {bucketName: string, path: string}} pathToDataEncryptionKeyPath
- * @property {(bucketName: string, path: string) => boolean} pathIsProjectFolder
+ * @property {(bucketName: string, path: string) => string} pathToProjectFolder
- * @property {() => Promise<Array<Buffer>>} getKeyEncryptionKeys
+ * @property {() => Promise<Array<RootKeyEncryptionKey>>} getRootKeyEncryptionKeys
  */
 
 /**
@@ -52,10 +55,47 @@ function isForbiddenError(err) {
   return cause.statusCode === 403
 }
 
+class RootKeyEncryptionKey {
+  /** @type {Buffer} */
+  #keyEncryptionKey
+  /** @type {Buffer} */
+  #salt
+
+  /**
+   * @param {Buffer} keyEncryptionKey
+   * @param {Buffer} salt
+   */
+  constructor(keyEncryptionKey, salt) {
+    if (keyEncryptionKey.byteLength !== AES256_KEY_LENGTH) {
+      throw new Error(`kek is not ${AES256_KEY_LENGTH} bytes long`)
+    }
+    this.#keyEncryptionKey = keyEncryptionKey
+    this.#salt = salt
+  }
+
+  /**
+   * @param {string} prefix
+   * @return {Promise<SSECOptions>}
+   */
+  async forProject(prefix) {
+    return new SSECOptions(
+      Buffer.from(
+        await hkdf(
+          'sha256',
+          this.#keyEncryptionKey,
+          this.#salt,
+          prefix,
+          AES256_KEY_LENGTH
+        )
+      )
+    )
+  }
+}
+
 class PerProjectEncryptedS3Persistor extends S3Persistor {
   /** @type {Settings} */
   #settings
-  /** @type {Promise<Array<SSECOptions>>} */
+  /** @type {Promise<Array<RootKeyEncryptionKey>>} */
   #availableKeyEncryptionKeysPromise
 
   /**
@@ -65,10 +105,10 @@ class PerProjectEncryptedS3Persistor extends S3Persistor {
     super(settings)
     this.#settings = settings
     this.#availableKeyEncryptionKeysPromise = this.#settings
-      .getKeyEncryptionKeys()
+      .getRootKeyEncryptionKeys()
-      .then(keysAsBuffer => {
+      .then(rootKEKs => {
-        if (keysAsBuffer.length === 0) throw new Error('no kek provided')
+        if (rootKEKs.length === 0) throw new Error('no root kek provided')
-        return keysAsBuffer.map(buffer => new SSECOptions(buffer))
+        return rootKEKs
       })
   }
 
@@ -76,9 +116,16 @@ class PerProjectEncryptedS3Persistor extends S3Persistor {
     await this.#availableKeyEncryptionKeysPromise
   }
 
-  async #getCurrentKeyEncryptionKey() {
+  /**
-    const available = await this.#availableKeyEncryptionKeysPromise
+   * @param {string} bucketName
-    return available[0]
+   * @param {string} path
+   * @return {Promise<SSECOptions>}
+   */
+  async #getCurrentKeyEncryptionKey(bucketName, path) {
+    const [currentRootKEK] = await this.#availableKeyEncryptionKeysPromise
+    return await currentRootKEK.forProject(
+      this.#settings.pathToProjectFolder(bucketName, path)
+    )
   }
 
   /**
@@ -87,7 +134,10 @@ class PerProjectEncryptedS3Persistor extends S3Persistor {
    */
   async getDataEncryptionKeySize(bucketName, path) {
     const dekPath = this.#settings.pathToDataEncryptionKeyPath(bucketName, path)
-    for (const ssecOptions of await this.#availableKeyEncryptionKeysPromise) {
+    for (const rootKEK of await this.#availableKeyEncryptionKeysPromise) {
+      const ssecOptions = await rootKEK.forProject(
+        this.#settings.pathToProjectFolder(bucketName, path)
+      )
       try {
         return await super.getObjectSize(dekPath.bucketName, dekPath.path, {
           ssecOptions,
@@ -138,7 +188,7 @@ class PerProjectEncryptedS3Persistor extends S3Persistor {
       {
         // Do not overwrite any objects if already created
         ifNoneMatch: '*',
-        ssecOptions: await this.#getCurrentKeyEncryptionKey(),
+        ssecOptions: await this.#getCurrentKeyEncryptionKey(bucketName, path),
       }
     )
     return new SSECOptions(dataEncryptionKey)
@@ -153,7 +203,10 @@ class PerProjectEncryptedS3Persistor extends S3Persistor {
     const dekPath = this.#settings.pathToDataEncryptionKeyPath(bucketName, path)
     let res
     let kekIndex = 0
-    for (const ssecOptions of await this.#availableKeyEncryptionKeysPromise) {
+    for (const rootKEK of await this.#availableKeyEncryptionKeysPromise) {
+      const ssecOptions = await rootKEK.forProject(
+        this.#settings.pathToProjectFolder(bucketName, path)
+      )
       try {
         res = await super.getObjectStream(dekPath.bucketName, dekPath.path, {
           ssecOptions,
@@ -171,12 +224,16 @@ class PerProjectEncryptedS3Persistor extends S3Persistor {
     await Stream.promises.pipeline(res, buf)
 
     if (kekIndex !== 0 && this.#settings.automaticallyRotateDEKEncryption) {
+      const ssecOptions = await this.#getCurrentKeyEncryptionKey(
+        bucketName,
+        path
+      )
       try {
         await super.sendStream(
           dekPath.bucketName,
           dekPath.path,
           Stream.Readable.from([buf.getContents()]),
-          { ssecOptions: await this.#getCurrentKeyEncryptionKey() }
+          { ssecOptions }
         )
       } catch (err) {
         if (this.#settings.ignoreErrorsFromDEKReEncryption) {
@@ -252,7 +309,7 @@ class PerProjectEncryptedS3Persistor extends S3Persistor {
   async deleteDirectory(bucketName, path, continuationToken) {
     // Note: Listing/Deleting a prefix does not require SSE-C credentials.
     await super.deleteDirectory(bucketName, path, continuationToken)
-    if (this.#settings.pathIsProjectFolder(bucketName, path)) {
+    if (this.#settings.pathToProjectFolder(bucketName, path) === path) {
       const dekPath = this.#settings.pathToDataEncryptionKeyPath(
         bucketName,
         path
@@ -357,4 +414,8 @@ class CachedPerProjectEncryptedS3Persistor {
   }
 }
 
-module.exports = PerProjectEncryptedS3Persistor
+module.exports = {
+  PerProjectEncryptedS3Persistor,
+  CachedPerProjectEncryptedS3Persistor,
+  RootKeyEncryptionKey,
+}

libraries/object-persistor/src/PersistorFactory.js

@@ -4,7 +4,9 @@ const GcsPersistor = require('./GcsPersistor')
 const { S3Persistor } = require('./S3Persistor')
 const FSPersistor = require('./FSPersistor')
 const MigrationPersistor = require('./MigrationPersistor')
-const PerProjectEncryptedS3Persistor = require('./PerProjectEncryptedS3Persistor')
+const {
+  PerProjectEncryptedS3Persistor,
+} = require('./PerProjectEncryptedS3Persistor')
 
 function getPersistor(backend, settings) {
   switch (backend) {

services/filestore/test/acceptance/js/FilestoreTests.js

@@ -39,7 +39,10 @@ const {
   NotImplementedError,
   NoKEKMatchedError,
 } = require('@overleaf/object-persistor/src/Errors')
-const PerProjectEncryptedS3Persistor = require('@overleaf/object-persistor/src/PerProjectEncryptedS3Persistor')
+const {
+  PerProjectEncryptedS3Persistor,
+  RootKeyEncryptionKey,
+} = require('@overleaf/object-persistor/src/PerProjectEncryptedS3Persistor')
 const crypto = require('crypto')
 
 describe('Filestore', function () {
@@ -1118,33 +1121,39 @@ describe('Filestore', function () {
         })
 
         describe('kek rotation', function () {
-          const newKEK = crypto.generateKeySync('aes', { length: 256 }).export()
+          const newKEK = new RootKeyEncryptionKey(
-          const oldKEK = crypto.generateKeySync('aes', { length: 256 }).export()
+            crypto.generateKeySync('aes', { length: 256 }).export(),
+            Buffer.alloc(32)
+          )
+          const oldKEK = new RootKeyEncryptionKey(
+            crypto.generateKeySync('aes', { length: 256 }).export(),
+            Buffer.alloc(32)
+          )
           const migrationStep0 = new PerProjectEncryptedS3Persistor({
             ...s3SSECConfig(),
             automaticallyRotateDEKEncryption: false,
-            async getKeyEncryptionKeys() {
+            async getRootKeyEncryptionKeys() {
               return [oldKEK] // only old key
             },
           })
           const migrationStep1 = new PerProjectEncryptedS3Persistor({
             ...s3SSECConfig(),
             automaticallyRotateDEKEncryption: false,
-            async getKeyEncryptionKeys() {
+            async getRootKeyEncryptionKeys() {
               return [oldKEK, newKEK] // new key as fallback
             },
           })
           const migrationStep2 = new PerProjectEncryptedS3Persistor({
             ...s3SSECConfig(),
             automaticallyRotateDEKEncryption: true, // <- different compared to partiallyRotated
-            async getKeyEncryptionKeys() {
+            async getRootKeyEncryptionKeys() {
               return [newKEK, oldKEK] // old keys as fallback
             },
           })
           const migrationStep3 = new PerProjectEncryptedS3Persistor({
             ...s3SSECConfig(),
             automaticallyRotateDEKEncryption: true,
-            async getKeyEncryptionKeys() {
+            async getRootKeyEncryptionKeys() {
               return [newKEK] // only new key
             },
           })

services/filestore/test/acceptance/js/TestConfig.js

@@ -2,6 +2,9 @@ const fs = require('fs')
 const Path = require('path')
 const crypto = require('crypto')
 const https = require('https')
+const {
+  RootKeyEncryptionKey,
+} = require('@overleaf/object-persistor/src/PerProjectEncryptedS3Persistor')
 
 // use functions to get a fresh copy, not a reference, each time
 function s3BaseConfig() {
@@ -26,15 +29,21 @@ function s3Config() {
   }
 }
 
-const S3SSECKeys = [crypto.generateKeySync('aes', { length: 256 }).export()]
+const S3SSECKeys = [
+  new RootKeyEncryptionKey(
+    crypto.generateKeySync('aes', { length: 256 }).export(),
+    Buffer.alloc(32)
+  ),
+]
 
 function s3SSECConfig() {
   return {
     ...s3Config(),
     ignoreErrorsFromDEKReEncryption: false,
     automaticallyRotateDEKEncryption: true,
-    pathIsProjectFolder(_bucketName, path) {
+    pathToProjectFolder(_bucketName, path) {
-      return !!path.match(/^[a-f0-9]+\/$/)
+      const [projectFolder] = path.match(/^[a-f0-9]+\//)
+      return projectFolder
     },
     pathToDataEncryptionKeyPath(_bucketName, path) {
       const [projectFolder] = path.match(/^[a-f0-9]+\//)
@@ -43,7 +52,7 @@ function s3SSECConfig() {
         path: Path.join(projectFolder, 'dek'),
       }
     },
-    async getKeyEncryptionKeys() {
+    async getRootKeyEncryptionKeys() {
       return S3SSECKeys
     },
   }