Merge pull request #14419 from overleaf/em-history-lib-async-await

Move overleaf-editor-core code to async/await

GitOrigin-RevId: 4ab8a58ba2ab402ff60a40e831b9c4a2c4701177
This commit is contained in:
Eric Mc Sween 2023-08-22 10:48:24 -04:00 committed by Copybot
parent d54bcc4aa9
commit 808fd2c0f9
23 changed files with 186 additions and 237 deletions

View file

@ -2,7 +2,7 @@
const _ = require('lodash')
const assert = require('check-types').assert
const BPromise = require('bluebird')
const pMap = require('p-map')
const AuthorList = require('./author_list')
const Operation = require('./operation')
@ -217,12 +217,12 @@ class Change {
*
* @param {string} kind see {File#load}
* @param {BlobStore} blobStore
* @return {Promise}
* @return {Promise<void>}
*/
loadFiles(kind, blobStore) {
return BPromise.each(this.operations, operation =>
operation.loadFiles(kind, blobStore)
)
async loadFiles(kind, blobStore) {
for (const operation of this.operations) {
await operation.loadFiles(kind, blobStore)
}
}
/**
@ -301,20 +301,19 @@ class Change {
return Change.fromRaw(this.toRaw())
}
store(blobStore, concurrency) {
async store(blobStore, concurrency) {
assert.maybe.number(concurrency, 'bad concurrency')
const raw = this.toRaw()
raw.authors = _.uniq(raw.authors)
return BPromise.map(
const rawOperations = await pMap(
this.operations,
operation => operation.store(blobStore),
{ concurrency: concurrency || 1 }
).then(rawOperations => {
raw.operations = rawOperations
return raw
})
)
raw.operations = rawOperations
return raw
}
canBeComposedWith(other) {

View file

@ -157,10 +157,10 @@ class Chunk {
*
* @param {string} kind
* @param {BlobStore} blobStore
* @return {Promise}
* @return {Promise<void>}
*/
loadFiles(kind, blobStore) {
return this.history.loadFiles(kind, blobStore)
async loadFiles(kind, blobStore) {
await this.history.loadFiles(kind, blobStore)
}
}

View file

@ -15,11 +15,6 @@ const StringFileData = require('./file_data/string_file_data')
* @typedef {import("./operation/text_operation")} TextOperation
*/
/**
* @template T
* @typedef {import("bluebird")<T>} BPromise
*/
class NotEditableError extends OError {
constructor() {
super('File is not editable')
@ -206,11 +201,10 @@ class File {
* @param {BlobStore} blobStore
* @return {Promise.<File>} for this
*/
load(kind, blobStore) {
return this.data.load(kind, blobStore).then(data => {
this.data = data
return this
})
async load(kind, blobStore) {
const data = await this.data.load(kind, blobStore)
this.data = data
return this
}
/**
@ -219,13 +213,12 @@ class File {
* the hash.
*
* @param {BlobStore} blobStore
* @return {BPromise<Object>} a raw HashFile
* @return {Promise<Object>} a raw HashFile
*/
store(blobStore) {
return this.data.store(blobStore).then(raw => {
storeRawMetadata(this.metadata, raw)
return raw
})
async store(blobStore) {
const raw = await this.data.store(blobStore)
storeRawMetadata(this.metadata, raw)
return raw
}
}

View file

@ -1,7 +1,6 @@
'use strict'
const assert = require('check-types').assert
const BPromise = require('bluebird')
const Blob = require('../blob')
const FileData = require('./')
@ -47,23 +46,23 @@ class BinaryFileData extends FileData {
}
/** @inheritdoc */
toEager() {
return BPromise.resolve(this)
async toEager() {
return this
}
/** @inheritdoc */
toLazy() {
return BPromise.resolve(this)
async toLazy() {
return this
}
/** @inheritdoc */
toHollow() {
return BPromise.try(() => FileData.createHollow(this.byteLength, null))
async toHollow() {
return FileData.createHollow(this.byteLength, null)
}
/** @inheritdoc */
store() {
return BPromise.resolve({ hash: this.hash })
async store() {
return { hash: this.hash }
}
}

View file

@ -1,7 +1,6 @@
'use strict'
const assert = require('check-types').assert
const BPromise = require('bluebird')
const Blob = require('../blob')
const FileData = require('./')
@ -33,30 +32,27 @@ class HashFileData extends FileData {
}
/** @inheritdoc */
toEager(blobStore) {
return this.toLazy(blobStore).then(lazyFileData =>
lazyFileData.toEager(blobStore)
)
async toEager(blobStore) {
const lazyFileData = await this.toLazy(blobStore)
return await lazyFileData.toEager(blobStore)
}
/** @inheritdoc */
toLazy(blobStore) {
return blobStore.getBlob(this.hash).then(blob => {
if (!blob) throw new Error('blob not found: ' + this.hash)
return FileData.createLazyFromBlob(blob)
})
async toLazy(blobStore) {
const blob = await blobStore.getBlob(this.hash)
if (!blob) throw new Error('blob not found: ' + this.hash)
return FileData.createLazyFromBlob(blob)
}
/** @inheritdoc */
toHollow(blobStore) {
return blobStore.getBlob(this.hash).then(function (blob) {
return FileData.createHollow(blob.getByteLength(), blob.getStringLength())
})
async toHollow(blobStore) {
const blob = await blobStore.getBlob(this.hash)
return FileData.createHollow(blob.getByteLength(), blob.getStringLength())
}
/** @inheritdoc */
store() {
return BPromise.resolve({ hash: this.hash })
async store() {
return { hash: this.hash }
}
}

View file

@ -1,7 +1,6 @@
'use strict'
const assert = require('check-types').assert
const BPromise = require('bluebird')
const FileData = require('./')
@ -37,8 +36,8 @@ class HollowBinaryFileData extends FileData {
}
/** @inheritdoc */
toHollow() {
return BPromise.resolve(this)
async toHollow() {
return this
}
}

View file

@ -1,7 +1,6 @@
'use strict'
const assert = require('check-types').assert
const BPromise = require('bluebird')
const FileData = require('./')
@ -41,8 +40,8 @@ class HollowStringFileData extends FileData {
}
/** @inheritdoc */
toHollow() {
return BPromise.resolve(this)
async toHollow() {
return this
}
/** @inheritdoc */

View file

@ -1,7 +1,6 @@
'use strict'
const assert = require('check-types').assert
const BPromise = require('bluebird')
const Blob = require('../blob')
@ -95,52 +94,46 @@ class FileData {
/**
* @function
* @param {BlobStore} blobStore
* @return {BPromise<FileData>}
* @return {Promise<FileData>}
* @abstract
* @see FileData#load
*/
toEager(blobStore) {
return BPromise.reject(
new Error('toEager not implemented for ' + JSON.stringify(this))
)
async toEager(blobStore) {
throw new Error('toEager not implemented for ' + JSON.stringify(this))
}
/**
* @function
* @param {BlobStore} blobStore
* @return {BPromise<FileData>}
* @return {Promise<FileData>}
* @abstract
* @see FileData#load
*/
toLazy(blobStore) {
return BPromise.reject(
new Error('toLazy not implemented for ' + JSON.stringify(this))
)
async toLazy(blobStore) {
throw new Error('toLazy not implemented for ' + JSON.stringify(this))
}
/**
* @function
* @param {BlobStore} blobStore
* @return {BPromise<FileData>}
* @return {Promise<FileData>}
* @abstract
* @see FileData#load
*/
toHollow(blobStore) {
return BPromise.reject(
new Error('toHollow not implemented for ' + JSON.stringify(this))
)
async toHollow(blobStore) {
throw new Error('toHollow not implemented for ' + JSON.stringify(this))
}
/**
* @see File#load
* @param {string} kind
* @param {BlobStore} blobStore
* @return {BPromise<FileData>}
* @return {Promise<FileData>}
*/
load(kind, blobStore) {
if (kind === 'eager') return this.toEager(blobStore)
if (kind === 'lazy') return this.toLazy(blobStore)
if (kind === 'hollow') return this.toHollow(blobStore)
async load(kind, blobStore) {
if (kind === 'eager') return await this.toEager(blobStore)
if (kind === 'lazy') return await this.toLazy(blobStore)
if (kind === 'hollow') return await this.toHollow(blobStore)
throw new Error('bad file data load kind: ' + kind)
}
@ -148,13 +141,11 @@ class FileData {
* @see File#store
* @function
* @param {BlobStore} blobStore
* @return {BPromise<Object>} a raw HashFile
* @return {Promise<Object>} a raw HashFile
* @abstract
*/
store(blobStore) {
return BPromise.reject(
new Error('store not implemented for ' + JSON.stringify(this))
)
async store(blobStore) {
throw new Error('store not implemented for ' + JSON.stringify(this))
}
}

View file

@ -2,7 +2,6 @@
const _ = require('lodash')
const assert = require('check-types').assert
const BPromise = require('bluebird')
const Blob = require('../blob')
const FileData = require('./')
@ -84,22 +83,19 @@ class LazyStringFileData extends FileData {
}
/** @inheritdoc */
toEager(blobStore) {
return blobStore.getString(this.hash).then(content => {
return new EagerStringFileData(
computeContent(this.textOperations, content)
)
})
async toEager(blobStore) {
const content = await blobStore.getString(this.hash)
return new EagerStringFileData(computeContent(this.textOperations, content))
}
/** @inheritdoc */
toLazy() {
return BPromise.resolve(this)
async toLazy() {
return this
}
/** @inheritdoc */
toHollow() {
return BPromise.try(() => FileData.createHollow(null, this.stringLength))
async toHollow() {
return FileData.createHollow(null, this.stringLength)
}
/** @inheritdoc */
@ -109,20 +105,19 @@ class LazyStringFileData extends FileData {
}
/** @inheritdoc */
store(blobStore) {
if (this.textOperations.length === 0)
return BPromise.resolve({ hash: this.hash })
return blobStore
.getString(this.hash)
.then(content => {
return blobStore.putString(computeContent(this.textOperations, content))
})
.then(blob => {
this.hash = blob.getHash()
this.stringLength = blob.getStringLength()
this.textOperations.length = 0
return { hash: this.hash }
})
async store(blobStore) {
if (this.textOperations.length === 0) {
return { hash: this.hash }
}
const content = await blobStore.getString(this.hash)
const blob = await blobStore.putString(
computeContent(this.textOperations, content)
)
this.hash = blob.getHash()
this.stringLength = blob.getStringLength()
this.textOperations.length = 0
return { hash: this.hash }
}
}

View file

@ -1,7 +1,6 @@
'use strict'
const assert = require('check-types').assert
const BPromise = require('bluebird')
const FileData = require('./')
@ -57,22 +56,19 @@ class StringFileData extends FileData {
}
/** @inheritdoc */
toEager() {
return BPromise.resolve(this)
async toEager() {
return this
}
/** @inheritdoc */
toHollow() {
return BPromise.try(() =>
FileData.createHollow(this.getByteLength(), this.getStringLength())
)
async toHollow() {
return FileData.createHollow(this.getByteLength(), this.getStringLength())
}
/** @inheritdoc */
store(blobStore) {
return blobStore.putString(this.content).then(function (blob) {
return { hash: blob.getHash() }
})
async store(blobStore) {
const blob = await blobStore.putString(this.content)
return { hash: blob.getHash() }
}
}

View file

@ -1,10 +1,9 @@
'use strict'
const BPromise = require('bluebird')
const _ = require('lodash')
const assert = require('check-types').assert
const OError = require('@overleaf/o-error')
const pMap = require('p-map')
const File = require('./file')
const safePathname = require('./safe_pathname')
@ -233,22 +232,21 @@ class FileMap {
* Map the files in this map to new values asynchronously, with an optional
* limit on concurrency.
* @param {function} iteratee like for _.mapValues
* @param {number} [concurrency] as for BPromise.map
* @return {Object}
* @param {number} [concurrency]
* @return {Promise<Object>}
*/
mapAsync(iteratee, concurrency) {
async mapAsync(iteratee, concurrency) {
assert.maybe.number(concurrency, 'bad concurrency')
const pathnames = this.getPathnames()
return BPromise.map(
const files = await pMap(
pathnames,
file => {
return iteratee(this.getFile(file), file, pathnames)
},
{ concurrency: concurrency || 1 }
).then(files => {
return _.zipObject(pathnames, files)
})
)
return _.zipObject(pathnames, files)
}
}

View file

@ -1,7 +1,7 @@
'use strict'
const assert = require('check-types').assert
const BPromise = require('bluebird')
const pMap = require('p-map')
const Change = require('./change')
const Snapshot = require('./snapshot')
@ -85,16 +85,19 @@ class History {
*
* @param {string} kind see {File#load}
* @param {BlobStore} blobStore
* @return {Promise}
* @return {Promise<void>}
*/
loadFiles(kind, blobStore) {
function loadChangeFiles(change) {
return change.loadFiles(kind, blobStore)
async loadFiles(kind, blobStore) {
async function loadChangeFiles(changes) {
for (const change of changes) {
await change.loadFiles(kind, blobStore)
}
}
return BPromise.join(
await Promise.all([
this.snapshot.loadFiles(kind, blobStore),
BPromise.each(this.changes, loadChangeFiles)
)
loadChangeFiles(this.changes),
])
}
/**
@ -107,21 +110,21 @@ class History {
* operations
* @return {Promise.<Object>}
*/
store(blobStore, concurrency) {
async store(blobStore, concurrency) {
assert.maybe.number(concurrency, 'bad concurrency')
function storeChange(change) {
return change.store(blobStore, concurrency)
async function storeChange(change) {
return await change.store(blobStore, concurrency)
}
return BPromise.join(
const [rawSnapshot, rawChanges] = await Promise.all([
this.snapshot.store(blobStore, concurrency),
BPromise.map(this.changes, storeChange, { concurrency: concurrency || 1 })
).then(([rawSnapshot, rawChanges]) => {
return {
snapshot: rawSnapshot,
changes: rawChanges,
}
})
pMap(this.changes, storeChange, { concurrency: concurrency || 1 }),
])
return {
snapshot: rawSnapshot,
changes: rawChanges,
}
}
}

View file

@ -59,14 +59,13 @@ class AddFileOperation extends Operation {
}
/** @inheritdoc */
loadFiles(kind, blobStore) {
return this.file.load(kind, blobStore)
async loadFiles(kind, blobStore) {
return await this.file.load(kind, blobStore)
}
store(blobStore) {
return this.file.store(blobStore).then(rawFile => {
return { pathname: this.pathname, file: rawFile }
})
async store(blobStore) {
const rawFile = await this.file.store(blobStore)
return { pathname: this.pathname, file: rawFile }
}
/**

View file

@ -2,7 +2,6 @@
const _ = require('lodash')
const assert = require('check-types').assert
const BPromise = require('bluebird')
const TextOperation = require('./text_operation')
@ -79,11 +78,9 @@ class Operation {
*
* @param {string} kind see {File#load}
* @param {BlobStore} blobStore
* @return {Promise}
* @return {Promise<void>}
*/
loadFiles(kind, blobStore) {
return BPromise.resolve()
}
async loadFiles(kind, blobStore) {}
/**
* Return a version of this operation that is suitable for long term storage.
@ -93,8 +90,8 @@ class Operation {
* @param {BlobStore} blobStore
* @return {Promise.<Object>}
*/
store(blobStore) {
return BPromise.try(() => this.toRaw())
async store(blobStore) {
return this.toRaw()
}
/**

View file

@ -1,7 +1,6 @@
'use strict'
const _ = require('lodash')
const BPromise = require('bluebird')
const ChangeNote = require('./change_note')
const ChangeRequest = require('./change_request')
@ -113,7 +112,7 @@ class OtClient {
*/
this.waitForVersion = function otClientWaitForVersion(version) {
if (!_waiting[version]) _waiting[version] = []
return new BPromise(function (resolve, reject) {
return new Promise(function (resolve, reject) {
_waiting[version].push(resolve)
})
}

View file

@ -1,12 +1,13 @@
'use strict'
const assert = require('check-types').assert
const BPromise = require('bluebird')
const OError = require('@overleaf/o-error')
const FileMap = require('./file_map')
const V2DocVersions = require('./v2_doc_versions')
const FILE_LOAD_CONCURRENCY = 50
/**
* @typedef {import("./types").BlobStore} BlobStore
* @typedef {import("./change")} Change
@ -194,10 +195,14 @@ class Snapshot {
*
* @param {string} kind see {File#load}
* @param {BlobStore} blobStore
* @return {Promise}
* @return {Promise<Object>} an object where keys are the pathnames and
* values are the files in the snapshot
*/
loadFiles(kind, blobStore) {
return BPromise.props(this.fileMap.map(file => file.load(kind, blobStore)))
async loadFiles(kind, blobStore) {
return await this.fileMap.mapAsync(
file => file.load(kind, blobStore),
FILE_LOAD_CONCURRENCY
)
}
/**
@ -208,22 +213,22 @@ class Snapshot {
* @param {number} [concurrency]
* @return {Promise.<Object>}
*/
store(blobStore, concurrency) {
async store(blobStore, concurrency) {
assert.maybe.number(concurrency, 'bad concurrency')
const projectVersion = this.projectVersion
const rawV2DocVersions = this.v2DocVersions
? this.v2DocVersions.toRaw()
: undefined
return this.fileMap
.mapAsync(file => file.store(blobStore), concurrency)
.then(rawFiles => {
return {
files: rawFiles,
projectVersion,
v2DocVersions: rawV2DocVersions,
}
})
const rawFiles = await this.fileMap.mapAsync(
file => file.store(blobStore),
concurrency
)
return {
files: rawFiles,
projectVersion,
v2DocVersions: rawV2DocVersions,
}
}
/**

View file

@ -1,9 +1,8 @@
import Blob from './blob'
import BPromise from 'bluebird'
export type BlobStore = {
getString(hash: string): BPromise<string>
putString(content: string): BPromise<Blob>
getString(hash: string): Promise<string>
putString(content: string): Promise<Blob>
}
export type StringFileRawData = {

View file

@ -16,7 +16,6 @@
"license": "Proprietary",
"private": true,
"devDependencies": {
"@types/bluebird": "^3.5.30",
"chai": "^3.3.0",
"istanbul": "^0.4.5",
"mocha": "^10.2.0",
@ -24,8 +23,8 @@
},
"dependencies": {
"@overleaf/o-error": "*",
"bluebird": "^3.1.1",
"check-types": "^5.1.0",
"lodash": "^4.17.19"
"lodash": "^4.17.19",
"p-map": "^4.0.0"
}
}

View file

@ -26,7 +26,7 @@ describe('File', function () {
const file = File.fromHash(File.EMPTY_FILE_HASH, metadata)
expect(file.toRaw()).to.eql({
hash: File.EMPTY_FILE_HASH,
metadata: metadata,
metadata,
})
delete file.getMetadata().main
@ -45,34 +45,31 @@ describe('File', function () {
})
describe('store', function () {
it('does not return empty metadata', function () {
it('does not return empty metadata', async function () {
const file = File.fromHash(File.EMPTY_FILE_HASH)
const fakeBlobStore = new FakeBlobStore()
return file.store(fakeBlobStore).then(raw => {
expect(raw).to.eql({ hash: File.EMPTY_FILE_HASH })
})
const raw = await file.store(fakeBlobStore)
expect(raw).to.eql({ hash: File.EMPTY_FILE_HASH })
})
it('returns non-empty metadata', function () {
it('returns non-empty metadata', async function () {
const metadata = { main: true }
const file = File.fromHash(File.EMPTY_FILE_HASH, metadata)
const fakeBlobStore = new FakeBlobStore()
return file.store(fakeBlobStore).then(raw => {
expect(raw).to.eql({
hash: File.EMPTY_FILE_HASH,
metadata: metadata,
})
const raw = await file.store(fakeBlobStore)
expect(raw).to.eql({
hash: File.EMPTY_FILE_HASH,
metadata,
})
})
it('returns a deep clone of metadata', function () {
it('returns a deep clone of metadata', async function () {
const metadata = { externalFile: { id: 123 } }
const file = File.fromHash(File.EMPTY_FILE_HASH, metadata)
const fakeBlobStore = new FakeBlobStore()
return file.store(fakeBlobStore).then(raw => {
raw.metadata.externalFile.id = 456
expect(file.getMetadata().externalFile.id).to.equal(123)
})
const raw = await file.store(fakeBlobStore)
raw.metadata.externalFile.id = 456
expect(file.getMetadata().externalFile.id).to.equal(123)
})
})

View file

@ -1,7 +1,6 @@
'use strict'
const { expect } = require('chai')
const BPromise = require('bluebird')
const _ = require('lodash')
const ot = require('..')
@ -177,26 +176,20 @@ describe('FileMap', function () {
expect(() => fileMap.removeFile('b')).to.throw(FileMap.FileNotFoundError)
})
it('has mapAsync', function () {
it('has mapAsync', async function () {
const concurrency = 1
return BPromise.map(
[
[[], {}],
[['a'], { a: 'a-a' }], // the test is to map to "content-pathname"
[['a', 'b'], { a: 'a-a', b: 'b-b' }],
],
test => {
const input = test[0]
const expectedOutput = test[1]
const fileMap = makeFileMap(input)
return fileMap
.mapAsync((file, pathname) => {
return file.getContent() + '-' + pathname
}, concurrency)
.then(result => {
expect(result).to.deep.equal(expectedOutput)
})
}
)
for (const test of [
[[], {}],
[['a'], { a: 'a-a' }], // the test is to map to "content-pathname"
[['a', 'b'], { a: 'a-a', b: 'b-b' }],
]) {
const input = test[0]
const expectedOutput = test[1]
const fileMap = makeFileMap(input)
const result = await fileMap.mapAsync((file, pathname) => {
return file.getContent() + '-' + pathname
}, concurrency)
expect(result).to.deep.equal(expectedOutput)
}
})
})

View file

@ -2,11 +2,6 @@
* @typedef {import("../..").Blob } Blob
*/
/**
* @template T
* @typedef {import("bluebird")<T>} BPromise
*/
/**
* Fake blob store for tests
*/
@ -15,7 +10,7 @@ class FakeBlobStore {
* Get a string from the blob store
*
* @param {string} hash
* @return {BPromise<string>}
* @return {Promise<string>}
*/
getString(hash) {
throw new Error('Not implemented')
@ -25,7 +20,7 @@ class FakeBlobStore {
* Store a string in the blob store
*
* @param {string} content
* @return {BPromise<Blob>}
* @return {Promise<Blob>}
*/
putString(content) {
throw new Error('Not implemented')

8
package-lock.json generated
View file

@ -465,12 +465,11 @@
"license": "Proprietary",
"dependencies": {
"@overleaf/o-error": "*",
"bluebird": "^3.1.1",
"check-types": "^5.1.0",
"lodash": "^4.17.19"
"lodash": "^4.17.19",
"p-map": "^4.0.0"
},
"devDependencies": {
"@types/bluebird": "^3.5.30",
"chai": "^3.3.0",
"istanbul": "^0.4.5",
"mocha": "^10.2.0",
@ -74774,13 +74773,12 @@
"version": "file:libraries/overleaf-editor-core",
"requires": {
"@overleaf/o-error": "*",
"@types/bluebird": "^3.5.30",
"bluebird": "^3.1.1",
"chai": "^3.3.0",
"check-types": "^5.1.0",
"istanbul": "^0.4.5",
"lodash": "^4.17.19",
"mocha": "^10.2.0",
"p-map": "^4.0.0",
"typescript": "^5.0.4"
},
"dependencies": {

View file

@ -161,7 +161,7 @@ describe('overleaf ot', function () {
.then(() => projectId)
})
.tap(projectId => {
.then(projectId => {
// Fetch empty file blob
return client.apis.Project.getProjectBlob({
project_id: projectId,