github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #6661 from overleaf/spd-local-tests

Browse source 
Move acceptance test mocks to nonstandard ports and add options for running locally

GitOrigin-RevId: bd8f70ac8d80599daccc51cfe7b90a2ad8d8c3d8
This commit is contained in:
Simon Detheridge 2022-08-09 13:03:30 +01:00 committed by Copybot
parent f1ee27a518
commit 9953822175
22 changed files with 172 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
services/web/app/src/infrastructure/mongodb.js
View file

@ -1,5 +1,6 @@
const Settings = require('@overleaf/settings')
const { MongoClient, ObjectId } = require('mongodb')
const OError = require('@overleaf/o-error')
const { addConnectionDrainer } = require('./GracefulShutdown')
if (
@ -86,6 +87,20 @@ async function getCollectionNames() {
return collections.map(collection => collection.collectionName)
}
async function dropTestDatabase() {
const internalDb = (await clientPromise).db()
const dbName = internalDb.databaseName
const env = process.env.NODE_ENV
if (dbName !== 'test-sharelatex' || env !== 'test') {
throw new OError(
`Refusing to clear database '${dbName}' in environment '${env}'`
)
}
await internalDb.dropDatabase()
}
/**
* WARNING: Consider using a pre-populated collection from `db` to avoid typos!
*/
@ -99,5 +114,6 @@ module.exports = {
ObjectId,
getCollectionNames,
getCollectionInternal,
dropTestDatabase,
waitForDb,
}

6
services/web/config/settings.defaults.js
View file

@ -1,3 +1,4 @@
const Path = require('path')
const { merge } = require('@overleaf/settings/merge')
let defaultFeatures, siteUrl
@ -105,6 +106,7 @@ module.exports = {
host: process.env.REDIS_HOST || 'localhost',
port: process.env.REDIS_PORT || '6379',
password: process.env.REDIS_PASSWORD || '',
db: process.env.REDIS_DB,
maxRetriesPerRequest: parseInt(
process.env.REDIS_MAX_RETRIES_PER_REQUEST || '20'
),
@ -565,8 +567,8 @@ module.exports = {
// If we ever need to write something to disk (e.g. incoming requests
// that need processing but may be too big for memory, then write
// them to disk here).
dumpFolder: './data/dumpFolder',
uploadFolder: './data/uploads',
dumpFolder: Path.resolve(__dirname, '../data/dumpFolder'),
uploadFolder: Path.resolve(__dirname, '../data/uploads'),
},
// Automatic Snapshots

2
services/web/docker-compose.ci.yml
View file

@ -110,4 +110,4 @@ services:
environment:
SAML_BASE_URL_PATH: 'http://saml/simplesaml/'
SAML_TEST_SP_ENTITY_ID: 'sharelatex-test-saml'
SAML_TEST_SP_LOCATION: 'http://www.overleaf.test:3000/saml/callback'
SAML_TEST_SP_LOCATION: 'http://www.overleaf.test:23000/saml/callback'

11
services/web/docker-compose.common.env
View file

@ -1,9 +1,10 @@
ADMIN_PRIVILEGE_AVAILABLE=true
BCRYPT_ROUNDS=1
WEB_PORT=23000
REDIS_HOST=redis
QUEUES_REDIS_HOST=redis
ANALYTICS_QUEUES_REDIS_HOST=redis
MONGO_URL=mongodb://mongo/sharelatex
MONGO_URL=mongodb://mongo/test-sharelatex
SHARELATEX_ALLOW_PUBLIC_ACCESS=true
LINKED_URL_PROXY=http://localhost:6543
ENABLED_LINKED_FILE_TYPES=url,project_file,project_output_file,mendeley,zotero
@ -12,15 +13,15 @@ NODE_OPTIONS=--unhandled-rejections=strict
LOCK_MANAGER_MAX_LOCK_WAIT_TIME=30000
COOKIE_DOMAIN=.overleaf.test
ADMIN_URL=http://admin.overleaf.test
PUBLIC_URL=http://www.overleaf.test:3000
PUBLIC_URL=http://www.overleaf.test:23000
HTTP_TEST_HOST=www.overleaf.test
OT_JWT_AUTH_KEY=very secret key
OT_JWT_AUTH_KEY="very secret key"
EXTERNAL_AUTH=none
RECAPTCHA_ENDPOINT=http://localhost:2222/recaptcha/api/siteverify
# Server-Pro LDAP
SHARELATEX_LDAP_URL=ldap://ldap:389
SHARELATEX_LDAP_SEARCH_BASE=ou=people,dc=planetexpress,dc=com
SHARELATEX_LDAP_SEARCH_FILTER=(uid={{username}})
SHARELATEX_LDAP_SEARCH_FILTER="(uid={{username}})"
SHARELATEX_LDAP_BIND_DN=cn=admin,dc=planetexpress,dc=com
SHARELATEX_LDAP_BIND_CREDENTIALS=GoodNewsEveryone
SHARELATEX_LDAP_EMAIL_ATT=mail
@ -31,7 +32,7 @@ SHARELATEX_LDAP_UPDATE_USER_DETAILS_ON_LOGIN=true
SHARELATEX_SAML_ENTRYPOINT=http://saml/simplesaml/saml2/idp/SSOService.php
SHARELATEX_SAML_CALLBACK_URL=http://saml/saml/callback
SHARELATEX_SAML_ISSUER=sharelatex-test-saml
SHARELATEX_SAML_IDENTITY_SERVICE_NAME=SAML Test Server
SHARELATEX_SAML_IDENTITY_SERVICE_NAME="SAML Test Server"
SHARELATEX_SAML_EMAIL_FIELD=email
SHARELATEX_SAML_FIRST_NAME_FIELD=givenName
SHARELATEX_SAML_LAST_NAME_FIELD=sn

2
services/web/docker-compose.yml
View file

@ -108,4 +108,4 @@ services:
environment:
SAML_BASE_URL_PATH: 'http://saml/simplesaml/'
SAML_TEST_SP_ENTITY_ID: 'sharelatex-test-saml'
SAML_TEST_SP_LOCATION: 'http://www.overleaf.test:3000/saml/callback'
SAML_TEST_SP_LOCATION: 'http://www.overleaf.test:23000/saml/callback'

19
services/web/local-dev.env Normal file
View file

@ -0,0 +1,19 @@
REDIS_HOST=localhost
QUEUES_REDIS_HOST=localhost
PUBSUB_REDIS_HOST=localhost
GCLOUD_2_REDIS_HOST=localhost
QUEUES_REDIS_HOST=localhost
POSTGRES_HOST=localhost
MONGO_URL=mongodb://localhost/sharelatex
SHARELATEX_LDAP_URL=ldap://localhost:22389
SAML_BASE_URL_PATH=http://localhost:22280/simplesaml/
SHARELATEX_SAML_ENTRYPOINT=http://localhost:22280/simplesaml/saml2/idp/SSOService.php
SHARELATEX_SAML_CALLBACK_URL=http://www.dev-overleaf.com/saml/callback
COOKIE_DOMAIN=.dev-overleaf.com
PUBLIC_URL=https://www.dev-overleaf.com/
CURRENT_IMAGE_NAME=texlive-full:2021.1
LINKED_URL_PROXY=http://localhost:8081/proxy/secret-proxy-token
SHARELATEX_CONFIG=$(pwd)/../../config/settings.web.js
ENABLE_PDF_CACHING=true
HAVE_I_BEEN_PWNED_ENABLED=true
NODE_ENV=development

18
services/web/local-test.env Normal file
View file

@ -0,0 +1,18 @@
CLEANUP_MONGO=true
REDIS_HOST=localhost
REDIS_DB=5
QUEUES_REDIS_HOST=localhost
PUBSUB_REDIS_HOST=localhost
GCLOUD_2_REDIS_HOST=localhost
QUEUES_REDIS_HOST=localhost
POSTGRES_HOST=localhost
MONGO_URL=mongodb://localhost/test-sharelatex
SHARELATEX_LDAP_URL=ldap://localhost:22389
SAML_BASE_URL_PATH=http://localhost:22280/simplesaml/
SHARELATEX_SAML_ENTRYPOINT=http://localhost:22280/simplesaml/saml2/idp/SSOService.php
SHARELATEX_SAML_CALLBACK_URL=http://localhost:22280/saml/callback
COOKIE_DOMAIN=
PUBLIC_URL=http://localhost:23000
HTTP_TEST_HOST=localhost

4
services/web/modules/server-ce-scripts/test/acceptance/src/ServerCEScriptsTests.js
View file

@ -50,9 +50,7 @@ describe('ServerCEScripts', function () {
it('should exit with code 1 on error', function () {
try {
run(
'REDIS_HOST=localhost node modules/server-ce-scripts/scripts/check-redis'
)
run('REDIS_PORT=42 node modules/server-ce-scripts/scripts/check-redis')
} catch (e) {
expect(e.status).to.equal(1)
return

21
services/web/package.json
View file

@ -36,7 +36,26 @@
"cypress:run-ct": "SHARELATEX_CONFIG=$PWD/config/settings.webpack.js cypress run --component",
"lezer-latex:generate": "node modules/source-editor/scripts/lezer-latex/generate.js",
"lezer-latex:run": "node modules/source-editor/scripts/lezer-latex/run.mjs",
"routes": "bin/routes"
"routes": "bin/routes",
"routes": "bin/routes",
"local:nodemon": "set -a;. ../../config/dev-environment.env;. ../../config/local.env;. ../../config/local-dev.env;. ./docker-compose.common.env;. ./local-dev.env; set +a; echo $SHARELATEX_CONFIG; WEB_PORT=13000 LISTEN_ADDRESS=0.0.0.0 npm run nodemon",
"local:webpack": "set -a;. ../../config/dev-environment.env;. ../../config/local.env;. ../../config/local-dev.env;. ./docker-compose.common.env;. ./local-dev.env; set +a; PORT=13808 npm run webpack",
"local:test:acceptance:run_dir": "set -a;. $(pwd)/docker-compose.common.env;. $(pwd)/local-test.env; set +a; npm run test:acceptance:run_dir",
"local:test:acceptance:run_app": "SHARELATEX_CONFIG=$(pwd)/test/acceptance/config/settings.test.${OVERLEAF_APP}.js npm run local:test:acceptance:run_dir -- $(pwd)/test/acceptance/src",
"local:test:acceptance:run_module": "if [ ! -d $(pwd)/modules/${MODULE}/test/acceptance ]; then echo \"Module '${MODULE}' does not have acceptance tests\"; exit 0; fi; SHARELATEX_CONFIG=$(pwd)/modules/${MODULE}/test/acceptance/config/settings.test.js BASE_CONFIG=$(pwd)/test/acceptance/config/settings.test.${OVERLEAF_APP}.js npm run local:test:acceptance:run_dir -- $(pwd)/modules/${MODULE}/test/acceptance",
"local:test:acceptance:run_modules": "SHARELATEX_CONFIG=$(pwd)/test/acceptance/config/settings.test.${OVERLEAF_APP}.js node $(pwd)/test/acceptance/getModuleTargets --name-only | xargs -n1 sh -c 'MODULE=$0 npm run local:test:acceptance:run_module || exit 255' $1",
"local:test:acceptance:app:saas": "OVERLEAF_APP=saas npm run local:test:acceptance:run_app",
"local:test:acceptance:app:server-pro": "OVERLEAF_APP=server-pro npm run local:test:acceptance:run_app",
"local:test:acceptance:app:server-ce": "OVERLEAF_APP=server-ce npm run local:test:acceptance:run_app",
"local:test:acceptance:app": "echo saas server-pro server-ce | xargs -n1 sh -c 'npm run local:test:acceptance:app:${0} || exit 255'",
"local:test:acceptance:modules:saas": "OVERLEAF_APP=saas npm run local:test:acceptance:run_modules",
"local:test:acceptance:modules:server-pro": "OVERLEAF_APP=server-pro npm run local:test:acceptance:run_modules",
"local:test:acceptance:modules:server-ce": "OVERLEAF_APP=server-ce npm run local:test:acceptance:run_modules",
"local:test:acceptance:modules": "echo saas server-ce server-pro | xargs -n1 sh -c 'npm run local:test:acceptance:modules:${0} || exit 255'",
"local:test:acceptance": "npm run local:test:acceptance:app && npm run local:test:acceptance:modules",
"local:test:unit": "npm run test:unit:all",
"local:test:frontend": "npm run test:frontend",
"local:test": "npm run local:test:unit && npm run local:test:frontend && npm run test:karma:single && npm run local:test:acceptance"
},
"browserslist": [
"last 1 year",

36
services/web/test/acceptance/config/settings.test.defaults.js
View file

@ -30,6 +30,7 @@ module.exports = {
},
web: {
url: 'http://localhost:23000',
user: httpAuthUser,
pass: httpAuthPass,
},
@ -38,6 +39,41 @@ module.exports = {
enabled: false,
url: 'http://localhost:1337',
},
documentupdater: {
url: 'http://localhost:23003',
},
spelling: {
url: 'http://localhost:23005',
host: 'localhost',
},
trackchanges: {
url: 'http://localhost:23015',
},
docstore: {
url: 'http://localhost:23016',
pubUrl: 'http://localhost:23016',
},
chat: {
internal_url: 'http://localhost:23010',
},
filestore: {
url: 'http://localhost:23009',
},
clsi: {
url: 'http://localhost:23013',
},
realTime: {
url: 'http://localhost:23026',
},
contacts: {
url: 'http://localhost:23036',
},
notifications: {
url: 'http://localhost:23042',
},
webpack: {
url: 'http://localhost:23808',
},
},
// for registration via SL, set enableLegacyRegistration to true

14
services/web/test/acceptance/config/settings.test.saas.js
View file

@ -11,15 +11,21 @@ const overrides = {
enableSubscriptions: true,
apis: {
thirdPartyDataStore: {
url: `http://localhost:23002`,
},
analytics: {
url: `http://localhost:23050`,
},
project_history: {
sendProjectStructureOps: true,
initializeHistoryForNewProjects: true,
displayHistoryForNewProjects: true,
url: `http://localhost:3054`,
url: `http://localhost:23054`,
},
recurly: {
url: 'http://localhost:6034',
url: 'http://localhost:26034',
subdomain: 'test',
apiKey: 'private-nonsense',
webhookUser: 'recurly',
@ -32,13 +38,13 @@ const overrides = {
},
v1: {
url: 'http://localhost:5000',
url: `http://localhost:25000`,
user: 'overleaf',
pass: 'password',
},
v1_history: {
url: `http://localhost:3100/api`,
url: `http://localhost:23100/api`,
user: 'overleaf',
pass: 'password',
},

6
services/web/test/acceptance/getModuleTargets.js
View file

@ -5,4 +5,8 @@ const Settings = require('@overleaf/settings')
const MODULES = Settings.moduleImportSequence
const TARGET = process.argv.slice(2).pop() || 'test_acceptance'
console.debug(MODULES.map(name => `modules/${name}/${TARGET}`).join('\n'))
if (TARGET === '--name-only') {
console.debug(MODULES.join('\n'))
} else {
console.debug(MODULES.map(name => `modules/${name}/${TARGET}`).join('\n'))
}

22
services/web/test/acceptance/src/Init.js
View file

@ -18,18 +18,18 @@ const mockOpts = {
debug: ['1', 'true', 'TRUE'].includes(process.env.DEBUG_MOCKS),
}
MockChatApi.initialize(3010, mockOpts)
MockClsiApi.initialize(3013, mockOpts)
MockDocstoreApi.initialize(3016, mockOpts)
MockDocUpdaterApi.initialize(3003, mockOpts)
MockFilestoreApi.initialize(3009, mockOpts)
MockNotificationsApi.initialize(3042, mockOpts)
MockSpellingApi.initialize(3005, mockOpts)
MockChatApi.initialize(23010, mockOpts)
MockClsiApi.initialize(23013, mockOpts)
MockDocstoreApi.initialize(23016, mockOpts)
MockDocUpdaterApi.initialize(23003, mockOpts)
MockFilestoreApi.initialize(23009, mockOpts)
MockNotificationsApi.initialize(23042, mockOpts)
MockSpellingApi.initialize(23005, mockOpts)
MockHaveIBeenPwnedApi.initialize(1337, mockOpts)
if (Features.hasFeature('saas')) {
MockAnalyticsApi.initialize(3050, mockOpts)
MockProjectHistoryApi.initialize(3054, mockOpts)
MockV1Api.initialize(5000, mockOpts)
MockV1HistoryApi.initialize(3100, mockOpts)
MockAnalyticsApi.initialize(23050, mockOpts)
MockProjectHistoryApi.initialize(23054, mockOpts)
MockV1Api.initialize(25000, mockOpts)
MockV1HistoryApi.initialize(23100, mockOpts)
}

7
services/web/test/acceptance/src/PasswordResetTests.js
View file

@ -1,16 +1,9 @@
const { expect } = require('chai')
const RateLimiter = require('../../../app/src/infrastructure/RateLimiter')
const UserHelper = require('./helpers/UserHelper')
const { db } = require('../../../app/src/infrastructure/mongodb')
describe('PasswordReset', function () {
let email, response, user, userHelper, token, emailQuery
afterEach(async function () {
await RateLimiter.promises.clearRateLimit(
'password_reset_rate_limit',
'127.0.0.1'
)
})
beforeEach(async function () {
userHelper = new UserHelper()
email = userHelper.getDefaultEmail()

2
services/web/test/acceptance/src/helpers/InitApp.js
View file

@ -18,7 +18,7 @@ MockReCAPTCHAApi.initialize(2222)
let server
before('start main app', function (done) {
server = App.listen(3000, 'localhost', done)
server = App.listen(23000, 'localhost', done)
})
before('start queue workers', function () {

9
services/web/test/acceptance/src/helpers/MongoHelper.js
View file

@ -1,5 +1,9 @@
const { execFile } = require('child_process')
const { waitForDb, db } = require('../../../../app/src/infrastructure/mongodb')
const {
waitForDb,
db,
dropTestDatabase,
} = require('../../../../app/src/infrastructure/mongodb')
const Settings = require('@overleaf/settings')
const DEFAULT_ENV = 'saas'
@ -7,6 +11,9 @@ const DEFAULT_ENV = 'saas'
module.exports = {
initialize() {
before(waitForDb)
if (process.env.CLEANUP_MONGO === 'true') {
before(dropTestDatabase)
}
before(function (done) {
const args = [

11
services/web/test/acceptance/src/helpers/RedisHelper.js
View file

@ -1,11 +1,10 @@
const RateLimiter = require('../../../../app/src/infrastructure/RateLimiter')
async function clearOverleafLoginRateLimit() {
await RateLimiter.promises.clearRateLimit('overleaf-login', '127.0.0.1')
}
const RedisWrapper = require('../../../../app/src/infrastructure/RedisWrapper')
const client = RedisWrapper.client('ratelimiter')
module.exports = {
initialize() {
before(clearOverleafLoginRateLimit)
beforeEach('clear redis', function (done) {
client.flushdb(done)
})
},
}

2
services/web/test/acceptance/src/helpers/UserHelper.js
View file

@ -162,7 +162,7 @@ class UserHelper {
* @returns {string} baseUrl
*/
static baseUrl() {
return `http://${process.env.HTTP_TEST_HOST || 'localhost'}:3000`
return `http://${process.env.HTTP_TEST_HOST || 'localhost'}:23000`
}
/* static async instantiation methods */

2
services/web/test/acceptance/src/helpers/request.js
View file

@ -1,6 +1,6 @@
// TODO: This file was created by bulk-decaffeinate.
// Sanity-check the conversion and remove this comment.
const BASE_URL = `http://${process.env.HTTP_TEST_HOST || 'localhost'}:3000`
const BASE_URL = `http://${process.env.HTTP_TEST_HOST || 'localhost'}:23000`
const request = require('request').defaults({
baseUrl: BASE_URL,
followRedirect: false,

2
services/web/test/acceptance/src/mocks/MockV1HistoryApi.js
View file

@ -69,7 +69,7 @@ class MockV1HistoryApi extends AbstractMockApi {
this.requestedZipPacks++
this.events.emit('v1-history-pack-zip')
res.json({
zipUrl: `http://localhost:3100/fake-zip-download/${req.params.project_id}/version/${req.params.version}`,
zipUrl: `http://localhost:23100/fake-zip-download/${req.params.project_id}/version/${req.params.version}`,
})
}
)

11
services/web/test/unit/src/Settings/SettingsTests.js
View file

@ -2,13 +2,14 @@ const chai = require('chai')
const { expect } = chai
function clearSettingsCache() {
delete require.cache[
require.resolve('../../../../config/settings.defaults.js')
]
const monorepoPath = require
.resolve('../../../../config/settings.defaults.js')
.replace(/\/services\/web\/config\/settings\.defaults\.js$/, '')
const settingsDeps = Object.keys(require.cache).filter(
x =>
x.includes('/@overleaf/settings/') ||
x.includes('/overleaf/libraries/settings')
x.includes('/@overleaf/settings') ||
x.includes(`${monorepoPath}/libraries/settings`) ||
x.includes(`${monorepoPath}/services/web/config`)
)
settingsDeps.forEach(dep => delete require.cache[dep])
}

2
services/web/webpack.config.dev.js
View file

@ -52,7 +52,7 @@ module.exports = merge(base, {
devServer: {
// Expose dev server at www.dev-overleaf.com
host: '0.0.0.0',
port: 3808,
port: parseInt(process.env.PORT, 10) || 3808,
client: {
webSocketURL: 'auto://0.0.0.0:0/ws',
},