From 773319d55051b894cfb747be1d8d2995991aebeb Mon Sep 17 00:00:00 2001 From: Alf Eaton Date: Tue, 11 Oct 2022 09:19:52 +0100 Subject: [PATCH] Merge pull request #9725 from overleaf/ae-overleaf-develop-ce Add Server CE development environment GitOrigin-RevId: c91d9eb58b1dbc3be24646e77033cd3242e707f0 --- develop/.gitignore | 3 + develop/README.md | 79 +++++++++++++ develop/bin/dev | 3 + develop/bin/down | 3 + develop/bin/logs | 9 ++ develop/bin/shell | 3 + develop/bin/up | 3 + develop/compiles/.gitkeep | 0 develop/dev.env | 18 +++ develop/docker-compose.dev.yml | 125 +++++++++++++++++++++ develop/docker-compose.yml | 178 ++++++++++++++++++++++++++++++ develop/texlive/Dockerfile | 8 ++ develop/webpack.config.dev-env.js | 23 ++++ patches/README.md | 1 + patches/ngcomponent+4.1.0.patch | 9 ++ patches/react2angular+4.0.6.patch | 9 ++ 16 files changed, 474 insertions(+) create mode 100644 develop/.gitignore create mode 100644 develop/README.md create mode 100755 develop/bin/dev create mode 100755 develop/bin/down create mode 100755 develop/bin/logs create mode 100755 develop/bin/shell create mode 100755 develop/bin/up create mode 100644 develop/compiles/.gitkeep create mode 100644 develop/dev.env create mode 100644 develop/docker-compose.dev.yml create mode 100644 develop/docker-compose.yml create mode 100644 develop/texlive/Dockerfile create mode 100644 develop/webpack.config.dev-env.js create mode 100644 patches/README.md create mode 100644 patches/ngcomponent+4.1.0.patch create mode 100644 patches/react2angular+4.0.6.patch diff --git a/develop/.gitignore b/develop/.gitignore new file mode 100644 index 0000000000..bb5888b864 --- /dev/null +++ b/develop/.gitignore @@ -0,0 +1,3 @@ +/compiles/* +!.gitkeep +.env \ No newline at end of file diff --git a/develop/README.md b/develop/README.md new file mode 100644 index 0000000000..c23b18795c --- /dev/null +++ b/develop/README.md @@ -0,0 +1,79 @@ +# Overleaf Community Edition, development environment + +## Building and running + +In this `develop` directory, build and start the services: + +```shell +bin/up +``` + +Once the services are running, open to create the first admin account. + +After making any changes to the code, run `bin/up` manually to rebuild the changed Docker images and recreate the changed containers. + +## TeX Live + +Compiling a PDF requires building a TeX Live image to handle the compilation inside Docker: + +```shell +docker build texlive -t texlive-full +``` + +> **Note** +> To compile on a macOS host, you'll need to override the path to the Docker socket by creating a `.env` file in this directory, containing +> `DOCKER_SOCKET_PATH=/var/run/docker.sock.raw` + +## Development + +To avoid running `bin/up` after every code change, you can run Overleaf +Community Edition in _development mode_, where services will automatically +update on code changes. To do this, use the included `bin/dev` script: + +```shell +bin/dev +``` + +This will start all services using `nodemon`, which will automatically monitor +the code and restart the services as necessary. This will incur a performance +hit, especially on macOS, so in order to only start a subset of the services, +provide a space-separated list to the `bin/dev` script. + +```shell +bin/dev [service1] [service2] ... [serviceN] +``` + +> **Note** +> Starting the `web` service in _development mode_ will only update the `web` +> service when backend code changes. In order to automatically update frontend +> code as well, make sure to start the `webpack` service in _development mode_ +> as well. + +## Debugging + +When run in _development mode_ most services expose a debugging port to which +you can attach a debugger such as +[the inspector in Chrome's Dev Tools](chrome://inspect/) or one integrated into +an IDE. The following table shows the port exposed on the **host machine** for +each service: + +| Service | Port | +| ------------------ | ---- | +| `web` | 9229 | +| `clsi` | 9230 | +| `chat` | 9231 | +| `contacts` | 9232 | +| `docstore` | 9233 | +| `document-updater` | 9234 | +| `filestore` | 9235 | +| `notifications` | 9236 | +| `real-time` | 9237 | +| `track-changes` | 9238 | + +To attach to a service using Chrome's _remote debugging_, go to + and make sure _Discover network targets_ is checked. Next +click _Configure..._ and add an entry `localhost:[service port]` for each of the +services you want to attach a debugger to. + +After adding an entry, the service will show up as a _Remote Target_ that you +can inspect and debug. diff --git a/develop/bin/dev b/develop/bin/dev new file mode 100755 index 0000000000..1dca01c92b --- /dev/null +++ b/develop/bin/dev @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +docker-compose -f docker-compose.yml -f docker-compose.dev.yml up --no-deps --detach "$@" diff --git a/develop/bin/down b/develop/bin/down new file mode 100755 index 0000000000..db6c58e092 --- /dev/null +++ b/develop/bin/down @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +docker-compose down diff --git a/develop/bin/logs b/develop/bin/logs new file mode 100755 index 0000000000..ca44dca017 --- /dev/null +++ b/develop/bin/logs @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +docker-compose logs --follow --tail 10 --no-color "$@" \ + | ggrep --line-buffered --invert-match "global.gc" \ + | ggrep --line-buffered --invert-match "health.check" \ + | ggrep --line-buffered --invert-match "slow event loop" \ + | ggrep --line-buffered --invert-match "process.memoryUsage" \ + | ggrep --line-buffered --only-matching "\{.*" \ + | bunyan --output short diff --git a/develop/bin/shell b/develop/bin/shell new file mode 100755 index 0000000000..564df7f5e5 --- /dev/null +++ b/develop/bin/shell @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +docker-compose exec -it "$@" /bin/bash diff --git a/develop/bin/up b/develop/bin/up new file mode 100755 index 0000000000..3ea6b03ff4 --- /dev/null +++ b/develop/bin/up @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +docker-compose up --build --detach "$@" diff --git a/develop/compiles/.gitkeep b/develop/compiles/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/develop/dev.env b/develop/dev.env new file mode 100644 index 0000000000..689f278aaf --- /dev/null +++ b/develop/dev.env @@ -0,0 +1,18 @@ +CHAT_HOST=chat +CLSI_HOST=clsi +CONTACTS_HOST=contacts +DOCSTORE_HOST=docstore +DOCUMENT_UPDATER_HOST=document-updater +FILESTORE_HOST=filestore +GRACEFUL_SHUTDOWN_DELAY=0 +LISTEN_ADDRESS=0.0.0.0 +MONGO_HOST=mongo +NOTIFICATIONS_HOST=notifications +REALTIME_HOST=real-time +REDIS_HOST=redis +SPELLING_HOST=spelling +TRACK_CHANGES_HOST=track-changes +WEBPACK_HOST=webpack +WEB_API_PASSWORD=sharelatex +WEB_API_USER=sharelatex +WEB_HOST=web \ No newline at end of file diff --git a/develop/docker-compose.dev.yml b/develop/docker-compose.dev.yml new file mode 100644 index 0000000000..f38391a660 --- /dev/null +++ b/develop/docker-compose.dev.yml @@ -0,0 +1,125 @@ +services: + clsi: + command: ["npm", "run", "nodemon"] + environment: + - NODE_APP_OPTIONS=--inspect=0.0.0.0:9229 + ports: + - "127.0.0.1:9230:9229" + volumes: + - ../services/clsi/app:/overleaf/services/clsi/app + - ../services/clsi/app.js:/overleaf/services/clsi/app.js + - ../services/clsi/config:/overleaf/services/clsi/config + + chat: + command: ["npm", "run", "nodemon"] + environment: + - NODE_APP_OPTIONS=--inspect=0.0.0.0:9229 + ports: + - "127.0.0.1:9231:9229" + volumes: + - ../services/chat/app:/overleaf/services/chat/app + - ../services/chat/app.js:/overleaf/services/chat/app.js + - ../services/chat/config:/overleaf/services/chat/config + + contacts: + command: ["npm", "run", "nodemon"] + environment: + - NODE_APP_OPTIONS=--inspect=0.0.0.0:9229 + ports: + - "127.0.0.1:9232:9229" + volumes: + - ../services/contacts/app:/overleaf/services/contacts/app + - ../services/contacts/app.js:/overleaf/services/contacts/app.js + - ../services/contacts/config:/overleaf/services/contacts/config + + docstore: + command: ["npm", "run", "nodemon"] + environment: + - NODE_APP_OPTIONS=--inspect=0.0.0.0:9229 + ports: + - "127.0.0.1:9233:9229" + volumes: + - ../services/docstore/app:/overleaf/services/docstore/app + - ../services/docstore/app.js:/overleaf/services/docstore/app.js + - ../services/docstore/config:/overleaf/services/docstore/config + + document-updater: + command: ["npm", "run", "nodemon"] + environment: + - NODE_APP_OPTIONS=--inspect=0.0.0.0:9229 + ports: + - "127.0.0.1:9234:9229" + volumes: + - ../services/document-updater/app:/overleaf/services/document-updater/app + - ../services/document-updater/app.js:/overleaf/services/document-updater/app.js + - ../services/document-updater/config:/overleaf/services/document-updater/config + + filestore: + command: ["npm", "run", "nodemon"] + environment: + - NODE_APP_OPTIONS=--inspect=0.0.0.0:9229 + ports: + - "127.0.0.1:9235:9229" + volumes: + - ../services/filestore/app:/overleaf/services/filestore/app + - ../services/filestore/app.js:/overleaf/services/filestore/app.js + - ../services/filestore/config:/overleaf/services/filestore/config + + notifications: + command: ["npm", "run", "nodemon"] + environment: + - NODE_APP_OPTIONS=--inspect=0.0.0.0:9229 + ports: + - "127.0.0.1:9236:9229" + volumes: + - ../services/notifications/app:/overleaf/services/notifications/app + - ../services/notifications/app.js:/overleaf/services/notifications/app.js + - ../services/notifications/config:/overleaf/services/notifications/config + + real-time: + command: ["npm", "run", "nodemon"] + environment: + - NODE_APP_OPTIONS=--inspect=0.0.0.0:9229 + ports: + - "127.0.0.1:9237:9229" + volumes: + - ../services/real-time/app:/overleaf/services/real-time/app + - ../services/real-time/app.js:/overleaf/services/real-time/app.js + - ../services/real-time/config:/overleaf/services/real-time/config + + track-changes: + command: ["npm", "run", "nodemon"] + environment: + - NODE_APP_OPTIONS=--inspect=0.0.0.0:9229 + ports: + - "127.0.0.1:9238:9229" + volumes: + - ../services/track-changes/app:/overleaf/services/track-changes/app + - ../services/track-changes/app.js:/overleaf/services/track-changes/app.js + - ../services/track-changes/config:/overleaf/services/track-changes/config + + web: + command: ["npm", "run", "nodemon"] + environment: + - NODE_APP_OPTIONS=--inspect=0.0.0.0:9229 + ports: + - "127.0.0.1:9229:9229" + volumes: + - ../services/web/app:/overleaf/services/web/app + - ../services/web/app.js:/overleaf/services/web/app.js + - ../services/web/config:/overleaf/services/web/config + - ../services/web/locales:/overleaf/services/web/locales + - ../services/web/modules:/overleaf/services/web/modules + - ../services/web/public:/overleaf/services/web/public + + webpack: + volumes: + - ../services/web/app:/overleaf/services/web/app + - ../services/web/config:/overleaf/services/web/config + - ../services/web/frontend:/overleaf/services/web/frontend + - ../services/web/locales:/overleaf/services/web/locales + - ../services/web/modules:/overleaf/services/web/modules + - ../services/web/public:/overleaf/services/web/public + - ../services/web/transform:/overleaf/services/web/transform + - ../services/web/types:/overleaf/services/web/types + - ../services/web/webpack-plugins:/overleaf/services/web/webpack-plugins diff --git a/develop/docker-compose.yml b/develop/docker-compose.yml new file mode 100644 index 0000000000..c345faf127 --- /dev/null +++ b/develop/docker-compose.yml @@ -0,0 +1,178 @@ +volumes: + clsi-cache: + clsi-output: + filestore-public-files: + filestore-template-files: + filestore-uploads: + filestore-user-files: + mongo-data: + npm-cache: + redis-data: + sharelatex-data: + spelling-cache: + web-data: + +services: + chat: + build: + context: .. + dockerfile: services/chat/Dockerfile + env_file: + - dev.env + volumes: + - npm-cache:/root/.npm + + clsi: + build: + context: .. + dockerfile: services/clsi/Dockerfile + env_file: + - dev.env + environment: + - DOCKER_RUNNER=true + - TEXLIVE_IMAGE=texlive-full # docker build texlive -t texlive-full + - COMPILES_HOST_DIR=${PWD}/compiles + user: root + volumes: + - npm-cache:/root/.npm + - ${PWD}/compiles:/overleaf/services/clsi/compiles + - ${DOCKER_SOCKET_PATH:-/var/run/docker.sock}:/var/run/docker.sock + - clsi-cache:/overleaf/services/clsi/cache + - clsi-output:/overleaf/services/clsi/output + + contacts: + build: + context: .. + dockerfile: services/contacts/Dockerfile + env_file: + - dev.env + volumes: + - npm-cache:/root/.npm + + docstore: + build: + context: .. + dockerfile: services/docstore/Dockerfile + env_file: + - dev.env + volumes: + - npm-cache:/root/.npm + + document-updater: + build: + context: .. + dockerfile: services/document-updater/Dockerfile + env_file: + - dev.env + volumes: + - npm-cache:/root/.npm + + filestore: + build: + context: .. + dockerfile: services/filestore/Dockerfile + env_file: + - dev.env +# environment: +# - ENABLE_CONVERSIONS=true + volumes: + - npm-cache:/root/.npm + - filestore-public-files:/overleaf/services/filestore/public_files + - filestore-template-files:/overleaf/services/filestore/template_files + - filestore-uploads:/overleaf/services/filestore/uploads + - filestore-user-files:/overleaf/services/filestore/user_files + + mongo: + image: mongo:4.4 + ports: + - "127.0.0.1:27017:27017" # for debugging + volumes: + - npm-cache:/root/.npm + - mongo-data:/data/db + + notifications: + build: + context: .. + dockerfile: services/notifications/Dockerfile + env_file: + - dev.env + volumes: + - npm-cache:/root/.npm + + real-time: + build: + context: .. + dockerfile: services/real-time/Dockerfile + env_file: + - dev.env + volumes: + - npm-cache:/root/.npm + + redis: + image: redis:5 + ports: + - "127.0.0.1:6379:6379" # for debugging + volumes: + - redis-data:/data + + spelling: + build: + context: .. + dockerfile: services/spelling/Dockerfile + env_file: + - dev.env + volumes: + - npm-cache:/root/.npm + - spelling-cache:/overleaf/services/spelling/cache + + track-changes: + build: + context: .. + dockerfile: services/track-changes/Dockerfile + env_file: + - dev.env + volumes: + - npm-cache:/root/.npm + + web: + build: + context: .. + dockerfile: services/web/Dockerfile + target: dev + env_file: + - dev.env + environment: + - APP_NAME=Overleaf Community Edition + - ENABLED_LINKED_FILE_TYPES=project_file,project_output_file + - EMAIL_CONFIRMATION_DISABLED=true + - NODE_ENV=development + - SHARELATEX_ALLOW_PUBLIC_ACCESS=true + command: ["node", "app.js"] + volumes: + - npm-cache:/root/.npm + - sharelatex-data:/var/lib/sharelatex + - web-data:/overleaf/services/web/data + depends_on: + - mongo + - redis + - chat + - clsi + - contacts + - docstore + - document-updater + - filestore + - notifications + - real-time + - spelling + + webpack: + build: + context: .. + dockerfile: services/web/Dockerfile + target: dev + command: ["npx", "webpack", "serve", "--config", "webpack.config.dev-env.js"] + ports: + - "127.0.0.1:80:3808" + volumes: + - npm-cache:/root/.npm + - ./webpack.config.dev-env.js:/overleaf/services/web/webpack.config.dev-env.js diff --git a/develop/texlive/Dockerfile b/develop/texlive/Dockerfile new file mode 100644 index 0000000000..944681a25f --- /dev/null +++ b/develop/texlive/Dockerfile @@ -0,0 +1,8 @@ +FROM debian:testing-slim + +RUN apt-get update +RUN apt-cache depends texlive-full | grep "Depends: " | grep -v -- "-doc" | grep -v -- "-lang-" | sed 's/Depends: //' | xargs apt-get install -y --no-install-recommends +RUN apt-get install -y --no-install-recommends fontconfig inkscape pandoc python3-pygments + +RUN useradd tex +USER tex diff --git a/develop/webpack.config.dev-env.js b/develop/webpack.config.dev-env.js new file mode 100644 index 0000000000..49f85408a8 --- /dev/null +++ b/develop/webpack.config.dev-env.js @@ -0,0 +1,23 @@ +const { merge } = require('webpack-merge') + +const base = require('./webpack.config.dev') + +module.exports = merge(base, { + devServer: { + allowedHosts: 'auto', + devMiddleware: { + index: false, + }, + proxy: [ + { + context: '/socket.io/**', + target: 'http://real-time:3026', + ws: true, + }, + { + context: ['!**/*.js', '!**/*.css', '!**/*.json'], + target: 'http://web:3000', + }, + ], + }, +}) diff --git a/patches/README.md b/patches/README.md new file mode 100644 index 0000000000..c158e27699 --- /dev/null +++ b/patches/README.md @@ -0,0 +1 @@ +The patches in this folder are applied by `patch-package` to dependencies, particularly those which need changes that are difficult to apply upstream. diff --git a/patches/ngcomponent+4.1.0.patch b/patches/ngcomponent+4.1.0.patch new file mode 100644 index 0000000000..d8bc68175e --- /dev/null +++ b/patches/ngcomponent+4.1.0.patch @@ -0,0 +1,9 @@ +diff --git a/node_modules/ngcomponent/index.ts b/node_modules/ngcomponent/index.ts +index 5fe33c5..8e1c6fc 100644 +--- a/node_modules/ngcomponent/index.ts ++++ b/node_modules/ngcomponent/index.ts +@@ -1,3 +1,4 @@ ++// @ts-nocheck + import { IChangesObject } from 'angular' + import assign = require('lodash/assign') + import mapValues = require('lodash/mapValues') diff --git a/patches/react2angular+4.0.6.patch b/patches/react2angular+4.0.6.patch new file mode 100644 index 0000000000..afcf627f67 --- /dev/null +++ b/patches/react2angular+4.0.6.patch @@ -0,0 +1,9 @@ +diff --git a/node_modules/react2angular/index.tsx b/node_modules/react2angular/index.tsx +index 5cee831..a07e040 100644 +--- a/node_modules/react2angular/index.tsx ++++ b/node_modules/react2angular/index.tsx +@@ -1,3 +1,4 @@ ++// @ts-nocheck + import { IAugmentedJQuery, IComponentOptions } from 'angular' + import fromPairs = require('lodash.frompairs') + import NgComponent from 'ngcomponent'