merge multiple repositories into an existing monorepo

- merged using: 'monorepo_add.sh libraries-redis-wrapper:libraries/redis-wrapper'
- see https://github.com/shopsys/monorepo-tools

libraries/redis-wrapper/.editorconfig

@@ -0,0 +1,11 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+indent_size = 2
+indent_style = space
+insert_final_newline = true
+max_line_length = 79
+tab_width = 4
+trim_trailing_whitespace = true

libraries/redis-wrapper/.eslintrc.json

@@ -0,0 +1,23 @@
+{
+  "extends": [
+    "standard",
+    "plugin:prettier/recommended",
+    "plugin:mocha/recommended",
+    "plugin:chai-expect/recommended",
+    "plugin:chai-friendly/recommended"
+  ],
+  "env": {
+    "node": true
+  },
+  "parserOptions": {
+    "ecmaVersion": 2018
+  },
+  "overrides": [
+    {
+      "files": ["test/**/*.js"],
+      "env": {
+        "mocha": true
+      }
+    }
+  ]
+}

libraries/redis-wrapper/.gitignore

@@ -0,0 +1,14 @@
+**.swp
+
+app.js
+app/js/
+test/unit/js/
+public/build/
+
+node_modules/
+
+/public/js/chat.js
+plato/
+
+.npmrc
+Dockerfile

libraries/redis-wrapper/.prettierrc.json

@@ -0,0 +1,4 @@
+{
+  "semi": false,
+  "singleQuote": true
+}

libraries/redis-wrapper/Errors.js

@@ -0,0 +1,15 @@
+const OError = require('@overleaf/o-error')
+
+class RedisError extends OError {}
+class RedisHealthCheckFailed extends RedisError {}
+class RedisHealthCheckTimedOut extends RedisHealthCheckFailed {}
+class RedisHealthCheckWriteError extends RedisHealthCheckFailed {}
+class RedisHealthCheckVerifyError extends RedisHealthCheckFailed {}
+
+module.exports = {
+  RedisError,
+  RedisHealthCheckFailed,
+  RedisHealthCheckTimedOut,
+  RedisHealthCheckWriteError,
+  RedisHealthCheckVerifyError,
+}

libraries/redis-wrapper/index.js

@@ -0,0 +1,173 @@
+const crypto = require('crypto')
+const os = require('os')
+const { promisify } = require('util')
+
+const Redis = require('ioredis')
+
+const {
+  RedisHealthCheckTimedOut,
+  RedisHealthCheckWriteError,
+  RedisHealthCheckVerifyError,
+} = require('./Errors')
+
+const HEARTBEAT_TIMEOUT = 2000
+
+// generate unique values for health check
+const HOST = os.hostname()
+const PID = process.pid
+const RND = crypto.randomBytes(4).toString('hex')
+let COUNT = 0
+
+function createClient(opts) {
+  const standardOpts = Object.assign({}, opts)
+  delete standardOpts.key_schema
+
+  if (standardOpts.retry_max_delay == null) {
+    standardOpts.retry_max_delay = 5000 // ms
+  }
+
+  if (standardOpts.endpoints) {
+    throw new Error(
+      '@overleaf/redis-wrapper: redis-sentinel is no longer supported'
+    )
+  }
+
+  let client
+  if (standardOpts.cluster) {
+    delete standardOpts.cluster
+    client = new Redis.Cluster(opts.cluster, standardOpts)
+  } else {
+    client = new Redis(standardOpts)
+  }
+  monkeyPatchIoRedisExec(client)
+  client.healthCheck = (callback) => {
+    if (callback) {
+      // callback based invocation
+      healthCheck(client).then(callback).catch(callback)
+    } else {
+      // Promise based invocation
+      return healthCheck(client)
+    }
+  }
+  return client
+}
+
+async function healthCheck(client) {
+  // check the redis connection by storing and retrieving a unique key/value pair
+  const uniqueToken = `host=${HOST}:pid=${PID}:random=${RND}:time=${Date.now()}:count=${COUNT++}`
+
+  // o-error context
+  const context = {
+    uniqueToken,
+    stage: 'add context for a timeout',
+  }
+
+  await runWithTimeout({
+    runner: runCheck(client, uniqueToken, context),
+    timeout: HEARTBEAT_TIMEOUT,
+    context,
+  })
+}
+
+async function runCheck(client, uniqueToken, context) {
+  const healthCheckKey = `_redis-wrapper:healthCheckKey:{${uniqueToken}}`
+  const healthCheckValue = `_redis-wrapper:healthCheckValue:{${uniqueToken}}`
+
+  // set the unique key/value pair
+  context.stage = 'write'
+  const writeAck = await client
+    .set(healthCheckKey, healthCheckValue, 'EX', 60)
+    .catch((err) => {
+      throw new RedisHealthCheckWriteError('write errored', context, err)
+    })
+  if (writeAck !== 'OK') {
+    context.writeAck = writeAck
+    throw new RedisHealthCheckWriteError('write failed', context)
+  }
+
+  // check that we can retrieve the unique key/value pair
+  context.stage = 'verify'
+  const [roundTrippedHealthCheckValue, deleteAck] = await client
+    .multi()
+    .get(healthCheckKey)
+    .del(healthCheckKey)
+    .exec()
+    .catch((err) => {
+      throw new RedisHealthCheckVerifyError(
+        'read/delete errored',
+        context,
+        err
+      )
+    })
+  if (roundTrippedHealthCheckValue !== healthCheckValue) {
+    context.roundTrippedHealthCheckValue = roundTrippedHealthCheckValue
+    throw new RedisHealthCheckVerifyError('read failed', context)
+  }
+  if (deleteAck !== 1) {
+    context.deleteAck = deleteAck
+    throw new RedisHealthCheckVerifyError('delete failed', context)
+  }
+}
+
+function unwrapMultiResult(result, callback) {
+  // ioredis exec returns a results like:
+  // [ [null, 42], [null, "foo"] ]
+  // where the first entries in each 2-tuple are
+  // presumably errors for each individual command,
+  // and the second entry is the result. We need to transform
+  // this into the same result as the old redis driver:
+  // [ 42, "foo" ]
+  //
+  // Basically reverse:
+  // https://github.com/luin/ioredis/blob/v4.17.3/lib/utils/index.ts#L75-L92
+  const filteredResult = []
+  for (const [err, value] of result || []) {
+    if (err) {
+      return callback(err)
+    } else {
+      filteredResult.push(value)
+    }
+  }
+  callback(null, filteredResult)
+}
+const unwrapMultiResultPromisified = promisify(unwrapMultiResult)
+
+function monkeyPatchIoRedisExec(client) {
+  const _multi = client.multi
+  client.multi = function () {
+    const multi = _multi.apply(client, arguments)
+    const _exec = multi.exec
+    multi.exec = (callback) => {
+      if (callback) {
+        // callback based invocation
+        _exec.call(multi, (error, result) => {
+          // The command can fail all-together due to syntax errors
+          if (error) return callback(error)
+          unwrapMultiResult(result, callback)
+        })
+      } else {
+        // Promise based invocation
+        return _exec.call(multi).then(unwrapMultiResultPromisified)
+      }
+    }
+    return multi
+  }
+}
+
+async function runWithTimeout({ runner, timeout, context }) {
+  let healthCheckDeadline
+  await Promise.race([
+    new Promise((resolve, reject) => {
+      healthCheckDeadline = setTimeout(() => {
+        // attach the timeout when hitting the timeout only
+        context.timeout = timeout
+        reject(new RedisHealthCheckTimedOut('timeout', context))
+      }, timeout)
+    }),
+    runner.finally(() => clearTimeout(healthCheckDeadline)),
+  ])
+}
+
+module.exports = {
+  createClient,
+}

libraries/redis-wrapper/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "@overleaf/redis-wrapper",
+  "version": "2.0.1",
+  "description": "Redis wrapper for node which will either use cluster or single instance redis",
+  "main": "index.js",
+  "files": [
+    "index.js",
+    "Errors.js"
+  ],
+  "author": "Overleaf (https://www.overleaf.com)",
+  "repository": "github:overleaf/redis-wrapper",
+  "license": "ISC",
+  "scripts": {
+    "lint": "eslint --max-warnings 0 .",
+    "format": "prettier-eslint $PWD'/**/*.js' --list-different",
+    "format:fix": "prettier-eslint $PWD'/**/*.js' --write",
+    "test": "mocha --recursive test/unit/src/"
+  },
+  "peerDependencies": {
+    "@overleaf/o-error": "^3.1.0"
+  },
+  "dependencies": {
+    "ioredis": "~4.27.1"
+  },
+  "devDependencies": {
+    "@overleaf/o-error": "^3.3.1",
+    "chai": "^4.3.4",
+    "eslint": "^6.8.0",
+    "eslint-config-prettier": "^6.10.1",
+    "eslint-config-standard": "^14.1.1",
+    "eslint-plugin-chai-expect": "^2.1.0",
+    "eslint-plugin-chai-friendly": "^0.5.0",
+    "eslint-plugin-import": "^2.20.2",
+    "eslint-plugin-mocha": "^6.3.0",
+    "eslint-plugin-node": "^11.1.0",
+    "eslint-plugin-prettier": "^3.4.0",
+    "eslint-plugin-promise": "^4.3.1",
+    "eslint-plugin-standard": "^4.1.0",
+    "logger-sharelatex": "^2.2.0",
+    "mocha": "^8.3.2",
+    "prettier": "^2.2.1",
+    "prettier-eslint-cli": "^5.0.1",
+    "sandboxed-module": "^2.0.4",
+    "sinon": "^9.2.4"
+  }
+}

libraries/redis-wrapper/test/scripts/cluster/clear-dbs.sh

@@ -0,0 +1,4 @@
+while true; do
+  seq 0 8 \
+  | xargs -I% redis-cli -p 700% FLUSHALL > /dev/null
+done

libraries/redis-wrapper/test/scripts/cluster/cluster.js

@@ -0,0 +1,26 @@
+/*
+  execute this script with a redis cluster running to test the health check.
+  starting and stopping shards with this script running is a good test.
+
+  to create a new cluster, use $ ./create-redis-cluster.sh
+  to run a chaos monkey, use $ ./clear-dbs.sh
+*/
+
+const redis = require('../../../')
+const logger = require('logger-sharelatex')
+
+const rclient = redis.createClient({
+  cluster: Array.from({ length: 9 }).map((value, index) => {
+    return { host: '127.0.0.1', port: 7000 + index }
+  }),
+})
+
+setInterval(() => {
+  rclient.healthCheck((err) => {
+    if (err) {
+      logger.error({ err }, 'HEALTH CHECK FAILED')
+    } else {
+      logger.log('HEALTH CHECK OK')
+    }
+  })
+}, 1000)

libraries/redis-wrapper/test/scripts/cluster/create-cluster.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+
+# USAGE: $0 [NUMBER_OF_NODES, default: 9] [DATA_DIR, default: a new temp dir]
+#
+# ports are assigned from 7000 on
+#
+# NOTE: the cluster setup requires redis 5+
+
+set -ex
+
+COUNT=${1:-9}
+DATA=$2
+
+if [[ -z "$DATA" ]]; then
+  IS_TEMP=1
+  TEMP=`mktemp -d`
+  DATA="$TEMP"
+fi
+
+HAS_DATA=
+if [[ -e "$DATA/7000/node.conf" ]]; then
+    HAS_DATA=1
+fi
+
+PIDs=""
+
+cleanup() {
+  # ensure that we delete the temp dir, no matter how the kill cmd exists
+  set +e
+  # invoke kill with at least one PID
+  echo "$PIDs" | xargs -r kill
+  if [[ ! -z "$IS_TEMP" ]]; then
+    rm -rf "$TEMP"
+  fi
+}
+trap cleanup exit
+
+for NUM in `seq "$COUNT"`; do
+  PORT=`expr 6999 + "$NUM"`
+  CWD="$DATA/$PORT"
+  mkdir -p "$CWD"
+  pushd "$CWD"
+  redis-server \
+    --appendonly no \
+    --cluster-enabled yes \
+    --cluster-config-file node.conf \
+    --port "$PORT" \
+    --save "" \
+    > /dev/null \
+  &
+  PIDs="$PIDs $!"
+  popd
+done
+
+# initial nodes
+if [[ -z "$HAS_DATA" ]]; then
+  # confirm the setup
+  echo yes \
+  | redis-cli --cluster create 127.0.0.1:7000 127.0.0.1:7001 127.0.0.1:7002
+fi
+
+# scale up as requested
+for NUM in `seq 4 "$COUNT"`; do
+  PORT=`expr 6999 + "$NUM"`
+  GUARD="$DATA/$PORT/.joined"
+  if [[ ! -e "$GUARD" ]]; then
+    redis-cli --cluster add-node "127.0.0.1:$PORT" 127.0.0.1:7000 --cluster-slave
+    touch "$GUARD"
+  fi
+done
+
+echo "CLUSTER IS READY" >&2
+wait

libraries/redis-wrapper/test/scripts/standalone.js

@@ -0,0 +1,17 @@
+// execute this script with a redis container running to test the health check
+// starting and stopping redis with this script running is a good test
+
+const redis = require('../../')
+const logger = require('logger-sharelatex')
+
+const rclient = redis.createClient({ host: 'localhost', port: '6379' })
+
+setInterval(() => {
+  rclient.healthCheck((err) => {
+    if (err) {
+      logger.error({ err }, 'HEALTH CHECK FAILED')
+    } else {
+      logger.log('HEALTH CHECK OK')
+    }
+  })
+}, 1000)

libraries/redis-wrapper/test/unit/src/test.js

@@ -0,0 +1,169 @@
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS206: Consider reworking classes to avoid initClass
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+const should = require('chai').should()
+const SandboxedModule = require('sandboxed-module')
+const assert = require('assert')
+const path = require('path')
+const sinon = require('sinon')
+const modulePath = path.join(__dirname, './../../../index.js')
+const { expect } = require('chai')
+
+describe('index', function () {
+  beforeEach(function () {
+    let Cluster, IoRedis, ioredisConstructor
+    this.settings = {}
+    this.ioredisConstructor = ioredisConstructor = sinon.stub()
+
+    this.ioredis = IoRedis = (function () {
+      let createIoRedis
+      IoRedis = class IoRedis {
+        static initClass() {
+          this.prototype.on = sinon.stub()
+          createIoRedis = ioredisConstructor
+        }
+
+        constructor() {
+          return createIoRedis.apply(this, arguments)
+        }
+      }
+      IoRedis.initClass()
+      return IoRedis
+    })()
+    this.ioredis.Cluster = Cluster = (function () {
+      Cluster = class Cluster {
+        static initClass() {
+          this.prototype.on = sinon.stub()
+        }
+
+        constructor(config, options) {
+          this.config = config
+          this.options = options
+        }
+      }
+      Cluster.initClass()
+      return Cluster
+    })()
+    this.redis = SandboxedModule.require(modulePath, {
+      requires: {
+        ioredis: this.ioredis,
+      },
+      globals: {
+        process: process,
+      },
+    })
+    return (this.auth_pass = '1234 pass')
+  })
+
+  describe('redis-sentinel', function () {
+    it('should throw an error when creating a client', function () {
+      const redisSentinelOptions = {
+        endpoints: ['127.0.0.1:1234', '127.0.0.1:2345', '127.0.0.1:3456'],
+      }
+      const createNewClient = () => {
+        this.redis.createClient(redisSentinelOptions)
+      }
+      expect(createNewClient).to.throw(
+        '@overleaf/redis-wrapper: redis-sentinel is no longer supported'
+      )
+    })
+  })
+
+  describe('single node redis', function () {
+    beforeEach(function () {
+      return (this.standardOpts = {
+        auth_pass: this.auth_pass,
+        port: 1234,
+        host: 'redis.mysite.env',
+      })
+    })
+
+    it('should work without opts', function () {
+      this.redis.createClient()
+    })
+
+    it('should use the ioredis driver in single-instance mode if a non array is passed', function () {
+      const client = this.redis.createClient(this.standardOpts)
+      return assert.equal(client.constructor, this.ioredis)
+    })
+
+    return it('should call createClient for the ioredis driver in single-instance mode if a non array is passed', function () {
+      const client = this.redis.createClient(this.standardOpts)
+      return this.ioredisConstructor
+        .calledWith(sinon.match(this.standardOpts))
+        .should.equal(true)
+    })
+  })
+
+  describe('cluster', function () {
+    beforeEach(function () {
+      this.cluster = [{ mock: 'cluster' }, { mock: 'cluster2' }]
+      this.extraOptions = { keepAlive: 100 }
+      return (this.settings = {
+        cluster: this.cluster,
+        redisOptions: this.extraOptions,
+        key_schema: {
+          foo(x) {
+            return `${x}`
+          },
+        },
+      })
+    })
+
+    it('should pass the options correctly though with no options', function () {
+      const client = this.redis.createClient({ cluster: this.cluster })
+      assert(client instanceof this.ioredis.Cluster)
+      return client.config.should.deep.equal(this.cluster)
+    })
+
+    it('should not pass the key_schema through to the driver', function () {
+      const client = this.redis.createClient({
+        cluster: this.cluster,
+        key_schema: 'foobar',
+      })
+      assert(client instanceof this.ioredis.Cluster)
+      client.config.should.deep.equal(this.cluster)
+      return expect(client.options).to.deep.equal({ retry_max_delay: 5000 })
+    })
+
+    return it('should pass the options correctly though with additional options', function () {
+      const client = this.redis.createClient(this.settings)
+      assert(client instanceof this.ioredis.Cluster)
+      client.config.should.deep.equal(this.cluster)
+      // need to use expect here because of _.clone in sandbox
+      return expect(client.options).to.deep.equal({
+        redisOptions: this.extraOptions,
+        retry_max_delay: 5000,
+      })
+    })
+  })
+
+  return describe('monkey patch ioredis exec', function () {
+    beforeEach(function () {
+      this.callback = sinon.stub()
+      this.results = []
+      this.multiOrig = { exec: sinon.stub().yields(null, this.results) }
+      this.client = { multi: sinon.stub().returns(this.multiOrig) }
+      this.ioredisConstructor.returns(this.client)
+      this.redis.createClient(this.client)
+      return (this.multi = this.client.multi())
+    })
+
+    it('should return the old redis format for an array', function () {
+      this.results[0] = [null, 42]
+      this.results[1] = [null, 'foo']
+      this.multi.exec(this.callback)
+      return this.callback.calledWith(null, [42, 'foo']).should.equal(true)
+    })
+
+    return it('should return the old redis format when there is an error', function () {
+      this.results[0] = [null, 42]
+      this.results[1] = ['error', 'foo']
+      this.multi.exec(this.callback)
+      return this.callback.calledWith('error').should.equal(true)
+    })
+  })
+})