merge multiple repositories into an existing monorepo

- merged using: 'monorepo_add.sh services-git-bridge:services/git-bridge'
- see https://github.com/shopsys/monorepo-tools

services/git-bridge/.dockerignore

@@ -0,0 +1,9 @@
+*
+!start.sh
+!/conf
+!/lib
+!/src/main
+!/pom.xml
+!/Makefile
+!/LICENSE
+!/vendor

services/git-bridge/.github/dependabot.yml

@@ -0,0 +1,21 @@
+version: 2
+updates:
+  - package-ecosystem: "maven"
+    directory: "/"
+    schedule:
+      interval: "daily"
+
+    pull-request-branch-name:
+      # Separate sections of the branch name with a hyphen
+      # Docker images use the branch name and do not support slashes in tags
+      # https://github.com/overleaf/google-ops/issues/822
+      # https://docs.github.com/en/github/administering-a-repository/configuration-options-for-dependency-updates#pull-request-branch-nameseparator
+      separator: "-"
+
+    # Block informal upgrades -- security upgrades use a separate queue.
+    # https://docs.github.com/en/github/administering-a-repository/configuration-options-for-dependency-updates#open-pull-requests-limit
+    open-pull-requests-limit: 0
+
+    labels:
+      - "dependencies"
+      - "type:maintenance"

services/git-bridge/.gitignore

@@ -0,0 +1,53 @@
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# Let's not share anything because we're using Maven.
+
+.idea
+*.iml
+
+# User-specific stuff:
+.idea/workspace.xml
+.idea/tasks.xml
+.idea/dictionaries
+.idea/vcs.xml
+.idea/jsLibraryMappings.xml
+
+# Sensitive or high-churn files:
+.idea/dataSources.ids
+.idea/dataSources.xml
+.idea/dataSources.local.xml
+.idea/sqlDataSources.xml
+.idea/dynamic.xml
+.idea/uiDesigner.xml
+
+# Gradle:
+.idea/gradle.xml
+.idea/libraries
+
+# Mongo Explorer plugin:
+.idea/mongoSettings.xml
+
+## File-based project format:
+*.iws
+
+## Plugin-specific files:
+
+# IntelliJ
+/out/
+target/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Local configuration files
+conf/runtime.json

services/git-bridge/Dockerfile

@@ -0,0 +1,58 @@
+# Dockerfile for git-bridge
+
+FROM maven:3-jdk-11 as base
+
+RUN apt-get update && apt-get install -y make git sqlite3 \
+ && rm -rf /var/lib/apt/lists
+
+COPY vendor/envsubst /opt/envsubst
+RUN chmod +x /opt/envsubst
+
+RUN useradd --create-home node
+
+FROM base as builder
+
+COPY . /app
+
+WORKDIR /app
+
+RUN make package \
+# The name of the created jar contains the current version tag.
+# Rename it to a static path that can be used for copying.
+&&  find /app/target \
+      -name 'writelatex-git-bridge*jar-with-dependencies.jar' \
+      -exec mv {} /git-bridge.jar \;
+
+FROM openjdk:11-jre
+
+RUN apt-get update && apt-get install -y git sqlite3 procps htop net-tools sockstat libjemalloc2 \
+ && rm -rf /var/lib/apt/lists
+
+ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.2
+
+# Install Google Cloud Profiler agent
+RUN mkdir -p /opt/cprof && \
+  wget -q -O- https://storage.googleapis.com/cloud-profiler/java/latest/profiler_java_agent.tar.gz \
+  | tar xzv -C /opt/cprof
+
+# Install Google Cloud Debugger agent
+RUN mkdir /opt/cdbg && \
+  wget -qO- https://storage.googleapis.com/cloud-debugger/compute-java/debian-wheezy/cdbg_java_agent_gce.tar.gz | \
+  tar xvz -C /opt/cdbg
+
+RUN useradd --create-home node
+
+COPY --from=builder /git-bridge.jar /
+
+COPY vendor/envsubst /opt/envsubst
+RUN chmod +x /opt/envsubst
+
+COPY conf/envsubst_template.json envsubst_template.json
+COPY start.sh start.sh
+
+RUN mkdir conf
+RUN chown node:node conf
+
+USER node
+
+CMD ["/start.sh"]

services/git-bridge/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Winston Li
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

services/git-bridge/Makefile

@@ -0,0 +1,31 @@
+# git-bridge makefile
+
+MVN_OPTS := "--no-transfer-progress"
+
+runtime-conf:
+	/opt/envsubst < conf/envsubst_template.json > conf/runtime.json
+
+
+run: package runtime-conf
+	java $(GIT_BRIDGE_JVM_ARGS) -jar \
+	target/writelatex-git-bridge-1.0-SNAPSHOT-jar-with-dependencies.jar \
+	conf/runtime.json
+
+
+build:
+	mvn $(MVN_OPTS) package -DskipTests
+
+
+test:
+	mvn $(MVN_OPTS) test
+
+
+clean:
+	mvn $(MVN_OPTS) clean
+
+
+package: clean
+	mvn $(MVN_OPTS) package -DskipTests
+
+
+.PHONY: run package build clean test runtime-conf

services/git-bridge/README.md

@@ -0,0 +1,138 @@
+# writelatex-git-bridge
+
+## Docker
+
+The `Dockerfile` contains all the requirements for building and running the
+ writelatex-git-bridge.
+
+```bash
+# build the image
+docker build -t writelatex-git-bridge .
+
+# run it with the demo config
+docker run -v `pwd`/conf/local.json:/conf/runtime.json writelatex-git-bridge
+```
+
+## Native install
+
+### Required packages
+
+  * `maven` (for building, running tests and packaging)
+  * `jdk-8` (for compiling and running)
+
+### Commands
+
+To be run from the base directory:
+
+**Build jar**:
+`mvn package`
+
+**Run tests**:
+`mvn test`
+
+**Clean**:
+`mvn clean`
+
+To be run from the dev-environment:
+
+**Build jar**:
+`bin/run git-bridge make package`
+
+**Run tests**:
+`bin/run git-bridge make test`
+
+**Clean**:
+`bin/run git-bridge make clean`
+
+### Installation
+
+Install dependencies:
+
+```
+sudo apt-get update
+sudo apt-get install -y maven
+sudo apt-get install -y openjdk-8-jdk
+sudo update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java
+sudo update-alternatives --set javac /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/javac
+```
+
+Create a config file according to the format below.
+
+Run `mvn package` to build, test, and package it into a jar at `target/writelatex-git-bridge-1.0-SNAPSHOT-jar-with-dependencies.jar`.
+
+Use `java -jar <path_to_jar> <path_to_config_file>` to run the server.
+
+## Runtime Configuration
+
+The configuration file is in `.json` format.
+
+    {
+        "port" (int): the port number,
+        "rootGitDirectory" (string): the directory in which to store
+                                     git repos and the db/atts,
+        "apiBaseUrl" (string): base url for the snapshot api,
+        "username" (string, optional): username for http basic auth,
+        "password" (string, optional): password for http basic auth,
+        "postbackBaseUrl" (string): the postback url,
+        "serviceName" (string): current name of writeLaTeX
+                                in case it ever changes,
+        "oauth2" (object): { null or missing if oauth2 shouldn't be used
+            "oauth2ClientID" (string): oauth2 client ID,
+            "oauth2ClientSecret" (string): oauth2 client secret,
+            "oauth2Server" (string): oauth2 server,
+                                     with protocol and
+                                     without trailing slash
+        },
+        "repoStore" (object, optional): { configure the repo store
+            "maxFileSize" (long, optional): maximum size of a file, inclusive
+        },
+        "swapStore" (object, optional): { the place to swap projects to.
+                                          if null, type defaults to
+                                          "noop"
+            "type" (string): "s3", "memory", "noop" (not recommended),
+            "awsAccessKey" (string, optional): only for s3,
+            "awsSecret" (string, optional): only for s3,
+            "s3BucketName" (string, optional): only for s3
+        },
+        "swapJob" (object, optional): { configure the project
+                                        swapping job.
+                                        if null, defaults to no-op
+            "minProjects" (int64): lower bound on number of projects
+                                   present. The swap job will never go
+                                   below this, regardless of what the
+                                   watermark shows. Regardless, if
+                                   minProjects prevents an eviction,
+                                   the swap job will WARN,
+            "lowGiB" (int32): the low watermark for swapping,
+                              i.e. swap until disk usage is below this,
+            "highGiB" (int32): the high watermark for swapping,
+                               i.e. start swapping when
+                               disk usage becomes this,
+            "intervalMillis" (int64): amount of time in between running
+                                      swap job and checking watermarks.
+                                      3600000 is 1 hour
+        }
+    }
+
+You have to restart the server for configuration changes to take effect.
+
+
+## Creating OAuth app
+
+In dev-env, run the following command in mongo to create the oauth application
+for git-bridge.
+
+```
+db.oauthApplications.insert({
+  "clientSecret" : "e6b2e9eee7ae2bb653823250bb69594a91db0547fe3790a7135acb497108e62d",
+  "grants" : [
+    "password"
+  ],
+  "id" : "264c723c925c13590880751f861f13084934030c13b4452901e73bdfab226edc",
+  "name" : "Overleaf Git Bridge",
+  "redirectUris" : [],
+  "scopes" : [
+    "git_bridge"
+  ]
+})
+```

services/git-bridge/conf/envsubst_template.json

@@ -0,0 +1,33 @@
+{
+  "port": ${GIT_BRIDGE_PORT:-8000},
+  "bindIp": "${GIT_BRIDGE_BIND_IP:-0.0.0.0}",
+  "idleTimeout": ${GIT_BRIDGE_IDLE_TIMEOUT:-30000},
+  "rootGitDirectory": "${GIT_BRIDGE_ROOT_DIR:-/tmp/wlgb}",
+  "apiBaseUrl": "${GIT_BRIDGE_API_BASE_URL:-https://localhost/api/v0}",
+  "postbackBaseUrl": "${GIT_BRIDGE_POSTBACK_BASE_URL:-https://localhost}",
+  "serviceName": "${GIT_BRIDGE_SERVICE_NAME:-Overleaf}",
+  "oauth2": {
+    "oauth2ClientID": "${GIT_BRIDGE_OAUTH2_CLIENT_ID}",
+    "oauth2ClientSecret": "${GIT_BRIDGE_OAUTH2_CLIENT_SECRET}",
+    "oauth2Server": "${GIT_BRIDGE_OAUTH2_SERVER:-https://localhost}"
+  },
+  "repoStore": {
+    "maxFileNum": ${GIT_BRIDGE_REPOSTORE_MAX_FILE_NUM:-2000},
+    "maxFileSize": ${GIT_BRIDGE_REPOSTORE_MAX_FILE_SIZE:-52428800}
+  },
+  "swapStore": {
+    "type": "${GIT_BRIDGE_SWAPSTORE_TYPE:-noop}",
+    "awsAccessKey": "${GIT_BRIDGE_SWAPSTORE_AWS_ACCESS_KEY}",
+    "awsSecret": "${GIT_BRIDGE_SWAPSTORE_AWS_SECRET}",
+    "s3BucketName": "${GIT_BRIDGE_SWAPSTORE_S3_BUCKET_NAME}",
+    "awsRegion": "${GIT_BRIDGE_SWAPSTORE_AWS_REGION:-us-east-1}"
+  },
+  "swapJob": {
+    "minProjects": ${GIT_BRIDGE_SWAPJOB_MIN_PROJECTS:-50},
+    "lowGiB": ${GIT_BRIDGE_SWAPJOB_LOW_GIB:-128},
+    "highGiB": ${GIT_BRIDGE_SWAPJOB_HIGH_GIB:-256},
+    "intervalMillis": ${GIT_BRIDGE_SWAPJOB_INTERVAL_MILLIS:-3600000},
+    "compressionMethod": "${GIT_BRIDGE_SWAPJOB_COMPRESSION_METHOD:-gzip}"
+  },
+  "sqliteHeapLimitBytes": ${GIT_BRIDGE_SQLITE_HEAP_LIMIT_BYTES:-0}
+}

services/git-bridge/conf/example_config.json

@@ -0,0 +1,33 @@
+{
+    "port": 8080,
+    "bindIp": "127.0.0.1",
+    "idleTimeout": 30000,
+    "rootGitDirectory": "/tmp/wlgb",
+    "apiBaseUrl": "https://localhost/api/v0",
+    "postbackBaseUrl": "https://localhost",
+    "serviceName": "Overleaf",
+    "oauth2": {
+        "oauth2ClientID": "asdf",
+        "oauth2ClientSecret": "asdf",
+        "oauth2Server": "https://localhost"
+    },
+    "repoStore": {
+        "maxFileNum": 2000,
+        "maxFileSize": 52428800
+    },
+    "swapStore": {
+        "type": "s3",
+        "awsAccessKey": "asdf",
+        "awsSecret": "asdf",
+        "s3BucketName": "com.overleaf.testbucket",
+        "awsRegion": "us-east-1"
+    },
+    "swapJob": {
+        "minProjects": 50,
+        "lowGiB": 128,
+        "highGiB": 256,
+        "intervalMillis": 3600000,
+        "compressionMethod": "gzip"
+    },
+    "sqliteHeapLimitBytes": 512000000
+}

services/git-bridge/conf/local.json

@@ -0,0 +1,28 @@
+{
+    "port": 8000,
+    "bindIp": "0.0.0.0",
+    "idleTimeout": 30000,
+    "rootGitDirectory": "/tmp/wlgb",
+    "apiBaseUrl": "http://v2.overleaf.test:4000/api/v0",
+    "postbackBaseUrl": "http://git-bridge:8000",
+    "serviceName": "Overleaf",
+    "oauth2": {
+        "oauth2ClientID": "264c723c925c13590880751f861f13084934030c13b4452901e73bdfab226edc",
+        "oauth2ClientSecret": "e6b2e9eee7ae2bb653823250bb69594a91db0547fe3790a7135acb497108e62d",
+        "oauth2Server": "http://v2.overleaf.test:4000"
+    },
+    "repoStore": {
+        "maxFileNum": 2000,
+        "maxFileSize": 52428800
+    },
+    "swapStore": {
+        "type": "noop"
+    },
+    "swapJob": {
+        "minProjects": 50,
+        "lowGiB": 128,
+        "highGiB": 256,
+        "intervalMillis": 3600000,
+        "compressionMethod": "gzip"
+    }
+}

services/git-bridge/newrelic.yml

@@ -0,0 +1,300 @@
+# This file configures the New Relic Agent.  New Relic monitors
+# Java applications with deep visibility and low overhead.  For more details and additional
+# configuration options visit https://docs.newrelic.com/docs/java/java-agent-configuration.
+#
+# This configuration file is custom generated for Winston
+#
+# This section is for settings common to all environments.
+# Do not add anything above this next line.
+common: &default_settings
+
+  # ============================== LICENSE KEY ===============================
+  # You must specify the license key associated with your New Relic
+  # account. For example, if your license key is 12345 use this:
+  # license_key: '12345'
+  # The key binds your Agent's data to your account in the New Relic service.
+  license_key: '<LICENSE KEY>'
+
+  # Agent Enabled
+  # Use this setting to disable the agent instead of removing it from the startup command.
+  # Default is true.
+  agent_enabled: true
+
+  # Set the name of your application as you'd like it show up in New Relic.
+  # If enable_auto_app_naming is false, the agent reports all data to this application.
+  # Otherwise, the agent reports only background tasks (transactions for non-web applications)
+  # to this application. To report data to more than one application 
+  # (useful for rollup reporting), separate the application names with ";".
+  # For example, to report data to "My Application" and "My Application 2" use this:
+  # app_name: My Application;My Application 2
+  # This setting is required. Up to 3 different application names can be specified.
+  # The first application name must be unique.
+  app_name: Git Bridge
+
+  # To enable high security, set this property to true. When in high
+  # security mode, the agent will use SSL and obfuscated SQL. Additionally,
+  # request parameters and message parameters will not be sent to New Relic.
+  high_security: false
+
+  # Set to true to enable support for auto app naming.
+  # The name of each web app is detected automatically
+  # and the agent reports data separately for each one.
+  # This provides a finer-grained performance breakdown for
+  # web apps in New Relic.
+  # Default is false.
+  enable_auto_app_naming: false
+
+  # Set to true to enable component-based transaction naming.
+  # Set to false to use the URI of a web request as the name of the transaction.
+  # Default is true.
+  enable_auto_transaction_naming: true
+
+  # The agent uses its own log file to keep its logging
+  # separate from that of your application.  Specify the log level here.
+  # This setting is dynamic, so changes do not require restarting your application.
+  # The levels in increasing order of verboseness are:
+  #   off, severe, warning, info, fine, finer, finest
+  # Default is info.
+  log_level: info
+
+  # Log all data sent to and from New Relic in plain text.
+  # This setting is dynamic, so changes do not require restarting your application.
+  # Default is false.
+  audit_mode: false
+
+  # The number of backup log files to save.
+  # Default is 1.
+  log_file_count: 1
+
+  # The maximum number of kbytes to write to any one log file.
+  # The log_file_count must be set greater than 1.
+  # Default is 0 (no limit).
+  log_limit_in_kbytes: 0
+
+  # Override other log rolling configuration and roll the logs daily.
+  # Default is false.
+  log_daily: false
+
+  # The name of the log file.
+  # Default is newrelic_agent.log.
+  log_file_name: newrelic_agent.log
+
+  # The log file directory.
+  # Default is the logs directory in the newrelic.jar parent directory.
+  #log_file_path:
+
+  # The agent communicates with New Relic via https by
+  # default.  If you want to communicate with newrelic via http,
+  # then turn off SSL by setting this value to false.
+  # This work is done asynchronously to the threads that process your
+  # application code, so response times will not be directly affected
+  # by this change.
+  # Default is true.
+  ssl: true
+
+  # Proxy settings for connecting to the New Relic server:
+  # If a proxy is used, the host setting is required.  Other settings
+  # are optional.  Default port is 8080.  The username and password
+  # settings will be used to authenticate to Basic Auth challenges
+  # from a proxy server.
+  #proxy_host: hostname
+  #proxy_port: 8080
+  #proxy_user: username
+  #proxy_password: password
+
+  # Limits the number of lines to capture for each stack trace. 
+  # Default is 30
+  max_stack_trace_lines: 30
+
+  # Provides the ability to configure the attributes sent to New Relic. These
+  # attributes can be found in transaction traces, traced errors, Insight's 
+  # transaction events, and Insight's page views.
+  attributes:
+  
+    # When true, attributes will be sent to New Relic. The default is true.
+    enabled: true
+    
+    #A comma separated list of attribute keys whose values should 
+    # be sent to New Relic.
+    #include:
+    
+    # A comma separated list of attribute keys whose values should 
+    # not be sent to New Relic.
+    #exclude:
+    
+
+  # Transaction tracer captures deep information about slow
+  # transactions and sends this to the New Relic service once a
+  # minute. Included in the transaction is the exact call sequence of
+  # the transactions including any SQL statements issued.
+  transaction_tracer:
+
+    # Transaction tracer is enabled by default. Set this to false to turn it off.
+    # This feature is not available to Lite accounts and is automatically disabled.
+    # Default is true.
+    enabled: true
+
+    # Threshold in seconds for when to collect a transaction
+    # trace. When the response time of a controller action exceeds
+    # this threshold, a transaction trace will be recorded and sent to
+    # New Relic. Valid values are any float value, or (default) "apdex_f",
+    # which will use the threshold for the "Frustrated" Apdex level
+    # (greater than four times the apdex_t value).
+    # Default is apdex_f.
+    transaction_threshold: apdex_f
+
+    # When transaction tracer is on, SQL statements can optionally be
+    # recorded. The recorder has three modes, "off" which sends no
+    # SQL, "raw" which sends the SQL statement in its original form,
+    # and "obfuscated", which strips out numeric and string literals.
+    # Default is obfuscated.
+    record_sql: obfuscated
+
+    # Set this to true to log SQL statements instead of recording them.
+    # SQL is logged using the record_sql mode.
+    # Default is false.
+    log_sql: false
+
+    # Threshold in seconds for when to collect stack trace for a SQL
+    # call. In other words, when SQL statements exceed this threshold,
+    # then capture and send to New Relic the current stack trace. This is
+    # helpful for pinpointing where long SQL calls originate from.
+    # Default is 0.5 seconds.
+    stack_trace_threshold: 0.5
+
+    # Determines whether the agent will capture query plans for slow
+    # SQL queries. Only supported for MySQL and PostgreSQL.
+    # Default is true.
+    explain_enabled: true
+
+    # Threshold for query execution time below which query plans will not 
+    # not be captured.  Relevant only when `explain_enabled` is true.
+    # Default is 0.5 seconds.
+    explain_threshold: 0.5
+
+    # Use this setting to control the variety of transaction traces.
+    # The higher the setting, the greater the variety.
+    # Set this to 0 to always report the slowest transaction trace.
+    # Default is 20.
+    top_n: 20
+
+  # Error collector captures information about uncaught exceptions and
+  # sends them to New Relic for viewing.
+  error_collector:
+
+    # This property enables the collection of errors. If the property is not
+    # set or the property is set to false, then errors will not be collected.
+    # Default is true.
+    enabled: true
+
+    # Use this property to exclude specific exceptions from being reported as errors
+    # by providing a comma separated list of full class names.
+    # The default is to exclude akka.actor.ActorKilledException. If you want to override
+    # this, you must provide any new value as an empty list is ignored.
+    ignore_errors: akka.actor.ActorKilledException
+
+    # Use this property to exclude specific http status codes from being reported as errors
+    # by providing a comma separated list of status codes.
+    # The default is to exclude 404s. If you want to override
+    # this, you must provide any new value as an empty list is ignored.
+    ignore_status_codes: 404
+
+  # Transaction Events are used for Histograms and Percentiles. Unaggregated data is collected
+  # for each web transaction and sent to the server on harvest. 
+  transaction_events:
+
+    # Set to false to disable transaction events.
+    # Default is true.
+    enabled: true
+
+    # Events are collected up to the configured amount. Afterwards, events are sampled to
+    # maintain an even distribution across the harvest cycle.
+    # Default is 2000.  Setting to 0 will disable.
+    max_samples_stored: 2000
+
+  # Cross Application Tracing adds request and response headers to
+  # external calls using supported HTTP libraries to provide better
+  # performance data when calling applications monitored by other New Relic Agents.
+  cross_application_tracer:
+
+    # Set to false to disable cross application tracing.
+    # Default is true.
+    enabled: true
+
+  # Thread profiler measures wall clock time, CPU time, and method call counts
+  # in your application's threads as they run.
+  # This feature is not available to Lite accounts and is automatically disabled.
+  thread_profiler:
+
+    # Set to false to disable the thread profiler.
+    # Default is true.
+    enabled: true
+
+  # New Relic Real User Monitoring gives you insight into the performance real users are
+  # experiencing with your website. This is accomplished by measuring the time it takes for
+  # your users' browsers to download and render your web pages by injecting a small amount
+  # of JavaScript code into the header and footer of each page. 
+  browser_monitoring:
+
+    # By default the agent automatically inserts API calls in compiled JSPs to
+    # inject the monitoring JavaScript into web pages. Not all rendering engines are supported.
+    # See https://docs.newrelic.com/docs/java/real-user-monitoring-in-java#manual_instrumentation
+    # for instructions to add these manually to your pages.
+    # Set this attribute to false to turn off this behavior.
+    auto_instrument: true
+
+  class_transformer:
+    # This instrumentation reports the name of the user principal returned from 
+    # HttpServletRequest.getUserPrincipal() when servlets and filters are invoked.
+    com.newrelic.instrumentation.servlet-user:
+      enabled: false
+
+    com.newrelic.instrumentation.spring-aop-2:
+      enabled: false
+
+    # Classes loaded by classloaders in this list will not be instrumented.
+    # This is a useful optimization for runtimes which use classloaders to
+    # load dynamic classes which the agent would not instrument.
+    classloader_excludes:
+      groovy.lang.GroovyClassLoader$InnerLoader,
+      org.codehaus.groovy.runtime.callsite.CallSiteClassLoader,
+      com.collaxa.cube.engine.deployment.BPELClassLoader,
+      org.springframework.data.convert.ClassGeneratingEntityInstantiator$ObjectInstantiatorClassGenerator,
+      org.mvel2.optimizers.impl.asm.ASMAccessorOptimizer$ContextClassLoader,
+      gw.internal.gosu.compiler.SingleServingGosuClassLoader,
+
+  # User-configurable custom labels for this agent.  Labels are name-value pairs.
+  # There is a maximum of 64 labels per agent.  Names and values are limited to 255 characters.
+  # Names and values may not contain colons (:) or semicolons (;).
+  labels:
+
+    # An example label
+    #label_name: label_value
+
+
+# Application Environments
+# ------------------------------------------
+# Environment specific settings are in this section.
+# You can use the environment to override the default settings.
+# For example, to change the app_name setting.
+# Use -Dnewrelic.environment=<environment> on the Java startup command line
+# to set the environment.
+# The default environment is production.
+
+# NOTE if your application has other named environments, you should
+# provide configuration settings for these environments here.
+
+development:
+  <<: *default_settings
+  app_name: Git Bridge (Development)
+
+test:
+  <<: *default_settings
+  app_name: Git Bridge (Test)
+
+production:
+  <<: *default_settings
+
+staging:
+  <<: *default_settings
+  app_name: Git Bridge (Staging)

services/git-bridge/pom.xml

@@ -0,0 +1,229 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>uk.ac.ic.wlgitbridge</groupId>
+    <artifactId>writelatex-git-bridge</artifactId>
+    <version>1.0-SNAPSHOT</version>
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <!--<maven.test.skip>true</maven.test.skip>-->
+    </properties>
+    <build>
+        <plugins>
+            <!-- https://mvnrepository.com/artifact/org.apache.maven.plugins/maven-compiler-plugin -->
+            <plugin>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.7.0</version>
+                <configuration>
+                    <source>1.8</source>
+                    <target>1.8</target>
+                    <compilerArgument></compilerArgument>
+                </configuration>
+            </plugin>
+            <!-- Workaround, test loader crashes without this configuration option -->
+            <!-- See: https://stackoverflow.com/questions/53010200/maven-surefire-could-not-find-forkedbooter-class -->
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <configuration>
+                  <argLine>-Djdk.net.URLClassPath.disableClassPathURLCheck=true</argLine>
+                </configuration>
+            </plugin>
+            <!-- https://mvnrepository.com/artifact/org.apache.maven.plugins/maven-assembly-plugin -->
+            <plugin>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>3.1.0</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <archive>
+                        <manifest>
+                            <mainClass>uk.ac.ic.wlgitbridge.Main</mainClass>
+                        </manifest>
+                    </archive>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+    <dependencies>
+        <!-- https://mvnrepository.com/artifact/junit/junit -->
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.13.2</version>
+            <scope>test</scope>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.jmock/jmock-junit4 -->
+        <dependency>
+            <groupId>org.jmock</groupId>
+            <artifactId>jmock-junit4</artifactId>
+            <version>2.8.4</version>
+            <scope>test</scope>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.eclipse.jetty/jetty-servlet -->
+        <dependency>
+            <groupId>org.eclipse.jetty</groupId>
+            <artifactId>jetty-servlet</artifactId>
+            <version>9.4.38.v20210224</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.eclipse.jetty/jetty-server -->
+        <dependency>
+            <groupId>org.eclipse.jetty</groupId>
+            <artifactId>jetty-server</artifactId>
+            <version>9.4.38.v20210224</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/com.google.code.gson/gson -->
+        <dependency>
+            <groupId>com.google.code.gson</groupId>
+            <artifactId>gson</artifactId>
+            <version>2.8.2</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.asynchttpclient/async-http-client -->
+        <dependency>
+            <groupId>org.asynchttpclient</groupId>
+            <artifactId>async-http-client</artifactId>
+            <version>2.3.0</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.eclipse.jgit/org.eclipse.jgit -->
+        <dependency>
+            <groupId>org.eclipse.jgit</groupId>
+            <artifactId>org.eclipse.jgit</artifactId>
+            <version>5.12.0.202106070339-r</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.eclipse.jgit/org.eclipse.jgit.http.server -->
+        <dependency>
+            <groupId>org.eclipse.jgit</groupId>
+            <artifactId>org.eclipse.jgit.http.server</artifactId>
+            <version>5.12.0.202106070339-r</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.xerial/sqlite-jdbc -->
+        <dependency>
+            <groupId>org.xerial</groupId>
+            <artifactId>sqlite-jdbc</artifactId>
+            <version>3.36.0.1</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/joda-time/joda-time -->
+        <dependency>
+            <groupId>joda-time</groupId>
+            <artifactId>joda-time</artifactId>
+            <version>2.9.9</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/com.google.oauth-client/google-oauth-client -->
+        <dependency>
+            <groupId>com.google.oauth-client</groupId>
+            <artifactId>google-oauth-client</artifactId>
+            <version>1.23.0</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/com.google.http-client/google-http-client -->
+        <dependency>
+            <groupId>com.google.http-client</groupId>
+            <artifactId>google-http-client</artifactId>
+            <version>1.23.0</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/com.google.http-client/google-http-client-gson -->
+        <dependency>
+            <groupId>com.google.http-client</groupId>
+            <artifactId>google-http-client-gson</artifactId>
+            <version>1.23.0</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+            <version>3.12.0</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/ch.qos.logback/logback-classic -->
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-classic</artifactId>
+            <version>1.2.3</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/com.google.guava/guava -->
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>30.1.1-jre</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.mock-server/mockserver-netty -->
+        <dependency>
+            <groupId>org.mock-server</groupId>
+            <artifactId>mockserver-netty</artifactId>
+            <version>5.3.0</version>
+            <scope>test</scope>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.mockito/mockito-core -->
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <version>3.11.1</version>
+            <scope>test</scope>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk -->
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk</artifactId>
+            <version>1.11.274</version>
+        </dependency>
+        <!-- API, java.xml.bind module -->
+        <dependency>
+            <groupId>jakarta.xml.bind</groupId>
+            <artifactId>jakarta.xml.bind-api</artifactId>
+            <version>2.3.2</version>
+        </dependency>
+
+        <!-- Runtime, com.sun.xml.bind module -->
+        <dependency>
+            <groupId>org.glassfish.jaxb</groupId>
+            <artifactId>jaxb-runtime</artifactId>
+            <version>2.3.2</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient -->
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+            <version>4.5.5</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/commons-io/commons-io -->
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+            <version>2.10.0</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-compress -->
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-compress</artifactId>
+            <version>1.20</version>
+        </dependency>
+        <!-- prometheus metrics -->
+        <dependency>
+            <groupId>io.prometheus</groupId>
+            <artifactId>simpleclient</artifactId>
+            <version>0.10.0</version>
+        </dependency>
+        <!-- Hotspot JVM metrics -->
+        <dependency>
+            <groupId>io.prometheus</groupId>
+            <artifactId>simpleclient_hotspot</artifactId>
+            <version>0.10.0</version>
+        </dependency>
+        <!-- Expose metrics via a servlet -->
+        <dependency>
+            <groupId>io.prometheus</groupId>
+            <artifactId>simpleclient_servlet</artifactId>
+            <version>0.10.0</version>
+        </dependency>
+    </dependencies>
+</project>

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/Main.java

@@ -0,0 +1,49 @@
+package uk.ac.ic.wlgitbridge;
+
+import uk.ac.ic.wlgitbridge.application.GitBridgeApp;
+import uk.ac.ic.wlgitbridge.bridge.Bridge;
+import uk.ac.ic.wlgitbridge.server.GitBridgeServer;
+import uk.ac.ic.wlgitbridge.util.Log;
+
+import java.util.Arrays;
+
+/**
+ * Created by Winston on 01/11/14.
+ */
+
+/**
+ * This is the entry point into the Git Bridge.
+ *
+ * It is responsible for creating the {@link GitBridgeApp} and then running it.
+ *
+ * The {@link GitBridgeApp} parses args and creates the {@link GitBridgeServer}.
+ *
+ * The {@link GitBridgeServer} creates the {@link Bridge}, among other things.
+ *
+ * The {@link Bridge} is the heart of the Git Bridge. Start there, and follow
+ * the links outwards (which lead back to the Git users and the postback from
+ * the snapshot API) and inwards (which lead into the components of the Git
+ * Bridge: the configurable repo store, db store, and swap store, along with
+ * the project lock, the swap job, the snapshot API, the resource cache
+ * and the postback manager).
+ */
+public class Main {
+
+    public static void main(String[] args) {
+        Log.info(
+                "Git Bridge started with args: "
+                        + Arrays.toString(args)
+        );
+        try {
+            new GitBridgeApp(args).run();
+        } catch (Throwable t) {
+            /* So that we get a timestamp */
+            Log.error(
+                    "Fatal exception thrown to top level, exiting: ",
+                    t
+            );
+            System.exit(1);
+        }
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/GitBridgeApp.java

@@ -0,0 +1,102 @@
+package uk.ac.ic.wlgitbridge.application;
+
+import uk.ac.ic.wlgitbridge.application.config.Config;
+import uk.ac.ic.wlgitbridge.application.exception.ArgsException;
+import uk.ac.ic.wlgitbridge.application.exception.ConfigFileException;
+import uk.ac.ic.wlgitbridge.server.GitBridgeServer;
+import uk.ac.ic.wlgitbridge.util.Log;
+
+import javax.servlet.ServletException;
+import java.io.IOException;
+
+/**
+ * Created by Winston on 02/11/14.
+ */
+
+/**
+ * Class that represents the application. Parses arguments and gives them to the
+ * server, or dies with a usage message.
+ */
+public class GitBridgeApp implements Runnable {
+
+    public static final int EXIT_CODE_FAILED = 1;
+    private static final String USAGE_MESSAGE =
+            "usage: writelatex-git-bridge [config_file]";
+
+    private String configFilePath;
+    Config config;
+    private GitBridgeServer server;
+
+    /**
+     * Constructs an instance of the WriteLatex-Git Bridge application.
+     * @param args args from main, which should be in the format [config_file]
+     */
+    public GitBridgeApp(String[] args) {
+        try {
+            parseArguments(args);
+            loadConfigFile();
+            Log.info("Config loaded: {}", config.getSanitisedString());
+        } catch (ArgsException e) {
+            printUsage();
+            System.exit(EXIT_CODE_FAILED);
+        } catch (ConfigFileException e) {
+            Log.error(
+                    "The property for " +
+                    e.getMissingMember() +
+                    " is invalid. Check your config file."
+            );
+            System.exit(EXIT_CODE_FAILED);
+        } catch (IOException e) {
+            Log.error("Invalid config file. Check the file path.");
+            System.exit(EXIT_CODE_FAILED);
+        }
+        try {
+            server = new GitBridgeServer(config);
+        } catch (ServletException e) {
+            Log.error(
+                    "Servlet exception when instantiating GitBridgeServer",
+                    e
+            );
+        }
+    }
+
+    /**
+     * Starts the server with the port number and root directory path given in
+     * the command-line arguments.
+     */
+    @Override
+    public void run() {
+        server.start();
+    }
+
+    public void stop() {
+        server.stop();
+    }
+
+    /* Helper methods */
+
+    private void parseArguments(String[] args) throws ArgsException {
+        checkArgumentsLength(args);
+        parseConfigFilePath(args);
+    }
+
+    private void checkArgumentsLength(String[] args) throws ArgsException {
+        if (args.length < 1) {
+            throw new ArgsException();
+        }
+    }
+
+    private void parseConfigFilePath(String[] args) throws ArgsException {
+        configFilePath = args[0];
+    }
+
+    private void loadConfigFile() throws ConfigFileException, IOException {
+        Log.info("Loading config file at path: " + configFilePath);
+        config = new Config(configFilePath);
+    }
+
+    private void printUsage() {
+        System.err.println(USAGE_MESSAGE);
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java

@@ -0,0 +1,211 @@
+package uk.ac.ic.wlgitbridge.application.config;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import uk.ac.ic.wlgitbridge.application.exception.ConfigFileException;
+import uk.ac.ic.wlgitbridge.bridge.repo.RepoStoreConfig;
+import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJobConfig;
+import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStoreConfig;
+import uk.ac.ic.wlgitbridge.snapshot.base.JSONSource;
+import uk.ac.ic.wlgitbridge.util.Instance;
+
+import javax.annotation.Nullable;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.Reader;
+import java.util.Optional;
+
+/**
+ * Created by Winston on 05/12/14.
+ */
+public class Config implements JSONSource {
+
+    static Config asSanitised(Config config) {
+        return new Config(
+                config.port,
+                config.bindIp,
+                config.idleTimeout,
+                config.rootGitDirectory,
+                config.apiBaseURL,
+                config.postbackURL,
+                config.serviceName,
+                Oauth2.asSanitised(config.oauth2),
+                config.repoStore,
+                SwapStoreConfig.sanitisedCopy(config.swapStore),
+                config.swapJob,
+                config.sqliteHeapLimitBytes
+        );
+    }
+
+    private int port;
+    private String bindIp;
+    private int idleTimeout;
+    private String rootGitDirectory;
+    private String apiBaseURL;
+    private String postbackURL;
+    private String serviceName;
+    @Nullable
+    private Oauth2 oauth2;
+    @Nullable
+    private RepoStoreConfig repoStore;
+    @Nullable
+    private SwapStoreConfig swapStore;
+    @Nullable
+    private SwapJobConfig swapJob;
+    private int sqliteHeapLimitBytes = 0;
+
+    public Config(
+            String configFilePath
+    ) throws ConfigFileException,
+             IOException {
+        this(new FileReader(configFilePath));
+    }
+
+    Config(Reader reader) {
+        fromJSON(new Gson().fromJson(reader, JsonElement.class));
+    }
+
+    public Config(
+            int port,
+            String bindIp,
+            int idleTimeout,
+            String rootGitDirectory,
+            String apiBaseURL,
+            String postbackURL,
+            String serviceName,
+            Oauth2 oauth2,
+            RepoStoreConfig repoStore,
+            SwapStoreConfig swapStore,
+            SwapJobConfig swapJob,
+            int sqliteHeapLimitBytes
+    ) {
+        this.port = port;
+        this.bindIp = bindIp;
+        this.idleTimeout = idleTimeout;
+        this.rootGitDirectory = rootGitDirectory;
+        this.apiBaseURL = apiBaseURL;
+        this.postbackURL = postbackURL;
+        this.serviceName = serviceName;
+        this.oauth2 = oauth2;
+        this.repoStore = repoStore;
+        this.swapStore = swapStore;
+        this.swapJob = swapJob;
+        this.sqliteHeapLimitBytes = sqliteHeapLimitBytes;
+    }
+
+    @Override
+    public void fromJSON(JsonElement json) {
+        JsonObject configObject = json.getAsJsonObject();
+        port = getElement(configObject, "port").getAsInt();
+        bindIp = getElement(configObject, "bindIp").getAsString();
+        idleTimeout = getElement(configObject, "idleTimeout").getAsInt();
+        rootGitDirectory = getElement(
+                configObject,
+                "rootGitDirectory"
+        ).getAsString();
+        String apiBaseURL = getElement(
+                configObject,
+                "apiBaseUrl"
+        ).getAsString();
+        if (!apiBaseURL.endsWith("/")) {
+            apiBaseURL += "/";
+        }
+        this.apiBaseURL = apiBaseURL;
+        serviceName = getElement(configObject, "serviceName").getAsString();
+        postbackURL = getElement(configObject, "postbackBaseUrl").getAsString();
+        if (!postbackURL.endsWith("/")) {
+            postbackURL += "/";
+        }
+        oauth2 = new Gson().fromJson(configObject.get("oauth2"), Oauth2.class);
+        repoStore = new Gson().fromJson(
+                configObject.get("repoStore"), RepoStoreConfig.class);
+        swapStore = new Gson().fromJson(
+                configObject.get("swapStore"),
+                SwapStoreConfig.class
+        );
+        swapJob = new Gson().fromJson(
+                configObject.get("swapJob"),
+                SwapJobConfig.class
+        );
+        if (configObject.has("sqliteHeapLimitBytes")) {
+            sqliteHeapLimitBytes = getElement(configObject, "sqliteHeapLimitBytes").getAsInt();
+        }
+    }
+
+    public String getSanitisedString() {
+        return Instance.prettyGson.toJson(Config.asSanitised(this));
+    }
+
+    public int getPort() {
+        return port;
+    }
+
+    public String getBindIp() {
+        return bindIp;
+    }
+
+    public int getIdleTimeout() {
+        return idleTimeout;
+    }
+
+    public String getRootGitDirectory() {
+        return rootGitDirectory;
+    }
+
+    public int getSqliteHeapLimitBytes() {
+        return this.sqliteHeapLimitBytes;
+    }
+
+    public String getAPIBaseURL() {
+        return apiBaseURL;
+    }
+
+    public String getServiceName() {
+        return serviceName;
+    }
+
+    public String getPostbackURL() {
+        return postbackURL;
+    }
+
+    public boolean isUsingOauth2() {
+        return oauth2 != null;
+    }
+
+    public Oauth2 getOauth2() {
+        if (!isUsingOauth2()) {
+            throw new AssertionError("Getting oauth2 when not using it");
+        }
+        return oauth2;
+    }
+
+    public Optional<RepoStoreConfig> getRepoStore() {
+        return Optional.ofNullable(repoStore);
+    }
+
+    public Optional<SwapStoreConfig> getSwapStore() {
+        return Optional.ofNullable(swapStore);
+    }
+
+    public Optional<SwapJobConfig> getSwapJob() {
+        return Optional.ofNullable(swapJob);
+    }
+
+    private JsonElement getElement(JsonObject configObject, String name) {
+        JsonElement element = configObject.get(name);
+        if (element == null) {
+            throw new RuntimeException(new ConfigFileException(name));
+        }
+        return element;
+    }
+
+    private String getOptionalString(JsonObject configObject, String name) {
+        JsonElement element = configObject.get(name);
+        if (element == null || !element.isJsonPrimitive()) {
+            return "";
+        }
+        return element.getAsString();
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Oauth2.java

@@ -0,0 +1,42 @@
+package uk.ac.ic.wlgitbridge.application.config;
+
+/**
+ * Created by winston on 25/10/15.
+ */
+public class Oauth2 {
+
+    private final String oauth2ClientID;
+    private final String oauth2ClientSecret;
+    private final String oauth2Server;
+
+    public Oauth2(
+            String oauth2ClientID,
+            String oauth2ClientSecret,
+            String oauth2Server
+    ) {
+        this.oauth2ClientID = oauth2ClientID;
+        this.oauth2ClientSecret = oauth2ClientSecret;
+        this.oauth2Server = oauth2Server;
+    }
+
+    public String getOauth2ClientID() {
+        return oauth2ClientID;
+    }
+
+    public String getOauth2ClientSecret() {
+        return oauth2ClientSecret;
+    }
+
+    public String getOauth2Server() {
+        return oauth2Server;
+    }
+
+    public static Oauth2 asSanitised(Oauth2 oauth2) {
+        return new Oauth2(
+                "<oauth2ClientID>",
+                "<oauth2ClientSecret>",
+                oauth2.oauth2Server
+        );
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/exception/ArgsException.java

@@ -0,0 +1,6 @@
+package uk.ac.ic.wlgitbridge.application.exception;
+
+/**
+ * Created by Winston on 03/11/14.
+ */
+public class ArgsException extends Exception {}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/exception/ConfigFileException.java

@@ -0,0 +1,18 @@
+package uk.ac.ic.wlgitbridge.application.exception;
+
+/**
+ * Created by Winston on 05/12/14.
+ */
+public class ConfigFileException extends Exception {
+
+    private final String missingMember;
+
+    public ConfigFileException(String missingMember) {
+        this.missingMember = missingMember;
+    }
+
+    public String getMissingMember() {
+        return missingMember;
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/jetty/NullLogger.java

@@ -0,0 +1,85 @@
+package uk.ac.ic.wlgitbridge.application.jetty;
+
+import org.eclipse.jetty.util.log.Logger;
+
+/**
+ * Created by Winston on 03/11/14.
+ */
+public class NullLogger implements Logger {
+
+    @Override
+    public String getName() {
+        return "null_logger";
+    }
+
+    @Override
+    public void warn(String s, Object... objects) {
+
+    }
+
+    @Override
+    public void warn(Throwable throwable) {
+
+    }
+
+    @Override
+    public void warn(String s, Throwable throwable) {
+
+    }
+
+    @Override
+    public void info(String s, Object... objects) {
+
+    }
+
+    @Override
+    public void info(Throwable throwable) {
+
+    }
+
+    @Override
+    public void info(String s, Throwable throwable) {
+
+    }
+
+    @Override
+    public boolean isDebugEnabled() {
+        return false;
+    }
+
+    @Override
+    public void setDebugEnabled(boolean b) {
+
+    }
+
+    @Override
+    public void debug(String s, Object... objects) {
+
+    }
+
+    @Override
+    public void debug(String s, long l) {
+
+    }
+
+    @Override
+    public void debug(Throwable throwable) {
+
+    }
+
+    @Override
+    public void debug(String s, Throwable throwable) {
+
+    }
+
+    @Override
+    public Logger getLogger(String s) {
+        return this;
+    }
+
+    @Override
+    public void ignore(Throwable throwable) {
+
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java

@@ -0,0 +1,839 @@
+package uk.ac.ic.wlgitbridge.bridge;
+
+import com.google.api.client.auth.oauth2.Credential;
+import org.eclipse.jgit.errors.RepositoryNotFoundException;
+import uk.ac.ic.wlgitbridge.application.config.Config;
+import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
+import uk.ac.ic.wlgitbridge.bridge.db.ProjectState;
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SqliteDBStore;
+import uk.ac.ic.wlgitbridge.bridge.gc.GcJob;
+import uk.ac.ic.wlgitbridge.bridge.gc.GcJobImpl;
+import uk.ac.ic.wlgitbridge.bridge.lock.LockGuard;
+import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
+import uk.ac.ic.wlgitbridge.bridge.repo.*;
+import uk.ac.ic.wlgitbridge.bridge.resource.ResourceCache;
+import uk.ac.ic.wlgitbridge.bridge.resource.UrlResourceCache;
+import uk.ac.ic.wlgitbridge.bridge.snapshot.NetSnapshotApi;
+import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApi;
+import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApiFacade;
+import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJob;
+import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJobImpl;
+import uk.ac.ic.wlgitbridge.bridge.swap.store.S3SwapStore;
+import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore;
+import uk.ac.ic.wlgitbridge.data.CandidateSnapshot;
+import uk.ac.ic.wlgitbridge.data.ProjectLockImpl;
+import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
+import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
+import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
+import uk.ac.ic.wlgitbridge.data.model.Snapshot;
+import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
+import uk.ac.ic.wlgitbridge.git.exception.SizeLimitExceededException;
+import uk.ac.ic.wlgitbridge.git.exception.FileLimitExceededException;
+import uk.ac.ic.wlgitbridge.git.handler.WLReceivePackFactory;
+import uk.ac.ic.wlgitbridge.git.handler.WLRepositoryResolver;
+import uk.ac.ic.wlgitbridge.git.handler.WLUploadPackFactory;
+import uk.ac.ic.wlgitbridge.git.handler.hook.WriteLatexPutHook;
+import uk.ac.ic.wlgitbridge.server.FileHandler;
+import uk.ac.ic.wlgitbridge.server.PostbackContents;
+import uk.ac.ic.wlgitbridge.server.PostbackHandler;
+import uk.ac.ic.wlgitbridge.snapshot.base.MissingRepositoryException;
+import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
+import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocResult;
+import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotAttachment;
+import uk.ac.ic.wlgitbridge.snapshot.push.PostbackManager;
+import uk.ac.ic.wlgitbridge.snapshot.push.PostbackPromise;
+import uk.ac.ic.wlgitbridge.snapshot.push.PushResult;
+import uk.ac.ic.wlgitbridge.snapshot.push.exception.*;
+import uk.ac.ic.wlgitbridge.util.Log;
+
+import java.io.File;
+import java.io.IOException;
+import java.sql.Timestamp;
+import java.time.LocalDateTime;
+import java.util.*;
+
+/**
+ * This is the heart of the Git Bridge. You plug in all the parts (project
+ * lock, repo store, db store, swap store, snapshot api, resource cache and
+ * postback manager) is called by Git user requests and Overleaf postback
+ * requests.
+ *
+ * Follow these links to go "outward" (to input from Git users and Overleaf):
+ *
+ * 1. JGit hooks, which handle user Git requests:
+ *
+ *    @see WLRepositoryResolver - used on all requests associate a repo with a
+ *                                project name, or fail
+ *
+*     @see WLUploadPackFactory - used to handle clones and fetches
+ *
+ *    @see WLReceivePackFactory - used to handle pushes by setting a hook
+ *    @see WriteLatexPutHook - the hook used to handle pushes
+ *
+ * 2. The Postback Servlet, which handles postbacks from the Overleaf app
+ *    to confirm that a project is pushed. If a postback is lost, it's fine, we
+ *    just update ourselves on the next access.
+ *
+ *    @see PostbackHandler - the entry point for postbacks
+ *
+ * Follow these links to go "inward" (to the Git Bridge components):
+ *
+ * 1. The Project Lock, used to synchronise accesses to projects and shutdown
+ *    the Git Bridge gracefully by preventing further lock acquiring.
+ *
+ *    @see ProjectLock - the interface used for the Project Lock
+ *    @see ProjectLockImpl - the default concrete implementation
+ *
+ * 2. The Repo Store, used to provide repository objects.
+ *
+ *    The default implementation uses Git on the file system.
+ *
+ *    @see RepoStore - the interface for the Repo Store
+ *    @see FSGitRepoStore - the default concrete implementation
+ *    @see ProjectRepo - an interface for an actual repo instance
+ *    @see GitProjectRepo - the default concrete implementation
+ *
+ * 3. The DB Store, used to store persistent data such as the latest version
+ *    of each project that we have (used for querying the Snapshot API), along
+ *    with caching remote blobs.
+ *
+ *    The default implementation is SQLite based.
+ *
+ *    @see DBStore - the interface for the DB store
+ *    @see SqliteDBStore - the default concrete implementation
+ *
+ * 4. The Swap Store, used to swap projects to when the disk goes over a
+ *    certain data usage.
+ *
+ *    The default implementation tarbzips projects to/from Amazon S3.
+ *
+ *    @see SwapStore - the interface for the Swap Store
+ *    @see S3SwapStore - the default concrete implementation
+ *
+ * 5. The Swap Job, which performs the actual swapping on the swap store based
+ *    on various configuration options.
+ *
+ *    @see SwapJob - the interface for the Swap Job
+ *    @see SwapJobImpl - the default concrete implementation
+ *
+ * 6. The Snapshot API, which provides data from the Overleaf app.
+ *
+ *    @see SnapshotApiFacade - wraps a concrete instance of the Snapshot API.
+ *    @see SnapshotApi - the interface for the Snapshot API.
+ *    @see NetSnapshotApi - the default concrete implementation
+ *
+ * 7. The Resource Cache, which provides the data for attachment resources from
+ *    URLs. It will generally fetch from the source on a cache miss.
+ *
+ *    The default implementation uses the DB Store to maintain a mapping from
+ *    URLs to files in an actual repo.
+ *
+ *    @see ResourceCache - the interface for the Resource Cache
+ *    @see UrlResourceCache - the default concrete implementation
+ *
+ * 8. The Postback Manager, which keeps track of pending postbacks. It stores a
+ *    mapping from project names to postback promises.
+ *
+ *    @see PostbackManager - the class
+ *    @see PostbackPromise - the object waited on for a postback.
+ *
+ */
+public class Bridge {
+
+    private final Config config;
+
+    private final ProjectLock lock;
+
+    private final RepoStore repoStore;
+    private final DBStore dbStore;
+    private final SwapStore swapStore;
+    private final SwapJob swapJob;
+    private final GcJob gcJob;
+
+    private final SnapshotApiFacade snapshotAPI;
+    private final ResourceCache resourceCache;
+
+    private final PostbackManager postbackManager;
+
+    /**
+     * Creates a Bridge from its configurable parts, which are the repo, db and
+     * swap store, and the swap job config.
+     *
+     * This should be the method used to create a Bridge.
+     * @param config The config to use
+     * @param repoStore The repo store to use
+     * @param dbStore The db store to use
+     * @param swapStore The swap store to use
+     * @param snapshotApi The snapshot api to use
+     * @return The constructed Bridge.
+     */
+    public static Bridge make(
+            Config config,
+            RepoStore repoStore,
+            DBStore dbStore,
+            SwapStore swapStore,
+            SnapshotApi snapshotApi
+    ) {
+        ProjectLock lock = new ProjectLockImpl((int threads) ->
+                Log.info("Waiting for " + threads + " projects...")
+        );
+        return new Bridge(
+                config,
+                lock,
+                repoStore,
+                dbStore,
+                swapStore,
+                SwapJob.fromConfig(
+                        config.getSwapJob(),
+                        lock,
+                        repoStore,
+                        dbStore,
+                        swapStore
+                ),
+                new GcJobImpl(repoStore, lock),
+                new SnapshotApiFacade(snapshotApi),
+                new UrlResourceCache(dbStore)
+        );
+    }
+
+    /**
+     * Creates a bridge from all of its components, not just its configurable
+     * parts. This is for substituting mock/stub components for testing.
+     * It's also used by Bridge.make to actually construct the bridge.
+     * @param lock the {@link ProjectLock} to use
+     * @param repoStore the {@link RepoStore} to use
+     * @param dbStore the {@link DBStore} to use
+     * @param swapStore the {@link SwapStore} to use
+     * @param swapJob the {@link SwapJob} to use
+     * @param gcJob
+     * @param snapshotAPI the {@link SnapshotApi} to use
+     * @param resourceCache the {@link ResourceCache} to use
+     */
+    Bridge(
+            Config config,
+            ProjectLock lock,
+            RepoStore repoStore,
+            DBStore dbStore,
+            SwapStore swapStore,
+            SwapJob swapJob,
+            GcJob gcJob,
+            SnapshotApiFacade snapshotAPI,
+            ResourceCache resourceCache
+    ) {
+        this.config = config;
+        this.lock = lock;
+        this.repoStore = repoStore;
+        this.dbStore = dbStore;
+        this.swapStore = swapStore;
+        this.snapshotAPI = snapshotAPI;
+        this.resourceCache = resourceCache;
+        this.swapJob = swapJob;
+        this.gcJob = gcJob;
+        postbackManager = new PostbackManager();
+        Runtime.getRuntime().addShutdownHook(new Thread(this::doShutdown));
+        repoStore.purgeNonexistentProjects(dbStore.getProjectNames());
+    }
+
+    /**
+     * This performs the graceful shutdown of the Bridge, which is called by the
+     * shutdown hook. It acquires the project write lock, which prevents
+     * work being done for new projects (which acquire the read lock).
+     * Once it has the write lock, there are no readers left, so the git bridge
+     * can shut down gracefully.
+     *
+     * It is also used by the tests.
+     */
+    void doShutdown() {
+        Log.info("Shutdown received.");
+        Log.info("Stopping SwapJob");
+        swapJob.stop();
+        Log.info("Stopping GcJob");
+        gcJob.stop();
+        Log.info("Waiting for projects");
+        lock.lockAll();
+        Log.info("Bye");
+    }
+
+    /**
+     * Starts the swap job, which will begin checking whether projects should be
+     * swapped with a configurable frequency.
+     */
+    public void startBackgroundJobs() {
+        swapJob.start();
+        gcJob.start();
+    }
+
+    public boolean healthCheck() {
+       try {
+         dbStore.getNumProjects();
+         File rootDirectory = new File("/");
+         if (!rootDirectory.exists()) {
+           throw new Exception("bad filesystem state, root directory does not exist");
+         }
+         Log.info("[HealthCheck] passed");
+         return true;
+       } catch (Exception e)  {
+         Log.error("[HealthCheck] FAILED!", e);
+         return false;
+       }
+    }
+
+    /**
+     * Performs a check of inconsistencies in the DB. This was used to upgrade
+     * the schema.
+     */
+    public void checkDB() {
+        Log.info("Checking DB");
+        File rootDir = repoStore.getRootDirectory();
+        for (File f : rootDir.listFiles()) {
+            if (f.getName().equals(".wlgb")) {
+                continue;
+            }
+            String projName = f.getName();
+            try (LockGuard __ = lock.lockGuard(projName)) {
+                File dotGit = new File(f, ".git");
+                if (!dotGit.exists()) {
+                    Log.warn("Project: {} has no .git", projName);
+                    continue;
+                }
+                ProjectState state = dbStore.getProjectState(projName);
+                if (state != ProjectState.NOT_PRESENT) {
+                    continue;
+                }
+                Log.warn(
+                        "Project: {} not in swap_store, adding",
+                        projName
+                );
+                dbStore.setLastAccessedTime(
+                        projName,
+                        new Timestamp(dotGit.lastModified())
+                );
+            }
+        }
+    }
+
+    /**
+     * Synchronises the given repository with Overleaf.
+     *
+     * It acquires the project lock and calls
+     * {@link #getUpdatedRepoCritical(Optional, String, GetDocResult)}.
+     * @param oauth2 The oauth2 to use
+     * @param projectName The name of the project
+     * @throws IOException
+     * @throws GitUserException
+     */
+    public ProjectRepo getUpdatedRepo(
+            Optional<Credential> oauth2,
+            String projectName
+    ) throws IOException, GitUserException {
+        try (LockGuard __ = lock.lockGuard(projectName)) {
+            Optional<GetDocResult> maybeDoc = snapshotAPI.getDoc(oauth2, projectName);
+            if (!maybeDoc.isPresent()) {
+                throw new RepositoryNotFoundException(projectName);
+            }
+            GetDocResult doc = maybeDoc.get();
+            Log.info("[{}] Updating repository", projectName);
+            return getUpdatedRepoCritical(oauth2, projectName, doc);
+        }
+    }
+
+    /**
+     * Synchronises the given repository with Overleaf.
+     *
+     * Pre: the project lock must be acquired for the given repo.
+     *
+     * 1. Queries the project state for the given project name.
+     *    a. NOT_PRESENT = We've never seen it before, and the row for the
+     *                     project doesn't even exist. The project definitely
+     *                     exists because we would have aborted otherwise.
+     *    b. PRESENT = The project is on disk.
+     *    c. SWAPPED = The project is in the {@link SwapStore}
+     *
+     * If the project has never been cloned, it is git init'd. If the project
+     * is in swap, it is restored to disk. Otherwise, the project was already
+     * present.
+     *
+     * With the project present, snapshots are downloaded from the snapshot
+     * API with {@link #updateProject(Optional, ProjectRepo)}.
+     *
+     * Then, the last accessed time of the project is set to the current time.
+     * This is to support the LRU of the swap store.
+     * @param oauth2
+     * @param projectName The name of the project
+     * @throws IOException
+     * @throws GitUserException
+     */
+    private ProjectRepo getUpdatedRepoCritical(
+            Optional<Credential> oauth2,
+            String projectName,
+            GetDocResult doc
+    ) throws IOException, GitUserException {
+        ProjectRepo repo;
+        ProjectState state = dbStore.getProjectState(projectName);
+        switch (state) {
+        case NOT_PRESENT:
+            Log.info("[{}] Repo not present", projectName);
+            String migratedFromID = doc.getMigratedFromID();
+            if (migratedFromID != null) {
+                Log.info("[{}] Has a migratedFromId: {}", projectName, migratedFromID);
+                try (LockGuard __ = lock.lockGuard(migratedFromID)) {
+                    ProjectState sourceState = dbStore.getProjectState(migratedFromID);
+                    switch (sourceState) {
+                        case NOT_PRESENT:
+                            // Normal init-repo
+                            Log.info("[{}] migrated-from project not present, proceed as normal",
+                                 projectName
+                            );
+                            repo = repoStore.initRepo(projectName);
+                            break;
+                        case SWAPPED:
+                            // Swap back and then copy
+                            swapJob.restore(migratedFromID);
+                            /* Fallthrough */
+                        default:
+                            // Copy data, and set version to zero
+                            Log.info("[{}] Init from other project: {}",
+                                projectName,
+                                migratedFromID
+                            );
+                            repo = repoStore.initRepoFromExisting(projectName, migratedFromID);
+                            dbStore.setLatestVersionForProject(migratedFromID, 0);
+                            dbStore.setLastAccessedTime(
+                                    migratedFromID,
+                                    Timestamp.valueOf(LocalDateTime.now())
+
+                            );
+                    }
+                }
+                break;
+            } else {
+                repo = repoStore.initRepo(projectName);
+                break;
+            }
+        case SWAPPED:
+            swapJob.restore(projectName);
+            /* Fallthrough */
+        default:
+            repo = repoStore.getExistingRepo(projectName);
+        }
+        updateProject(oauth2, repo);
+        dbStore.setLastAccessedTime(
+                projectName,
+                Timestamp.valueOf(LocalDateTime.now())
+        );
+        return repo;
+    }
+
+    /**
+     * The public call to push a project.
+     *
+     * It acquires the lock and calls {@link #pushCritical(
+     *      Optional,
+     *      String,
+     *      RawDirectory,
+     *      RawDirectory
+     * )}, catching exceptions, logging, and rethrowing them.
+     * @param oauth2 The oauth2 to use for the snapshot API
+     * @param projectName The name of the project to push to
+     * @param directoryContents The new contents of the project
+     * @param oldDirectoryContents The old contents of the project
+     * @param hostname
+     * @throws SnapshotPostException
+     * @throws IOException
+     * @throws MissingRepositoryException
+     * @throws ForbiddenException
+     * @throws GitUserException
+     */
+    public void push(
+            Optional<Credential> oauth2,
+            String projectName,
+            RawDirectory directoryContents,
+            RawDirectory oldDirectoryContents,
+            String hostname
+    ) throws SnapshotPostException, IOException, MissingRepositoryException, ForbiddenException, GitUserException {
+        Log.debug("[{}] pushing to Overleaf", projectName);
+        try (LockGuard __ = lock.lockGuard(projectName)) {
+            pushCritical(
+                    oauth2,
+                    projectName,
+                    directoryContents,
+                    oldDirectoryContents
+            );
+        } catch (SevereSnapshotPostException e) {
+            Log.warn(
+                    "[" + projectName + "] Failed to put to Overleaf",
+                    e
+            );
+            throw e;
+        } catch (SnapshotPostException e) {
+            /* Stack trace should be printed further up */
+            Log.warn(
+                    "[{}] Exception when waiting for postback: {}",
+                    projectName,
+                    e.getClass().getSimpleName()
+            );
+            throw e;
+        } catch (IOException e) {
+            Log.warn("[{}] IOException on put: {}", projectName, e);
+            throw e;
+        }
+
+        gcJob.queueForGc(projectName);
+    }
+
+    /**
+     * Does the work of pushing to a project, assuming the project lock is held.
+     * The {@link WriteLatexPutHook} is the original caller, and when we return
+     * without throwing, the commit is committed.
+     *
+     * We start off by creating a postback key, which is given in the url when
+     * the Overleaf app tries to access the atts.
+     *
+     * Then creates a {@link CandidateSnapshot} from the old and new project
+     * contents. The
+     * {@link CandidateSnapshot} is created using
+     * {@link #createCandidateSnapshot(String, RawDirectory, RawDirectory)},
+     * which creates the snapshot object and writes the push files to the
+     * atts directory, which is served by the {@link PostbackHandler}.
+     * The files are deleted at the end of a try-with-resources block.
+     *
+     * Then 3 things are used to make the push request to the snapshot API:
+     * 1. The oauth2
+     * 2. The candidate snapshot
+     * 3. The postback key
+     *
+     * If the snapshot API reports this as not successful, we immediately throw
+     * an {@link OutOfDateException}, which goes back to the user.
+     *
+     * Otherwise, we wait (with a timeout) on a promise from the postback
+     * manager, which can throw back to the user.
+     *
+     * If this is successful, we approve the snapshot with
+     * {@link #approveSnapshot(int, CandidateSnapshot)}, which updates our side
+     * of the push: the latest version and the URL index store.
+     *
+     * Then, we set the last accessed time for the swap store.
+     *
+     * Finally, after we return, the push to the repo from the hook is
+     * successful and the repo gets updated.
+     *
+     * @param oauth2
+     * @param projectName
+     * @param directoryContents
+     * @param oldDirectoryContents
+     * @throws IOException
+     * @throws MissingRepositoryException
+     * @throws ForbiddenException
+     * @throws SnapshotPostException
+     * @throws GitUserException
+     */
+    private void pushCritical(
+            Optional<Credential> oauth2,
+            String projectName,
+            RawDirectory directoryContents,
+            RawDirectory oldDirectoryContents
+    ) throws IOException, MissingRepositoryException, ForbiddenException, SnapshotPostException, GitUserException {
+        Optional<Long> maxFileNum = config
+                  .getRepoStore()
+                  .flatMap(RepoStoreConfig::getMaxFileNum);
+        if (maxFileNum.isPresent()) {
+          long maxFileNum_ = maxFileNum.get();
+          if (directoryContents.getFileTable().size() > maxFileNum_) {
+            Log.debug("[{}] Too many files: {}/{}", projectName, directoryContents.getFileTable().size(), maxFileNum_);
+            throw new FileLimitExceededException(directoryContents.getFileTable().size(), maxFileNum_);
+          }
+        }
+        Log.info("[{}] Pushing files ({} new, {} old)", projectName, directoryContents.getFileTable().size(), oldDirectoryContents.getFileTable().size());
+        String postbackKey = postbackManager.makeKeyForProject(projectName);
+        Log.info(
+                "[{}] Created postback key: {}",
+                projectName,
+                postbackKey
+        );
+        try (
+                CandidateSnapshot candidate = createCandidateSnapshot(
+                                projectName,
+                                directoryContents,
+                                oldDirectoryContents
+                );
+        ) {
+            Log.info(
+                    "[{}] Candidate snapshot created: {}",
+                    projectName,
+                    candidate
+            );
+            PushResult result
+                    = snapshotAPI.push(oauth2, candidate, postbackKey);
+            if (result.wasSuccessful()) {
+                Log.info(
+                        "[{}] Push to Overleaf successful",
+                        projectName
+                );
+                Log.info("[{}] Waiting for postback...", projectName);
+                int versionID =
+                        postbackManager.waitForVersionIdOrThrow(projectName);
+                Log.info(
+                        "[{}] Got version ID for push: {}",
+                        projectName,
+                        versionID
+                );
+                approveSnapshot(versionID, candidate);
+                Log.info(
+                        "[{}] Approved version ID: {}",
+                        projectName,
+                        versionID
+                );
+                dbStore.setLastAccessedTime(
+                        projectName,
+                        Timestamp.valueOf(LocalDateTime.now())
+                );
+            } else {
+                Log.warn(
+                        "[{}] Went out of date while waiting for push",
+                        projectName
+                );
+                throw new OutOfDateException();
+            }
+        }
+    }
+
+    /**
+     * A public call that should originate from the {@link FileHandler}.
+     *
+     * The {@link FileHandler} serves atts to the Overleaf app during a push.
+     * The Overleaf app includes the postback key in the request, which was
+     * originally given on a push request.
+     *
+     * This method checks that the postback key matches, and throws if not.
+     *
+     * The FileHandler should not serve the file if this throws.
+     * @param projectName The project name that this key belongs to
+     * @param postbackKey The key
+     * @throws InvalidPostbackKeyException If the key doesn't match
+     */
+    public void checkPostbackKey(String projectName, String postbackKey)
+            throws InvalidPostbackKeyException {
+        postbackManager.checkPostbackKey(projectName, postbackKey);
+    }
+
+    /**
+     * A public call that originates from the postback thread
+     * {@link PostbackContents#processPostback()}, i.e. once the Overleaf app
+     * has fetched all the atts and has committed the push and is happy, it
+     * calls back here, fulfilling the promise that the push
+     * {@link #push(Optional, String, RawDirectory, RawDirectory, String)}
+     * is waiting on.
+     *
+     * The Overleaf app will have invented a new version for the push, which is
+     * passed to the promise for the original push request to update the app.
+     * @param projectName The name of the project being pushed to
+     * @param postbackKey The postback key being used
+     * @param versionID the new version id to use
+     * @throws UnexpectedPostbackException if the postback key is invalid
+     */
+    public void postbackReceivedSuccessfully(
+            String projectName,
+            String postbackKey,
+            int versionID
+    ) throws UnexpectedPostbackException {
+        Log.info(
+                "[{}]" +
+                        " Postback received by postback thread, version: {}",
+                projectName,
+                versionID);
+        postbackManager.postVersionIDForProject(
+                projectName,
+                versionID,
+                postbackKey
+        );
+    }
+
+    /**
+     * As with {@link #postbackReceivedSuccessfully(String, String, int)},
+     * but with an exception instead.
+     *
+     * This is based on the JSON body of the postback from the Overleaf app.
+     *
+     * The most likely problem is an {@link OutOfDateException}.
+     * @param projectName The name of the project
+     * @param postbackKey The postback key being used
+     * @param exception The exception encountered
+     * @throws UnexpectedPostbackException If the postback key is invalid
+     */
+    public void postbackReceivedWithException(
+            String projectName,
+            String postbackKey,
+            SnapshotPostException exception
+    ) throws UnexpectedPostbackException {
+        Log.warn("[{}] Postback received with exception", projectName);
+        postbackManager.postExceptionForProject(
+                projectName,
+                exception,
+                postbackKey
+        );
+    }
+
+    /* PRIVATE */
+
+    /**
+     * Called by {@link #getUpdatedRepoCritical(Optional, String)}
+     *
+     * Does the actual work of getting the snapshots for a project from the
+     * snapshot API and committing them to a repo.
+     *
+     * If any snapshots were found, sets the latest version for the project.
+     *
+     * @param oauth2
+     * @param repo
+     * @throws IOException
+     * @throws GitUserException
+     */
+    private void updateProject(
+            Optional<Credential> oauth2,
+            ProjectRepo repo
+    ) throws IOException, GitUserException {
+        String projectName = repo.getProjectName();
+        int latestVersionId = dbStore.getLatestVersionForProject(projectName);
+        Deque<Snapshot> snapshots = snapshotAPI.getSnapshots(
+                oauth2, projectName, latestVersionId);
+
+        makeCommitsFromSnapshots(repo, snapshots);
+
+        // TODO: in case crashes around here, add an
+        // "updating_from_commit" column to the DB as a way to rollback the
+        // any failed partial updates before re-trying
+        // Also need to consider the empty state (a new git init'd repo being
+        // the rollback target)
+        if (!snapshots.isEmpty()) {
+            dbStore.setLatestVersionForProject(
+                    projectName,
+                    snapshots.getLast().getVersionID()
+            );
+        }
+    }
+
+    /**
+     * Called by {@link #updateProject(Optional, ProjectRepo)}.
+     *
+     * Performs the actual Git commits on the disk.
+     *
+     * Each commit adds files to the db store
+     * ({@link ResourceCache#get(String, String, String, Map, Map, Optional)},
+     * and then removes any files that were deleted.
+     * @param repo The repository to commit to
+     * @param snapshots The snapshots to commit
+     * @throws IOException If an IOException occurred
+     * @throws SizeLimitExceededException If one of the files was too big.
+     */
+    private void makeCommitsFromSnapshots(
+            ProjectRepo repo,
+            Collection<Snapshot> snapshots
+    ) throws IOException, GitUserException {
+        String name = repo.getProjectName();
+        Optional<Long> maxSize = config
+                .getRepoStore()
+                .flatMap(RepoStoreConfig::getMaxFileSize);
+        for (Snapshot snapshot : snapshots) {
+            RawDirectory directory = repo.getDirectory();
+            Map<String, RawFile> fileTable = directory.getFileTable();
+            List<RawFile> files = new ArrayList<>();
+            files.addAll(snapshot.getSrcs());
+            for (RawFile file : files) {
+                long size = file.size();
+                /* Can't throw in ifPresent... */
+                if (maxSize.isPresent()) {
+                    long maxSize_ = maxSize.get();
+                    if (size >= maxSize_) {
+                        throw new SizeLimitExceededException(
+                                Optional.of(file.getPath()), size, maxSize_);
+                    }
+                }
+            }
+            Map<String, byte[]> fetchedUrls = new HashMap<>();
+            for (SnapshotAttachment snapshotAttachment : snapshot.getAtts()) {
+                files.add(
+                        resourceCache.get(
+                                name,
+                                snapshotAttachment.getUrl(),
+                                snapshotAttachment.getPath(),
+                                fileTable,
+                                fetchedUrls,
+                                maxSize
+                        )
+                );
+            }
+            Log.info(
+                    "[{}] Committing version ID: {}",
+                    name,
+                    snapshot.getVersionID()
+            );
+            Collection<String> missingFiles = repo.commitAndGetMissing(
+                    new GitDirectoryContents(
+                            files,
+                            repoStore.getRootDirectory(),
+                            name,
+                            snapshot
+                    )
+            );
+            dbStore.deleteFilesForProject(
+                    name,
+                    missingFiles.toArray(new String[missingFiles.size()])
+            );
+        }
+    }
+
+    /**
+     * Called by
+     * {@link #pushCritical(Optional, String, RawDirectory, RawDirectory)}.
+     *
+     * This call consists of 2 things: Creating the candidate snapshot,
+     * and writing the atts to the atts directory.
+     *
+     * The candidate snapshot RAIIs away those atts (use try-with-resources).
+     * @param projectName The name of the project
+     * @param directoryContents The new directory contents
+     * @param oldDirectoryContents The old directory contents
+     * @return The {@link CandidateSnapshot} created
+     * @throws IOException If an I/O exception occurred on writing
+     */
+    private CandidateSnapshot createCandidateSnapshot(
+            String projectName,
+            RawDirectory directoryContents,
+            RawDirectory oldDirectoryContents
+    ) throws IOException {
+        CandidateSnapshot candidateSnapshot = new CandidateSnapshot(
+                projectName,
+                dbStore.getLatestVersionForProject(projectName),
+                directoryContents,
+                oldDirectoryContents
+        );
+        candidateSnapshot.writeServletFiles(repoStore.getRootDirectory());
+        return candidateSnapshot;
+    }
+
+    /**
+     * Called by
+     * {@link #pushCritical(Optional, String, RawDirectory, RawDirectory)}.
+     *
+     * This method approves a push by setting the latest version and removing
+     * any deleted files from the db store (files were already added by the
+     * resources cache).
+     * @param versionID
+     * @param candidateSnapshot
+     */
+    private void approveSnapshot(
+            int versionID,
+            CandidateSnapshot candidateSnapshot
+    ) {
+        List<String> deleted = candidateSnapshot.getDeleted();
+        dbStore.setLatestVersionForProject(
+                candidateSnapshot.getProjectName(),
+                versionID
+        );
+        dbStore.deleteFilesForProject(
+                candidateSnapshot.getProjectName(),
+                deleted.toArray(new String[deleted.size()])
+        );
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBInitException.java

@@ -0,0 +1,20 @@
+package uk.ac.ic.wlgitbridge.bridge.db;
+
+/**
+ * Created by winston on 23/08/2016.
+ */
+public class DBInitException extends RuntimeException {
+
+    public DBInitException(String message) {
+        super(message);
+    }
+
+    public DBInitException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public DBInitException(Throwable cause) {
+        super(cause);
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java

@@ -0,0 +1,44 @@
+package uk.ac.ic.wlgitbridge.bridge.db;
+
+import java.sql.Timestamp;
+import java.util.List;
+
+/**
+ * Created by winston on 20/08/2016.
+ */
+public interface DBStore {
+
+    int getNumProjects();
+
+    List<String> getProjectNames();
+
+    void setLatestVersionForProject(String project, int versionID);
+
+    int getLatestVersionForProject(String project);
+
+    void addURLIndexForProject(String projectName, String url, String path);
+
+    void deleteFilesForProject(String project, String... files);
+
+    String getPathForURLInProject(String projectName, String url);
+
+    String getOldestUnswappedProject();
+
+    void swap(String projectName, String compressionMethod);
+
+    void restore(String projectName);
+
+    String getSwapCompression(String projectName);
+
+    int getNumUnswappedProjects();
+
+    ProjectState getProjectState(String projectName);
+
+    /**
+     * Sets the last accessed time for the given project name.
+     * @param projectName the project's name
+     * @param time the time, or null if the project is to be swapped
+     */
+    void setLastAccessedTime(String projectName, Timestamp time);
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/ProjectState.java

@@ -0,0 +1,12 @@
+package uk.ac.ic.wlgitbridge.bridge.db;
+
+/**
+ * Created by winston on 24/08/2016.
+ */
+public enum ProjectState {
+
+    NOT_PRESENT,
+    PRESENT,
+    SWAPPED
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/noop/NoopDbStore.java

@@ -0,0 +1,75 @@
+package uk.ac.ic.wlgitbridge.bridge.db.noop;
+
+import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
+import uk.ac.ic.wlgitbridge.bridge.db.ProjectState;
+
+import java.sql.Timestamp;
+import java.util.List;
+
+public class NoopDbStore implements DBStore {
+
+    @Override
+    public int getNumProjects() {
+        return 0;
+    }
+
+    @Override
+    public List<String> getProjectNames() {
+        return null;
+    }
+
+    @Override
+    public void setLatestVersionForProject(String project, int versionID) {
+
+    }
+
+    @Override
+    public int getLatestVersionForProject(String project) {
+        return 0;
+    }
+
+    @Override
+    public void addURLIndexForProject(String projectName, String url, String path) {
+
+    }
+
+    @Override
+    public void deleteFilesForProject(String project, String... files) {
+
+    }
+
+    @Override
+    public String getPathForURLInProject(String projectName, String url) {
+        return null;
+    }
+
+    @Override
+    public String getOldestUnswappedProject() {
+        return null;
+    }
+
+    @Override
+    public int getNumUnswappedProjects() {
+        return 0;
+    }
+
+    @Override
+    public ProjectState getProjectState(String projectName) {
+        return null;
+    }
+
+    @Override
+    public void setLastAccessedTime(String projectName, Timestamp time) {
+    }
+
+    @Override
+    public void swap(String projectName, String compressionMethod) {}
+
+    @Override
+    public void restore(String projectName) {}
+
+    @Override
+    public String getSwapCompression(String projectName) {
+        return null;
+    }
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SQLQuery.java

@@ -0,0 +1,13 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+/**
+ * Created by Winston on 20/11/14.
+ */
+public interface SQLQuery<T> extends SQLUpdate {
+
+    public T processResultSet(ResultSet resultSet) throws SQLException;
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SQLUpdate.java

@@ -0,0 +1,18 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+/**
+ * Created by Winston on 20/11/14.
+ */
+public interface SQLUpdate {
+
+    String getSQL();
+    default void addParametersToStatement(
+            PreparedStatement statement
+    ) throws SQLException {
+
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStore.java

@@ -0,0 +1,228 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite;
+
+import com.google.common.base.Preconditions;
+import uk.ac.ic.wlgitbridge.bridge.db.DBInitException;
+import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
+import uk.ac.ic.wlgitbridge.bridge.db.ProjectState;
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.query.*;
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter.*;
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create.*;
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.delete.*;
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert.*;
+
+import java.io.File;
+import java.sql.*;
+import java.util.List;
+import java.util.stream.Stream;
+
+/**
+ * Created by Winston on 17/11/14.
+ */
+public class SqliteDBStore implements DBStore {
+
+    private final Connection connection;
+    private int heapLimitBytes = 0;
+
+    public SqliteDBStore(File dbFile) {
+        this(dbFile, 0);
+    }
+
+    public SqliteDBStore(File dbFile, int heapLimitBytes) {
+        this.heapLimitBytes = heapLimitBytes;
+        try {
+            connection = openConnectionTo(dbFile);
+            createTables();
+        } catch (Throwable t) {
+            throw new DBInitException(t);
+        }
+    }
+
+    @Override
+    public int getNumProjects() {
+        return query(new GetNumProjects());
+    }
+
+    @Override
+    public List<String> getProjectNames() {
+        return query(new GetProjectNamesSQLQuery());
+    }
+
+    @Override
+    public void setLatestVersionForProject(
+            String projectName,
+            int versionID
+    ) {
+        update(new SetProjectSQLUpdate(projectName, versionID));
+    }
+
+    @Override
+    public int getLatestVersionForProject(
+            String projectName
+    ) {
+        return query(new GetLatestVersionForProjectSQLQuery(projectName));
+    }
+
+    @Override
+    public void addURLIndexForProject(
+            String projectName,
+            String url,
+            String path
+    ) {
+        update(new AddURLIndexSQLUpdate(projectName, url, path));
+    }
+
+    @Override
+    public void deleteFilesForProject(
+            String projectName,
+            String... paths
+    ) {
+        update(new DeleteFilesForProjectSQLUpdate(projectName, paths));
+    }
+
+    @Override
+    public String getPathForURLInProject(
+            String projectName,
+            String url
+    ) {
+        return query(new GetPathForURLInProjectSQLQuery(projectName, url));
+    }
+
+    @Override
+    public String getOldestUnswappedProject() {
+        return query(new GetOldestProjectName());
+    }
+
+    @Override
+    public int getNumUnswappedProjects() {
+        return query(new GetNumUnswappedProjects());
+    }
+
+    @Override
+    public ProjectState getProjectState(String projectName) {
+        return query(new GetProjectState(projectName));
+    }
+
+    @Override
+    public void setLastAccessedTime(
+            String projectName,
+            Timestamp lastAccessed
+    ) {
+        update(new SetProjectLastAccessedTime(projectName, lastAccessed));
+    }
+
+    @Override
+    public void swap(String projectName, String compressionMethod) {
+      update(new UpdateSwap(projectName, compressionMethod));
+    }
+
+    @Override
+    public void restore(String projectName) {
+      update(new UpdateRestore(projectName));
+    }
+
+    @Override
+    public String getSwapCompression(String projectName) {
+        return query(new GetSwapCompression(projectName));
+    }
+
+    private Connection openConnectionTo(File dbFile) {
+        File parentDir = dbFile.getParentFile();
+        if (!parentDir.exists() && !parentDir.mkdirs()) {
+            throw new DBInitException(
+                    parentDir.getAbsolutePath() + " directory didn't exist, " +
+                            "and unable to create. Check your permissions."
+            );
+        }
+        try {
+            Class.forName("org.sqlite.JDBC");
+        } catch (ClassNotFoundException e) {
+            throw new DBInitException(e);
+        }
+        try {
+            return DriverManager.getConnection(
+                    "jdbc:sqlite:" + dbFile.getAbsolutePath()
+            );
+        } catch (SQLException e) {
+            throw new DBInitException("Unable to connect to DB", e);
+        }
+    }
+
+    private void createTables() {
+        /* Migrations */
+        /* We need to eat exceptions from here */
+        try { doUpdate(new SetSoftHeapLimitPragma(this.heapLimitBytes)); } catch (SQLException ignore) {}
+        try { doUpdate(new ProjectsAddLastAccessed()); } catch (SQLException ignore) {}
+        try { doUpdate(new ProjectsAddSwapTime()); } catch (SQLException ignore) {}
+        try { doUpdate(new ProjectsAddRestoreTime()); } catch (SQLException ignore) {}
+        try { doUpdate(new ProjectsAddSwapCompression()); } catch (SQLException ignore) {}
+
+        /* Create tables (if they don't exist) */
+        Stream.of(
+                new CreateProjectsTableSQLUpdate(),
+                new CreateProjectsIndexLastAccessed(),
+                new CreateURLIndexStoreSQLUpdate(),
+                new CreateIndexURLIndexStore()
+        ).forEach(this::update);
+
+        /* In the case of needing to change the schema, we need to check that
+           migrations didn't just fail */
+        Preconditions.checkState(query(new LastAccessedColumnExists()));
+        Preconditions.checkState(query(new SwapTimeColumnExists()));
+        Preconditions.checkState(query(new RestoreTimeColumnExists()));
+        Preconditions.checkState(query(new SwapCompressionColumnExists()));
+    }
+
+    private void update(SQLUpdate update) {
+        try {
+            doUpdate(update);
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private <T> T query(SQLQuery<T> query) {
+        try {
+            return doQuery(query);
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private void doUpdate(SQLUpdate update) throws SQLException {
+        PreparedStatement statement = null;
+        try {
+            statement = connection.prepareStatement(update.getSQL());
+            update.addParametersToStatement(statement);
+            statement.executeUpdate();
+        } catch (SQLException e) {
+            throw e;
+        } finally {
+            try {
+                statement.close();
+            } catch (Throwable t) {
+                throw new SQLException(t);
+            }
+        }
+    }
+
+    private <T> T doQuery(SQLQuery<T> query) throws SQLException {
+        PreparedStatement statement = null;
+        ResultSet results = null;
+        try {
+            statement = connection.prepareStatement(query.getSQL());
+            query.addParametersToStatement(statement);
+            results = statement.executeQuery();
+            return query.processResultSet(results);
+        } catch (SQLException e) {
+            throw e;
+        } finally {
+            if (statement != null) {
+                statement.close();
+            }
+            if (results != null) {
+                results.close();
+            }
+        }
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetLatestVersionForProjectSQLQuery.java

@@ -0,0 +1,43 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
+
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+/**
+ * Created by Winston on 20/11/14.
+ */
+public class GetLatestVersionForProjectSQLQuery implements SQLQuery<Integer> {
+
+    private static final String GET_VERSION_IDS_FOR_PROJECT_NAME =
+            "SELECT `version_id` FROM `projects` WHERE `name` = ?";
+
+    private final String projectName;
+
+    public GetLatestVersionForProjectSQLQuery(String projectName) {
+        this.projectName = projectName;
+    }
+
+    @Override
+    public Integer processResultSet(ResultSet resultSet) throws SQLException {
+        int versionID = 0;
+        while (resultSet.next()) {
+            versionID = resultSet.getInt("version_id");
+        }
+        return versionID;
+    }
+
+    @Override
+    public String getSQL() {
+        return GET_VERSION_IDS_FOR_PROJECT_NAME;
+    }
+
+    @Override
+    public void addParametersToStatement(
+            PreparedStatement statement
+    ) throws SQLException {
+        statement.setString(1, projectName);
+    }
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetNumProjects.java

@@ -0,0 +1,30 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+/**
+ * Created by winston on 24/08/2016.
+ */
+public class GetNumProjects implements SQLQuery<Integer> {
+
+    private static final String GET_NUM_PROJECTS =
+            "SELECT COUNT(*)\n" +
+            "    FROM `projects`";
+
+    @Override
+    public String getSQL() {
+        return GET_NUM_PROJECTS;
+    }
+
+    @Override
+    public Integer processResultSet(ResultSet resultSet) throws SQLException {
+        while (resultSet.next()) {
+            return resultSet.getInt("COUNT(*)");
+        }
+        throw new IllegalStateException("Count always returns results");
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetNumUnswappedProjects.java

@@ -0,0 +1,31 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+/**
+ * Created by winston on 24/08/2016.
+ */
+public class GetNumUnswappedProjects implements SQLQuery<Integer> {
+
+    private static final String GET_NUM_UNSWAPPED_PROJECTS =
+            "SELECT COUNT(*)\n" +
+            "    FROM `projects`\n" +
+            "    WHERE `last_accessed` IS NOT NULL";
+
+    @Override
+    public String getSQL() {
+        return GET_NUM_UNSWAPPED_PROJECTS;
+    }
+
+    @Override
+    public Integer processResultSet(ResultSet resultSet) throws SQLException {
+        while (resultSet.next()) {
+            return resultSet.getInt("COUNT(*)");
+        }
+        throw new IllegalStateException("Count always returns results");
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetOldestProjectName.java

@@ -0,0 +1,31 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+/**
+ * Created by winston on 23/08/2016.
+ */
+public class GetOldestProjectName implements SQLQuery<String> {
+
+    private static final String GET_OLDEST_PROJECT_NAME =
+            "SELECT `name`, MIN(`last_accessed`)\n" +
+            "    FROM `projects` \n" +
+            "    WHERE `last_accessed` IS NOT NULL;";
+
+    @Override
+    public String getSQL() {
+        return GET_OLDEST_PROJECT_NAME;
+    }
+
+    @Override
+    public String processResultSet(ResultSet resultSet) throws SQLException {
+        while (resultSet.next()) {
+            return resultSet.getString("name");
+        }
+        return null;
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetPathForURLInProjectSQLQuery.java

@@ -0,0 +1,50 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
+
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+/**
+ * Created by Winston on 20/11/14.
+ */
+public class GetPathForURLInProjectSQLQuery implements SQLQuery<String> {
+
+    private static final String GET_URL_INDEXES_FOR_PROJECT_NAME =
+            "SELECT `path` "
+                    + "FROM `url_index_store` "
+                    + "WHERE `project_name` = ? "
+                    + "AND `url` = ?";
+
+    private final String projectName;
+    private final String url;
+
+    public GetPathForURLInProjectSQLQuery(String projectName, String url) {
+        this.projectName = projectName;
+        this.url = url;
+    }
+
+    @Override
+    public String processResultSet(ResultSet resultSet) throws SQLException {
+        String path = null;
+        while (resultSet.next()) {
+            path = resultSet.getString("path");
+        }
+        return path;
+    }
+
+    @Override
+    public String getSQL() {
+        return GET_URL_INDEXES_FOR_PROJECT_NAME;
+    }
+
+    @Override
+    public void addParametersToStatement(
+            PreparedStatement statement
+    ) throws SQLException {
+        statement.setString(1, projectName);
+        statement.setString(2, url);
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetProjectNamesSQLQuery.java

@@ -0,0 +1,34 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Created by Winston on 21/02/15.
+ */
+public class GetProjectNamesSQLQuery implements SQLQuery<List<String>> {
+
+    private static final String GET_URL_INDEXES_FOR_PROJECT_NAME =
+            "SELECT `name` FROM `projects`";
+
+    @Override
+    public List<String> processResultSet(
+            ResultSet resultSet
+    ) throws SQLException {
+        List<String> projectNames = new ArrayList<>();
+        while (resultSet.next()) {
+            projectNames.add(resultSet.getString("name"));
+        }
+        return projectNames;
+    }
+
+    @Override
+    public String getSQL() {
+        return GET_URL_INDEXES_FOR_PROJECT_NAME;
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetProjectState.java

@@ -0,0 +1,51 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
+
+import uk.ac.ic.wlgitbridge.bridge.db.ProjectState;
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
+
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+/**
+ * Created by winston on 24/08/2016.
+ */
+public class GetProjectState implements SQLQuery<ProjectState> {
+
+    private static final String GET_PROJECT_STATE =
+            "SELECT `last_accessed`\n" +
+            "    FROM `projects`\n" +
+            "    WHERE `name` = ?";
+
+    private final String projectName;
+
+    public GetProjectState(String projectName) {
+        this.projectName = projectName;
+    }
+
+    @Override
+    public String getSQL() {
+        return GET_PROJECT_STATE;
+    }
+
+    @Override
+    public ProjectState processResultSet(
+            ResultSet resultSet
+    ) throws SQLException {
+        while (resultSet.next()) {
+            if (resultSet.getTimestamp("last_accessed") == null) {
+                return ProjectState.SWAPPED;
+            }
+            return ProjectState.PRESENT;
+        }
+        return ProjectState.NOT_PRESENT;
+    }
+
+    @Override
+    public void addParametersToStatement(
+            PreparedStatement statement
+    ) throws SQLException {
+        statement.setString(1, projectName);
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetSwapCompression.java

@@ -0,0 +1,39 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
+
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+public class GetSwapCompression implements SQLQuery<String> {
+  private static final String GET_SWAP_COMPRESSION =
+    "SELECT `swap_compression` FROM `projects` WHERE `name` = ?";
+
+  private final String projectName;
+
+  public GetSwapCompression(String projectName) {
+    this.projectName = projectName;
+  }
+
+  @Override
+  public String processResultSet(ResultSet resultSet) throws SQLException {
+    String compression = null;
+    while (resultSet.next()) {
+      compression = resultSet.getString("swap_compression");
+    }
+    return compression;
+  }
+
+  @Override
+  public String getSQL() {
+    return GET_SWAP_COMPRESSION;
+  }
+
+  @Override
+  public void addParametersToStatement(
+    PreparedStatement statement
+  ) throws SQLException {
+    statement.setString(1, projectName);
+  }
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/LastAccessedColumnExists.java

@@ -0,0 +1,31 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+/**
+ * Created by winston on 04/09/2016.
+ */
+public class LastAccessedColumnExists implements SQLQuery<Boolean> {
+
+    private static final String LAST_ACCESSED_COLUMN_EXISTS =
+            "PRAGMA table_info(`projects`)";
+
+    @Override
+    public String getSQL() {
+        return LAST_ACCESSED_COLUMN_EXISTS;
+    }
+
+    @Override
+    public Boolean processResultSet(ResultSet resultSet) throws SQLException {
+        while (resultSet.next()) {
+            if (resultSet.getString(2).equals("last_accessed")) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/RestoreTimeColumnExists.java

@@ -0,0 +1,26 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+public class RestoreTimeColumnExists implements SQLQuery<Boolean> {
+    private static final String RESTORE_TIME_COLUMN_EXISTS =
+            "PRAGMA table_info(`projects`)";
+
+    @Override
+    public String getSQL() {
+        return RESTORE_TIME_COLUMN_EXISTS;
+    }
+
+    @Override
+    public Boolean processResultSet(ResultSet resultSet) throws SQLException {
+        while (resultSet.next()) {
+            if (resultSet.getString(2).equals("restore_time")) {
+                return true;
+            }
+        }
+        return false;
+    }
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/SwapCompressionColumnExists.java

@@ -0,0 +1,27 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+public class SwapCompressionColumnExists implements SQLQuery<Boolean> {
+    private static final String SWAP_COMPRESSION_COLUMN_EXISTS =
+            "PRAGMA table_info(`projects`)";
+
+    @Override
+    public String getSQL() {
+        return SWAP_COMPRESSION_COLUMN_EXISTS;
+    }
+
+    @Override
+    public Boolean processResultSet(ResultSet resultSet) throws SQLException {
+        while (resultSet.next()) {
+            if (resultSet.getString(2).equals("swap_compression")) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/SwapTimeColumnExists.java

@@ -0,0 +1,27 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+public class SwapTimeColumnExists implements SQLQuery<Boolean> {
+    private static final String SWAP_TIME_COLUMN_EXISTS =
+            "PRAGMA table_info(`projects`)";
+
+    @Override
+    public String getSQL() {
+        return SWAP_TIME_COLUMN_EXISTS;
+    }
+
+    @Override
+    public Boolean processResultSet(ResultSet resultSet) throws SQLException {
+        while (resultSet.next()) {
+            if (resultSet.getString(2).equals("swap_time")) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddLastAccessed.java

@@ -0,0 +1,19 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
+
+/**
+ * Created by winston on 03/09/2016.
+ */
+public class ProjectsAddLastAccessed implements SQLUpdate {
+
+    private static final String PROJECTS_ADD_LAST_ACCESSED =
+            "ALTER TABLE `projects`\n" +
+            "ADD COLUMN `last_accessed` DATETIME NULL DEFAULT 0";
+
+    @Override
+    public String getSQL() {
+        return PROJECTS_ADD_LAST_ACCESSED;
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddRestoreTime.java

@@ -0,0 +1,14 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
+
+public class ProjectsAddRestoreTime implements SQLUpdate {
+  private static final String PROJECTS_ADD_RESTORE_TIME =
+    "ALTER TABLE `projects`\n" +
+      "ADD COLUMN `restore_time` DATETIME NULL;\n";
+
+  @Override
+  public String getSQL() {
+    return PROJECTS_ADD_RESTORE_TIME;
+  }
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddSwapCompression.java

@@ -0,0 +1,14 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
+
+public class ProjectsAddSwapCompression implements SQLUpdate {
+  private static final String PROJECTS_ADD_SWAP_COMPRESSION =
+    "ALTER TABLE `projects`\n" +
+      "ADD COLUMN `swap_compression` VARCHAR NULL;\n";
+
+  @Override
+  public String getSQL() {
+    return PROJECTS_ADD_SWAP_COMPRESSION;
+  }
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddSwapTime.java

@@ -0,0 +1,15 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
+
+public class ProjectsAddSwapTime implements SQLUpdate {
+  private static final String PROJECTS_ADD_SWAP_TIME =
+    "ALTER TABLE `projects`\n" +
+      "ADD COLUMN `swap_time` DATETIME NULL;\n";
+
+  @Override
+  public String getSQL() {
+    return PROJECTS_ADD_SWAP_TIME;
+  }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/SetSoftHeapLimitPragma.java

@@ -0,0 +1,17 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
+
+public class SetSoftHeapLimitPragma implements SQLUpdate {
+    private int heapLimitBytes = 0;
+
+    public SetSoftHeapLimitPragma(int heapLimitBytes) {
+        this.heapLimitBytes = heapLimitBytes;
+    }
+
+    @Override
+    public String getSQL() {
+        return "PRAGMA soft_heap_limit="+this.heapLimitBytes+";";
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateIndexURLIndexStore.java

@@ -0,0 +1,19 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
+
+/**
+ * Created by Winston on 21/02/15.
+ */
+public class CreateIndexURLIndexStore implements SQLUpdate {
+
+    public static final String CREATE_INDEX_URL_INDEX_STORE =
+            "CREATE UNIQUE INDEX IF NOT EXISTS `project_path_index` " +
+            "ON `url_index_store`(`project_name`, `path`);\n";
+
+    @Override
+    public String getSQL() {
+        return CREATE_INDEX_URL_INDEX_STORE;
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateProjectsIndexLastAccessed.java

@@ -0,0 +1,19 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
+
+/**
+ * Created by winston on 23/08/2016.
+ */
+public class CreateProjectsIndexLastAccessed implements SQLUpdate {
+
+    private static final String CREATE_PROJECTS_INDEX_LAST_ACCESSED =
+            "CREATE INDEX IF NOT EXISTS `projects_index_last_accessed`\n" +
+            "    ON `projects`(`last_accessed`)";
+
+    @Override
+    public String getSQL() {
+        return CREATE_PROJECTS_INDEX_LAST_ACCESSED;
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateProjectsTableSQLUpdate.java

@@ -0,0 +1,26 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
+
+/**
+ * Created by Winston on 20/11/14.
+ */
+public class CreateProjectsTableSQLUpdate implements SQLUpdate {
+
+    private static final String CREATE_PROJECTS_TABLE =
+            "CREATE TABLE IF NOT EXISTS `projects` (\n" +
+            "    `name` VARCHAR NOT NULL DEFAULT '',\n" +
+            "    `version_id` INT NOT NULL DEFAULT 0,\n" +
+            "    `last_accessed` DATETIME NULL DEFAULT 0,\n" +
+            "    `swap_time` DATETIME NULL,\n" +
+            "    `restore_time` DATETIME NULL,\n" +
+            "    `swap_compression` VARCHAR NULL,\n" +
+            "    PRIMARY KEY (`name`)\n" +
+            ")";
+
+    @Override
+    public String getSQL() {
+        return CREATE_PROJECTS_TABLE;
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateURLIndexStoreSQLUpdate.java

@@ -0,0 +1,28 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
+
+/**
+ * Created by Winston on 20/11/14.
+ */
+public class CreateURLIndexStoreSQLUpdate implements SQLUpdate {
+
+    private static final String CREATE_URL_INDEX_STORE =
+            "CREATE TABLE IF NOT EXISTS `url_index_store` (\n"+
+            "  `project_name` varchar(10) NOT NULL DEFAULT '',\n"+
+            "  `url` text NOT NULL,\n"+
+            "  `path` text NOT NULL,\n"+
+            "  PRIMARY KEY (`project_name`,`url`),\n"+
+            "  CONSTRAINT `url_index_store_ibfk_1` " +
+                    "FOREIGN KEY (`project_name`) " +
+                    "REFERENCES `projects` (`name`) " +
+                    "ON DELETE CASCADE " +
+                    "ON UPDATE CASCADE\n"+
+            ");\n";
+
+    @Override
+    public String getSQL() {
+        return CREATE_URL_INDEX_STORE;
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/delete/DeleteFilesForProjectSQLUpdate.java

@@ -0,0 +1,53 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.delete;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+/**
+ * Created by Winston on 20/11/14.
+ */
+public class DeleteFilesForProjectSQLUpdate implements SQLUpdate {
+
+    private static final String DELETE_URL_INDEXES_FOR_PROJECT_NAME =
+            "DELETE FROM `url_index_store` " +
+                    "WHERE `project_name` = ? AND path IN (";
+
+    private final String projectName;
+    private final String[] paths;
+
+    public DeleteFilesForProjectSQLUpdate(
+            String projectName,
+            String... paths
+    ) {
+        this.projectName = projectName;
+        this.paths = paths;
+    }
+
+    @Override
+    public String getSQL() {
+        StringBuilder sb = new StringBuilder(
+                DELETE_URL_INDEXES_FOR_PROJECT_NAME
+        );
+        for (int i = 0; i < paths.length; i++) {
+            sb.append("?");
+            if (i < paths.length - 1) {
+                sb.append(", ");
+            }
+        }
+        sb.append(");\n");
+        return sb.toString();
+    }
+
+    @Override
+    public void addParametersToStatement(
+            PreparedStatement statement
+    ) throws SQLException {
+        statement.setString(1, projectName);
+        for (int i = 0; i < paths.length; i++) {
+            statement.setString(i + 2, paths[i]);
+        }
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/AddURLIndexSQLUpdate.java

@@ -0,0 +1,45 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+/**
+ * Created by Winston on 20/11/14.
+ */
+public class AddURLIndexSQLUpdate implements SQLUpdate {
+
+    private static final String ADD_URL_INDEX =
+            "INSERT OR REPLACE INTO `url_index_store`(" +
+                    "`project_name`, " +
+                    "`url`, " +
+                    "`path`" +
+            ") VALUES " +
+            "(?, ?, ?)\n";
+
+    private final String projectName;
+    private final String url;
+    private final String path;
+
+    public AddURLIndexSQLUpdate(String projectName, String url, String path) {
+        this.projectName = projectName;
+        this.url = url;
+        this.path = path;
+    }
+
+    @Override
+    public String getSQL() {
+        return ADD_URL_INDEX;
+    }
+
+    @Override
+    public void addParametersToStatement(
+            PreparedStatement statement
+    ) throws SQLException {
+        statement.setString(1, projectName);
+        statement.setString(2, url);
+        statement.setString(3, path);
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/SetProjectLastAccessedTime.java

@@ -0,0 +1,43 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+
+/**
+ * Created by winston on 23/08/2016.
+ */
+public class SetProjectLastAccessedTime implements SQLUpdate {
+
+    private static final String SET_PROJECT_LAST_ACCESSED_TIME =
+            "UPDATE `projects`\n" +
+            "SET `last_accessed` = ?\n" +
+            "WHERE `name` = ?";
+
+    private final String projectName;
+    private final Timestamp lastAccessed;
+
+    public SetProjectLastAccessedTime(
+            String projectName,
+            Timestamp lastAccessed
+    ) {
+        this.projectName = projectName;
+        this.lastAccessed = lastAccessed;
+    }
+
+    @Override
+    public String getSQL() {
+        return SET_PROJECT_LAST_ACCESSED_TIME;
+    }
+
+    @Override
+    public void addParametersToStatement(
+            PreparedStatement statement
+    ) throws SQLException {
+        statement.setTimestamp(1, lastAccessed);
+        statement.setString(2, projectName);
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/SetProjectSQLUpdate.java

@@ -0,0 +1,39 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+/**
+ * Created by Winston on 20/11/14.
+ */
+public class SetProjectSQLUpdate implements SQLUpdate {
+
+    private static final String SET_PROJECT =
+            "INSERT OR REPLACE "
+                    + "INTO `projects`(`name`, `version_id`, `last_accessed`) "
+                    + "VALUES (?, ?, DATETIME('now'));\n";
+
+    private final String projectName;
+    private final int versionID;
+
+    public SetProjectSQLUpdate(String projectName, int versionID) {
+        this.projectName = projectName;
+        this.versionID = versionID;
+    }
+
+    @Override
+    public String getSQL() {
+        return SET_PROJECT;
+    }
+
+    @Override
+    public void addParametersToStatement(
+            PreparedStatement statement
+    ) throws SQLException {
+        statement.setString(1, projectName);
+        statement.setInt(2, versionID);
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/UpdateRestore.java

@@ -0,0 +1,40 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.time.LocalDateTime;
+
+public class UpdateRestore implements SQLUpdate {
+  private static final String UPDATE_RESTORE =
+    "UPDATE `projects`\n" +
+      "SET `last_accessed` = ?,\n" +
+      "    `swap_time` = NULL,\n" +
+      "    `restore_time` = ?,\n" +
+      "    `swap_compression` = NULL\n" +
+      "WHERE `name` = ?;\n";
+
+  private final String projectName;
+  private final Timestamp now;
+
+  public UpdateRestore(String projectName) {
+    this.projectName = projectName;
+    this.now = Timestamp.valueOf(LocalDateTime.now());
+  }
+
+  @Override
+  public String getSQL() {
+    return UPDATE_RESTORE;
+  }
+
+  @Override
+  public void addParametersToStatement(
+    PreparedStatement statement
+  ) throws SQLException {
+    statement.setTimestamp(1, now);
+    statement.setTimestamp(2, now);
+    statement.setString(3, projectName);
+  }
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/UpdateSwap.java

@@ -0,0 +1,42 @@
+package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert;
+
+import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.time.LocalDateTime;
+
+public class UpdateSwap implements SQLUpdate {
+  private static final String UPDATE_SWAP =
+    "UPDATE `projects`\n" +
+      "SET `last_accessed` = NULL,\n" +
+      "    `swap_time` = ?,\n" +
+      "    `restore_time` = NULL,\n" +
+      "    `swap_compression` = ?\n" +
+      "WHERE `name` = ?;\n";
+
+  private final String projectName;
+  private final String compression;
+  private final Timestamp now;
+
+  public UpdateSwap(String projectName, String compression) {
+    this.projectName = projectName;
+    this.compression = compression;
+    this.now = Timestamp.valueOf(LocalDateTime.now());
+  }
+
+  @Override
+  public String getSQL() {
+    return UPDATE_SWAP;
+  }
+
+  @Override
+  public void addParametersToStatement(
+    PreparedStatement statement
+  ) throws SQLException {
+    statement.setTimestamp(1, now);
+    statement.setString(2, compression);
+    statement.setString(3, projectName);
+  }
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/gc/GcJob.java

@@ -0,0 +1,34 @@
+package uk.ac.ic.wlgitbridge.bridge.gc;
+
+import uk.ac.ic.wlgitbridge.bridge.Bridge;
+import uk.ac.ic.wlgitbridge.bridge.repo.ProjectRepo;
+import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
+
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+
+/**
+ * Is started by the bridge. Every time a project is updated, we queue it for
+ * GC which executes every hour or so.
+ *
+ * We don't queue it into a more immediate Executor because there is no way to
+ * know if a call to {@link Bridge#updateProject(Optional, ProjectRepo)},
+ * which releases the lock, is going to call
+ * {@link Bridge#push(Optional, String, RawDirectory, RawDirectory, String)}.
+ *
+ * We don't want the GC to run in between an update and a push.
+ */
+public interface GcJob {
+
+    void start();
+
+    void stop();
+
+    void onPreGc(Runnable preGc);
+
+    void onPostGc(Runnable postGc);
+
+    void queueForGc(String projectName);
+
+    CompletableFuture<Void> waitForRun();
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/gc/GcJobImpl.java

@@ -0,0 +1,141 @@
+package uk.ac.ic.wlgitbridge.bridge.gc;
+
+import uk.ac.ic.wlgitbridge.bridge.lock.LockGuard;
+import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
+import uk.ac.ic.wlgitbridge.bridge.repo.ProjectRepo;
+import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
+import uk.ac.ic.wlgitbridge.util.Log;
+import uk.ac.ic.wlgitbridge.util.TimerUtils;
+
+import java.io.IOException;
+import java.util.*;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * Implementation of {@link GcJob} using its own Timer and a synchronized
+ * queue.
+ */
+public class GcJobImpl implements GcJob {
+
+    private final RepoStore repoStore;
+    private final ProjectLock locks;
+
+    private final long intervalMs;
+    private final Timer timer;
+
+    private final Set<String> gcQueue;
+
+    /**
+     * Hooks in case they are needed, e.g. for testing.
+     */
+    private AtomicReference<Runnable> preGc;
+    private AtomicReference<Runnable> postGc;
+
+    /* We need to iterate over and empty it after every run */
+    private final Lock jobWaitersLock;
+    private final List<CompletableFuture<Void>> jobWaiters;
+
+    public GcJobImpl(RepoStore repoStore, ProjectLock locks, long intervalMs) {
+        this.repoStore = repoStore;
+        this.locks = locks;
+        this.intervalMs = intervalMs;
+        timer = new Timer();
+        gcQueue = Collections.newSetFromMap(new ConcurrentHashMap<>());
+        preGc = new AtomicReference<>(() -> {});
+        postGc = new AtomicReference<>(() -> {});
+        jobWaitersLock = new ReentrantLock();
+        jobWaiters = new ArrayList<>();
+    }
+
+    public GcJobImpl(RepoStore repoStore, ProjectLock locks) {
+        this(
+                repoStore,
+                locks,
+                TimeUnit.MILLISECONDS.convert(1, TimeUnit.HOURS)
+        );
+    }
+
+    @Override
+    public void start() {
+        Log.info("Starting GC job to run every [{}] ms", intervalMs);
+        timer.scheduleAtFixedRate(
+                TimerUtils.makeTimerTask(this::doGC),
+                intervalMs,
+                intervalMs
+        );
+    }
+
+    @Override
+    public void stop() {
+        Log.info("Stopping GC job");
+        timer.cancel();
+    }
+
+    @Override
+    public void onPreGc(Runnable preGc) {
+        this.preGc.set(preGc);
+    }
+
+    @Override
+    public void onPostGc(Runnable postGc) {
+        this.postGc.set(postGc);
+    }
+
+    /**
+     * Needs to be callable from any thread.
+     * @param projectName
+     */
+    @Override
+    public void queueForGc(String projectName) {
+        gcQueue.add(projectName);
+    }
+
+    @Override
+    public CompletableFuture<Void> waitForRun() {
+        CompletableFuture<Void> ret = new CompletableFuture<>();
+        jobWaitersLock.lock();
+        try {
+            jobWaiters.add(ret);
+        } finally {
+            jobWaitersLock.unlock();
+        }
+        return ret;
+    }
+
+    private void doGC() {
+        Log.info("GC job running");
+        int numGcs = 0;
+        preGc.get().run();
+        for (
+                Iterator<String> it = gcQueue.iterator();
+                it.hasNext();
+                it.remove(), ++numGcs
+        ) {
+            String proj = it.next();
+            Log.info("[{}] Running GC job on project", proj);
+            try (LockGuard __ = locks.lockGuard(proj)) {
+                try {
+                    ProjectRepo repo = repoStore.getExistingRepo(proj);
+                    repo.runGC();
+                    repo.deleteIncomingPacks();
+                } catch (IOException e) {
+                    Log.info("[{}] Failed to GC project", proj);
+                }
+            }
+        }
+        Log.info("GC job finished, num gcs: {}", numGcs);
+        jobWaitersLock.lock();
+        try {
+            jobWaiters.forEach(w -> w.complete(null));
+        } finally {
+            jobWaitersLock.unlock();
+        }
+        postGc.get().run();
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/lock/LockGuard.java

@@ -0,0 +1,10 @@
+package uk.ac.ic.wlgitbridge.bridge.lock;
+
+/**
+ * Created by winston on 24/08/2016.
+ */
+public interface LockGuard extends AutoCloseable {
+
+    void close();
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/lock/ProjectLock.java

@@ -0,0 +1,23 @@
+package uk.ac.ic.wlgitbridge.bridge.lock;
+
+/**
+ * Project Lock class.
+ *
+ * The locks should be re-entrant. For example, we are usually holding the lock
+ * when a project must be restored, which tries to acquire the lock again.
+ */
+public interface ProjectLock {
+
+    void lockAll();
+
+    void lockForProject(String projectName);
+
+    void unlockForProject(String projectName);
+
+    /* RAII hahaha */
+    default LockGuard lockGuard(String projectName) {
+        lockForProject(projectName);
+        return () -> unlockForProject(projectName);
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStore.java

@@ -0,0 +1,238 @@
+package uk.ac.ic.wlgitbridge.bridge.repo;
+
+import com.google.api.client.repackaged.com.google.common.base.Preconditions;
+import org.apache.commons.io.FileUtils;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.Repository;
+import uk.ac.ic.wlgitbridge.util.Log;
+import uk.ac.ic.wlgitbridge.util.Project;
+import uk.ac.ic.wlgitbridge.util.Tar;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Function;
+
+import static uk.ac.ic.wlgitbridge.util.Util.deleteInDirectoryApartFrom;
+
+/**
+ * Created by winston on 20/08/2016.
+ */
+public class FSGitRepoStore implements RepoStore {
+
+    private static final long DEFAULT_MAX_FILE_SIZE = 50 * 1024 * 1024;
+
+    private final String repoStorePath;
+
+    private final File rootDirectory;
+
+    private final long maxFileSize;
+
+    private final Function<File, Long> fsSizer;
+
+    public FSGitRepoStore(
+            String repoStorePath,
+            Optional<Long> maxFileSize
+    ) {
+        this(
+                repoStorePath,
+                maxFileSize.orElse(DEFAULT_MAX_FILE_SIZE),
+                d -> d.getTotalSpace() - d.getFreeSpace()
+        );
+    }
+
+    public FSGitRepoStore(
+            String repoStorePath,
+            long maxFileSize,
+            Function<File, Long> fsSizer
+    ) {
+        this.repoStorePath = repoStorePath;
+        rootDirectory = initRootGitDirectory(repoStorePath);
+        this.maxFileSize = maxFileSize;
+        this.fsSizer = fsSizer;
+    }
+
+    @Override
+    public String getRepoStorePath() {
+        return repoStorePath;
+    }
+
+    @Override
+    public File getRootDirectory() {
+        return rootDirectory;
+    }
+
+    @Override
+    public ProjectRepo initRepo(String project) throws IOException {
+        GitProjectRepo ret = GitProjectRepo.fromName(project);
+        ret.initRepo(this);
+        return new WalkOverrideGitRepo(
+                ret, Optional.of(maxFileSize), Optional.empty());
+    }
+
+     @Override
+     public ProjectRepo initRepoFromExisting(
+             String project, String fromProject
+     ) throws IOException {
+         String repoRoot = getRepoStorePath();
+         String sourcePath = repoRoot + "/" + fromProject;
+         String destinationPath = repoRoot + "/" + project;
+         Log.info("[{}] Init repo by copying data from: {}, to: {}",
+             project,
+             sourcePath,
+             destinationPath
+         );
+         File source = new File(sourcePath);
+         File destination = new File(destinationPath);
+         FileUtils.copyDirectory(source, destination);
+         GitProjectRepo ret = GitProjectRepo.fromName(project);
+         ret.useExistingRepository(this);
+         return new WalkOverrideGitRepo(
+                 ret, Optional.of(maxFileSize), Optional.empty());
+     }
+
+    @Override
+    public ProjectRepo getExistingRepo(String project) throws IOException {
+        GitProjectRepo ret = GitProjectRepo.fromName(project);
+        ret.useExistingRepository(this);
+        return new WalkOverrideGitRepo(
+                ret, Optional.of(maxFileSize), Optional.empty());
+    }
+
+    @Override
+    public ProjectRepo useJGitRepo(Repository repo, ObjectId commitId) {
+        GitProjectRepo ret = GitProjectRepo.fromJGitRepo(repo);
+        return new WalkOverrideGitRepo(
+                ret, Optional.of(maxFileSize), Optional.of(commitId));
+    }
+
+    /* TODO: Perhaps we should just delete bad directories on the fly. */
+    @Override
+    public void purgeNonexistentProjects(
+            Collection<String> existingProjectNames
+    ) {
+        List<String> excludedFromDeletion =
+                new ArrayList<>(existingProjectNames);
+        excludedFromDeletion.add(".wlgb");
+        deleteInDirectoryApartFrom(
+                rootDirectory,
+                excludedFromDeletion.toArray(new String[] {})
+        );
+    }
+
+    @Override
+    public long totalSize() {
+        return fsSizer.apply(rootDirectory);
+    }
+
+    @Override
+    public InputStream bzip2Project(
+            String projectName,
+            long[] sizePtr
+    ) throws IOException {
+        Project.checkValidProjectName(projectName);
+        Log.info("[{}] bzip2 project", projectName);
+        return Tar.bz2.zip(getDotGitForProject(projectName), sizePtr);
+    }
+
+    @Override
+    public InputStream gzipProject(
+        String projectName,
+        long[] sizePtr
+    ) throws IOException {
+        Project.checkValidProjectName(projectName);
+        Log.info("[{}] gzip project", projectName);
+        return Tar.gzip.zip(getDotGitForProject(projectName), sizePtr);
+    }
+
+    @Override
+    public void gcProject(String projectName) throws IOException {
+        Project.checkValidProjectName(projectName);
+        ProjectRepo repo = getExistingRepo(projectName);
+        repo.runGC();
+    }
+
+    @Override
+    public void remove(String projectName) throws IOException {
+        Project.checkValidProjectName(projectName);
+        FileUtils.deleteDirectory(new File(rootDirectory, projectName));
+    }
+
+    @Override
+    public void unbzip2Project(
+            String projectName,
+            InputStream dataStream
+    ) throws IOException {
+        Preconditions.checkArgument(
+                Project.isValidProjectName(projectName),
+                "[%s] invalid project name: ",
+                projectName
+        );
+        Preconditions.checkState(
+                getDirForProject(projectName).mkdirs(),
+                "[%s] directories for " +
+                        "evicted project already exist",
+                projectName
+        );
+        Log.info("[{}] un-bzip2 project", projectName);
+        Tar.bz2.unzip(dataStream, getDirForProject(projectName));
+    }
+
+    @Override
+    public void ungzipProject(
+      String projectName,
+      InputStream dataStream
+    ) throws IOException {
+        Preconditions.checkArgument(
+            Project.isValidProjectName(projectName),
+            "[%s] invalid project name: ",
+            projectName
+        );
+        Preconditions.checkState(
+            getDirForProject(projectName).mkdirs(),
+          "[%s] directories for " +
+            "evicted project already exist",
+            projectName
+        );
+        Log.info("[{}] un-gzip project", projectName);
+        Tar.gzip.unzip(dataStream, getDirForProject(projectName));
+    }
+
+    private File getDirForProject(String projectName) {
+        Project.checkValidProjectName(projectName);
+        return Paths.get(
+                rootDirectory.getAbsolutePath()
+        ).resolve(
+                projectName
+        ).toFile();
+    }
+
+    private File getDotGitForProject(String projectName) {
+        Project.checkValidProjectName(projectName);
+        return Paths.get(
+                rootDirectory.getAbsolutePath()
+        ).resolve(
+                projectName
+        ).resolve(
+                ".git"
+        ).toFile();
+    }
+
+    private File initRootGitDirectory(String rootGitDirectoryPath) {
+        File rootGitDirectory = new File(rootGitDirectoryPath);
+        rootGitDirectory.mkdirs();
+        Preconditions.checkArgument(
+                rootGitDirectory.isDirectory(),
+                "given root git directory " +
+                        "is not a directory: %s",
+                rootGitDirectory.getAbsolutePath()
+        );
+        return rootGitDirectory;
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/GitProjectRepo.java

@@ -0,0 +1,280 @@
+package uk.ac.ic.wlgitbridge.bridge.repo;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.io.IOUtils;
+import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.api.ResetCommand;
+import org.eclipse.jgit.api.errors.GitAPIException;
+import org.eclipse.jgit.lib.PersonIdent;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.storage.file.FileRepositoryBuilder;
+import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
+import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
+import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
+import uk.ac.ic.wlgitbridge.git.util.RepositoryObjectTreeWalker;
+import uk.ac.ic.wlgitbridge.util.Log;
+import uk.ac.ic.wlgitbridge.util.Project;
+import uk.ac.ic.wlgitbridge.util.Util;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileVisitResult;
+import java.nio.file.FileVisitor;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.*;
+
+/**
+ * Class representing a Git repository.
+ *
+ * It stores the projectName and repo separately because the hooks need to be
+ * able to construct one of these without knowing whether the repo exists yet.
+ *
+ * It can then be passed to the Bridge, which will either
+ * {@link #initRepo(RepoStore)} for a never-seen-before repo, or
+ * {@link #useExistingRepository(RepoStore)} for an existing repo.
+ *
+ * Make sure to acquire the project lock before calling methods here.
+ */
+public class GitProjectRepo implements ProjectRepo {
+
+    private final String projectName;
+    private Optional<Repository> repository;
+
+    public static GitProjectRepo fromJGitRepo(Repository repo) {
+        return new GitProjectRepo(
+                repo.getWorkTree().getName(), Optional.of(repo));
+    }
+
+    public static GitProjectRepo fromName(String projectName) {
+        return new GitProjectRepo(projectName, Optional.empty());
+    }
+
+    GitProjectRepo(String projectName, Optional<Repository> repository) {
+        Preconditions.checkArgument(Project.isValidProjectName(projectName));
+        this.projectName = projectName;
+        this.repository = repository;
+    }
+
+    @Override
+    public String getProjectName() {
+        return projectName;
+    }
+
+    @Override
+    public void initRepo(
+            RepoStore repoStore
+    ) throws IOException {
+        initRepositoryField(repoStore);
+        Preconditions.checkState(repository.isPresent());
+        Repository repo = this.repository.get();
+        // TODO: assert that this is a fresh repo. At the moment, we can't be
+        // sure whether the repo to be init'd doesn't exist or is just fresh
+        // and we crashed / aborted while committing
+        if (repo.getObjectDatabase().exists()) return;
+        repo.create();
+    }
+
+    @Override
+    public void useExistingRepository(
+            RepoStore repoStore
+    ) throws IOException {
+        initRepositoryField(repoStore);
+        Preconditions.checkState(repository.isPresent());
+        Preconditions.checkState(
+                repository.get().getObjectDatabase().exists()
+        );
+    }
+
+    @Override
+    public RawDirectory getDirectory()
+            throws IOException, GitUserException {
+        Preconditions.checkState(repository.isPresent());
+        return new RepositoryObjectTreeWalker(
+                repository.get()
+        ).getDirectoryContents(Optional.empty());
+    }
+
+    @Override
+    public Collection<String> commitAndGetMissing(
+            GitDirectoryContents contents
+    ) throws IOException {
+        try {
+            return doCommitAndGetMissing(contents);
+        } catch (GitAPIException e) {
+            throw new IOException(e);
+        }
+    }
+
+    @Override
+    public void runGC() throws IOException {
+        Preconditions.checkState(
+                repository.isPresent(),
+                "Repo is not present"
+        );
+        File dir = getProjectDir();
+        Preconditions.checkState(dir.isDirectory());
+        Log.info("[{}] Running git gc", projectName);
+        Process proc = new ProcessBuilder(
+                "git", "gc"
+        ).directory(dir).start();
+        int exitCode;
+        try {
+            exitCode = proc.waitFor();
+            Log.info("Exit: {}", exitCode);
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        }
+        if (exitCode != 0) {
+            Log.warn("[{}] Git gc failed", dir.getAbsolutePath());
+            Log.warn(IOUtils.toString(
+                    proc.getInputStream(),
+                    StandardCharsets.UTF_8
+            ));
+            Log.warn(IOUtils.toString(
+                    proc.getErrorStream(),
+                    StandardCharsets.UTF_8
+            ));
+            throw new IOException("git gc error");
+        }
+        Log.info("[{}] git gc successful", projectName);
+    }
+
+    @Override
+    public void deleteIncomingPacks() throws IOException {
+        Log.info(
+                "[{}] Checking for garbage `incoming` files",
+                projectName
+        );
+        Files.walkFileTree(getDotGitDir().toPath(), new FileVisitor<Path>() {
+            @Override
+            public FileVisitResult preVisitDirectory(
+                    Path dir,
+                    BasicFileAttributes attrs
+            ) throws IOException {
+                return FileVisitResult.CONTINUE;
+            }
+
+            @Override
+            public FileVisitResult visitFile(
+                    Path file,
+                    BasicFileAttributes attrs
+            ) throws IOException {
+                File file_ = file.toFile();
+                String name = file_.getName();
+                if (name.startsWith("incoming_") && name.endsWith(".pack")) {
+                    Log.info("Deleting garbage `incoming` file: {}", file_);
+                    Preconditions.checkState(file_.delete());
+                }
+                return FileVisitResult.CONTINUE;
+            }
+
+            @Override
+            public FileVisitResult visitFileFailed(
+                    Path file,
+                    IOException exc
+            ) throws IOException {
+                Preconditions.checkNotNull(file);
+                Preconditions.checkNotNull(exc);
+                Log.warn("Failed to visit file: " + file, exc);
+                return FileVisitResult.TERMINATE;
+            }
+
+            @Override
+            public FileVisitResult postVisitDirectory(
+                    Path dir,
+                    IOException exc
+            ) throws IOException {
+                Preconditions.checkNotNull(dir);
+                if (exc != null) {
+                    return FileVisitResult.TERMINATE;
+                }
+                return FileVisitResult.CONTINUE;
+            }
+
+        });
+    }
+
+    @Override
+    public File getProjectDir() {
+        return getJGitRepository().getDirectory().getParentFile();
+    }
+
+    public void resetHard() throws IOException {
+        Git git = new Git(getJGitRepository());
+        try {
+            git.reset().setMode(ResetCommand.ResetType.HARD).call();
+        } catch (GitAPIException e) {
+            throw new IOException(e);
+        }
+    }
+
+    @Override
+    public Repository getJGitRepository() {
+        return repository.get();
+    }
+
+    public File getDotGitDir() {
+        return getJGitRepository().getWorkTree();
+    }
+
+    private void initRepositoryField(RepoStore repoStore) throws IOException {
+        Preconditions.checkNotNull(repoStore);
+        Preconditions.checkArgument(Project.isValidProjectName(projectName));
+        Preconditions.checkState(!repository.isPresent());
+        repository = Optional.of(createJGitRepository(repoStore, projectName));
+    }
+
+    private Repository createJGitRepository(
+            RepoStore repoStore,
+            String projName
+    ) throws IOException {
+        File repoDir = new File(repoStore.getRootDirectory(), projName);
+        return new FileRepositoryBuilder().setWorkTree(repoDir).build();
+    }
+
+    private Collection<String> doCommitAndGetMissing(
+            GitDirectoryContents contents
+    ) throws IOException, GitAPIException {
+        Preconditions.checkState(repository.isPresent());
+        Repository repo = getJGitRepository();
+        resetHard();
+        String name = getProjectName();
+        Log.info("[{}] Writing commit", name);
+        contents.write();
+        Git git = new Git(getJGitRepository());
+        Log.info("[{}] Getting missing files", name);
+        Set<String> missingFiles = git.status().call().getMissing();
+        for (String missing : missingFiles) {
+            Log.info("[{}] Git rm {}", name, missing);
+            git.rm().setCached(true).addFilepattern(missing).call();
+        }
+        Log.info("[{}] Calling Git add", name);
+        git.add(
+        ).setWorkingTreeIterator(
+                new NoGitignoreIterator(repo)
+        ).addFilepattern(".").call();
+        Log.info("[{}] Calling Git commit", name);
+        git.commit(
+        ).setAuthor(
+                new PersonIdent(
+                        contents.getUserName(),
+                        contents.getUserEmail(),
+                        contents.getWhen(),
+                        TimeZone.getDefault()
+                )
+        ).setMessage(
+                contents.getCommitMessage()
+        ).call();
+        Log.info(
+                "[{}] Deleting files in directory: {}",
+                name,
+                contents.getDirectory().getAbsolutePath()
+        );
+        Util.deleteInDirectoryApartFrom(contents.getDirectory(), ".git");
+        return missingFiles;
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/NoGitignoreIterator.java

@@ -0,0 +1,87 @@
+package uk.ac.ic.wlgitbridge.bridge.repo;
+
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.treewalk.AbstractTreeIterator;
+import org.eclipse.jgit.treewalk.FileTreeIterator;
+import org.eclipse.jgit.treewalk.WorkingTreeIterator;
+import org.eclipse.jgit.treewalk.WorkingTreeOptions;
+import org.eclipse.jgit.util.FS;
+
+import java.io.File;
+import java.lang.reflect.Field;
+
+/**
+ * Created by winston on 08/10/2016.
+ */
+public class NoGitignoreIterator extends FileTreeIterator {
+
+    private static final Field ignoreNodeField;
+
+    static {
+        try {
+            ignoreNodeField = WorkingTreeIterator.class.getDeclaredField(
+                    "ignoreNode"
+            );
+        } catch (NoSuchFieldException e) {
+            throw new RuntimeException(e);
+        }
+        ignoreNodeField.setAccessible(true);
+    }
+
+    public NoGitignoreIterator(Repository repo) {
+        super(repo);
+    }
+
+    public NoGitignoreIterator(
+            Repository repo,
+            FileModeStrategy fileModeStrategy
+    ) {
+        super(repo, fileModeStrategy);
+    }
+
+    public NoGitignoreIterator(File root, FS fs, WorkingTreeOptions options) {
+        super(root, fs, options);
+    }
+
+    public NoGitignoreIterator(
+            File root,
+            FS fs,
+            WorkingTreeOptions options,
+            FileModeStrategy fileModeStrategy
+    ) {
+        super(root, fs, options, fileModeStrategy);
+    }
+
+    protected NoGitignoreIterator(FileTreeIterator p, File root, FS fs) {
+        super(p, root, fs);
+    }
+
+    protected NoGitignoreIterator(
+            WorkingTreeIterator p,
+            File root,
+            FS fs,
+            FileModeStrategy fileModeStrategy
+    ) {
+        super(p, root, fs, fileModeStrategy);
+    }
+
+    // Note: the `list` is a list of top-level entities in this directory,
+    //       not a full list of files in the tree.
+    @Override
+    protected void init(Entry[] list) {
+        super.init(list);
+        try {
+            ignoreNodeField.set(this, null);
+        } catch (IllegalAccessException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    // When entering a sub-directory, create a new instance of this class,
+    // so we can also ignore gitignore specifications in sub-directories
+    @Override
+    protected AbstractTreeIterator enterSubtree() {
+      String fullPath = getDirectory().getAbsolutePath() + "/" + current().getName();
+      return new NoGitignoreIterator(this, new File(fullPath), fs, fileModeStrategy);
+    }
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/ProjectRepo.java

@@ -0,0 +1,41 @@
+package uk.ac.ic.wlgitbridge.bridge.repo;
+
+import org.eclipse.jgit.lib.Repository;
+import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
+import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
+import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+
+/**
+ * Created by winston on 20/08/2016.
+ */
+public interface ProjectRepo {
+
+    String getProjectName();
+
+    void initRepo(
+            RepoStore repoStore
+    ) throws IOException;
+
+    void useExistingRepository(
+            RepoStore repoStore
+    ) throws IOException;
+
+    RawDirectory getDirectory(
+    ) throws IOException, GitUserException;
+
+    Collection<String> commitAndGetMissing(
+            GitDirectoryContents gitDirectoryContents
+    ) throws IOException, GitUserException;
+
+    void runGC() throws IOException;
+
+    void deleteIncomingPacks() throws IOException;
+
+    File getProjectDir();
+
+    Repository getJGitRepository();
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java

@@ -0,0 +1,104 @@
+package uk.ac.ic.wlgitbridge.bridge.repo;
+
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.Repository;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collection;
+
+/**
+ * Created by winston on 20/08/2016.
+ */
+public interface RepoStore {
+
+    /* Still need to get rid of these two methods.
+       Main dependency: GitRepoStore needs a Repository which needs a directory.
+       Instead, use a visitor or something. */
+    String getRepoStorePath();
+
+    File getRootDirectory();
+
+    ProjectRepo initRepo(String project) throws IOException;
+
+    ProjectRepo initRepoFromExisting(String project, String fromProject) throws IOException;
+
+    ProjectRepo getExistingRepo(String project) throws IOException;
+
+    ProjectRepo useJGitRepo(Repository repo, ObjectId commitId);
+
+    void purgeNonexistentProjects(
+            Collection<String> existingProjectNames
+    );
+
+    long totalSize();
+
+    /**
+     * Tars and bzip2s the .git directory of the given project. Throws an
+     * IOException if the project doesn't exist. The returned stream is a copy
+     * of the original .git directory, which must be deleted using remove().
+     */
+    InputStream bzip2Project(
+            String projectName,
+            long[] sizePtr
+    ) throws IOException;
+
+    default InputStream bzip2Project(
+            String projectName
+    ) throws IOException {
+        return bzip2Project(projectName, null);
+    }
+
+    /**
+     * Tars and gzips the .git directory of the given project. Throws an
+     * IOException if the project doesn't exist. The returned stream is a copy
+     * of the original .git directory, which must be deleted using remove().
+     */
+    InputStream gzipProject(
+        String projectName,
+        long[] sizePtr
+    ) throws IOException;
+
+    default InputStream gzipProject(
+        String projectName
+    ) throws IOException {
+        return gzipProject(projectName, null);
+    }
+
+    void gcProject(String projectName) throws IOException;
+
+    /**
+     * Called after {@link #bzip2Project(String, long[])}'s has been safely
+     * uploaded to the swap store. Removes all traces of the project from disk,
+     * i.e. not just its .git, but the whole project's git directory.
+     * @param projectName
+     * @throws IOException
+     */
+    void remove(String projectName) throws IOException;
+
+    /**
+     * Unbzip2s the given data stream into a .git directory for projectName.
+     * Creates the project's git directory.
+     * If projectName already exists, throws an IOException.
+     * @param projectName the name of the project, e.g. abc123
+     * @param dataStream the data stream containing the bzipped contents.
+     */
+    void unbzip2Project(
+            String projectName,
+            InputStream dataStream
+    ) throws IOException;
+
+    /**
+     * Ungzips the given data stream into a .git directory for projectName.
+     * Creates the project's git directory.
+     * If projectName already exists, throws an IOException.
+     * @param projectName the name of the project, e.g. abc123
+     * @param dataStream the data stream containing the gzip contents.
+     */
+    void ungzipProject(
+        String projectName,
+        InputStream dataStream
+    ) throws IOException;
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStoreConfig.java

@@ -0,0 +1,29 @@
+package uk.ac.ic.wlgitbridge.bridge.repo;
+
+import javax.annotation.Nullable;
+import java.util.Optional;
+
+/**
+ * Created by winston on 02/07/2017.
+ */
+public class RepoStoreConfig {
+
+    @Nullable
+    private final Long maxFileSize;
+
+    @Nullable
+    private final Long maxFileNum;
+
+    public RepoStoreConfig(Long maxFileSize, Long maxFileNum) {
+        this.maxFileSize = maxFileSize;
+        this.maxFileNum = maxFileNum;
+    }
+
+    public Optional<Long> getMaxFileSize() {
+        return Optional.ofNullable(maxFileSize);
+    }
+
+    public Optional<Long> getMaxFileNum() {
+        return Optional.ofNullable(maxFileNum);
+    }
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/WalkOverrideGitRepo.java

@@ -0,0 +1,95 @@
+package uk.ac.ic.wlgitbridge.bridge.repo;
+
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.Repository;
+import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
+import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
+import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
+import uk.ac.ic.wlgitbridge.git.util.RepositoryObjectTreeWalker;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Optional;
+
+/**
+ * This class takes a GitProjectRepo and delegates all calls to it.
+ *
+ * The purpose is to insert a file size check in {@link #getDirectory()}.
+ *
+ * We delegate instead of subclass because we can't override the static
+ * constructors in {@link GitProjectRepo}.
+ */
+public class WalkOverrideGitRepo implements ProjectRepo {
+
+    private final GitProjectRepo gitRepo;
+
+    private final Optional<Long> maxFileSize;
+
+    private final Optional<ObjectId> commitId;
+
+    public WalkOverrideGitRepo(
+            GitProjectRepo gitRepo,
+            Optional<Long> maxFileSize,
+            Optional<ObjectId> commitId
+    ) {
+        this.gitRepo = gitRepo;
+        this.maxFileSize = maxFileSize;
+        this.commitId = commitId;
+    }
+
+    @Override
+    public String getProjectName() {
+        return gitRepo.getProjectName();
+    }
+
+    @Override
+    public void initRepo(RepoStore repoStore) throws IOException {
+        gitRepo.initRepo(repoStore);
+    }
+
+    @Override
+    public void useExistingRepository(RepoStore repoStore) throws IOException {
+        gitRepo.useExistingRepository(repoStore);
+    }
+
+    @Override
+    public RawDirectory getDirectory() throws IOException, GitUserException {
+        Repository repo = gitRepo.getJGitRepository();
+        RepositoryObjectTreeWalker walker;
+        if (commitId.isPresent()) {
+            walker = new RepositoryObjectTreeWalker(repo, commitId.get());
+        } else {
+            walker = new RepositoryObjectTreeWalker(repo);
+        }
+        return walker.getDirectoryContents(maxFileSize);
+    }
+
+    @Override
+    public Collection<String> commitAndGetMissing(
+            GitDirectoryContents gitDirectoryContents
+    ) throws GitUserException, IOException {
+        return gitRepo.commitAndGetMissing(gitDirectoryContents);
+    }
+
+    @Override
+    public void runGC() throws IOException {
+        gitRepo.runGC();
+    }
+
+    @Override
+    public void deleteIncomingPacks() throws IOException {
+        gitRepo.deleteIncomingPacks();
+    }
+
+    @Override
+    public File getProjectDir() {
+        return gitRepo.getProjectDir();
+    }
+
+    @Override
+    public Repository getJGitRepository() {
+        return gitRepo.getJGitRepository();
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/resource/ResourceCache.java

@@ -0,0 +1,24 @@
+package uk.ac.ic.wlgitbridge.bridge.resource;
+
+import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
+import uk.ac.ic.wlgitbridge.git.exception.SizeLimitExceededException;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * Created by winston on 20/08/2016.
+ */
+public interface ResourceCache {
+
+    RawFile get(
+            String projectName,
+            String url,
+            String newPath,
+            Map<String, RawFile> fileTable,
+            Map<String, byte[]> fetchedUrls,
+            Optional<Long> maxFileSize
+    ) throws IOException, SizeLimitExceededException;
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/resource/UrlResourceCache.java

@@ -0,0 +1,142 @@
+package uk.ac.ic.wlgitbridge.bridge.resource;
+
+import static org.asynchttpclient.Dsl.*;
+import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
+import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
+import uk.ac.ic.wlgitbridge.data.filestore.RepositoryFile;
+import uk.ac.ic.wlgitbridge.git.exception.SizeLimitExceededException;
+import uk.ac.ic.wlgitbridge.io.http.ning.NingHttpClient;
+import uk.ac.ic.wlgitbridge.io.http.ning.NingHttpClientFacade;
+import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
+import uk.ac.ic.wlgitbridge.util.Log;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ExecutionException;
+
+/**
+ * Created by winston on 20/08/2016.
+ */
+public class UrlResourceCache implements ResourceCache {
+
+    private final DBStore dbStore;
+
+    private final NingHttpClientFacade http;
+
+    UrlResourceCache(DBStore dbStore, NingHttpClientFacade http) {
+        this.dbStore = dbStore;
+        this.http = http;
+    }
+
+    public UrlResourceCache(DBStore dbStore) {
+        this(dbStore, new NingHttpClient(asyncHttpClient()));
+    }
+
+    @Override
+    public RawFile get(
+            String projectName,
+            String url,
+            String newPath,
+            Map<String, RawFile> fileTable,
+            Map<String, byte[]> fetchedUrls,
+            Optional<Long> maxFileSize
+    ) throws IOException, SizeLimitExceededException {
+        String path = dbStore.getPathForURLInProject(projectName, getCacheKeyFromUrl(url));
+        byte[] contents;
+        if (path == null) {
+            path = newPath;
+            contents = fetch(projectName, url, path, maxFileSize);
+            fetchedUrls.put(url, contents);
+        } else {
+            Log.info("Found (" + projectName + "): " + url);
+            Log.info("At (" + projectName + "): " + path);
+            contents = fetchedUrls.get(url);
+            if (contents == null) {
+                RawFile rawFile = fileTable.get(path);
+                if (rawFile == null) {
+                    Log.warn(
+                            "File " + path
+                                    + " was not in the current commit, "
+                                    + "or the git tree, yet path was not null. "
+                                    + "File url is: "
+                                    + url
+                    );
+                    contents = fetch(projectName, url, path, maxFileSize);
+                } else {
+                    contents = rawFile.getContents();
+                }
+            }
+        }
+        return new RepositoryFile(newPath, contents);
+    }
+
+    private byte[] fetch(
+            String projectName,
+            final String url,
+            String path,
+            Optional<Long> maxFileSize
+    ) throws FailedConnectionException, SizeLimitExceededException {
+        byte[] contents;
+        Log.info("GET -> " + url);
+        try {
+            contents = http.get(url, hs -> {
+                List<String> contentLengths = hs.getAll("Content-Length");
+                if (!maxFileSize.isPresent()) {
+                    return true;
+                }
+                if (contentLengths.isEmpty()) {
+                    return true;
+                }
+                long contentLength = Long.parseLong(contentLengths.get(0));
+                long maxFileSize_ = maxFileSize.get();
+                if (contentLength <= maxFileSize_) {
+                    return true;
+                }
+                throw new SizeLimitExceededException(
+                        Optional.of(path), contentLength, maxFileSize_
+                );
+            });
+        } catch (ExecutionException e) {
+            Throwable cause = e.getCause();
+            if (cause instanceof SizeLimitExceededException) {
+                throw (SizeLimitExceededException) cause;
+            }
+            Log.warn(
+                    "ExecutionException when fetching project: " +
+                            projectName +
+                            ", url: " +
+                            url +
+                            ", path: " +
+                            path,
+                    e
+            );
+            throw new FailedConnectionException();
+        }
+        if (maxFileSize.isPresent() && contents.length > maxFileSize.get()) {
+            throw new SizeLimitExceededException(
+                    Optional.of(path), contents.length, maxFileSize.get());
+        }
+        dbStore.addURLIndexForProject(projectName, getCacheKeyFromUrl(url), path);
+        return contents;
+    }
+
+    /**
+     * Construct a suitable cache key from the given file URL.
+     *
+     * The file URL returned by the web service may contain a token parameter
+     * used for authentication. This token changes for every request, so we
+     * need to strip it from the query string before using the URL as a cache
+     * key.
+     */
+    private String getCacheKeyFromUrl(String url) {
+        // We're not doing proper URL parsing here, but it should be enough to
+        // remove the token without touching the important parts of the URL.
+        //
+        // The URL looks like:
+        //
+        // https://history.overleaf.com/api/projects/:project_id/blobs/:hash?token=:token&_path=:path
+        return url.replaceAll("token=[^&]*", "token=REMOVED");
+    }
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/snapshot/NetSnapshotApi.java

@@ -0,0 +1,55 @@
+package uk.ac.ic.wlgitbridge.bridge.snapshot;
+
+import com.google.api.client.auth.oauth2.Credential;
+import uk.ac.ic.wlgitbridge.data.CandidateSnapshot;
+import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocRequest;
+import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocResult;
+import uk.ac.ic.wlgitbridge.snapshot.getforversion.GetForVersionRequest;
+import uk.ac.ic.wlgitbridge.snapshot.getforversion.GetForVersionResult;
+import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.GetSavedVersRequest;
+import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.GetSavedVersResult;
+import uk.ac.ic.wlgitbridge.snapshot.push.PushRequest;
+import uk.ac.ic.wlgitbridge.snapshot.push.PushResult;
+
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+
+/**
+ * Created by winston on 20/08/2016.
+ */
+public class NetSnapshotApi implements SnapshotApi {
+
+    @Override
+    public CompletableFuture<GetDocResult> getDoc(
+            Optional<Credential> oauth2, String projectName) {
+        return new GetDocRequest(opt(oauth2), projectName).request();
+    }
+
+    @Override
+    public CompletableFuture<GetForVersionResult> getForVersion(
+            Optional<Credential> oauth2, String projectName, int versionId) {
+        return new GetForVersionRequest(
+                opt(oauth2), projectName, versionId).request();
+    }
+
+    @Override
+    public CompletableFuture<GetSavedVersResult> getSavedVers(
+            Optional<Credential> oauth2, String projectName) {
+        return new GetSavedVersRequest(opt(oauth2), projectName).request();
+    }
+
+    @Override
+    public CompletableFuture<PushResult> push(
+            Optional<Credential> oauth2,
+            CandidateSnapshot candidateSnapshot,
+            String postbackKey
+    ) {
+        return new PushRequest(
+                opt(oauth2), candidateSnapshot, postbackKey).request();
+    }
+
+    private static Credential opt(Optional<Credential> oauth2) {
+        return oauth2.orElse(null);
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/snapshot/SnapshotApi.java

@@ -0,0 +1,54 @@
+package uk.ac.ic.wlgitbridge.bridge.snapshot;
+
+import com.google.api.client.auth.oauth2.Credential;
+import uk.ac.ic.wlgitbridge.data.CandidateSnapshot;
+import uk.ac.ic.wlgitbridge.snapshot.base.MissingRepositoryException;
+import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
+import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
+import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocResult;
+import uk.ac.ic.wlgitbridge.snapshot.getforversion.GetForVersionResult;
+import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.GetSavedVersResult;
+import uk.ac.ic.wlgitbridge.snapshot.push.PushResult;
+
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+
+/**
+ * Created by winston on 20/08/2016.
+ */
+public interface SnapshotApi {
+
+    CompletableFuture<GetDocResult> getDoc(
+            Optional<Credential> oauth2, String projectName);
+
+    CompletableFuture<GetForVersionResult> getForVersion(
+            Optional<Credential> oauth2, String projectName, int versionId);
+
+    CompletableFuture<GetSavedVersResult> getSavedVers(
+            Optional<Credential> oauth2, String projectName);
+
+    CompletableFuture<PushResult> push(
+            Optional<Credential> oauth2,
+            CandidateSnapshot candidateSnapshot,
+            String postbackKey);
+
+    static <T> T getResult(CompletableFuture<T> result)
+            throws MissingRepositoryException, FailedConnectionException, ForbiddenException {
+        try {
+            return result.join();
+        } catch (CompletionException e) {
+            try {
+                throw e.getCause();
+            } catch (MissingRepositoryException
+                    | FailedConnectionException
+                    | ForbiddenException
+                    | RuntimeException r) {
+                throw r;
+            } catch (Throwable __) {
+                throw e;
+            }
+        }
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/snapshot/SnapshotApiFacade.java

@@ -0,0 +1,162 @@
+package uk.ac.ic.wlgitbridge.bridge.snapshot;
+
+import com.google.api.client.auth.oauth2.Credential;
+import uk.ac.ic.wlgitbridge.data.CandidateSnapshot;
+import uk.ac.ic.wlgitbridge.data.model.Snapshot;
+import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
+import uk.ac.ic.wlgitbridge.snapshot.base.MissingRepositoryException;
+import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
+import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
+import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocResult;
+import uk.ac.ic.wlgitbridge.snapshot.getforversion.GetForVersionResult;
+import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotData;
+import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.GetSavedVersResult;
+import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.SnapshotInfo;
+import uk.ac.ic.wlgitbridge.snapshot.push.PushResult;
+import uk.ac.ic.wlgitbridge.snapshot.push.exception.InvalidProjectException;
+
+import java.util.*;
+import java.util.concurrent.CompletableFuture;
+import java.util.stream.Collectors;
+
+/**
+ * Created by winston on 02/07/2017.
+ */
+public class SnapshotApiFacade {
+
+    private final SnapshotApi api;
+
+    public SnapshotApiFacade(SnapshotApi api) {
+        this.api = api;
+    }
+
+    public boolean projectExists(
+            Optional<Credential> oauth2,
+            String projectName
+    ) throws FailedConnectionException, GitUserException {
+        try {
+            SnapshotApi
+                    .getResult(api.getDoc(oauth2, projectName))
+                    .getVersionID();
+            return true;
+        } catch (InvalidProjectException e) {
+            return false;
+        }
+    }
+
+    public Optional<GetDocResult> getDoc(
+            Optional<Credential> oauth2,
+            String projectName
+    ) throws FailedConnectionException, GitUserException {
+        try {
+            GetDocResult doc = SnapshotApi
+                    .getResult(api.getDoc(oauth2, projectName));
+            doc.getVersionID();
+            return Optional.of(doc);
+        } catch (InvalidProjectException e) {
+            return Optional.empty();
+        }
+    }
+
+    public Deque<Snapshot> getSnapshots(
+            Optional<Credential> oauth2,
+            String projectName,
+            int afterVersionId
+    ) throws GitUserException, FailedConnectionException {
+        List<SnapshotInfo> snapshotInfos = getSnapshotInfosAfterVersion(
+                oauth2,
+                projectName,
+                afterVersionId
+        );
+        List<SnapshotData> snapshotDatas = getMatchingSnapshotData(
+                oauth2,
+                projectName,
+                snapshotInfos
+        );
+        return combine(snapshotInfos, snapshotDatas);
+    }
+
+    public PushResult push(
+            Optional<Credential> oauth2,
+            CandidateSnapshot candidateSnapshot,
+            String postbackKey
+    ) throws MissingRepositoryException, FailedConnectionException, ForbiddenException {
+        return SnapshotApi.getResult(api.push(
+                oauth2, candidateSnapshot, postbackKey));
+    }
+
+    private List<SnapshotInfo> getSnapshotInfosAfterVersion(
+            Optional<Credential> oauth2,
+            String projectName,
+            int version
+    ) throws FailedConnectionException, GitUserException {
+        SortedSet<SnapshotInfo> versions = new TreeSet<>();
+        CompletableFuture<GetDocResult> getDoc
+                = api.getDoc(oauth2, projectName);
+        CompletableFuture<GetSavedVersResult> savedVers
+                = api.getSavedVers(oauth2, projectName);
+        GetDocResult latestDoc = SnapshotApi.getResult(getDoc);
+        int latest = latestDoc.getVersionID();
+        // Handle edge-case for projects with no changes, that were imported
+        // to v2. In which case both `latest` and `version` will be zero.
+        // See: https://github.com/overleaf/writelatex-git-bridge/pull/50
+        if (latest > version || (latest == 0 && version == 0)) {
+            for (
+                    SnapshotInfo snapshotInfo :
+                    SnapshotApi.getResult(savedVers).getSavedVers()
+            ) {
+                if (snapshotInfo.getVersionId() > version) {
+                    versions.add(snapshotInfo);
+                }
+            }
+            versions.add(new SnapshotInfo(
+                    latest,
+                    latestDoc.getCreatedAt(),
+                    latestDoc.getName(),
+                    latestDoc.getEmail()
+            ));
+
+        }
+        return new ArrayList<>(versions);
+    }
+
+    private List<SnapshotData> getMatchingSnapshotData(
+            Optional<Credential> oauth2,
+            String projectName,
+            List<SnapshotInfo> snapshotInfos
+    ) throws FailedConnectionException, ForbiddenException {
+        List<CompletableFuture<GetForVersionResult>> firedRequests
+                = fireDataRequests(oauth2, projectName, snapshotInfos);
+        List<SnapshotData> snapshotDataList = new ArrayList<>();
+        for (CompletableFuture<GetForVersionResult> fired : firedRequests) {
+            snapshotDataList.add(fired.join().getSnapshotData());
+        }
+        return snapshotDataList;
+    }
+
+    private List<CompletableFuture<GetForVersionResult>> fireDataRequests(
+            Optional<Credential> oauth2,
+            String projectName,
+            List<SnapshotInfo> snapshotInfos
+    ) {
+        return snapshotInfos
+                .stream()
+                .map(snap -> api.getForVersion(
+                        oauth2, projectName, snap.getVersionId()))
+                .collect(Collectors.toList());
+    }
+
+    private static Deque<Snapshot> combine(
+            List<SnapshotInfo> snapshotInfos,
+            List<SnapshotData> snapshotDatas
+    ) {
+        Deque<Snapshot> snapshots = new LinkedList<>();
+        Iterator<SnapshotInfo> infos = snapshotInfos.iterator();
+        Iterator<SnapshotData> datas = snapshotDatas.iterator();
+        while (infos.hasNext()) {
+            snapshots.add(new Snapshot(infos.next(), datas.next()));
+        }
+        return snapshots;
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/NoopSwapJob.java

@@ -0,0 +1,20 @@
+package uk.ac.ic.wlgitbridge.bridge.swap.job;
+
+/**
+ * Created by winston on 24/08/2016.
+ */
+public class NoopSwapJob implements SwapJob {
+
+    @Override
+    public void start() {}
+
+    @Override
+    public void stop() {}
+
+    @Override
+    public void evict(String projName) {}
+
+    @Override
+    public void restore(String projName) {}
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJob.java

@@ -0,0 +1,124 @@
+package uk.ac.ic.wlgitbridge.bridge.swap.job;
+
+import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
+import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
+import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
+import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore;
+
+import java.io.IOException;
+import java.util.Optional;
+
+/**
+ * Created by winston on 20/08/2016.
+ */
+public interface SwapJob {
+
+    enum CompressionMethod { Bzip2, Gzip }
+
+    static CompressionMethod stringToCompressionMethod(String compressionString) {
+        if (compressionString == null) {
+            return null;
+        }
+        CompressionMethod result;
+        switch (compressionString) {
+            case "gzip":
+                result = CompressionMethod.Gzip;
+                break;
+            case "bzip2":
+                result = CompressionMethod.Bzip2;
+                break;
+            default:
+                result = null;
+                break;
+        }
+        return result;
+    }
+
+    static String compressionMethodAsString(CompressionMethod compressionMethod) {
+        if (compressionMethod == null) {
+            return null;
+        }
+        String result;
+        switch (compressionMethod) {
+            case Gzip:
+                result =  "gzip";
+                break;
+            case Bzip2:
+                result =  "bzip2";
+                break;
+            default:
+                result =  null;
+                break;
+        }
+        return result;
+    }
+
+    static SwapJob fromConfig(
+            Optional<SwapJobConfig> cfg,
+            ProjectLock lock,
+            RepoStore repoStore,
+            DBStore dbStore,
+            SwapStore swapStore
+    ) {
+        if (cfg.isPresent()) {
+            return new SwapJobImpl(
+                    cfg.get(),
+                    lock,
+                    repoStore,
+                    dbStore,
+                    swapStore
+            );
+        }
+        return new NoopSwapJob();
+    }
+
+    /**
+     * Starts the swap job, which should schedule an attempted swap at the given
+     * configured interval (config["swapJob"]["intervalMillis"]
+     */
+    void start();
+
+    /**
+     * Stops the stop job.
+     */
+    void stop();
+
+    /**
+     * Called by the swap job when a project should be evicted.
+     *
+     * Pre:
+     * 1. projName must be in repoStore
+     * 2. projName should not be in swapStore
+     * 3. projName should be PRESENT in dbStore (last_accessed is not null)
+     *
+     * Acquires the project lock and performs an eviction of projName.
+     *
+     * Post:
+     * 1. projName should not in repoStore
+     * 2. projName must be in swapStore
+     * 3. projName must be SWAPPED in dbStore (last_accessed is null)
+     * @param projName
+     * @throws IOException
+     */
+    void evict(String projName) throws IOException;
+
+    /**
+     * Called on a project when it must be restored.
+     *
+     * Pre:
+     * 1. projName should not be in repoStore
+     * 2. projName must be in swapStore
+     * 3. projName must be SWAPPED in dbStore (last_accessed is null)
+     *
+     * Acquires the project lock and restores projName.
+     *
+     * Post:
+     * 1. projName must be in repoStore
+     * 2. projName should not in swapStore
+     * 3. projName should be PRESENT in dbStore (last_accessed is not null)
+     * @param projName
+     * @throws IOException
+     */
+    void restore(String projName) throws IOException;
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobConfig.java

@@ -0,0 +1,55 @@
+package uk.ac.ic.wlgitbridge.bridge.swap.job;
+
+import uk.ac.ic.wlgitbridge.util.Log;
+import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJob.CompressionMethod;
+
+/**
+ * Created by winston on 23/08/2016.
+ */
+public class SwapJobConfig {
+
+    private final int minProjects;
+    private final int lowGiB;
+    private final int highGiB;
+    private final long intervalMillis;
+    private final String compressionMethod;
+
+    public SwapJobConfig(
+            int minProjects,
+            int lowGiB,
+            int highGiB,
+            long intervalMillis,
+            String compressionMethod
+    ) {
+        this.minProjects = minProjects;
+        this.lowGiB = lowGiB;
+        this.highGiB = highGiB;
+        this.intervalMillis = intervalMillis;
+        this.compressionMethod = compressionMethod;
+    }
+
+    public int getMinProjects() {
+        return minProjects;
+    }
+
+    public int getLowGiB() {
+        return lowGiB;
+    }
+
+    public int getHighGiB() {
+        return highGiB;
+    }
+
+    public long getIntervalMillis() {
+        return intervalMillis;
+    }
+
+    public SwapJob.CompressionMethod getCompressionMethod() {
+      CompressionMethod result = SwapJob.stringToCompressionMethod(compressionMethod);
+      if (result == null) {
+        Log.info("SwapJobConfig: un-supported compressionMethod '{}', default to 'bzip2'", compressionMethod);
+        result = CompressionMethod.Bzip2;
+      }
+      return result;
+    }
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImpl.java

@@ -0,0 +1,262 @@
+package uk.ac.ic.wlgitbridge.bridge.swap.job;
+
+import com.google.api.client.repackaged.com.google.common.base.Preconditions;
+import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
+import uk.ac.ic.wlgitbridge.bridge.lock.LockGuard;
+import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
+import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
+import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore;
+import uk.ac.ic.wlgitbridge.util.Log;
+import uk.ac.ic.wlgitbridge.util.TimerUtils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.sql.Timestamp;
+import java.time.Duration;
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.Timer;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Created by winston on 20/08/2016.
+ */
+public class SwapJobImpl implements SwapJob {
+
+    private static final long GiB = (1l << 30);
+
+    int minProjects;
+    long lowWatermarkBytes;
+    long highWatermarkBytes;
+    Duration interval;
+
+    private final ProjectLock lock;
+    private final RepoStore repoStore;
+    private final DBStore dbStore;
+    private final SwapStore swapStore;
+    private final CompressionMethod compressionMethod;
+
+    private final Timer timer;
+
+    final AtomicInteger swaps;
+
+    public SwapJobImpl(
+            SwapJobConfig cfg,
+            ProjectLock lock,
+            RepoStore repoStore,
+            DBStore dbStore,
+            SwapStore swapStore
+    ) {
+        this(
+                cfg.getMinProjects(),
+                GiB * cfg.getLowGiB(),
+                GiB * cfg.getHighGiB(),
+                Duration.ofMillis(cfg.getIntervalMillis()),
+                cfg.getCompressionMethod(),
+                lock,
+                repoStore,
+                dbStore,
+                swapStore
+        );
+    }
+
+    SwapJobImpl(
+            int minProjects,
+            long lowWatermarkBytes,
+            long highWatermarkBytes,
+            Duration interval,
+            CompressionMethod method,
+            ProjectLock lock,
+            RepoStore repoStore,
+            DBStore dbStore,
+            SwapStore swapStore
+    ) {
+        this.minProjects = minProjects;
+        this.lowWatermarkBytes = lowWatermarkBytes;
+        this.highWatermarkBytes = highWatermarkBytes;
+        this.interval = interval;
+        this.compressionMethod = method;
+        this.lock = lock;
+        this.repoStore = repoStore;
+        this.dbStore = dbStore;
+        this.swapStore = swapStore;
+        timer = new Timer();
+        swaps = new AtomicInteger(0);
+    }
+
+    @Override
+    public void start() {
+        timer.schedule(
+                TimerUtils.makeTimerTask(this::doSwap),
+                0
+        );
+    }
+
+    @Override
+    public void stop() {
+        timer.cancel();
+    }
+
+    private void doSwap() {
+        try {
+            doSwap_();
+        } catch (Throwable t) {
+            Log.warn("Exception thrown during swap job", t);
+        }
+        timer.schedule(
+                TimerUtils.makeTimerTask(this::doSwap),
+                interval.toMillis()
+        );
+    }
+
+    private void doSwap_() {
+        ArrayList<String> exceptionProjectNames = new ArrayList<String>();
+
+        Log.info("Running swap number {}", swaps.get() + 1);
+        long totalSize = repoStore.totalSize();
+        Log.info("Size is {}/{} (high)", totalSize, highWatermarkBytes);
+        if (totalSize < highWatermarkBytes) {
+            Log.info("No need to swap.");
+            swaps.incrementAndGet();
+            return;
+        }
+        int numProjects = dbStore.getNumProjects();
+        // while we have too many projects on disk
+        while (
+                (totalSize = repoStore.totalSize()) > lowWatermarkBytes &&
+                (numProjects = dbStore.getNumUnswappedProjects()) > minProjects
+        ) {
+            // check if we've had too many exceptions so far
+            if (exceptionProjectNames.size() >= 20) {
+                StringBuilder sb = new StringBuilder();
+                for (String s: exceptionProjectNames) {
+                    sb.append(s);
+                    sb.append(' ');
+                }
+                Log.error(
+                    "Too many exceptions while running swap, giving up on this run: {}",
+                    sb.toString()
+                );
+                break;
+            }
+            // get the oldest project and try to swap it
+            String projectName = dbStore.getOldestUnswappedProject();
+            try {
+                evict(projectName);
+            } catch (Exception e) {
+                Log.warn("[{}] Exception while swapping, mark project and move on", projectName, e);
+                // NOTE: this is something of a hack. If a project fails to swap we get stuck in a
+                // loop where `dbStore.getOldestUnswappedProject()` gives the same failing project over and over again,
+                // which fills up the disk with errors. By touching the access time we can mark the project as a
+                // non-candidate for swapping. Ideally we should be checking the logs for these log events and fixing
+                // whatever is wrong with the project
+                dbStore.setLastAccessedTime(
+                    projectName,
+                    Timestamp.valueOf(LocalDateTime.now())
+                );
+                exceptionProjectNames.add(projectName);
+            }
+        }
+        if (totalSize > lowWatermarkBytes) {
+            Log.warn(
+                    "Finished swapping, but total size is still too high."
+            );
+        }
+        Log.info(
+                "Size: {}/{} (low), " +
+                        "{} (high), " +
+                        "projects on disk: {}/{}, " +
+                        "min projects on disk: {}",
+                totalSize,
+                lowWatermarkBytes,
+                highWatermarkBytes,
+                numProjects,
+                dbStore.getNumProjects(),
+                minProjects
+        );
+        swaps.incrementAndGet();
+    }
+
+    /**
+     * @see SwapJob#evict(String) for high-level description.
+     *
+     * 1. Acquires the project lock.
+     * 2. Gets a bz2 stream and size of a project from the repo store, or throws
+     * 3. Uploads the bz2 stream and size to the projName in the swapStore.
+     * 4. Sets the last accessed time in the dbStore to null, which makes our
+     *    state SWAPPED
+     * 5. Removes the project from the repo store.
+     * @param projName
+     * @throws IOException
+     */
+    @Override
+    public void evict(String projName) throws IOException {
+        Preconditions.checkNotNull(projName, "projName was null");
+        Log.info("Evicting project: {}", projName);
+        try (LockGuard __ = lock.lockGuard(projName)) {
+            try {
+                repoStore.gcProject(projName);
+            } catch (Exception e) {
+                Log.error("[{}] Exception while running gc on project: {}", projName, e);
+            }
+            long[] sizePtr = new long[1];
+            try (InputStream blob = getBlobStream(projName, sizePtr)) {
+                swapStore.upload(projName, blob, sizePtr[0]);
+                String compression = SwapJob.compressionMethodAsString(compressionMethod);
+                if (compression == null) {
+                  throw new RuntimeException("invalid compression method, should not happen");
+                }
+                dbStore.swap(projName, compression);
+                repoStore.remove(projName);
+            }
+        }
+        Log.info("Evicted project: {}", projName);
+    }
+
+    private InputStream getBlobStream(String projName, long[] sizePtr) throws IOException {
+        if (compressionMethod == CompressionMethod.Gzip) {
+          return repoStore.gzipProject(projName, sizePtr);
+        } else if (compressionMethod == CompressionMethod.Bzip2) {
+          return repoStore.bzip2Project(projName, sizePtr);
+        } else {
+          throw new RuntimeException("invalid compression method, should not happen");
+        }
+    }
+
+    /**
+     * @see SwapJob#restore(String) for high-level description.
+     *
+     * 1. Acquires the project lock.
+     * 2. Gets a bz2 stream for the project from the swapStore.
+     * 3. Fully downloads and places the bz2 stream back in the repo store.
+     * 4. Sets the last accessed time in the dbStore to now, which makes our
+     *    state PRESENT and the last project to be evicted.
+     * @param projName
+     * @throws IOException
+     */
+    @Override
+    public void restore(String projName) throws IOException {
+        try (LockGuard __ = lock.lockGuard(projName)) {
+            try (InputStream zipped = swapStore.openDownloadStream(projName)) {
+                String compression = dbStore.getSwapCompression(projName);
+                if (compression == null) {
+                    throw new RuntimeException("Missing compression method during restore, should not happen");
+                }
+                if ("gzip".equals(compression)) {
+                  repoStore.ungzipProject(
+                    projName,
+                    zipped
+                  );
+                } else if ("bzip2".equals(compression)) {
+                  repoStore.unbzip2Project(
+                    projName,
+                    zipped
+                  );
+                }
+                swapStore.remove(projName);
+                dbStore.restore(projName);
+            }
+        }
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/InMemorySwapStore.java

@@ -0,0 +1,54 @@
+package uk.ac.ic.wlgitbridge.bridge.swap.store;
+
+import org.apache.commons.io.IOUtils;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Created by winston on 23/08/2016.
+ */
+public class InMemorySwapStore implements SwapStore {
+
+    private final Map<String, byte[]> store;
+
+    public InMemorySwapStore() {
+        store = new HashMap<>();
+    }
+
+    public InMemorySwapStore(SwapStoreConfig __) {
+        this();
+    }
+
+    @Override
+    public void upload(
+            String projectName,
+            InputStream uploadStream,
+            long contentLength
+    ) throws IOException {
+        store.put(
+                projectName,
+                IOUtils.toByteArray(uploadStream, contentLength)
+        );
+    }
+
+    @Override
+    public InputStream openDownloadStream(String projectName) {
+        byte[] buf = store.get(projectName);
+        if (buf == null) {
+            throw new IllegalArgumentException(
+                    "no such project in swap store: " + projectName
+            );
+        }
+        return new ByteArrayInputStream(buf);
+    }
+
+    @Override
+    public void remove(String projectName) {
+        store.remove(projectName);
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/NoopSwapStore.java

@@ -0,0 +1,28 @@
+package uk.ac.ic.wlgitbridge.bridge.swap.store;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+
+/**
+ * Created by winston on 24/08/2016.
+ */
+public class NoopSwapStore implements SwapStore {
+
+    public NoopSwapStore(SwapStoreConfig __) {}
+
+    @Override
+    public void upload(
+            String projectName,
+            InputStream uploadStream,
+            long contentLength
+    ) {}
+
+    @Override
+    public InputStream openDownloadStream(String projectName) {
+        return new ByteArrayInputStream(new byte[0]);
+    }
+
+    @Override
+    public void remove(String projectName) {}
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/S3SwapStore.java

@@ -0,0 +1,86 @@
+package uk.ac.ic.wlgitbridge.bridge.swap.store;
+
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.model.*;
+
+import java.io.InputStream;
+
+/**
+ * Created by winston on 21/08/2016.
+ */
+public class S3SwapStore implements SwapStore {
+
+    private final AmazonS3 s3;
+
+    private final String bucketName;
+
+    public S3SwapStore(SwapStoreConfig cfg) {
+        this(
+                cfg.getAwsAccessKey(),
+                cfg.getAwsSecret(),
+                cfg.getS3BucketName(),
+                cfg.getAwsRegion()
+        );
+    }
+
+    S3SwapStore(
+            String accessKey,
+            String secret,
+            String bucketName,
+            String region
+    ) {
+        String regionToUse = null;
+        if (region == null) {
+            regionToUse = "us-east-1";
+        } else {
+            regionToUse = region;
+        }
+        s3 = AmazonS3ClientBuilder
+                .standard()
+                .withRegion(regionToUse)
+                .withCredentials(
+                        new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secret))
+                ).build();
+        this.bucketName = bucketName;
+    }
+
+    @Override
+    public void upload(
+            String projectName,
+            InputStream uploadStream,
+            long contentLength
+    ) {
+        ObjectMetadata metadata = new ObjectMetadata();
+        metadata.setContentLength(contentLength);
+        PutObjectRequest put = new PutObjectRequest(
+                bucketName,
+                projectName,
+                uploadStream,
+                metadata
+        );
+        PutObjectResult res = s3.putObject(put);
+    }
+
+    @Override
+    public InputStream openDownloadStream(String projectName) {
+        GetObjectRequest get = new GetObjectRequest(
+                bucketName,
+                projectName
+        );
+        S3Object res = s3.getObject(get);
+        return res.getObjectContent();
+    }
+
+    @Override
+    public void remove(String projectName) {
+        DeleteObjectRequest del = new DeleteObjectRequest(
+                bucketName,
+                projectName
+        );
+        s3.deleteObject(del);
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/SwapStore.java

@@ -0,0 +1,44 @@
+package uk.ac.ic.wlgitbridge.bridge.swap.store;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.Function;
+
+/**
+ * Created by winston on 20/08/2016.
+ */
+public interface SwapStore {
+
+    Map<String, Function<SwapStoreConfig, SwapStore>> swapStores =
+            new HashMap<String, Function<SwapStoreConfig, SwapStore>>() {
+
+                {
+                    put("noop", NoopSwapStore::new);
+                    put("memory", InMemorySwapStore::new);
+                    put("s3", S3SwapStore::new);
+                }
+
+            };
+
+    static SwapStore fromConfig(
+            Optional<SwapStoreConfig> cfg
+    ) {
+        SwapStoreConfig cfg_ = cfg.orElse(SwapStoreConfig.NOOP);
+        String type = cfg_.getType();
+        return swapStores.get(type).apply(cfg_);
+    }
+
+    void upload(
+            String projectName,
+            InputStream uploadStream,
+            long contentLength
+    ) throws IOException;
+
+    InputStream openDownloadStream(String projectName);
+
+    void remove(String projectName);
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/SwapStoreConfig.java

@@ -0,0 +1,85 @@
+package uk.ac.ic.wlgitbridge.bridge.swap.store;
+
+/**
+ * Created by winston on 24/08/2016.
+ */
+public class SwapStoreConfig {
+
+    public static final SwapStoreConfig NOOP = new SwapStoreConfig(
+            "noop",
+            null,
+            null,
+            null,
+            null
+    );
+
+    private String type;
+    private String awsAccessKey;
+    private String awsSecret;
+    private String s3BucketName;
+    private String awsRegion;
+
+    public SwapStoreConfig() {}
+
+    public SwapStoreConfig(
+            String awsAccessKey,
+            String awsSecret,
+            String s3BucketName,
+            String awsRegion
+    ) {
+        this(
+                "s3",
+                awsAccessKey,
+                awsSecret,
+                s3BucketName,
+                awsRegion
+        );
+    }
+
+    SwapStoreConfig(
+            String type,
+            String awsAccessKey,
+            String awsSecret,
+            String s3BucketName,
+            String awsRegion
+    ) {
+        this.type = type;
+        this.awsAccessKey = awsAccessKey;
+        this.awsSecret = awsSecret;
+        this.s3BucketName = s3BucketName;
+        this.awsRegion = awsRegion;
+    }
+
+    public String getType() {
+        return type;
+    }
+
+    public String getAwsAccessKey() {
+        return awsAccessKey;
+    }
+
+    public String getAwsSecret() {
+        return awsSecret;
+    }
+
+    public String getS3BucketName() {
+        return s3BucketName;
+    }
+
+    public String getAwsRegion() { return awsRegion; }
+
+    public SwapStoreConfig sanitisedCopy() {
+        return new SwapStoreConfig(
+                type,
+                awsAccessKey == null ? null : "<awsAccessKey>",
+                awsSecret == null ? null : "<awsSecret>",
+                s3BucketName,
+                awsRegion
+        );
+    }
+
+    public static SwapStoreConfig sanitisedCopy(SwapStoreConfig swapStore) {
+        return swapStore == null ? null : swapStore.sanitisedCopy();
+    }
+    
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/util/CastUtil.java

@@ -0,0 +1,18 @@
+package uk.ac.ic.wlgitbridge.bridge.util;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Created by winston on 01/07/2017.
+ */
+public class CastUtil {
+
+    public static int assumeInt(long l) {
+        Preconditions.checkArgument(
+                l <= (long) Integer.MAX_VALUE
+                        && l >= (long) Integer.MIN_VALUE,
+                l + " cannot fit inside an int");
+        return (int) l;
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/CandidateSnapshot.java

@@ -0,0 +1,150 @@
+package uk.ac.ic.wlgitbridge.data;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
+import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
+import uk.ac.ic.wlgitbridge.util.Util;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * Created by Winston on 16/11/14.
+ */
+public class CandidateSnapshot implements AutoCloseable {
+
+    private final String projectName;
+    private final int currentVersion;
+    private final List<ServletFile> files;
+    private final List<String> deleted;
+    private File attsDirectory;
+
+    public CandidateSnapshot(
+            String projectName,
+            int currentVersion,
+            RawDirectory directoryContents,
+            RawDirectory oldDirectoryContents
+    ) {
+        this.projectName = projectName;
+        this.currentVersion = currentVersion;
+        files = diff(directoryContents, oldDirectoryContents);
+        deleted = deleted(directoryContents, oldDirectoryContents);
+    }
+
+    private List<ServletFile> diff(
+            RawDirectory directoryContents,
+            RawDirectory oldDirectoryContents
+    ) {
+        List<ServletFile> files = new LinkedList<ServletFile>();
+        Map<String, RawFile> fileTable = directoryContents.getFileTable();
+        Map<String, RawFile> oldFileTable = oldDirectoryContents.getFileTable();
+        for (Entry<String, RawFile> entry : fileTable.entrySet()) {
+            RawFile file = entry.getValue();
+            files.add(new ServletFile(file, oldFileTable.get(file.getPath())));
+        }
+        return files;
+    }
+
+    private List<String> deleted(
+            RawDirectory directoryContents,
+            RawDirectory oldDirectoryContents
+    ) {
+        List<String> deleted = new LinkedList<String>();
+        Map<String, RawFile> fileTable = directoryContents.getFileTable();
+        for (
+                Entry<String, RawFile> entry :
+                oldDirectoryContents.getFileTable().entrySet()
+        ) {
+            String path = entry.getKey();
+            RawFile newFile = fileTable.get(path);
+            if (newFile == null) {
+                deleted.add(path);
+            }
+        }
+        return deleted;
+    }
+
+    public void writeServletFiles(File rootGitDirectory) throws IOException {
+        attsDirectory = new File(
+                rootGitDirectory,
+                ".wlgb/atts/" + projectName
+        );
+        for (ServletFile file : files) {
+            if (file.isChanged()) {
+                file.writeToDiskWithName(attsDirectory, file.getUniqueIdentifier());
+            }
+        }
+    }
+
+    public void deleteServletFiles() throws IOException {
+        if (attsDirectory != null) {
+            Util.deleteDirectory(attsDirectory);
+        }
+    }
+
+    public JsonElement getJsonRepresentation(String postbackKey) {
+        String projectURL = Util.getPostbackURL() + "api/" + projectName;
+        JsonObject jsonObject = new JsonObject();
+        jsonObject.addProperty("latestVerId", currentVersion);
+        jsonObject.add("files", getFilesAsJson(projectURL, postbackKey));
+        jsonObject.addProperty(
+                "postbackUrl", projectURL + "/" + postbackKey + "/postback"
+        );
+        return jsonObject;
+    }
+
+    private JsonArray getFilesAsJson(String projectURL, String postbackKey) {
+        JsonArray filesArray = new JsonArray();
+        for (ServletFile file : files) {
+            filesArray.add(getFileAsJson(file, projectURL, postbackKey));
+        }
+        return filesArray;
+    }
+
+    private JsonObject getFileAsJson(
+            ServletFile file,
+            String projectURL,
+            String postbackKey
+    ) {
+        JsonObject jsonFile = new JsonObject();
+        jsonFile.addProperty("name", file.getPath());
+        if (file.isChanged()) {
+            String identifier = file.getUniqueIdentifier();
+            String url = projectURL + "/" + identifier + "?key=" + postbackKey;
+            jsonFile.addProperty("url", url);
+        }
+        return jsonFile;
+    }
+
+    public String getProjectName() {
+        return projectName;
+    }
+
+    public List<String> getDeleted() {
+        return deleted;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder();
+        sb.append("VersionId: ");
+        sb.append(currentVersion);
+        sb.append(", files: ");
+        sb.append(files);
+        sb.append(", deleted: ");
+        sb.append(deleted);
+        return sb.toString();
+    }
+
+    @Override
+    public void close() throws IOException {
+        deleteServletFiles();
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/LockAllWaiter.java

@@ -0,0 +1,10 @@
+package uk.ac.ic.wlgitbridge.data;
+
+/**
+ * Created by Winston on 21/02/15.
+ */
+public interface LockAllWaiter {
+
+    void threadsRemaining(int threads);
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/ProjectLockImpl.java

@@ -0,0 +1,89 @@
+package uk.ac.ic.wlgitbridge.data;
+
+import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import uk.ac.ic.wlgitbridge.util.Log;
+
+/**
+ * Created by Winston on 20/11/14.
+ */
+public class ProjectLockImpl implements ProjectLock {
+
+    private final Map<String, Lock> projectLocks;
+    private final ReentrantReadWriteLock rwlock;
+    private final Lock rlock;
+    private final ReentrantReadWriteLock.WriteLock wlock;
+    private LockAllWaiter waiter;
+    private boolean waiting;
+
+    public ProjectLockImpl() {
+        projectLocks = new HashMap<String, Lock>();
+        rwlock = new ReentrantReadWriteLock();
+        rlock = rwlock.readLock();
+        wlock = rwlock.writeLock();
+        waiting = false;
+    }
+
+    public ProjectLockImpl(LockAllWaiter waiter) {
+        this();
+        setWaiter(waiter);
+    }
+
+    @Override
+    public void lockForProject(String projectName) {
+        Log.debug("[{}] taking project lock", projectName);
+        getLockForProjectName(projectName).lock();
+        Log.debug("[{}] taking reentrant lock", projectName);
+        rlock.lock();
+        Log.debug("[{}] taken locks", projectName);
+    }
+
+    @Override
+    public void unlockForProject(String projectName) {
+        Log.debug("[{}] releasing project lock", projectName);
+        getLockForProjectName(projectName).unlock();
+        Log.debug("[{}] releasing reentrant lock", projectName);
+        rlock.unlock();
+        Log.debug("[{}] released locks", projectName);
+        if (waiting) {
+            Log.debug("[{}] waiting for remaining threads", projectName);
+            trySignal();
+        }
+    }
+
+    private void trySignal() {
+        int threads = rwlock.getReadLockCount();
+        Log.debug("-> waiting for {} threads", threads);
+        if (waiter != null && threads > 0) {
+            waiter.threadsRemaining(threads);
+        }
+        Log.debug("-> finished waiting for threads");
+    }
+
+    public void lockAll() {
+        Log.debug("-> locking all threads");
+        waiting = true;
+        trySignal();
+        Log.debug("-> locking reentrant write lock");
+        wlock.lock();
+    }
+
+    private synchronized Lock getLockForProjectName(String projectName) {
+        Lock lock = projectLocks.get(projectName);
+        if (lock == null) {
+            lock = new ReentrantLock();
+            projectLocks.put(projectName, lock);
+        }
+        return lock;
+    }
+
+    public void setWaiter(LockAllWaiter waiter) {
+        this.waiter = waiter;
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/ServletFile.java

@@ -0,0 +1,46 @@
+package uk.ac.ic.wlgitbridge.data;
+
+import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
+import java.util.UUID;
+
+/**
+ * Created by Winston on 21/02/15.
+ */
+public class ServletFile extends RawFile {
+
+    private final RawFile file;
+    private final boolean changed;
+    private String uuid;
+
+    public ServletFile(RawFile file, RawFile oldFile) {
+        this.file = file;
+        this.uuid = UUID.randomUUID().toString();
+        changed = !equals(oldFile);
+    }
+
+    public String getUniqueIdentifier() { return uuid; }
+
+    @Override
+    public String getPath() {
+        return file.getPath();
+    }
+
+    @Override
+    public byte[] getContents() {
+        return file.getContents();
+    }
+
+    @Override
+    public long size() {
+        return getContents().length;
+    }
+
+    public boolean isChanged() {
+        return changed;
+    }
+
+    @Override
+    public String toString() {
+        return getPath();
+    }
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/filestore/GitDirectoryContents.java

@@ -0,0 +1,84 @@
+package uk.ac.ic.wlgitbridge.data.filestore;
+
+import uk.ac.ic.wlgitbridge.data.model.Snapshot;
+import uk.ac.ic.wlgitbridge.util.Util;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Date;
+import java.util.List;
+
+/**
+ * Created by Winston on 14/11/14.
+ */
+public class GitDirectoryContents {
+
+    private final List<RawFile> files;
+    private final File gitDirectory;
+    private final String userName;
+    private final String userEmail;
+    private final String commitMessage;
+    private final Date when;
+
+    public GitDirectoryContents(
+            List<RawFile> files,
+            File rootGitDirectory,
+            String projectName,
+            String userName,
+            String userEmail,
+            String commitMessage,
+            Date when
+    ) {
+        this.files = files;
+        this.gitDirectory = new File(rootGitDirectory, projectName);
+        this.userName = userName;
+        this.userEmail = userEmail;
+        this.commitMessage = commitMessage;
+        this.when = when;
+    }
+
+    public GitDirectoryContents(
+            List<RawFile> files,
+            File rootGitDirectory,
+            String projectName,
+            Snapshot snapshot
+    ) {
+        this(
+                files,
+                rootGitDirectory,
+                projectName,
+                snapshot.getUserName(),
+                snapshot.getUserEmail(),
+                snapshot.getComment(),
+                snapshot.getCreatedAt()
+        );
+    }
+
+    public void write() throws IOException {
+        Util.deleteInDirectoryApartFrom(gitDirectory, ".git");
+        for (RawFile fileNode : files) {
+            fileNode.writeToDisk(gitDirectory);
+        }
+    }
+
+    public File getDirectory() {
+        return gitDirectory;
+    }
+
+    public String getUserName() {
+        return userName;
+    }
+
+    public String getUserEmail() {
+        return userEmail;
+    }
+
+    public String getCommitMessage() {
+        return commitMessage;
+    }
+
+    public Date getWhen() {
+        return when;
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/filestore/RawDirectory.java

@@ -0,0 +1,23 @@
+package uk.ac.ic.wlgitbridge.data.filestore;
+
+import uk.ac.ic.wlgitbridge.git.exception.SizeLimitExceededException;
+
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * Created by Winston on 16/11/14.
+ */
+public class RawDirectory {
+
+    private final Map<String, RawFile> fileTable;
+
+    public RawDirectory(Map<String, RawFile> fileTable) {
+        this.fileTable = fileTable;
+    }
+
+    public Map<String, RawFile> getFileTable() {
+        return fileTable;
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/filestore/RawFile.java

@@ -0,0 +1,46 @@
+package uk.ac.ic.wlgitbridge.data.filestore;
+
+import uk.ac.ic.wlgitbridge.util.Log;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Arrays;
+
+/**
+ * Created by Winston on 16/11/14.
+ */
+public abstract class RawFile {
+
+    public abstract String getPath();
+
+    public abstract byte[] getContents();
+
+    public abstract long size();
+
+    public final void writeToDisk(File directory) throws IOException {
+        writeToDiskWithName(directory, getPath());
+    }
+
+    public final void writeToDiskWithName(File directory, String name) throws IOException {
+        File file = new File(directory, name);
+        file.getParentFile().mkdirs();
+        file.createNewFile();
+        OutputStream out = new FileOutputStream(file);
+        out.write(getContents());
+        out.close();
+        Log.info("Wrote file: {}", file.getAbsolutePath());
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof RawFile)) {
+            return false;
+        }
+        RawFile that = (RawFile) obj;
+        return getPath().equals(that.getPath())
+                && Arrays.equals(getContents(), that.getContents());
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/filestore/RepositoryFile.java

@@ -0,0 +1,31 @@
+package uk.ac.ic.wlgitbridge.data.filestore;
+
+/**
+ * Created by Winston on 16/11/14.
+ */
+public class RepositoryFile extends RawFile {
+
+    private final String path;
+    private final byte[] contents;
+
+    public RepositoryFile(String path, byte[] contents) {
+        this.path = path;
+        this.contents = contents;
+    }
+
+    @Override
+    public String getPath() {
+        return path;
+    }
+
+    @Override
+    public byte[] getContents() {
+        return contents;
+    }
+
+    @Override
+    public long size() {
+        return contents.length;
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/Snapshot.java

@@ -0,0 +1,77 @@
+package uk.ac.ic.wlgitbridge.data.model;
+
+import org.joda.time.DateTime;
+import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotAttachment;
+import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotData;
+import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotFile;
+import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.SnapshotInfo;
+import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.WLUser;
+
+import java.util.Date;
+import java.util.List;
+
+/**
+ * Created by Winston on 03/11/14.
+ */
+public class Snapshot implements Comparable<Snapshot> {
+
+    private final int versionID;
+    private final String comment;
+    private final String userName;
+    private final String userEmail;
+    private final Date createdAt;
+
+    private final List<SnapshotFile> srcs;
+    private final List<SnapshotAttachment> atts;
+
+    public Snapshot(SnapshotInfo info, SnapshotData data) {
+        versionID = info.getVersionId();
+        comment = info.getComment();
+        WLUser user = info.getUser();
+        userName = user.getName();
+        userEmail = user.getEmail();
+        createdAt = new DateTime(info.getCreatedAt()).toDate();
+
+        srcs = data.getSrcs();
+        atts = data.getAtts();
+    }
+
+    public int getVersionID() {
+        return versionID;
+    }
+
+    public String getComment() {
+        return comment;
+    }
+
+    public String getUserName() {
+        return userName;
+    }
+
+    public String getUserEmail() {
+        return userEmail;
+    }
+
+    public List<SnapshotFile> getSrcs() {
+        return srcs;
+    }
+
+    public List<SnapshotAttachment> getAtts() {
+        return atts;
+    }
+
+    public Date getCreatedAt() {
+        return createdAt;
+    }
+
+    @Override
+    public int compareTo(Snapshot snapshot) {
+        return Integer.compare(versionID, snapshot.versionID);
+    }
+
+    @Override
+    public String toString() {
+        return String.valueOf(versionID);
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/exception/FileLimitExceededException.java

@@ -0,0 +1,34 @@
+package uk.ac.ic.wlgitbridge.git.exception;
+
+import uk.ac.ic.wlgitbridge.util.Util;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+
+public class FileLimitExceededException extends GitUserException {
+
+    private final long numFiles;
+
+    private final long maxFiles;
+
+    public FileLimitExceededException(long numFiles, long maxFiles) {
+        this.numFiles = numFiles;
+        this.maxFiles = maxFiles;
+    }
+
+    @Override
+    public String getMessage() {
+        return "too many files";
+    }
+
+    @Override
+    public List<String> getDescriptionLines() {
+        return Arrays.asList(
+            "repository contains " +
+            numFiles + " files, which exceeds the limit of " +
+            maxFiles + " files"
+        );
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/exception/GitUserException.java

@@ -0,0 +1,14 @@
+package uk.ac.ic.wlgitbridge.git.exception;
+
+import java.util.List;
+
+/**
+ * Created by winston on 20/08/2016.
+ */
+public abstract class GitUserException extends Exception {
+
+    public abstract String getMessage();
+
+    public abstract List<String> getDescriptionLines();
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/exception/InvalidGitRepository.java

@@ -0,0 +1,22 @@
+package uk.ac.ic.wlgitbridge.git.exception;
+
+import java.util.Arrays;
+import java.util.List;
+
+public class InvalidGitRepository extends GitUserException {
+
+    @Override
+    public String getMessage() {
+        return "invalid git repo";
+    }
+
+    @Override
+    public List<String> getDescriptionLines() {
+        return Arrays.asList(
+                "Your Git repository contains a reference we cannot resolve.",
+                "If your project contains a Git submodule,",
+                "please remove it and try again."
+        );
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/exception/SizeLimitExceededException.java

@@ -0,0 +1,40 @@
+package uk.ac.ic.wlgitbridge.git.exception;
+
+import uk.ac.ic.wlgitbridge.util.Util;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+
+public class SizeLimitExceededException extends GitUserException {
+
+    private final Optional<String> path;
+
+    private final long actualSize;
+
+    private final long maxSize;
+
+    public SizeLimitExceededException(
+            Optional<String> path, long actualSize, long maxSize) {
+        this.path = path;
+        this.actualSize = actualSize;
+        this.maxSize = maxSize;
+    }
+
+    @Override
+    public String getMessage() {
+        return "file too big";
+    }
+
+    @Override
+    public List<String> getDescriptionLines() {
+        String filename =
+                path.isPresent() ? "File '" + path.get() + "' is" : "There's a file";
+        return Arrays.asList(
+                filename + " too large to push to "
+                        + Util.getServiceName() + " via git",
+                        "the recommended maximum file size is 50 MiB"
+        );
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/exception/SnapshotAPIException.java

@@ -0,0 +1,10 @@
+package uk.ac.ic.wlgitbridge.git.exception;
+
+import uk.ac.ic.wlgitbridge.snapshot.base.JSONSource;
+
+/**
+ * Created by winston on 20/08/2016.
+ */
+public abstract class SnapshotAPIException
+        extends GitUserException
+        implements JSONSource {}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLReceivePackFactory.java

@@ -0,0 +1,81 @@
+package uk.ac.ic.wlgitbridge.git.handler;
+
+import com.google.api.client.auth.oauth2.Credential;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.transport.ReceivePack;
+import org.eclipse.jgit.transport.resolver.ReceivePackFactory;
+import uk.ac.ic.wlgitbridge.bridge.Bridge;
+import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
+import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApi;
+import uk.ac.ic.wlgitbridge.git.handler.hook.WriteLatexPutHook;
+import uk.ac.ic.wlgitbridge.git.servlet.WLGitServlet;
+import uk.ac.ic.wlgitbridge.server.Oauth2Filter;
+import uk.ac.ic.wlgitbridge.util.Log;
+import uk.ac.ic.wlgitbridge.util.Util;
+
+import javax.servlet.http.HttpServletRequest;
+import java.util.Optional;
+
+/**
+ * Created by Winston on 02/11/14.
+ */
+/**
+ * One of the "big three" interfaces created by {@link WLGitServlet} to handle
+ * user Git requests.
+ *
+ * This class just puts a {@link WriteLatexPutHook} into the {@link ReceivePack}
+ * that it returns.
+ */
+public class WLReceivePackFactory
+        implements ReceivePackFactory<HttpServletRequest> {
+
+    private final RepoStore repoStore;
+
+    private final Bridge bridge;
+
+    public WLReceivePackFactory(RepoStore repoStore, Bridge bridge) {
+        this.repoStore = repoStore;
+        this.bridge = bridge;
+    }
+
+    /**
+     * Puts a {@link WriteLatexPutHook} into the returned {@link ReceivePack}.
+     *
+     * The {@link WriteLatexPutHook} needs our hostname, which we get from the
+     * original {@link HttpServletRequest}, used to provide a postback URL to
+     * the {@link SnapshotApi}. We also give it the oauth2 that we injected in
+     * the {@link Oauth2Filter}, and the {@link Bridge}.
+     *
+     * At this point, the repository will have been synced to the latest on
+     * Overleaf, but it's possible that an update happens on Overleaf while our
+     * put hook is running. In this case, we fail, and the user tries again,
+     * triggering another sync, and so on.
+     * @param httpServletRequest the original request
+     * @param repository the JGit {@link Repository} provided by
+     * {@link WLRepositoryResolver}
+     * @return a correctly hooked {@link ReceivePack}
+     */
+    @Override
+    public ReceivePack create(
+            HttpServletRequest httpServletRequest,
+            Repository repository
+    ) {
+        Log.info(
+                "[{}] Creating receive-pack",
+                repository.getWorkTree().getName()
+        );
+        Optional<Credential> oauth2 = Optional.ofNullable(
+                (Credential) httpServletRequest.getAttribute(
+                        Oauth2Filter.ATTRIBUTE_KEY));
+        ReceivePack receivePack = new ReceivePack(repository);
+        String hostname = Util.getPostbackURL();
+        if (hostname == null) {
+            hostname = httpServletRequest.getLocalName();
+        }
+        receivePack.setPreReceiveHook(
+                new WriteLatexPutHook(repoStore, bridge, hostname, oauth2)
+        );
+        return receivePack;
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLRepositoryResolver.java

@@ -0,0 +1,126 @@
+package uk.ac.ic.wlgitbridge.git.handler;
+
+import com.google.api.client.auth.oauth2.Credential;
+import org.eclipse.jgit.errors.RepositoryNotFoundException;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.transport.ServiceMayNotContinueException;
+import org.eclipse.jgit.transport.resolver.RepositoryResolver;
+import org.eclipse.jgit.transport.resolver.ServiceNotAuthorizedException;
+import uk.ac.ic.wlgitbridge.bridge.Bridge;
+import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
+import uk.ac.ic.wlgitbridge.git.handler.hook.WriteLatexPutHook;
+import uk.ac.ic.wlgitbridge.git.servlet.WLGitServlet;
+import uk.ac.ic.wlgitbridge.server.GitBridgeServer;
+import uk.ac.ic.wlgitbridge.server.Oauth2Filter;
+import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
+import uk.ac.ic.wlgitbridge.util.Log;
+import uk.ac.ic.wlgitbridge.util.Util;
+
+import javax.servlet.http.HttpServletRequest;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Optional;
+
+/**
+ * Created by Winston on 02/11/14.
+ */
+/**
+ * One of the "big three" interfaces created by {@link WLGitServlet} to handle
+ * user Git requests.
+ *
+ * This class is used by all Git requests to resolve a project name to a
+ * JGit {@link Repository}, or fail by throwing an exception.
+ *
+ * It has a single method, {@link #open(HttpServletRequest, String)}, which
+ * calls into the {@link Bridge} to synchronise the project with Overleaf, i.e.
+ * bringing it onto disk and applying commits to it until it is up-to-date with
+ * Overleaf.
+ */
+public class WLRepositoryResolver
+        implements RepositoryResolver<HttpServletRequest> {
+
+    private final Bridge bridge;
+
+    public WLRepositoryResolver(Bridge bridge) {
+        this.bridge = bridge;
+    }
+
+    /**
+     * Calls into the Bridge to resolve a project name to a JGit
+     * {@link Repository}, or throw an exception.
+     *
+     * On success, the repository will have been brought onto disk and updated
+     * to the latest (synced).
+     *
+     * In the case of clones and fetches, upload packs are created from the
+     * returned JGit {@link Repository} by the {@link WLUploadPackFactory}.
+     *
+     * The project lock is acquired for this process so it can't be swapped out.
+     *
+     * However, it can still be swapped out between this and a Git push. The
+     * push would fail due to the project changed on Overleaf between the sync
+     * and the actual push to Overleaf (performed by the
+     * {@link WLReceivePackFactory} and {@link WriteLatexPutHook}. In this case,
+     * the user will have to try again (which prompts another update, etc. until
+     * this no longer happens).
+     * @param httpServletRequest The HttpServletRequest as required by the
+     * interface. We injected the oauth2 creds into it with
+     * {@link Oauth2Filter}, which was set up by the {@link GitBridgeServer}.
+     * @param name The name of the project
+     * @return the JGit {@link Repository}.
+     * @throws RepositoryNotFoundException If the project does not exist
+     * @throws ServiceNotAuthorizedException If the user did not auth when
+     * required to
+     * @throws ServiceMayNotContinueException If any other general user
+     * exception occurs that must be propogated back to the user, e.g.
+     * internal errors (IOException, etc), too large file, and so on.
+     */
+    @Override
+    public Repository open(
+            HttpServletRequest httpServletRequest,
+            String name
+    ) throws RepositoryNotFoundException,
+             ServiceNotAuthorizedException,
+             ServiceMayNotContinueException {
+        Log.info("[{}] Request to open git repo", name);
+        Optional<Credential> oauth2 = Optional.ofNullable(
+                (Credential) httpServletRequest.getAttribute(
+                        Oauth2Filter.ATTRIBUTE_KEY));
+        String projName = Util.removeAllSuffixes(name, "/", ".git");
+        try {
+            return bridge.getUpdatedRepo(oauth2, projName).getJGitRepository();
+        } catch (RepositoryNotFoundException e) {
+            Log.info("Repository not found: " + name);
+            throw e;
+            /*
+        } catch (ServiceNotAuthorizedException e) {
+            cannot occur
+        } catch (ServiceNotEnabledException e) {
+            cannot occur
+            */
+        } catch (ServiceMayNotContinueException e) {
+            /* Such as FailedConnectionException */
+            throw e;
+        } catch (RuntimeException e) {
+            Log.warn(
+                    "Runtime exception when trying to open repo: " + projName,
+                    e
+            );
+            throw new ServiceMayNotContinueException(e);
+        } catch (ForbiddenException e) {
+            throw new ServiceNotAuthorizedException();
+        } catch (GitUserException e) {
+            throw new ServiceMayNotContinueException(
+                e.getMessage() + "\n" +
+                    String.join("\n", e.getDescriptionLines()),
+                e);
+        } catch (IOException e) {
+            Log.warn(
+                    "IOException when trying to open repo: " + projName,
+                    e
+            );
+            throw new ServiceMayNotContinueException("Internal server error.");
+        }
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLUploadPackFactory.java

@@ -0,0 +1,44 @@
+package uk.ac.ic.wlgitbridge.git.handler;
+
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.transport.UploadPack;
+import org.eclipse.jgit.transport.resolver.UploadPackFactory;
+import uk.ac.ic.wlgitbridge.git.servlet.WLGitServlet;
+import uk.ac.ic.wlgitbridge.util.Log;
+
+import javax.servlet.http.HttpServletRequest;
+
+/**
+ * Created by Winston on 02/11/14.
+ */
+/**
+ * One of the "big three" interfaces created by {@link WLGitServlet} to handle
+ * user Git requests.
+ *
+ * The actual class doesn't do much, and most of the work is done when the
+ * project name is being resolved by the {@link WLRepositoryResolver}.
+ */
+public class WLUploadPackFactory
+        implements UploadPackFactory<HttpServletRequest> {
+
+    /**
+     * This does nothing special. Synchronising the project with Overleaf will
+     * have been performed by {@link WLRepositoryResolver}.
+     * @param __ Not used, required by the {@link UploadPackFactory} interface
+     * @param repository The JGit repository provided by the
+     * {@link WLRepositoryResolver}
+     * @return the {@link UploadPack}, used by JGit to serve the request
+     */
+    @Override
+    public UploadPack create(
+            HttpServletRequest __,
+            Repository repository
+    ) {
+        Log.info(
+                "[{}] Creating upload-pack",
+                repository.getWorkTree().getName()
+        );
+       return new UploadPack(repository);
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/hook/WriteLatexPutHook.java

@@ -0,0 +1,178 @@
+package uk.ac.ic.wlgitbridge.git.handler.hook;
+
+import com.google.api.client.auth.oauth2.Credential;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.transport.PreReceiveHook;
+import org.eclipse.jgit.transport.ReceiveCommand;
+import org.eclipse.jgit.transport.ReceiveCommand.Result;
+import org.eclipse.jgit.transport.ReceivePack;
+import uk.ac.ic.wlgitbridge.bridge.Bridge;
+import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
+import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
+import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
+import uk.ac.ic.wlgitbridge.git.handler.WLReceivePackFactory;
+import uk.ac.ic.wlgitbridge.git.handler.hook.exception.ForcedPushException;
+import uk.ac.ic.wlgitbridge.git.handler.hook.exception.WrongBranchException;
+import uk.ac.ic.wlgitbridge.snapshot.push.exception.InternalErrorException;
+import uk.ac.ic.wlgitbridge.snapshot.push.exception.OutOfDateException;
+import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
+import uk.ac.ic.wlgitbridge.util.Log;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Optional;
+
+/**
+ * Created by Winston on 03/11/14.
+ */
+/**
+ * Created by {@link WLReceivePackFactory} to update the {@link Bridge} for a
+ * user's Git push request, or fail with an error. The hook is able to approve
+ * or reject a request.
+ */
+public class WriteLatexPutHook implements PreReceiveHook {
+
+    private final RepoStore repoStore;
+
+    private final Bridge bridge;
+    private final String hostname;
+    private final Optional<Credential> oauth2;
+
+    /**
+     * The constructor to use, which provides the hook with the {@link Bridge},
+     * the hostname (used to construct a URL to give to Overleaf to postback),
+     * and the oauth2 (used to authenticate with the Snapshot API).
+     * @param repoStore
+     * @param bridge the {@link Bridge}
+     * @param hostname the hostname used for postback from the Snapshot API
+     * @param oauth2 used to authenticate with the snapshot API, or null
+     */
+    public WriteLatexPutHook(
+            RepoStore repoStore,
+            Bridge bridge,
+            String hostname,
+            Optional<Credential> oauth2
+    ) {
+        this.repoStore = repoStore;
+        this.bridge = bridge;
+        this.hostname = hostname;
+        this.oauth2 = oauth2;
+    }
+
+    @Override
+    public void onPreReceive(
+            ReceivePack receivePack,
+            Collection<ReceiveCommand> receiveCommands
+    ) {
+        Log.debug("-> Handling {} commands in {}", receiveCommands.size(), receivePack.getRepository().getDirectory().getAbsolutePath());
+        for (ReceiveCommand receiveCommand : receiveCommands) {
+            try {
+                handleReceiveCommand(
+                        oauth2,
+                        receivePack.getRepository(),
+                        receiveCommand
+                );
+            } catch (IOException e) {
+                Log.debug("IOException on pre receive: {}", e.getMessage());
+                receivePack.sendError(e.getMessage());
+                receiveCommand.setResult(
+                        Result.REJECTED_OTHER_REASON,
+                        e.getMessage()
+                );
+            } catch (OutOfDateException e) {
+                Log.debug("OutOfDateException on pre receive: {}", e.getMessage());
+                receiveCommand.setResult(Result.REJECTED_NONFASTFORWARD);
+            } catch (GitUserException e) {
+                Log.debug("GitUserException on pre receive: {}", e.getMessage());
+                handleSnapshotPostException(receivePack, receiveCommand, e);
+            } catch (Throwable t) {
+                Log.warn("Throwable on pre receive: {}", t.getMessage());
+                handleSnapshotPostException(
+                        receivePack,
+                        receiveCommand,
+                        new InternalErrorException()
+                );
+            }
+        }
+        Log.debug("-> Handled {} commands in {}", receiveCommands.size(), receivePack.getRepository().getDirectory().getAbsolutePath());
+    }
+
+    private void handleSnapshotPostException(
+            ReceivePack receivePack,
+            ReceiveCommand receiveCommand,
+            GitUserException e
+    ) {
+        String message = e.getMessage();
+        receivePack.sendError(message);
+        StringBuilder msg = new StringBuilder();
+        for (
+                Iterator<String> it = e.getDescriptionLines().iterator();
+                it.hasNext();
+        ) {
+            String line = it.next();
+            msg.append("hint: ");
+            msg.append(line);
+            if (it.hasNext()) {
+                msg.append('\n');
+            }
+        }
+        receivePack.sendMessage("");
+        receivePack.sendMessage(msg.toString());
+        receiveCommand.setResult(Result.REJECTED_OTHER_REASON, message);
+    }
+
+    private void handleReceiveCommand(
+            Optional<Credential> oauth2,
+            Repository repository,
+            ReceiveCommand receiveCommand
+    ) throws IOException, GitUserException {
+        checkBranch(receiveCommand);
+        checkForcedPush(receiveCommand);
+        bridge.push(
+                oauth2,
+                repository.getWorkTree().getName(),
+                getPushedDirectoryContents(repository,
+                        receiveCommand),
+                getOldDirectoryContents(repository),
+                hostname
+        );
+    }
+
+    private void checkBranch(
+            ReceiveCommand receiveCommand
+    ) throws WrongBranchException {
+        if (!receiveCommand.getRefName().equals("refs/heads/master")) {
+            throw new WrongBranchException();
+        }
+    }
+
+    private void checkForcedPush(
+            ReceiveCommand receiveCommand
+    ) throws ForcedPushException {
+        if (
+                receiveCommand.getType()
+                        == ReceiveCommand.Type.UPDATE_NONFASTFORWARD
+        ) {
+            throw new ForcedPushException();
+        }
+    }
+
+    private RawDirectory getPushedDirectoryContents(
+            Repository repository,
+            ReceiveCommand receiveCommand
+    ) throws IOException, GitUserException {
+        return repoStore
+                .useJGitRepo(repository, receiveCommand.getNewId())
+                .getDirectory();
+    }
+
+    private RawDirectory getOldDirectoryContents(
+            Repository repository
+    ) throws IOException, GitUserException {
+        return repoStore
+                .useJGitRepo(repository, repository.resolve("HEAD"))
+                .getDirectory();
+    }
+
+}

services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/hook/exception/ForcedPushException.java

@@ -0,0 +1,37 @@
+package uk.ac.ic.wlgitbridge.git.handler.hook.exception;
+
+import com.google.gson.JsonElement;
+import uk.ac.ic.wlgitbridge.util.Util;
+import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Created by Winston on 16/11/14.
+ */
+public class ForcedPushException extends SnapshotPostException {
+
+    private static final String[] DESCRIPTION_LINES = {
+            "You can't git push --force to a "
+                    + Util.getServiceName()
+                    + " project.",
+            "Try to put your changes on top of the current head.",
+            "If everything else fails, delete and reclone your repository, "
+                    + "make your changes, then push again."
+    };
+
+    @Override
+    public String getMessage() {
+        return "forced push prohibited";
+    }
+
+    @Override
+    public List<String> getDescriptionLines() {
+        return Arrays.asList(DESCRIPTION_LINES);
+    }
+
+    @Override
+    public void fromJSON(JsonElement json) {}
+
+}