github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-14 20:40:17 -05:00
Code Issues Projects Releases Packages Wiki Activity

merge multiple repositories into an existing monorepo

Browse source 
- merged using: 'monorepo_add.sh services-git-bridge:services/git-bridge'
- see https://github.com/shopsys/monorepo-tools
This commit is contained in:
Jakob Ackermann 2021-08-05 08:33:29 +01:00
commit 4fd44292d0
No known key found for this signature in database
GPG key ID: 30C56800FCA3828A
547 changed files with 20788 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
services/git-bridge/.dockerignore Normal file
View file

@ -0,0 +1,9 @@
*
!start.sh
!/conf
!/lib
!/src/main
!/pom.xml
!/Makefile
!/LICENSE
!/vendor

21
services/git-bridge/.github/dependabot.yml vendored Normal file
View file

@ -0,0 +1,21 @@
version: 2
updates:
- package-ecosystem: "maven"
directory: "/"
schedule:
interval: "daily"
pull-request-branch-name:
# Separate sections of the branch name with a hyphen
# Docker images use the branch name and do not support slashes in tags
# https://github.com/overleaf/google-ops/issues/822
# https://docs.github.com/en/github/administering-a-repository/configuration-options-for-dependency-updates#pull-request-branch-nameseparator
separator: "-"
# Block informal upgrades -- security upgrades use a separate queue.
# https://docs.github.com/en/github/administering-a-repository/configuration-options-for-dependency-updates#open-pull-requests-limit
open-pull-requests-limit: 0
labels:
- "dependencies"
- "type:maintenance"

53
services/git-bridge/.gitignore vendored Normal file
View file

@ -0,0 +1,53 @@
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
# Let's not share anything because we're using Maven.
.idea
*.iml
# User-specific stuff:
.idea/workspace.xml
.idea/tasks.xml
.idea/dictionaries
.idea/vcs.xml
.idea/jsLibraryMappings.xml
# Sensitive or high-churn files:
.idea/dataSources.ids
.idea/dataSources.xml
.idea/dataSources.local.xml
.idea/sqlDataSources.xml
.idea/dynamic.xml
.idea/uiDesigner.xml
# Gradle:
.idea/gradle.xml
.idea/libraries
# Mongo Explorer plugin:
.idea/mongoSettings.xml
## File-based project format:
*.iws
## Plugin-specific files:
# IntelliJ
/out/
target/
# mpeltonen/sbt-idea plugin
.idea_modules/
# JIRA plugin
atlassian-ide-plugin.xml
# Crashlytics plugin (for Android Studio and IntelliJ)
com_crashlytics_export_strings.xml
crashlytics.properties
crashlytics-build.properties
fabric.properties
# Local configuration files
conf/runtime.json

58
services/git-bridge/Dockerfile Normal file
View file

@ -0,0 +1,58 @@
# Dockerfile for git-bridge
FROM maven:3-jdk-11 as base
RUN apt-get update && apt-get install -y make git sqlite3 \
&& rm -rf /var/lib/apt/lists
COPY vendor/envsubst /opt/envsubst
RUN chmod +x /opt/envsubst
RUN useradd --create-home node
FROM base as builder
COPY . /app
WORKDIR /app
RUN make package \
# The name of the created jar contains the current version tag.
# Rename it to a static path that can be used for copying.
&& find /app/target \
-name 'writelatex-git-bridge*jar-with-dependencies.jar' \
-exec mv {} /git-bridge.jar \;
FROM openjdk:11-jre
RUN apt-get update && apt-get install -y git sqlite3 procps htop net-tools sockstat libjemalloc2 \
&& rm -rf /var/lib/apt/lists
ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.2
# Install Google Cloud Profiler agent
RUN mkdir -p /opt/cprof && \
wget -q -O- https://storage.googleapis.com/cloud-profiler/java/latest/profiler_java_agent.tar.gz \
| tar xzv -C /opt/cprof
# Install Google Cloud Debugger agent
RUN mkdir /opt/cdbg && \
wget -qO- https://storage.googleapis.com/cloud-debugger/compute-java/debian-wheezy/cdbg_java_agent_gce.tar.gz | \
tar xvz -C /opt/cdbg
RUN useradd --create-home node
COPY --from=builder /git-bridge.jar /
COPY vendor/envsubst /opt/envsubst
RUN chmod +x /opt/envsubst
COPY conf/envsubst_template.json envsubst_template.json
COPY start.sh start.sh
RUN mkdir conf
RUN chown node:node conf
USER node
CMD ["/start.sh"]

22
services/git-bridge/LICENSE Normal file
View file

@ -0,0 +1,22 @@
The MIT License (MIT)
Copyright (c) 2014 Winston Li
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

31
services/git-bridge/Makefile Normal file
View file

@ -0,0 +1,31 @@
# git-bridge makefile
MVN_OPTS := "--no-transfer-progress"
runtime-conf:
/opt/envsubst < conf/envsubst_template.json > conf/runtime.json
run: package runtime-conf
java $(GIT_BRIDGE_JVM_ARGS) -jar \
target/writelatex-git-bridge-1.0-SNAPSHOT-jar-with-dependencies.jar \
conf/runtime.json
build:
mvn $(MVN_OPTS) package -DskipTests
test:
mvn $(MVN_OPTS) test
clean:
mvn $(MVN_OPTS) clean
package: clean
mvn $(MVN_OPTS) package -DskipTests
.PHONY: run package build clean test runtime-conf

138
services/git-bridge/README.md Normal file
View file

@ -0,0 +1,138 @@
# writelatex-git-bridge
## Docker
The `Dockerfile` contains all the requirements for building and running the
writelatex-git-bridge.
```bash
# build the image
docker build -t writelatex-git-bridge .
# run it with the demo config
docker run -v `pwd`/conf/local.json:/conf/runtime.json writelatex-git-bridge
```
## Native install
### Required packages
* `maven` (for building, running tests and packaging)
* `jdk-8` (for compiling and running)
### Commands
To be run from the base directory:
**Build jar**:
`mvn package`
**Run tests**:
`mvn test`
**Clean**:
`mvn clean`
To be run from the dev-environment:
**Build jar**:
`bin/run git-bridge make package`
**Run tests**:
`bin/run git-bridge make test`
**Clean**:
`bin/run git-bridge make clean`
### Installation
Install dependencies:
```
sudo apt-get update
sudo apt-get install -y maven
sudo apt-get install -y openjdk-8-jdk
sudo update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java
sudo update-alternatives --set javac /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/javac
```
Create a config file according to the format below.
Run `mvn package` to build, test, and package it into a jar at `target/writelatex-git-bridge-1.0-SNAPSHOT-jar-with-dependencies.jar`.
Use `java -jar <path_to_jar> <path_to_config_file>` to run the server.
## Runtime Configuration
The configuration file is in `.json` format.
{
"port" (int): the port number,
"rootGitDirectory" (string): the directory in which to store
git repos and the db/atts,
"apiBaseUrl" (string): base url for the snapshot api,
"username" (string, optional): username for http basic auth,
"password" (string, optional): password for http basic auth,
"postbackBaseUrl" (string): the postback url,
"serviceName" (string): current name of writeLaTeX
in case it ever changes,
"oauth2" (object): { null or missing if oauth2 shouldn't be used
"oauth2ClientID" (string): oauth2 client ID,
"oauth2ClientSecret" (string): oauth2 client secret,
"oauth2Server" (string): oauth2 server,
with protocol and
without trailing slash
},
"repoStore" (object, optional): { configure the repo store
"maxFileSize" (long, optional): maximum size of a file, inclusive
},
"swapStore" (object, optional): { the place to swap projects to.
if null, type defaults to
"noop"
"type" (string): "s3", "memory", "noop" (not recommended),
"awsAccessKey" (string, optional): only for s3,
"awsSecret" (string, optional): only for s3,
"s3BucketName" (string, optional): only for s3
},
"swapJob" (object, optional): { configure the project
swapping job.
if null, defaults to no-op
"minProjects" (int64): lower bound on number of projects
present. The swap job will never go
below this, regardless of what the
watermark shows. Regardless, if
minProjects prevents an eviction,
the swap job will WARN,
"lowGiB" (int32): the low watermark for swapping,
i.e. swap until disk usage is below this,
"highGiB" (int32): the high watermark for swapping,
i.e. start swapping when
disk usage becomes this,
"intervalMillis" (int64): amount of time in between running
swap job and checking watermarks.
3600000 is 1 hour
}
}
You have to restart the server for configuration changes to take effect.
## Creating OAuth app
In dev-env, run the following command in mongo to create the oauth application
for git-bridge.
```
db.oauthApplications.insert({
"clientSecret" : "e6b2e9eee7ae2bb653823250bb69594a91db0547fe3790a7135acb497108e62d",
"grants" : [
"password"
],
"id" : "264c723c925c13590880751f861f13084934030c13b4452901e73bdfab226edc",
"name" : "Overleaf Git Bridge",
"redirectUris" : [],
"scopes" : [
"git_bridge"
]
})
```

33
services/git-bridge/conf/envsubst_template.json Normal file
View file

@ -0,0 +1,33 @@
{
"port": ${GIT_BRIDGE_PORT:-8000},
"bindIp": "${GIT_BRIDGE_BIND_IP:-0.0.0.0}",
"idleTimeout": ${GIT_BRIDGE_IDLE_TIMEOUT:-30000},
"rootGitDirectory": "${GIT_BRIDGE_ROOT_DIR:-/tmp/wlgb}",
"apiBaseUrl": "${GIT_BRIDGE_API_BASE_URL:-https://localhost/api/v0}",
"postbackBaseUrl": "${GIT_BRIDGE_POSTBACK_BASE_URL:-https://localhost}",
"serviceName": "${GIT_BRIDGE_SERVICE_NAME:-Overleaf}",
"oauth2": {
"oauth2ClientID": "${GIT_BRIDGE_OAUTH2_CLIENT_ID}",
"oauth2ClientSecret": "${GIT_BRIDGE_OAUTH2_CLIENT_SECRET}",
"oauth2Server": "${GIT_BRIDGE_OAUTH2_SERVER:-https://localhost}"
},
"repoStore": {
"maxFileNum": ${GIT_BRIDGE_REPOSTORE_MAX_FILE_NUM:-2000},
"maxFileSize": ${GIT_BRIDGE_REPOSTORE_MAX_FILE_SIZE:-52428800}
},
"swapStore": {
"type": "${GIT_BRIDGE_SWAPSTORE_TYPE:-noop}",
"awsAccessKey": "${GIT_BRIDGE_SWAPSTORE_AWS_ACCESS_KEY}",
"awsSecret": "${GIT_BRIDGE_SWAPSTORE_AWS_SECRET}",
"s3BucketName": "${GIT_BRIDGE_SWAPSTORE_S3_BUCKET_NAME}",
"awsRegion": "${GIT_BRIDGE_SWAPSTORE_AWS_REGION:-us-east-1}"
},
"swapJob": {
"minProjects": ${GIT_BRIDGE_SWAPJOB_MIN_PROJECTS:-50},
"lowGiB": ${GIT_BRIDGE_SWAPJOB_LOW_GIB:-128},
"highGiB": ${GIT_BRIDGE_SWAPJOB_HIGH_GIB:-256},
"intervalMillis": ${GIT_BRIDGE_SWAPJOB_INTERVAL_MILLIS:-3600000},
"compressionMethod": "${GIT_BRIDGE_SWAPJOB_COMPRESSION_METHOD:-gzip}"
},
"sqliteHeapLimitBytes": ${GIT_BRIDGE_SQLITE_HEAP_LIMIT_BYTES:-0}
}

33
services/git-bridge/conf/example_config.json Normal file
View file

@ -0,0 +1,33 @@
{
"port": 8080,
"bindIp": "127.0.0.1",
"idleTimeout": 30000,
"rootGitDirectory": "/tmp/wlgb",
"apiBaseUrl": "https://localhost/api/v0",
"postbackBaseUrl": "https://localhost",
"serviceName": "Overleaf",
"oauth2": {
"oauth2ClientID": "asdf",
"oauth2ClientSecret": "asdf",
"oauth2Server": "https://localhost"
},
"repoStore": {
"maxFileNum": 2000,
"maxFileSize": 52428800
},
"swapStore": {
"type": "s3",
"awsAccessKey": "asdf",
"awsSecret": "asdf",
"s3BucketName": "com.overleaf.testbucket",
"awsRegion": "us-east-1"
},
"swapJob": {
"minProjects": 50,
"lowGiB": 128,
"highGiB": 256,
"intervalMillis": 3600000,
"compressionMethod": "gzip"
},
"sqliteHeapLimitBytes": 512000000
}

28
services/git-bridge/conf/local.json Normal file
View file

@ -0,0 +1,28 @@
{
"port": 8000,
"bindIp": "0.0.0.0",
"idleTimeout": 30000,
"rootGitDirectory": "/tmp/wlgb",
"apiBaseUrl": "http://v2.overleaf.test:4000/api/v0",
"postbackBaseUrl": "http://git-bridge:8000",
"serviceName": "Overleaf",
"oauth2": {
"oauth2ClientID": "264c723c925c13590880751f861f13084934030c13b4452901e73bdfab226edc",
"oauth2ClientSecret": "e6b2e9eee7ae2bb653823250bb69594a91db0547fe3790a7135acb497108e62d",
"oauth2Server": "http://v2.overleaf.test:4000"
},
"repoStore": {
"maxFileNum": 2000,
"maxFileSize": 52428800
},
"swapStore": {
"type": "noop"
},
"swapJob": {
"minProjects": 50,
"lowGiB": 128,
"highGiB": 256,
"intervalMillis": 3600000,
"compressionMethod": "gzip"
}
}

BIN
services/git-bridge/lib/newrelic.jar Normal file
View file

Binary file not shown.

300
services/git-bridge/newrelic.yml Normal file
View file

@ -0,0 +1,300 @@
# This file configures the New Relic Agent. New Relic monitors
# Java applications with deep visibility and low overhead. For more details and additional
# configuration options visit https://docs.newrelic.com/docs/java/java-agent-configuration.
#
# This configuration file is custom generated for Winston
#
# This section is for settings common to all environments.
# Do not add anything above this next line.
common: &default_settings
# ============================== LICENSE KEY ===============================
# You must specify the license key associated with your New Relic
# account. For example, if your license key is 12345 use this:
# license_key: '12345'
# The key binds your Agent's data to your account in the New Relic service.
license_key: '<LICENSE KEY>'
# Agent Enabled
# Use this setting to disable the agent instead of removing it from the startup command.
# Default is true.
agent_enabled: true
# Set the name of your application as you'd like it show up in New Relic.
# If enable_auto_app_naming is false, the agent reports all data to this application.
# Otherwise, the agent reports only background tasks (transactions for non-web applications)
# to this application. To report data to more than one application
# (useful for rollup reporting), separate the application names with ";".
# For example, to report data to "My Application" and "My Application 2" use this:
# app_name: My Application;My Application 2
# This setting is required. Up to 3 different application names can be specified.
# The first application name must be unique.
app_name: Git Bridge
# To enable high security, set this property to true. When in high
# security mode, the agent will use SSL and obfuscated SQL. Additionally,
# request parameters and message parameters will not be sent to New Relic.
high_security: false
# Set to true to enable support for auto app naming.
# The name of each web app is detected automatically
# and the agent reports data separately for each one.
# This provides a finer-grained performance breakdown for
# web apps in New Relic.
# Default is false.
enable_auto_app_naming: false
# Set to true to enable component-based transaction naming.
# Set to false to use the URI of a web request as the name of the transaction.
# Default is true.
enable_auto_transaction_naming: true
# The agent uses its own log file to keep its logging
# separate from that of your application. Specify the log level here.
# This setting is dynamic, so changes do not require restarting your application.
# The levels in increasing order of verboseness are:
# off, severe, warning, info, fine, finer, finest
# Default is info.
log_level: info
# Log all data sent to and from New Relic in plain text.
# This setting is dynamic, so changes do not require restarting your application.
# Default is false.
audit_mode: false
# The number of backup log files to save.
# Default is 1.
log_file_count: 1
# The maximum number of kbytes to write to any one log file.
# The log_file_count must be set greater than 1.
# Default is 0 (no limit).
log_limit_in_kbytes: 0
# Override other log rolling configuration and roll the logs daily.
# Default is false.
log_daily: false
# The name of the log file.
# Default is newrelic_agent.log.
log_file_name: newrelic_agent.log
# The log file directory.
# Default is the logs directory in the newrelic.jar parent directory.
#log_file_path:
# The agent communicates with New Relic via https by
# default. If you want to communicate with newrelic via http,
# then turn off SSL by setting this value to false.
# This work is done asynchronously to the threads that process your
# application code, so response times will not be directly affected
# by this change.
# Default is true.
ssl: true
# Proxy settings for connecting to the New Relic server:
# If a proxy is used, the host setting is required. Other settings
# are optional. Default port is 8080. The username and password
# settings will be used to authenticate to Basic Auth challenges
# from a proxy server.
#proxy_host: hostname
#proxy_port: 8080
#proxy_user: username
#proxy_password: password
# Limits the number of lines to capture for each stack trace.
# Default is 30
max_stack_trace_lines: 30
# Provides the ability to configure the attributes sent to New Relic. These
# attributes can be found in transaction traces, traced errors, Insight's
# transaction events, and Insight's page views.
attributes:
# When true, attributes will be sent to New Relic. The default is true.
enabled: true
#A comma separated list of attribute keys whose values should
# be sent to New Relic.
#include:
# A comma separated list of attribute keys whose values should
# not be sent to New Relic.
#exclude:
# Transaction tracer captures deep information about slow
# transactions and sends this to the New Relic service once a
# minute. Included in the transaction is the exact call sequence of
# the transactions including any SQL statements issued.
transaction_tracer:
# Transaction tracer is enabled by default. Set this to false to turn it off.
# This feature is not available to Lite accounts and is automatically disabled.
# Default is true.
enabled: true
# Threshold in seconds for when to collect a transaction
# trace. When the response time of a controller action exceeds
# this threshold, a transaction trace will be recorded and sent to
# New Relic. Valid values are any float value, or (default) "apdex_f",
# which will use the threshold for the "Frustrated" Apdex level
# (greater than four times the apdex_t value).
# Default is apdex_f.
transaction_threshold: apdex_f
# When transaction tracer is on, SQL statements can optionally be
# recorded. The recorder has three modes, "off" which sends no
# SQL, "raw" which sends the SQL statement in its original form,
# and "obfuscated", which strips out numeric and string literals.
# Default is obfuscated.
record_sql: obfuscated
# Set this to true to log SQL statements instead of recording them.
# SQL is logged using the record_sql mode.
# Default is false.
log_sql: false
# Threshold in seconds for when to collect stack trace for a SQL
# call. In other words, when SQL statements exceed this threshold,
# then capture and send to New Relic the current stack trace. This is
# helpful for pinpointing where long SQL calls originate from.
# Default is 0.5 seconds.
stack_trace_threshold: 0.5
# Determines whether the agent will capture query plans for slow
# SQL queries. Only supported for MySQL and PostgreSQL.
# Default is true.
explain_enabled: true
# Threshold for query execution time below which query plans will not
# not be captured. Relevant only when `explain_enabled` is true.
# Default is 0.5 seconds.
explain_threshold: 0.5
# Use this setting to control the variety of transaction traces.
# The higher the setting, the greater the variety.
# Set this to 0 to always report the slowest transaction trace.
# Default is 20.
top_n: 20
# Error collector captures information about uncaught exceptions and
# sends them to New Relic for viewing.
error_collector:
# This property enables the collection of errors. If the property is not
# set or the property is set to false, then errors will not be collected.
# Default is true.
enabled: true
# Use this property to exclude specific exceptions from being reported as errors
# by providing a comma separated list of full class names.
# The default is to exclude akka.actor.ActorKilledException. If you want to override
# this, you must provide any new value as an empty list is ignored.
ignore_errors: akka.actor.ActorKilledException
# Use this property to exclude specific http status codes from being reported as errors
# by providing a comma separated list of status codes.
# The default is to exclude 404s. If you want to override
# this, you must provide any new value as an empty list is ignored.
ignore_status_codes: 404
# Transaction Events are used for Histograms and Percentiles. Unaggregated data is collected
# for each web transaction and sent to the server on harvest.
transaction_events:
# Set to false to disable transaction events.
# Default is true.
enabled: true
# Events are collected up to the configured amount. Afterwards, events are sampled to
# maintain an even distribution across the harvest cycle.
# Default is 2000. Setting to 0 will disable.
max_samples_stored: 2000
# Cross Application Tracing adds request and response headers to
# external calls using supported HTTP libraries to provide better
# performance data when calling applications monitored by other New Relic Agents.
cross_application_tracer:
# Set to false to disable cross application tracing.
# Default is true.
enabled: true
# Thread profiler measures wall clock time, CPU time, and method call counts
# in your application's threads as they run.
# This feature is not available to Lite accounts and is automatically disabled.
thread_profiler:
# Set to false to disable the thread profiler.
# Default is true.
enabled: true
# New Relic Real User Monitoring gives you insight into the performance real users are
# experiencing with your website. This is accomplished by measuring the time it takes for
# your users' browsers to download and render your web pages by injecting a small amount
# of JavaScript code into the header and footer of each page.
browser_monitoring:
# By default the agent automatically inserts API calls in compiled JSPs to
# inject the monitoring JavaScript into web pages. Not all rendering engines are supported.
# See https://docs.newrelic.com/docs/java/real-user-monitoring-in-java#manual_instrumentation
# for instructions to add these manually to your pages.
# Set this attribute to false to turn off this behavior.
auto_instrument: true
class_transformer:
# This instrumentation reports the name of the user principal returned from
# HttpServletRequest.getUserPrincipal() when servlets and filters are invoked.
com.newrelic.instrumentation.servlet-user:
enabled: false
com.newrelic.instrumentation.spring-aop-2:
enabled: false
# Classes loaded by classloaders in this list will not be instrumented.
# This is a useful optimization for runtimes which use classloaders to
# load dynamic classes which the agent would not instrument.
classloader_excludes:
groovy.lang.GroovyClassLoader$InnerLoader,
org.codehaus.groovy.runtime.callsite.CallSiteClassLoader,
com.collaxa.cube.engine.deployment.BPELClassLoader,
org.springframework.data.convert.ClassGeneratingEntityInstantiator$ObjectInstantiatorClassGenerator,
org.mvel2.optimizers.impl.asm.ASMAccessorOptimizer$ContextClassLoader,
gw.internal.gosu.compiler.SingleServingGosuClassLoader,
# User-configurable custom labels for this agent. Labels are name-value pairs.
# There is a maximum of 64 labels per agent. Names and values are limited to 255 characters.
# Names and values may not contain colons (:) or semicolons (;).
labels:
# An example label
#label_name: label_value
# Application Environments
# ------------------------------------------
# Environment specific settings are in this section.
# You can use the environment to override the default settings.
# For example, to change the app_name setting.
# Use -Dnewrelic.environment=<environment> on the Java startup command line
# to set the environment.
# The default environment is production.
# NOTE if your application has other named environments, you should
# provide configuration settings for these environments here.
development:
<<: *default_settings
app_name: Git Bridge (Development)
test:
<<: *default_settings
app_name: Git Bridge (Test)
production:
<<: *default_settings
staging:
<<: *default_settings
app_name: Git Bridge (Staging)

229
services/git-bridge/pom.xml Normal file
View file

@ -0,0 +1,229 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>uk.ac.ic.wlgitbridge</groupId>
<artifactId>writelatex-git-bridge</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<!--<maven.test.skip>true</maven.test.skip>-->
</properties>
<build>
<plugins>
<!-- https://mvnrepository.com/artifact/org.apache.maven.plugins/maven-compiler-plugin -->
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.7.0</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<compilerArgument></compilerArgument>
</configuration>
</plugin>
<!-- Workaround, test loader crashes without this configuration option -->
<!-- See: https://stackoverflow.com/questions/53010200/maven-surefire-could-not-find-forkedbooter-class -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<argLine>-Djdk.net.URLClassPath.disableClassPathURLCheck=true</argLine>
</configuration>
</plugin>
<!-- https://mvnrepository.com/artifact/org.apache.maven.plugins/maven-assembly-plugin -->
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.1.0</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
<configuration>
<archive>
<manifest>
<mainClass>uk.ac.ic.wlgitbridge.Main</mainClass>
</manifest>
</archive>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
</plugin>
</plugins>
</build>
<dependencies>
<!-- https://mvnrepository.com/artifact/junit/junit -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.13.2</version>
<scope>test</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.jmock/jmock-junit4 -->
<dependency>
<groupId>org.jmock</groupId>
<artifactId>jmock-junit4</artifactId>
<version>2.8.4</version>
<scope>test</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.eclipse.jetty/jetty-servlet -->
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-servlet</artifactId>
<version>9.4.38.v20210224</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.eclipse.jetty/jetty-server -->
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
<version>9.4.38.v20210224</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.google.code.gson/gson -->
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.8.2</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.asynchttpclient/async-http-client -->
<dependency>
<groupId>org.asynchttpclient</groupId>
<artifactId>async-http-client</artifactId>
<version>2.3.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.eclipse.jgit/org.eclipse.jgit -->
<dependency>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit</artifactId>
<version>5.12.0.202106070339-r</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.eclipse.jgit/org.eclipse.jgit.http.server -->
<dependency>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit.http.server</artifactId>
<version>5.12.0.202106070339-r</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.xerial/sqlite-jdbc -->
<dependency>
<groupId>org.xerial</groupId>
<artifactId>sqlite-jdbc</artifactId>
<version>3.36.0.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/joda-time/joda-time -->
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>2.9.9</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.google.oauth-client/google-oauth-client -->
<dependency>
<groupId>com.google.oauth-client</groupId>
<artifactId>google-oauth-client</artifactId>
<version>1.23.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.google.http-client/google-http-client -->
<dependency>
<groupId>com.google.http-client</groupId>
<artifactId>google-http-client</artifactId>
<version>1.23.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.google.http-client/google-http-client-gson -->
<dependency>
<groupId>com.google.http-client</groupId>
<artifactId>google-http-client-gson</artifactId>
<version>1.23.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.12.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/ch.qos.logback/logback-classic -->
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.2.3</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.google.guava/guava -->
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>30.1.1-jre</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.mock-server/mockserver-netty -->
<dependency>
<groupId>org.mock-server</groupId>
<artifactId>mockserver-netty</artifactId>
<version>5.3.0</version>
<scope>test</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.mockito/mockito-core -->
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>3.11.1</version>
<scope>test</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk -->
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.11.274</version>
</dependency>
<!-- API, java.xml.bind module -->
<dependency>
<groupId>jakarta.xml.bind</groupId>
<artifactId>jakarta.xml.bind-api</artifactId>
<version>2.3.2</version>
</dependency>
<!-- Runtime, com.sun.xml.bind module -->
<dependency>
<groupId>org.glassfish.jaxb</groupId>
<artifactId>jaxb-runtime</artifactId>
<version>2.3.2</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient -->
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.5</version>
</dependency>
<!-- https://mvnrepository.com/artifact/commons-io/commons-io -->
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.10.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-compress -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>1.20</version>
</dependency>
<!-- prometheus metrics -->
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient</artifactId>
<version>0.10.0</version>
</dependency>
<!-- Hotspot JVM metrics -->
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient_hotspot</artifactId>
<version>0.10.0</version>
</dependency>
<!-- Expose metrics via a servlet -->
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient_servlet</artifactId>
<version>0.10.0</version>
</dependency>
</dependencies>
</project>

49
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/Main.java Normal file
View file

@ -0,0 +1,49 @@
package uk.ac.ic.wlgitbridge;
import uk.ac.ic.wlgitbridge.application.GitBridgeApp;
import uk.ac.ic.wlgitbridge.bridge.Bridge;
import uk.ac.ic.wlgitbridge.server.GitBridgeServer;
import uk.ac.ic.wlgitbridge.util.Log;
import java.util.Arrays;
/**
* Created by Winston on 01/11/14.
*/
/**
* This is the entry point into the Git Bridge.
*
* It is responsible for creating the {@link GitBridgeApp} and then running it.
*
* The {@link GitBridgeApp} parses args and creates the {@link GitBridgeServer}.
*
* The {@link GitBridgeServer} creates the {@link Bridge}, among other things.
*
* The {@link Bridge} is the heart of the Git Bridge. Start there, and follow
* the links outwards (which lead back to the Git users and the postback from
* the snapshot API) and inwards (which lead into the components of the Git
* Bridge: the configurable repo store, db store, and swap store, along with
* the project lock, the swap job, the snapshot API, the resource cache
* and the postback manager).
*/
public class Main {
public static void main(String[] args) {
Log.info(
"Git Bridge started with args: "
+ Arrays.toString(args)
);
try {
new GitBridgeApp(args).run();
} catch (Throwable t) {
/* So that we get a timestamp */
Log.error(
"Fatal exception thrown to top level, exiting: ",
t
);
System.exit(1);
}
}
}

102
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/GitBridgeApp.java Normal file
View file

@ -0,0 +1,102 @@
package uk.ac.ic.wlgitbridge.application;
import uk.ac.ic.wlgitbridge.application.config.Config;
import uk.ac.ic.wlgitbridge.application.exception.ArgsException;
import uk.ac.ic.wlgitbridge.application.exception.ConfigFileException;
import uk.ac.ic.wlgitbridge.server.GitBridgeServer;
import uk.ac.ic.wlgitbridge.util.Log;
import javax.servlet.ServletException;
import java.io.IOException;
/**
* Created by Winston on 02/11/14.
*/
/**
* Class that represents the application. Parses arguments and gives them to the
* server, or dies with a usage message.
*/
public class GitBridgeApp implements Runnable {
public static final int EXIT_CODE_FAILED = 1;
private static final String USAGE_MESSAGE =
"usage: writelatex-git-bridge [config_file]";
private String configFilePath;
Config config;
private GitBridgeServer server;
/**
* Constructs an instance of the WriteLatex-Git Bridge application.
* @param args args from main, which should be in the format [config_file]
*/
public GitBridgeApp(String[] args) {
try {
parseArguments(args);
loadConfigFile();
Log.info("Config loaded: {}", config.getSanitisedString());
} catch (ArgsException e) {
printUsage();
System.exit(EXIT_CODE_FAILED);
} catch (ConfigFileException e) {
Log.error(
"The property for " +
e.getMissingMember() +
" is invalid. Check your config file."
);
System.exit(EXIT_CODE_FAILED);
} catch (IOException e) {
Log.error("Invalid config file. Check the file path.");
System.exit(EXIT_CODE_FAILED);
}
try {
server = new GitBridgeServer(config);
} catch (ServletException e) {
Log.error(
"Servlet exception when instantiating GitBridgeServer",
e
);
}
}
/**
* Starts the server with the port number and root directory path given in
* the command-line arguments.
*/
@Override
public void run() {
server.start();
}
public void stop() {
server.stop();
}
/* Helper methods */
private void parseArguments(String[] args) throws ArgsException {
checkArgumentsLength(args);
parseConfigFilePath(args);
}
private void checkArgumentsLength(String[] args) throws ArgsException {
if (args.length < 1) {
throw new ArgsException();
}
}
private void parseConfigFilePath(String[] args) throws ArgsException {
configFilePath = args[0];
}
private void loadConfigFile() throws ConfigFileException, IOException {
Log.info("Loading config file at path: " + configFilePath);
config = new Config(configFilePath);
}
private void printUsage() {
System.err.println(USAGE_MESSAGE);
}
}

211
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java Normal file
View file

@ -0,0 +1,211 @@
package uk.ac.ic.wlgitbridge.application.config;
import com.google.gson.Gson;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import uk.ac.ic.wlgitbridge.application.exception.ConfigFileException;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStoreConfig;
import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJobConfig;
import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStoreConfig;
import uk.ac.ic.wlgitbridge.snapshot.base.JSONSource;
import uk.ac.ic.wlgitbridge.util.Instance;
import javax.annotation.Nullable;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.util.Optional;
/**
* Created by Winston on 05/12/14.
*/
public class Config implements JSONSource {
static Config asSanitised(Config config) {
return new Config(
config.port,
config.bindIp,
config.idleTimeout,
config.rootGitDirectory,
config.apiBaseURL,
config.postbackURL,
config.serviceName,
Oauth2.asSanitised(config.oauth2),
config.repoStore,
SwapStoreConfig.sanitisedCopy(config.swapStore),
config.swapJob,
config.sqliteHeapLimitBytes
);
}
private int port;
private String bindIp;
private int idleTimeout;
private String rootGitDirectory;
private String apiBaseURL;
private String postbackURL;
private String serviceName;
@Nullable
private Oauth2 oauth2;
@Nullable
private RepoStoreConfig repoStore;
@Nullable
private SwapStoreConfig swapStore;
@Nullable
private SwapJobConfig swapJob;
private int sqliteHeapLimitBytes = 0;
public Config(
String configFilePath
) throws ConfigFileException,
IOException {
this(new FileReader(configFilePath));
}
Config(Reader reader) {
fromJSON(new Gson().fromJson(reader, JsonElement.class));
}
public Config(
int port,
String bindIp,
int idleTimeout,
String rootGitDirectory,
String apiBaseURL,
String postbackURL,
String serviceName,
Oauth2 oauth2,
RepoStoreConfig repoStore,
SwapStoreConfig swapStore,
SwapJobConfig swapJob,
int sqliteHeapLimitBytes
) {
this.port = port;
this.bindIp = bindIp;
this.idleTimeout = idleTimeout;
this.rootGitDirectory = rootGitDirectory;
this.apiBaseURL = apiBaseURL;
this.postbackURL = postbackURL;
this.serviceName = serviceName;
this.oauth2 = oauth2;
this.repoStore = repoStore;
this.swapStore = swapStore;
this.swapJob = swapJob;
this.sqliteHeapLimitBytes = sqliteHeapLimitBytes;
}
@Override
public void fromJSON(JsonElement json) {
JsonObject configObject = json.getAsJsonObject();
port = getElement(configObject, "port").getAsInt();
bindIp = getElement(configObject, "bindIp").getAsString();
idleTimeout = getElement(configObject, "idleTimeout").getAsInt();
rootGitDirectory = getElement(
configObject,
"rootGitDirectory"
).getAsString();
String apiBaseURL = getElement(
configObject,
"apiBaseUrl"
).getAsString();
if (!apiBaseURL.endsWith("/")) {
apiBaseURL += "/";
}
this.apiBaseURL = apiBaseURL;
serviceName = getElement(configObject, "serviceName").getAsString();
postbackURL = getElement(configObject, "postbackBaseUrl").getAsString();
if (!postbackURL.endsWith("/")) {
postbackURL += "/";
}
oauth2 = new Gson().fromJson(configObject.get("oauth2"), Oauth2.class);
repoStore = new Gson().fromJson(
configObject.get("repoStore"), RepoStoreConfig.class);
swapStore = new Gson().fromJson(
configObject.get("swapStore"),
SwapStoreConfig.class
);
swapJob = new Gson().fromJson(
configObject.get("swapJob"),
SwapJobConfig.class
);
if (configObject.has("sqliteHeapLimitBytes")) {
sqliteHeapLimitBytes = getElement(configObject, "sqliteHeapLimitBytes").getAsInt();
}
}
public String getSanitisedString() {
return Instance.prettyGson.toJson(Config.asSanitised(this));
}
public int getPort() {
return port;
}
public String getBindIp() {
return bindIp;
}
public int getIdleTimeout() {
return idleTimeout;
}
public String getRootGitDirectory() {
return rootGitDirectory;
}
public int getSqliteHeapLimitBytes() {
return this.sqliteHeapLimitBytes;
}
public String getAPIBaseURL() {
return apiBaseURL;
}
public String getServiceName() {
return serviceName;
}
public String getPostbackURL() {
return postbackURL;
}
public boolean isUsingOauth2() {
return oauth2 != null;
}
public Oauth2 getOauth2() {
if (!isUsingOauth2()) {
throw new AssertionError("Getting oauth2 when not using it");
}
return oauth2;
}
public Optional<RepoStoreConfig> getRepoStore() {
return Optional.ofNullable(repoStore);
}
public Optional<SwapStoreConfig> getSwapStore() {
return Optional.ofNullable(swapStore);
}
public Optional<SwapJobConfig> getSwapJob() {
return Optional.ofNullable(swapJob);
}
private JsonElement getElement(JsonObject configObject, String name) {
JsonElement element = configObject.get(name);
if (element == null) {
throw new RuntimeException(new ConfigFileException(name));
}
return element;
}
private String getOptionalString(JsonObject configObject, String name) {
JsonElement element = configObject.get(name);
if (element == null || !element.isJsonPrimitive()) {
return "";
}
return element.getAsString();
}
}

42
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Oauth2.java Normal file
View file

@ -0,0 +1,42 @@
package uk.ac.ic.wlgitbridge.application.config;
/**
* Created by winston on 25/10/15.
*/
public class Oauth2 {
private final String oauth2ClientID;
private final String oauth2ClientSecret;
private final String oauth2Server;
public Oauth2(
String oauth2ClientID,
String oauth2ClientSecret,
String oauth2Server
) {
this.oauth2ClientID = oauth2ClientID;
this.oauth2ClientSecret = oauth2ClientSecret;
this.oauth2Server = oauth2Server;
}
public String getOauth2ClientID() {
return oauth2ClientID;
}
public String getOauth2ClientSecret() {
return oauth2ClientSecret;
}
public String getOauth2Server() {
return oauth2Server;
}
public static Oauth2 asSanitised(Oauth2 oauth2) {
return new Oauth2(
"<oauth2ClientID>",
"<oauth2ClientSecret>",
oauth2.oauth2Server
);
}
}

6
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/exception/ArgsException.java Normal file
View file

@ -0,0 +1,6 @@
package uk.ac.ic.wlgitbridge.application.exception;
/**
* Created by Winston on 03/11/14.
*/
public class ArgsException extends Exception {}

18
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/exception/ConfigFileException.java Normal file
View file

@ -0,0 +1,18 @@
package uk.ac.ic.wlgitbridge.application.exception;
/**
* Created by Winston on 05/12/14.
*/
public class ConfigFileException extends Exception {
private final String missingMember;
public ConfigFileException(String missingMember) {
this.missingMember = missingMember;
}
public String getMissingMember() {
return missingMember;
}
}

85
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/jetty/NullLogger.java Normal file
View file

@ -0,0 +1,85 @@
package uk.ac.ic.wlgitbridge.application.jetty;
import org.eclipse.jetty.util.log.Logger;
/**
* Created by Winston on 03/11/14.
*/
public class NullLogger implements Logger {
@Override
public String getName() {
return "null_logger";
}
@Override
public void warn(String s, Object... objects) {
}
@Override
public void warn(Throwable throwable) {
}
@Override
public void warn(String s, Throwable throwable) {
}
@Override
public void info(String s, Object... objects) {
}
@Override
public void info(Throwable throwable) {
}
@Override
public void info(String s, Throwable throwable) {
}
@Override
public boolean isDebugEnabled() {
return false;
}
@Override
public void setDebugEnabled(boolean b) {
}
@Override
public void debug(String s, Object... objects) {
}
@Override
public void debug(String s, long l) {
}
@Override
public void debug(Throwable throwable) {
}
@Override
public void debug(String s, Throwable throwable) {
}
@Override
public Logger getLogger(String s) {
return this;
}
@Override
public void ignore(Throwable throwable) {
}
}

839
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java Normal file
View file

@ -0,0 +1,839 @@
package uk.ac.ic.wlgitbridge.bridge;
import com.google.api.client.auth.oauth2.Credential;
import org.eclipse.jgit.errors.RepositoryNotFoundException;
import uk.ac.ic.wlgitbridge.application.config.Config;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.db.ProjectState;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SqliteDBStore;
import uk.ac.ic.wlgitbridge.bridge.gc.GcJob;
import uk.ac.ic.wlgitbridge.bridge.gc.GcJobImpl;
import uk.ac.ic.wlgitbridge.bridge.lock.LockGuard;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.bridge.repo.*;
import uk.ac.ic.wlgitbridge.bridge.resource.ResourceCache;
import uk.ac.ic.wlgitbridge.bridge.resource.UrlResourceCache;
import uk.ac.ic.wlgitbridge.bridge.snapshot.NetSnapshotApi;
import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApi;
import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApiFacade;
import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJob;
import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJobImpl;
import uk.ac.ic.wlgitbridge.bridge.swap.store.S3SwapStore;
import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore;
import uk.ac.ic.wlgitbridge.data.CandidateSnapshot;
import uk.ac.ic.wlgitbridge.data.ProjectLockImpl;
import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import uk.ac.ic.wlgitbridge.data.model.Snapshot;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.git.exception.SizeLimitExceededException;
import uk.ac.ic.wlgitbridge.git.exception.FileLimitExceededException;
import uk.ac.ic.wlgitbridge.git.handler.WLReceivePackFactory;
import uk.ac.ic.wlgitbridge.git.handler.WLRepositoryResolver;
import uk.ac.ic.wlgitbridge.git.handler.WLUploadPackFactory;
import uk.ac.ic.wlgitbridge.git.handler.hook.WriteLatexPutHook;
import uk.ac.ic.wlgitbridge.server.FileHandler;
import uk.ac.ic.wlgitbridge.server.PostbackContents;
import uk.ac.ic.wlgitbridge.server.PostbackHandler;
import uk.ac.ic.wlgitbridge.snapshot.base.MissingRepositoryException;
import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocResult;
import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotAttachment;
import uk.ac.ic.wlgitbridge.snapshot.push.PostbackManager;
import uk.ac.ic.wlgitbridge.snapshot.push.PostbackPromise;
import uk.ac.ic.wlgitbridge.snapshot.push.PushResult;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.*;
import uk.ac.ic.wlgitbridge.util.Log;
import java.io.File;
import java.io.IOException;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.util.*;
/**
* This is the heart of the Git Bridge. You plug in all the parts (project
* lock, repo store, db store, swap store, snapshot api, resource cache and
* postback manager) is called by Git user requests and Overleaf postback
* requests.
*
* Follow these links to go "outward" (to input from Git users and Overleaf):
*
* 1. JGit hooks, which handle user Git requests:
*
* @see WLRepositoryResolver - used on all requests associate a repo with a
* project name, or fail
*
* @see WLUploadPackFactory - used to handle clones and fetches
*
* @see WLReceivePackFactory - used to handle pushes by setting a hook
* @see WriteLatexPutHook - the hook used to handle pushes
*
* 2. The Postback Servlet, which handles postbacks from the Overleaf app
* to confirm that a project is pushed. If a postback is lost, it's fine, we
* just update ourselves on the next access.
*
* @see PostbackHandler - the entry point for postbacks
*
* Follow these links to go "inward" (to the Git Bridge components):
*
* 1. The Project Lock, used to synchronise accesses to projects and shutdown
* the Git Bridge gracefully by preventing further lock acquiring.
*
* @see ProjectLock - the interface used for the Project Lock
* @see ProjectLockImpl - the default concrete implementation
*
* 2. The Repo Store, used to provide repository objects.
*
* The default implementation uses Git on the file system.
*
* @see RepoStore - the interface for the Repo Store
* @see FSGitRepoStore - the default concrete implementation
* @see ProjectRepo - an interface for an actual repo instance
* @see GitProjectRepo - the default concrete implementation
*
* 3. The DB Store, used to store persistent data such as the latest version
* of each project that we have (used for querying the Snapshot API), along
* with caching remote blobs.
*
* The default implementation is SQLite based.
*
* @see DBStore - the interface for the DB store
* @see SqliteDBStore - the default concrete implementation
*
* 4. The Swap Store, used to swap projects to when the disk goes over a
* certain data usage.
*
* The default implementation tarbzips projects to/from Amazon S3.
*
* @see SwapStore - the interface for the Swap Store
* @see S3SwapStore - the default concrete implementation
*
* 5. The Swap Job, which performs the actual swapping on the swap store based
* on various configuration options.
*
* @see SwapJob - the interface for the Swap Job
* @see SwapJobImpl - the default concrete implementation
*
* 6. The Snapshot API, which provides data from the Overleaf app.
*
* @see SnapshotApiFacade - wraps a concrete instance of the Snapshot API.
* @see SnapshotApi - the interface for the Snapshot API.
* @see NetSnapshotApi - the default concrete implementation
*
* 7. The Resource Cache, which provides the data for attachment resources from
* URLs. It will generally fetch from the source on a cache miss.
*
* The default implementation uses the DB Store to maintain a mapping from
* URLs to files in an actual repo.
*
* @see ResourceCache - the interface for the Resource Cache
* @see UrlResourceCache - the default concrete implementation
*
* 8. The Postback Manager, which keeps track of pending postbacks. It stores a
* mapping from project names to postback promises.
*
* @see PostbackManager - the class
* @see PostbackPromise - the object waited on for a postback.
*
*/
public class Bridge {
private final Config config;
private final ProjectLock lock;
private final RepoStore repoStore;
private final DBStore dbStore;
private final SwapStore swapStore;
private final SwapJob swapJob;
private final GcJob gcJob;
private final SnapshotApiFacade snapshotAPI;
private final ResourceCache resourceCache;
private final PostbackManager postbackManager;
/**
* Creates a Bridge from its configurable parts, which are the repo, db and
* swap store, and the swap job config.
*
* This should be the method used to create a Bridge.
* @param config The config to use
* @param repoStore The repo store to use
* @param dbStore The db store to use
* @param swapStore The swap store to use
* @param snapshotApi The snapshot api to use
* @return The constructed Bridge.
*/
public static Bridge make(
Config config,
RepoStore repoStore,
DBStore dbStore,
SwapStore swapStore,
SnapshotApi snapshotApi
) {
ProjectLock lock = new ProjectLockImpl((int threads) ->
Log.info("Waiting for " + threads + " projects...")
);
return new Bridge(
config,
lock,
repoStore,
dbStore,
swapStore,
SwapJob.fromConfig(
config.getSwapJob(),
lock,
repoStore,
dbStore,
swapStore
),
new GcJobImpl(repoStore, lock),
new SnapshotApiFacade(snapshotApi),
new UrlResourceCache(dbStore)
);
}
/**
* Creates a bridge from all of its components, not just its configurable
* parts. This is for substituting mock/stub components for testing.
* It's also used by Bridge.make to actually construct the bridge.
* @param lock the {@link ProjectLock} to use
* @param repoStore the {@link RepoStore} to use
* @param dbStore the {@link DBStore} to use
* @param swapStore the {@link SwapStore} to use
* @param swapJob the {@link SwapJob} to use
* @param gcJob
* @param snapshotAPI the {@link SnapshotApi} to use
* @param resourceCache the {@link ResourceCache} to use
*/
Bridge(
Config config,
ProjectLock lock,
RepoStore repoStore,
DBStore dbStore,
SwapStore swapStore,
SwapJob swapJob,
GcJob gcJob,
SnapshotApiFacade snapshotAPI,
ResourceCache resourceCache
) {
this.config = config;
this.lock = lock;
this.repoStore = repoStore;
this.dbStore = dbStore;
this.swapStore = swapStore;
this.snapshotAPI = snapshotAPI;
this.resourceCache = resourceCache;
this.swapJob = swapJob;
this.gcJob = gcJob;
postbackManager = new PostbackManager();
Runtime.getRuntime().addShutdownHook(new Thread(this::doShutdown));
repoStore.purgeNonexistentProjects(dbStore.getProjectNames());
}
/**
* This performs the graceful shutdown of the Bridge, which is called by the
* shutdown hook. It acquires the project write lock, which prevents
* work being done for new projects (which acquire the read lock).
* Once it has the write lock, there are no readers left, so the git bridge
* can shut down gracefully.
*
* It is also used by the tests.
*/
void doShutdown() {
Log.info("Shutdown received.");
Log.info("Stopping SwapJob");
swapJob.stop();
Log.info("Stopping GcJob");
gcJob.stop();
Log.info("Waiting for projects");
lock.lockAll();
Log.info("Bye");
}
/**
* Starts the swap job, which will begin checking whether projects should be
* swapped with a configurable frequency.
*/
public void startBackgroundJobs() {
swapJob.start();
gcJob.start();
}
public boolean healthCheck() {
try {
dbStore.getNumProjects();
File rootDirectory = new File("/");
if (!rootDirectory.exists()) {
throw new Exception("bad filesystem state, root directory does not exist");
}
Log.info("[HealthCheck] passed");
return true;
} catch (Exception e) {
Log.error("[HealthCheck] FAILED!", e);
return false;
}
}
/**
* Performs a check of inconsistencies in the DB. This was used to upgrade
* the schema.
*/
public void checkDB() {
Log.info("Checking DB");
File rootDir = repoStore.getRootDirectory();
for (File f : rootDir.listFiles()) {
if (f.getName().equals(".wlgb")) {
continue;
}
String projName = f.getName();
try (LockGuard __ = lock.lockGuard(projName)) {
File dotGit = new File(f, ".git");
if (!dotGit.exists()) {
Log.warn("Project: {} has no .git", projName);
continue;
}
ProjectState state = dbStore.getProjectState(projName);
if (state != ProjectState.NOT_PRESENT) {
continue;
}
Log.warn(
"Project: {} not in swap_store, adding",
projName
);
dbStore.setLastAccessedTime(
projName,
new Timestamp(dotGit.lastModified())
);
}
}
}
/**
* Synchronises the given repository with Overleaf.
*
* It acquires the project lock and calls
* {@link #getUpdatedRepoCritical(Optional, String, GetDocResult)}.
* @param oauth2 The oauth2 to use
* @param projectName The name of the project
* @throws IOException
* @throws GitUserException
*/
public ProjectRepo getUpdatedRepo(
Optional<Credential> oauth2,
String projectName
) throws IOException, GitUserException {
try (LockGuard __ = lock.lockGuard(projectName)) {
Optional<GetDocResult> maybeDoc = snapshotAPI.getDoc(oauth2, projectName);
if (!maybeDoc.isPresent()) {
throw new RepositoryNotFoundException(projectName);
}
GetDocResult doc = maybeDoc.get();
Log.info("[{}] Updating repository", projectName);
return getUpdatedRepoCritical(oauth2, projectName, doc);
}
}
/**
* Synchronises the given repository with Overleaf.
*
* Pre: the project lock must be acquired for the given repo.
*
* 1. Queries the project state for the given project name.
* a. NOT_PRESENT = We've never seen it before, and the row for the
* project doesn't even exist. The project definitely
* exists because we would have aborted otherwise.
* b. PRESENT = The project is on disk.
* c. SWAPPED = The project is in the {@link SwapStore}
*
* If the project has never been cloned, it is git init'd. If the project
* is in swap, it is restored to disk. Otherwise, the project was already
* present.
*
* With the project present, snapshots are downloaded from the snapshot
* API with {@link #updateProject(Optional, ProjectRepo)}.
*
* Then, the last accessed time of the project is set to the current time.
* This is to support the LRU of the swap store.
* @param oauth2
* @param projectName The name of the project
* @throws IOException
* @throws GitUserException
*/
private ProjectRepo getUpdatedRepoCritical(
Optional<Credential> oauth2,
String projectName,
GetDocResult doc
) throws IOException, GitUserException {
ProjectRepo repo;
ProjectState state = dbStore.getProjectState(projectName);
switch (state) {
case NOT_PRESENT:
Log.info("[{}] Repo not present", projectName);
String migratedFromID = doc.getMigratedFromID();
if (migratedFromID != null) {
Log.info("[{}] Has a migratedFromId: {}", projectName, migratedFromID);
try (LockGuard __ = lock.lockGuard(migratedFromID)) {
ProjectState sourceState = dbStore.getProjectState(migratedFromID);
switch (sourceState) {
case NOT_PRESENT:
// Normal init-repo
Log.info("[{}] migrated-from project not present, proceed as normal",
projectName
);
repo = repoStore.initRepo(projectName);
break;
case SWAPPED:
// Swap back and then copy
swapJob.restore(migratedFromID);
/* Fallthrough */
default:
// Copy data, and set version to zero
Log.info("[{}] Init from other project: {}",
projectName,
migratedFromID
);
repo = repoStore.initRepoFromExisting(projectName, migratedFromID);
dbStore.setLatestVersionForProject(migratedFromID, 0);
dbStore.setLastAccessedTime(
migratedFromID,
Timestamp.valueOf(LocalDateTime.now())
);
}
}
break;
} else {
repo = repoStore.initRepo(projectName);
break;
}
case SWAPPED:
swapJob.restore(projectName);
/* Fallthrough */
default:
repo = repoStore.getExistingRepo(projectName);
}
updateProject(oauth2, repo);
dbStore.setLastAccessedTime(
projectName,
Timestamp.valueOf(LocalDateTime.now())
);
return repo;
}
/**
* The public call to push a project.
*
* It acquires the lock and calls {@link #pushCritical(
* Optional,
* String,
* RawDirectory,
* RawDirectory
* )}, catching exceptions, logging, and rethrowing them.
* @param oauth2 The oauth2 to use for the snapshot API
* @param projectName The name of the project to push to
* @param directoryContents The new contents of the project
* @param oldDirectoryContents The old contents of the project
* @param hostname
* @throws SnapshotPostException
* @throws IOException
* @throws MissingRepositoryException
* @throws ForbiddenException
* @throws GitUserException
*/
public void push(
Optional<Credential> oauth2,
String projectName,
RawDirectory directoryContents,
RawDirectory oldDirectoryContents,
String hostname
) throws SnapshotPostException, IOException, MissingRepositoryException, ForbiddenException, GitUserException {
Log.debug("[{}] pushing to Overleaf", projectName);
try (LockGuard __ = lock.lockGuard(projectName)) {
pushCritical(
oauth2,
projectName,
directoryContents,
oldDirectoryContents
);
} catch (SevereSnapshotPostException e) {
Log.warn(
"[" + projectName + "] Failed to put to Overleaf",
e
);
throw e;
} catch (SnapshotPostException e) {
/* Stack trace should be printed further up */
Log.warn(
"[{}] Exception when waiting for postback: {}",
projectName,
e.getClass().getSimpleName()
);
throw e;
} catch (IOException e) {
Log.warn("[{}] IOException on put: {}", projectName, e);
throw e;
}
gcJob.queueForGc(projectName);
}
/**
* Does the work of pushing to a project, assuming the project lock is held.
* The {@link WriteLatexPutHook} is the original caller, and when we return
* without throwing, the commit is committed.
*
* We start off by creating a postback key, which is given in the url when
* the Overleaf app tries to access the atts.
*
* Then creates a {@link CandidateSnapshot} from the old and new project
* contents. The
* {@link CandidateSnapshot} is created using
* {@link #createCandidateSnapshot(String, RawDirectory, RawDirectory)},
* which creates the snapshot object and writes the push files to the
* atts directory, which is served by the {@link PostbackHandler}.
* The files are deleted at the end of a try-with-resources block.
*
* Then 3 things are used to make the push request to the snapshot API:
* 1. The oauth2
* 2. The candidate snapshot
* 3. The postback key
*
* If the snapshot API reports this as not successful, we immediately throw
* an {@link OutOfDateException}, which goes back to the user.
*
* Otherwise, we wait (with a timeout) on a promise from the postback
* manager, which can throw back to the user.
*
* If this is successful, we approve the snapshot with
* {@link #approveSnapshot(int, CandidateSnapshot)}, which updates our side
* of the push: the latest version and the URL index store.
*
* Then, we set the last accessed time for the swap store.
*
* Finally, after we return, the push to the repo from the hook is
* successful and the repo gets updated.
*
* @param oauth2
* @param projectName
* @param directoryContents
* @param oldDirectoryContents
* @throws IOException
* @throws MissingRepositoryException
* @throws ForbiddenException
* @throws SnapshotPostException
* @throws GitUserException
*/
private void pushCritical(
Optional<Credential> oauth2,
String projectName,
RawDirectory directoryContents,
RawDirectory oldDirectoryContents
) throws IOException, MissingRepositoryException, ForbiddenException, SnapshotPostException, GitUserException {
Optional<Long> maxFileNum = config
.getRepoStore()
.flatMap(RepoStoreConfig::getMaxFileNum);
if (maxFileNum.isPresent()) {
long maxFileNum_ = maxFileNum.get();
if (directoryContents.getFileTable().size() > maxFileNum_) {
Log.debug("[{}] Too many files: {}/{}", projectName, directoryContents.getFileTable().size(), maxFileNum_);
throw new FileLimitExceededException(directoryContents.getFileTable().size(), maxFileNum_);
}
}
Log.info("[{}] Pushing files ({} new, {} old)", projectName, directoryContents.getFileTable().size(), oldDirectoryContents.getFileTable().size());
String postbackKey = postbackManager.makeKeyForProject(projectName);
Log.info(
"[{}] Created postback key: {}",
projectName,
postbackKey
);
try (
CandidateSnapshot candidate = createCandidateSnapshot(
projectName,
directoryContents,
oldDirectoryContents
);
) {
Log.info(
"[{}] Candidate snapshot created: {}",
projectName,
candidate
);
PushResult result
= snapshotAPI.push(oauth2, candidate, postbackKey);
if (result.wasSuccessful()) {
Log.info(
"[{}] Push to Overleaf successful",
projectName
);
Log.info("[{}] Waiting for postback...", projectName);
int versionID =
postbackManager.waitForVersionIdOrThrow(projectName);
Log.info(
"[{}] Got version ID for push: {}",
projectName,
versionID
);
approveSnapshot(versionID, candidate);
Log.info(
"[{}] Approved version ID: {}",
projectName,
versionID
);
dbStore.setLastAccessedTime(
projectName,
Timestamp.valueOf(LocalDateTime.now())
);
} else {
Log.warn(
"[{}] Went out of date while waiting for push",
projectName
);
throw new OutOfDateException();
}
}
}
/**
* A public call that should originate from the {@link FileHandler}.
*
* The {@link FileHandler} serves atts to the Overleaf app during a push.
* The Overleaf app includes the postback key in the request, which was
* originally given on a push request.
*
* This method checks that the postback key matches, and throws if not.
*
* The FileHandler should not serve the file if this throws.
* @param projectName The project name that this key belongs to
* @param postbackKey The key
* @throws InvalidPostbackKeyException If the key doesn't match
*/
public void checkPostbackKey(String projectName, String postbackKey)
throws InvalidPostbackKeyException {
postbackManager.checkPostbackKey(projectName, postbackKey);
}
/**
* A public call that originates from the postback thread
* {@link PostbackContents#processPostback()}, i.e. once the Overleaf app
* has fetched all the atts and has committed the push and is happy, it
* calls back here, fulfilling the promise that the push
* {@link #push(Optional, String, RawDirectory, RawDirectory, String)}
* is waiting on.
*
* The Overleaf app will have invented a new version for the push, which is
* passed to the promise for the original push request to update the app.
* @param projectName The name of the project being pushed to
* @param postbackKey The postback key being used
* @param versionID the new version id to use
* @throws UnexpectedPostbackException if the postback key is invalid
*/
public void postbackReceivedSuccessfully(
String projectName,
String postbackKey,
int versionID
) throws UnexpectedPostbackException {
Log.info(
"[{}]" +
" Postback received by postback thread, version: {}",
projectName,
versionID);
postbackManager.postVersionIDForProject(
projectName,
versionID,
postbackKey
);
}
/**
* As with {@link #postbackReceivedSuccessfully(String, String, int)},
* but with an exception instead.
*
* This is based on the JSON body of the postback from the Overleaf app.
*
* The most likely problem is an {@link OutOfDateException}.
* @param projectName The name of the project
* @param postbackKey The postback key being used
* @param exception The exception encountered
* @throws UnexpectedPostbackException If the postback key is invalid
*/
public void postbackReceivedWithException(
String projectName,
String postbackKey,
SnapshotPostException exception
) throws UnexpectedPostbackException {
Log.warn("[{}] Postback received with exception", projectName);
postbackManager.postExceptionForProject(
projectName,
exception,
postbackKey
);
}
/* PRIVATE */
/**
* Called by {@link #getUpdatedRepoCritical(Optional, String)}
*
* Does the actual work of getting the snapshots for a project from the
* snapshot API and committing them to a repo.
*
* If any snapshots were found, sets the latest version for the project.
*
* @param oauth2
* @param repo
* @throws IOException
* @throws GitUserException
*/
private void updateProject(
Optional<Credential> oauth2,
ProjectRepo repo
) throws IOException, GitUserException {
String projectName = repo.getProjectName();
int latestVersionId = dbStore.getLatestVersionForProject(projectName);
Deque<Snapshot> snapshots = snapshotAPI.getSnapshots(
oauth2, projectName, latestVersionId);
makeCommitsFromSnapshots(repo, snapshots);
// TODO: in case crashes around here, add an
// "updating_from_commit" column to the DB as a way to rollback the
// any failed partial updates before re-trying
// Also need to consider the empty state (a new git init'd repo being
// the rollback target)
if (!snapshots.isEmpty()) {
dbStore.setLatestVersionForProject(
projectName,
snapshots.getLast().getVersionID()
);
}
}
/**
* Called by {@link #updateProject(Optional, ProjectRepo)}.
*
* Performs the actual Git commits on the disk.
*
* Each commit adds files to the db store
* ({@link ResourceCache#get(String, String, String, Map, Map, Optional)},
* and then removes any files that were deleted.
* @param repo The repository to commit to
* @param snapshots The snapshots to commit
* @throws IOException If an IOException occurred
* @throws SizeLimitExceededException If one of the files was too big.
*/
private void makeCommitsFromSnapshots(
ProjectRepo repo,
Collection<Snapshot> snapshots
) throws IOException, GitUserException {
String name = repo.getProjectName();
Optional<Long> maxSize = config
.getRepoStore()
.flatMap(RepoStoreConfig::getMaxFileSize);
for (Snapshot snapshot : snapshots) {
RawDirectory directory = repo.getDirectory();
Map<String, RawFile> fileTable = directory.getFileTable();
List<RawFile> files = new ArrayList<>();
files.addAll(snapshot.getSrcs());
for (RawFile file : files) {
long size = file.size();
/* Can't throw in ifPresent... */
if (maxSize.isPresent()) {
long maxSize_ = maxSize.get();
if (size >= maxSize_) {
throw new SizeLimitExceededException(
Optional.of(file.getPath()), size, maxSize_);
}
}
}
Map<String, byte[]> fetchedUrls = new HashMap<>();
for (SnapshotAttachment snapshotAttachment : snapshot.getAtts()) {
files.add(
resourceCache.get(
name,
snapshotAttachment.getUrl(),
snapshotAttachment.getPath(),
fileTable,
fetchedUrls,
maxSize
)
);
}
Log.info(
"[{}] Committing version ID: {}",
name,
snapshot.getVersionID()
);
Collection<String> missingFiles = repo.commitAndGetMissing(
new GitDirectoryContents(
files,
repoStore.getRootDirectory(),
name,
snapshot
)
);
dbStore.deleteFilesForProject(
name,
missingFiles.toArray(new String[missingFiles.size()])
);
}
}
/**
* Called by
* {@link #pushCritical(Optional, String, RawDirectory, RawDirectory)}.
*
* This call consists of 2 things: Creating the candidate snapshot,
* and writing the atts to the atts directory.
*
* The candidate snapshot RAIIs away those atts (use try-with-resources).
* @param projectName The name of the project
* @param directoryContents The new directory contents
* @param oldDirectoryContents The old directory contents
* @return The {@link CandidateSnapshot} created
* @throws IOException If an I/O exception occurred on writing
*/
private CandidateSnapshot createCandidateSnapshot(
String projectName,
RawDirectory directoryContents,
RawDirectory oldDirectoryContents
) throws IOException {
CandidateSnapshot candidateSnapshot = new CandidateSnapshot(
projectName,
dbStore.getLatestVersionForProject(projectName),
directoryContents,
oldDirectoryContents
);
candidateSnapshot.writeServletFiles(repoStore.getRootDirectory());
return candidateSnapshot;
}
/**
* Called by
* {@link #pushCritical(Optional, String, RawDirectory, RawDirectory)}.
*
* This method approves a push by setting the latest version and removing
* any deleted files from the db store (files were already added by the
* resources cache).
* @param versionID
* @param candidateSnapshot
*/
private void approveSnapshot(
int versionID,
CandidateSnapshot candidateSnapshot
) {
List<String> deleted = candidateSnapshot.getDeleted();
dbStore.setLatestVersionForProject(
candidateSnapshot.getProjectName(),
versionID
);
dbStore.deleteFilesForProject(
candidateSnapshot.getProjectName(),
deleted.toArray(new String[deleted.size()])
);
}
}

20
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBInitException.java Normal file
View file

@ -0,0 +1,20 @@
package uk.ac.ic.wlgitbridge.bridge.db;
/**
* Created by winston on 23/08/2016.
*/
public class DBInitException extends RuntimeException {
public DBInitException(String message) {
super(message);
}
public DBInitException(String message, Throwable cause) {
super(message, cause);
}
public DBInitException(Throwable cause) {
super(cause);
}
}

44
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java Normal file
View file

@ -0,0 +1,44 @@
package uk.ac.ic.wlgitbridge.bridge.db;
import java.sql.Timestamp;
import java.util.List;
/**
* Created by winston on 20/08/2016.
*/
public interface DBStore {
int getNumProjects();
List<String> getProjectNames();
void setLatestVersionForProject(String project, int versionID);
int getLatestVersionForProject(String project);
void addURLIndexForProject(String projectName, String url, String path);
void deleteFilesForProject(String project, String... files);
String getPathForURLInProject(String projectName, String url);
String getOldestUnswappedProject();
void swap(String projectName, String compressionMethod);
void restore(String projectName);
String getSwapCompression(String projectName);
int getNumUnswappedProjects();
ProjectState getProjectState(String projectName);
/**
* Sets the last accessed time for the given project name.
* @param projectName the project's name
* @param time the time, or null if the project is to be swapped
*/
void setLastAccessedTime(String projectName, Timestamp time);
}

12
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/ProjectState.java Normal file
View file

@ -0,0 +1,12 @@
package uk.ac.ic.wlgitbridge.bridge.db;
/**
* Created by winston on 24/08/2016.
*/
public enum ProjectState {
NOT_PRESENT,
PRESENT,
SWAPPED
}

75
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/noop/NoopDbStore.java Normal file
View file

@ -0,0 +1,75 @@
package uk.ac.ic.wlgitbridge.bridge.db.noop;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.db.ProjectState;
import java.sql.Timestamp;
import java.util.List;
public class NoopDbStore implements DBStore {
@Override
public int getNumProjects() {
return 0;
}
@Override
public List<String> getProjectNames() {
return null;
}
@Override
public void setLatestVersionForProject(String project, int versionID) {
}
@Override
public int getLatestVersionForProject(String project) {
return 0;
}
@Override
public void addURLIndexForProject(String projectName, String url, String path) {
}
@Override
public void deleteFilesForProject(String project, String... files) {
}
@Override
public String getPathForURLInProject(String projectName, String url) {
return null;
}
@Override
public String getOldestUnswappedProject() {
return null;
}
@Override
public int getNumUnswappedProjects() {
return 0;
}
@Override
public ProjectState getProjectState(String projectName) {
return null;
}
@Override
public void setLastAccessedTime(String projectName, Timestamp time) {
}
@Override
public void swap(String projectName, String compressionMethod) {}
@Override
public void restore(String projectName) {}
@Override
public String getSwapCompression(String projectName) {
return null;
}
}

13
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SQLQuery.java Normal file
View file

@ -0,0 +1,13 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite;
import java.sql.ResultSet;
import java.sql.SQLException;
/**
* Created by Winston on 20/11/14.
*/
public interface SQLQuery<T> extends SQLUpdate {
public T processResultSet(ResultSet resultSet) throws SQLException;
}

18
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SQLUpdate.java Normal file
View file

@ -0,0 +1,18 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite;
import java.sql.PreparedStatement;
import java.sql.SQLException;
/**
* Created by Winston on 20/11/14.
*/
public interface SQLUpdate {
String getSQL();
default void addParametersToStatement(
PreparedStatement statement
) throws SQLException {
}
}

228
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/SqliteDBStore.java Normal file
View file

@ -0,0 +1,228 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite;
import com.google.common.base.Preconditions;
import uk.ac.ic.wlgitbridge.bridge.db.DBInitException;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.db.ProjectState;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.query.*;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter.*;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create.*;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.delete.*;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert.*;
import java.io.File;
import java.sql.*;
import java.util.List;
import java.util.stream.Stream;
/**
* Created by Winston on 17/11/14.
*/
public class SqliteDBStore implements DBStore {
private final Connection connection;
private int heapLimitBytes = 0;
public SqliteDBStore(File dbFile) {
this(dbFile, 0);
}
public SqliteDBStore(File dbFile, int heapLimitBytes) {
this.heapLimitBytes = heapLimitBytes;
try {
connection = openConnectionTo(dbFile);
createTables();
} catch (Throwable t) {
throw new DBInitException(t);
}
}
@Override
public int getNumProjects() {
return query(new GetNumProjects());
}
@Override
public List<String> getProjectNames() {
return query(new GetProjectNamesSQLQuery());
}
@Override
public void setLatestVersionForProject(
String projectName,
int versionID
) {
update(new SetProjectSQLUpdate(projectName, versionID));
}
@Override
public int getLatestVersionForProject(
String projectName
) {
return query(new GetLatestVersionForProjectSQLQuery(projectName));
}
@Override
public void addURLIndexForProject(
String projectName,
String url,
String path
) {
update(new AddURLIndexSQLUpdate(projectName, url, path));
}
@Override
public void deleteFilesForProject(
String projectName,
String... paths
) {
update(new DeleteFilesForProjectSQLUpdate(projectName, paths));
}
@Override
public String getPathForURLInProject(
String projectName,
String url
) {
return query(new GetPathForURLInProjectSQLQuery(projectName, url));
}
@Override
public String getOldestUnswappedProject() {
return query(new GetOldestProjectName());
}
@Override
public int getNumUnswappedProjects() {
return query(new GetNumUnswappedProjects());
}
@Override
public ProjectState getProjectState(String projectName) {
return query(new GetProjectState(projectName));
}
@Override
public void setLastAccessedTime(
String projectName,
Timestamp lastAccessed
) {
update(new SetProjectLastAccessedTime(projectName, lastAccessed));
}
@Override
public void swap(String projectName, String compressionMethod) {
update(new UpdateSwap(projectName, compressionMethod));
}
@Override
public void restore(String projectName) {
update(new UpdateRestore(projectName));
}
@Override
public String getSwapCompression(String projectName) {
return query(new GetSwapCompression(projectName));
}
private Connection openConnectionTo(File dbFile) {
File parentDir = dbFile.getParentFile();
if (!parentDir.exists() && !parentDir.mkdirs()) {
throw new DBInitException(
parentDir.getAbsolutePath() + " directory didn't exist, " +
"and unable to create. Check your permissions."
);
}
try {
Class.forName("org.sqlite.JDBC");
} catch (ClassNotFoundException e) {
throw new DBInitException(e);
}
try {
return DriverManager.getConnection(
"jdbc:sqlite:" + dbFile.getAbsolutePath()
);
} catch (SQLException e) {
throw new DBInitException("Unable to connect to DB", e);
}
}
private void createTables() {
/* Migrations */
/* We need to eat exceptions from here */
try { doUpdate(new SetSoftHeapLimitPragma(this.heapLimitBytes)); } catch (SQLException ignore) {}
try { doUpdate(new ProjectsAddLastAccessed()); } catch (SQLException ignore) {}
try { doUpdate(new ProjectsAddSwapTime()); } catch (SQLException ignore) {}
try { doUpdate(new ProjectsAddRestoreTime()); } catch (SQLException ignore) {}
try { doUpdate(new ProjectsAddSwapCompression()); } catch (SQLException ignore) {}
/* Create tables (if they don't exist) */
Stream.of(
new CreateProjectsTableSQLUpdate(),
new CreateProjectsIndexLastAccessed(),
new CreateURLIndexStoreSQLUpdate(),
new CreateIndexURLIndexStore()
).forEach(this::update);
/* In the case of needing to change the schema, we need to check that
migrations didn't just fail */
Preconditions.checkState(query(new LastAccessedColumnExists()));
Preconditions.checkState(query(new SwapTimeColumnExists()));
Preconditions.checkState(query(new RestoreTimeColumnExists()));
Preconditions.checkState(query(new SwapCompressionColumnExists()));
}
private void update(SQLUpdate update) {
try {
doUpdate(update);
} catch (SQLException e) {
throw new RuntimeException(e);
}
}
private <T> T query(SQLQuery<T> query) {
try {
return doQuery(query);
} catch (SQLException e) {
throw new RuntimeException(e);
}
}
private void doUpdate(SQLUpdate update) throws SQLException {
PreparedStatement statement = null;
try {
statement = connection.prepareStatement(update.getSQL());
update.addParametersToStatement(statement);
statement.executeUpdate();
} catch (SQLException e) {
throw e;
} finally {
try {
statement.close();
} catch (Throwable t) {
throw new SQLException(t);
}
}
}
private <T> T doQuery(SQLQuery<T> query) throws SQLException {
PreparedStatement statement = null;
ResultSet results = null;
try {
statement = connection.prepareStatement(query.getSQL());
query.addParametersToStatement(statement);
results = statement.executeQuery();
return query.processResultSet(results);
} catch (SQLException e) {
throw e;
} finally {
if (statement != null) {
statement.close();
}
if (results != null) {
results.close();
}
}
}
}

43
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetLatestVersionForProjectSQLQuery.java Normal file
View file

@ -0,0 +1,43 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
/**
* Created by Winston on 20/11/14.
*/
public class GetLatestVersionForProjectSQLQuery implements SQLQuery<Integer> {
private static final String GET_VERSION_IDS_FOR_PROJECT_NAME =
"SELECT `version_id` FROM `projects` WHERE `name` = ?";
private final String projectName;
public GetLatestVersionForProjectSQLQuery(String projectName) {
this.projectName = projectName;
}
@Override
public Integer processResultSet(ResultSet resultSet) throws SQLException {
int versionID = 0;
while (resultSet.next()) {
versionID = resultSet.getInt("version_id");
}
return versionID;
}
@Override
public String getSQL() {
return GET_VERSION_IDS_FOR_PROJECT_NAME;
}
@Override
public void addParametersToStatement(
PreparedStatement statement
) throws SQLException {
statement.setString(1, projectName);
}
}

30
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetNumProjects.java Normal file
View file

@ -0,0 +1,30 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.ResultSet;
import java.sql.SQLException;
/**
* Created by winston on 24/08/2016.
*/
public class GetNumProjects implements SQLQuery<Integer> {
private static final String GET_NUM_PROJECTS =
"SELECT COUNT(*)\n" +
" FROM `projects`";
@Override
public String getSQL() {
return GET_NUM_PROJECTS;
}
@Override
public Integer processResultSet(ResultSet resultSet) throws SQLException {
while (resultSet.next()) {
return resultSet.getInt("COUNT(*)");
}
throw new IllegalStateException("Count always returns results");
}
}

31
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetNumUnswappedProjects.java Normal file
View file

@ -0,0 +1,31 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.ResultSet;
import java.sql.SQLException;
/**
* Created by winston on 24/08/2016.
*/
public class GetNumUnswappedProjects implements SQLQuery<Integer> {
private static final String GET_NUM_UNSWAPPED_PROJECTS =
"SELECT COUNT(*)\n" +
" FROM `projects`\n" +
" WHERE `last_accessed` IS NOT NULL";
@Override
public String getSQL() {
return GET_NUM_UNSWAPPED_PROJECTS;
}
@Override
public Integer processResultSet(ResultSet resultSet) throws SQLException {
while (resultSet.next()) {
return resultSet.getInt("COUNT(*)");
}
throw new IllegalStateException("Count always returns results");
}
}

31
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetOldestProjectName.java Normal file
View file

@ -0,0 +1,31 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.ResultSet;
import java.sql.SQLException;
/**
* Created by winston on 23/08/2016.
*/
public class GetOldestProjectName implements SQLQuery<String> {
private static final String GET_OLDEST_PROJECT_NAME =
"SELECT `name`, MIN(`last_accessed`)\n" +
" FROM `projects` \n" +
" WHERE `last_accessed` IS NOT NULL;";
@Override
public String getSQL() {
return GET_OLDEST_PROJECT_NAME;
}
@Override
public String processResultSet(ResultSet resultSet) throws SQLException {
while (resultSet.next()) {
return resultSet.getString("name");
}
return null;
}
}

50
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetPathForURLInProjectSQLQuery.java Normal file
View file

@ -0,0 +1,50 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
/**
* Created by Winston on 20/11/14.
*/
public class GetPathForURLInProjectSQLQuery implements SQLQuery<String> {
private static final String GET_URL_INDEXES_FOR_PROJECT_NAME =
"SELECT `path` "
+ "FROM `url_index_store` "
+ "WHERE `project_name` = ? "
+ "AND `url` = ?";
private final String projectName;
private final String url;
public GetPathForURLInProjectSQLQuery(String projectName, String url) {
this.projectName = projectName;
this.url = url;
}
@Override
public String processResultSet(ResultSet resultSet) throws SQLException {
String path = null;
while (resultSet.next()) {
path = resultSet.getString("path");
}
return path;
}
@Override
public String getSQL() {
return GET_URL_INDEXES_FOR_PROJECT_NAME;
}
@Override
public void addParametersToStatement(
PreparedStatement statement
) throws SQLException {
statement.setString(1, projectName);
statement.setString(2, url);
}
}

34
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetProjectNamesSQLQuery.java Normal file
View file

@ -0,0 +1,34 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
/**
* Created by Winston on 21/02/15.
*/
public class GetProjectNamesSQLQuery implements SQLQuery<List<String>> {
private static final String GET_URL_INDEXES_FOR_PROJECT_NAME =
"SELECT `name` FROM `projects`";
@Override
public List<String> processResultSet(
ResultSet resultSet
) throws SQLException {
List<String> projectNames = new ArrayList<>();
while (resultSet.next()) {
projectNames.add(resultSet.getString("name"));
}
return projectNames;
}
@Override
public String getSQL() {
return GET_URL_INDEXES_FOR_PROJECT_NAME;
}
}

51
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetProjectState.java Normal file
View file

@ -0,0 +1,51 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.ProjectState;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
/**
* Created by winston on 24/08/2016.
*/
public class GetProjectState implements SQLQuery<ProjectState> {
private static final String GET_PROJECT_STATE =
"SELECT `last_accessed`\n" +
" FROM `projects`\n" +
" WHERE `name` = ?";
private final String projectName;
public GetProjectState(String projectName) {
this.projectName = projectName;
}
@Override
public String getSQL() {
return GET_PROJECT_STATE;
}
@Override
public ProjectState processResultSet(
ResultSet resultSet
) throws SQLException {
while (resultSet.next()) {
if (resultSet.getTimestamp("last_accessed") == null) {
return ProjectState.SWAPPED;
}
return ProjectState.PRESENT;
}
return ProjectState.NOT_PRESENT;
}
@Override
public void addParametersToStatement(
PreparedStatement statement
) throws SQLException {
statement.setString(1, projectName);
}
}

39
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/GetSwapCompression.java Normal file
View file

@ -0,0 +1,39 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
public class GetSwapCompression implements SQLQuery<String> {
private static final String GET_SWAP_COMPRESSION =
"SELECT `swap_compression` FROM `projects` WHERE `name` = ?";
private final String projectName;
public GetSwapCompression(String projectName) {
this.projectName = projectName;
}
@Override
public String processResultSet(ResultSet resultSet) throws SQLException {
String compression = null;
while (resultSet.next()) {
compression = resultSet.getString("swap_compression");
}
return compression;
}
@Override
public String getSQL() {
return GET_SWAP_COMPRESSION;
}
@Override
public void addParametersToStatement(
PreparedStatement statement
) throws SQLException {
statement.setString(1, projectName);
}
}

31
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/LastAccessedColumnExists.java Normal file
View file

@ -0,0 +1,31 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.ResultSet;
import java.sql.SQLException;
/**
* Created by winston on 04/09/2016.
*/
public class LastAccessedColumnExists implements SQLQuery<Boolean> {
private static final String LAST_ACCESSED_COLUMN_EXISTS =
"PRAGMA table_info(`projects`)";
@Override
public String getSQL() {
return LAST_ACCESSED_COLUMN_EXISTS;
}
@Override
public Boolean processResultSet(ResultSet resultSet) throws SQLException {
while (resultSet.next()) {
if (resultSet.getString(2).equals("last_accessed")) {
return true;
}
}
return false;
}
}

26
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/RestoreTimeColumnExists.java Normal file
View file

@ -0,0 +1,26 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.ResultSet;
import java.sql.SQLException;
public class RestoreTimeColumnExists implements SQLQuery<Boolean> {
private static final String RESTORE_TIME_COLUMN_EXISTS =
"PRAGMA table_info(`projects`)";
@Override
public String getSQL() {
return RESTORE_TIME_COLUMN_EXISTS;
}
@Override
public Boolean processResultSet(ResultSet resultSet) throws SQLException {
while (resultSet.next()) {
if (resultSet.getString(2).equals("restore_time")) {
return true;
}
}
return false;
}
}

27
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/SwapCompressionColumnExists.java Normal file
View file

@ -0,0 +1,27 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.ResultSet;
import java.sql.SQLException;
public class SwapCompressionColumnExists implements SQLQuery<Boolean> {
private static final String SWAP_COMPRESSION_COLUMN_EXISTS =
"PRAGMA table_info(`projects`)";
@Override
public String getSQL() {
return SWAP_COMPRESSION_COLUMN_EXISTS;
}
@Override
public Boolean processResultSet(ResultSet resultSet) throws SQLException {
while (resultSet.next()) {
if (resultSet.getString(2).equals("swap_compression")) {
return true;
}
}
return false;
}
}

27
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/query/SwapTimeColumnExists.java Normal file
View file

@ -0,0 +1,27 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
import java.sql.ResultSet;
import java.sql.SQLException;
public class SwapTimeColumnExists implements SQLQuery<Boolean> {
private static final String SWAP_TIME_COLUMN_EXISTS =
"PRAGMA table_info(`projects`)";
@Override
public String getSQL() {
return SWAP_TIME_COLUMN_EXISTS;
}
@Override
public Boolean processResultSet(ResultSet resultSet) throws SQLException {
while (resultSet.next()) {
if (resultSet.getString(2).equals("swap_time")) {
return true;
}
}
return false;
}
}

19
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddLastAccessed.java Normal file
View file

@ -0,0 +1,19 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
/**
* Created by winston on 03/09/2016.
*/
public class ProjectsAddLastAccessed implements SQLUpdate {
private static final String PROJECTS_ADD_LAST_ACCESSED =
"ALTER TABLE `projects`\n" +
"ADD COLUMN `last_accessed` DATETIME NULL DEFAULT 0";
@Override
public String getSQL() {
return PROJECTS_ADD_LAST_ACCESSED;
}
}

14
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddRestoreTime.java Normal file
View file

@ -0,0 +1,14 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
public class ProjectsAddRestoreTime implements SQLUpdate {
private static final String PROJECTS_ADD_RESTORE_TIME =
"ALTER TABLE `projects`\n" +
"ADD COLUMN `restore_time` DATETIME NULL;\n";
@Override
public String getSQL() {
return PROJECTS_ADD_RESTORE_TIME;
}
}

14
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddSwapCompression.java Normal file
View file

@ -0,0 +1,14 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
public class ProjectsAddSwapCompression implements SQLUpdate {
private static final String PROJECTS_ADD_SWAP_COMPRESSION =
"ALTER TABLE `projects`\n" +
"ADD COLUMN `swap_compression` VARCHAR NULL;\n";
@Override
public String getSQL() {
return PROJECTS_ADD_SWAP_COMPRESSION;
}
}

15
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/ProjectsAddSwapTime.java Normal file
View file

@ -0,0 +1,15 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
public class ProjectsAddSwapTime implements SQLUpdate {
private static final String PROJECTS_ADD_SWAP_TIME =
"ALTER TABLE `projects`\n" +
"ADD COLUMN `swap_time` DATETIME NULL;\n";
@Override
public String getSQL() {
return PROJECTS_ADD_SWAP_TIME;
}
}

17
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/alter/SetSoftHeapLimitPragma.java Normal file
View file

@ -0,0 +1,17 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
public class SetSoftHeapLimitPragma implements SQLUpdate {
private int heapLimitBytes = 0;
public SetSoftHeapLimitPragma(int heapLimitBytes) {
this.heapLimitBytes = heapLimitBytes;
}
@Override
public String getSQL() {
return "PRAGMA soft_heap_limit="+this.heapLimitBytes+";";
}
}

19
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateIndexURLIndexStore.java Normal file
View file

@ -0,0 +1,19 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
/**
* Created by Winston on 21/02/15.
*/
public class CreateIndexURLIndexStore implements SQLUpdate {
public static final String CREATE_INDEX_URL_INDEX_STORE =
"CREATE UNIQUE INDEX IF NOT EXISTS `project_path_index` " +
"ON `url_index_store`(`project_name`, `path`);\n";
@Override
public String getSQL() {
return CREATE_INDEX_URL_INDEX_STORE;
}
}

19
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateProjectsIndexLastAccessed.java Normal file
View file

@ -0,0 +1,19 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
/**
* Created by winston on 23/08/2016.
*/
public class CreateProjectsIndexLastAccessed implements SQLUpdate {
private static final String CREATE_PROJECTS_INDEX_LAST_ACCESSED =
"CREATE INDEX IF NOT EXISTS `projects_index_last_accessed`\n" +
" ON `projects`(`last_accessed`)";
@Override
public String getSQL() {
return CREATE_PROJECTS_INDEX_LAST_ACCESSED;
}
}

26
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateProjectsTableSQLUpdate.java Normal file
View file

@ -0,0 +1,26 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
/**
* Created by Winston on 20/11/14.
*/
public class CreateProjectsTableSQLUpdate implements SQLUpdate {
private static final String CREATE_PROJECTS_TABLE =
"CREATE TABLE IF NOT EXISTS `projects` (\n" +
" `name` VARCHAR NOT NULL DEFAULT '',\n" +
" `version_id` INT NOT NULL DEFAULT 0,\n" +
" `last_accessed` DATETIME NULL DEFAULT 0,\n" +
" `swap_time` DATETIME NULL,\n" +
" `restore_time` DATETIME NULL,\n" +
" `swap_compression` VARCHAR NULL,\n" +
" PRIMARY KEY (`name`)\n" +
")";
@Override
public String getSQL() {
return CREATE_PROJECTS_TABLE;
}
}

28
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/create/CreateURLIndexStoreSQLUpdate.java Normal file
View file

@ -0,0 +1,28 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
/**
* Created by Winston on 20/11/14.
*/
public class CreateURLIndexStoreSQLUpdate implements SQLUpdate {
private static final String CREATE_URL_INDEX_STORE =
"CREATE TABLE IF NOT EXISTS `url_index_store` (\n"+
" `project_name` varchar(10) NOT NULL DEFAULT '',\n"+
" `url` text NOT NULL,\n"+
" `path` text NOT NULL,\n"+
" PRIMARY KEY (`project_name`,`url`),\n"+
" CONSTRAINT `url_index_store_ibfk_1` " +
"FOREIGN KEY (`project_name`) " +
"REFERENCES `projects` (`name`) " +
"ON DELETE CASCADE " +
"ON UPDATE CASCADE\n"+
");\n";
@Override
public String getSQL() {
return CREATE_URL_INDEX_STORE;
}
}

53
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/delete/DeleteFilesForProjectSQLUpdate.java Normal file
View file

@ -0,0 +1,53 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.delete;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
import java.sql.PreparedStatement;
import java.sql.SQLException;
/**
* Created by Winston on 20/11/14.
*/
public class DeleteFilesForProjectSQLUpdate implements SQLUpdate {
private static final String DELETE_URL_INDEXES_FOR_PROJECT_NAME =
"DELETE FROM `url_index_store` " +
"WHERE `project_name` = ? AND path IN (";
private final String projectName;
private final String[] paths;
public DeleteFilesForProjectSQLUpdate(
String projectName,
String... paths
) {
this.projectName = projectName;
this.paths = paths;
}
@Override
public String getSQL() {
StringBuilder sb = new StringBuilder(
DELETE_URL_INDEXES_FOR_PROJECT_NAME
);
for (int i = 0; i < paths.length; i++) {
sb.append("?");
if (i < paths.length - 1) {
sb.append(", ");
}
}
sb.append(");\n");
return sb.toString();
}
@Override
public void addParametersToStatement(
PreparedStatement statement
) throws SQLException {
statement.setString(1, projectName);
for (int i = 0; i < paths.length; i++) {
statement.setString(i + 2, paths[i]);
}
}
}

45
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/AddURLIndexSQLUpdate.java Normal file
View file

@ -0,0 +1,45 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
import java.sql.PreparedStatement;
import java.sql.SQLException;
/**
* Created by Winston on 20/11/14.
*/
public class AddURLIndexSQLUpdate implements SQLUpdate {
private static final String ADD_URL_INDEX =
"INSERT OR REPLACE INTO `url_index_store`(" +
"`project_name`, " +
"`url`, " +
"`path`" +
") VALUES " +
"(?, ?, ?)\n";
private final String projectName;
private final String url;
private final String path;
public AddURLIndexSQLUpdate(String projectName, String url, String path) {
this.projectName = projectName;
this.url = url;
this.path = path;
}
@Override
public String getSQL() {
return ADD_URL_INDEX;
}
@Override
public void addParametersToStatement(
PreparedStatement statement
) throws SQLException {
statement.setString(1, projectName);
statement.setString(2, url);
statement.setString(3, path);
}
}

43
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/SetProjectLastAccessedTime.java Normal file
View file

@ -0,0 +1,43 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Timestamp;
/**
* Created by winston on 23/08/2016.
*/
public class SetProjectLastAccessedTime implements SQLUpdate {
private static final String SET_PROJECT_LAST_ACCESSED_TIME =
"UPDATE `projects`\n" +
"SET `last_accessed` = ?\n" +
"WHERE `name` = ?";
private final String projectName;
private final Timestamp lastAccessed;
public SetProjectLastAccessedTime(
String projectName,
Timestamp lastAccessed
) {
this.projectName = projectName;
this.lastAccessed = lastAccessed;
}
@Override
public String getSQL() {
return SET_PROJECT_LAST_ACCESSED_TIME;
}
@Override
public void addParametersToStatement(
PreparedStatement statement
) throws SQLException {
statement.setTimestamp(1, lastAccessed);
statement.setString(2, projectName);
}
}

39
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/SetProjectSQLUpdate.java Normal file
View file

@ -0,0 +1,39 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
import java.sql.PreparedStatement;
import java.sql.SQLException;
/**
* Created by Winston on 20/11/14.
*/
public class SetProjectSQLUpdate implements SQLUpdate {
private static final String SET_PROJECT =
"INSERT OR REPLACE "
+ "INTO `projects`(`name`, `version_id`, `last_accessed`) "
+ "VALUES (?, ?, DATETIME('now'));\n";
private final String projectName;
private final int versionID;
public SetProjectSQLUpdate(String projectName, int versionID) {
this.projectName = projectName;
this.versionID = versionID;
}
@Override
public String getSQL() {
return SET_PROJECT;
}
@Override
public void addParametersToStatement(
PreparedStatement statement
) throws SQLException {
statement.setString(1, projectName);
statement.setInt(2, versionID);
}
}

40
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/UpdateRestore.java Normal file
View file

@ -0,0 +1,40 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.time.LocalDateTime;
public class UpdateRestore implements SQLUpdate {
private static final String UPDATE_RESTORE =
"UPDATE `projects`\n" +
"SET `last_accessed` = ?,\n" +
" `swap_time` = NULL,\n" +
" `restore_time` = ?,\n" +
" `swap_compression` = NULL\n" +
"WHERE `name` = ?;\n";
private final String projectName;
private final Timestamp now;
public UpdateRestore(String projectName) {
this.projectName = projectName;
this.now = Timestamp.valueOf(LocalDateTime.now());
}
@Override
public String getSQL() {
return UPDATE_RESTORE;
}
@Override
public void addParametersToStatement(
PreparedStatement statement
) throws SQLException {
statement.setTimestamp(1, now);
statement.setTimestamp(2, now);
statement.setString(3, projectName);
}
}

42
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/sqlite/update/insert/UpdateSwap.java Normal file
View file

@ -0,0 +1,42 @@
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert;
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.time.LocalDateTime;
public class UpdateSwap implements SQLUpdate {
private static final String UPDATE_SWAP =
"UPDATE `projects`\n" +
"SET `last_accessed` = NULL,\n" +
" `swap_time` = ?,\n" +
" `restore_time` = NULL,\n" +
" `swap_compression` = ?\n" +
"WHERE `name` = ?;\n";
private final String projectName;
private final String compression;
private final Timestamp now;
public UpdateSwap(String projectName, String compression) {
this.projectName = projectName;
this.compression = compression;
this.now = Timestamp.valueOf(LocalDateTime.now());
}
@Override
public String getSQL() {
return UPDATE_SWAP;
}
@Override
public void addParametersToStatement(
PreparedStatement statement
) throws SQLException {
statement.setTimestamp(1, now);
statement.setString(2, compression);
statement.setString(3, projectName);
}
}

34
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/gc/GcJob.java Normal file
View file

@ -0,0 +1,34 @@
package uk.ac.ic.wlgitbridge.bridge.gc;
import uk.ac.ic.wlgitbridge.bridge.Bridge;
import uk.ac.ic.wlgitbridge.bridge.repo.ProjectRepo;
import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
/**
* Is started by the bridge. Every time a project is updated, we queue it for
* GC which executes every hour or so.
*
* We don't queue it into a more immediate Executor because there is no way to
* know if a call to {@link Bridge#updateProject(Optional, ProjectRepo)},
* which releases the lock, is going to call
* {@link Bridge#push(Optional, String, RawDirectory, RawDirectory, String)}.
*
* We don't want the GC to run in between an update and a push.
*/
public interface GcJob {
void start();
void stop();
void onPreGc(Runnable preGc);
void onPostGc(Runnable postGc);
void queueForGc(String projectName);
CompletableFuture<Void> waitForRun();
}

141
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/gc/GcJobImpl.java Normal file
View file

@ -0,0 +1,141 @@
package uk.ac.ic.wlgitbridge.bridge.gc;
import uk.ac.ic.wlgitbridge.bridge.lock.LockGuard;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.bridge.repo.ProjectRepo;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.util.Log;
import uk.ac.ic.wlgitbridge.util.TimerUtils;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
/**
* Implementation of {@link GcJob} using its own Timer and a synchronized
* queue.
*/
public class GcJobImpl implements GcJob {
private final RepoStore repoStore;
private final ProjectLock locks;
private final long intervalMs;
private final Timer timer;
private final Set<String> gcQueue;
/**
* Hooks in case they are needed, e.g. for testing.
*/
private AtomicReference<Runnable> preGc;
private AtomicReference<Runnable> postGc;
/* We need to iterate over and empty it after every run */
private final Lock jobWaitersLock;
private final List<CompletableFuture<Void>> jobWaiters;
public GcJobImpl(RepoStore repoStore, ProjectLock locks, long intervalMs) {
this.repoStore = repoStore;
this.locks = locks;
this.intervalMs = intervalMs;
timer = new Timer();
gcQueue = Collections.newSetFromMap(new ConcurrentHashMap<>());
preGc = new AtomicReference<>(() -> {});
postGc = new AtomicReference<>(() -> {});
jobWaitersLock = new ReentrantLock();
jobWaiters = new ArrayList<>();
}
public GcJobImpl(RepoStore repoStore, ProjectLock locks) {
this(
repoStore,
locks,
TimeUnit.MILLISECONDS.convert(1, TimeUnit.HOURS)
);
}
@Override
public void start() {
Log.info("Starting GC job to run every [{}] ms", intervalMs);
timer.scheduleAtFixedRate(
TimerUtils.makeTimerTask(this::doGC),
intervalMs,
intervalMs
);
}
@Override
public void stop() {
Log.info("Stopping GC job");
timer.cancel();
}
@Override
public void onPreGc(Runnable preGc) {
this.preGc.set(preGc);
}
@Override
public void onPostGc(Runnable postGc) {
this.postGc.set(postGc);
}
/**
* Needs to be callable from any thread.
* @param projectName
*/
@Override
public void queueForGc(String projectName) {
gcQueue.add(projectName);
}
@Override
public CompletableFuture<Void> waitForRun() {
CompletableFuture<Void> ret = new CompletableFuture<>();
jobWaitersLock.lock();
try {
jobWaiters.add(ret);
} finally {
jobWaitersLock.unlock();
}
return ret;
}
private void doGC() {
Log.info("GC job running");
int numGcs = 0;
preGc.get().run();
for (
Iterator<String> it = gcQueue.iterator();
it.hasNext();
it.remove(), ++numGcs
) {
String proj = it.next();
Log.info("[{}] Running GC job on project", proj);
try (LockGuard __ = locks.lockGuard(proj)) {
try {
ProjectRepo repo = repoStore.getExistingRepo(proj);
repo.runGC();
repo.deleteIncomingPacks();
} catch (IOException e) {
Log.info("[{}] Failed to GC project", proj);
}
}
}
Log.info("GC job finished, num gcs: {}", numGcs);
jobWaitersLock.lock();
try {
jobWaiters.forEach(w -> w.complete(null));
} finally {
jobWaitersLock.unlock();
}
postGc.get().run();
}
}

10
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/lock/LockGuard.java Normal file
View file

@ -0,0 +1,10 @@
package uk.ac.ic.wlgitbridge.bridge.lock;
/**
* Created by winston on 24/08/2016.
*/
public interface LockGuard extends AutoCloseable {
void close();
}

23
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/lock/ProjectLock.java Normal file
View file

@ -0,0 +1,23 @@
package uk.ac.ic.wlgitbridge.bridge.lock;
/**
* Project Lock class.
*
* The locks should be re-entrant. For example, we are usually holding the lock
* when a project must be restored, which tries to acquire the lock again.
*/
public interface ProjectLock {
void lockAll();
void lockForProject(String projectName);
void unlockForProject(String projectName);
/* RAII hahaha */
default LockGuard lockGuard(String projectName) {
lockForProject(projectName);
return () -> unlockForProject(projectName);
}
}

238
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStore.java Normal file
View file

@ -0,0 +1,238 @@
package uk.ac.ic.wlgitbridge.bridge.repo;
import com.google.api.client.repackaged.com.google.common.base.Preconditions;
import org.apache.commons.io.FileUtils;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Repository;
import uk.ac.ic.wlgitbridge.util.Log;
import uk.ac.ic.wlgitbridge.util.Project;
import uk.ac.ic.wlgitbridge.util.Tar;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
import java.util.function.Function;
import static uk.ac.ic.wlgitbridge.util.Util.deleteInDirectoryApartFrom;
/**
* Created by winston on 20/08/2016.
*/
public class FSGitRepoStore implements RepoStore {
private static final long DEFAULT_MAX_FILE_SIZE = 50 * 1024 * 1024;
private final String repoStorePath;
private final File rootDirectory;
private final long maxFileSize;
private final Function<File, Long> fsSizer;
public FSGitRepoStore(
String repoStorePath,
Optional<Long> maxFileSize
) {
this(
repoStorePath,
maxFileSize.orElse(DEFAULT_MAX_FILE_SIZE),
d -> d.getTotalSpace() - d.getFreeSpace()
);
}
public FSGitRepoStore(
String repoStorePath,
long maxFileSize,
Function<File, Long> fsSizer
) {
this.repoStorePath = repoStorePath;
rootDirectory = initRootGitDirectory(repoStorePath);
this.maxFileSize = maxFileSize;
this.fsSizer = fsSizer;
}
@Override
public String getRepoStorePath() {
return repoStorePath;
}
@Override
public File getRootDirectory() {
return rootDirectory;
}
@Override
public ProjectRepo initRepo(String project) throws IOException {
GitProjectRepo ret = GitProjectRepo.fromName(project);
ret.initRepo(this);
return new WalkOverrideGitRepo(
ret, Optional.of(maxFileSize), Optional.empty());
}
@Override
public ProjectRepo initRepoFromExisting(
String project, String fromProject
) throws IOException {
String repoRoot = getRepoStorePath();
String sourcePath = repoRoot + "/" + fromProject;
String destinationPath = repoRoot + "/" + project;
Log.info("[{}] Init repo by copying data from: {}, to: {}",
project,
sourcePath,
destinationPath
);
File source = new File(sourcePath);
File destination = new File(destinationPath);
FileUtils.copyDirectory(source, destination);
GitProjectRepo ret = GitProjectRepo.fromName(project);
ret.useExistingRepository(this);
return new WalkOverrideGitRepo(
ret, Optional.of(maxFileSize), Optional.empty());
}
@Override
public ProjectRepo getExistingRepo(String project) throws IOException {
GitProjectRepo ret = GitProjectRepo.fromName(project);
ret.useExistingRepository(this);
return new WalkOverrideGitRepo(
ret, Optional.of(maxFileSize), Optional.empty());
}
@Override
public ProjectRepo useJGitRepo(Repository repo, ObjectId commitId) {
GitProjectRepo ret = GitProjectRepo.fromJGitRepo(repo);
return new WalkOverrideGitRepo(
ret, Optional.of(maxFileSize), Optional.of(commitId));
}
/* TODO: Perhaps we should just delete bad directories on the fly. */
@Override
public void purgeNonexistentProjects(
Collection<String> existingProjectNames
) {
List<String> excludedFromDeletion =
new ArrayList<>(existingProjectNames);
excludedFromDeletion.add(".wlgb");
deleteInDirectoryApartFrom(
rootDirectory,
excludedFromDeletion.toArray(new String[] {})
);
}
@Override
public long totalSize() {
return fsSizer.apply(rootDirectory);
}
@Override
public InputStream bzip2Project(
String projectName,
long[] sizePtr
) throws IOException {
Project.checkValidProjectName(projectName);
Log.info("[{}] bzip2 project", projectName);
return Tar.bz2.zip(getDotGitForProject(projectName), sizePtr);
}
@Override
public InputStream gzipProject(
String projectName,
long[] sizePtr
) throws IOException {
Project.checkValidProjectName(projectName);
Log.info("[{}] gzip project", projectName);
return Tar.gzip.zip(getDotGitForProject(projectName), sizePtr);
}
@Override
public void gcProject(String projectName) throws IOException {
Project.checkValidProjectName(projectName);
ProjectRepo repo = getExistingRepo(projectName);
repo.runGC();
}
@Override
public void remove(String projectName) throws IOException {
Project.checkValidProjectName(projectName);
FileUtils.deleteDirectory(new File(rootDirectory, projectName));
}
@Override
public void unbzip2Project(
String projectName,
InputStream dataStream
) throws IOException {
Preconditions.checkArgument(
Project.isValidProjectName(projectName),
"[%s] invalid project name: ",
projectName
);
Preconditions.checkState(
getDirForProject(projectName).mkdirs(),
"[%s] directories for " +
"evicted project already exist",
projectName
);
Log.info("[{}] un-bzip2 project", projectName);
Tar.bz2.unzip(dataStream, getDirForProject(projectName));
}
@Override
public void ungzipProject(
String projectName,
InputStream dataStream
) throws IOException {
Preconditions.checkArgument(
Project.isValidProjectName(projectName),
"[%s] invalid project name: ",
projectName
);
Preconditions.checkState(
getDirForProject(projectName).mkdirs(),
"[%s] directories for " +
"evicted project already exist",
projectName
);
Log.info("[{}] un-gzip project", projectName);
Tar.gzip.unzip(dataStream, getDirForProject(projectName));
}
private File getDirForProject(String projectName) {
Project.checkValidProjectName(projectName);
return Paths.get(
rootDirectory.getAbsolutePath()
).resolve(
projectName
).toFile();
}
private File getDotGitForProject(String projectName) {
Project.checkValidProjectName(projectName);
return Paths.get(
rootDirectory.getAbsolutePath()
).resolve(
projectName
).resolve(
".git"
).toFile();
}
private File initRootGitDirectory(String rootGitDirectoryPath) {
File rootGitDirectory = new File(rootGitDirectoryPath);
rootGitDirectory.mkdirs();
Preconditions.checkArgument(
rootGitDirectory.isDirectory(),
"given root git directory " +
"is not a directory: %s",
rootGitDirectory.getAbsolutePath()
);
return rootGitDirectory;
}
}

280
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/GitProjectRepo.java Normal file
View file

@ -0,0 +1,280 @@
package uk.ac.ic.wlgitbridge.bridge.repo;
import com.google.common.base.Preconditions;
import org.apache.commons.io.IOUtils;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.ResetCommand;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.lib.PersonIdent;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.storage.file.FileRepositoryBuilder;
import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.git.util.RepositoryObjectTreeWalker;
import uk.ac.ic.wlgitbridge.util.Log;
import uk.ac.ic.wlgitbridge.util.Project;
import uk.ac.ic.wlgitbridge.util.Util;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileVisitResult;
import java.nio.file.FileVisitor;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.*;
/**
* Class representing a Git repository.
*
* It stores the projectName and repo separately because the hooks need to be
* able to construct one of these without knowing whether the repo exists yet.
*
* It can then be passed to the Bridge, which will either
* {@link #initRepo(RepoStore)} for a never-seen-before repo, or
* {@link #useExistingRepository(RepoStore)} for an existing repo.
*
* Make sure to acquire the project lock before calling methods here.
*/
public class GitProjectRepo implements ProjectRepo {
private final String projectName;
private Optional<Repository> repository;
public static GitProjectRepo fromJGitRepo(Repository repo) {
return new GitProjectRepo(
repo.getWorkTree().getName(), Optional.of(repo));
}
public static GitProjectRepo fromName(String projectName) {
return new GitProjectRepo(projectName, Optional.empty());
}
GitProjectRepo(String projectName, Optional<Repository> repository) {
Preconditions.checkArgument(Project.isValidProjectName(projectName));
this.projectName = projectName;
this.repository = repository;
}
@Override
public String getProjectName() {
return projectName;
}
@Override
public void initRepo(
RepoStore repoStore
) throws IOException {
initRepositoryField(repoStore);
Preconditions.checkState(repository.isPresent());
Repository repo = this.repository.get();
// TODO: assert that this is a fresh repo. At the moment, we can't be
// sure whether the repo to be init'd doesn't exist or is just fresh
// and we crashed / aborted while committing
if (repo.getObjectDatabase().exists()) return;
repo.create();
}
@Override
public void useExistingRepository(
RepoStore repoStore
) throws IOException {
initRepositoryField(repoStore);
Preconditions.checkState(repository.isPresent());
Preconditions.checkState(
repository.get().getObjectDatabase().exists()
);
}
@Override
public RawDirectory getDirectory()
throws IOException, GitUserException {
Preconditions.checkState(repository.isPresent());
return new RepositoryObjectTreeWalker(
repository.get()
).getDirectoryContents(Optional.empty());
}
@Override
public Collection<String> commitAndGetMissing(
GitDirectoryContents contents
) throws IOException {
try {
return doCommitAndGetMissing(contents);
} catch (GitAPIException e) {
throw new IOException(e);
}
}
@Override
public void runGC() throws IOException {
Preconditions.checkState(
repository.isPresent(),
"Repo is not present"
);
File dir = getProjectDir();
Preconditions.checkState(dir.isDirectory());
Log.info("[{}] Running git gc", projectName);
Process proc = new ProcessBuilder(
"git", "gc"
).directory(dir).start();
int exitCode;
try {
exitCode = proc.waitFor();
Log.info("Exit: {}", exitCode);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
if (exitCode != 0) {
Log.warn("[{}] Git gc failed", dir.getAbsolutePath());
Log.warn(IOUtils.toString(
proc.getInputStream(),
StandardCharsets.UTF_8
));
Log.warn(IOUtils.toString(
proc.getErrorStream(),
StandardCharsets.UTF_8
));
throw new IOException("git gc error");
}
Log.info("[{}] git gc successful", projectName);
}
@Override
public void deleteIncomingPacks() throws IOException {
Log.info(
"[{}] Checking for garbage `incoming` files",
projectName
);
Files.walkFileTree(getDotGitDir().toPath(), new FileVisitor<Path>() {
@Override
public FileVisitResult preVisitDirectory(
Path dir,
BasicFileAttributes attrs
) throws IOException {
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(
Path file,
BasicFileAttributes attrs
) throws IOException {
File file_ = file.toFile();
String name = file_.getName();
if (name.startsWith("incoming_") && name.endsWith(".pack")) {
Log.info("Deleting garbage `incoming` file: {}", file_);
Preconditions.checkState(file_.delete());
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFileFailed(
Path file,
IOException exc
) throws IOException {
Preconditions.checkNotNull(file);
Preconditions.checkNotNull(exc);
Log.warn("Failed to visit file: " + file, exc);
return FileVisitResult.TERMINATE;
}
@Override
public FileVisitResult postVisitDirectory(
Path dir,
IOException exc
) throws IOException {
Preconditions.checkNotNull(dir);
if (exc != null) {
return FileVisitResult.TERMINATE;
}
return FileVisitResult.CONTINUE;
}
});
}
@Override
public File getProjectDir() {
return getJGitRepository().getDirectory().getParentFile();
}
public void resetHard() throws IOException {
Git git = new Git(getJGitRepository());
try {
git.reset().setMode(ResetCommand.ResetType.HARD).call();
} catch (GitAPIException e) {
throw new IOException(e);
}
}
@Override
public Repository getJGitRepository() {
return repository.get();
}
public File getDotGitDir() {
return getJGitRepository().getWorkTree();
}
private void initRepositoryField(RepoStore repoStore) throws IOException {
Preconditions.checkNotNull(repoStore);
Preconditions.checkArgument(Project.isValidProjectName(projectName));
Preconditions.checkState(!repository.isPresent());
repository = Optional.of(createJGitRepository(repoStore, projectName));
}
private Repository createJGitRepository(
RepoStore repoStore,
String projName
) throws IOException {
File repoDir = new File(repoStore.getRootDirectory(), projName);
return new FileRepositoryBuilder().setWorkTree(repoDir).build();
}
private Collection<String> doCommitAndGetMissing(
GitDirectoryContents contents
) throws IOException, GitAPIException {
Preconditions.checkState(repository.isPresent());
Repository repo = getJGitRepository();
resetHard();
String name = getProjectName();
Log.info("[{}] Writing commit", name);
contents.write();
Git git = new Git(getJGitRepository());
Log.info("[{}] Getting missing files", name);
Set<String> missingFiles = git.status().call().getMissing();
for (String missing : missingFiles) {
Log.info("[{}] Git rm {}", name, missing);
git.rm().setCached(true).addFilepattern(missing).call();
}
Log.info("[{}] Calling Git add", name);
git.add(
).setWorkingTreeIterator(
new NoGitignoreIterator(repo)
).addFilepattern(".").call();
Log.info("[{}] Calling Git commit", name);
git.commit(
).setAuthor(
new PersonIdent(
contents.getUserName(),
contents.getUserEmail(),
contents.getWhen(),
TimeZone.getDefault()
)
).setMessage(
contents.getCommitMessage()
).call();
Log.info(
"[{}] Deleting files in directory: {}",
name,
contents.getDirectory().getAbsolutePath()
);
Util.deleteInDirectoryApartFrom(contents.getDirectory(), ".git");
return missingFiles;
}
}

87
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/NoGitignoreIterator.java Normal file
View file

@ -0,0 +1,87 @@
package uk.ac.ic.wlgitbridge.bridge.repo;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.treewalk.AbstractTreeIterator;
import org.eclipse.jgit.treewalk.FileTreeIterator;
import org.eclipse.jgit.treewalk.WorkingTreeIterator;
import org.eclipse.jgit.treewalk.WorkingTreeOptions;
import org.eclipse.jgit.util.FS;
import java.io.File;
import java.lang.reflect.Field;
/**
* Created by winston on 08/10/2016.
*/
public class NoGitignoreIterator extends FileTreeIterator {
private static final Field ignoreNodeField;
static {
try {
ignoreNodeField = WorkingTreeIterator.class.getDeclaredField(
"ignoreNode"
);
} catch (NoSuchFieldException e) {
throw new RuntimeException(e);
}
ignoreNodeField.setAccessible(true);
}
public NoGitignoreIterator(Repository repo) {
super(repo);
}
public NoGitignoreIterator(
Repository repo,
FileModeStrategy fileModeStrategy
) {
super(repo, fileModeStrategy);
}
public NoGitignoreIterator(File root, FS fs, WorkingTreeOptions options) {
super(root, fs, options);
}
public NoGitignoreIterator(
File root,
FS fs,
WorkingTreeOptions options,
FileModeStrategy fileModeStrategy
) {
super(root, fs, options, fileModeStrategy);
}
protected NoGitignoreIterator(FileTreeIterator p, File root, FS fs) {
super(p, root, fs);
}
protected NoGitignoreIterator(
WorkingTreeIterator p,
File root,
FS fs,
FileModeStrategy fileModeStrategy
) {
super(p, root, fs, fileModeStrategy);
}
// Note: the `list` is a list of top-level entities in this directory,
// not a full list of files in the tree.
@Override
protected void init(Entry[] list) {
super.init(list);
try {
ignoreNodeField.set(this, null);
} catch (IllegalAccessException e) {
throw new RuntimeException(e);
}
}
// When entering a sub-directory, create a new instance of this class,
// so we can also ignore gitignore specifications in sub-directories
@Override
protected AbstractTreeIterator enterSubtree() {
String fullPath = getDirectory().getAbsolutePath() + "/" + current().getName();
return new NoGitignoreIterator(this, new File(fullPath), fs, fileModeStrategy);
}
}

41
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/ProjectRepo.java Normal file
View file

@ -0,0 +1,41 @@
package uk.ac.ic.wlgitbridge.bridge.repo;
import org.eclipse.jgit.lib.Repository;
import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
/**
* Created by winston on 20/08/2016.
*/
public interface ProjectRepo {
String getProjectName();
void initRepo(
RepoStore repoStore
) throws IOException;
void useExistingRepository(
RepoStore repoStore
) throws IOException;
RawDirectory getDirectory(
) throws IOException, GitUserException;
Collection<String> commitAndGetMissing(
GitDirectoryContents gitDirectoryContents
) throws IOException, GitUserException;
void runGC() throws IOException;
void deleteIncomingPacks() throws IOException;
File getProjectDir();
Repository getJGitRepository();
}

104
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java Normal file
View file

@ -0,0 +1,104 @@
package uk.ac.ic.wlgitbridge.bridge.repo;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Repository;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collection;
/**
* Created by winston on 20/08/2016.
*/
public interface RepoStore {
/* Still need to get rid of these two methods.
Main dependency: GitRepoStore needs a Repository which needs a directory.
Instead, use a visitor or something. */
String getRepoStorePath();
File getRootDirectory();
ProjectRepo initRepo(String project) throws IOException;
ProjectRepo initRepoFromExisting(String project, String fromProject) throws IOException;
ProjectRepo getExistingRepo(String project) throws IOException;
ProjectRepo useJGitRepo(Repository repo, ObjectId commitId);
void purgeNonexistentProjects(
Collection<String> existingProjectNames
);
long totalSize();
/**
* Tars and bzip2s the .git directory of the given project. Throws an
* IOException if the project doesn't exist. The returned stream is a copy
* of the original .git directory, which must be deleted using remove().
*/
InputStream bzip2Project(
String projectName,
long[] sizePtr
) throws IOException;
default InputStream bzip2Project(
String projectName
) throws IOException {
return bzip2Project(projectName, null);
}
/**
* Tars and gzips the .git directory of the given project. Throws an
* IOException if the project doesn't exist. The returned stream is a copy
* of the original .git directory, which must be deleted using remove().
*/
InputStream gzipProject(
String projectName,
long[] sizePtr
) throws IOException;
default InputStream gzipProject(
String projectName
) throws IOException {
return gzipProject(projectName, null);
}
void gcProject(String projectName) throws IOException;
/**
* Called after {@link #bzip2Project(String, long[])}'s has been safely
* uploaded to the swap store. Removes all traces of the project from disk,
* i.e. not just its .git, but the whole project's git directory.
* @param projectName
* @throws IOException
*/
void remove(String projectName) throws IOException;
/**
* Unbzip2s the given data stream into a .git directory for projectName.
* Creates the project's git directory.
* If projectName already exists, throws an IOException.
* @param projectName the name of the project, e.g. abc123
* @param dataStream the data stream containing the bzipped contents.
*/
void unbzip2Project(
String projectName,
InputStream dataStream
) throws IOException;
/**
* Ungzips the given data stream into a .git directory for projectName.
* Creates the project's git directory.
* If projectName already exists, throws an IOException.
* @param projectName the name of the project, e.g. abc123
* @param dataStream the data stream containing the gzip contents.
*/
void ungzipProject(
String projectName,
InputStream dataStream
) throws IOException;
}

29
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStoreConfig.java Normal file
View file

@ -0,0 +1,29 @@
package uk.ac.ic.wlgitbridge.bridge.repo;
import javax.annotation.Nullable;
import java.util.Optional;
/**
* Created by winston on 02/07/2017.
*/
public class RepoStoreConfig {
@Nullable
private final Long maxFileSize;
@Nullable
private final Long maxFileNum;
public RepoStoreConfig(Long maxFileSize, Long maxFileNum) {
this.maxFileSize = maxFileSize;
this.maxFileNum = maxFileNum;
}
public Optional<Long> getMaxFileSize() {
return Optional.ofNullable(maxFileSize);
}
public Optional<Long> getMaxFileNum() {
return Optional.ofNullable(maxFileNum);
}
}

95
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/WalkOverrideGitRepo.java Normal file
View file

@ -0,0 +1,95 @@
package uk.ac.ic.wlgitbridge.bridge.repo;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Repository;
import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.git.util.RepositoryObjectTreeWalker;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.Optional;
/**
* This class takes a GitProjectRepo and delegates all calls to it.
*
* The purpose is to insert a file size check in {@link #getDirectory()}.
*
* We delegate instead of subclass because we can't override the static
* constructors in {@link GitProjectRepo}.
*/
public class WalkOverrideGitRepo implements ProjectRepo {
private final GitProjectRepo gitRepo;
private final Optional<Long> maxFileSize;
private final Optional<ObjectId> commitId;
public WalkOverrideGitRepo(
GitProjectRepo gitRepo,
Optional<Long> maxFileSize,
Optional<ObjectId> commitId
) {
this.gitRepo = gitRepo;
this.maxFileSize = maxFileSize;
this.commitId = commitId;
}
@Override
public String getProjectName() {
return gitRepo.getProjectName();
}
@Override
public void initRepo(RepoStore repoStore) throws IOException {
gitRepo.initRepo(repoStore);
}
@Override
public void useExistingRepository(RepoStore repoStore) throws IOException {
gitRepo.useExistingRepository(repoStore);
}
@Override
public RawDirectory getDirectory() throws IOException, GitUserException {
Repository repo = gitRepo.getJGitRepository();
RepositoryObjectTreeWalker walker;
if (commitId.isPresent()) {
walker = new RepositoryObjectTreeWalker(repo, commitId.get());
} else {
walker = new RepositoryObjectTreeWalker(repo);
}
return walker.getDirectoryContents(maxFileSize);
}
@Override
public Collection<String> commitAndGetMissing(
GitDirectoryContents gitDirectoryContents
) throws GitUserException, IOException {
return gitRepo.commitAndGetMissing(gitDirectoryContents);
}
@Override
public void runGC() throws IOException {
gitRepo.runGC();
}
@Override
public void deleteIncomingPacks() throws IOException {
gitRepo.deleteIncomingPacks();
}
@Override
public File getProjectDir() {
return gitRepo.getProjectDir();
}
@Override
public Repository getJGitRepository() {
return gitRepo.getJGitRepository();
}
}

24
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/resource/ResourceCache.java Normal file
View file

@ -0,0 +1,24 @@
package uk.ac.ic.wlgitbridge.bridge.resource;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import uk.ac.ic.wlgitbridge.git.exception.SizeLimitExceededException;
import java.io.IOException;
import java.util.Map;
import java.util.Optional;
/**
* Created by winston on 20/08/2016.
*/
public interface ResourceCache {
RawFile get(
String projectName,
String url,
String newPath,
Map<String, RawFile> fileTable,
Map<String, byte[]> fetchedUrls,
Optional<Long> maxFileSize
) throws IOException, SizeLimitExceededException;
}

142
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/resource/UrlResourceCache.java Normal file
View file

@ -0,0 +1,142 @@
package uk.ac.ic.wlgitbridge.bridge.resource;
import static org.asynchttpclient.Dsl.*;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import uk.ac.ic.wlgitbridge.data.filestore.RepositoryFile;
import uk.ac.ic.wlgitbridge.git.exception.SizeLimitExceededException;
import uk.ac.ic.wlgitbridge.io.http.ning.NingHttpClient;
import uk.ac.ic.wlgitbridge.io.http.ning.NingHttpClientFacade;
import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
import uk.ac.ic.wlgitbridge.util.Log;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
/**
* Created by winston on 20/08/2016.
*/
public class UrlResourceCache implements ResourceCache {
private final DBStore dbStore;
private final NingHttpClientFacade http;
UrlResourceCache(DBStore dbStore, NingHttpClientFacade http) {
this.dbStore = dbStore;
this.http = http;
}
public UrlResourceCache(DBStore dbStore) {
this(dbStore, new NingHttpClient(asyncHttpClient()));
}
@Override
public RawFile get(
String projectName,
String url,
String newPath,
Map<String, RawFile> fileTable,
Map<String, byte[]> fetchedUrls,
Optional<Long> maxFileSize
) throws IOException, SizeLimitExceededException {
String path = dbStore.getPathForURLInProject(projectName, getCacheKeyFromUrl(url));
byte[] contents;
if (path == null) {
path = newPath;
contents = fetch(projectName, url, path, maxFileSize);
fetchedUrls.put(url, contents);
} else {
Log.info("Found (" + projectName + "): " + url);
Log.info("At (" + projectName + "): " + path);
contents = fetchedUrls.get(url);
if (contents == null) {
RawFile rawFile = fileTable.get(path);
if (rawFile == null) {
Log.warn(
"File " + path
+ " was not in the current commit, "
+ "or the git tree, yet path was not null. "
+ "File url is: "
+ url
);
contents = fetch(projectName, url, path, maxFileSize);
} else {
contents = rawFile.getContents();
}
}
}
return new RepositoryFile(newPath, contents);
}
private byte[] fetch(
String projectName,
final String url,
String path,
Optional<Long> maxFileSize
) throws FailedConnectionException, SizeLimitExceededException {
byte[] contents;
Log.info("GET -> " + url);
try {
contents = http.get(url, hs -> {
List<String> contentLengths = hs.getAll("Content-Length");
if (!maxFileSize.isPresent()) {
return true;
}
if (contentLengths.isEmpty()) {
return true;
}
long contentLength = Long.parseLong(contentLengths.get(0));
long maxFileSize_ = maxFileSize.get();
if (contentLength <= maxFileSize_) {
return true;
}
throw new SizeLimitExceededException(
Optional.of(path), contentLength, maxFileSize_
);
});
} catch (ExecutionException e) {
Throwable cause = e.getCause();
if (cause instanceof SizeLimitExceededException) {
throw (SizeLimitExceededException) cause;
}
Log.warn(
"ExecutionException when fetching project: " +
projectName +
", url: " +
url +
", path: " +
path,
e
);
throw new FailedConnectionException();
}
if (maxFileSize.isPresent() && contents.length > maxFileSize.get()) {
throw new SizeLimitExceededException(
Optional.of(path), contents.length, maxFileSize.get());
}
dbStore.addURLIndexForProject(projectName, getCacheKeyFromUrl(url), path);
return contents;
}
/**
* Construct a suitable cache key from the given file URL.
*
* The file URL returned by the web service may contain a token parameter
* used for authentication. This token changes for every request, so we
* need to strip it from the query string before using the URL as a cache
* key.
*/
private String getCacheKeyFromUrl(String url) {
// We're not doing proper URL parsing here, but it should be enough to
// remove the token without touching the important parts of the URL.
//
// The URL looks like:
//
// https://history.overleaf.com/api/projects/:project_id/blobs/:hash?token=:token&_path=:path
return url.replaceAll("token=[^&]*", "token=REMOVED");
}
}

55
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/snapshot/NetSnapshotApi.java Normal file
View file

@ -0,0 +1,55 @@
package uk.ac.ic.wlgitbridge.bridge.snapshot;
import com.google.api.client.auth.oauth2.Credential;
import uk.ac.ic.wlgitbridge.data.CandidateSnapshot;
import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocRequest;
import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocResult;
import uk.ac.ic.wlgitbridge.snapshot.getforversion.GetForVersionRequest;
import uk.ac.ic.wlgitbridge.snapshot.getforversion.GetForVersionResult;
import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.GetSavedVersRequest;
import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.GetSavedVersResult;
import uk.ac.ic.wlgitbridge.snapshot.push.PushRequest;
import uk.ac.ic.wlgitbridge.snapshot.push.PushResult;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
/**
* Created by winston on 20/08/2016.
*/
public class NetSnapshotApi implements SnapshotApi {
@Override
public CompletableFuture<GetDocResult> getDoc(
Optional<Credential> oauth2, String projectName) {
return new GetDocRequest(opt(oauth2), projectName).request();
}
@Override
public CompletableFuture<GetForVersionResult> getForVersion(
Optional<Credential> oauth2, String projectName, int versionId) {
return new GetForVersionRequest(
opt(oauth2), projectName, versionId).request();
}
@Override
public CompletableFuture<GetSavedVersResult> getSavedVers(
Optional<Credential> oauth2, String projectName) {
return new GetSavedVersRequest(opt(oauth2), projectName).request();
}
@Override
public CompletableFuture<PushResult> push(
Optional<Credential> oauth2,
CandidateSnapshot candidateSnapshot,
String postbackKey
) {
return new PushRequest(
opt(oauth2), candidateSnapshot, postbackKey).request();
}
private static Credential opt(Optional<Credential> oauth2) {
return oauth2.orElse(null);
}
}

54
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/snapshot/SnapshotApi.java Normal file
View file

@ -0,0 +1,54 @@
package uk.ac.ic.wlgitbridge.bridge.snapshot;
import com.google.api.client.auth.oauth2.Credential;
import uk.ac.ic.wlgitbridge.data.CandidateSnapshot;
import uk.ac.ic.wlgitbridge.snapshot.base.MissingRepositoryException;
import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocResult;
import uk.ac.ic.wlgitbridge.snapshot.getforversion.GetForVersionResult;
import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.GetSavedVersResult;
import uk.ac.ic.wlgitbridge.snapshot.push.PushResult;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
/**
* Created by winston on 20/08/2016.
*/
public interface SnapshotApi {
CompletableFuture<GetDocResult> getDoc(
Optional<Credential> oauth2, String projectName);
CompletableFuture<GetForVersionResult> getForVersion(
Optional<Credential> oauth2, String projectName, int versionId);
CompletableFuture<GetSavedVersResult> getSavedVers(
Optional<Credential> oauth2, String projectName);
CompletableFuture<PushResult> push(
Optional<Credential> oauth2,
CandidateSnapshot candidateSnapshot,
String postbackKey);
static <T> T getResult(CompletableFuture<T> result)
throws MissingRepositoryException, FailedConnectionException, ForbiddenException {
try {
return result.join();
} catch (CompletionException e) {
try {
throw e.getCause();
} catch (MissingRepositoryException
| FailedConnectionException
| ForbiddenException
| RuntimeException r) {
throw r;
} catch (Throwable __) {
throw e;
}
}
}
}

162
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/snapshot/SnapshotApiFacade.java Normal file
View file

@ -0,0 +1,162 @@
package uk.ac.ic.wlgitbridge.bridge.snapshot;
import com.google.api.client.auth.oauth2.Credential;
import uk.ac.ic.wlgitbridge.data.CandidateSnapshot;
import uk.ac.ic.wlgitbridge.data.model.Snapshot;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.snapshot.base.MissingRepositoryException;
import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocResult;
import uk.ac.ic.wlgitbridge.snapshot.getforversion.GetForVersionResult;
import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotData;
import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.GetSavedVersResult;
import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.SnapshotInfo;
import uk.ac.ic.wlgitbridge.snapshot.push.PushResult;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.InvalidProjectException;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
/**
* Created by winston on 02/07/2017.
*/
public class SnapshotApiFacade {
private final SnapshotApi api;
public SnapshotApiFacade(SnapshotApi api) {
this.api = api;
}
public boolean projectExists(
Optional<Credential> oauth2,
String projectName
) throws FailedConnectionException, GitUserException {
try {
SnapshotApi
.getResult(api.getDoc(oauth2, projectName))
.getVersionID();
return true;
} catch (InvalidProjectException e) {
return false;
}
}
public Optional<GetDocResult> getDoc(
Optional<Credential> oauth2,
String projectName
) throws FailedConnectionException, GitUserException {
try {
GetDocResult doc = SnapshotApi
.getResult(api.getDoc(oauth2, projectName));
doc.getVersionID();
return Optional.of(doc);
} catch (InvalidProjectException e) {
return Optional.empty();
}
}
public Deque<Snapshot> getSnapshots(
Optional<Credential> oauth2,
String projectName,
int afterVersionId
) throws GitUserException, FailedConnectionException {
List<SnapshotInfo> snapshotInfos = getSnapshotInfosAfterVersion(
oauth2,
projectName,
afterVersionId
);
List<SnapshotData> snapshotDatas = getMatchingSnapshotData(
oauth2,
projectName,
snapshotInfos
);
return combine(snapshotInfos, snapshotDatas);
}
public PushResult push(
Optional<Credential> oauth2,
CandidateSnapshot candidateSnapshot,
String postbackKey
) throws MissingRepositoryException, FailedConnectionException, ForbiddenException {
return SnapshotApi.getResult(api.push(
oauth2, candidateSnapshot, postbackKey));
}
private List<SnapshotInfo> getSnapshotInfosAfterVersion(
Optional<Credential> oauth2,
String projectName,
int version
) throws FailedConnectionException, GitUserException {
SortedSet<SnapshotInfo> versions = new TreeSet<>();
CompletableFuture<GetDocResult> getDoc
= api.getDoc(oauth2, projectName);
CompletableFuture<GetSavedVersResult> savedVers
= api.getSavedVers(oauth2, projectName);
GetDocResult latestDoc = SnapshotApi.getResult(getDoc);
int latest = latestDoc.getVersionID();
// Handle edge-case for projects with no changes, that were imported
// to v2. In which case both `latest` and `version` will be zero.
// See: https://github.com/overleaf/writelatex-git-bridge/pull/50
if (latest > version || (latest == 0 && version == 0)) {
for (
SnapshotInfo snapshotInfo :
SnapshotApi.getResult(savedVers).getSavedVers()
) {
if (snapshotInfo.getVersionId() > version) {
versions.add(snapshotInfo);
}
}
versions.add(new SnapshotInfo(
latest,
latestDoc.getCreatedAt(),
latestDoc.getName(),
latestDoc.getEmail()
));
}
return new ArrayList<>(versions);
}
private List<SnapshotData> getMatchingSnapshotData(
Optional<Credential> oauth2,
String projectName,
List<SnapshotInfo> snapshotInfos
) throws FailedConnectionException, ForbiddenException {
List<CompletableFuture<GetForVersionResult>> firedRequests
= fireDataRequests(oauth2, projectName, snapshotInfos);
List<SnapshotData> snapshotDataList = new ArrayList<>();
for (CompletableFuture<GetForVersionResult> fired : firedRequests) {
snapshotDataList.add(fired.join().getSnapshotData());
}
return snapshotDataList;
}
private List<CompletableFuture<GetForVersionResult>> fireDataRequests(
Optional<Credential> oauth2,
String projectName,
List<SnapshotInfo> snapshotInfos
) {
return snapshotInfos
.stream()
.map(snap -> api.getForVersion(
oauth2, projectName, snap.getVersionId()))
.collect(Collectors.toList());
}
private static Deque<Snapshot> combine(
List<SnapshotInfo> snapshotInfos,
List<SnapshotData> snapshotDatas
) {
Deque<Snapshot> snapshots = new LinkedList<>();
Iterator<SnapshotInfo> infos = snapshotInfos.iterator();
Iterator<SnapshotData> datas = snapshotDatas.iterator();
while (infos.hasNext()) {
snapshots.add(new Snapshot(infos.next(), datas.next()));
}
return snapshots;
}
}

20
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/NoopSwapJob.java Normal file
View file

@ -0,0 +1,20 @@
package uk.ac.ic.wlgitbridge.bridge.swap.job;
/**
* Created by winston on 24/08/2016.
*/
public class NoopSwapJob implements SwapJob {
@Override
public void start() {}
@Override
public void stop() {}
@Override
public void evict(String projName) {}
@Override
public void restore(String projName) {}
}

124
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJob.java Normal file
View file

@ -0,0 +1,124 @@
package uk.ac.ic.wlgitbridge.bridge.swap.job;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore;
import java.io.IOException;
import java.util.Optional;
/**
* Created by winston on 20/08/2016.
*/
public interface SwapJob {
enum CompressionMethod { Bzip2, Gzip }
static CompressionMethod stringToCompressionMethod(String compressionString) {
if (compressionString == null) {
return null;
}
CompressionMethod result;
switch (compressionString) {
case "gzip":
result = CompressionMethod.Gzip;
break;
case "bzip2":
result = CompressionMethod.Bzip2;
break;
default:
result = null;
break;
}
return result;
}
static String compressionMethodAsString(CompressionMethod compressionMethod) {
if (compressionMethod == null) {
return null;
}
String result;
switch (compressionMethod) {
case Gzip:
result = "gzip";
break;
case Bzip2:
result = "bzip2";
break;
default:
result = null;
break;
}
return result;
}
static SwapJob fromConfig(
Optional<SwapJobConfig> cfg,
ProjectLock lock,
RepoStore repoStore,
DBStore dbStore,
SwapStore swapStore
) {
if (cfg.isPresent()) {
return new SwapJobImpl(
cfg.get(),
lock,
repoStore,
dbStore,
swapStore
);
}
return new NoopSwapJob();
}
/**
* Starts the swap job, which should schedule an attempted swap at the given
* configured interval (config["swapJob"]["intervalMillis"]
*/
void start();
/**
* Stops the stop job.
*/
void stop();
/**
* Called by the swap job when a project should be evicted.
*
* Pre:
* 1. projName must be in repoStore
* 2. projName should not be in swapStore
* 3. projName should be PRESENT in dbStore (last_accessed is not null)
*
* Acquires the project lock and performs an eviction of projName.
*
* Post:
* 1. projName should not in repoStore
* 2. projName must be in swapStore
* 3. projName must be SWAPPED in dbStore (last_accessed is null)
* @param projName
* @throws IOException
*/
void evict(String projName) throws IOException;
/**
* Called on a project when it must be restored.
*
* Pre:
* 1. projName should not be in repoStore
* 2. projName must be in swapStore
* 3. projName must be SWAPPED in dbStore (last_accessed is null)
*
* Acquires the project lock and restores projName.
*
* Post:
* 1. projName must be in repoStore
* 2. projName should not in swapStore
* 3. projName should be PRESENT in dbStore (last_accessed is not null)
* @param projName
* @throws IOException
*/
void restore(String projName) throws IOException;
}

55
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobConfig.java Normal file
View file

@ -0,0 +1,55 @@
package uk.ac.ic.wlgitbridge.bridge.swap.job;
import uk.ac.ic.wlgitbridge.util.Log;
import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJob.CompressionMethod;
/**
* Created by winston on 23/08/2016.
*/
public class SwapJobConfig {
private final int minProjects;
private final int lowGiB;
private final int highGiB;
private final long intervalMillis;
private final String compressionMethod;
public SwapJobConfig(
int minProjects,
int lowGiB,
int highGiB,
long intervalMillis,
String compressionMethod
) {
this.minProjects = minProjects;
this.lowGiB = lowGiB;
this.highGiB = highGiB;
this.intervalMillis = intervalMillis;
this.compressionMethod = compressionMethod;
}
public int getMinProjects() {
return minProjects;
}
public int getLowGiB() {
return lowGiB;
}
public int getHighGiB() {
return highGiB;
}
public long getIntervalMillis() {
return intervalMillis;
}
public SwapJob.CompressionMethod getCompressionMethod() {
CompressionMethod result = SwapJob.stringToCompressionMethod(compressionMethod);
if (result == null) {
Log.info("SwapJobConfig: un-supported compressionMethod '{}', default to 'bzip2'", compressionMethod);
result = CompressionMethod.Bzip2;
}
return result;
}
}

262
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/job/SwapJobImpl.java Normal file
View file

@ -0,0 +1,262 @@
package uk.ac.ic.wlgitbridge.bridge.swap.job;
import com.google.api.client.repackaged.com.google.common.base.Preconditions;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.lock.LockGuard;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore;
import uk.ac.ic.wlgitbridge.util.Log;
import uk.ac.ic.wlgitbridge.util.TimerUtils;
import java.io.IOException;
import java.io.InputStream;
import java.sql.Timestamp;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Timer;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Created by winston on 20/08/2016.
*/
public class SwapJobImpl implements SwapJob {
private static final long GiB = (1l << 30);
int minProjects;
long lowWatermarkBytes;
long highWatermarkBytes;
Duration interval;
private final ProjectLock lock;
private final RepoStore repoStore;
private final DBStore dbStore;
private final SwapStore swapStore;
private final CompressionMethod compressionMethod;
private final Timer timer;
final AtomicInteger swaps;
public SwapJobImpl(
SwapJobConfig cfg,
ProjectLock lock,
RepoStore repoStore,
DBStore dbStore,
SwapStore swapStore
) {
this(
cfg.getMinProjects(),
GiB * cfg.getLowGiB(),
GiB * cfg.getHighGiB(),
Duration.ofMillis(cfg.getIntervalMillis()),
cfg.getCompressionMethod(),
lock,
repoStore,
dbStore,
swapStore
);
}
SwapJobImpl(
int minProjects,
long lowWatermarkBytes,
long highWatermarkBytes,
Duration interval,
CompressionMethod method,
ProjectLock lock,
RepoStore repoStore,
DBStore dbStore,
SwapStore swapStore
) {
this.minProjects = minProjects;
this.lowWatermarkBytes = lowWatermarkBytes;
this.highWatermarkBytes = highWatermarkBytes;
this.interval = interval;
this.compressionMethod = method;
this.lock = lock;
this.repoStore = repoStore;
this.dbStore = dbStore;
this.swapStore = swapStore;
timer = new Timer();
swaps = new AtomicInteger(0);
}
@Override
public void start() {
timer.schedule(
TimerUtils.makeTimerTask(this::doSwap),
0
);
}
@Override
public void stop() {
timer.cancel();
}
private void doSwap() {
try {
doSwap_();
} catch (Throwable t) {
Log.warn("Exception thrown during swap job", t);
}
timer.schedule(
TimerUtils.makeTimerTask(this::doSwap),
interval.toMillis()
);
}
private void doSwap_() {
ArrayList<String> exceptionProjectNames = new ArrayList<String>();
Log.info("Running swap number {}", swaps.get() + 1);
long totalSize = repoStore.totalSize();
Log.info("Size is {}/{} (high)", totalSize, highWatermarkBytes);
if (totalSize < highWatermarkBytes) {
Log.info("No need to swap.");
swaps.incrementAndGet();
return;
}
int numProjects = dbStore.getNumProjects();
// while we have too many projects on disk
while (
(totalSize = repoStore.totalSize()) > lowWatermarkBytes &&
(numProjects = dbStore.getNumUnswappedProjects()) > minProjects
) {
// check if we've had too many exceptions so far
if (exceptionProjectNames.size() >= 20) {
StringBuilder sb = new StringBuilder();
for (String s: exceptionProjectNames) {
sb.append(s);
sb.append(' ');
}
Log.error(
"Too many exceptions while running swap, giving up on this run: {}",
sb.toString()
);
break;
}
// get the oldest project and try to swap it
String projectName = dbStore.getOldestUnswappedProject();
try {
evict(projectName);
} catch (Exception e) {
Log.warn("[{}] Exception while swapping, mark project and move on", projectName, e);
// NOTE: this is something of a hack. If a project fails to swap we get stuck in a
// loop where `dbStore.getOldestUnswappedProject()` gives the same failing project over and over again,
// which fills up the disk with errors. By touching the access time we can mark the project as a
// non-candidate for swapping. Ideally we should be checking the logs for these log events and fixing
// whatever is wrong with the project
dbStore.setLastAccessedTime(
projectName,
Timestamp.valueOf(LocalDateTime.now())
);
exceptionProjectNames.add(projectName);
}
}
if (totalSize > lowWatermarkBytes) {
Log.warn(
"Finished swapping, but total size is still too high."
);
}
Log.info(
"Size: {}/{} (low), " +
"{} (high), " +
"projects on disk: {}/{}, " +
"min projects on disk: {}",
totalSize,
lowWatermarkBytes,
highWatermarkBytes,
numProjects,
dbStore.getNumProjects(),
minProjects
);
swaps.incrementAndGet();
}
/**
* @see SwapJob#evict(String) for high-level description.
*
* 1. Acquires the project lock.
* 2. Gets a bz2 stream and size of a project from the repo store, or throws
* 3. Uploads the bz2 stream and size to the projName in the swapStore.
* 4. Sets the last accessed time in the dbStore to null, which makes our
* state SWAPPED
* 5. Removes the project from the repo store.
* @param projName
* @throws IOException
*/
@Override
public void evict(String projName) throws IOException {
Preconditions.checkNotNull(projName, "projName was null");
Log.info("Evicting project: {}", projName);
try (LockGuard __ = lock.lockGuard(projName)) {
try {
repoStore.gcProject(projName);
} catch (Exception e) {
Log.error("[{}] Exception while running gc on project: {}", projName, e);
}
long[] sizePtr = new long[1];
try (InputStream blob = getBlobStream(projName, sizePtr)) {
swapStore.upload(projName, blob, sizePtr[0]);
String compression = SwapJob.compressionMethodAsString(compressionMethod);
if (compression == null) {
throw new RuntimeException("invalid compression method, should not happen");
}
dbStore.swap(projName, compression);
repoStore.remove(projName);
}
}
Log.info("Evicted project: {}", projName);
}
private InputStream getBlobStream(String projName, long[] sizePtr) throws IOException {
if (compressionMethod == CompressionMethod.Gzip) {
return repoStore.gzipProject(projName, sizePtr);
} else if (compressionMethod == CompressionMethod.Bzip2) {
return repoStore.bzip2Project(projName, sizePtr);
} else {
throw new RuntimeException("invalid compression method, should not happen");
}
}
/**
* @see SwapJob#restore(String) for high-level description.
*
* 1. Acquires the project lock.
* 2. Gets a bz2 stream for the project from the swapStore.
* 3. Fully downloads and places the bz2 stream back in the repo store.
* 4. Sets the last accessed time in the dbStore to now, which makes our
* state PRESENT and the last project to be evicted.
* @param projName
* @throws IOException
*/
@Override
public void restore(String projName) throws IOException {
try (LockGuard __ = lock.lockGuard(projName)) {
try (InputStream zipped = swapStore.openDownloadStream(projName)) {
String compression = dbStore.getSwapCompression(projName);
if (compression == null) {
throw new RuntimeException("Missing compression method during restore, should not happen");
}
if ("gzip".equals(compression)) {
repoStore.ungzipProject(
projName,
zipped
);
} else if ("bzip2".equals(compression)) {
repoStore.unbzip2Project(
projName,
zipped
);
}
swapStore.remove(projName);
dbStore.restore(projName);
}
}
}
}

54
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/InMemorySwapStore.java Normal file
View file

@ -0,0 +1,54 @@
package uk.ac.ic.wlgitbridge.bridge.swap.store;
import org.apache.commons.io.IOUtils;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
/**
* Created by winston on 23/08/2016.
*/
public class InMemorySwapStore implements SwapStore {
private final Map<String, byte[]> store;
public InMemorySwapStore() {
store = new HashMap<>();
}
public InMemorySwapStore(SwapStoreConfig __) {
this();
}
@Override
public void upload(
String projectName,
InputStream uploadStream,
long contentLength
) throws IOException {
store.put(
projectName,
IOUtils.toByteArray(uploadStream, contentLength)
);
}
@Override
public InputStream openDownloadStream(String projectName) {
byte[] buf = store.get(projectName);
if (buf == null) {
throw new IllegalArgumentException(
"no such project in swap store: " + projectName
);
}
return new ByteArrayInputStream(buf);
}
@Override
public void remove(String projectName) {
store.remove(projectName);
}
}

28
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/NoopSwapStore.java Normal file
View file

@ -0,0 +1,28 @@
package uk.ac.ic.wlgitbridge.bridge.swap.store;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
/**
* Created by winston on 24/08/2016.
*/
public class NoopSwapStore implements SwapStore {
public NoopSwapStore(SwapStoreConfig __) {}
@Override
public void upload(
String projectName,
InputStream uploadStream,
long contentLength
) {}
@Override
public InputStream openDownloadStream(String projectName) {
return new ByteArrayInputStream(new byte[0]);
}
@Override
public void remove(String projectName) {}
}

86
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/S3SwapStore.java Normal file
View file

@ -0,0 +1,86 @@
package uk.ac.ic.wlgitbridge.bridge.swap.store;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.*;
import java.io.InputStream;
/**
* Created by winston on 21/08/2016.
*/
public class S3SwapStore implements SwapStore {
private final AmazonS3 s3;
private final String bucketName;
public S3SwapStore(SwapStoreConfig cfg) {
this(
cfg.getAwsAccessKey(),
cfg.getAwsSecret(),
cfg.getS3BucketName(),
cfg.getAwsRegion()
);
}
S3SwapStore(
String accessKey,
String secret,
String bucketName,
String region
) {
String regionToUse = null;
if (region == null) {
regionToUse = "us-east-1";
} else {
regionToUse = region;
}
s3 = AmazonS3ClientBuilder
.standard()
.withRegion(regionToUse)
.withCredentials(
new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secret))
).build();
this.bucketName = bucketName;
}
@Override
public void upload(
String projectName,
InputStream uploadStream,
long contentLength
) {
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(contentLength);
PutObjectRequest put = new PutObjectRequest(
bucketName,
projectName,
uploadStream,
metadata
);
PutObjectResult res = s3.putObject(put);
}
@Override
public InputStream openDownloadStream(String projectName) {
GetObjectRequest get = new GetObjectRequest(
bucketName,
projectName
);
S3Object res = s3.getObject(get);
return res.getObjectContent();
}
@Override
public void remove(String projectName) {
DeleteObjectRequest del = new DeleteObjectRequest(
bucketName,
projectName
);
s3.deleteObject(del);
}
}

44
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/SwapStore.java Normal file
View file

@ -0,0 +1,44 @@
package uk.ac.ic.wlgitbridge.bridge.swap.store;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
/**
* Created by winston on 20/08/2016.
*/
public interface SwapStore {
Map<String, Function<SwapStoreConfig, SwapStore>> swapStores =
new HashMap<String, Function<SwapStoreConfig, SwapStore>>() {
{
put("noop", NoopSwapStore::new);
put("memory", InMemorySwapStore::new);
put("s3", S3SwapStore::new);
}
};
static SwapStore fromConfig(
Optional<SwapStoreConfig> cfg
) {
SwapStoreConfig cfg_ = cfg.orElse(SwapStoreConfig.NOOP);
String type = cfg_.getType();
return swapStores.get(type).apply(cfg_);
}
void upload(
String projectName,
InputStream uploadStream,
long contentLength
) throws IOException;
InputStream openDownloadStream(String projectName);
void remove(String projectName);
}

85
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/store/SwapStoreConfig.java Normal file
View file

@ -0,0 +1,85 @@
package uk.ac.ic.wlgitbridge.bridge.swap.store;
/**
* Created by winston on 24/08/2016.
*/
public class SwapStoreConfig {
public static final SwapStoreConfig NOOP = new SwapStoreConfig(
"noop",
null,
null,
null,
null
);
private String type;
private String awsAccessKey;
private String awsSecret;
private String s3BucketName;
private String awsRegion;
public SwapStoreConfig() {}
public SwapStoreConfig(
String awsAccessKey,
String awsSecret,
String s3BucketName,
String awsRegion
) {
this(
"s3",
awsAccessKey,
awsSecret,
s3BucketName,
awsRegion
);
}
SwapStoreConfig(
String type,
String awsAccessKey,
String awsSecret,
String s3BucketName,
String awsRegion
) {
this.type = type;
this.awsAccessKey = awsAccessKey;
this.awsSecret = awsSecret;
this.s3BucketName = s3BucketName;
this.awsRegion = awsRegion;
}
public String getType() {
return type;
}
public String getAwsAccessKey() {
return awsAccessKey;
}
public String getAwsSecret() {
return awsSecret;
}
public String getS3BucketName() {
return s3BucketName;
}
public String getAwsRegion() { return awsRegion; }
public SwapStoreConfig sanitisedCopy() {
return new SwapStoreConfig(
type,
awsAccessKey == null ? null : "<awsAccessKey>",
awsSecret == null ? null : "<awsSecret>",
s3BucketName,
awsRegion
);
}
public static SwapStoreConfig sanitisedCopy(SwapStoreConfig swapStore) {
return swapStore == null ? null : swapStore.sanitisedCopy();
}
}

18
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/util/CastUtil.java Normal file
View file

@ -0,0 +1,18 @@
package uk.ac.ic.wlgitbridge.bridge.util;
import com.google.common.base.Preconditions;
/**
* Created by winston on 01/07/2017.
*/
public class CastUtil {
public static int assumeInt(long l) {
Preconditions.checkArgument(
l <= (long) Integer.MAX_VALUE
&& l >= (long) Integer.MIN_VALUE,
l + " cannot fit inside an int");
return (int) l;
}
}

150
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/CandidateSnapshot.java Normal file
View file

@ -0,0 +1,150 @@
package uk.ac.ic.wlgitbridge.data;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
import uk.ac.ic.wlgitbridge.util.Util;
import java.io.File;
import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
/**
* Created by Winston on 16/11/14.
*/
public class CandidateSnapshot implements AutoCloseable {
private final String projectName;
private final int currentVersion;
private final List<ServletFile> files;
private final List<String> deleted;
private File attsDirectory;
public CandidateSnapshot(
String projectName,
int currentVersion,
RawDirectory directoryContents,
RawDirectory oldDirectoryContents
) {
this.projectName = projectName;
this.currentVersion = currentVersion;
files = diff(directoryContents, oldDirectoryContents);
deleted = deleted(directoryContents, oldDirectoryContents);
}
private List<ServletFile> diff(
RawDirectory directoryContents,
RawDirectory oldDirectoryContents
) {
List<ServletFile> files = new LinkedList<ServletFile>();
Map<String, RawFile> fileTable = directoryContents.getFileTable();
Map<String, RawFile> oldFileTable = oldDirectoryContents.getFileTable();
for (Entry<String, RawFile> entry : fileTable.entrySet()) {
RawFile file = entry.getValue();
files.add(new ServletFile(file, oldFileTable.get(file.getPath())));
}
return files;
}
private List<String> deleted(
RawDirectory directoryContents,
RawDirectory oldDirectoryContents
) {
List<String> deleted = new LinkedList<String>();
Map<String, RawFile> fileTable = directoryContents.getFileTable();
for (
Entry<String, RawFile> entry :
oldDirectoryContents.getFileTable().entrySet()
) {
String path = entry.getKey();
RawFile newFile = fileTable.get(path);
if (newFile == null) {
deleted.add(path);
}
}
return deleted;
}
public void writeServletFiles(File rootGitDirectory) throws IOException {
attsDirectory = new File(
rootGitDirectory,
".wlgb/atts/" + projectName
);
for (ServletFile file : files) {
if (file.isChanged()) {
file.writeToDiskWithName(attsDirectory, file.getUniqueIdentifier());
}
}
}
public void deleteServletFiles() throws IOException {
if (attsDirectory != null) {
Util.deleteDirectory(attsDirectory);
}
}
public JsonElement getJsonRepresentation(String postbackKey) {
String projectURL = Util.getPostbackURL() + "api/" + projectName;
JsonObject jsonObject = new JsonObject();
jsonObject.addProperty("latestVerId", currentVersion);
jsonObject.add("files", getFilesAsJson(projectURL, postbackKey));
jsonObject.addProperty(
"postbackUrl", projectURL + "/" + postbackKey + "/postback"
);
return jsonObject;
}
private JsonArray getFilesAsJson(String projectURL, String postbackKey) {
JsonArray filesArray = new JsonArray();
for (ServletFile file : files) {
filesArray.add(getFileAsJson(file, projectURL, postbackKey));
}
return filesArray;
}
private JsonObject getFileAsJson(
ServletFile file,
String projectURL,
String postbackKey
) {
JsonObject jsonFile = new JsonObject();
jsonFile.addProperty("name", file.getPath());
if (file.isChanged()) {
String identifier = file.getUniqueIdentifier();
String url = projectURL + "/" + identifier + "?key=" + postbackKey;
jsonFile.addProperty("url", url);
}
return jsonFile;
}
public String getProjectName() {
return projectName;
}
public List<String> getDeleted() {
return deleted;
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("VersionId: ");
sb.append(currentVersion);
sb.append(", files: ");
sb.append(files);
sb.append(", deleted: ");
sb.append(deleted);
return sb.toString();
}
@Override
public void close() throws IOException {
deleteServletFiles();
}
}

10
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/LockAllWaiter.java Normal file
View file

@ -0,0 +1,10 @@
package uk.ac.ic.wlgitbridge.data;
/**
* Created by Winston on 21/02/15.
*/
public interface LockAllWaiter {
void threadsRemaining(int threads);
}

89
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/ProjectLockImpl.java Normal file
View file

@ -0,0 +1,89 @@
package uk.ac.ic.wlgitbridge.data;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import uk.ac.ic.wlgitbridge.util.Log;
/**
* Created by Winston on 20/11/14.
*/
public class ProjectLockImpl implements ProjectLock {
private final Map<String, Lock> projectLocks;
private final ReentrantReadWriteLock rwlock;
private final Lock rlock;
private final ReentrantReadWriteLock.WriteLock wlock;
private LockAllWaiter waiter;
private boolean waiting;
public ProjectLockImpl() {
projectLocks = new HashMap<String, Lock>();
rwlock = new ReentrantReadWriteLock();
rlock = rwlock.readLock();
wlock = rwlock.writeLock();
waiting = false;
}
public ProjectLockImpl(LockAllWaiter waiter) {
this();
setWaiter(waiter);
}
@Override
public void lockForProject(String projectName) {
Log.debug("[{}] taking project lock", projectName);
getLockForProjectName(projectName).lock();
Log.debug("[{}] taking reentrant lock", projectName);
rlock.lock();
Log.debug("[{}] taken locks", projectName);
}
@Override
public void unlockForProject(String projectName) {
Log.debug("[{}] releasing project lock", projectName);
getLockForProjectName(projectName).unlock();
Log.debug("[{}] releasing reentrant lock", projectName);
rlock.unlock();
Log.debug("[{}] released locks", projectName);
if (waiting) {
Log.debug("[{}] waiting for remaining threads", projectName);
trySignal();
}
}
private void trySignal() {
int threads = rwlock.getReadLockCount();
Log.debug("-> waiting for {} threads", threads);
if (waiter != null && threads > 0) {
waiter.threadsRemaining(threads);
}
Log.debug("-> finished waiting for threads");
}
public void lockAll() {
Log.debug("-> locking all threads");
waiting = true;
trySignal();
Log.debug("-> locking reentrant write lock");
wlock.lock();
}
private synchronized Lock getLockForProjectName(String projectName) {
Lock lock = projectLocks.get(projectName);
if (lock == null) {
lock = new ReentrantLock();
projectLocks.put(projectName, lock);
}
return lock;
}
public void setWaiter(LockAllWaiter waiter) {
this.waiter = waiter;
}
}

46
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/ServletFile.java Normal file
View file

@ -0,0 +1,46 @@
package uk.ac.ic.wlgitbridge.data;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import java.util.UUID;
/**
* Created by Winston on 21/02/15.
*/
public class ServletFile extends RawFile {
private final RawFile file;
private final boolean changed;
private String uuid;
public ServletFile(RawFile file, RawFile oldFile) {
this.file = file;
this.uuid = UUID.randomUUID().toString();
changed = !equals(oldFile);
}
public String getUniqueIdentifier() { return uuid; }
@Override
public String getPath() {
return file.getPath();
}
@Override
public byte[] getContents() {
return file.getContents();
}
@Override
public long size() {
return getContents().length;
}
public boolean isChanged() {
return changed;
}
@Override
public String toString() {
return getPath();
}
}

84
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/filestore/GitDirectoryContents.java Normal file
View file

@ -0,0 +1,84 @@
package uk.ac.ic.wlgitbridge.data.filestore;
import uk.ac.ic.wlgitbridge.data.model.Snapshot;
import uk.ac.ic.wlgitbridge.util.Util;
import java.io.File;
import java.io.IOException;
import java.util.Date;
import java.util.List;
/**
* Created by Winston on 14/11/14.
*/
public class GitDirectoryContents {
private final List<RawFile> files;
private final File gitDirectory;
private final String userName;
private final String userEmail;
private final String commitMessage;
private final Date when;
public GitDirectoryContents(
List<RawFile> files,
File rootGitDirectory,
String projectName,
String userName,
String userEmail,
String commitMessage,
Date when
) {
this.files = files;
this.gitDirectory = new File(rootGitDirectory, projectName);
this.userName = userName;
this.userEmail = userEmail;
this.commitMessage = commitMessage;
this.when = when;
}
public GitDirectoryContents(
List<RawFile> files,
File rootGitDirectory,
String projectName,
Snapshot snapshot
) {
this(
files,
rootGitDirectory,
projectName,
snapshot.getUserName(),
snapshot.getUserEmail(),
snapshot.getComment(),
snapshot.getCreatedAt()
);
}
public void write() throws IOException {
Util.deleteInDirectoryApartFrom(gitDirectory, ".git");
for (RawFile fileNode : files) {
fileNode.writeToDisk(gitDirectory);
}
}
public File getDirectory() {
return gitDirectory;
}
public String getUserName() {
return userName;
}
public String getUserEmail() {
return userEmail;
}
public String getCommitMessage() {
return commitMessage;
}
public Date getWhen() {
return when;
}
}

23
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/filestore/RawDirectory.java Normal file
View file

@ -0,0 +1,23 @@
package uk.ac.ic.wlgitbridge.data.filestore;
import uk.ac.ic.wlgitbridge.git.exception.SizeLimitExceededException;
import java.util.Map;
import java.util.Optional;
/**
* Created by Winston on 16/11/14.
*/
public class RawDirectory {
private final Map<String, RawFile> fileTable;
public RawDirectory(Map<String, RawFile> fileTable) {
this.fileTable = fileTable;
}
public Map<String, RawFile> getFileTable() {
return fileTable;
}
}

46
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/filestore/RawFile.java Normal file
View file

@ -0,0 +1,46 @@
package uk.ac.ic.wlgitbridge.data.filestore;
import uk.ac.ic.wlgitbridge.util.Log;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Arrays;
/**
* Created by Winston on 16/11/14.
*/
public abstract class RawFile {
public abstract String getPath();
public abstract byte[] getContents();
public abstract long size();
public final void writeToDisk(File directory) throws IOException {
writeToDiskWithName(directory, getPath());
}
public final void writeToDiskWithName(File directory, String name) throws IOException {
File file = new File(directory, name);
file.getParentFile().mkdirs();
file.createNewFile();
OutputStream out = new FileOutputStream(file);
out.write(getContents());
out.close();
Log.info("Wrote file: {}", file.getAbsolutePath());
}
@Override
public boolean equals(Object obj) {
if (!(obj instanceof RawFile)) {
return false;
}
RawFile that = (RawFile) obj;
return getPath().equals(that.getPath())
&& Arrays.equals(getContents(), that.getContents());
}
}

31
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/filestore/RepositoryFile.java Normal file
View file

@ -0,0 +1,31 @@
package uk.ac.ic.wlgitbridge.data.filestore;
/**
* Created by Winston on 16/11/14.
*/
public class RepositoryFile extends RawFile {
private final String path;
private final byte[] contents;
public RepositoryFile(String path, byte[] contents) {
this.path = path;
this.contents = contents;
}
@Override
public String getPath() {
return path;
}
@Override
public byte[] getContents() {
return contents;
}
@Override
public long size() {
return contents.length;
}
}

77
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/Snapshot.java Normal file
View file

@ -0,0 +1,77 @@
package uk.ac.ic.wlgitbridge.data.model;
import org.joda.time.DateTime;
import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotAttachment;
import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotData;
import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotFile;
import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.SnapshotInfo;
import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.WLUser;
import java.util.Date;
import java.util.List;
/**
* Created by Winston on 03/11/14.
*/
public class Snapshot implements Comparable<Snapshot> {
private final int versionID;
private final String comment;
private final String userName;
private final String userEmail;
private final Date createdAt;
private final List<SnapshotFile> srcs;
private final List<SnapshotAttachment> atts;
public Snapshot(SnapshotInfo info, SnapshotData data) {
versionID = info.getVersionId();
comment = info.getComment();
WLUser user = info.getUser();
userName = user.getName();
userEmail = user.getEmail();
createdAt = new DateTime(info.getCreatedAt()).toDate();
srcs = data.getSrcs();
atts = data.getAtts();
}
public int getVersionID() {
return versionID;
}
public String getComment() {
return comment;
}
public String getUserName() {
return userName;
}
public String getUserEmail() {
return userEmail;
}
public List<SnapshotFile> getSrcs() {
return srcs;
}
public List<SnapshotAttachment> getAtts() {
return atts;
}
public Date getCreatedAt() {
return createdAt;
}
@Override
public int compareTo(Snapshot snapshot) {
return Integer.compare(versionID, snapshot.versionID);
}
@Override
public String toString() {
return String.valueOf(versionID);
}
}

34
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/exception/FileLimitExceededException.java Normal file
View file

@ -0,0 +1,34 @@
package uk.ac.ic.wlgitbridge.git.exception;
import uk.ac.ic.wlgitbridge.util.Util;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
public class FileLimitExceededException extends GitUserException {
private final long numFiles;
private final long maxFiles;
public FileLimitExceededException(long numFiles, long maxFiles) {
this.numFiles = numFiles;
this.maxFiles = maxFiles;
}
@Override
public String getMessage() {
return "too many files";
}
@Override
public List<String> getDescriptionLines() {
return Arrays.asList(
"repository contains " +
numFiles + " files, which exceeds the limit of " +
maxFiles + " files"
);
}
}

14
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/exception/GitUserException.java Normal file
View file

@ -0,0 +1,14 @@
package uk.ac.ic.wlgitbridge.git.exception;
import java.util.List;
/**
* Created by winston on 20/08/2016.
*/
public abstract class GitUserException extends Exception {
public abstract String getMessage();
public abstract List<String> getDescriptionLines();
}

22
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/exception/InvalidGitRepository.java Normal file
View file

@ -0,0 +1,22 @@
package uk.ac.ic.wlgitbridge.git.exception;
import java.util.Arrays;
import java.util.List;
public class InvalidGitRepository extends GitUserException {
@Override
public String getMessage() {
return "invalid git repo";
}
@Override
public List<String> getDescriptionLines() {
return Arrays.asList(
"Your Git repository contains a reference we cannot resolve.",
"If your project contains a Git submodule,",
"please remove it and try again."
);
}
}

40
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/exception/SizeLimitExceededException.java Normal file
View file

@ -0,0 +1,40 @@
package uk.ac.ic.wlgitbridge.git.exception;
import uk.ac.ic.wlgitbridge.util.Util;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
public class SizeLimitExceededException extends GitUserException {
private final Optional<String> path;
private final long actualSize;
private final long maxSize;
public SizeLimitExceededException(
Optional<String> path, long actualSize, long maxSize) {
this.path = path;
this.actualSize = actualSize;
this.maxSize = maxSize;
}
@Override
public String getMessage() {
return "file too big";
}
@Override
public List<String> getDescriptionLines() {
String filename =
path.isPresent() ? "File '" + path.get() + "' is" : "There's a file";
return Arrays.asList(
filename + " too large to push to "
+ Util.getServiceName() + " via git",
"the recommended maximum file size is 50 MiB"
);
}
}

10
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/exception/SnapshotAPIException.java Normal file
View file

@ -0,0 +1,10 @@
package uk.ac.ic.wlgitbridge.git.exception;
import uk.ac.ic.wlgitbridge.snapshot.base.JSONSource;
/**
* Created by winston on 20/08/2016.
*/
public abstract class SnapshotAPIException
extends GitUserException
implements JSONSource {}

81
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLReceivePackFactory.java Normal file
View file

@ -0,0 +1,81 @@
package uk.ac.ic.wlgitbridge.git.handler;
import com.google.api.client.auth.oauth2.Credential;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.ReceivePack;
import org.eclipse.jgit.transport.resolver.ReceivePackFactory;
import uk.ac.ic.wlgitbridge.bridge.Bridge;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApi;
import uk.ac.ic.wlgitbridge.git.handler.hook.WriteLatexPutHook;
import uk.ac.ic.wlgitbridge.git.servlet.WLGitServlet;
import uk.ac.ic.wlgitbridge.server.Oauth2Filter;
import uk.ac.ic.wlgitbridge.util.Log;
import uk.ac.ic.wlgitbridge.util.Util;
import javax.servlet.http.HttpServletRequest;
import java.util.Optional;
/**
* Created by Winston on 02/11/14.
*/
/**
* One of the "big three" interfaces created by {@link WLGitServlet} to handle
* user Git requests.
*
* This class just puts a {@link WriteLatexPutHook} into the {@link ReceivePack}
* that it returns.
*/
public class WLReceivePackFactory
implements ReceivePackFactory<HttpServletRequest> {
private final RepoStore repoStore;
private final Bridge bridge;
public WLReceivePackFactory(RepoStore repoStore, Bridge bridge) {
this.repoStore = repoStore;
this.bridge = bridge;
}
/**
* Puts a {@link WriteLatexPutHook} into the returned {@link ReceivePack}.
*
* The {@link WriteLatexPutHook} needs our hostname, which we get from the
* original {@link HttpServletRequest}, used to provide a postback URL to
* the {@link SnapshotApi}. We also give it the oauth2 that we injected in
* the {@link Oauth2Filter}, and the {@link Bridge}.
*
* At this point, the repository will have been synced to the latest on
* Overleaf, but it's possible that an update happens on Overleaf while our
* put hook is running. In this case, we fail, and the user tries again,
* triggering another sync, and so on.
* @param httpServletRequest the original request
* @param repository the JGit {@link Repository} provided by
* {@link WLRepositoryResolver}
* @return a correctly hooked {@link ReceivePack}
*/
@Override
public ReceivePack create(
HttpServletRequest httpServletRequest,
Repository repository
) {
Log.info(
"[{}] Creating receive-pack",
repository.getWorkTree().getName()
);
Optional<Credential> oauth2 = Optional.ofNullable(
(Credential) httpServletRequest.getAttribute(
Oauth2Filter.ATTRIBUTE_KEY));
ReceivePack receivePack = new ReceivePack(repository);
String hostname = Util.getPostbackURL();
if (hostname == null) {
hostname = httpServletRequest.getLocalName();
}
receivePack.setPreReceiveHook(
new WriteLatexPutHook(repoStore, bridge, hostname, oauth2)
);
return receivePack;
}
}

126
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLRepositoryResolver.java Normal file
View file

@ -0,0 +1,126 @@
package uk.ac.ic.wlgitbridge.git.handler;
import com.google.api.client.auth.oauth2.Credential;
import org.eclipse.jgit.errors.RepositoryNotFoundException;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.ServiceMayNotContinueException;
import org.eclipse.jgit.transport.resolver.RepositoryResolver;
import org.eclipse.jgit.transport.resolver.ServiceNotAuthorizedException;
import uk.ac.ic.wlgitbridge.bridge.Bridge;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.git.handler.hook.WriteLatexPutHook;
import uk.ac.ic.wlgitbridge.git.servlet.WLGitServlet;
import uk.ac.ic.wlgitbridge.server.GitBridgeServer;
import uk.ac.ic.wlgitbridge.server.Oauth2Filter;
import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
import uk.ac.ic.wlgitbridge.util.Log;
import uk.ac.ic.wlgitbridge.util.Util;
import javax.servlet.http.HttpServletRequest;
import java.io.IOException;
import java.util.Arrays;
import java.util.Optional;
/**
* Created by Winston on 02/11/14.
*/
/**
* One of the "big three" interfaces created by {@link WLGitServlet} to handle
* user Git requests.
*
* This class is used by all Git requests to resolve a project name to a
* JGit {@link Repository}, or fail by throwing an exception.
*
* It has a single method, {@link #open(HttpServletRequest, String)}, which
* calls into the {@link Bridge} to synchronise the project with Overleaf, i.e.
* bringing it onto disk and applying commits to it until it is up-to-date with
* Overleaf.
*/
public class WLRepositoryResolver
implements RepositoryResolver<HttpServletRequest> {
private final Bridge bridge;
public WLRepositoryResolver(Bridge bridge) {
this.bridge = bridge;
}
/**
* Calls into the Bridge to resolve a project name to a JGit
* {@link Repository}, or throw an exception.
*
* On success, the repository will have been brought onto disk and updated
* to the latest (synced).
*
* In the case of clones and fetches, upload packs are created from the
* returned JGit {@link Repository} by the {@link WLUploadPackFactory}.
*
* The project lock is acquired for this process so it can't be swapped out.
*
* However, it can still be swapped out between this and a Git push. The
* push would fail due to the project changed on Overleaf between the sync
* and the actual push to Overleaf (performed by the
* {@link WLReceivePackFactory} and {@link WriteLatexPutHook}. In this case,
* the user will have to try again (which prompts another update, etc. until
* this no longer happens).
* @param httpServletRequest The HttpServletRequest as required by the
* interface. We injected the oauth2 creds into it with
* {@link Oauth2Filter}, which was set up by the {@link GitBridgeServer}.
* @param name The name of the project
* @return the JGit {@link Repository}.
* @throws RepositoryNotFoundException If the project does not exist
* @throws ServiceNotAuthorizedException If the user did not auth when
* required to
* @throws ServiceMayNotContinueException If any other general user
* exception occurs that must be propogated back to the user, e.g.
* internal errors (IOException, etc), too large file, and so on.
*/
@Override
public Repository open(
HttpServletRequest httpServletRequest,
String name
) throws RepositoryNotFoundException,
ServiceNotAuthorizedException,
ServiceMayNotContinueException {
Log.info("[{}] Request to open git repo", name);
Optional<Credential> oauth2 = Optional.ofNullable(
(Credential) httpServletRequest.getAttribute(
Oauth2Filter.ATTRIBUTE_KEY));
String projName = Util.removeAllSuffixes(name, "/", ".git");
try {
return bridge.getUpdatedRepo(oauth2, projName).getJGitRepository();
} catch (RepositoryNotFoundException e) {
Log.info("Repository not found: " + name);
throw e;
/*
} catch (ServiceNotAuthorizedException e) {
cannot occur
} catch (ServiceNotEnabledException e) {
cannot occur
*/
} catch (ServiceMayNotContinueException e) {
/* Such as FailedConnectionException */
throw e;
} catch (RuntimeException e) {
Log.warn(
"Runtime exception when trying to open repo: " + projName,
e
);
throw new ServiceMayNotContinueException(e);
} catch (ForbiddenException e) {
throw new ServiceNotAuthorizedException();
} catch (GitUserException e) {
throw new ServiceMayNotContinueException(
e.getMessage() + "\n" +
String.join("\n", e.getDescriptionLines()),
e);
} catch (IOException e) {
Log.warn(
"IOException when trying to open repo: " + projName,
e
);
throw new ServiceMayNotContinueException("Internal server error.");
}
}
}

44
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLUploadPackFactory.java Normal file
View file

@ -0,0 +1,44 @@
package uk.ac.ic.wlgitbridge.git.handler;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.UploadPack;
import org.eclipse.jgit.transport.resolver.UploadPackFactory;
import uk.ac.ic.wlgitbridge.git.servlet.WLGitServlet;
import uk.ac.ic.wlgitbridge.util.Log;
import javax.servlet.http.HttpServletRequest;
/**
* Created by Winston on 02/11/14.
*/
/**
* One of the "big three" interfaces created by {@link WLGitServlet} to handle
* user Git requests.
*
* The actual class doesn't do much, and most of the work is done when the
* project name is being resolved by the {@link WLRepositoryResolver}.
*/
public class WLUploadPackFactory
implements UploadPackFactory<HttpServletRequest> {
/**
* This does nothing special. Synchronising the project with Overleaf will
* have been performed by {@link WLRepositoryResolver}.
* @param __ Not used, required by the {@link UploadPackFactory} interface
* @param repository The JGit repository provided by the
* {@link WLRepositoryResolver}
* @return the {@link UploadPack}, used by JGit to serve the request
*/
@Override
public UploadPack create(
HttpServletRequest __,
Repository repository
) {
Log.info(
"[{}] Creating upload-pack",
repository.getWorkTree().getName()
);
return new UploadPack(repository);
}
}

178
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/hook/WriteLatexPutHook.java Normal file
View file

@ -0,0 +1,178 @@
package uk.ac.ic.wlgitbridge.git.handler.hook;
import com.google.api.client.auth.oauth2.Credential;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.PreReceiveHook;
import org.eclipse.jgit.transport.ReceiveCommand;
import org.eclipse.jgit.transport.ReceiveCommand.Result;
import org.eclipse.jgit.transport.ReceivePack;
import uk.ac.ic.wlgitbridge.bridge.Bridge;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.git.handler.WLReceivePackFactory;
import uk.ac.ic.wlgitbridge.git.handler.hook.exception.ForcedPushException;
import uk.ac.ic.wlgitbridge.git.handler.hook.exception.WrongBranchException;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.InternalErrorException;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.OutOfDateException;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
import uk.ac.ic.wlgitbridge.util.Log;
import java.io.IOException;
import java.util.Collection;
import java.util.Iterator;
import java.util.Optional;
/**
* Created by Winston on 03/11/14.
*/
/**
* Created by {@link WLReceivePackFactory} to update the {@link Bridge} for a
* user's Git push request, or fail with an error. The hook is able to approve
* or reject a request.
*/
public class WriteLatexPutHook implements PreReceiveHook {
private final RepoStore repoStore;
private final Bridge bridge;
private final String hostname;
private final Optional<Credential> oauth2;
/**
* The constructor to use, which provides the hook with the {@link Bridge},
* the hostname (used to construct a URL to give to Overleaf to postback),
* and the oauth2 (used to authenticate with the Snapshot API).
* @param repoStore
* @param bridge the {@link Bridge}
* @param hostname the hostname used for postback from the Snapshot API
* @param oauth2 used to authenticate with the snapshot API, or null
*/
public WriteLatexPutHook(
RepoStore repoStore,
Bridge bridge,
String hostname,
Optional<Credential> oauth2
) {
this.repoStore = repoStore;
this.bridge = bridge;
this.hostname = hostname;
this.oauth2 = oauth2;
}
@Override
public void onPreReceive(
ReceivePack receivePack,
Collection<ReceiveCommand> receiveCommands
) {
Log.debug("-> Handling {} commands in {}", receiveCommands.size(), receivePack.getRepository().getDirectory().getAbsolutePath());
for (ReceiveCommand receiveCommand : receiveCommands) {
try {
handleReceiveCommand(
oauth2,
receivePack.getRepository(),
receiveCommand
);
} catch (IOException e) {
Log.debug("IOException on pre receive: {}", e.getMessage());
receivePack.sendError(e.getMessage());
receiveCommand.setResult(
Result.REJECTED_OTHER_REASON,
e.getMessage()
);
} catch (OutOfDateException e) {
Log.debug("OutOfDateException on pre receive: {}", e.getMessage());
receiveCommand.setResult(Result.REJECTED_NONFASTFORWARD);
} catch (GitUserException e) {
Log.debug("GitUserException on pre receive: {}", e.getMessage());
handleSnapshotPostException(receivePack, receiveCommand, e);
} catch (Throwable t) {
Log.warn("Throwable on pre receive: {}", t.getMessage());
handleSnapshotPostException(
receivePack,
receiveCommand,
new InternalErrorException()
);
}
}
Log.debug("-> Handled {} commands in {}", receiveCommands.size(), receivePack.getRepository().getDirectory().getAbsolutePath());
}
private void handleSnapshotPostException(
ReceivePack receivePack,
ReceiveCommand receiveCommand,
GitUserException e
) {
String message = e.getMessage();
receivePack.sendError(message);
StringBuilder msg = new StringBuilder();
for (
Iterator<String> it = e.getDescriptionLines().iterator();
it.hasNext();
) {
String line = it.next();
msg.append("hint: ");
msg.append(line);
if (it.hasNext()) {
msg.append('\n');
}
}
receivePack.sendMessage("");
receivePack.sendMessage(msg.toString());
receiveCommand.setResult(Result.REJECTED_OTHER_REASON, message);
}
private void handleReceiveCommand(
Optional<Credential> oauth2,
Repository repository,
ReceiveCommand receiveCommand
) throws IOException, GitUserException {
checkBranch(receiveCommand);
checkForcedPush(receiveCommand);
bridge.push(
oauth2,
repository.getWorkTree().getName(),
getPushedDirectoryContents(repository,
receiveCommand),
getOldDirectoryContents(repository),
hostname
);
}
private void checkBranch(
ReceiveCommand receiveCommand
) throws WrongBranchException {
if (!receiveCommand.getRefName().equals("refs/heads/master")) {
throw new WrongBranchException();
}
}
private void checkForcedPush(
ReceiveCommand receiveCommand
) throws ForcedPushException {
if (
receiveCommand.getType()
== ReceiveCommand.Type.UPDATE_NONFASTFORWARD
) {
throw new ForcedPushException();
}
}
private RawDirectory getPushedDirectoryContents(
Repository repository,
ReceiveCommand receiveCommand
) throws IOException, GitUserException {
return repoStore
.useJGitRepo(repository, receiveCommand.getNewId())
.getDirectory();
}
private RawDirectory getOldDirectoryContents(
Repository repository
) throws IOException, GitUserException {
return repoStore
.useJGitRepo(repository, repository.resolve("HEAD"))
.getDirectory();
}
}

37
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/hook/exception/ForcedPushException.java Normal file
View file

@ -0,0 +1,37 @@
package uk.ac.ic.wlgitbridge.git.handler.hook.exception;
import com.google.gson.JsonElement;
import uk.ac.ic.wlgitbridge.util.Util;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
import java.util.Arrays;
import java.util.List;
/**
* Created by Winston on 16/11/14.
*/
public class ForcedPushException extends SnapshotPostException {
private static final String[] DESCRIPTION_LINES = {
"You can't git push --force to a "
+ Util.getServiceName()
+ " project.",
"Try to put your changes on top of the current head.",
"If everything else fails, delete and reclone your repository, "
+ "make your changes, then push again."
};
@Override
public String getMessage() {
return "forced push prohibited";
}
@Override
public List<String> getDescriptionLines() {
return Arrays.asList(DESCRIPTION_LINES);
}
@Override
public void fromJSON(JsonElement json) {}
}

Some files were not shown because too many files have changed in this diff Show more