github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2025-04-11 11:34:05 +00:00
Code Issues Projects Releases Packages Wiki Activity

Large refactor of parts into distinct components / interfaces

Browse source
This commit is contained in:
Winston Li 2016-08-20 19:27:23 +01:00 • committed by Michael Mazour
parent 5b810b64ba
commit 692b979098
25 changed files with 705 additions and 371 deletions

6
services/git-bridge/pom.xml
View file

@ -134,5 +134,11 @@
<version>3.10.4</version>
<scope>test</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.mockito/mockito-core -->
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>1.10.19</version>
</dependency>
</dependencies>
</project>

206
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java
View file

@ -2,15 +2,27 @@ package uk.ac.ic.wlgitbridge.bridge;
import com.google.api.client.auth.oauth2.Credential;
import org.eclipse.jgit.transport.ServiceMayNotContinueException;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.bridge.resource.ResourceCache;
import uk.ac.ic.wlgitbridge.bridge.resource.UrlResourceCache;
import uk.ac.ic.wlgitbridge.bridge.snapshot.NetSnapshotAPI;
import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotAPI;
import uk.ac.ic.wlgitbridge.bridge.swap.SwapJob;
import uk.ac.ic.wlgitbridge.bridge.swap.SwapJobImpl;
import uk.ac.ic.wlgitbridge.bridge.swap.SwapStore;
import uk.ac.ic.wlgitbridge.data.CandidateSnapshot;
import uk.ac.ic.wlgitbridge.data.ProjectLock;
import uk.ac.ic.wlgitbridge.data.ShutdownHook;
import uk.ac.ic.wlgitbridge.data.ProjectLockImpl;
import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
import uk.ac.ic.wlgitbridge.data.model.DataStore;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import uk.ac.ic.wlgitbridge.data.model.Snapshot;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocRequest;
import uk.ac.ic.wlgitbridge.snapshot.getdoc.exception.InvalidProjectException;
import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotAttachment;
import uk.ac.ic.wlgitbridge.snapshot.push.PostbackManager;
import uk.ac.ic.wlgitbridge.snapshot.push.PushRequest;
import uk.ac.ic.wlgitbridge.snapshot.push.PushResult;
@ -18,29 +30,91 @@ import uk.ac.ic.wlgitbridge.snapshot.push.exception.*;
import uk.ac.ic.wlgitbridge.util.Log;
import java.io.IOException;
import java.util.*;
/**
* Created by Winston on 16/11/14.
*/
public class Bridge {
private final DataStore dataStore;
private final PostbackManager postbackManager;
private final ProjectLock mainProjectLock;
private final ProjectLock lock;
public Bridge(String rootGitDirectoryPath) {
dataStore = new DataStore(rootGitDirectoryPath);
postbackManager = new PostbackManager();
mainProjectLock = new ProjectLock();
Runtime.getRuntime().addShutdownHook(new ShutdownHook(mainProjectLock));
private final RepoStore repoStore;
private final DBStore dbStore;
private final SwapStore swapStore;
private final SnapshotAPI snapshotAPI;
private final ResourceCache resourceCache;
private final SwapJob swapJob;
private final PostbackManager postbackManager;
public static Bridge make(
RepoStore repoStore,
DBStore dbStore,
SwapStore swapStore
) {
ProjectLock lock = new ProjectLockImpl((int threads) ->
Log.info("Waiting for " + threads + " projects...")
);
return new Bridge(
lock,
repoStore,
dbStore,
swapStore,
new NetSnapshotAPI(),
new UrlResourceCache(dbStore),
new SwapJobImpl(
lock,
repoStore,
dbStore,
swapStore
)
);
}
Bridge(
ProjectLock lock,
RepoStore repoStore,
DBStore dbStore,
SwapStore swapStore,
SnapshotAPI snapshotAPI,
ResourceCache resourceCache,
SwapJob swapJob
) {
this.lock = lock;
this.repoStore = repoStore;
this.dbStore = dbStore;
this.swapStore = swapStore;
this.snapshotAPI = snapshotAPI;
this.resourceCache = resourceCache;
this.swapJob = swapJob;
postbackManager = new PostbackManager();
Runtime.getRuntime().addShutdownHook(new Thread(this::doShutdown));
repoStore.purgeNonexistentProjects(dbStore.getProjectNames());
}
void doShutdown() {
Log.info("Shutdown received.");
Log.info("Stopping SwapJob");
swapJob.stop();
Log.info("Waiting for projects");
lock.lockAll();
Log.info("Bye");
}
public void startSwapJob(int intervalMillis) {
swapJob.start(intervalMillis);
}
/* TODO: Remove these when WLBridged is moved into RepoStore */
public void lockForProject(String projectName) {
mainProjectLock.lockForProject(projectName);
lock.lockForProject(projectName);
}
public void unlockForProject(String projectName) {
mainProjectLock.unlockForProject(projectName);
lock.unlockForProject(projectName);
}
public boolean repositoryExists(Credential oauth2, String projectName)
@ -64,7 +138,7 @@ public class Bridge {
) throws IOException,
GitUserException {
Log.info("[{}] Fetching", repo.getProjectName());
dataStore.updateProjectWithName(oauth2, repo);
updateProjectWithName(oauth2, repo);
}
public void
@ -74,7 +148,7 @@ public class Bridge {
RawDirectory oldDirectoryContents,
String hostname)
throws SnapshotPostException, IOException, ForbiddenException {
mainProjectLock.lockForProject(projectName);
lock.lockForProject(projectName);
CandidateSnapshot candidate = null;
try {
Log.info("[{}] Pushing", projectName);
@ -85,7 +159,7 @@ public class Bridge {
postbackKey
);
candidate =
dataStore.createCandidateSnapshot(
createCandidateSnapshot(
projectName,
directoryContents,
oldDirectoryContents
@ -115,7 +189,7 @@ public class Bridge {
projectName,
versionID
);
dataStore.approveSnapshot(versionID, candidate);
approveSnapshot(versionID, candidate);
Log.info(
"[{}] Approved version ID: {}",
projectName,
@ -152,7 +226,7 @@ public class Bridge {
projectName
);
}
mainProjectLock.unlockForProject(projectName);
lock.unlockForProject(projectName);
}
}
@ -190,4 +264,100 @@ public class Bridge {
);
}
/* PRIVATE */
private void updateProjectWithName(
Credential oauth2,
ProjectRepo repo
) throws IOException, GitUserException {
String projectName = repo.getProjectName();
Deque<Snapshot> snapshots =
snapshotAPI.getSnapshotsForProjectAfterVersion(
oauth2,
projectName,
dbStore.getLatestVersionForProject(projectName)
);
makeCommitsFromSnapshots(repo, snapshots);
if (!snapshots.isEmpty()) {
dbStore.setLatestVersionForProject(
projectName,
snapshots.getLast().getVersionID()
);
}
}
private void makeCommitsFromSnapshots(ProjectRepo repo,
Collection<Snapshot> snapshots)
throws IOException, GitUserException {
String name = repo.getProjectName();
for (Snapshot snapshot : snapshots) {
Map<String, RawFile> fileTable = repo.getFiles();
List<RawFile> files = new LinkedList<>();
files.addAll(snapshot.getSrcs());
Map<String, byte[]> fetchedUrls = new HashMap<>();
for (SnapshotAttachment snapshotAttachment : snapshot.getAtts()) {
files.add(
resourceCache.get(
name,
snapshotAttachment.getUrl(),
snapshotAttachment.getPath(),
fileTable,
fetchedUrls
)
);
}
Log.info(
"[{}] Committing version ID: {}",
name,
snapshot.getVersionID()
);
Collection<String> missingFiles = repo.commitAndGetMissing(
new GitDirectoryContents(
files,
repoStore.getRootDirectory(),
name,
snapshot
)
);
dbStore.deleteFilesForProject(
name,
missingFiles.toArray(new String[missingFiles.size()])
);
}
}
private CandidateSnapshot createCandidateSnapshot(
String projectName,
RawDirectory directoryContents,
RawDirectory oldDirectoryContents
) throws IOException {
CandidateSnapshot candidateSnapshot = new CandidateSnapshot(
projectName,
dbStore.getLatestVersionForProject(projectName),
directoryContents,
oldDirectoryContents
);
candidateSnapshot.writeServletFiles(repoStore.getRootDirectory());
return candidateSnapshot;
}
private void approveSnapshot(
int versionID,
CandidateSnapshot candidateSnapshot
) {
List<String> deleted = candidateSnapshot.getDeleted();
dbStore.setLatestVersionForProject(
candidateSnapshot.getProjectName(),
versionID
);
dbStore.deleteFilesForProject(
candidateSnapshot.getProjectName(),
deleted.toArray(new String[deleted.size()])
);
}
}

7
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/WLBridgedProject.java
View file

@ -40,7 +40,12 @@ public class WLBridgedProject {
}
}
private void updateRepositoryFromSnapshots(Credential oauth2, Repository repository) throws RepositoryNotFoundException, ServiceMayNotContinueException, GitUserException {
private void updateRepositoryFromSnapshots(
Credential oauth2,
Repository repository
) throws RepositoryNotFoundException,
ServiceMayNotContinueException,
GitUserException {
try {
bridgeAPI.getWritableRepositories(
oauth2,

22
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/DBStore.java Normal file
View file

@ -0,0 +1,22 @@
package uk.ac.ic.wlgitbridge.bridge.db;
import java.util.List;
/**
* Created by winston on 20/08/2016.
*/
public interface DBStore {
List<String> getProjectNames();
void setLatestVersionForProject(String project, int versionID);
int getLatestVersionForProject(String project);
void addURLIndexForProject(String projectName, String url, String path);
void deleteFilesForProject(String project, String... files);
String getPathForURLInProject(String projectName, String url);
}

10
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/db/SqlitePersistentStore.java → services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/db/SqliteDBStore.java
View file

@ -1,4 +1,4 @@
package uk.ac.ic.wlgitbridge.data.model.db;
package uk.ac.ic.wlgitbridge.bridge.db;
import uk.ac.ic.wlgitbridge.data.model.db.sql.SQLiteWLDatabase;
import uk.ac.ic.wlgitbridge.util.Log;
@ -9,15 +9,15 @@ import java.util.Arrays;
import java.util.List;
/**
* Created by Winston on 19/11/14.
* Created by winston on 20/08/2016.
*/
public class SqlitePersistentStore implements PersistentStore {
public class SqliteDBStore implements DBStore {
private final SQLiteWLDatabase database;
public SqlitePersistentStore(File rootGitDirectory) {
public SqliteDBStore(File rootDirectory) {
try {
database = new SQLiteWLDatabase(rootGitDirectory);
database = new SQLiteWLDatabase(rootDirectory);
} catch (SQLException e) {
throw new RuntimeException(e);
} catch (ClassNotFoundException e) {

12
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/lock/ProjectLock.java Normal file
View file

@ -0,0 +1,12 @@
package uk.ac.ic.wlgitbridge.bridge.lock;
/**
* Created by winston on 20/08/2016.
*/
public interface ProjectLock {
void lockAll();
void lockForProject(String projectName);
void unlockForProject(String projectName);
}

52
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSRepoStore.java Normal file
View file

@ -0,0 +1,52 @@
package uk.ac.ic.wlgitbridge.bridge.repo;
import uk.ac.ic.wlgitbridge.util.Util;
import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
/**
* Created by winston on 20/08/2016.
*/
public class FSRepoStore implements RepoStore {
private final String repoStorePath;
private final File rootDirectory;
public FSRepoStore(String repoStorePath) {
this.repoStorePath = repoStorePath;
rootDirectory = initRootGitDirectory(repoStorePath);
}
@Override
public String getRepoStorePath() {
return repoStorePath;
}
@Override
public File getRootDirectory() {
return rootDirectory;
}
@Override
public void purgeNonexistentProjects(
Collection<String> existingProjectNames
) {
List<String> excludedFromDeletion =
new ArrayList<>(existingProjectNames);
excludedFromDeletion.add(".wlgb");
Util.deleteInDirectoryApartFrom(
rootDirectory,
excludedFromDeletion.toArray(new String[] {})
);
}
private File initRootGitDirectory(String rootGitDirectoryPath) {
File rootGitDirectory = new File(rootGitDirectoryPath);
rootGitDirectory.mkdirs();
return rootGitDirectory;
}
}

22
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java Normal file
View file

@ -0,0 +1,22 @@
package uk.ac.ic.wlgitbridge.bridge.repo;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import java.io.File;
import java.util.Collection;
/**
* Created by winston on 20/08/2016.
*/
public interface RepoStore {
String getRepoStorePath();
File getRootDirectory();
void purgeNonexistentProjects(
Collection<String> existingProjectNames
);
}

13
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/resource/ResourceCache.java Normal file
View file

@ -0,0 +1,13 @@
package uk.ac.ic.wlgitbridge.bridge.resource;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import java.io.IOException;
import java.util.Map;
/**
* Created by winston on 20/08/2016.
*/
public interface ResourceCache {
RawFile get(String projectName, String url, String newPath, Map<String, RawFile> fileTable, Map<String, byte[]> fetchedUrls) throws IOException;
}

107
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/resource/UrlResourceCache.java Normal file
View file

@ -0,0 +1,107 @@
package uk.ac.ic.wlgitbridge.bridge.resource;
import com.ning.http.client.AsyncCompletionHandler;
import com.ning.http.client.HttpResponseBodyPart;
import com.ning.http.client.Response;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import uk.ac.ic.wlgitbridge.data.filestore.RepositoryFile;
import uk.ac.ic.wlgitbridge.snapshot.base.Request;
import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
import uk.ac.ic.wlgitbridge.util.Log;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Map;
import java.util.concurrent.ExecutionException;
/**
* Created by winston on 20/08/2016.
*/
public class UrlResourceCache implements ResourceCache {
private final DBStore dbStore;
public UrlResourceCache(DBStore dbStore) {
this.dbStore = dbStore;
}
@Override
public RawFile get(String projectName, String url, String newPath, Map<String, RawFile> fileTable, Map<String, byte[]> fetchedUrls) throws IOException {
String path = dbStore.getPathForURLInProject(projectName, url);
byte[] contents;
if (path == null) {
path = newPath;
contents = fetch(projectName, url, path);
fetchedUrls.put(url, contents);
} else {
Log.info("Found (" + projectName + "): " + url);
Log.info("At (" + projectName + "): " + path);
contents = fetchedUrls.get(url);
if (contents == null) {
RawFile rawFile = fileTable.get(path);
if (rawFile == null) {
Log.warn(
"File " + path + " was not in the current commit, or the git tree, yet path was not null. " +
"File url is: " + url
);
contents = fetch(projectName, url, path);
} else {
contents = rawFile.getContents();
}
}
}
return new RepositoryFile(newPath, contents);
}
private byte[] fetch(String projectName, final String url, String path) throws FailedConnectionException {
byte[] contents;
Log.info("GET -> " + url);
try {
contents = Request.httpClient.prepareGet(url).execute(new AsyncCompletionHandler<byte[]>() {
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
@Override
public STATE onBodyPartReceived(HttpResponseBodyPart bodyPart) throws Exception {
bytes.write(bodyPart.getBodyPartBytes());
return STATE.CONTINUE;
}
@Override
public byte[] onCompleted(Response response) throws Exception {
byte[] data = bytes.toByteArray();
bytes.close();
Log.info(response.getStatusCode() + " " + response.getStatusText() + " (" + data.length + "B) -> " + url);
return data;
}
}).get();
} catch (InterruptedException e) {
Log.warn(
"Interrupted when fetching project: " +
projectName +
", url: " +
url +
", path: " +
path,
e
);
throw new FailedConnectionException();
} catch (ExecutionException e) {
Log.warn(
"ExecutionException when fetching project: " +
projectName +
", url: " +
url +
", path: " +
path,
e
);
throw new FailedConnectionException();
}
dbStore.addURLIndexForProject(projectName, url, path);
return contents;
}
}

79
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/snapshot/NetSnapshotAPI.java Normal file
View file

@ -0,0 +1,79 @@
package uk.ac.ic.wlgitbridge.bridge.snapshot;
import com.google.api.client.auth.oauth2.Credential;
import uk.ac.ic.wlgitbridge.data.model.Snapshot;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocRequest;
import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocResult;
import uk.ac.ic.wlgitbridge.snapshot.getforversion.GetForVersionRequest;
import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotData;
import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.GetSavedVersRequest;
import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.SnapshotInfo;
import java.util.*;
/**
* Created by winston on 20/08/2016.
*/
public class NetSnapshotAPI implements SnapshotAPI {
@Override
public Deque<Snapshot> getSnapshotsForProjectAfterVersion(Credential oauth2, String projectName, int version) throws FailedConnectionException, GitUserException {
List<SnapshotInfo> snapshotInfos = getSnapshotInfosAfterVersion(oauth2, projectName, version);
List<SnapshotData> snapshotDatas = getMatchingSnapshotData(oauth2, projectName, snapshotInfos);
LinkedList<Snapshot> snapshots = combine(snapshotInfos, snapshotDatas);
return snapshots;
}
private List<SnapshotInfo> getSnapshotInfosAfterVersion(Credential oauth2, String projectName, int version) throws FailedConnectionException, GitUserException {
SortedSet<SnapshotInfo> versions = new TreeSet<SnapshotInfo>();
GetDocRequest getDoc = new GetDocRequest(oauth2, projectName);
GetSavedVersRequest getSavedVers = new GetSavedVersRequest(oauth2, projectName);
getDoc.request();
getSavedVers.request();
GetDocResult latestDoc = getDoc.getResult();
int latest = latestDoc.getVersionID();
if (latest > version) {
for (SnapshotInfo snapshotInfo : getSavedVers.getResult().getSavedVers()) {
if (snapshotInfo.getVersionId() > version) {
versions.add(snapshotInfo);
}
}
versions.add(new SnapshotInfo(latest, latestDoc.getCreatedAt(), latestDoc.getName(), latestDoc.getEmail()));
}
return new LinkedList<SnapshotInfo>(versions);
}
private List<SnapshotData> getMatchingSnapshotData(Credential oauth2, String projectName, List<SnapshotInfo> snapshotInfos) throws FailedConnectionException, ForbiddenException {
List<GetForVersionRequest> firedRequests = fireDataRequests(oauth2, projectName, snapshotInfos);
List<SnapshotData> snapshotDataList = new LinkedList<SnapshotData>();
for (GetForVersionRequest fired : firedRequests) {
snapshotDataList.add(fired.getResult().getSnapshotData());
}
return snapshotDataList;
}
private List<GetForVersionRequest> fireDataRequests(Credential oauth2, String projectName, List<SnapshotInfo> snapshotInfos) {
List<GetForVersionRequest> requests = new LinkedList<GetForVersionRequest>();
for (SnapshotInfo snapshotInfo : snapshotInfos) {
GetForVersionRequest request = new GetForVersionRequest(oauth2, projectName, snapshotInfo.getVersionId());
requests.add(request);
request.request();
}
return requests;
}
private LinkedList<Snapshot> combine(List<SnapshotInfo> snapshotInfos, List<SnapshotData> snapshotDatas) {
LinkedList<Snapshot> snapshots = new LinkedList<Snapshot>();
Iterator<SnapshotInfo> infos = snapshotInfos.iterator();
Iterator<SnapshotData> datas = snapshotDatas.iterator();
while (infos.hasNext()) {
snapshots.add(new Snapshot(infos.next(), datas.next()));
}
return snapshots;
}
}

21
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/snapshot/SnapshotAPI.java Normal file
View file

@ -0,0 +1,21 @@
package uk.ac.ic.wlgitbridge.bridge.snapshot;
import com.google.api.client.auth.oauth2.Credential;
import uk.ac.ic.wlgitbridge.data.model.Snapshot;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
import java.util.Deque;
/**
* Created by winston on 20/08/2016.
*/
public interface SnapshotAPI {
Deque<Snapshot> getSnapshotsForProjectAfterVersion(
Credential oauth2,
String projectName,
int latestVersion
) throws FailedConnectionException, GitUserException;
}

12
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJob.java Normal file
View file

@ -0,0 +1,12 @@
package uk.ac.ic.wlgitbridge.bridge.swap;
/**
* Created by winston on 20/08/2016.
*/
public interface SwapJob {
void start(int intervalMillis);
void stop();
}

53
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapJobImpl.java Normal file
View file

@ -0,0 +1,53 @@
package uk.ac.ic.wlgitbridge.bridge.swap;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.util.Util;
import java.util.Timer;
/**
* Created by winston on 20/08/2016.
*/
public class SwapJobImpl implements SwapJob {
private final ProjectLock lock;
private final RepoStore repoStore;
private final SwapStore swapStore;
private final DBStore dbStore;
private final Timer timer;
public SwapJobImpl(
ProjectLock lock,
RepoStore repoStore,
DBStore dbStore, SwapStore swapStore
) {
this.lock = lock;
this.repoStore = repoStore;
this.swapStore = swapStore;
this.dbStore = dbStore;
timer = new Timer();
}
@Override
public void start(int intervalMillis) {
timer.scheduleAtFixedRate(
Util.makeTimerTask(this::doSwap),
0,
intervalMillis
);
}
@Override
public void stop() {
timer.cancel();
}
private void doSwap() {
throw new UnsupportedOperationException();
}
}

7
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/swap/SwapStore.java Normal file
View file

@ -0,0 +1,7 @@
package uk.ac.ic.wlgitbridge.bridge.swap;
/**
* Created by winston on 20/08/2016.
*/
public interface SwapStore {
}

11
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/ProjectLock.java → services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/ProjectLockImpl.java
View file

@ -1,5 +1,7 @@
package uk.ac.ic.wlgitbridge.data;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.locks.Lock;
@ -9,7 +11,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
/**
* Created by Winston on 20/11/14.
*/
public class ProjectLock {
public class ProjectLockImpl implements ProjectLock {
private final Map<String, Lock> projectLocks;
private final ReentrantReadWriteLock rwlock;
@ -18,7 +20,7 @@ public class ProjectLock {
private LockAllWaiter waiter;
private boolean waiting;
public ProjectLock() {
public ProjectLockImpl() {
projectLocks = new HashMap<String, Lock>();
rwlock = new ReentrantReadWriteLock();
rlock = rwlock.readLock();
@ -26,6 +28,11 @@ public class ProjectLock {
waiting = false;
}
public ProjectLockImpl(LockAllWaiter waiter) {
this();
setWaiter(waiter);
}
public void lockForProject(String projectName) {
getLockForProjectName(projectName).lock();
rlock.lock();

30
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/ShutdownHook.java
View file

@ -1,30 +0,0 @@
package uk.ac.ic.wlgitbridge.data;
import uk.ac.ic.wlgitbridge.util.Log;
/**
* Created by Winston on 21/02/15.
*/
public class ShutdownHook extends Thread implements LockAllWaiter {
private final ProjectLock projectLock;
public ShutdownHook(ProjectLock projectLock) {
this.projectLock = projectLock;
projectLock.setWaiter(this);
}
@Override
public void run() {
Log.info("Shutdown received.");
projectLock.lockAll();
Log.info("No projects to wait for.");
Log.info("Bye");
}
@Override
public void threadsRemaining(int threads) {
Log.info("Waiting for " + threads + " projects...");
}
}

68
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/SnapshotFetcher.java
View file

@ -1,78 +1,10 @@
package uk.ac.ic.wlgitbridge.data;
import com.google.api.client.auth.oauth2.Credential;
import uk.ac.ic.wlgitbridge.data.model.Snapshot;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocRequest;
import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocResult;
import uk.ac.ic.wlgitbridge.snapshot.getforversion.GetForVersionRequest;
import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotData;
import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.GetSavedVersRequest;
import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.SnapshotInfo;
import java.util.*;
/**
* Created by Winston on 07/11/14.
*/
public class SnapshotFetcher {
public LinkedList<Snapshot> getSnapshotsForProjectAfterVersion(Credential oauth2, String projectName, int version) throws FailedConnectionException, GitUserException {
List<SnapshotInfo> snapshotInfos = getSnapshotInfosAfterVersion(oauth2, projectName, version);
List<SnapshotData> snapshotDatas = getMatchingSnapshotData(oauth2, projectName, snapshotInfos);
LinkedList<Snapshot> snapshots = combine(snapshotInfos, snapshotDatas);
return snapshots;
}
private List<SnapshotInfo> getSnapshotInfosAfterVersion(Credential oauth2, String projectName, int version) throws FailedConnectionException, GitUserException {
SortedSet<SnapshotInfo> versions = new TreeSet<SnapshotInfo>();
GetDocRequest getDoc = new GetDocRequest(oauth2, projectName);
GetSavedVersRequest getSavedVers = new GetSavedVersRequest(oauth2, projectName);
getDoc.request();
getSavedVers.request();
GetDocResult latestDoc = getDoc.getResult();
int latest = latestDoc.getVersionID();
if (latest > version) {
for (SnapshotInfo snapshotInfo : getSavedVers.getResult().getSavedVers()) {
if (snapshotInfo.getVersionId() > version) {
versions.add(snapshotInfo);
}
}
versions.add(new SnapshotInfo(latest, latestDoc.getCreatedAt(), latestDoc.getName(), latestDoc.getEmail()));
}
return new LinkedList<SnapshotInfo>(versions);
}
private List<SnapshotData> getMatchingSnapshotData(Credential oauth2, String projectName, List<SnapshotInfo> snapshotInfos) throws FailedConnectionException, ForbiddenException {
List<GetForVersionRequest> firedRequests = fireDataRequests(oauth2, projectName, snapshotInfos);
List<SnapshotData> snapshotDataList = new LinkedList<SnapshotData>();
for (GetForVersionRequest fired : firedRequests) {
snapshotDataList.add(fired.getResult().getSnapshotData());
}
return snapshotDataList;
}
private List<GetForVersionRequest> fireDataRequests(Credential oauth2, String projectName, List<SnapshotInfo> snapshotInfos) {
List<GetForVersionRequest> requests = new LinkedList<GetForVersionRequest>();
for (SnapshotInfo snapshotInfo : snapshotInfos) {
GetForVersionRequest request = new GetForVersionRequest(oauth2, projectName, snapshotInfo.getVersionId());
requests.add(request);
request.request();
}
return requests;
}
private LinkedList<Snapshot> combine(List<SnapshotInfo> snapshotInfos, List<SnapshotData> snapshotDatas) {
LinkedList<Snapshot> snapshots = new LinkedList<Snapshot>();
Iterator<SnapshotInfo> infos = snapshotInfos.iterator();
Iterator<SnapshotData> datas = snapshotDatas.iterator();
while (infos.hasNext()) {
snapshots.add(new Snapshot(infos.next(), datas.next()));
}
return snapshots;
}
}

143
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/DataStore.java
View file

@ -1,143 +0,0 @@
package uk.ac.ic.wlgitbridge.data.model;
import com.google.api.client.auth.oauth2.Credential;
import uk.ac.ic.wlgitbridge.bridge.ProjectRepo;
import uk.ac.ic.wlgitbridge.data.CandidateSnapshot;
import uk.ac.ic.wlgitbridge.data.SnapshotFetcher;
import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import uk.ac.ic.wlgitbridge.data.model.db.PersistentStore;
import uk.ac.ic.wlgitbridge.data.model.db.SqlitePersistentStore;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotAttachment;
import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
import uk.ac.ic.wlgitbridge.util.Log;
import uk.ac.ic.wlgitbridge.util.Util;
import java.io.File;
import java.io.IOException;
import java.util.*;
/**
* Created by Winston on 06/11/14.
*/
public class DataStore {
private final File rootGitDirectory;
private final PersistentStore persistentStore;
private final SnapshotFetcher snapshotFetcher;
private final ResourceFetcher resourceFetcher;
public DataStore(String rootGitDirectoryPath) {
rootGitDirectory = initRootGitDirectory(rootGitDirectoryPath);
persistentStore = new SqlitePersistentStore(rootGitDirectory);
List<String> excludedFromDeletion = persistentStore.getProjectNames();
excludedFromDeletion.add(".wlgb");
Util.deleteInDirectoryApartFrom(
rootGitDirectory,
excludedFromDeletion.toArray(new String[] {})
);
snapshotFetcher = new SnapshotFetcher();
resourceFetcher = new ResourceFetcher(persistentStore);
}
public void updateProjectWithName(
Credential oauth2,
ProjectRepo repo
) throws IOException, GitUserException {
String projectName = repo.getProjectName();
LinkedList<Snapshot> snapshots =
snapshotFetcher.getSnapshotsForProjectAfterVersion(
oauth2,
projectName,
persistentStore.getLatestVersionForProject(projectName)
);
makeCommitsFromSnapshots(repo, snapshots);
if (!snapshots.isEmpty()) {
persistentStore.setLatestVersionForProject(
projectName,
snapshots.getLast().getVersionID()
);
}
}
private void makeCommitsFromSnapshots(ProjectRepo repo,
List<Snapshot> snapshots)
throws IOException, GitUserException {
String name = repo.getProjectName();
for (Snapshot snapshot : snapshots) {
Map<String, RawFile> fileTable = repo.getFiles();
List<RawFile> files = new LinkedList<>();
files.addAll(snapshot.getSrcs());
Map<String, byte[]> fetchedUrls = new HashMap<>();
for (SnapshotAttachment snapshotAttachment : snapshot.getAtts()) {
files.add(
resourceFetcher.get(
name,
snapshotAttachment.getUrl(),
snapshotAttachment.getPath(),
fileTable,
fetchedUrls
)
);
}
Log.info(
"[{}] Committing version ID: {}",
name,
snapshot.getVersionID()
);
Collection<String> missingFiles = repo.commitAndGetMissing(
new GitDirectoryContents(
files,
rootGitDirectory,
name,
snapshot
)
);
persistentStore.deleteFilesForProject(
name,
missingFiles.toArray(new String[missingFiles.size()])
);
}
}
public CandidateSnapshot createCandidateSnapshot(
String projectName,
RawDirectory directoryContents,
RawDirectory oldDirectoryContents
) throws SnapshotPostException,
IOException {
CandidateSnapshot candidateSnapshot = new CandidateSnapshot(
projectName,
persistentStore.getLatestVersionForProject(projectName),
directoryContents,
oldDirectoryContents
);
candidateSnapshot.writeServletFiles(rootGitDirectory);
return candidateSnapshot;
}
public void approveSnapshot(int versionID,
CandidateSnapshot candidateSnapshot) {
List<String> deleted = candidateSnapshot.getDeleted();
persistentStore.setLatestVersionForProject(
candidateSnapshot.getProjectName(),
versionID
);
persistentStore.deleteFilesForProject(
candidateSnapshot.getProjectName(),
deleted.toArray(new String[deleted.size()])
);
}
private File initRootGitDirectory(String rootGitDirectoryPath) {
File rootGitDirectory = new File(rootGitDirectoryPath);
rootGitDirectory.mkdirs();
return rootGitDirectory;
}
}

96
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/data/model/ResourceFetcher.java
View file

@ -1,105 +1,9 @@
package uk.ac.ic.wlgitbridge.data.model;
import com.ning.http.client.AsyncCompletionHandler;
import com.ning.http.client.HttpResponseBodyPart;
import com.ning.http.client.Response;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import uk.ac.ic.wlgitbridge.data.filestore.RepositoryFile;
import uk.ac.ic.wlgitbridge.data.model.db.PersistentStore;
import uk.ac.ic.wlgitbridge.snapshot.base.Request;
import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
import uk.ac.ic.wlgitbridge.util.Log;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Map;
import java.util.concurrent.ExecutionException;
/**
* Created by Winston on 21/02/15.
*/
public class ResourceFetcher {
private final PersistentStore persistentStore;
public ResourceFetcher(PersistentStore persistentStore) {
this.persistentStore = persistentStore;
}
public RawFile get(String projectName, String url, String newPath, Map<String, RawFile> fileTable, Map<String, byte[]> fetchedUrls) throws IOException {
String path = persistentStore.getPathForURLInProject(projectName, url);
byte[] contents;
if (path == null) {
path = newPath;
contents = fetch(projectName, url, path);
fetchedUrls.put(url, contents);
} else {
Log.info("Found (" + projectName + "): " + url);
Log.info("At (" + projectName + "): " + path);
contents = fetchedUrls.get(url);
if (contents == null) {
RawFile rawFile = fileTable.get(path);
if (rawFile == null) {
Log.warn(
"File " + path + " was not in the current commit, or the git tree, yet path was not null. " +
"File url is: " + url
);
contents = fetch(projectName, url, path);
} else {
contents = rawFile.getContents();
}
}
}
return new RepositoryFile(newPath, contents);
}
private byte[] fetch(String projectName, final String url, String path) throws FailedConnectionException {
byte[] contents;
Log.info("GET -> " + url);
try {
contents = Request.httpClient.prepareGet(url).execute(new AsyncCompletionHandler<byte[]>() {
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
@Override
public STATE onBodyPartReceived(HttpResponseBodyPart bodyPart) throws Exception {
bytes.write(bodyPart.getBodyPartBytes());
return STATE.CONTINUE;
}
@Override
public byte[] onCompleted(Response response) throws Exception {
byte[] data = bytes.toByteArray();
bytes.close();
Log.info(response.getStatusCode() + " " + response.getStatusText() + " (" + data.length + "B) -> " + url);
return data;
}
}).get();
} catch (InterruptedException e) {
Log.warn(
"Interrupted when fetching project: " +
projectName +
", url: " +
url +
", path: " +
path,
e
);
throw new FailedConnectionException();
} catch (ExecutionException e) {
Log.warn(
"ExecutionException when fetching project: " +
projectName +
", url: " +
url +
", path: " +
path,
e
);
throw new FailedConnectionException();
}
persistentStore.addURLIndexForProject(projectName, url, path);
return contents;
}
}

14
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java
View file

@ -9,6 +9,11 @@ import org.eclipse.jetty.servlet.ServletHolder;
import uk.ac.ic.wlgitbridge.application.config.Config;
import uk.ac.ic.wlgitbridge.application.jetty.NullLogger;
import uk.ac.ic.wlgitbridge.bridge.Bridge;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.db.SqliteDBStore;
import uk.ac.ic.wlgitbridge.bridge.repo.FSRepoStore;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.bridge.swap.SwapStore;
import uk.ac.ic.wlgitbridge.git.exception.InvalidRootDirectoryPathException;
import uk.ac.ic.wlgitbridge.git.servlet.WLGitServlet;
import uk.ac.ic.wlgitbridge.snapshot.base.SnapshotAPIRequest;
@ -43,7 +48,14 @@ public class GitBridgeServer {
org.eclipse.jetty.util.log.Log.setLog(new NullLogger());
this.port = config.getPort();
this.rootGitDirectoryPath = config.getRootGitDirectory();
bridgeAPI = new Bridge(rootGitDirectoryPath);
RepoStore repoStore = new FSRepoStore(rootGitDirectoryPath);
DBStore dbStore = new SqliteDBStore(repoStore.getRootDirectory());
SwapStore swapStore = new SwapStore() {};
bridgeAPI = Bridge.make(
repoStore,
dbStore,
swapStore
);
jettyServer = new Server(port);
configureJettyServer(config);
SnapshotAPIRequest.setBasicAuth(config.getUsername(), config.getPassword());

9
services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Util.java
View file

@ -19,6 +19,15 @@ public class Util {
private static String POSTBACK_URL;
private static final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSSSSS");
public static TimerTask makeTimerTask(Runnable lamb) {
return new TimerTask() {
@Override
public void run() {
lamb.run();
}
};
}
public static String entries(int entries) {
if (entries == 1) {
return "entry";

58
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java Normal file
View file

@ -0,0 +1,58 @@
package uk.ac.ic.wlgitbridge.bridge;
import org.junit.Before;
import org.junit.Test;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
import uk.ac.ic.wlgitbridge.bridge.resource.ResourceCache;
import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotAPI;
import uk.ac.ic.wlgitbridge.bridge.swap.SwapJob;
import uk.ac.ic.wlgitbridge.bridge.swap.SwapStore;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
/**
* Created by winston on 20/08/2016.
*/
public class BridgeTest {
private Bridge bridge;
private ProjectLock lock;
private RepoStore repoStore;
private DBStore dbStore;
private SwapStore swapStore;
private SnapshotAPI snapshotAPI;
private ResourceCache resourceCache;
private SwapJob swapJob;
@Before
public void setup() {
lock = mock(ProjectLock.class);
repoStore = mock(RepoStore.class);
dbStore = mock(DBStore.class);
swapStore = mock(SwapStore.class);
snapshotAPI = mock(SnapshotAPI.class);
swapJob = mock(SwapJob.class);
bridge = new Bridge(
lock,
repoStore,
dbStore,
swapStore,
snapshotAPI,
resourceCache,
swapJob
);
}
@Test
public void shutdownStopsSwapJob() {
bridge.startSwapJob(1000);
bridge.doShutdown();
verify(swapJob).start(1000);
verify(swapJob).stop();
}
}

14
services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/data/model/ResourceFetcherTest.java
View file

@ -9,8 +9,10 @@ import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.mockserver.client.server.MockServerClient;
import org.mockserver.junit.MockServerRule;
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
import uk.ac.ic.wlgitbridge.bridge.resource.ResourceCache;
import uk.ac.ic.wlgitbridge.bridge.resource.UrlResourceCache;
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
import uk.ac.ic.wlgitbridge.data.model.db.PersistentStore;
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
import uk.ac.ic.wlgitbridge.git.util.RepositoryObjectTreeWalker;
@ -50,23 +52,23 @@ public class ResourceFetcherTest {
);
final Mockery context = new Mockery();
final PersistentStore persistentStore = context.mock(PersistentStore.class);
final DBStore dbStore = context.mock(DBStore.class);
context.checking(new Expectations() {{
// It should fetch the file once it finds it is missing.
oneOf(persistentStore).getPathForURLInProject(testProjectName, testUrl);
oneOf(dbStore).getPathForURLInProject(testProjectName, testUrl);
will(returnValue(oldTestPath));
// It should update the URL index store once it has fetched; at present, it does not actually change the stored path.
oneOf(persistentStore).addURLIndexForProject(testProjectName, testUrl, oldTestPath);
oneOf(dbStore).addURLIndexForProject(testProjectName, testUrl, oldTestPath);
}});
ResourceFetcher resourceFetcher = new ResourceFetcher(persistentStore);
ResourceCache resources = new UrlResourceCache(dbStore);
TemporaryFolder repositoryFolder = new TemporaryFolder();
repositoryFolder.create();
Repository repository = new FileRepositoryBuilder().setWorkTree(repositoryFolder.getRoot()).build();
Map<String, RawFile> fileTable = new RepositoryObjectTreeWalker(repository).getDirectoryContents().getFileTable();
Map<String, byte[]> fetchedUrls = new HashMap<String, byte[]>();
resourceFetcher.get(testProjectName, testUrl, newTestPath, fileTable, fetchedUrls);
resources.get(testProjectName, testUrl, newTestPath, fileTable, fetchedUrls);
// We don't bother caching in this case, at present.
assertEquals(0, fetchedUrls.size());

4
services/git-bridge/writelatex-git-bridge.iml
View file

@ -13,7 +13,7 @@
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" scope="TEST" name="Maven: junit:junit:4.12" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-core:1.3" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: org.hamcrest:hamcrest-core:1.3" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: org.jmock:jmock-junit4:2.8.2" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: org.jmock:jmock:2.8.2" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: org.jmock:jmock-testjar:2.8.2" level="project" />
@ -107,5 +107,7 @@
<orderEntry type="library" scope="TEST" name="Maven: org.bouncycastle:bcprov-jdk15on:1.52" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: janino:janino:2.5.10" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: org.mock-server:mockserver-logging:3.10.4" level="project" />
<orderEntry type="library" name="Maven: org.mockito:mockito-core:1.10.19" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: org.objenesis:objenesis:2.1" level="project" />
</component>
</module>