diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java index 10a74c56bf..31796c2ebc 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/Bridge.java @@ -1,7 +1,7 @@ package uk.ac.ic.wlgitbridge.bridge; import com.google.api.client.auth.oauth2.Credential; -import org.eclipse.jgit.transport.ServiceMayNotContinueException; +import org.eclipse.jgit.errors.RepositoryNotFoundException; import uk.ac.ic.wlgitbridge.bridge.db.DBStore; import uk.ac.ic.wlgitbridge.bridge.db.ProjectState; import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SqliteDBStore; @@ -15,13 +15,14 @@ import uk.ac.ic.wlgitbridge.bridge.repo.ProjectRepo; import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore; import uk.ac.ic.wlgitbridge.bridge.resource.ResourceCache; import uk.ac.ic.wlgitbridge.bridge.resource.UrlResourceCache; -import uk.ac.ic.wlgitbridge.bridge.snapshot.NetSnapshotAPI; -import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotAPI; +import uk.ac.ic.wlgitbridge.bridge.snapshot.NetSnapshotApi; +import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApi; import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJob; import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJobConfig; import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJobImpl; import uk.ac.ic.wlgitbridge.bridge.swap.store.S3SwapStore; import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore; +import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApiFacade; import uk.ac.ic.wlgitbridge.data.CandidateSnapshot; import uk.ac.ic.wlgitbridge.data.ProjectLockImpl; import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents; @@ -38,12 +39,9 @@ import uk.ac.ic.wlgitbridge.server.FileHandler; import uk.ac.ic.wlgitbridge.server.PostbackContents; import uk.ac.ic.wlgitbridge.server.PostbackHandler; import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException; -import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocRequest; -import uk.ac.ic.wlgitbridge.snapshot.getdoc.exception.InvalidProjectException; import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotAttachment; import uk.ac.ic.wlgitbridge.snapshot.push.PostbackManager; import uk.ac.ic.wlgitbridge.snapshot.push.PostbackPromise; -import uk.ac.ic.wlgitbridge.snapshot.push.PushRequest; import uk.ac.ic.wlgitbridge.snapshot.push.PushResult; import uk.ac.ic.wlgitbridge.snapshot.push.exception.*; import uk.ac.ic.wlgitbridge.util.Log; @@ -120,8 +118,8 @@ import java.util.*; * * 6. The Snapshot API, which provides data from the Overleaf app. * - * @see SnapshotAPI - the interface for the Snapshot API. - * @see NetSnapshotAPI - the default concrete implementation + * @see SnapshotApi - the interface for the Snapshot API. + * @see NetSnapshotApi - the default concrete implementation * * 7. The Resource Cache, which provides the data for attachment resources from * URLs. It will generally fetch from the source on a cache miss. @@ -149,7 +147,7 @@ public class Bridge { private final SwapJob swapJob; private final GcJob gcJob; - private final SnapshotAPI snapshotAPI; + private final SnapshotApiFacade snapshotAPI; private final ResourceCache resourceCache; private final PostbackManager postbackManager; @@ -169,7 +167,8 @@ public class Bridge { RepoStore repoStore, DBStore dbStore, SwapStore swapStore, - Optional swapJobConfig + Optional swapJobConfig, + SnapshotApi snapshotApi ) { ProjectLock lock = new ProjectLockImpl((int threads) -> Log.info("Waiting for " + threads + " projects...") @@ -187,7 +186,7 @@ public class Bridge { swapStore ), new GcJobImpl(repoStore, lock), - new NetSnapshotAPI(), + new SnapshotApiFacade(snapshotApi), new UrlResourceCache(dbStore) ); } @@ -202,7 +201,7 @@ public class Bridge { * @param swapStore the {@link SwapStore} to use * @param swapJob the {@link SwapJob} to use * @param gcJob - * @param snapshotAPI the {@link SnapshotAPI} to use + * @param snapshotAPI the {@link SnapshotApi} to use * @param resourceCache the {@link ResourceCache} to use */ Bridge( @@ -212,7 +211,7 @@ public class Bridge { SwapStore swapStore, SwapJob swapJob, GcJob gcJob, - SnapshotAPI snapshotAPI, + SnapshotApiFacade snapshotAPI, ResourceCache resourceCache ) { this.lock = lock; @@ -291,63 +290,26 @@ public class Bridge { } } - /** - * Checks if a project exists by asking the snapshot API. - * - * The snapshot API is the source of truth because we can't know by - * ourselves whether a project exists. If a user creates a project on the - * app, and clones, the project is not on the git bridge disk and must ask - * the snapshot API whether it exists. - * - * 1. Acquires the project lock. - * 2. Makes a docs request and tries to get the version ID. - * 3. If the version ID is valid, returns true. - * 4. Otherwise, the version ID is invalid, and throws - * InvalidProjectException, returning false. - * - * @param oauth2 The oauth2 to use for the snapshot API - * @param projectName The project name - * @return true iff the project exists - * @throws ServiceMayNotContinueException if the connection fails - * @throws GitUserException if the user is not allowed access - */ - public boolean projectExists( - Credential oauth2, - String projectName - ) throws ServiceMayNotContinueException, - GitUserException { - Log.info("[{}] Checking that project exists", projectName); - try (LockGuard __ = lock.lockGuard(projectName)) { - GetDocRequest getDocRequest = new GetDocRequest( - oauth2, - projectName - ); - getDocRequest.request(); - getDocRequest.getResult().getVersionID(); - return true; - } catch (InvalidProjectException e) { - return false; - } - } - /** * Synchronises the given repository with Overleaf. * * It acquires the project lock and calls - * {@link #updateRepositoryCritical(Credential, ProjectRepo)} + * {@link #getUpdatedRepoCritical(Optional, String)}. * @param oauth2 The oauth2 to use - * @param repo the repository to use + * @param projectName The name of the project * @throws IOException * @throws GitUserException */ - public void updateRepository( - Credential oauth2, - ProjectRepo repo + public ProjectRepo getUpdatedRepo( + Optional oauth2, + String projectName ) throws IOException, GitUserException { - String projectName = repo.getProjectName(); try (LockGuard __ = lock.lockGuard(projectName)) { + if (!snapshotAPI.projectExists(oauth2, projectName)) { + throw new RepositoryNotFoundException(projectName); + } Log.info("[{}] Updating repository", projectName); - updateRepositoryCritical(oauth2, repo); + return getUpdatedRepoCritical(oauth2, projectName); } } @@ -359,9 +321,7 @@ public class Bridge { * 1. Queries the project state for the given project name. * a. NOT_PRESENT = We've never seen it before, and the row for the * project doesn't even exist. The project definitely - * exists because - * {@link #projectExists(Credential, String)} would - * have had to return true to get here. + * exists because we would have aborted otherwise. * b. PRESENT = The project is on disk. * c. SWAPPED = The project is in the {@link SwapStore} * @@ -370,43 +330,44 @@ public class Bridge { * present. * * With the project present, snapshots are downloaded from the snapshot - * API with {@link #updateProject(Credential, ProjectRepo)}. + * API with {@link #updateProject(Optional, ProjectRepo)}. * * Then, the last accessed time of the project is set to the current time. * This is to support the LRU of the swap store. * @param oauth2 - * @param repo + * @param projectName The name of the project * @throws IOException * @throws GitUserException */ - private void updateRepositoryCritical( - Credential oauth2, - ProjectRepo repo + private ProjectRepo getUpdatedRepoCritical( + Optional oauth2, + String projectName ) throws IOException, GitUserException { - String projectName = repo.getProjectName(); + ProjectRepo repo; ProjectState state = dbStore.getProjectState(projectName); switch (state) { case NOT_PRESENT: - repo.initRepo(repoStore); + repo = repoStore.initRepo(projectName); break; case SWAPPED: swapJob.restore(projectName); /* Fallthrough */ default: - repo.useExistingRepository(repoStore); + repo = repoStore.getExistingRepo(projectName); } updateProject(oauth2, repo); dbStore.setLastAccessedTime( projectName, Timestamp.valueOf(LocalDateTime.now()) ); + return repo; } /** * The public call to push a project. * * It acquires the lock and calls {@link #pushCritical( - * Credential, + * Optional, * String, * RawDirectory, * RawDirectory @@ -421,7 +382,7 @@ public class Bridge { * @throws ForbiddenException */ public void push( - Credential oauth2, + Optional oauth2, String projectName, RawDirectory directoryContents, RawDirectory oldDirectoryContents, @@ -499,7 +460,7 @@ public class Bridge { * @throws SnapshotPostException */ private void pushCritical( - Credential oauth2, + Optional oauth2, String projectName, RawDirectory directoryContents, RawDirectory oldDirectoryContents @@ -523,13 +484,8 @@ public class Bridge { projectName, candidate ); - PushRequest pushRequest = new PushRequest( - oauth2, - candidate, - postbackKey - ); - pushRequest.request(); - PushResult result = pushRequest.getResult(); + PushResult result + = snapshotAPI.push(oauth2, candidate, postbackKey); if (result.wasSuccessful()) { Log.info( "[{}] Push to Overleaf successful", @@ -587,7 +543,7 @@ public class Bridge { * {@link PostbackContents#processPostback()}, i.e. once the Overleaf app * has fetched all the atts and has committed the push and is happy, it * calls back here, fulfilling the promise that the push - * {@link #push(Credential, String, RawDirectory, RawDirectory, String)} + * {@link #push(Optional, String, RawDirectory, RawDirectory, String)} * is waiting on. * * The Overleaf app will have invented a new version for the push, which is @@ -642,7 +598,7 @@ public class Bridge { /* PRIVATE */ /** - * Called by {@link #updateRepositoryCritical(Credential, ProjectRepo)}. + * Called by {@link #getUpdatedRepoCritical(Optional, String)} * * Does the actual work of getting the snapshots for a project from the * snapshot API and committing them to a repo. @@ -655,16 +611,13 @@ public class Bridge { * @throws GitUserException */ private void updateProject( - Credential oauth2, + Optional oauth2, ProjectRepo repo ) throws IOException, GitUserException { String projectName = repo.getProjectName(); - Deque snapshots = - snapshotAPI.getSnapshotsForProjectAfterVersion( - oauth2, - projectName, - dbStore.getLatestVersionForProject(projectName) - ); + int latestVersionId = dbStore.getLatestVersionForProject(projectName); + Deque snapshots = snapshotAPI.getSnapshots( + oauth2, projectName, latestVersionId); makeCommitsFromSnapshots(repo, snapshots); @@ -677,7 +630,7 @@ public class Bridge { } /** - * Called by {@link #updateProject(Credential, ProjectRepo)}. + * Called by {@link #updateProject(Optional, ProjectRepo)}. * * Performs the actual Git commits on the disk. * @@ -732,7 +685,7 @@ public class Bridge { /** * Called by - * {@link #pushCritical(Credential, String, RawDirectory, RawDirectory)}. + * {@link #pushCritical(Optional, String, RawDirectory, RawDirectory)}. * * This call consists of 2 things: Creating the candidate snapshot, * and writing the atts to the atts directory. @@ -761,7 +714,7 @@ public class Bridge { /** * Called by - * {@link #pushCritical(Credential, String, RawDirectory, RawDirectory)}. + * {@link #pushCritical(Optional, String, RawDirectory, RawDirectory)}. * * This method approves a push by setting the latest version and removing * any deleted files from the db store (files were already added by the diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/gc/GcJob.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/gc/GcJob.java index dea5d37ece..74f965d7a4 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/gc/GcJob.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/gc/GcJob.java @@ -1,10 +1,10 @@ package uk.ac.ic.wlgitbridge.bridge.gc; -import com.google.api.client.auth.oauth2.Credential; import uk.ac.ic.wlgitbridge.bridge.Bridge; import uk.ac.ic.wlgitbridge.bridge.repo.ProjectRepo; import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory; +import java.util.Optional; import java.util.concurrent.CompletableFuture; /** @@ -12,9 +12,9 @@ import java.util.concurrent.CompletableFuture; * GC which executes every hour or so. * * We don't queue it into a more immediate Executor because there is no way to - * know if a call to {@link Bridge#updateProject(Credential, ProjectRepo)}, + * know if a call to {@link Bridge#updateProject(Optional, ProjectRepo)}, * which releases the lock, is going to call - * {@link Bridge#push(Credential, String, RawDirectory, RawDirectory, String)}. + * {@link Bridge#push(Optional, String, RawDirectory, RawDirectory, String)}. * * We don't want the GC to run in between an update and a push. */ diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStore.java index cbace4118c..eef1a92f86 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStore.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/FSGitRepoStore.java @@ -45,6 +45,13 @@ public class FSGitRepoStore implements RepoStore { return rootDirectory; } + @Override + public ProjectRepo initRepo(String project) throws IOException { + GitProjectRepo ret = new GitProjectRepo(project); + ret.initRepo(this); + return ret; + } + @Override public ProjectRepo getExistingRepo(String project) throws IOException { GitProjectRepo ret = new GitProjectRepo(project); diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/GitProjectRepo.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/GitProjectRepo.java index 9dbc9229c7..dd83169598 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/GitProjectRepo.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/GitProjectRepo.java @@ -199,6 +199,7 @@ public class GitProjectRepo implements ProjectRepo { } } + @Override public Repository getJGitRepository() { return repository.get(); } diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/ProjectRepo.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/ProjectRepo.java index 80988b14f3..dc875f416b 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/ProjectRepo.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/ProjectRepo.java @@ -1,5 +1,6 @@ package uk.ac.ic.wlgitbridge.bridge.repo; +import org.eclipse.jgit.lib.Repository; import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents; import uk.ac.ic.wlgitbridge.data.filestore.RawFile; import uk.ac.ic.wlgitbridge.git.exception.GitUserException; @@ -37,4 +38,5 @@ public interface ProjectRepo { File getProjectDir(); + Repository getJGitRepository(); } diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java index 466d8441c7..000ae25ca2 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/repo/RepoStore.java @@ -17,6 +17,8 @@ public interface RepoStore { File getRootDirectory(); + ProjectRepo initRepo(String project) throws IOException; + ProjectRepo getExistingRepo(String project) throws IOException; void purgeNonexistentProjects( diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/snapshot/NetSnapshotApi.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/snapshot/NetSnapshotApi.java new file mode 100644 index 0000000000..2e1af6c70d --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/snapshot/NetSnapshotApi.java @@ -0,0 +1,55 @@ +package uk.ac.ic.wlgitbridge.bridge.snapshot; + +import com.google.api.client.auth.oauth2.Credential; +import uk.ac.ic.wlgitbridge.data.CandidateSnapshot; +import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocRequest; +import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocResult; +import uk.ac.ic.wlgitbridge.snapshot.getforversion.GetForVersionRequest; +import uk.ac.ic.wlgitbridge.snapshot.getforversion.GetForVersionResult; +import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.GetSavedVersRequest; +import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.GetSavedVersResult; +import uk.ac.ic.wlgitbridge.snapshot.push.PushRequest; +import uk.ac.ic.wlgitbridge.snapshot.push.PushResult; + +import java.util.Optional; +import java.util.concurrent.CompletableFuture; + +/** + * Created by winston on 20/08/2016. + */ +public class NetSnapshotApi implements SnapshotApi { + + @Override + public CompletableFuture getDoc( + Optional oath2, String projectName) { + return new GetDocRequest(opt(oath2), projectName).request(); + } + + @Override + public CompletableFuture getForVersion( + Optional oauth2, String projectName, int versionId) { + return new GetForVersionRequest( + opt(oauth2), projectName, versionId).request(); + } + + @Override + public CompletableFuture getSavedVers( + Optional oauth2, String projectName) { + return new GetSavedVersRequest(opt(oauth2), projectName).request(); + } + + @Override + public CompletableFuture push( + Optional oauth2, + CandidateSnapshot candidateSnapshot, + String postbackKey + ) { + return new PushRequest( + opt(oauth2), candidateSnapshot, postbackKey).request(); + } + + private static Credential opt(Optional oath2) { + return oath2.orElse(null); + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/snapshot/SnapshotAPI.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/snapshot/SnapshotAPI.java deleted file mode 100644 index d2c1cbc435..0000000000 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/snapshot/SnapshotAPI.java +++ /dev/null @@ -1,21 +0,0 @@ -package uk.ac.ic.wlgitbridge.bridge.snapshot; - -import com.google.api.client.auth.oauth2.Credential; -import uk.ac.ic.wlgitbridge.data.model.Snapshot; -import uk.ac.ic.wlgitbridge.git.exception.GitUserException; -import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException; - -import java.util.Deque; - -/** - * Created by winston on 20/08/2016. - */ -public interface SnapshotAPI { - - Deque getSnapshotsForProjectAfterVersion( - Credential oauth2, - String projectName, - int latestVersion - ) throws FailedConnectionException, GitUserException; - -} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/snapshot/SnapshotApi.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/snapshot/SnapshotApi.java new file mode 100644 index 0000000000..cd7ce7eaac --- /dev/null +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/snapshot/SnapshotApi.java @@ -0,0 +1,52 @@ +package uk.ac.ic.wlgitbridge.bridge.snapshot; + +import com.google.api.client.auth.oauth2.Credential; +import uk.ac.ic.wlgitbridge.data.CandidateSnapshot; +import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException; +import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException; +import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocResult; +import uk.ac.ic.wlgitbridge.snapshot.getforversion.GetForVersionResult; +import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.GetSavedVersResult; +import uk.ac.ic.wlgitbridge.snapshot.push.PushResult; + +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; + +/** + * Created by winston on 20/08/2016. + */ +public interface SnapshotApi { + + CompletableFuture getDoc( + Optional oath2, String projectName); + + CompletableFuture getForVersion( + Optional oauth2, String projectName, int versionId); + + CompletableFuture getSavedVers( + Optional oauth2, String projectName); + + CompletableFuture push( + Optional oauth2, + CandidateSnapshot candidateSnapshot, + String postbackKey); + + static T getResult(CompletableFuture result) + throws FailedConnectionException, ForbiddenException { + try { + return result.join(); + } catch (CompletionException e) { + try { + throw e.getCause(); + } catch (FailedConnectionException + | ForbiddenException + | RuntimeException r) { + throw r; + } catch (Throwable __) { + throw e; + } + } + } + +} diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/snapshot/NetSnapshotAPI.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/snapshot/SnapshotApiFacade.java similarity index 52% rename from services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/snapshot/NetSnapshotAPI.java rename to services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/snapshot/SnapshotApiFacade.java index a77f88c7a5..ea4d388101 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/snapshot/NetSnapshotAPI.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/bridge/snapshot/SnapshotApiFacade.java @@ -1,34 +1,57 @@ package uk.ac.ic.wlgitbridge.bridge.snapshot; import com.google.api.client.auth.oauth2.Credential; +import uk.ac.ic.wlgitbridge.data.CandidateSnapshot; import uk.ac.ic.wlgitbridge.data.model.Snapshot; import uk.ac.ic.wlgitbridge.git.exception.GitUserException; import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException; import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException; -import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocRequest; import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocResult; -import uk.ac.ic.wlgitbridge.snapshot.getforversion.GetForVersionRequest; +import uk.ac.ic.wlgitbridge.snapshot.getforversion.GetForVersionResult; import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotData; -import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.GetSavedVersRequest; +import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.GetSavedVersResult; import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.SnapshotInfo; +import uk.ac.ic.wlgitbridge.snapshot.push.PushResult; +import uk.ac.ic.wlgitbridge.snapshot.push.exception.InvalidProjectException; import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; /** - * Created by winston on 20/08/2016. + * Created by winston on 02/07/2017. */ -public class NetSnapshotAPI implements SnapshotAPI { +public class SnapshotApiFacade { - @Override - public Deque getSnapshotsForProjectAfterVersion( - Credential oauth2, - String projectName, - int version + private final SnapshotApi api; + + public SnapshotApiFacade(SnapshotApi api) { + this.api = api; + } + + public boolean projectExists( + Optional oauth2, + String projectName ) throws FailedConnectionException, GitUserException { + try { + SnapshotApi + .getResult(api.getDoc(oauth2, projectName)) + .getVersionID(); + return true; + } catch (InvalidProjectException e) { + return false; + } + } + + public Deque getSnapshots( + Optional oauth2, + String projectName, + int afterVersionId + ) throws GitUserException, FailedConnectionException { List snapshotInfos = getSnapshotInfosAfterVersion( oauth2, projectName, - version + afterVersionId ); List snapshotDatas = getMatchingSnapshotData( oauth2, @@ -38,25 +61,31 @@ public class NetSnapshotAPI implements SnapshotAPI { return combine(snapshotInfos, snapshotDatas); } + public PushResult push( + Optional oauth2, + CandidateSnapshot candidateSnapshot, + String postbackKey + ) throws FailedConnectionException, ForbiddenException { + return SnapshotApi.getResult(api.push( + oauth2, candidateSnapshot, postbackKey)); + } + private List getSnapshotInfosAfterVersion( - Credential oauth2, + Optional oauth2, String projectName, int version ) throws FailedConnectionException, GitUserException { SortedSet versions = new TreeSet<>(); - GetDocRequest getDoc = new GetDocRequest(oauth2, projectName); - GetSavedVersRequest getSavedVers = new GetSavedVersRequest( - oauth2, - projectName - ); - getDoc.request(); - getSavedVers.request(); - GetDocResult latestDoc = getDoc.getResult(); + CompletableFuture getDoc + = api.getDoc(oauth2, projectName); + CompletableFuture savedVers + = api.getSavedVers(oauth2, projectName); + GetDocResult latestDoc = SnapshotApi.getResult(getDoc); int latest = latestDoc.getVersionID(); if (latest > version) { for ( SnapshotInfo snapshotInfo : - getSavedVers.getResult().getSavedVers() + SnapshotApi.getResult(savedVers).getSavedVers() ) { if (snapshotInfo.getVersionId() > version) { versions.add(snapshotInfo); @@ -70,45 +99,36 @@ public class NetSnapshotAPI implements SnapshotAPI { )); } - return new LinkedList(versions); + return new ArrayList<>(versions); } private List getMatchingSnapshotData( - Credential oauth2, + Optional oauth2, String projectName, List snapshotInfos ) throws FailedConnectionException, ForbiddenException { - List firedRequests = fireDataRequests( - oauth2, - projectName, - snapshotInfos - ); + List> firedRequests + = fireDataRequests(oauth2, projectName, snapshotInfos); List snapshotDataList = new ArrayList<>(); - for (GetForVersionRequest fired : firedRequests) { - snapshotDataList.add(fired.getResult().getSnapshotData()); + for (CompletableFuture fired : firedRequests) { + snapshotDataList.add(fired.join().getSnapshotData()); } return snapshotDataList; } - private List fireDataRequests( - Credential oauth2, + private List> fireDataRequests( + Optional oauth2, String projectName, List snapshotInfos ) { - List requests = new ArrayList<>(); - for (SnapshotInfo snapshotInfo : snapshotInfos) { - GetForVersionRequest request = new GetForVersionRequest( - oauth2, - projectName, - snapshotInfo.getVersionId() - ); - requests.add(request); - request.request(); - } - return requests; + return snapshotInfos + .stream() + .map(snap -> api.getForVersion( + oauth2, projectName, snap.getVersionId())) + .collect(Collectors.toList()); } - private Deque combine( + private static Deque combine( List snapshotInfos, List snapshotDatas ) { diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLReceivePackFactory.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLReceivePackFactory.java index 39056999da..70bf90de74 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLReceivePackFactory.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLReceivePackFactory.java @@ -5,7 +5,7 @@ import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.transport.ReceivePack; import org.eclipse.jgit.transport.resolver.ReceivePackFactory; import uk.ac.ic.wlgitbridge.bridge.Bridge; -import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotAPI; +import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApi; import uk.ac.ic.wlgitbridge.git.handler.hook.WriteLatexPutHook; import uk.ac.ic.wlgitbridge.git.servlet.WLGitServlet; import uk.ac.ic.wlgitbridge.server.Oauth2Filter; @@ -13,6 +13,7 @@ import uk.ac.ic.wlgitbridge.util.Log; import uk.ac.ic.wlgitbridge.util.Util; import javax.servlet.http.HttpServletRequest; +import java.util.Optional; /** * Created by Winston on 02/11/14. @@ -38,7 +39,7 @@ public class WLReceivePackFactory * * The {@link WriteLatexPutHook} needs our hostname, which we get from the * original {@link HttpServletRequest}, used to provide a postback URL to - * the {@link SnapshotAPI}. We also give it the oauth2 that we injected in + * the {@link SnapshotApi}. We also give it the oauth2 that we injected in * the {@link Oauth2Filter}, and the {@link Bridge}. * * At this point, the repository will have been synced to the latest on @@ -59,9 +60,9 @@ public class WLReceivePackFactory "[{}] Creating receive-pack", repository.getWorkTree().getName() ); - Credential oauth2 = (Credential) httpServletRequest.getAttribute( - Oauth2Filter.ATTRIBUTE_KEY - ); + Optional oauth2 = Optional.ofNullable( + (Credential) httpServletRequest.getAttribute( + Oauth2Filter.ATTRIBUTE_KEY)); ReceivePack receivePack = new ReceivePack(repository); String hostname = Util.getPostbackURL(); if (hostname == null) { diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLRepositoryResolver.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLRepositoryResolver.java index 049845424b..022101fc5b 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLRepositoryResolver.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/WLRepositoryResolver.java @@ -7,7 +7,6 @@ import org.eclipse.jgit.transport.ServiceMayNotContinueException; import org.eclipse.jgit.transport.resolver.RepositoryResolver; import org.eclipse.jgit.transport.resolver.ServiceNotAuthorizedException; import uk.ac.ic.wlgitbridge.bridge.Bridge; -import uk.ac.ic.wlgitbridge.bridge.repo.GitProjectRepo; import uk.ac.ic.wlgitbridge.git.exception.GitUserException; import uk.ac.ic.wlgitbridge.git.handler.hook.WriteLatexPutHook; import uk.ac.ic.wlgitbridge.git.servlet.WLGitServlet; @@ -19,6 +18,7 @@ import uk.ac.ic.wlgitbridge.util.Util; import javax.servlet.http.HttpServletRequest; import java.io.IOException; +import java.util.Optional; /** * Created by Winston on 02/11/14. @@ -82,17 +82,12 @@ public class WLRepositoryResolver ServiceNotAuthorizedException, ServiceMayNotContinueException { Log.info("[{}] Request to open git repo", name); - Credential oauth2 = (Credential) httpServletRequest.getAttribute( - Oauth2Filter.ATTRIBUTE_KEY - ); + Optional oauth2 = Optional.ofNullable( + (Credential) httpServletRequest.getAttribute( + Oauth2Filter.ATTRIBUTE_KEY)); String projName = Util.removeAllSuffixes(name, "/", ".git"); try { - if (!bridge.projectExists(oauth2, projName)) { - throw new RepositoryNotFoundException(projName); - } - GitProjectRepo repo = new GitProjectRepo(projName); - bridge.updateRepository(oauth2, repo); - return repo.getJGitRepository(); + return bridge.getUpdatedRepo(oauth2, projName).getJGitRepository(); } catch (RepositoryNotFoundException e) { Log.info("Repository not found: " + name); throw e; diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/hook/WriteLatexPutHook.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/hook/WriteLatexPutHook.java index 50fe9ae56d..b1da5ed31c 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/hook/WriteLatexPutHook.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/handler/hook/WriteLatexPutHook.java @@ -21,6 +21,7 @@ import uk.ac.ic.wlgitbridge.util.Log; import java.io.IOException; import java.util.Collection; import java.util.Iterator; +import java.util.Optional; /** * Created by Winston on 03/11/14. @@ -34,7 +35,7 @@ public class WriteLatexPutHook implements PreReceiveHook { private final Bridge bridge; private final String hostname; - private final Credential oauth2; + private final Optional oauth2; /** * The constructor to use, which provides the hook with the {@link Bridge}, @@ -47,7 +48,7 @@ public class WriteLatexPutHook implements PreReceiveHook { public WriteLatexPutHook( Bridge bridge, String hostname, - Credential oauth2 + Optional oauth2 ) { this.bridge = bridge; this.hostname = hostname; @@ -112,7 +113,7 @@ public class WriteLatexPutHook implements PreReceiveHook { } private void handleReceiveCommand( - Credential oauth2, + Optional oauth2, Repository repository, ReceiveCommand receiveCommand ) throws IOException, GitUserException { diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/util/RepositoryObjectTreeWalker.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/util/RepositoryObjectTreeWalker.java index 71bb8a1418..836b0b319d 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/util/RepositoryObjectTreeWalker.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/git/util/RepositoryObjectTreeWalker.java @@ -49,10 +49,17 @@ public class RepositoryObjectTreeWalker { public RawDirectory getDirectoryContents( ) throws IOException, SizeLimitExceededException, InvalidGitRepository { - return getDirectoryContents(50 * 1024 * 1024); + return getDirectoryContents(Optional.empty()); } - private RawDirectory getDirectoryContents(long maxFileSize) + public RawDirectory getDirectoryContents(long maxFileSize) + throws InvalidGitRepository, + SizeLimitExceededException, + IOException { + return getDirectoryContents(Optional.of(maxFileSize)); + } + + private RawDirectory getDirectoryContents(Optional maxFileSize) throws IOException, SizeLimitExceededException, InvalidGitRepository { @@ -73,7 +80,7 @@ public class RepositoryObjectTreeWalker { return treeWalk; } - private Map walkGitObjectTree(long maxFileSize) + private Map walkGitObjectTree(Optional maxFileSize) throws IOException, SizeLimitExceededException, InvalidGitRepository { @@ -90,9 +97,9 @@ public class RepositoryObjectTreeWalker { } ObjectLoader obj = repository.open(objectId); long size = obj.getSize(); - if (size > maxFileSize) { + if (maxFileSize.isPresent() && size > maxFileSize.get()) { throw new SizeLimitExceededException( - Optional.ofNullable(path), size, maxFileSize); + Optional.ofNullable(path), size, maxFileSize.get()); } try (ByteArrayOutputStream o = new ByteArrayOutputStream( CastUtil.assumeInt(size))) { diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java index 1150b4a642..1984f056eb 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java @@ -13,6 +13,8 @@ import uk.ac.ic.wlgitbridge.bridge.db.DBStore; import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SqliteDBStore; import uk.ac.ic.wlgitbridge.bridge.repo.FSGitRepoStore; import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore; +import uk.ac.ic.wlgitbridge.bridge.snapshot.NetSnapshotApi; +import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApi; import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore; import uk.ac.ic.wlgitbridge.git.servlet.WLGitServlet; import uk.ac.ic.wlgitbridge.snapshot.base.SnapshotAPIRequest; @@ -57,14 +59,16 @@ public class GitBridgeServer { ).resolve(".wlgb").resolve("wlgb.db").toFile() ); SwapStore swapStore = SwapStore.fromConfig(config.getSwapStore()); + SnapshotApi snapshotApi = new NetSnapshotApi(); bridge = Bridge.make( repoStore, dbStore, swapStore, - config.getSwapJob() + config.getSwapJob(), + snapshotApi ); jettyServer = new Server(port); - configureJettyServer(config); + configureJettyServer(config, snapshotApi); SnapshotAPIRequest.setBasicAuth( config.getUsername(), config.getPassword() @@ -105,11 +109,12 @@ public class GitBridgeServer { } private void configureJettyServer( - Config config + Config config, + SnapshotApi snapshotApi ) throws ServletException { HandlerCollection handlers = new HandlerList(); handlers.addHandler(initApiHandler()); - handlers.addHandler(initGitHandler(config)); + handlers.addHandler(initGitHandler(config, snapshotApi)); jettyServer.setHandler(handlers); } @@ -130,12 +135,13 @@ public class GitBridgeServer { } private Handler initGitHandler( - Config config + Config config, + SnapshotApi snapshotApi ) throws ServletException { final ServletContextHandler servletContextHandler = new ServletContextHandler(ServletContextHandler.SESSIONS); if (config.isUsingOauth2()) { - Filter filter = new Oauth2Filter(config.getOauth2()); + Filter filter = new Oauth2Filter(snapshotApi, config.getOauth2()); servletContextHandler.addFilter( new FilterHolder(filter), "/*", diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/Oauth2Filter.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/Oauth2Filter.java index cd5140d614..e33294f75e 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/Oauth2Filter.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/Oauth2Filter.java @@ -5,6 +5,7 @@ import com.google.api.client.http.GenericUrl; import org.apache.commons.codec.binary.Base64; import org.eclipse.jetty.server.Request; import uk.ac.ic.wlgitbridge.application.config.Oauth2; +import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApi; import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException; import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocRequest; import uk.ac.ic.wlgitbridge.util.Instance; @@ -17,6 +18,7 @@ import javax.servlet.http.HttpServletResponse; import java.io.IOException; import java.io.PrintWriter; import java.io.UnsupportedEncodingException; +import java.util.Optional; import java.util.StringTokenizer; /** @@ -26,9 +28,12 @@ public class Oauth2Filter implements Filter { public static final String ATTRIBUTE_KEY = "oauth2"; + private final SnapshotApi snapshotApi; + private final Oauth2 oauth2; - public Oauth2Filter(Oauth2 oauth2) { + public Oauth2Filter(SnapshotApi snapshotApi, Oauth2 oauth2) { + this.snapshotApi = snapshotApi; this.oauth2 = oauth2; } @@ -60,7 +65,8 @@ public class Oauth2Filter implements Filter { GetDocRequest doc = new GetDocRequest(project); doc.request(); try { - doc.getResult(); + SnapshotApi.getResult( + snapshotApi.getDoc(Optional.empty(), project)); } catch (ForbiddenException e) { Log.info("[{}] Auth needed", project); getAndInjectCredentials( diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/base/Request.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/base/Request.java index 6404ea9f85..58491fb736 100644 --- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/base/Request.java +++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/snapshot/base/Request.java @@ -9,8 +9,7 @@ import uk.ac.ic.wlgitbridge.util.Log; import javax.servlet.http.HttpServletResponse; import java.io.IOException; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; +import java.util.concurrent.*; /** * Created by Winston on 06/11/14. @@ -19,6 +18,8 @@ public abstract class Request { public static final AsyncHttpClient httpClient = new AsyncHttpClient(); + private static final Executor executor = Executors.newCachedThreadPool(); + private final String url; private Future future; @@ -27,7 +28,7 @@ public abstract class Request { this.url = url; } - public void request() { + public CompletableFuture request() { switch (httpMethod()) { case GET: performGetRequest(); @@ -38,9 +39,18 @@ public abstract class Request { default: break; } + CompletableFuture ret = new CompletableFuture<>(); + executor.execute(() -> { + try { + ret.complete(getResult()); + } catch (Throwable t) { + ret.completeExceptionally(t); + } + }); + return ret; } - public T getResult() throws FailedConnectionException, ForbiddenException { + private T getResult() throws FailedConnectionException, ForbiddenException { try { HttpResponse response = future.get(); Log.info( diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java index 93d4f46973..ad05047661 100644 --- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java +++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java @@ -9,14 +9,14 @@ import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock; import uk.ac.ic.wlgitbridge.bridge.repo.ProjectRepo; import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore; import uk.ac.ic.wlgitbridge.bridge.resource.ResourceCache; -import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotAPI; +import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApiFacade; import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJob; import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore; -import uk.ac.ic.wlgitbridge.data.model.Snapshot; import uk.ac.ic.wlgitbridge.git.exception.GitUserException; import java.io.IOException; import java.util.ArrayDeque; +import java.util.Optional; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyInt; @@ -34,7 +34,7 @@ public class BridgeTest { private RepoStore repoStore; private DBStore dbStore; private SwapStore swapStore; - private SnapshotAPI snapshotAPI; + private SnapshotApiFacade snapshotAPI; private ResourceCache resourceCache; private SwapJob swapJob; private GcJob gcJob; @@ -45,7 +45,7 @@ public class BridgeTest { repoStore = mock(RepoStore.class); dbStore = mock(DBStore.class); swapStore = mock(SwapStore.class); - snapshotAPI = mock(SnapshotAPI.class); + snapshotAPI = mock(SnapshotApiFacade.class); resourceCache = mock(ResourceCache.class); swapJob = mock(SwapJob.class); gcJob = mock(GcJob.class); @@ -75,17 +75,18 @@ public class BridgeTest { public void updatingRepositorySetsLastAccessedTime( ) throws IOException, GitUserException { ProjectRepo repo = mock(ProjectRepo.class); - when(repo.getProjectName()).thenReturn("asdf"); + when(repoStore.getExistingRepo("asdf")).thenReturn(repo); when(dbStore.getProjectState("asdf")).thenReturn(ProjectState.PRESENT); + when(snapshotAPI.projectExists(Optional.empty(), "asdf")).thenReturn(true); when( - snapshotAPI.getSnapshotsForProjectAfterVersion( + snapshotAPI.getSnapshots( any(), any(), anyInt() ) - ).thenReturn(new ArrayDeque()); - bridge.updateRepository(null, repo); + ).thenReturn(new ArrayDeque<>()); + bridge.getUpdatedRepo(Optional.empty(), "asdf"); verify(dbStore).setLastAccessedTime(eq("asdf"), any()); } -} \ No newline at end of file +}