From 3baa74876c0925a86ba3bdcdffe9b6c03310cc60 Mon Sep 17 00:00:00 2001 From: minux Date: Thu, 27 Oct 2022 12:57:00 +0900 Subject: [PATCH] Support local-to-remote mirroring (#717) Motivation We currently support only remote-to-local mirroring. We should support the opposite direction as well. Modifications: - Implement `GitMirror#mirrorToLocalRemote()` which used to throw `UnsupportedOperationException` before. Result: - Close #53 - You can now enable mirroring from Central Dogma to a remote Git server. --- .../it/mirror/git/GitMirrorTest.java | 13 +- .../git/LocalToRemoteGitMirrorTest.java | 526 ++++++++++++++++++ .../mirror/DefaultMirroringService.java | 4 +- .../server/internal/mirror/GitMirror.java | 466 ++++++++++++++-- .../server/internal/mirror/MirrorState.java | 7 +- .../server/mirror/MirrorUtil.java | 2 +- site/src/sphinx/concepts.rst | 2 +- site/src/sphinx/mirroring.rst | 5 + site/src/sphinx/setup-configuration.rst | 2 +- .../testing/internal/TestUtil.java | 3 +- 10 files changed, 967 insertions(+), 63 deletions(-) create mode 100644 it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java diff --git a/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorTest.java b/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorTest.java index 00fda5c962..c27ef8fb86 100644 --- a/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorTest.java +++ b/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorTest.java @@ -426,6 +426,8 @@ private void pushMirrorSettings(@Nullable String localPath, @Nullable String rem private void pushMirrorSettings(String localRepo, @Nullable String localPath, @Nullable String remotePath, @Nullable String gitignore) { + final String localPath0 = localPath == null ? "/" : localPath; + final String remoteUri = gitUri + firstNonNull(remotePath, ""); client.forRepo(projName, Project.REPO_META) .commit("Add /mirrors.json", Change.ofJsonUpsert("/mirrors.json", @@ -433,9 +435,10 @@ private void pushMirrorSettings(String localRepo, @Nullable String localPath, @N " \"type\": \"single\"," + " \"direction\": \"REMOTE_TO_LOCAL\"," + " \"localRepo\": \"" + localRepo + "\"," + - (localPath != null ? "\"localPath\": \"" + localPath + "\"," : "") + - " \"remoteUri\": \"" + gitUri + firstNonNull(remotePath, "") + '"' + - ",\"gitignore\": " + firstNonNull(gitignore, "\"\"") + + " \"localPath\": \"" + localPath0 + "\"," + + " \"remoteUri\": \"" + remoteUri + "\"," + + " \"schedule\": \"0 0 0 1 1 ? 2099\"," + + " \"gitignore\": " + firstNonNull(gitignore, "\"\"") + "}]")) .push().join(); } @@ -453,8 +456,8 @@ private void addToGitIndex(String path, String content) throws IOException, GitA addToGitIndex(git, gitWorkTree, path, content); } - private static void addToGitIndex(Git git, File gitWorkTree, - String path, String content) throws IOException, GitAPIException { + static void addToGitIndex(Git git, File gitWorkTree, + String path, String content) throws IOException, GitAPIException { final File file = Paths.get(gitWorkTree.getAbsolutePath(), path.split("/")).toFile(); file.getParentFile().mkdirs(); Files.asCharSink(file, StandardCharsets.UTF_8).write(content); diff --git a/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java b/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java new file mode 100644 index 0000000000..90d727ab26 --- /dev/null +++ b/it/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java @@ -0,0 +1,526 @@ +/* + * Copyright 2022 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.it.mirror.git; + +import static com.google.common.base.MoreObjects.firstNonNull; +import static com.linecorp.centraldogma.it.mirror.git.GitMirrorTest.addToGitIndex; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_COMMIT_SECTION; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_GPGSIGN; +import static org.eclipse.jgit.lib.Constants.R_HEADS; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import javax.annotation.Nullable; + +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.lib.FileMode; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectReader; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.StoredConfig; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.storage.file.FileRepositoryBuilder; +import org.eclipse.jgit.treewalk.TreeWalk; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import com.google.common.base.Strings; + +import com.linecorp.centraldogma.client.CentralDogma; +import com.linecorp.centraldogma.common.CentralDogmaException; +import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.Entry; +import com.linecorp.centraldogma.common.PathPattern; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.internal.Jackson; +import com.linecorp.centraldogma.server.CentralDogmaBuilder; +import com.linecorp.centraldogma.server.MirrorException; +import com.linecorp.centraldogma.server.MirroringService; +import com.linecorp.centraldogma.server.internal.mirror.MirrorState; +import com.linecorp.centraldogma.server.mirror.MirrorDirection; +import com.linecorp.centraldogma.server.storage.project.Project; +import com.linecorp.centraldogma.testing.internal.TemporaryFolderExtension; +import com.linecorp.centraldogma.testing.internal.TestUtil; +import com.linecorp.centraldogma.testing.junit.CentralDogmaExtension; + +class LocalToRemoteGitMirrorTest { + + private static final String LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME = ".mirror_state.json"; + + private static final int MAX_NUM_FILES = 32; + private static final long MAX_NUM_BYTES = 1048576; // 1 MiB + + private static final String REPO_FOO = "foo"; + + @RegisterExtension + static final CentralDogmaExtension dogma = new CentralDogmaExtension() { + @Override + protected void configure(CentralDogmaBuilder builder) { + builder.mirroringEnabled(true); + builder.maxNumFilesPerMirror(MAX_NUM_FILES); + builder.maxNumBytesPerMirror(MAX_NUM_BYTES); + } + }; + + private static CentralDogma client; + private static MirroringService mirroringService; + + @BeforeAll + static void init() { + client = dogma.client(); + mirroringService = dogma.mirroringService(); + } + + @RegisterExtension + final TemporaryFolderExtension gitRepoDir = new TemporaryFolderExtension() { + @Override + protected boolean runForEachTest() { + return true; + } + }; + + private Git git; + private File gitWorkTree; + private String gitUri; + + private String projName; + + @BeforeEach + void initGitRepo(TestInfo testInfo) throws Exception { + final String repoName = TestUtil.normalizedDisplayName(testInfo); + gitWorkTree = new File(gitRepoDir.getRoot().toFile(), repoName).getAbsoluteFile(); + final Repository gitRepo = new FileRepositoryBuilder().setWorkTree(gitWorkTree).build(); + createGitRepo(gitRepo); + + git = Git.wrap(gitRepo); + gitUri = "git+file://" + + (gitWorkTree.getPath().startsWith(File.separator) ? "" : '/') + + gitWorkTree.getPath().replace(File.separatorChar, '/') + + "/.git"; + // Start the master branch with an empty commit. + git.commit().setMessage("Initial commit").call(); + } + + private static void createGitRepo(Repository gitRepo) throws IOException { + gitRepo.create(); + + // Disable GPG signing. + final StoredConfig config = gitRepo.getConfig(); + config.setBoolean(CONFIG_COMMIT_SECTION, null, CONFIG_KEY_GPGSIGN, false); + config.save(); + } + + @BeforeEach + void initDogmaRepo(TestInfo testInfo) { + projName = TestUtil.normalizedDisplayName(testInfo); + client.createProject(projName).join(); + client.createRepository(projName, REPO_FOO).join(); + } + + @AfterEach + void destroyDogmaRepo() throws IOException { + client.removeProject(projName).join(); + } + + @ParameterizedTest + @CsvSource({ + "'', ''", + "/local/foo, /remote", + "/local, /remote/bar", + "/local/foo, /remote/bar" + }) + void localToRemote(String localPath, String remotePath) throws Exception { + pushMirrorSettings(localPath, remotePath, null); + + final ObjectId commitId = git.getRepository().exactRef(R_HEADS + "master").getObjectId(); + assertThat(getFileContent(commitId, remotePath + '/' + LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME)) + .isNull(); + // Mirror an empty Central Dogma repository, which will; + // - Create /.mirror_state.json + mirroringService.mirror().join(); + + final ObjectId commitId1 = git.getRepository().exactRef(R_HEADS + "master").getObjectId(); + assertThat(commitId).isNotEqualTo(commitId1); + byte[] content = getFileContent(commitId1, remotePath + '/' + LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME); + MirrorState mirrorState = Jackson.readValue(content, MirrorState.class); + assertThat(mirrorState.sourceRevision()).isEqualTo("1"); + + // Mirror once again without adding a commit. + mirroringService.mirror().join(); + + // Make sure no commit was added thus the source revision wasn't changed. + final ObjectId commitId2 = git.getRepository().exactRef(R_HEADS + "master").getObjectId(); + assertThat(commitId2).isEqualTo(commitId1); + content = getFileContent(commitId2, remotePath + '/' + LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME); + mirrorState = Jackson.readValue(content, MirrorState.class); + assertThat(mirrorState.sourceRevision()).isEqualTo("1"); + + // Create a new commit + client.forRepo(projName, REPO_FOO) + .commit("Add a commit", + Change.ofJsonUpsert(localPath + "/foo.json", "{\"a\":\"b\"}"), + Change.ofJsonUpsert(localPath + "/bar/foo.json", "{\"a\":\"c\"}"), + Change.ofTextUpsert(localPath + "/baz/foo.txt", "\"a\": \"b\"\n")) + .push().join(); + + mirroringService.mirror().join(); + final ObjectId commitId3 = git.getRepository().exactRef(R_HEADS + "master").getObjectId(); + assertThat(commitId3).isNotEqualTo(commitId2); + content = getFileContent(commitId3, remotePath + '/' + LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME); + mirrorState = Jackson.readValue(content, MirrorState.class); + assertThat(mirrorState.sourceRevision()).isEqualTo("2"); + assertThat(Jackson.writeValueAsString(Jackson.readTree( + getFileContent(commitId3, remotePath + "/foo.json")))).isEqualTo("{\"a\":\"b\"}"); + assertThat(Jackson.writeValueAsString(Jackson.readTree( + getFileContent(commitId3, remotePath + "/bar/foo.json")))).isEqualTo("{\"a\":\"c\"}"); + assertThat(new String(getFileContent(commitId3, remotePath + "/baz/foo.txt"))) + .isEqualTo("\"a\": \"b\"\n"); + + // Mirror once again without adding a commit. + mirroringService.mirror().join(); + + // Make sure no commit was added thus the source revision wasn't changed. + final ObjectId commitId4 = git.getRepository().exactRef(R_HEADS + "master").getObjectId(); + assertThat(commitId4).isEqualTo(commitId3); + content = getFileContent(commitId4, remotePath + '/' + LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME); + mirrorState = Jackson.readValue(content, MirrorState.class); + assertThat(mirrorState.sourceRevision()).isEqualTo("2"); + + // Create a new commit + client.forRepo(projName, REPO_FOO) + .commit("Remove foo.json and foo.txt", + Change.ofRemoval(localPath + "/foo.json"), + Change.ofRemoval(localPath + "/baz/foo.txt")) + .push().join(); + + mirroringService.mirror().join(); + final ObjectId commitId5 = git.getRepository().exactRef(R_HEADS + "master").getObjectId(); + assertThat(commitId5).isNotEqualTo(commitId4); + content = getFileContent(commitId5, remotePath + '/' + LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME); + mirrorState = Jackson.readValue(content, MirrorState.class); + assertThat(mirrorState.sourceRevision()).isEqualTo("3"); + assertThat(getFileContent(commitId5, remotePath + "/foo.json")).isNull(); + assertThat(getFileContent(commitId5, remotePath + "/baz/foo.txt")).isNull(); + assertThat(Jackson.writeValueAsString(Jackson.readTree( + getFileContent(commitId5, remotePath + "/bar/foo.json")))).isEqualTo("{\"a\":\"c\"}"); + + addToGitIndex(git, gitWorkTree, (remotePath + "/bar/foo.json").substring(1), "{\"a\":\"d\"}"); + git.commit().setMessage("Change the file arbitrarily").call(); + final ObjectId commitId6 = git.getRepository().exactRef(R_HEADS + "master").getObjectId(); + assertThat(Jackson.writeValueAsString(Jackson.readTree( + getFileContent(commitId6, remotePath + "/bar/foo.json")))).isEqualTo("{\"a\":\"d\"}"); + + mirroringService.mirror().join(); + final ObjectId commitId7 = git.getRepository().exactRef(R_HEADS + "master").getObjectId(); + assertThat(commitId7).isNotEqualTo(commitId6); + content = getFileContent(commitId7, remotePath + '/' + LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME); + mirrorState = Jackson.readValue(content, MirrorState.class); + assertThat(mirrorState.sourceRevision()).isEqualTo("3"); + // The arbitrarily changed file is overwritten. + assertThat(Jackson.writeValueAsString(Jackson.readTree( + getFileContent(commitId7, remotePath + "/bar/foo.json")))).isEqualTo("{\"a\":\"c\"}"); + } + + @Nullable + private byte[] getFileContent(ObjectId commitId, String fileName) throws IOException { + try (ObjectReader reader = git.getRepository().newObjectReader(); + TreeWalk treeWalk = new TreeWalk(reader); + RevWalk revWalk = new RevWalk(reader)) { + treeWalk.addTree(revWalk.parseTree(commitId).getId()); + + while (treeWalk.next()) { + if (treeWalk.getFileMode() == FileMode.TREE) { + treeWalk.enterSubtree(); + continue; + } + if (fileName.equals('/' + treeWalk.getPathString())) { + final ObjectId objectId = treeWalk.getObjectId(0); + return reader.open(objectId).getBytes(); + } + } + } + return null; + } + + @ParameterizedTest + @CsvSource({ + "'', ''", + "/local/foo, /remote", + "/local, /remote/bar", + "/local/foo, /remote/bar" + }) + void localToRemote_gitignore(String localPath, String remotePath) throws Exception { + pushMirrorSettings(localPath, remotePath, "\"/exclude_if_root.txt\\n**/exclude_dir\""); + checkGitignore(localPath, remotePath); + } + + @ParameterizedTest + @CsvSource({ + "'', ''", + "/local/foo, /remote", + "/local, /remote/bar", + "/local/foo, /remote/bar" + }) + void localToRemote_gitignore_with_array(String localPath, String remotePath) throws Exception { + pushMirrorSettings(localPath, remotePath, "[\"/exclude_if_root.txt\", \"exclude_dir\"]"); + checkGitignore(localPath, remotePath); + } + + @Test + void localToRemote_subdirectory() throws Exception { + pushMirrorSettings("/source/main", "/target", null); + + client.forRepo(projName, REPO_FOO) + .commit("Add a file that's not part of mirror", Change.ofTextUpsert("/not_mirrored.txt", "")) + .push().join(); + + // Mirror an empty git repository, which will; + // - Create /target/mirror_state.json + mirroringService.mirror().join(); + + final ObjectId commitId = git.getRepository().exactRef(R_HEADS + "master").getObjectId(); + byte[] content = getFileContent(commitId, "/target/" + LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME); + MirrorState mirrorState = Jackson.readValue(content, MirrorState.class); + assertThat(mirrorState.sourceRevision()).isEqualTo("2"); + + Set files = listFiles(commitId); + assertThat(files.size()).isOne(); // mirror state file. + + // Now, add some files to the git repository and mirror. + // Note that the files not under '/source' should not be mirrored. + client.forRepo(projName, REPO_FOO) + .commit("Add the release dates of the 'Infamous' series", + Change.ofTextUpsert("/source/main/first/light.txt", "26-Aug-2014"), // mirrored + Change.ofJsonUpsert("/second/son.json", "{\"release\": \"21-Mar-2014\"}")) // not mirrored + .push().join(); + mirroringService.mirror().join(); + + final ObjectId commitId1 = git.getRepository().exactRef(R_HEADS + "master").getObjectId(); + content = getFileContent(commitId1, "/target/" + LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME); + mirrorState = Jackson.readValue(content, MirrorState.class); + assertThat(mirrorState.sourceRevision()).isEqualTo("3"); + + files = listFiles(commitId1); + assertThat(files.size()).isSameAs(2); // mirror state file and target/first/light.txt + // Make sure 'target/first/light.txt' is mirrored. + assertThat(new String(getFileContent(commitId1, "/target/first/light.txt"))) + .isEqualTo("26-Aug-2014\n"); + } + + @Test + void localToRemote_tooManyFiles() throws Exception { + pushMirrorSettings(null, null, null); + + // Add more than allowed number of filed. + final ArrayList> changes = new ArrayList<>(); + for (int i = 0; i <= MAX_NUM_FILES; i++) { + changes.add(Change.ofTextUpsert("/" + i + ".txt", String.valueOf(i))); + } + client.forRepo(projName, REPO_FOO).commit("Add a bunch of numbered files", changes).push().join(); + + // Perform mirroring, which should fail. + assertThatThrownBy(() -> mirroringService.mirror().join()) + .hasCauseInstanceOf(MirrorException.class) + .hasMessageContaining("contains more than") + .hasMessageContaining("file"); + } + + @Test + void localToRemote_tooManyBytes() throws Exception { + pushMirrorSettings(null, null, null); + + // Add files whose total size exceeds the allowed maximum. + long remainder = MAX_NUM_BYTES + 1; + final int defaultFileSize = (int) (MAX_NUM_BYTES / MAX_NUM_FILES * 2); + final ArrayList> changes = new ArrayList<>(); + for (int i = 0;; i++) { + final int fileSize; + if (remainder > defaultFileSize) { + remainder -= defaultFileSize; + fileSize = defaultFileSize; + } else { + fileSize = (int) remainder; + remainder = 0; + } + + changes.add(Change.ofTextUpsert("/" + i + ".txt", Strings.repeat("*", fileSize))); + + if (remainder == 0) { + break; + } + } + client.forRepo(projName, REPO_FOO).commit("Add a bunch of numbered asterisks", changes).push().join(); + + // Perform mirroring, which should fail. + assertThatThrownBy(() -> mirroringService.mirror().join()) + .hasCauseInstanceOf(MirrorException.class) + .hasMessageContaining("contains more than") + .hasMessageContaining("byte"); + } + + @CsvSource({ "meta", "dogma" }) + @ParameterizedTest + void cannotMirrorInternalRepositories(String localRepo) { + assertThatThrownBy(() -> pushMirrorSettings(localRepo, "/", "/", null, MirrorDirection.LOCAL_TO_REMOTE)) + .hasCauseInstanceOf(CentralDogmaException.class) + .hasMessageContaining("invalid localRepo:"); + } + + private void pushMirrorSettings(@Nullable String localPath, @Nullable String remotePath, + @Nullable String gitignore) { + pushMirrorSettings(REPO_FOO, localPath, remotePath, gitignore, MirrorDirection.LOCAL_TO_REMOTE); + } + + private void pushMirrorSettings(String localRepo, @Nullable String localPath, @Nullable String remotePath, + @Nullable String gitignore, MirrorDirection direction) { + final String localPath0 = localPath == null ? "/" : localPath; + final String remoteUri = gitUri + firstNonNull(remotePath, ""); + client.forRepo(projName, Project.REPO_META) + .commit("Add /mirrors.json", + Change.ofJsonUpsert("/mirrors.json", + "[{" + + " \"type\": \"single\"," + + " \"direction\": \"" + direction + "\"," + + " \"localRepo\": \"" + localRepo + "\"," + + " \"localPath\": \"" + localPath0 + "\"," + + " \"remoteUri\": \"" + remoteUri + "\"," + + " \"schedule\": \"0 0 0 1 1 ? 2099\"," + + " \"gitignore\": " + firstNonNull(gitignore, "\"\"") + + "}]")) + .push().join(); + } + + private Set listFiles(ObjectId commitId) throws IOException { + try (ObjectReader reader = git.getRepository().newObjectReader(); + TreeWalk treeWalk = new TreeWalk(reader); + RevWalk revWalk = new RevWalk(reader)) { + treeWalk.addTree(revWalk.parseTree(commitId).getId()); + + final HashSet files = new HashSet<>(); + while (treeWalk.next()) { + if (treeWalk.getFileMode() == FileMode.TREE) { + treeWalk.enterSubtree(); + continue; + } + files.add('/' + treeWalk.getPathString()); + } + return files; + } + } + + private void checkGitignore(String localPath, String remotePath) throws IOException, GitAPIException { + // Mirror an empty git repository, which will; + // - Create /mirror_state.json + mirroringService.mirror().join(); + + // Make sure /mirror_state.json exists + final ObjectId commitId = git.getRepository().exactRef(R_HEADS + "master").getObjectId(); + byte[] content = getFileContent(commitId, remotePath + '/' + LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME); + MirrorState mirrorState = Jackson.readValue(content, MirrorState.class); + assertThat(mirrorState.sourceRevision()).isEqualTo("1"); + + // Now, add files to the local repository and mirror. + client.forRepo(projName, REPO_FOO) + .commit("Add the release dates of the 'Infamous' series", + Change.ofTextUpsert(localPath + "/light.txt", "26-Aug-2014"), + Change.ofTextUpsert(localPath + "/exclude_if_root.txt", "26-Aug-2014"), // excluded + Change.ofTextUpsert(localPath + "/subdir/exclude_if_root.txt", "26-Aug-2014"), + Change.ofTextUpsert(localPath + "/subdir/exclude_dir/foo.txt", "26-Aug-2014")) // excluded + .push().join(); + + mirroringService.mirror().join(); + + final ObjectId commitId1 = git.getRepository().exactRef(R_HEADS + "master").getObjectId(); + assertThat(commitId1).isNotEqualTo(commitId); + content = getFileContent(commitId1, remotePath + '/' + LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME); + mirrorState = Jackson.readValue(content, MirrorState.class); + assertThat(mirrorState.sourceRevision()).isEqualTo("2"); + // Remove first directory because it's localPath(). + assertThat(new String(getFileContent(commitId1, remotePath + "/light.txt"))).isEqualTo("26-Aug-2014\n"); + assertThat(new String(getFileContent(commitId1, remotePath + "/subdir/exclude_if_root.txt"))) + .isEqualTo("26-Aug-2014\n"); + + // Make sure the files that match gitignore are not mirrored. + assertThat(getFileContent(commitId1, remotePath + "/exclude_if_root.txt")).isNull(); + assertThat(getFileContent(commitId1, remotePath + "/subdir/exclude_dir/foo.txt")).isNull(); + } + + @Test + void changeDirection() throws Exception { + pushMirrorSettings(null, null, null); + + // Mirror an empty Central Dogma repository, which will; + // - Create /.mirror_state.json + mirroringService.mirror().join(); + + final ObjectId commitId1 = git.getRepository().exactRef(R_HEADS + "master").getObjectId(); + byte[] content = getFileContent(commitId1, '/' + LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME); + MirrorState mirrorState = Jackson.readValue(content, MirrorState.class); + assertThat(mirrorState.sourceRevision()).isEqualTo("1"); + + // Create a new commit + client.forRepo(projName, REPO_FOO) + .commit("Add a commit", + Change.ofJsonUpsert("/foo.json", "{\"a\":\"b\"}"), + Change.ofJsonUpsert("/bar/foo.json", "{\"a\":\"c\"}"), + Change.ofTextUpsert("/baz/foo.txt", "\"a\": \"b\"\n")) + .push().join(); + + mirroringService.mirror().join(); + + final ObjectId commitId2 = git.getRepository().exactRef(R_HEADS + "master").getObjectId(); + content = getFileContent(commitId2, '/' + LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME); + mirrorState = Jackson.readValue(content, MirrorState.class); + assertThat(mirrorState.sourceRevision()).isEqualTo("2"); + assertThat(Jackson.writeValueAsString(Jackson.readTree( + getFileContent(commitId2, "/foo.json")))).isEqualTo("{\"a\":\"b\"}"); + assertThat(Jackson.writeValueAsString(Jackson.readTree( + getFileContent(commitId2, "/bar/foo.json")))).isEqualTo("{\"a\":\"c\"}"); + assertThat(new String(getFileContent(commitId2, "/baz/foo.txt"))) + .isEqualTo("\"a\": \"b\"\n"); + + // Change the direction + pushMirrorSettings(REPO_FOO, null, null, null, MirrorDirection.REMOTE_TO_LOCAL); + addToGitIndex(git, gitWorkTree, "foo.json", "{\"a\":\"foo\"}"); + git.commit().setMessage("Modify foo.json").call(); + mirroringService.mirror().join(); + + final Map> entries = client.forRepo(projName, REPO_FOO) + .file(PathPattern.all()) + .get() + .join(); + assertThat(entries.size()).isEqualTo(2); + assertThat(entries.get("/foo.json")).isEqualTo( + Entry.ofJson(new Revision(3), "/foo.json", "{\"a\":\"foo\"}")); + + assertThat(entries.get("/mirror_state.json")).isNotNull(); + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringService.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringService.java index 0cce23add5..7860426a88 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringService.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringService.java @@ -138,7 +138,7 @@ public void onSuccess(@Nullable Object result) {} @Override public void onFailure(Throwable cause) { - logger.error("Git-to-CD mirroring scheduler stopped due to an unexpected exception:", cause); + logger.error("Git mirroring scheduler stopped due to an unexpected exception:", cause); } }, MoreExecutors.directExecutor()); } @@ -207,7 +207,7 @@ public void onSuccess(@Nullable Object result) {} @Override public void onFailure(Throwable cause) { - logger.warn("Unexpected Git-to-CD mirroring failure: {}", m, cause); + logger.warn("Unexpected Git mirroring failure: {}", m, cause); } }, MoreExecutors.directExecutor()); }); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/GitMirror.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/GitMirror.java index 54c488c787..cc1659899c 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/GitMirror.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/GitMirror.java @@ -18,15 +18,19 @@ import static com.linecorp.centraldogma.server.mirror.MirrorSchemes.SCHEME_GIT_SSH; import static com.linecorp.centraldogma.server.storage.repository.FindOptions.FIND_ALL_WITHOUT_CONTENT; +import static java.nio.charset.StandardCharsets.UTF_8; import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.net.URI; -import java.nio.charset.StandardCharsets; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; +import java.util.Objects; import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.annotation.Nullable; @@ -34,13 +38,24 @@ import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.RemoteSetUrlCommand; import org.eclipse.jgit.api.RemoteSetUrlCommand.UriType; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.dircache.DirCache; +import org.eclipse.jgit.dircache.DirCacheBuilder; +import org.eclipse.jgit.dircache.DirCacheEditor; +import org.eclipse.jgit.dircache.DirCacheEditor.DeletePath; +import org.eclipse.jgit.dircache.DirCacheEditor.PathEdit; +import org.eclipse.jgit.dircache.DirCacheEntry; import org.eclipse.jgit.ignore.IgnoreNode; import org.eclipse.jgit.ignore.IgnoreNode.MatchResult; +import org.eclipse.jgit.lib.CommitBuilder; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectInserter; import org.eclipse.jgit.lib.ObjectReader; +import org.eclipse.jgit.lib.PersonIdent; import org.eclipse.jgit.lib.RefUpdate; +import org.eclipse.jgit.lib.RefUpdate.Result; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.transport.FetchResult; import org.eclipse.jgit.transport.RefSpec; @@ -51,6 +66,7 @@ import org.slf4j.LoggerFactory; import com.cronutils.model.Cron; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.TreeNode; import com.fasterxml.jackson.databind.JsonNode; @@ -68,12 +84,23 @@ import com.linecorp.centraldogma.server.internal.mirror.credential.PublicKeyMirrorCredential; import com.linecorp.centraldogma.server.mirror.MirrorCredential; import com.linecorp.centraldogma.server.mirror.MirrorDirection; +import com.linecorp.centraldogma.server.storage.StorageException; import com.linecorp.centraldogma.server.storage.repository.Repository; public final class GitMirror extends AbstractMirror { private static final Logger logger = LoggerFactory.getLogger(GitMirror.class); + // We are going to hide this file from CD UI after we implement UI for mirroring. + private static final String MIRROR_STATE_FILE_NAME = "mirror_state.json"; + + // Prepend '.' because this file is metadata. + private static final String LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME = '.' + MIRROR_STATE_FILE_NAME; + + private static final Pattern CR = Pattern.compile("\r", Pattern.LITERAL); + + private static final byte[] EMPTY_BYTE = new byte[0]; + private static final Pattern DISALLOWED_CHARS = Pattern.compile("[^-_a-zA-Z]"); private static final Pattern CONSECUTIVE_UNDERSCORES = Pattern.compile("_+"); @@ -101,7 +128,62 @@ public GitMirror(Cron schedule, MirrorDirection direction, MirrorCredential cred @Override protected void mirrorLocalToRemote(File workDir, int maxNumFiles, long maxNumBytes) throws Exception { - throw new UnsupportedOperationException(); + try (Git git = openGit(workDir)) { + final String headBranchRefName = Constants.R_HEADS + remoteBranch(); + final ObjectId headCommitId = fetchRemoteHeadAndGetCommitId(git, headBranchRefName); + + final org.eclipse.jgit.lib.Repository gitRepository = git.getRepository(); + try (ObjectReader reader = gitRepository.newObjectReader(); + TreeWalk treeWalk = new TreeWalk(reader); + RevWalk revWalk = new RevWalk(reader)) { + + // Prepare to traverse the tree. We can get the tree ID by parsing the object ID. + final ObjectId headTreeId = revWalk.parseTree(headCommitId).getId(); + treeWalk.reset(headTreeId); + + final String mirrorStatePath = remotePath() + LOCAL_TO_REMOTE_MIRROR_STATE_FILE_NAME; + final Revision localHead = localRepo().normalizeNow(Revision.HEAD); + final Revision remoteCurrentRevision = remoteCurrentRevision(reader, treeWalk, mirrorStatePath); + if (localHead.equals(remoteCurrentRevision)) { + // The remote repository is up-to date. + logger.debug("The remote repository '{}#{}' already at {}. Local repository: '{}'", + remoteRepoUri(), remoteBranch(), localHead, localRepo().name()); + return; + } + + // Reset to traverse the tree from the first. + treeWalk.reset(headTreeId); + + // The staging area that keeps the entries of the new tree. + // It starts with the entries of the tree at the current head and then this method will apply + // the requested changes to build the new tree. + final DirCache dirCache = DirCache.newInCore(); + final DirCacheBuilder builder = dirCache.builder(); + builder.addTree(EMPTY_BYTE, 0, reader, headTreeId); + builder.finish(); + + try (ObjectInserter inserter = gitRepository.newObjectInserter()) { + addModifiedEntryToCache(localHead, dirCache, reader, inserter, + treeWalk, maxNumFiles, maxNumBytes); + // Add the mirror state file. + final MirrorState mirrorState = new MirrorState(localHead.text()); + applyPathEdit( + dirCache, new InsertText(mirrorStatePath.substring(1), // Strip the leading '/'. + inserter, + Jackson.writeValueAsPrettyString(mirrorState) + '\n')); + } + + final ObjectId nextCommitId = + commit(gitRepository, dirCache, headCommitId, localHead); + updateRef(gitRepository, revWalk, headBranchRefName, nextCommitId); + + git.push() + .setRefSpecs(new RefSpec(headBranchRefName)) + .setAtomic(true) + .setTimeout(GIT_TIMEOUT_SECS) + .call(); + } + } } @Override @@ -112,20 +194,8 @@ protected void mirrorRemoteToLocal(File workDir, CommandExecutor executor, final String summary; try (Git git = openGit(workDir)) { - final FetchCommand fetch = git.fetch(); - final String refName = Constants.R_HEADS + remoteBranch(); - final FetchResult fetchResult = fetch.setRefSpecs(new RefSpec(refName)) - .setCheckFetchedObjects(true) - .setRemoveDeletedRefs(true) - .setTagOpt(TagOpt.NO_TAGS) - .setTimeout(GIT_TIMEOUT_SECS) - .call(); - - final ObjectId id = fetchResult.getAdvertisedRef(refName).getObjectId(); - final RefUpdate refUpdate = git.getRepository().updateRef(refName); - refUpdate.setNewObjectId(id); - refUpdate.update(); - + final String headBranchRefName = Constants.R_HEADS + remoteBranch(); + final ObjectId id = fetchRemoteHeadAndGetCommitId(git, headBranchRefName); final Revision localRev = localRepo().normalizeNow(Revision.HEAD); try (ObjectReader reader = git.getRepository().newObjectReader(); @@ -136,7 +206,7 @@ protected void mirrorRemoteToLocal(File workDir, CommandExecutor executor, treeWalk.addTree(revWalk.parseTree(id).getId()); // Check if local repository needs update. - final String mirrorStatePath = localPath() + "mirror_state.json"; + final String mirrorStatePath = localPath() + MIRROR_STATE_FILE_NAME; final Entry mirrorState = localRepo().getOrNull(localRev, mirrorStatePath).join(); final String localSourceRevision; if (mirrorState == null || mirrorState.type() != EntryType.JSON) { @@ -175,37 +245,8 @@ protected void mirrorRemoteToLocal(File workDir, CommandExecutor executor, continue; } - // Recurse into a directory if necessary. if (fileMode == FileMode.TREE) { - // Enter if the directory is under remotePath. - // e.g. - // path == /foo/bar - // remotePath == /foo/ - if (path.startsWith(remotePath())) { - treeWalk.enterSubtree(); - continue; - } - - // Enter if the directory is equal to remotePath. - // e.g. - // path == /foo - // remotePath == /foo/ - final int pathLen = path.length() + 1; // Include the trailing '/'. - if (pathLen == remotePath().length() && remotePath().startsWith(path)) { - treeWalk.enterSubtree(); - continue; - } - - // Enter if the directory is parent of remotePath. - // e.g. - // path == /foo - // remotePath == /foo/bar/ - if (pathLen < remotePath().length() && remotePath().startsWith(path + '/')) { - treeWalk.enterSubtree(); - continue; - } - - // Skip the directory that are not under the remote path. + maybeEnterSubtree(treeWalk, remotePath(), path); continue; } @@ -227,13 +268,15 @@ protected void mirrorRemoteToLocal(File workDir, CommandExecutor executor, } if (++numFiles > maxNumFiles) { - throw new MirrorException("mirror contains more than " + maxNumFiles + " file(s)"); + throwMirrorException(maxNumFiles, "files"); + return; } final ObjectId objectId = treeWalk.getObjectId(0); final long contentLength = reader.getObjectSize(objectId, ObjectReader.OBJ_ANY); if (numBytes > maxNumBytes - contentLength) { - throw new MirrorException("mirror contains more than " + maxNumBytes + " byte(s)"); + throwMirrorException(maxNumBytes, "bytes"); + return; } numBytes += contentLength; @@ -244,7 +287,7 @@ protected void mirrorRemoteToLocal(File workDir, CommandExecutor executor, changes.putIfAbsent(localPath, Change.ofJsonUpsert(localPath, jsonNode)); break; case TEXT: - final String strVal = new String(content, StandardCharsets.UTF_8); + final String strVal = new String(content, UTF_8); changes.putIfAbsent(localPath, Change.ofTextUpsert(localPath, strVal)); break; } @@ -333,4 +376,327 @@ private Git openGit(File workDir) throws Exception { } } } + + @Nullable + private Revision remoteCurrentRevision( + ObjectReader reader, TreeWalk treeWalk, String mirrorStatePath) { + try { + while (treeWalk.next()) { + final FileMode fileMode = treeWalk.getFileMode(); + final String path = '/' + treeWalk.getPathString(); + + // Recurse into a directory if necessary. + if (fileMode == FileMode.TREE) { + if (remotePath().startsWith(path + '/')) { + treeWalk.enterSubtree(); + } + continue; + } + + if (!path.equals(mirrorStatePath)) { + continue; + } + + final byte[] content = currentEntryContent(reader, treeWalk); + final MirrorState mirrorState = Jackson.readValue(content, MirrorState.class); + return new Revision(mirrorState.sourceRevision()); + } + // There's no mirror state file which means this is the first mirroring or the file is removed. + return null; + } catch (Exception e) { + logger.warn("Unexpected exception while retrieving the remote source revision", e); + return null; + } + } + + private static ObjectId fetchRemoteHeadAndGetCommitId( + Git git, String headBranchRefName) throws GitAPIException, IOException { + final FetchCommand fetch = git.fetch(); + final FetchResult fetchResult = fetch.setRefSpecs(new RefSpec(headBranchRefName)) + .setCheckFetchedObjects(true) + .setRemoveDeletedRefs(true) + .setTagOpt(TagOpt.NO_TAGS) + .setTimeout(GIT_TIMEOUT_SECS) + .call(); + final ObjectId commitId = fetchResult.getAdvertisedRef(headBranchRefName).getObjectId(); + final RefUpdate refUpdate = git.getRepository().updateRef(headBranchRefName); + refUpdate.setNewObjectId(commitId); + refUpdate.update(); + return commitId; + } + + private Map> localHeadEntries(Revision localHead) { + final Map> localRawHeadEntries = localRepo().find(localHead, localPath() + "**") + .join(); + + final Stream>> entryStream = + localRawHeadEntries.entrySet() + .stream(); + if (ignoreNode == null) { + // Use HashMap to manipulate it. + return entryStream.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + final Map> sortedMap = + entryStream.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, + (v1, v2) -> v1, LinkedHashMap::new)); + // Use HashMap to manipulate it. + final HashMap> result = new HashMap<>(sortedMap.size()); + String lastIgnoredDirectory = null; + for (Map.Entry> entry : sortedMap.entrySet()) { + final String path = entry.getKey(); + final boolean isDirectory = entry.getValue().type() == EntryType.DIRECTORY; + final MatchResult ignoreResult = ignoreNode.isIgnored( + path.substring(localPath().length()), isDirectory); + if (ignoreResult == MatchResult.IGNORED) { + if (isDirectory) { + lastIgnoredDirectory = path; + } + continue; + } + if (ignoreResult == MatchResult.CHECK_PARENT) { + if (lastIgnoredDirectory != null && path.startsWith(lastIgnoredDirectory)) { + continue; + } + } + result.put(path, entry.getValue()); + } + + return result; + } + + private void addModifiedEntryToCache(Revision localHead, DirCache dirCache, ObjectReader reader, + ObjectInserter inserter, TreeWalk treeWalk, + int maxNumFiles, long maxNumBytes) throws IOException { + final Map> localHeadEntries = localHeadEntries(localHead); + long numFiles = 0; + long numBytes = 0; + while (treeWalk.next()) { + final FileMode fileMode = treeWalk.getFileMode(); + final String pathString = treeWalk.getPathString(); + final String remoteFilePath = '/' + pathString; + + // Recurse into a directory if necessary. + if (fileMode == FileMode.TREE) { + maybeEnterSubtree(treeWalk, remotePath(), remoteFilePath); + continue; + } + + if (fileMode != FileMode.REGULAR_FILE && fileMode != FileMode.EXECUTABLE_FILE) { + // Skip non-file entries. + continue; + } + + // Skip the entries that are not under the remote path. + if (!remoteFilePath.startsWith(remotePath())) { + continue; + } + + final String localFilePath = localPath() + remoteFilePath.substring(remotePath().length()); + + // Skip the entry whose path does not conform to CD's path rule. + if (!Util.isValidFilePath(localFilePath)) { + continue; + } + + final Entry entry = localHeadEntries.remove(localFilePath); + if (entry == null) { + // Remove a deleted entry. + applyPathEdit(dirCache, new DeletePath(pathString)); + continue; + } + + if (++numFiles > maxNumFiles) { + throwMirrorException(maxNumFiles, "files"); + return; + } + + final byte[] oldContent = currentEntryContent(reader, treeWalk); + final long contentLength = applyPathEdit(dirCache, inserter, pathString, entry, oldContent); + numBytes += contentLength; + if (numBytes > maxNumBytes) { + throwMirrorException(maxNumBytes, "bytes"); + return; + } + } + + // Add newly added entries. + for (Map.Entry> entry : localHeadEntries.entrySet()) { + final Entry value = entry.getValue(); + if (value.type() == EntryType.DIRECTORY) { + continue; + } + if (entry.getKey().endsWith(MIRROR_STATE_FILE_NAME)) { + continue; + } + + if (++numFiles > maxNumFiles) { + throwMirrorException(maxNumFiles, "files"); + return; + } + + final String convertedPath = remotePath().substring(1) + // Strip the leading '/' + entry.getKey().substring(localPath().length()); + final long contentLength = applyPathEdit(dirCache, inserter, convertedPath, value, null); + numBytes += contentLength; + if (numBytes > maxNumBytes) { + throwMirrorException(maxNumBytes, "bytes"); + } + } + } + + private static long applyPathEdit(DirCache dirCache, ObjectInserter inserter, String pathString, + Entry entry, @Nullable byte[] oldContent) + throws JsonProcessingException { + switch (EntryType.guessFromPath(pathString)) { + case JSON: + final JsonNode oldJsonNode = oldContent != null ? Jackson.readTree(oldContent) : null; + final JsonNode newJsonNode = (JsonNode) entry.content(); + + // Upsert only when the contents are really different. + if (!Objects.equals(newJsonNode, oldJsonNode)) { + // Use InsertText to store the content in pretty format + final String newContent = newJsonNode.toPrettyString() + '\n'; + applyPathEdit(dirCache, new InsertText(pathString, inserter, newContent)); + return newContent.length(); + } + break; + case TEXT: + final String sanitizedOldText = oldContent != null ? + sanitizeText(new String(oldContent, UTF_8)) : null; + final String sanitizedNewText = entry.contentAsText(); // Already sanitized when committing. + // Upsert only when the contents are really different. + if (!sanitizedNewText.equals(sanitizedOldText)) { + applyPathEdit(dirCache, new InsertText(pathString, inserter, sanitizedNewText)); + return sanitizedNewText.length(); + } + break; + } + return 0; + } + + private static void applyPathEdit(DirCache dirCache, PathEdit edit) { + final DirCacheEditor e = dirCache.editor(); + e.add(edit); + e.finish(); + } + + private static byte[] currentEntryContent(ObjectReader reader, TreeWalk treeWalk) throws IOException { + final ObjectId objectId = treeWalk.getObjectId(0); + return reader.open(objectId).getBytes(); + } + + private static void maybeEnterSubtree( + TreeWalk treeWalk, String remotePath, String path) throws IOException { + // Enter if the directory is under the remote path. + // e.g. + // path == /foo/bar + // remotePath == /foo/ + if (path.startsWith(remotePath)) { + treeWalk.enterSubtree(); + return; + } + + // Enter if the directory is equal to the remote path. + // e.g. + // path == /foo + // remotePath == /foo/ + final int pathLen = path.length() + 1; // Include the trailing '/'. + if (pathLen == remotePath.length() && remotePath.startsWith(path)) { + treeWalk.enterSubtree(); + return; + } + + // Enter if the directory is the parent of the remote path. + // e.g. + // path == /foo + // remotePath == /foo/bar/ + if (pathLen < remotePath.length() && remotePath.startsWith(path + '/')) { + treeWalk.enterSubtree(); + } + } + + /** + * Removes {@code \r} and appends {@code \n} on the last line if it does not end with {@code \n}. + */ + private static String sanitizeText(String text) { + if (text.indexOf('\r') >= 0) { + text = CR.matcher(text).replaceAll(""); + } + if (!text.isEmpty() && !text.endsWith("\n")) { + text += "\n"; + } + return text; + } + + private ObjectId commit(org.eclipse.jgit.lib.Repository gitRepository, DirCache dirCache, + ObjectId headCommitId, Revision localHead) throws IOException { + try (ObjectInserter inserter = gitRepository.newObjectInserter()) { + // flush the current index to repository and get the result tree object id. + final ObjectId nextTreeId = dirCache.writeTree(inserter); + // build a commit object + final PersonIdent personIdent = + new PersonIdent(MIRROR_AUTHOR.name(), MIRROR_AUTHOR.email(), + System.currentTimeMillis() / 1000L * 1000L, // Drop the milliseconds + 0); + + final CommitBuilder commitBuilder = new CommitBuilder(); + commitBuilder.setAuthor(personIdent); + commitBuilder.setCommitter(personIdent); + commitBuilder.setTreeId(nextTreeId); + commitBuilder.setEncoding(UTF_8); + commitBuilder.setParentId(headCommitId); + + final String summary = "Mirror '" + localRepo().name() + "' at " + localHead + + " to the repository '" + remoteRepoUri() + '#' + remoteBranch() + "'\n"; + logger.info(summary); + commitBuilder.setMessage(summary); + + final ObjectId nextCommitId = inserter.insert(commitBuilder); + inserter.flush(); + return nextCommitId; + } + } + + private T throwMirrorException(long number, String filesOrBytes) { + throw new MirrorException("mirror (" + remoteRepoUri() + '#' + remoteBranch() + + ") contains more than " + number + ' ' + filesOrBytes); + } + + static void updateRef(org.eclipse.jgit.lib.Repository jGitRepository, RevWalk revWalk, + String ref, ObjectId commitId) throws IOException { + final RefUpdate refUpdate = jGitRepository.updateRef(ref); + refUpdate.setNewObjectId(commitId); + + final Result res = refUpdate.update(revWalk); + switch (res) { + case NEW: + case FAST_FORWARD: + // Expected + break; + default: + throw new StorageException("unexpected refUpdate state: " + res); + } + } + + private static final class InsertText extends PathEdit { + private final ObjectInserter inserter; + private final String text; + + InsertText(String entryPath, ObjectInserter inserter, String text) { + super(entryPath); + this.inserter = inserter; + this.text = text; + } + + @Override + public void apply(DirCacheEntry ent) { + try { + ent.setObjectId(inserter.insert(Constants.OBJ_BLOB, text.getBytes(UTF_8))); + ent.setFileMode(FileMode.REGULAR_FILE); + } catch (IOException e) { + throw new StorageException("failed to create a new text blob", e); + } + } + } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirrorState.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirrorState.java index a69a7a9888..136fb2dfc9 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirrorState.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirrorState.java @@ -20,8 +20,10 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; -final class MirrorState { +@VisibleForTesting +public final class MirrorState { private final String sourceRevision; @@ -30,7 +32,8 @@ final class MirrorState { this.sourceRevision = requireNonNull(sourceRevision, "sourceRevision"); } - String sourceRevision() { + @JsonProperty("sourceRevision") + public String sourceRevision() { return sourceRevision; } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorUtil.java b/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorUtil.java index 3d5a17e59d..2fd6b8d2f5 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorUtil.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorUtil.java @@ -61,7 +61,7 @@ public static String normalizePath(String path) { * *

e.g. git+ssh://foo.com/bar.git/some-path#master is split into: * - remoteRepoUri: git+ssh://foo.com/bar.git - * - remotePath: /some-path + * - remotePath: /some-path/ * - remoteBranch: master * *

e.g. dogma://foo.com/bar/qux.dogma is split into: diff --git a/site/src/sphinx/concepts.rst b/site/src/sphinx/concepts.rst index 296220486d..a31e2c1cb4 100644 --- a/site/src/sphinx/concepts.rst +++ b/site/src/sphinx/concepts.rst @@ -42,7 +42,7 @@ Concepts - Meta repository - A *meta repository* is a repository whose name is ``meta`` in a project. It is dedicated to store the - metadata related with the project it belongs to, such as Git-to-CD mirroring settings. + metadata related with the project it belongs to, such as Git mirroring settings. - Commit diff --git a/site/src/sphinx/mirroring.rst b/site/src/sphinx/mirroring.rst index d7e7e4d564..2de09eec8d 100644 --- a/site/src/sphinx/mirroring.rst +++ b/site/src/sphinx/mirroring.rst @@ -217,6 +217,11 @@ If everything was configured correctly, the repository you specified in ``localR "sourceRevision": "22fb176e4d8096d709d34ffe985c5f3acea83ef2" } +Setting up a CD-to-Git mirror +----------------------------- +It's exactly the same as setting up a Git-to-CD mirror which is described above, except you need to specify +``direction`` with ``LOCAL_TO_REMOTE``. + Mirror limit settings --------------------- Central Dogma limits the number of files and the total size of the files in a mirror for its reliability. diff --git a/site/src/sphinx/setup-configuration.rst b/site/src/sphinx/setup-configuration.rst index ddbaaba7c2..cee560db3e 100644 --- a/site/src/sphinx/setup-configuration.rst +++ b/site/src/sphinx/setup-configuration.rst @@ -182,7 +182,7 @@ Core properties - ``mirroringEnabled`` (boolean) - - whether to enable Git-to-CD mirroring. It's enabled by default. For more information about mirroring, + - whether to enable Git mirroring. It's enabled by default. For more information about mirroring, refer to :ref:`mirroring`. - ``numMirroringThreads`` (integer) diff --git a/testing-internal/src/main/java/com/linecorp/centraldogma/testing/internal/TestUtil.java b/testing-internal/src/main/java/com/linecorp/centraldogma/testing/internal/TestUtil.java index ace83be7f6..c188724505 100644 --- a/testing-internal/src/main/java/com/linecorp/centraldogma/testing/internal/TestUtil.java +++ b/testing-internal/src/main/java/com/linecorp/centraldogma/testing/internal/TestUtil.java @@ -47,7 +47,8 @@ public static void assertJsonConversion(T value, Class valueType, String } public static String normalizedDisplayName(TestInfo testInfo) { - return DISALLOWED_CHARS.matcher(testInfo.getDisplayName()).replaceAll(""); + return DISALLOWED_CHARS.matcher(testInfo.getDisplayName() + testInfo.getTestMethod().get().getName()) + .replaceAll(""); } private TestUtil() {}