Add HTTP-to-CD mirroring

it/build.gradle

@@ -1,4 +1,5 @@
 dependencies {
+    testImplementation libs.armeria.junit5
     // jGit
     testImplementation libs.jgit
     // JSch

it/src/test/java/com/linecorp/centraldogma/it/mirror/http/HttpMirrorTest.java

@@ -0,0 +1,186 @@
+/*
+ * Copyright 2023 LINE Corporation
+ *
+ * LINE Corporation licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package com.linecorp.centraldogma.it.mirror.http;
+
+import static com.google.common.base.MoreObjects.firstNonNull;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import javax.annotation.Nullable;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInfo;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+import com.google.common.base.Strings;
+
+import com.linecorp.armeria.common.ContentTooLargeException;
+import com.linecorp.armeria.server.HttpStatusException;
+import com.linecorp.armeria.server.ServerBuilder;
+import com.linecorp.armeria.server.annotation.Get;
+import com.linecorp.armeria.server.annotation.Param;
+import com.linecorp.armeria.server.annotation.StatusCode;
+import com.linecorp.armeria.testing.junit5.server.ServerExtension;
+import com.linecorp.centraldogma.client.CentralDogma;
+import com.linecorp.centraldogma.common.Change;
+import com.linecorp.centraldogma.common.Entry;
+import com.linecorp.centraldogma.common.EntryNotFoundException;
+import com.linecorp.centraldogma.server.CentralDogmaBuilder;
+import com.linecorp.centraldogma.server.MirrorException;
+import com.linecorp.centraldogma.server.MirroringService;
+import com.linecorp.centraldogma.server.storage.project.Project;
+import com.linecorp.centraldogma.testing.internal.TestUtil;
+import com.linecorp.centraldogma.testing.junit.CentralDogmaExtension;
+
+class HttpMirrorTest {
+
+    private static final int MAX_NUM_BYTES = 1024; // 1 KiB
+
+    private static final String REPO_FOO = "foo";
+
+    @RegisterExtension
+    static final CentralDogmaExtension dogma = new CentralDogmaExtension() {
+        @Override
+        protected void configure(CentralDogmaBuilder builder) {
+            builder.mirroringEnabled(true);
+            builder.maxNumBytesPerMirror(MAX_NUM_BYTES);
+        }
+    };
+
+    @RegisterExtension
+    static final ServerExtension server = new ServerExtension() {
+        @Override
+        protected void configure(ServerBuilder sb) throws Exception {
+            sb.annotatedService(new Object() {
+                @Get("/get/:length")
+                public String get(@Param int length) {
+                    return Strings.repeat(".", length);
+                }
+
+                @Get("/204") // Generate a '204 No Content' response.
+                public void respond204() {}
+
+                @Get("/304")
+                @StatusCode(304)
+                public void respond304() {}
+            });
+        }
+    };
+
+    private static CentralDogma client;
+    private static MirroringService mirroringService;
+
+    @BeforeAll
+    static void init() {
+        client = dogma.client();
+        mirroringService = dogma.mirroringService();
+    }
+
+    private String projName;
+
+    @BeforeEach
+    void initDogmaRepo(TestInfo testInfo) {
+        projName = TestUtil.normalizedDisplayName(testInfo);
+        client.createProject(projName).join();
+        client.createRepository(projName, REPO_FOO).join();
+    }
+
+    @AfterEach
+    void destroyDogmaRepo() {
+        client.removeProject(projName).join();
+    }
+
+    @Test
+    void simple() throws Exception {
+        // Configure the server to mirror http://.../get/7 into /bar.txt.
+        pushMirrorSettings(REPO_FOO, "/bar.txt", 7);
+        testSuccessfulMirror(".......\n");
+    }
+
+    @Test
+    void tooLargeContent() throws Exception {
+        // Configure the server to mirror http://.../get/<MAX_NUM_BYTES + 1> into /bar.txt.
+        pushMirrorSettings(REPO_FOO, "/bar.txt", MAX_NUM_BYTES + 1);
+        testFailedMirror(ContentTooLargeException.class);
+    }
+
+    @Test
+    void shouldHandle204() throws Exception {
+        // Configure the server to mirror http://.../204 into /bar.txt.
+        pushMirrorSettings(REPO_FOO, "/bar.txt", server.httpUri() + "/204");
+        testSuccessfulMirror("");
+    }
+
+    @Test
+    void shouldRejectNon2xx() throws Exception {
+        // Configure the server to mirror http://.../get/<MAX_NUM_BYTES + 1> into /bar.txt.
+        pushMirrorSettings(REPO_FOO, "/bar.txt", server.httpUri() + "/304");
+        testFailedMirror(HttpStatusException.class);
+    }
+
+    private void pushMirrorSettings(String localRepo, @Nullable String localPath, int length) {
+        pushMirrorSettings(localRepo, localPath, remoteUri(length));
+    }
+
+    private void pushMirrorSettings(String localRepo, @Nullable String localPath, String remoteUri) {
+        client.forRepo(projName, Project.REPO_META)
+              .commit("Add /mirrors.json",
+                      Change.ofJsonUpsert("/mirrors.json",
+                                          "[{" +
+                                          "  \"type\": \"single\"," +
+                                          "  \"direction\": \"REMOTE_TO_LOCAL\"," +
+                                          "  \"localRepo\": \"" + localRepo + "\"," +
+                                          "  \"localPath\": \"" + firstNonNull(localPath, "/") + "\"," +
+                                          "  \"remoteUri\": \"" + remoteUri + "\"," +
+                                          "  \"schedule\": \"0 0 0 1 1 ? 2099\"" +
+                                          "}]"))
+              .push().join();
+    }
+
+    private void testSuccessfulMirror(String expectedContent) {
+        // Trigger the mirroring task.
+        mirroringService.mirror().join();
+
+        // On successful mirroring, /bar.txt should contain 7 periods.
+        final Entry<?> entry = client.forRepo(projName, REPO_FOO)
+                                     .file("/bar.txt")
+                                     .get().join();
+
+        assertThat(entry.contentAsText()).isEqualTo(expectedContent);
+    }
+
+    private void testFailedMirror(Class<? extends Throwable> rootCause) {
+        // Trigger the mirroring task, which will fail because the response was too large.
+        assertThatThrownBy(() -> mirroringService.mirror().join()).cause().satisfies(cause -> {
+            assertThat(cause).isInstanceOf(MirrorException.class)
+                             .hasCauseInstanceOf(rootCause);
+        });
+
+        // As a result, /bar.txt shouldn't exist.
+        assertThatThrownBy(() -> {
+            client.forRepo(projName, REPO_FOO)
+                  .file("/bar.txt")
+                  .get().join();
+        }).hasCauseInstanceOf(EntryNotFoundException.class);
+    }
+
+    private static String remoteUri(int length) {
+        return String.format("%s/get/%d", server.httpUri(), length);
+    }
+}

server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/AbstractMirror.java

@@ -52,7 +52,8 @@ public abstract class AbstractMirror implements Mirror {
     private final Repository localRepo;
     private final String localPath;
     private final URI remoteRepoUri;
-    private final String remotePath;
+    @Nullable
+    private final String remoteSubpath;
     @Nullable
     private final String remoteBranch;
     @Nullable
@@ -62,16 +63,16 @@ public abstract class AbstractMirror implements Mirror {
 
     protected AbstractMirror(Cron schedule, MirrorDirection direction, MirrorCredential credential,
                              Repository localRepo, String localPath,
-                             URI remoteRepoUri, String remotePath, @Nullable String remoteBranch,
+                             URI remoteRepoUri, @Nullable String remoteSubpath, @Nullable String remoteBranch,
                              @Nullable String gitignore) {
 
         this.schedule = requireNonNull(schedule, "schedule");
         this.direction = requireNonNull(direction, "direction");
         this.credential = requireNonNull(credential, "credential");
         this.localRepo = requireNonNull(localRepo, "localRepo");
-        this.localPath = normalizePath(requireNonNull(localPath, "localPath"));
+        this.localPath = requireNonNull(localPath, "localPath");
         this.remoteRepoUri = requireNonNull(remoteRepoUri, "remoteRepoUri");
-        this.remotePath = normalizePath(requireNonNull(remotePath, "remotePath"));
+        this.remoteSubpath = remoteSubpath;
         this.remoteBranch = remoteBranch;
         this.gitignore = gitignore;
 
@@ -81,7 +82,7 @@ protected AbstractMirror(Cron schedule, MirrorDirection direction, MirrorCredent
         // Use the properties' hash code so that the same properties result in the same jitter.
         jitterMillis = Math.abs(Objects.hash(this.schedule.asString(), this.direction,
                                              this.localRepo.parent().name(), this.localRepo.name(),
-                                             this.remoteRepoUri, this.remotePath, this.remoteBranch) /
+                                             this.remoteRepoUri, this.remoteSubpath, this.remoteBranch) /
                                 (Integer.MAX_VALUE / 60000));
     }
 
@@ -129,8 +130,8 @@ public final URI remoteRepoUri() {
     }
 
     @Override
-    public final String remotePath() {
-        return remotePath;
+    public String remoteSubpath() {
+        return remoteSubpath;
     }
 
     @Override
@@ -180,7 +181,7 @@ public String toString() {
                                                  .add("localRepo", localRepo.name())
                                                  .add("localPath", localPath)
                                                  .add("remoteRepo", remoteRepoUri)
-                                                 .add("remotePath", remotePath)
+                                                 .add("remoteSubpath", remoteSubpath)
                                                  .add("remoteBranch", remoteBranch)
                                                  .add("gitignore", gitignore)
                                                  .add("credential", credential);

server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/CentralDogmaMirror.java

@@ -16,11 +16,13 @@
 
 package com.linecorp.centraldogma.server.internal.mirror;
 
+import static com.linecorp.centraldogma.server.mirror.MirrorUtil.normalizePath;
 import static java.util.Objects.requireNonNull;
 
 import java.io.File;
 import java.net.URI;
 
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
 import com.cronutils.model.Cron;
@@ -37,16 +39,31 @@ public final class CentralDogmaMirror extends AbstractMirror {
 
     public CentralDogmaMirror(Cron schedule, MirrorDirection direction, MirrorCredential credential,
                               Repository localRepo, String localPath,
-                              URI remoteRepoUri, String remoteProject, String remoteRepo, String remotePath,
+                              URI remoteRepoUri, String remoteProject, String remoteRepo, String remoteSubpath,
                               @Nullable String gitignore) {
         // Central Dogma has no notion of 'branch', so we just pass null as a placeholder.
-        super(schedule, direction, credential, localRepo, localPath, remoteRepoUri, remotePath, null,
+        super(schedule,
+              direction,
+              credential,
+              localRepo,
+              normalizePath(requireNonNull(localPath, "localPath")),
+              remoteRepoUri,
+              normalizePath(requireNonNull(remoteSubpath, "remoteSubpath")),
+              null,
               gitignore);
 
         this.remoteProject = requireNonNull(remoteProject, "remoteProject");
         this.remoteRepo = requireNonNull(remoteRepo, "remoteRepo");
     }
 
+    @Nonnull
+    @Override
+    public String remoteSubpath() {
+        final String remoteSubpath = super.remoteSubpath();
+        assert remoteSubpath != null;
+        return remoteSubpath;
+    }
+
     String remoteProject() {
         return remoteProject;
     }