Clickhouse mode of study view (#11224)

Add Clickhouse implementation of study view endpoints (off by default) with legacy api intact --------- Co-authored-by: haynescd <[email protected]> --------- Co-authored-by: haynescd <[email protected]> Co-authored-by: Charles Haynes <[email protected]> Co-authored-by: Onur Sumer <[email protected]> Co-authored-by: Neel Kuila <[email protected]> Co-authored-by: Bryan Lai <[email protected]> Co-authored-by: Zhaoyuan (Ryan) Fu <[email protected]> Co-authored-by: Qi-Xuan Lu <[email protected]> Co-authored-by: Bryan Lai <[email protected]> Co-authored-by: Gaofei Zhao <[email protected]> Co-authored-by: Robert Sheridan <[email protected]> Co-authored-by: Zain Nasir <[email protected]> Co-authored-by: Zain Nasir <[email protected]> Co-authored-by: haynescd <[email protected]>
cBioPortal · Dec 19, 2024 · 39caae5 · 39caae5
1 parent be86bae
commit 39caae5
Show file tree

Hide file tree

Showing 137 changed files with 26,236 additions and 431 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -223,6 +223,134 @@ jobs:
             DOCKER_IMAGE_SESSION_SERVICE: cbioportal/session-service:0.5.0
             FRONTEND_TEST_DO_NOT_LOAD_EXTERNAL_FRONTEND: true
 
+    pull_cbioportal_test_codebase:
+      machine:
+        image: ubuntu-2204:2024.08.1
+      resource_class: medium
+      working_directory: /tmp/repos
+      steps:
+        - run:
+            name: Checkout cbioportal/cbioportal-test
+            environment:
+              TEST_REPO_URL: https://github.com/cBioPortal/cbioportal-test
+            command: |
+              git clone ${TEST_REPO_URL}
+        - persist_to_workspace:
+            root: /tmp/repos
+            paths:
+              - cbioportal-test
+
+    pull_cbioportal_frontend_codebase:
+      machine:
+        image: ubuntu-2204:2024.08.1
+      resource_class: medium
+      working_directory: /tmp/repos
+      steps:
+        - run:
+            name: Checkout cbioportal/cbioportal-frontend
+            environment:
+              FRONTEND_REPO_URL: https://github.com/cBioPortal/cbioportal-frontend.git
+              FRONTEND_REPO_BRANCH: master
+            command: |
+              git clone -b ${FRONTEND_REPO_BRANCH} --single-branch ${FRONTEND_REPO_URL}
+        - persist_to_workspace:
+            root: /tmp/repos
+            paths:
+              - cbioportal-frontend
+
+    build_push_image:
+      machine:
+        image: ubuntu-2204:2024.08.1
+      resource_class: medium
+      working_directory: /tmp/repos
+      steps:
+        - attach_workspace:
+            at: /tmp/repos
+        - checkout:
+            path: /tmp/repos/cbioportal
+        - run:
+            name: Build cBioPortal docker image
+            environment:
+              DOCKER_REPO: cbioportal/cbioportal-dev
+            command: |
+              export DOCKER_TAG=$CIRCLE_SHA1
+              URL="https://hub.docker.com/v2/repositories/cbioportal/cbioportal-dev/tags/$DOCKER_TAG-web-shenandoah"
+              TAG_FOUND=$(curl -s $URL | jq -r .name)
+              if [ $TAG_FOUND = "$DOCKER_TAG-web-shenandoah" ]; then
+                echo "Image already exists. Skipping build step!"
+                exit 0
+              fi
+              cd cbioportal-test
+              ./scripts/build-push-image.sh --src=/tmp/repos/cbioportal --push=true --skip_web_and_data=true
+              EXISTS=$(docker manifest inspect $DOCKER_REPO:$DOCKER_TAG-web-shenandoah > /dev/null; echo $?)
+              if [ $EXISTS -eq 0 ]; then
+                echo "Build succeeded!"
+              else
+                echo "Build failed!"
+                exit 1
+              fi
+
+    run_api_tests:
+      machine:
+        image: ubuntu-2204:2024.08.1
+        docker_layer_caching: true
+      resource_class: large
+      working_directory: /tmp/repos
+      steps:
+        - attach_workspace:
+            at: /tmp/repos
+        - checkout:
+            path: /tmp/repos/cbioportal
+        - run:
+            name: Instantiate a cbioportal instance
+            environment:
+              DOCKER_REPO: cbioportal/cbioportal-dev
+              APP_CLICKHOUSE_MODE: "true"
+            command: |
+              cd cbioportal-test
+              export DOCKER_IMAGE_CBIOPORTAL=$DOCKER_REPO:$CIRCLE_SHA1-web-shenandoah
+              nohup ./scripts/docker-compose.sh >> /tmp/repos/docker-compose-logs.txt 2>&1 &
+        - run:
+            name: Wait for cbioportal to be live at localhost
+            command: |
+              cd cbioportal-test
+              ./utils/check-connection.sh --url=localhost:8080
+        - run:
+            name: Confirm cbioportal config matches PR
+            command: |
+              cd cbioportal
+              echo "Matching gitCommitId..."
+              INSTANCE_COMMIT_ID=$(curl -s http://localhost:8080/api/info | jq -r '.["gitCommitId"]')
+              PR_COMMIT_ID=$CIRCLE_SHA1
+              if [ "$INSTANCE_COMMIT_ID" = "$PR_COMMIT_ID" ]; then
+                echo "gitCommitId successfully matched!"
+                echo "cBioPortal is ready:"
+                curl -s http://localhost:8080/api/info | jq
+                exit 0
+              else
+                echo "gitCommitIds do not match!"
+                echo "Instance Commit ID: $INSTANCE_COMMIT_ID"
+                echo "PR Commit ID: $PR_COMMIT_ID"
+                exit 1
+              fi
+        - run:
+            name: Run API Tests
+            environment:
+              API_TEST_HOST: http://localhost:8080
+              BACKEND_ROOT: /tmp/repos/cbioportal
+            command: |
+              cd cbioportal-frontend
+              nvm install 15.2.1
+              nvm use 15.2.1
+              npm install -g [email protected]
+              yarn --ignore-engines
+              yarn run apitests
+
+        - store_artifacts:
+            path: /tmp/repos/cbioportal-test/web-metadata.json
+        - store_artifacts:
+            path: /tmp/repos/docker-compose-logs.txt
+
 
 workflows:
     version: 2
@@ -238,3 +366,22 @@ workflows:
                     - build_backend
                     - pull_frontend_codebase
                     - install_yarn
+    api_tests:
+      jobs:
+        - pull_cbioportal_test_codebase
+        - pull_cbioportal_frontend_codebase
+        - wait_for_approval:
+            type: approval
+            requires:
+              - pull_cbioportal_test_codebase
+              - pull_cbioportal_frontend_codebase
+        - build_push_image:
+            context:
+              - api-tests
+            requires:
+              - wait_for_approval
+        - run_api_tests:
+            context:
+              - api-tests
+            requires:
+              - build_push_image
diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml
@@ -34,7 +34,7 @@ jobs:
           cp src/main/resources/application.properties.EXAMPLE src/main/resources/application.properties
       - name: Build and analyze
         env:
-          SONAR_TOKEN:  ${{ secrets.SONAR_TOKEN }}
+          SONAR_TOKEN:  de1b5cc660cd210dde840f492c371da6cc801763
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: mvn clean install verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar
       - name: 'Add host.testcontainers.internal to /etc/hosts'
@@ -45,6 +45,6 @@ jobs:
           mvn verify -Pintegration-test
       - name: Code Coverage
         env:
-          SONAR_TOKEN:  ${{ secrets.SONAR_TOKEN }}
+          SONAR_TOKEN:  de1b5cc660cd210dde840f492c371da6cc801763
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: mvn sonar:sonar -Pcoverage
diff --git a/pom.xml b/pom.xml
@@ -99,6 +99,7 @@
         <selenium.version>4.17.0</selenium.version>
 		<sentry.version>7.1.0</sentry.version>
         <apache_httpclient.version>5.2.1</apache_httpclient.version>
+        <clickhouse_testcontainer.version>1.19.7</clickhouse_testcontainer.version>
 
 
 		<!-- No sure what these are for -->
@@ -354,6 +355,31 @@
             <scope>test</scope>
         </dependency>
 
+        <dependency>
+            <groupId>com.clickhouse</groupId>
+            <artifactId>clickhouse-jdbc</artifactId>
+            <version>0.6.2</version>
+            <!-- use uber jar with all dependencies included, change classifier
+            to http for smaller jar -->
+            <classifier>all</classifier>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>clickhouse</artifactId>
+            <version>${clickhouse_testcontainer.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.mybatis.spring.boot</groupId>
+            <artifactId>mybatis-spring-boot-test-autoconfigure</artifactId>
+            <version>3.0.3</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-testcontainers</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <dependencyManagement>
@@ -418,9 +444,25 @@
 				<includes>
 					<include>cgds.sql</include>
 					<include>migration.sql</include>
+                    <include>clickhouse/clickhouse.sql</include>
+                    <include>clickhouse/materialized_views.sql</include>
 				</includes>
 			</resource>
 		</resources>
+        <testResources>
+            <testResource>
+                <directory>src/main/resources/db-scripts</directory>
+                <filtering>false</filtering>
+                <includes>
+                    <include>clickhouse/clickhouse.sql</include>
+                    <include>clickhouse/materialized_views.sql</include>
+                </includes> 
+            </testResource>
+            <testResource>
+                <directory>src/test/resources</directory>
+                <filtering>false</filtering>
+            </testResource>
+        </testResources>
 
 		<plugins>
 			<plugin>

diff --git a/src/main/java/org/cbioportal/PortalApplication.java b/src/main/java/org/cbioportal/PortalApplication.java
@@ -6,6 +6,7 @@
 import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration;
 import org.springframework.context.annotation.PropertySource;
 import org.springframework.context.annotation.PropertySources;
+import org.springframework.transaction.annotation.EnableTransactionManagement;
 
 @SpringBootApplication(exclude = {
     MongoAutoConfiguration.class,
@@ -18,6 +19,7 @@
     @PropertySource(ignoreResourceNotFound = true, value = "classpath:git.properties"),
     @PropertySource(ignoreResourceNotFound = true, value = "classpath:springdoc.properties")
 })
+@EnableTransactionManagement
 public class PortalApplication {
     public static void main(String[] args) {
         SpringApplication.run(PortalApplication.class, args);

diff --git a/src/main/java/org/cbioportal/WebAppConfig.java b/src/main/java/org/cbioportal/WebAppConfig.java
@@ -2,6 +2,7 @@
 
 import java.util.List;
 
+import org.cbioportal.web.ExecuterTimeInterceptor;
 import org.cbioportal.web.util.InvolvedCancerStudyExtractorInterceptor;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
@@ -12,6 +13,7 @@
 import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry;
 import org.springframework.web.servlet.config.annotation.ViewControllerRegistry;
 import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
+import org.springframework.web.servlet.handler.WebRequestHandlerInterceptorAdapter;
 
 // TODO Consider creating separate DispatcherServlets as in the original web.xml
 // See: https://stackoverflow.com/a/30686733/11651683
@@ -75,6 +77,9 @@ public HandlerInterceptor involvedCancerStudyExtractorInterceptor() {
 	@Override
 	public void addInterceptors(InterceptorRegistry registry) {
 		registry.addInterceptor(involvedCancerStudyExtractorInterceptor());
+        registry.addInterceptor(new WebRequestHandlerInterceptorAdapter(
+            new ExecuterTimeInterceptor()
+        )).addPathPatterns("/**");
 	}
 
 	@Override

diff --git a/src/main/java/org/cbioportal/model/AlterationType.java b/src/main/java/org/cbioportal/model/AlterationType.java
@@ -1,6 +1,8 @@
 package org.cbioportal.model;
 
 public enum AlterationType {
-    MUTATION,
-    COPY_NUMBER_ALTERATION
+    MUTATION_EXTENDED,
+    COPY_NUMBER_ALTERATION,
+    STRUCTURAL_VARIANT,
+    GENERIC_ASSAY;
 }
diff --git a/src/main/java/org/cbioportal/model/CaseListDataCount.java b/src/main/java/org/cbioportal/model/CaseListDataCount.java
@@ -1,6 +1,8 @@
 package org.cbioportal.model;
 
-public class CaseListDataCount {
+import java.io.Serializable;
+
+public class CaseListDataCount implements Serializable {
 
     private String label;
     private String value;

diff --git a/src/main/java/org/cbioportal/model/ClinicalViolinPlotBoxData.java b/src/main/java/org/cbioportal/model/ClinicalViolinPlotBoxData.java
@@ -1,6 +1,8 @@
 package org.cbioportal.model;
 
-public class ClinicalViolinPlotBoxData {
+import java.io.Serializable;
+
+public class ClinicalViolinPlotBoxData implements Serializable {
     private double whiskerLower;
     private double whiskerUpper;
     private double median;

diff --git a/src/main/java/org/cbioportal/model/ClinicalViolinPlotData.java b/src/main/java/org/cbioportal/model/ClinicalViolinPlotData.java
@@ -1,8 +1,9 @@
 package org.cbioportal.model;
 
+import java.io.Serializable;
 import java.util.List;
 
-public class ClinicalViolinPlotData {
+public class ClinicalViolinPlotData implements Serializable {
     private List<ClinicalViolinPlotRowData> rows;
     private Double axisStart;
     private Double axisEnd;

diff --git a/src/main/java/org/cbioportal/model/ClinicalViolinPlotIndividualPoint.java b/src/main/java/org/cbioportal/model/ClinicalViolinPlotIndividualPoint.java
@@ -1,6 +1,8 @@
 package org.cbioportal.model;
 
-public class ClinicalViolinPlotIndividualPoint {
+import java.io.Serializable;
+
+public class ClinicalViolinPlotIndividualPoint implements Serializable {
     private String sampleId;
     private String studyId;
     private double value;

diff --git a/src/main/java/org/cbioportal/model/ClinicalViolinPlotRowData.java b/src/main/java/org/cbioportal/model/ClinicalViolinPlotRowData.java
@@ -1,8 +1,9 @@
 package org.cbioportal.model;
 
+import java.io.Serializable;
 import java.util.List;
 
-public class ClinicalViolinPlotRowData {
+public class ClinicalViolinPlotRowData implements Serializable {
     private String category;
     private Integer numSamples;
     private List<Double> curveData;

diff --git a/src/main/java/org/cbioportal/model/GenericAssayDataCount.java b/src/main/java/org/cbioportal/model/GenericAssayDataCount.java
@@ -7,6 +7,13 @@ public class GenericAssayDataCount implements Serializable {
     private String value;
     private Integer count;
 
+    public GenericAssayDataCount() {}
+
+    public GenericAssayDataCount(String value, Integer count) {
+        this.value = value;
+        this.count = count;
+    }
+
     public String getValue() {
         return value;
     }

diff --git a/src/main/java/org/cbioportal/model/GenericAssayDataCountItem.java b/src/main/java/org/cbioportal/model/GenericAssayDataCountItem.java
@@ -7,6 +7,13 @@ public class GenericAssayDataCountItem implements Serializable {
     private String stableId;
     private List<GenericAssayDataCount> counts;
 
+    public GenericAssayDataCountItem() {}
+
+    public GenericAssayDataCountItem(String stableId, List<GenericAssayDataCount> counts) {
+        this.stableId = stableId;
+        this.counts = counts;
+    }
+
     public String getStableId() {
         return stableId;
     }

diff --git a/src/main/java/org/cbioportal/model/GenomicDataCount.java b/src/main/java/org/cbioportal/model/GenomicDataCount.java
@@ -10,6 +10,21 @@ public class GenomicDataCount implements Serializable {
     private Integer count;
     private Integer uniqueCount;
 
+    public GenomicDataCount() {}
+
+    public GenomicDataCount(String label, String value, Integer count) {
+        this.label = label;
+        this.value = value;
+        this.count = count;
+    }
+
+    public GenomicDataCount(String label, String value, Integer count, Integer uniqueCount) {
+        this.label = label;
+        this.value = value;
+        this.count = count;
+        this.uniqueCount = uniqueCount;
+    }
+
     public String getLabel() {
         return label;
     }