DOP-1160: adds deploy stage to world builder, updates list of projects

tools/world-builder/build.py

@@ -7,67 +7,81 @@
 import yaml
 from typing import Any, Dict, Optional, Set, List
 
-PAT_URL = re.compile('.+/(.+).git')
+PAT_URL = re.compile(".+/(.+).git")
 
 
 class BuildTask:
-    def __init__(self, name: str, branches: List[str], build_dir: str, build_command: str) -> None:
+    def __init__(
+        self,
+        name: str,
+        branches: List[str],
+        build_dir: str,
+        build_command: str,
+        deploy_command: str,
+    ) -> None:
         self.name = name
         self.branches = branches
         self.build_dir = build_dir
         self.build_command = build_command
-        self.built_flag_file = os.path.join(self.build_dir, '.built')
+        self.deploy_command = deploy_command
+        self.built_flag_file = os.path.join(self.build_dir, ".built")
 
     def build(self) -> None:
-        subprocess.check_call(['git', 'clean', '-xfd'], cwd=self.build_dir)
+        subprocess.check_call(["git", "clean", "-xfd"], cwd=self.build_dir)
         # Some of our properties die if these directories do not exist
-        for path in ('build', 'build/master', 'build/public', 'build/public/master'):
+        for path in ("build", "build/master", "build/public", "build/public/master"):
             try:
                 os.mkdir(os.path.join(self.build_dir, path))
             except OSError:
                 pass
 
         for branch in self.branches:
-            print('Building ' + self.build_dir)
-            subprocess.check_call(['git', 'checkout', '-q', branch], cwd=self.build_dir)
-            subprocess.check_call(['git', 'pull', '--ff-only', '-q'], cwd=self.build_dir)
+            print("Building " + self.build_dir)
+            subprocess.check_call(["git", "checkout", "-q", branch], cwd=self.build_dir)
+            subprocess.check_call(
+                ["git", "pull", "--ff-only", "-q"], cwd=self.build_dir
+            )
             subprocess.check_call(self.build_command, shell=True, cwd=self.build_dir)
 
+    def deploy(self) -> None:
+        for branch in self.branches:
+            print("Deploying " + self.build_dir + branch)
+            subprocess.check_call(["git", "checkout", "-q", branch], cwd=self.build_dir)
+            subprocess.check_call(self.deploy_command, shell=True, cwd=self.build_dir)
+
 
 def load_branches_from_yaml(path: str) -> List[str]:
-    with open(path, 'r') as f:
+    with open(path, "r") as f:
         loaded = yaml.safe_load(f)
 
-    return loaded['git']['branches']['published']
+    return loaded["git"]["branches"]["published"]
 
 
 def initialize_project(project: Dict[str, Any]) -> BuildTask:
-    git_url = project['git']
-    project_name = project.get('name', None)
+    git_url = project["git"]
+    project_name = project.get("name", None)
     if not project_name:
         match = PAT_URL.match(git_url)
         if not match:
-            raise ValueError('Cannot determine project name: {}'.format(git_url))
+            raise ValueError("Cannot determine project name: {}".format(git_url))
         project_name = match.group(1)
 
-    branches = project['branches']
+    branches = project["branches"]
     if isinstance(branches, str):
         branches = load_branches_from_yaml(branches)
 
-    output_dir = os.path.join('build', project_name)
+    output_dir = os.path.join("build", project_name)
     if not os.path.isdir(output_dir):
-        print('Cloning {}'.format(git_url))
-        subprocess.check_call(['git', 'clone', '-q', git_url, output_dir])
+        print("Cloning {}".format(git_url))
+        subprocess.check_call(["git", "clone", "-q", git_url, output_dir])
 
     return BuildTask(
-            project_name,
-            branches,
-            output_dir,
-            project['build'])
+        project_name, branches, output_dir, project["build"], project["deploy"]
+    )
 
 
 def main(path: str, projects: Optional[Set[str]]) -> None:
-    with open(path, 'r') as f:
+    with open(path, "r") as f:
         data = json.load(f)
 
     tasks = []
@@ -76,24 +90,37 @@ def main(path: str, projects: Optional[Set[str]]) -> None:
         if projects is None or task.name in projects:
             tasks.append(task)
 
-    errors = []  # type: List[str]
+    build_errors = []  # type: List[str]
     for task in tasks:
         try:
             task.build()
         except subprocess.SubprocessError:
-            errors.append(task.build_dir)
-            print('Failed to build in {}'.format(task.build_dir))
+            build_errors.append(task.build_dir)
+            print("Failed to build in {}".format(task.build_dir))
+            tasks.remove(task)  # so that we don't deploy the unbuilt thing
+
+    if build_errors:
+        print("\nFailed to build:")
+        for d in build_errors:
+            print("  " + d)
+
+    deploy_errors = []  # type: List[str]
+    for task in tasks:
+        try:
+            task.deploy()
+        except subprocess.SubprocessError:
+            deploy_errors.append(task.build)
 
-    if errors:
-        print('\nFailed to build:')
-        for d in errors:
-            print('  ' + d)
+    if deploy_errors:
+        print("\nFailed to deploy:")
+        for d in deploy_errors:
+            print(" " + d)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     if len(sys.argv) > 1:
         projects = set(sys.argv[1:])  # type: Optional[Set[str]]
     else:
         projects = None
 
-    main('projects.json', projects)
+    main("projects.json", projects)

tools/world-builder/projects.json

@@ -2,79 +2,118 @@
   {
     "git": "https://github.com/mongodb/docs-bi-connector.git",
     "branches": "../../data/bi-connector-published-branches.yaml",
-    "build": "mkdir -p build/master && make publish"
+    "build": "make publish",
+    "deploy": "make deploy"
   },
   {
     "git": "https://github.com/mongodb/docs-spark-connector.git",
     "branches": "../../data/spark-connector-published-branches.yaml",
-    "build": "giza make publish"
+    "build": "make publish",
+    "deploy": "make deploy"
   },
   {
     "git": "https://github.com/mongodb/docs.git",
     "branches": "../../data/manual-published-branches.yaml",
-    "build": "make publish"
+    "build": "make publish",
+    "deploy": "make deploy"
   },
   {
     "name": "mms-onprem",
     "git": "[email protected]:10gen/mms-docs.git",
     "branches": "../../data/mms-published-branches.yaml",
-    "build": "giza make publish-onprem"
+    "build": "make publish-onprem",
+    "deploy": "make deploy-opsmgr"
   },
   {
     "name": "mms-cloud",
     "git": "[email protected]:10gen/mms-docs.git",
     "branches": ["master"],
-    "build": "make publish-cloud"
-  },
-  {
-    "git": "[email protected]:10gen/baas-docs.git",
-    "branches": "../../data/stitch-published-branches.yaml",
-    "build": "make publish"
+    "build": "make publish-cloud",
+    "deploy": "make deploy-cloud"
   },
   {
     "git": "https://github.com/mongodb/docs-compass.git",
     "branches": "../../data/compass-published-branches.yaml",
-    "build": "make publish"
+    "build": "make publish",
+    "deploy": "make deploy"
   },
   {
     "git": "[email protected]:10gen/docs-charts.git",
     "branches": "../../data/charts-published-branches.yaml",
-    "build": "make publish"
+    "build": "make publish",
+    "deploy": "make deploy"
   },
   {
     "git": "[email protected]:10gen/cloud-docs.git",
     "branches": ["master"],
-    "build": "make publish"
+    "build": "make publish",
+    "deploy": "make deploy"
   },
   {
     "git": "https://github.com/mongodb/docs-php-library.git",
     "branches": "../../data/php-library-published-branches.yaml",
-    "build": "make publish"
+    "build": "make publish",
+    "deploy": "make deploy"
   },
   {
     "git": "https://github.com/mongodb/docs-ruby.git",
     "branches": "../../data/ruby-driver-published-branches.yaml",
-    "build": "make publish"
+    "build": "make publish",
+    "deploy": "make deploy"
   },
   {
     "git": "[email protected]:10gen/docs-mongoid.git",
     "branches": "../../data/mongoid-published-branches.yaml",
-    "build": "make publish"
+    "build": "make publish",
+    "deploy": "make deploy"
   },
   {
     "git": "[email protected]:mongodb/docs-meta.git",
     "branches": ["master"],
-    "build": "make publish"
+    "build": "make publish",
+    "deploy": "make deploy"
   },
   {
-    "git": "[email protected]:mongodb/docs-ecosystem.git",
+    "name": "guides",
+    "git": "[email protected]:10gen/docs-tutorials.git",
     "branches": ["master"],
-    "build": "make publish"
+    "build": "make publish",
+    "deploy": "make deploy"
   },
   {
-    "git": "https://github.com/mongodb/docs-landing.git",
+    "git": "git@github.com:mongodb/docs-commandline-tools.git",
     "branches": ["master"],
-    "build": "",
-    "docs-tools": false
+    "build": "make publish",
+    "deploy": "make deploy"
+  },
+  {
+    "git": "[email protected]:10gen/docs-kafka-connector.git",
+    "branches": "../../data/kafka-connector-published-branches.yaml",
+    "build": "make publish",
+    "deploy": "make deploy"
+  },
+  {
+    "git": "[email protected]:10gen/docs-mongocli.git",
+    "branches": "../../data/mongocli-published-branches.yaml",
+    "build": "make publish",
+    "deploy": "make deploy"
+  },
+  {
+    "git": "[email protected]:mongodb/docs-mongodb-shell.git",
+    "branches": ["master"],
+    "build": "make publish",
+    "deploy": "make deploy"
+  },
+  {
+    "git": "[email protected]:10gen/docs-k8s-operator.git",
+    "branches": "../../data/kubernetes-operator-published-branches.yaml",
+    "build": "make publish",
+    "deploy": "make deploy"
+  },
+  {
+    "git": "[email protected]:10gen/cloud-docs-osb.git",
+    "branches": "../../data/atlas-open-service-broker-published-branches.yaml",
+    "build": "make publish",
+    "deploy": "make deploy"
   }
 ]