Merge pull request #169 from cloudgraphdev/aa-pages

fix: get all pages from ecs,efs,elb,redshift,ses

src/services/ecsContainer/data.ts

@@ -43,30 +43,42 @@ export default async ({
     /**
      * Get the instance arns of all the containers in each cluster
      */
-    let containerInstanceArns: any = await Promise.all(
+    const containerInstanceArns: any = await Promise.all(
       ecsClusters.map(
         async ({ clusterName: cluster, region }) =>
-          new Promise(resolveEcsData =>
-            new ECS({ ...config, region, endpoint }).listContainerInstances(
-              { cluster },
-              (err, data) => {
-                if (err) {
-                  errorLog.generateAwsErrorLog({
-                    functionName: 'ecs:listContainerInstances',
-                    err,
-                  })
-                }
-
-                if (isEmpty(data)) {
-                  return resolveEcsData([])
-                }
+          new Promise(resolveEcsData => {
+            const ecs = new ECS({ ...config, region, endpoint })
+            const resources: ECS.StringList = []
+            const listContainerInstances = (nextToken?: string): void => {
+              ecs.listContainerInstances(
+                { cluster, nextToken },
+                (err, data) => {
+                  if (err) {
+                    errorLog.generateAwsErrorLog({
+                      functionName: 'ecs:listContainerInstances',
+                      err,
+                    })
+                  }
 
-                const { containerInstanceArns: containerInstances = [] } = data
+                  if (isEmpty(data?.containerInstanceArns)) {
+                    return resolveEcsData([])
+                  }
+                  resources.push(...data.containerInstanceArns)
 
-                resolveEcsData({ cluster, containerInstances, region })
-              }
-            )
-          )
+                  if (data.nextToken) {
+                    listContainerInstances(data.nextToken)
+                  } else {
+                    resolveEcsData({
+                      cluster,
+                      containerInstances: resources,
+                      region,
+                    })
+                  }
+                }
+              )
+            }
+            return listContainerInstances()
+          })
       )
     )
     /**

src/services/ecsTask/data.ts

@@ -48,27 +48,32 @@ export default async ({
     let ecsTaskArns: any = await Promise.all(
       ecsClusters.map(
         async ({ clusterName: cluster, region }) =>
-          new Promise(resolveEcsData =>
-            new ECS({ ...config, region, endpoint }).listTasks(
-              { cluster },
-              (err, data) => {
+          new Promise(resolveEcsData => {
+            const ecs = new ECS({ ...config, region, endpoint })
+            const resources: ECS.StringList = []
+            const listTasks = (nextToken?: string): void => {
+              ecs.listTasks({ cluster, nextToken }, (err, data) => {
                 if (err) {
                   errorLog.generateAwsErrorLog({
                     functionName: 'ecs:listTasks',
                     err,
                   })
                 }
 
-                if (isEmpty(data)) {
+                if (isEmpty(data?.taskArns)) {
                   return resolveEcsData([])
                 }
 
-                const { taskArns = [] } = data
-
-                resolveEcsData({ region, cluster, taskArns })
-              }
-            )
-          )
+                resources.push(...data.taskArns)
+                if (data.nextToken) {
+                  listTasks(data.nextToken)
+                } else {
+                  resolveEcsData({ region, cluster, taskArns: resources })
+                }
+              })
+            }
+            listTasks()
+          })
       )
     )
 

src/services/efsMountTarget/data.ts

@@ -46,41 +46,50 @@ export default async ({
 
     regions.split(',').map(region => {
       efsFileSystems.map(efs => {
-        const regionPromise = new Promise<void>(resolveMountTargets =>
-          new EFS({
+        const regionPromise = new Promise<void>(resolveMountTargets => {
+          const efsClient = new EFS({
             ...config,
             region: efs.region,
             endpoint,
-          }).describeMountTargets(
-            { FileSystemId: efs.FileSystemId },
-            async (err: AWSError, data: DescribeMountTargetsResponse) => {
-              if (err) {
-                errorLog.generateAwsErrorLog({
-                  functionName: 'efs:describeMountTargets',
-                  err,
-                })
-              }
+          })
+          const describeMountTargets = (nextToken?: string): void => {
+            efsClient.describeMountTargets(
+              { FileSystemId: efs.FileSystemId, Marker: nextToken },
+              async (err: AWSError, data: DescribeMountTargetsResponse) => {
+                if (err) {
+                  errorLog.generateAwsErrorLog({
+                    functionName: 'efs:describeMountTargets',
+                    err,
+                  })
+                }
 
-              if (isEmpty(data)) {
-                return resolveMountTargets()
-              }
+                if (isEmpty(data)) {
+                  return resolveMountTargets()
+                }
 
-              const { MountTargets: mountTargets = [] } = data || {}
+                const { MountTargets: mountTargets = [] } = data || {}
 
-              logger.debug(lt.fetchedEfsMountTargets(mountTargets.length))
+                logger.debug(lt.fetchedEfsMountTargets(mountTargets.length))
 
-              if (!isEmpty(mountTargets)) {
-                efsMountTargets.push(
-                  ...mountTargets.map(target => ({
-                    region,
-                    ...target,
-                  }))
-                )
+                if (!isEmpty(mountTargets)) {
+                  efsMountTargets.push(
+                    ...mountTargets.map(target => ({
+                      region,
+                      ...target,
+                    }))
+                  )
+                }
+                if (data.NextMarker) {
+                  describeMountTargets(data.NextMarker)
+                } else {
+                  resolveMountTargets()
+                }
               }
-              resolveMountTargets()
-            }
-          )
-        )
+            )
+          }
+
+          describeMountTargets()
+        })
         regionPromises.push(regionPromise)
       })
     })

src/services/elb/data.ts

@@ -54,33 +54,40 @@ const getElbTags = async (
     )
   })
 
-const listElbData = async (
-  elb: ELB
-): Promise<LoadBalancerDescription[]> =>
-  new Promise<LoadBalancerDescription[]>(resolve => {
-    let loadBalancerData: LoadBalancerDescription[] = []
-    elb.describeLoadBalancers(
-      (err: AWSError, data: DescribeAccessPointsOutput) => {
-        if (err) {
-          errorLog.generateAwsErrorLog({
-            functionName: 'elb:describeLoadBalancers',
-            err,
-          })
-        }
-        if (!isEmpty(data)) {
-          const { LoadBalancerDescriptions: loadBalancerDescriptions = [] } =
-            data
-          logger.debug(lt.fetchedElbs(loadBalancerDescriptions.length))
-          loadBalancerData = loadBalancerDescriptions.map(lbDescription => ({
-            ...lbDescription,
-          }))
-          resolve(loadBalancerData)
+const listElbData = async (elb: ELB): Promise<LoadBalancerDescription[]> => {
+  const loadBalancerData: LoadBalancerDescription[] = []
+  return new Promise<LoadBalancerDescription[]>(resolve => {
+    const listAllData = (nextToken?: string): void => {
+      elb.describeLoadBalancers(
+        { Marker: nextToken },
+        (err: AWSError, data: DescribeAccessPointsOutput) => {
+          if (err) {
+            errorLog.generateAwsErrorLog({
+              functionName: 'elb:describeLoadBalancers',
+              err,
+            })
+          }
+          if (!isEmpty(data)) {
+            const { LoadBalancerDescriptions: loadBalancerDescriptions = [] } =
+              data
+            logger.debug(lt.fetchedElbs(loadBalancerDescriptions.length))
+            loadBalancerData.push(
+              ...loadBalancerDescriptions.map(lbDescription => ({
+                ...lbDescription,
+              }))
+            )
+          }
+          if (data.NextMarker) {
+            listAllData(data.NextMarker)
+          } else {
+            resolve(loadBalancerData)
+          }
         }
-
-        resolve(loadBalancerData)
-      }
-    )
+      )
+    }
+    listAllData()
   })
+}
 
 const listElbAttributes = async (
   elb: ELB,
@@ -127,7 +134,7 @@ export interface RawAwsElb extends LoadBalancerDescription {
 export default async ({
   regions,
   config,
-  account
+  account,
 }: {
   regions: string
   account: string
@@ -143,7 +150,9 @@ export default async ({
       return new Promise<void>(async resolveElbData => {
         // Get Load Balancer Data
         const elbDescriptionData = await listElbData(elbInstance)
-        const elbNames: string[] = elbDescriptionData.map(elb => elb.LoadBalancerName)
+        const elbNames: string[] = elbDescriptionData.map(
+          elb => elb.LoadBalancerName
+        )
 
         if (!isEmpty(elbNames)) {
           // Get Tags

src/services/redshift/data.ts

@@ -35,41 +35,48 @@ export default async ({
     const rsData: RawAwsRedshiftCluster[] = []
     const regionPromises = []
 
-    regions.split(',').map(region => {
-      const regionPromise = new Promise<void>(resolveRegion =>
-        new RS({ ...config, region, endpoint }).describeClusters(
-          {},
-          (err: AWSError, data: ClustersMessage) => {
-            if (err) {
-              errorLog.generateAwsErrorLog({
-                functionName: 'redshift:describeClusters',
-                err,
-              })
-            }
+    regions.split(',').forEach(region => {
+      const regionPromise = new Promise<void>(resolveRegion => {
+        const rs = new RS({ ...config, region, endpoint })
+        const listClusters = (nextToken?: string): void => {
+          rs.describeClusters(
+            { Marker: nextToken },
+            (err: AWSError, data: ClustersMessage) => {
+              if (err) {
+                errorLog.generateAwsErrorLog({
+                  functionName: 'redshift:describeClusters',
+                  err,
+                })
+              }
 
-            if (isEmpty(data)) {
-              return resolveRegion()
-            }
+              if (isEmpty(data)) {
+                return resolveRegion()
+              }
 
-            const { Clusters: clusters = [] } = data
+              const { Clusters: clusters = [] } = data
 
-            if (isEmpty(clusters)) {
-              return resolveRegion()
-            }
+              if (isEmpty(clusters)) {
+                return resolveRegion()
+              }
 
-            logger.debug(lt.fetchedRedshiftClusters(clusters.length))
-            rsData.push(
-              ...clusters.map(({ ...cluster }) => ({
-                ...cluster,
-                region,
-                Tags: convertAwsTagsToTagMap(cluster.Tags as AwsTag[]),
-              }))
-            )
-
-            resolveRegion()
-          }
-        )
-      )
+              logger.debug(lt.fetchedRedshiftClusters(clusters.length))
+              rsData.push(
+                ...clusters.map(({ ...cluster }) => ({
+                  ...cluster,
+                  region,
+                  Tags: convertAwsTagsToTagMap(cluster.Tags as AwsTag[]),
+                }))
+              )
+              if (data.Marker) {
+                listClusters(data.Marker)
+              } else {
+                resolveRegion()
+              }
+            }
+          )
+        }
+        listClusters()
+      })
       regionPromises.push(regionPromise)
     })