[rubrik] Add managed volumes datastream

.github/CODEOWNERS

@@ -336,6 +336,7 @@
 /packages/rapid7_insightvm @elastic/security-service-integrations
 /packages/redis @elastic/obs-infraobs-integrations
 /packages/redisenterprise @elastic/obs-infraobs-integrations
+/packages/rubrik @elastic/obs-infraobs-integrations
 /packages/salesforce @elastic/obs-infraobs-integrations
 /packages/santa @elastic/security-service-integrations
 /packages/security_detection_engine @elastic/protections

packages/rubrik/LICENSE.txt

@@ -0,0 +1,93 @@
+Elastic License 2.0
+
+URL: https://www.elastic.co/licensing/elastic-license
+
+## Acceptance
+
+By using the software, you agree to all of the terms and conditions below.
+
+## Copyright License
+
+The licensor grants you a non-exclusive, royalty-free, worldwide,
+non-sublicensable, non-transferable license to use, copy, distribute, make
+available, and prepare derivative works of the software, in each case subject to
+the limitations and conditions below.
+
+## Limitations
+
+You may not provide the software to third parties as a hosted or managed
+service, where the service provides users with access to any substantial set of
+the features or functionality of the software.
+
+You may not move, change, disable, or circumvent the license key functionality
+in the software, and you may not remove or obscure any functionality in the
+software that is protected by the license key.
+
+You may not alter, remove, or obscure any licensing, copyright, or other notices
+of the licensor in the software. Any use of the licensor’s trademarks is subject
+to applicable law.
+
+## Patents
+
+The licensor grants you a license, under any patent claims the licensor can
+license, or becomes able to license, to make, have made, use, sell, offer for
+sale, import and have imported the software, in each case subject to the
+limitations and conditions in this license. This license does not cover any
+patent claims that you cause to be infringed by modifications or additions to
+the software. If you or your company make any written claim that the software
+infringes or contributes to infringement of any patent, your patent license for
+the software granted under these terms ends immediately. If your company makes
+such a claim, your patent license ends immediately for work on behalf of your
+company.
+
+## Notices
+
+You must ensure that anyone who gets a copy of any part of the software from you
+also gets a copy of these terms.
+
+If you modify the software, you must include in any modified copies of the
+software prominent notices stating that you have modified the software.
+
+## No Other Rights
+
+These terms do not imply any licenses other than those expressly granted in
+these terms.
+
+## Termination
+
+If you use the software in violation of these terms, such use is not licensed,
+and your licenses will automatically terminate. If the licensor provides you
+with a notice of your violation, and you cease all violation of this license no
+later than 30 days after you receive that notice, your licenses will be
+reinstated retroactively. However, if you violate these terms after such
+reinstatement, any additional violation of these terms will cause your licenses
+to terminate automatically and permanently.
+
+## No Liability
+
+*As far as the law allows, the software comes as is, without any warranty or
+condition, and the licensor will not be liable to you for any damages arising
+out of these terms or the use or nature of the software, under any kind of
+legal claim.*
+
+## Definitions
+
+The **licensor** is the entity offering these terms, and the **software** is the
+software the licensor makes available under these terms, including any portion
+of it.
+
+**you** refers to the individual or entity agreeing to these terms.
+
+**your company** is any legal entity, sole proprietorship, or other kind of
+organization that you work for, plus all organizations that have control over,
+are under the control of, or are under common control with that
+organization. **control** means ownership of substantially all the assets of an
+entity, or the power to direct its management and policies by vote, contract, or
+otherwise. Control can be direct or indirect.
+
+**your licenses** are all the licenses granted to you for the software under
+these terms.
+
+**use** means anything you do with the software requiring one of your licenses.
+
+**trademark** means trademarks, service marks, and similar rights.

packages/rubrik/_dev/build/build.yml

@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: "[email protected]"

packages/rubrik/_dev/build/docs/README.md

@@ -0,0 +1 @@
+# Instructions to setup Rubrik Integration

packages/rubrik/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "0.2.0"
+  changes:
+    - description: Create an integration for Rubrik with Filesets datastream
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/12111

packages/rubrik/data_stream/linux_filesets/_dev/test/pipeline/test-fileset-metrics.log

@@ -0,0 +1 @@
+{"node":{"host":{"osType":"LINUX"},"name":"Linux","reportWorkload":{"archiveSnapshots":0,"archiveStorage":0,"complianceStatus":"OUT_OF_COMPLIANCE","localStorage":1286184885,"name":"Linux","totalSnapshots":6}}}

packages/rubrik/data_stream/linux_filesets/_dev/test/pipeline/test-fileset-metrics.log-expected.json

@@ -0,0 +1,26 @@
+{
+    "expected": [
+        {
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "kind": "metric",
+                "original": "{\"node\":{\"host\":{\"osType\":\"LINUX\"},\"name\":\"Linux\",\"reportWorkload\":{\"archiveSnapshots\":0,\"archiveStorage\":0,\"complianceStatus\":\"OUT_OF_COMPLIANCE\",\"localStorage\":1286184885,\"name\":\"Linux\",\"totalSnapshots\":6}}}"
+            },
+            "rubrik": {
+                "fileset": {
+                    "archive_snapshots": 0,
+                    "archive_storage": 0,
+                    "compliance_status": "OUT_OF_COMPLIANCE",
+                    "host": {
+                        "os_type": "LINUX"
+                    },
+                    "local_storage": 1286184885,
+                    "name": "Linux",
+                    "total_snapshots": 6
+                }
+            }
+        }
+    ]
+}

packages/rubrik/data_stream/linux_filesets/agent/stream/httpjson.yml.hbs

@@ -0,0 +1,102 @@
+config_version: "2"
+interval: {{interval}}
+{{#if enable_request_tracer}}
+request.tracer.filename: "../../logs/httpjson/http-request-trace-*.ndjson"
+request.tracer.maxbackups: 5
+{{/if}}
+request.method: "POST"
+request.url: https://{{hostname}}.my.rubrik.com/api/graphql
+{{#if http_client_timeout}}
+request.timeout: {{http_client_timeout}}
+{{/if}}
+request.transforms:
+  - set:
+      target: "header.Content-Type"
+      value: "application/json"
+auth.oauth2:
+  client.id: {{client_id}}
+  client.secret: {{client_secret}}
+  token_url: {{token_url}}
+
+
+request.body:
+  query: |
+    query FilesetTemplateListQuery($hostRoot: HostRoot!, $first: Int, $after: String, $filter: [Filter!], $sortBy: HierarchySortByField, $sortOrder: SortOrder) {
+      filesetTemplates(hostRoot: $hostRoot, first: $first, after: $after, filter: $filter, sortBy: $sortBy, sortOrder: $sortOrder) {
+        edges {
+          node {
+            id
+            name
+            physicalChildConnection(filter: $filter) {
+              edges {
+                node {
+                  ... on LinuxFileset {
+                    name
+                    host {
+                      osType                 
+                    }
+                    reportWorkload {
+                      name
+                      complianceStatus
+                      archiveStorage
+                      localStorage
+                      totalSnapshots
+                      archiveSnapshots
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  variables: |
+    {
+      "hostRoot": "LINUX_HOST_ROOT",
+      {{#if first}}
+      "first": {{first}},
+      {{/if}}
+      "filter": [
+        {{#if filters}}
+          {{#each filters}}
+            {
+              "field": "{{field}}",
+              "texts": [{{#each values}}"{{this}}"{{#unless @last}}, {{/unless}}{{/each}}]
+            }{{#unless @last}},{{/unless}}
+          {{/each}}
+        {{/if}}
+      ]
+    }      
+
+response.split:
+  target: body.data.filesetTemplates.edges
+  type: array
+  keep_parent: false
+  ignore_empty_value: true
+  split:
+    target: body.node.physicalChildConnection.edges
+    type: array
+    keep_parent: false
+    ignore_empty_value: true
+
+
+{{#if tags.length}}
+tags:
+{{else if preserve_original_event}}
+tags:
+{{/if}}
+{{#each tags as |tag i|}}
+  - {{tag}}
+{{/each}}
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/rubrik/data_stream/linux_filesets/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,97 @@
+---
+description: Pipeline to parse rubrik.fileset workload data
+processors:
+  - set:
+      field: ecs.version
+      value: 8.11.0
+      ignore_failure: true
+      ignore_empty_value: true
+  - set:
+      field: event.kind
+      value: metric
+      ignore_failure: true
+      ignore_empty_value: true
+  - rename:
+      field: message
+      target_field: event.original
+      ignore_missing: true
+      if: ctx.event?.original == null
+      description: 'Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document.'
+  - remove:
+      field: message
+      ignore_missing: true
+      if: 'ctx.event?.original != null'
+      description: 'The `message` field is no longer required if the document has an `event.original` field.'
+  - json:
+      field: event.original
+      target_field: response
+      on_failure:
+        - set:
+            field: error.message
+            value: 'Received invalid JSON. Unable to parse the source log message'
+            ignore_failure: true
+  - rename:
+      field: response.node.name
+      target_field: rubrik.fileset.name
+      ignore_missing: true
+      if: 'ctx.response?.node?.name != null'
+  - rename:
+      field: response.node.host.osType
+      target_field: rubrik.fileset.host.os_type
+      ignore_missing: true
+      if: 'ctx.response?.node?.host?.osType != null'
+  - rename:
+      field: response.node.reportWorkload.complianceStatus
+      target_field: rubrik.fileset.compliance_status
+      ignore_missing: true
+      if: 'ctx.response?.node?.reportWorkload?.complianceStatus != null'
+  - rename:
+      field: response.node.reportWorkload.archiveSnapshots
+      target_field: rubrik.fileset.archive_snapshots
+      ignore_missing: true
+      if: 'ctx.response?.node?.reportWorkload?.archiveSnapshots != null'
+  - rename:
+      field: response.node.reportWorkload.archiveStorage
+      target_field: rubrik.fileset.archive_storage
+      ignore_missing: true
+      if: 'ctx.response?.node?.reportWorkload?.archiveStorage != null'
+  - rename:
+      field: response.node.reportWorkload.localStorage
+      target_field: rubrik.fileset.local_storage
+      ignore_missing: true
+      if: 'ctx.response?.node?.reportWorkload?.localStorage != null'
+  - rename:
+      field: response.node.reportWorkload.totalSnapshots
+      target_field: rubrik.fileset.total_snapshots
+      ignore_missing: true
+      if: 'ctx.response?.node?.reportWorkload?.totalSnapshots != null'
+  - remove:
+      field: response
+      ignore_missing: true
+      if: 'ctx.rubrik.fileset != null'
+  - script:
+      description: Removes null/empty values under rubrik fileset.
+      lang: painless
+      source: |
+        boolean drop(Object o) {
+          if (o == null || o == "") {
+            return true;
+          } else if (o instanceof Map) {
+            ((Map) o).values().removeIf(v -> drop(v));
+            return (((Map) o).size() == 0);
+          } else if (o instanceof List) {
+            ((List) o).removeIf(v -> drop(v));
+            return (((List) o).size() == 0);
+          }
+          return false;
+        }
+        if (ctx.containsKey('rubrik') && ctx.rubrik.containsKey('fileset')) {
+          drop(ctx.rubrik.fileset);
+        }         
+on_failure:
+  - set:
+      field: event.kind
+      value: pipeline_error
+  - set:
+      field: error.message
+      value: "{{{_ingest.on_failure_message}}}"     

packages/rubrik/data_stream/linux_filesets/fields/base-fields.yml

@@ -0,0 +1,12 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.

packages/rubrik/data_stream/linux_filesets/fields/fields.yml

@@ -0,0 +1,31 @@
+- name: rubrik.fileset
+  type: group
+  fields:
+    - name: name
+      type: keyword
+      description: The name of the Fileset.
+    - name: compliance_status
+      type: keyword
+      description: The current compliance status of the workload.
+    - name: archive_storage
+      type: long
+      unit: byte
+      metric_type: gauge
+      description: The amount of storage used by archived snapshots.
+    - name: local_storage
+      type: long
+      unit: byte
+      metric_type: gauge
+      description: The local storage size in bytes.
+    - name: total_snapshots
+      type: long
+      metric_type: gauge
+      description: The total number of snapshots present for the workload.
+    - name: archive_snapshots
+      type: long
+      metric_type: gauge
+      description: The number of snapshots that have been archived.
+    - name: host.os_type
+      type: keyword
+      dimension: true
+      description: The operating system type of the host.