diff --git a/plugins/command-manager/build.gradle b/plugins/command-manager/build.gradle index d62a1e87..688cff06 100644 --- a/plugins/command-manager/build.gradle +++ b/plugins/command-manager/build.gradle @@ -1,9 +1,13 @@ import org.opensearch.gradle.test.RestIntegTestTask +import java.util.concurrent.Callable buildscript { ext { opensearch_version = System.getProperty("opensearch.version", "2.18.0-SNAPSHOT") + opensearch_no_snapshot = opensearch_version.replace("-SNAPSHOT","") + opensearch_build = opensearch_no_snapshot + ".0" + job_scheduler_version = System.getProperty("job_scheduler.version", opensearch_build) wazuh_version = System.getProperty("version", "5.0.0") revision = System.getProperty("revision", "0") } @@ -24,7 +28,7 @@ apply plugin: 'java' apply plugin: 'idea' apply plugin: 'eclipse' apply plugin: 'opensearch.opensearchplugin' -apply plugin: 'opensearch.yaml-rest-test' +apply plugin: 'opensearch.rest-test' apply plugin: 'opensearch.pluginzip' def pluginName = 'wazuh-indexer-command-manager' @@ -67,6 +71,7 @@ opensearchplugin { name pluginName description pluginDescription classname "${projectPath}.${pathToPlugin}.${pluginClassName}" + extendedPlugins = ['opensearch-job-scheduler'] licenseFile rootProject.file('LICENSE.txt') noticeFile rootProject.file('NOTICE.txt') } @@ -88,6 +93,10 @@ def versions = [ imposter: "4.1.2" ] +configurations { + zipArchive +} + dependencies { implementation "org.apache.httpcomponents.client5:httpclient5:${versions.httpclient5}" implementation "org.apache.httpcomponents.core5:httpcore5-h2:${versions.httpcore5}" @@ -99,6 +108,11 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" implementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" + // Job Scheduler stuff + zipArchive group: 'org.opensearch.plugin', name: 'opensearch-job-scheduler', version: "${opensearch_build}" + // implementation "org.opensearch:opensearch:${opensearch_version}" + compileOnly "org.opensearch:opensearch-job-scheduler-spi:${job_scheduler_version}" + testImplementation "junit:junit:${versions.junit}" // imposter @@ -134,30 +148,37 @@ test { jvmArgs "-Djava.security.policy=./plugins/command-manager/src/main/plugin-metadata/plugin-security.policy/plugin-security.policy" } -task integTest(type: RestIntegTestTask) { - description = "Run tests against a cluster" - testClassesDirs = sourceSets.test.output.classesDirs - classpath = sourceSets.test.runtimeClasspath +def getJobSchedulerPlugin() { + provider(new Callable() { + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return configurations.zipArchive.asFileTree.matching { + include '**/opensearch-job-scheduler*' + }.singleFile + } + } + } + }) } -tasks.named("check").configure { dependsOn(integTest) } -integTest { +testClusters.integTest { + plugin(getJobSchedulerPlugin()) + testDistribution = "ARCHIVE" + // This installs our plugin into the testClusters + plugin(project.tasks.bundlePlugin.archiveFile) + // The --debug-jvm command-line option makes the cluster debuggable; this makes the tests debuggable if (System.getProperty("test.debug") != null) { jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005' } -} - -testClusters.integTest { - testDistribution = "INTEG_TEST" - //testDistribution = "ARCHIVE" - // This installs our plugin into the testClusters - plugin(project.tasks.bundlePlugin.archiveFile) // add customized keystore keystore 'm_api.auth.username', 'admin' keystore 'm_api.auth.password', 'test' - keystore 'm_api.uri', 'http://127.0.0.1:55000' // base URI of the M_API + keystore 'm_api.uri', 'https://127.0.0.1:55000' // base URI of the M_API } run { diff --git a/plugins/command-manager/openapi.yml b/plugins/command-manager/openapi.yml index a5706102..a2d44918 100644 --- a/plugins/command-manager/openapi.yml +++ b/plugins/command-manager/openapi.yml @@ -9,8 +9,8 @@ paths: post: tags: - "authentication" - summary: Add a new command to the queue. - description: Add a new command to the queue. + summary: Mock of the Wazuh Server M_API authentication endpoint. + description: Returns a JWT. responses: "200": description: OK @@ -18,20 +18,31 @@ paths: post: tags: - "commands" - summary: Add a new command to the queue. - description: Add a new command to the queue. + summary: Add commands. + description: Receives and processes an array of commands. requestBody: required: true content: "application/json": schema: - $ref: "#/components/schemas/Command" + $ref: "#/components/schemas/Commands" responses: "200": description: OK + "400": + description: parsing_exception + "500": + description: Internal server error (boom!) components: schemas: + Commands: + type: object + properties: + commands: + type: array + items: + $ref: '#/components/schemas/Command' Command: type: object properties: diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/CommandManagerPlugin.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/CommandManagerPlugin.java index 71b805fd..d52301ba 100644 --- a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/CommandManagerPlugin.java +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/CommandManagerPlugin.java @@ -8,33 +8,52 @@ */ package com.wazuh.commandmanager; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.index.IndexResponse; import org.opensearch.client.Client; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.UUIDs; import org.opensearch.common.settings.*; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.IndexScopedSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.settings.SettingsFilter; +import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; +import org.opensearch.jobscheduler.spi.JobSchedulerExtension; +import org.opensearch.jobscheduler.spi.ScheduledJobParser; +import org.opensearch.jobscheduler.spi.ScheduledJobRunner; +import org.opensearch.jobscheduler.spi.schedule.ScheduleParser; import org.opensearch.plugins.ActionPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.ReloadablePlugin; import org.opensearch.repositories.RepositoriesService; -import org.opensearch.rest.RestController; -import org.opensearch.rest.RestHandler; +import org.opensearch.rest.*; import org.opensearch.script.ScriptService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.watcher.ResourceWatcherService; import java.io.IOException; +import java.time.Instant; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.concurrent.CompletableFuture; import java.util.function.Supplier; import com.wazuh.commandmanager.index.CommandIndex; +import com.wazuh.commandmanager.jobscheduler.CommandManagerJobParameter; +import com.wazuh.commandmanager.jobscheduler.CommandManagerJobRunner; +import com.wazuh.commandmanager.jobscheduler.JobDocument; import com.wazuh.commandmanager.rest.RestPostCommandAction; import com.wazuh.commandmanager.settings.PluginSettings; import com.wazuh.commandmanager.utils.httpclient.HttpRestClient; @@ -42,15 +61,31 @@ /** * The Command Manager plugin exposes an HTTP API with a single endpoint to receive raw commands * from the Wazuh Server. These commands are processed, indexed and sent back to the Server for its - * delivery to, in most cases, the Agents. + * delivery to, in most cases, the Agents. The Command Manager plugin exposes an HTTP API with a + * single endpoint to receive raw commands from the Wazuh Server. These commands are processed, + * indexed and sent back to the Server for its delivery to, in most cases, the Agents. + * + *

The Command Manager plugin is also a JobScheduler extension plugin. */ -public class CommandManagerPlugin extends Plugin implements ActionPlugin, ReloadablePlugin { +public class CommandManagerPlugin extends Plugin + implements ActionPlugin, ReloadablePlugin, JobSchedulerExtension { public static final String COMMAND_MANAGER_BASE_URI = "/_plugins/_command_manager"; public static final String COMMANDS_URI = COMMAND_MANAGER_BASE_URI + "/commands"; public static final String COMMAND_MANAGER_INDEX_NAME = ".commands"; public static final String COMMAND_MANAGER_INDEX_TEMPLATE_NAME = "index-template-commands"; + public static final String COMMAND_DOCUMENT_PARENT_OBJECT_NAME = "command"; + public static final String JOB_INDEX_NAME = ".scheduled-commands"; + public static final String JOB_INDEX_TEMPLATE_NAME = "index-template-scheduled-commands"; + public static final Integer JOB_PERIOD_MINUTES = 1; + public static final Integer PAGE_SIZE = 100; + public static final Long DEFAULT_TIMEOUT_SECONDS = 20L; + public static final TimeValue PIT_KEEP_ALIVE_SECONDS = TimeValue.timeValueSeconds(30L); + + private static final Logger log = LogManager.getLogger(CommandManagerPlugin.class); + public static final String JOB_TYPE = "command_manager_scheduler_extension"; private CommandIndex commandIndex; + private JobDocument jobDocument; @Override public Collection createComponents( @@ -68,9 +103,51 @@ public Collection createComponents( this.commandIndex = new CommandIndex(client, clusterService, threadPool); PluginSettings.getInstance(environment.settings()); + // JobSchedulerExtension stuff + CommandManagerJobRunner.getInstance() + .setThreadPool(threadPool) + .setClient(client) + .setClusterService(clusterService) + .setEnvironment(environment); + + scheduleCommandJob(client, clusterService, threadPool); + return Collections.emptyList(); } + /** + * Indexes a document into the jobs index, so that JobScheduler plugin can run it + * + * @param client: The cluster client, used for indexing + * @param clusterService: Provides the addListener method. We use it to determine if this is a + * new cluster. + * @param threadPool: Used by jobDocument to create the document in a thread. + */ + private void scheduleCommandJob( + Client client, ClusterService clusterService, ThreadPool threadPool) { + clusterService.addListener( + event -> { + if (event.localNodeClusterManager() && event.isNewCluster()) { + jobDocument = JobDocument.getInstance(); + CompletableFuture indexResponseCompletableFuture = + jobDocument.create( + clusterService, + client, + threadPool, + UUIDs.base64UUID(), + getJobType(), + JOB_PERIOD_MINUTES); + indexResponseCompletableFuture.thenAccept( + indexResponse -> { + log.info( + "Scheduled task successfully, response: {}", + indexResponse.getResult().toString()); + }); + } + }); + } + + @Override public List getRestHandlers( Settings settings, RestController restController, @@ -100,6 +177,75 @@ public void reload(Settings settings) { // xxxService.refreshAndClearCache(commandManagerSettings); } + @Override + public String getJobType() { + return CommandManagerPlugin.JOB_TYPE; + } + + @Override + public String getJobIndex() { + return CommandManagerPlugin.JOB_INDEX_NAME; + } + + @Override + public ScheduledJobRunner getJobRunner() { + log.info("getJobRunner() executed"); + return CommandManagerJobRunner.getInstance(); + } + + @Override + public ScheduledJobParser getJobParser() { + log.info("getJobParser() executed"); + return (parser, id, jobDocVersion) -> { + CommandManagerJobParameter jobParameter = new CommandManagerJobParameter(); + XContentParserUtils.ensureExpectedToken( + XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + + while (!parser.nextToken().equals(XContentParser.Token.END_OBJECT)) { + String fieldName = parser.currentName(); + parser.nextToken(); + switch (fieldName) { + case CommandManagerJobParameter.NAME_FIELD: + jobParameter.setJobName(parser.text()); + break; + case CommandManagerJobParameter.ENABLED_FIELD: + jobParameter.setEnabled(parser.booleanValue()); + break; + case CommandManagerJobParameter.ENABLED_TIME_FIELD: + jobParameter.setEnabledTime(parseInstantValue(parser)); + break; + case CommandManagerJobParameter.LAST_UPDATE_TIME_FIELD: + jobParameter.setLastUpdateTime(parseInstantValue(parser)); + break; + case CommandManagerJobParameter.SCHEDULE_FIELD: + jobParameter.setSchedule(ScheduleParser.parse(parser)); + break; + default: + XContentParserUtils.throwUnknownToken( + parser.currentToken(), parser.getTokenLocation()); + } + } + return jobParameter; + }; + } + + /** + * Returns the proper Instant object with milliseconds from the Unix epoch when the current + * token actually holds a value. + * + * @param parser: The parser as provided by JobScheduler + */ + private Instant parseInstantValue(XContentParser parser) throws IOException { + if (XContentParser.Token.VALUE_NULL.equals(parser.currentToken())) { + return null; + } + if (parser.currentToken().isValue()) { + return Instant.ofEpochMilli(parser.longValue()); + } + XContentParserUtils.throwUnknownToken(parser.currentToken(), parser.getTokenLocation()); + return null; + } + /** * Close the resources opened by this plugin. * diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/index/CommandIndex.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/index/CommandIndex.java index ea8f2c27..8db3fd56 100644 --- a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/index/CommandIndex.java +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/index/CommandIndex.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.admin.indices.template.put.PutIndexTemplateRequest; +import org.opensearch.action.bulk.BulkRequest; import org.opensearch.action.index.IndexRequest; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.Client; @@ -24,6 +25,7 @@ import org.opensearch.threadpool.ThreadPool; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -59,36 +61,38 @@ public CommandIndex(Client client, ClusterService clusterService, ThreadPool thr * @param document instance of the document model to persist in the index. * @return A CompletableFuture with the RestStatus response from the operation */ + @Deprecated public CompletableFuture asyncCreate(Document document) { CompletableFuture future = new CompletableFuture<>(); ExecutorService executor = this.threadPool.executor(ThreadPool.Names.WRITE); - // Create index template if it does not exist. - if (!indexTemplateExists(CommandManagerPlugin.COMMAND_MANAGER_INDEX_TEMPLATE_NAME)) { - putIndexTemplate(CommandManagerPlugin.COMMAND_MANAGER_INDEX_TEMPLATE_NAME); - } else { - log.info( - "Index template {} already exists. Skipping creation.", - CommandManagerPlugin.COMMAND_MANAGER_INDEX_TEMPLATE_NAME); - } - log.info("Indexing command with id [{}]", document.getId()); try { - IndexRequest request = - new IndexRequest() - .index(CommandManagerPlugin.COMMAND_MANAGER_INDEX_NAME) - .source( - document.toXContent( - XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) - .id(document.getId()) - .create(true); + IndexRequest request = createIndexRequest(document); executor.submit( () -> { try (ThreadContext.StoredContext ignored = this.threadPool.getThreadContext().stashContext()) { + // Create index template if it does not exist. + if (!IndexTemplateUtils.indexTemplateExists( + this.clusterService, + CommandManagerPlugin.COMMAND_MANAGER_INDEX_TEMPLATE_NAME)) { + IndexTemplateUtils.putIndexTemplate( + this.client, + CommandManagerPlugin.COMMAND_MANAGER_INDEX_TEMPLATE_NAME); + } else { + log.info( + "Index template {} already exists. Skipping creation.", + CommandManagerPlugin.COMMAND_MANAGER_INDEX_TEMPLATE_NAME); + } + RestStatus restStatus = client.index(request).actionGet().status(); future.complete(restStatus); } catch (Exception e) { + log.error( + "Error indexing command with id [{}] due to {}", + document.getId(), + e.getMessage()); future.completeExceptionally(e); } }); @@ -98,6 +102,54 @@ public CompletableFuture asyncCreate(Document document) { return future; } + /** + * @param documents list of instances of the document model to persist in the index. + * @return A CompletableFuture with the RestStatus response from the operation + */ + public CompletableFuture asyncBulkCreate(ArrayList documents) { + CompletableFuture future = new CompletableFuture<>(); + ExecutorService executor = this.threadPool.executor(ThreadPool.Names.WRITE); + + BulkRequest bulkRequest = new BulkRequest(); + for (Document document : documents) { + log.info("Adding command with id [{}] to the bulk request", document.getId()); + try { + bulkRequest.add(createIndexRequest(document)); + } catch (IOException e) { + log.error( + "Error creating IndexRequest with document id [{}] due to {}", + document.getId(), + e); + } + } + + executor.submit( + () -> { + try (ThreadContext.StoredContext ignored = + this.threadPool.getThreadContext().stashContext()) { + // Create index template if it does not exist. + if (!IndexTemplateUtils.indexTemplateExists( + this.clusterService, + CommandManagerPlugin.COMMAND_MANAGER_INDEX_TEMPLATE_NAME)) { + IndexTemplateUtils.putIndexTemplate( + this.client, + CommandManagerPlugin.COMMAND_MANAGER_INDEX_TEMPLATE_NAME); + } else { + log.info( + "Index template {} already exists. Skipping creation.", + CommandManagerPlugin.COMMAND_MANAGER_INDEX_TEMPLATE_NAME); + } + + RestStatus restStatus = client.bulk(bulkRequest).actionGet().status(); + future.complete(restStatus); + } catch (Exception e) { + log.error("Error indexing commands with bulk due to {}", e.getMessage()); + future.completeExceptionally(e); + } + }); + return future; + } + /** * Checks for the existence of the given index template in the cluster. * @@ -118,7 +170,6 @@ public boolean indexTemplateExists(String template_name) { * @param templateName : The name if the index template to load */ public void putIndexTemplate(String templateName) { - ExecutorService executor = this.threadPool.executor(ThreadPool.Names.WRITE); try { // @throws IOException Map template = IndexTemplateUtils.fromFile(templateName + ".json"); @@ -130,21 +181,33 @@ public void putIndexTemplate(String templateName) { .name(templateName) .patterns((List) template.get("index_patterns")); - executor.submit( - () -> { - AcknowledgedResponse acknowledgedResponse = - this.client - .admin() - .indices() - .putTemplate(putIndexTemplateRequest) - .actionGet(); - if (acknowledgedResponse.isAcknowledged()) { - log.info("Index template [{}] created successfully", templateName); - } - }); + AcknowledgedResponse acknowledgedResponse = + this.client.admin().indices().putTemplate(putIndexTemplateRequest).actionGet(); + if (acknowledgedResponse.isAcknowledged()) { + log.info("Index template [{}] created successfully", templateName); + } } catch (IOException e) { log.error("Error reading index template [{}] from filesystem", templateName); } } + + /** + * Create an IndexRequest object from a Document object. + * + * @param document the document to create the IndexRequest for COMMAND_MANAGER_INDEX + * @return an IndexRequest object + * @throws IOException thrown by XContentFactory.jsonBuilder() + */ + private IndexRequest createIndexRequest(Document document) throws IOException { + IndexRequest request = + new IndexRequest() + .index(CommandManagerPlugin.COMMAND_MANAGER_INDEX_NAME) + .source( + document.toXContent( + XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .id(document.getId()) + .create(true); + return request; + } } diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/CommandManagerJobParameter.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/CommandManagerJobParameter.java new file mode 100644 index 00000000..c6da74a3 --- /dev/null +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/CommandManagerJobParameter.java @@ -0,0 +1,113 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package com.wazuh.commandmanager.jobscheduler; + +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.jobscheduler.spi.ScheduledJobParameter; +import org.opensearch.jobscheduler.spi.schedule.Schedule; + +import java.io.IOException; +import java.time.Instant; + +/** A model for the parameters and schema to be indexed to the jobs index. */ +public class CommandManagerJobParameter implements ScheduledJobParameter { + public static final String NAME_FIELD = "name"; + public static final String ENABLED_FIELD = "enabled"; + public static final String LAST_UPDATE_TIME_FIELD = "last_update_time"; + public static final String LAST_UPDATE_TIME_FIELD_READABLE = "last_update_time_field"; + public static final String SCHEDULE_FIELD = "schedule"; + public static final String ENABLED_TIME_FIELD = "enabled_time"; + public static final String ENABLED_TIME_FIELD_READABLE = "enabled_time_field"; + + private String jobName; + private Instant lastUpdateTime; + private Instant enabledTime; + private boolean isEnabled; + private Schedule schedule; + + public CommandManagerJobParameter() {} + + public CommandManagerJobParameter(String jobName, Schedule schedule) { + this.jobName = jobName; + this.schedule = schedule; + + Instant now = Instant.now(); + this.isEnabled = true; + this.enabledTime = now; + this.lastUpdateTime = now; + } + + @Override + public String getName() { + return this.jobName; + } + + @Override + public Instant getLastUpdateTime() { + return this.lastUpdateTime; + } + + @Override + public Instant getEnabledTime() { + return this.enabledTime; + } + + @Override + public Schedule getSchedule() { + return this.schedule; + } + + @Override + public boolean isEnabled() { + return this.isEnabled; + } + + public void setJobName(String jobName) { + this.jobName = jobName; + } + + public void setLastUpdateTime(Instant lastUpdateTime) { + this.lastUpdateTime = lastUpdateTime; + } + + public void setEnabledTime(Instant enabledTime) { + this.enabledTime = enabledTime; + } + + public void setEnabled(boolean enabled) { + isEnabled = enabled; + } + + public void setSchedule(Schedule schedule) { + this.schedule = schedule; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(NAME_FIELD, this.jobName); + builder.field(ENABLED_FIELD, this.isEnabled); + builder.field(SCHEDULE_FIELD, this.schedule); + if (this.enabledTime != null) { + builder.timeField( + ENABLED_TIME_FIELD, + ENABLED_TIME_FIELD_READABLE, + this.enabledTime.toEpochMilli()); + } + if (this.lastUpdateTime != null) { + builder.timeField( + LAST_UPDATE_TIME_FIELD, + LAST_UPDATE_TIME_FIELD_READABLE, + this.lastUpdateTime.toEpochMilli()); + } + builder.endObject(); + + return builder; + } +} diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/CommandManagerJobRunner.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/CommandManagerJobRunner.java new file mode 100644 index 00000000..3f2e32e0 --- /dev/null +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/CommandManagerJobRunner.java @@ -0,0 +1,93 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package com.wazuh.commandmanager.jobscheduler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.env.Environment; +import org.opensearch.jobscheduler.spi.JobExecutionContext; +import org.opensearch.jobscheduler.spi.ScheduledJobParameter; +import org.opensearch.jobscheduler.spi.ScheduledJobRunner; +import org.opensearch.threadpool.ThreadPool; + +import com.wazuh.commandmanager.CommandManagerPlugin; + +/** + * Implements the ScheduledJobRunner interface, which exposes the runJob() method, which executes + * the job's logic in its own thread. + */ +public class CommandManagerJobRunner implements ScheduledJobRunner { + + private static final Logger log = LogManager.getLogger(CommandManagerJobRunner.class); + private static CommandManagerJobRunner INSTANCE; + private ThreadPool threadPool; + private ClusterService clusterService; + + private Client client; + private Environment environment; + + private CommandManagerJobRunner() { + // Singleton class, use getJobRunner method instead of constructor + } + + public static CommandManagerJobRunner getInstance() { + log.info("Getting Job Runner Instance"); + if (INSTANCE != null) { + return INSTANCE; + } + synchronized (CommandManagerJobRunner.class) { + if (INSTANCE != null) { + return INSTANCE; + } + INSTANCE = new CommandManagerJobRunner(); + return INSTANCE; + } + } + + private boolean commandManagerIndexExists() { + return this.clusterService + .state() + .routingTable() + .hasIndex(CommandManagerPlugin.COMMAND_MANAGER_INDEX_NAME); + } + + @Override + public void runJob(ScheduledJobParameter jobParameter, JobExecutionContext context) { + if (!commandManagerIndexExists()) { + log.info( + "{} index not yet created, not running command manager jobs", + CommandManagerPlugin.COMMAND_MANAGER_INDEX_NAME); + return; + } + SearchThread searchThread = new SearchThread(this.client); + threadPool.generic().submit(searchThread); + } + + public CommandManagerJobRunner setClusterService(ClusterService clusterService) { + this.clusterService = clusterService; + return getInstance(); + } + + public CommandManagerJobRunner setClient(Client client) { + this.client = client; + return getInstance(); + } + + public CommandManagerJobRunner setEnvironment(Environment environment) { + this.environment = environment; + return getInstance(); + } + + public CommandManagerJobRunner setThreadPool(ThreadPool threadPool) { + this.threadPool = threadPool; + return getInstance(); + } +} diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/JobDocument.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/JobDocument.java new file mode 100644 index 00000000..5522c9c9 --- /dev/null +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/JobDocument.java @@ -0,0 +1,89 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package com.wazuh.commandmanager.jobscheduler; + +import com.wazuh.commandmanager.utils.IndexTemplateUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.threadpool.ThreadPool; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; + +import com.wazuh.commandmanager.CommandManagerPlugin; + +/** Indexes the command job to the Jobs index. */ +public class JobDocument { + private static final Logger log = LogManager.getLogger(JobDocument.class); + private static final JobDocument INSTANCE = new JobDocument(); + + private JobDocument() {} + + public static JobDocument getInstance() { + log.info("Getting JobDocument Instance"); + return INSTANCE; + } + + /** + * Writes a CommandManagerJobParameter type document to the jobs index + * + * @param client: The cluster's client + * @param threadPool: The cluster's threadPool + * @param id: The job ID to be used + * @param jobName: The name of the job + * @param interval: The interval the action is expected to run at + * @return a CompletableFuture that will hold the IndexResponse. + */ + public CompletableFuture create( + ClusterService clusterService, Client client, ThreadPool threadPool, String id, String jobName, Integer interval) { + CompletableFuture completableFuture = new CompletableFuture<>(); + ExecutorService executorService = threadPool.executor(ThreadPool.Names.WRITE); + CommandManagerJobParameter jobParameter = + new CommandManagerJobParameter( + jobName, new IntervalSchedule(Instant.now(), interval, ChronoUnit.MINUTES)); + try { + IndexRequest indexRequest = + new IndexRequest() + .index(CommandManagerPlugin.JOB_INDEX_NAME) + .id(id) + .source(jobParameter.toXContent(JsonXContent.contentBuilder(), null)) + .create(true); + executorService.submit( + () -> { + try (ThreadContext.StoredContext ignored = + threadPool.getThreadContext().stashContext()) { + if (!IndexTemplateUtils.indexTemplateExists(clusterService,CommandManagerPlugin.JOB_INDEX_TEMPLATE_NAME)) { + IndexTemplateUtils.putIndexTemplate(client, CommandManagerPlugin.JOB_INDEX_TEMPLATE_NAME); + } else { + log.info( + "Index template {} already exists. Skipping creation.", + CommandManagerPlugin.JOB_INDEX_NAME); + } + IndexResponse indexResponse = client.index(indexRequest).actionGet(); + completableFuture.complete(indexResponse); + } catch (Exception e) { + completableFuture.completeExceptionally(e); + } + }); + } catch (IOException e) { + log.error("Failed to index command with ID {}: {}", id, e); + } + return completableFuture; + } +} diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/SearchThread.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/SearchThread.java new file mode 100644 index 00000000..b8e52fe9 --- /dev/null +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/jobscheduler/SearchThread.java @@ -0,0 +1,324 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package com.wazuh.commandmanager.jobscheduler; + +import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; +import org.apache.hc.core5.net.URIBuilder; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.search.CreatePitRequest; +import org.opensearch.action.search.CreatePitResponse; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.client.Client; +import org.opensearch.common.action.ActionFuture; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.TermQueryBuilder; +import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHits; +import org.opensearch.search.builder.PointInTimeBuilder; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.sort.SortOrder; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.*; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; + +import com.wazuh.commandmanager.CommandManagerPlugin; +import com.wazuh.commandmanager.model.Command; +import com.wazuh.commandmanager.model.Document; +import com.wazuh.commandmanager.model.Status; +import com.wazuh.commandmanager.settings.PluginSettings; +import com.wazuh.commandmanager.utils.httpclient.AuthHttpRestClient; + +/** + * The class in charge of searching and managing commands in {@link Status#PENDING} status and of + * submitting them to the destination client. + */ +public class SearchThread implements Runnable { + public static final String COMMAND_STATUS_FIELD = + Command.COMMAND + "." + Command.STATUS; + public static final String COMMAND_ORDER_ID_FIELD = + Command.COMMAND + "." + Command.ORDER_ID; + public static final String COMMAND_TIMEOUT_FIELD = Command.COMMAND + "." + Command.TIMEOUT; + public static final String DELIVERY_TIMESTAMP_FIELD = Document.DELIVERY_TIMESTAMP; + private static final Logger log = LogManager.getLogger(SearchThread.class); + public static final String ORDERS_OBJECT = "/orders"; + private final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + private final Client client; + private SearchResponse currentPage = null; + + public SearchThread(Client client) { + this.client = client; + } + + /** + * Retrieves a nested value from a {@code Map} in a (somewhat) safe way. + * + * @param map The parent map to look at. + * @param key The key our nested object is found under. + * @param type The type we expect the nested object to be of. + * @param The type of the nested object. + * @return the nested object cast into the proper type. + */ + public static T getNestedObject(Map map, String key, Class type) { + Object value = map.get(key); + if (value == null) { + return null; + } + if (type.isInstance(value)) { + // Make a defensive copy for supported types like Map or List + if (value instanceof Map) { + return type.cast(new HashMap<>((Map) value)); + } else if (value instanceof List) { + return type.cast(new ArrayList<>((List) value)); + } + // Return the value directly if it is immutable (e.g., String, Integer) + return type.cast(value); + } else { + throw new ClassCastException( + "Expected " + + type.getName() + + " but found " + + value.getClass().getName()); + } + } + + /** + * Iterates over search results, updating their status field and submitting them to the + * destination + * + * @param searchResponse The search results page + * @throws IllegalStateException Rethrown from setSentStatus() + */ + @SuppressWarnings("unchecked") + public void handlePage(SearchResponse searchResponse) throws IllegalStateException { + SearchHits searchHits = searchResponse.getHits(); + ArrayList orders = new ArrayList<>(); + for (SearchHit hit : searchHits) { + Map orderMap = getNestedObject(hit.getSourceAsMap(), Command.COMMAND, Map.class); + if (orderMap != null) { + orderMap.put("document_id", hit.getId()); + orders.add(orderMap); + } + } + String payload = null; + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + payload = builder.map(Collections.singletonMap("orders", orders)).toString(); + } catch (IOException e) { + log.error("Error parsing hit contents: {}", e.getMessage()); + } + + if (payload != null) { + SimpleHttpResponse response = deliverOrders(payload); + if (response == null) { + return; + } + if (RestStatus.fromCode(response.getCode()) == RestStatus.CREATED + | RestStatus.fromCode(response.getCode()) == RestStatus.ACCEPTED + | RestStatus.fromCode(response.getCode()) == RestStatus.OK) { + for (SearchHit hit : searchHits) { + setSentStatus(hit); + } + } + } + } + + /** + * Send the command order over HTTP + * + * @param orders The list of order to send. + */ + private SimpleHttpResponse deliverOrders(String orders) { + try { + PluginSettings settings = PluginSettings.getInstance(); + URI uri = new URIBuilder(settings.getUri() + SearchThread.ORDERS_OBJECT).build(); + return AccessController.doPrivileged( + (PrivilegedAction) + () -> AuthHttpRestClient.getInstance().post(uri, orders, null)); + } catch (URISyntaxException e) { + log.error("Invalid URI: {}", e.getMessage()); + } + return null; + } + + /** + * Retrieves the hit's contents and updates the {@link Status} field to {@link Status#SENT}. + * + * @param hit The page's result we are to update. + * @throws IllegalStateException Raised by {@link ActionFuture#actionGet(long)}. + */ + @SuppressWarnings("unchecked") + private void setSentStatus(SearchHit hit) throws IllegalStateException { + Map commandMap = + getNestedObject( + hit.getSourceAsMap(), + CommandManagerPlugin.COMMAND_DOCUMENT_PARENT_OBJECT_NAME, + Map.class); + commandMap.put(Command.STATUS, Status.SENT); + hit.getSourceAsMap() + .put(CommandManagerPlugin.COMMAND_DOCUMENT_PARENT_OBJECT_NAME, commandMap); + IndexRequest indexRequest = + new IndexRequest() + .index(CommandManagerPlugin.COMMAND_MANAGER_INDEX_NAME) + .source(hit.getSourceAsMap()) + .id(hit.getId()); + this.client + .index(indexRequest) + .actionGet(CommandManagerPlugin.DEFAULT_TIMEOUT_SECONDS * 1000); + } + + /** + * Runs a PIT style query against the Commands index. + * + * @param pointInTimeBuilder A pit builder object used to run the query. + * @param searchAfter An array of objects containing the last page's values of the sort fields. + * @return The search response. + * @throws IllegalStateException Raised by {@link ActionFuture#actionGet(long)}. + */ + public SearchResponse pitQuery(PointInTimeBuilder pointInTimeBuilder, Object[] searchAfter) + throws IllegalStateException { + SearchRequest searchRequest = + new SearchRequest(CommandManagerPlugin.COMMAND_MANAGER_INDEX_NAME); + TermQueryBuilder termQueryBuilder = + QueryBuilders.termQuery(SearchThread.COMMAND_STATUS_FIELD, Status.PENDING); + TimeValue timeout = + TimeValue.timeValueSeconds(CommandManagerPlugin.DEFAULT_TIMEOUT_SECONDS); + this.searchSourceBuilder + .query(termQueryBuilder) + .size(CommandManagerPlugin.PAGE_SIZE) + .trackTotalHits(true) + .timeout(timeout) + .pointInTimeBuilder(pointInTimeBuilder); + if (this.searchSourceBuilder.sorts() == null) { + this.searchSourceBuilder.sort(SearchThread.DELIVERY_TIMESTAMP_FIELD, SortOrder.ASC); + } + if (searchAfter.length > 0) { + this.searchSourceBuilder.searchAfter(searchAfter); + } + searchRequest.source(this.searchSourceBuilder); + return this.client.search(searchRequest).actionGet(timeout); + } + + @Override + public void run() { + long consumableHits = 0L; + boolean firstPage = true; + PointInTimeBuilder pointInTimeBuilder = buildPit(); + try { + do { + this.currentPage = + pitQuery( + pointInTimeBuilder, + getSearchAfter(this.currentPage).orElse(new Object[0])); + if (firstPage) { + consumableHits = totalHits(); + firstPage = false; + } + if (consumableHits > 0) { + handlePage(this.currentPage); + consumableHits -= getPageLength(); + } + } while (consumableHits > 0); + } catch (ArrayIndexOutOfBoundsException e) { + log.error("ArrayIndexOutOfBoundsException retrieving page: {}", e.getMessage()); + } catch (IllegalStateException e) { + log.error("IllegalStateException retrieving page: {}", e.getMessage()); + } catch (Exception e) { + log.error("Generic exception retrieving page: {}", e.getMessage()); + } + } + + private long getPageLength() { + return this.currentPage.getHits().getHits().length; + } + + private long totalHits() { + if (this.currentPage.getHits().getTotalHits() != null) { + return this.currentPage.getHits().getTotalHits().value; + } else { + return 0; + } + } + + /** + * Gets the sort values of the last hit of a page. It is used by a PIT search to get the next + * page of results. + * + * @param searchResponse The current page + * @return The values of the sort fields of the last hit of a page whenever present. Otherwise, + * an empty Optional. + */ + private Optional getSearchAfter(SearchResponse searchResponse) { + if (searchResponse == null) { + return Optional.empty(); + } + try { + List hits = Arrays.asList(searchResponse.getHits().getHits()); + if (hits.isEmpty()) { + log.warn("Empty hits page, not getting searchAfter values"); + return Optional.empty(); + } + return Optional.ofNullable(hits.get(hits.size() - 1).getSortValues()); + } catch (NullPointerException | NoSuchElementException e) { + log.error("Could not get the page's searchAfter values: {}", e.getMessage()); + return Optional.empty(); + } + } + + /** + * Prepares a PointInTimeBuilder object to be used in a search. + * + * @return a PointInTimeBuilder or null. + */ + private PointInTimeBuilder buildPit() { + CompletableFuture future = new CompletableFuture<>(); + ActionListener actionListener = + new ActionListener<>() { + @Override + public void onResponse(CreatePitResponse createPitResponse) { + future.complete(createPitResponse); + } + + @Override + public void onFailure(Exception e) { + log.error(e.getMessage()); + future.completeExceptionally(e); + } + }; + this.client.createPit( + new CreatePitRequest( + CommandManagerPlugin.PIT_KEEP_ALIVE_SECONDS, + false, + CommandManagerPlugin.COMMAND_MANAGER_INDEX_NAME), + actionListener); + try { + return new PointInTimeBuilder(future.get().getId()); + } catch (CancellationException e) { + log.error("Building PIT was cancelled: {}", e.getMessage()); + } catch (ExecutionException e) { + log.error("Error building PIT: {}", e.getMessage()); + } catch (InterruptedException e) { + log.error("Building PIT was interrupted: {}", e.getMessage()); + } + return null; + } +} diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/model/Command.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/model/Command.java index 57b7ec5a..c5221248 100644 --- a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/model/Command.java +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/model/Command.java @@ -16,6 +16,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.List; import reactor.util.annotation.NonNull; @@ -61,6 +62,15 @@ public Command( this.status = Status.PENDING; } + /** + * Retrieves the timeout value for this command. + * + * @return the timeout value in milliseconds. + */ + public Integer getTimeout() { + return this.timeout; + } + /** * Parses the request's payload into the Command model. * @@ -127,6 +137,16 @@ public static Command parse(XContentParser parser) } } + public static List parseToArray(XContentParser parser) + throws IOException, IllegalArgumentException { + List commands = new ArrayList<>(); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + Command command = Command.parse(parser); + commands.add(command); + } + return commands; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(COMMAND); diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/model/Document.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/model/Document.java index b9ec3850..6a9eef76 100644 --- a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/model/Document.java +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/model/Document.java @@ -9,18 +9,28 @@ package com.wazuh.commandmanager.model; import org.opensearch.common.UUIDs; +import org.opensearch.common.time.DateFormatter; +import org.opensearch.common.time.DateUtils; +import org.opensearch.common.time.FormatNames; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import java.io.IOException; +import java.time.ZonedDateTime; import java.util.List; /** Command's target fields. */ public class Document implements ToXContentObject { + private static final String DATE_FORMAT = FormatNames.DATE_TIME_NO_MILLIS.getSnakeCaseName(); + private static final DateFormatter DATE_FORMATTER = DateFormatter.forPattern(DATE_FORMAT); + public static final String TIMESTAMP = "@timestamp"; + public static final String DELIVERY_TIMESTAMP = "delivery_timestamp"; private final Agent agent; private final Command command; private final String id; + private final ZonedDateTime timestamp; + private final ZonedDateTime deliveryTimestamp; /** * Default constructor @@ -32,6 +42,8 @@ public Document(Agent agent, Command command) { this.agent = agent; this.command = command; this.id = UUIDs.base64UUID(); + this.timestamp = DateUtils.nowWithMillisResolution(); + this.deliveryTimestamp = timestamp.plusSeconds(command.getTimeout()); } /** @@ -68,11 +80,22 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); this.agent.toXContent(builder, ToXContentObject.EMPTY_PARAMS); this.command.toXContent(builder, ToXContentObject.EMPTY_PARAMS); + builder.field(TIMESTAMP, DATE_FORMATTER.format(this.timestamp)); + builder.field(DELIVERY_TIMESTAMP, DATE_FORMATTER.format(this.deliveryTimestamp)); return builder.endObject(); } @Override public String toString() { - return "Document{" + "agent=" + agent + ", command=" + command + '}'; + return "Document{" + + "@timestamp=" + + timestamp + + ", delivery_timestamp=" + + deliveryTimestamp + + ", agent=" + + agent + + ", command=" + + command + + '}'; } } diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/model/Documents.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/model/Documents.java new file mode 100644 index 00000000..36ecd068 --- /dev/null +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/model/Documents.java @@ -0,0 +1,78 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package com.wazuh.commandmanager.model; + +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.ArrayList; + +public class Documents implements ToXContentObject { + private ArrayList documents; + + public Documents() { + this.documents = new ArrayList<>(); + } + + /** + * Default constructor + * + * @param documents + */ + public Documents(ArrayList documents) { + this.documents = documents; + } + + /** + * Get the list of Document objects. + * + * @return the list of documents. + */ + public ArrayList getDocuments() { + return documents; + } + + /** + * Set the list of Document objects. + * + * @param documents the list of documents to set. + */ + public void setDocuments(ArrayList documents) { + this.documents = documents; + } + + /** + * Adds a document to the list of documents. + * + * @param document The document to add to the list. + */ + public void addDocument(Document document) { + this.documents.add(document); + } + + /** + * Fit this object into a XContentBuilder parser, preparing it for the reply of POST /commands. + * + * @param builder XContentBuilder builder + * @param params ToXContent.EMPTY_PARAMS + * @return XContentBuilder builder with the representation of this object. + * @throws IOException parsing error. + */ + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startArray("_documents"); + for (Document document : this.documents) { + builder.startObject(); + builder.field("_id", document.getId()); + builder.endObject(); + } + return builder.endArray(); + } +} diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/rest/RestPostCommandAction.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/rest/RestPostCommandAction.java index 277ae153..fe786d67 100644 --- a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/rest/RestPostCommandAction.java +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/rest/RestPostCommandAction.java @@ -8,11 +8,9 @@ */ package com.wazuh.commandmanager.rest; -import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; @@ -22,15 +20,17 @@ import org.opensearch.rest.RestRequest; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Locale; +import java.util.concurrent.CompletableFuture; import com.wazuh.commandmanager.CommandManagerPlugin; import com.wazuh.commandmanager.index.CommandIndex; import com.wazuh.commandmanager.model.Agent; import com.wazuh.commandmanager.model.Command; import com.wazuh.commandmanager.model.Document; -import com.wazuh.commandmanager.utils.httpclient.HttpRestClientDemo; +import com.wazuh.commandmanager.model.Documents; import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; import static org.opensearch.rest.RestRequest.Method.POST; @@ -94,34 +94,60 @@ private RestChannelConsumer handlePost(RestRequest request) throws IOException { request.uri(), request.getRequestId(), request.header("Host")); - // Get request details + + /// Request validation + /// ================== + /// Fail fast. + if (!request.hasContent()) { + // Bad request if body doesn't exist + return channel -> { + channel.sendResponse( + new BytesRestResponse(RestStatus.BAD_REQUEST, "Body content is required")); + }; + } + + /// Request parsing + /// =============== + /// Retrieves and generates an array list of commands. XContentParser parser = request.contentParser(); + List commands = new ArrayList<>(); ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + // The array of commands is inside the "commands" JSON object. + // This line moves the parser pointer to this object. + parser.nextToken(); + if (parser.nextToken() == XContentParser.Token.START_ARRAY) { + commands = Command.parseToArray(parser); + } else { + log.error("Token does not match {}", parser.currentToken()); + } - Command command = Command.parse(parser); - Document document = - new Document( - new Agent(List.of("groups000")), // TODO read agent from .agents index - command); - - // Commands delivery to the Management API. - // Note: needs to be decoupled from the Rest handler (job scheduler task). - try { - String payload = - document.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS) - .toString(); - SimpleHttpResponse response = - HttpRestClientDemo.runWithResponse(payload, document.getId()); - log.info("Received response to POST request with code [{}]", response.getCode()); - log.info("Raw response:\n{}", response.getBodyText()); - } catch (Exception e) { - log.error("Error reading response: {}", e.getMessage()); + /// Commands expansion + /// ================== + /// Transforms the array of commands to orders. + /// While commands can be targeted to groups of agents, orders are targeted to individual + // agents. + /// Given a group of agents A with N agents, a total of N orders are generated. One for each + // agent. + Documents documents = new Documents(); + for (Command command : commands) { + Document document = + new Document( + new Agent(List.of("groups000")), // TODO read agent from .agents index + command); + documents.addDocument(document); } - // Send response + /// Orders indexing + /// ================== + /// The orders are inserted into the index. + CompletableFuture bulkRequestFuture = + this.commandIndex.asyncBulkCreate(documents.getDocuments()); + + /// Send response + /// ================== + /// Reply to the request. return channel -> { - this.commandIndex - .asyncCreate(document) + bulkRequestFuture .thenAccept( restStatus -> { try (XContentBuilder builder = channel.newBuilder()) { @@ -129,7 +155,7 @@ private RestChannelConsumer handlePost(RestRequest request) throws IOException { builder.field( "_index", CommandManagerPlugin.COMMAND_MANAGER_INDEX_NAME); - builder.field("_id", document.getId()); + documents.toXContent(builder, ToXContent.EMPTY_PARAMS); builder.field("result", restStatus.name()); builder.endObject(); channel.sendResponse( diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/utils/IndexTemplateUtils.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/utils/IndexTemplateUtils.java index 409b133c..6af5616a 100644 --- a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/utils/IndexTemplateUtils.java +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/utils/IndexTemplateUtils.java @@ -8,6 +8,14 @@ */ package com.wazuh.commandmanager.utils; +import com.wazuh.commandmanager.index.CommandIndex; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.admin.indices.template.put.PutIndexTemplateRequest; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.IndexTemplateMetadata; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; @@ -15,12 +23,14 @@ import java.io.IOException; import java.io.InputStream; +import java.util.List; import java.util.Map; import reactor.util.annotation.NonNull; /** Util functions to parse and manage index templates files. */ public class IndexTemplateUtils { + private static final Logger log = LogManager.getLogger(IndexTemplateUtils.class); /** Default constructor */ public IndexTemplateUtils() {} @@ -70,4 +80,46 @@ public static Map toMap(InputStream is) throws IOException { public static Map get(Map map, String key) { return (Map) map.get(key); } + + /** + * Checks for the existence of the given index template in the cluster. + * + * @param clusterService The cluster service used to check the node's existence + * @param templateName index template name within the resources folder + * @return whether the index template exists. + */ + public static boolean indexTemplateExists(ClusterService clusterService, String templateName) { + Map templates = + clusterService.state().metadata().templates(); + log.debug("Existing index templates: {} ", templates); + + return templates.containsKey(templateName); + } + + /** + * Inserts an index template + * @param templateName : The name if the index template to load + */ + public static void putIndexTemplate(Client client, String templateName) { + try { + // @throws IOException + Map template = IndexTemplateUtils.fromFile(templateName + ".json"); + + PutIndexTemplateRequest putIndexTemplateRequest = + new PutIndexTemplateRequest() + .mapping(IndexTemplateUtils.get(template, "mappings")) + .settings(IndexTemplateUtils.get(template, "settings")) + .name(templateName) + .patterns((List) template.get("index_patterns")); + + AcknowledgedResponse acknowledgedResponse = + client.admin().indices().putTemplate(putIndexTemplateRequest).actionGet(); + if (acknowledgedResponse.isAcknowledged()) { + log.info("Index template [{}] created successfully", templateName); + } + + } catch (IOException e) { + log.error("Error reading index template [{}] from filesystem", templateName); + } + } } diff --git a/plugins/command-manager/src/main/resources/META-INF/services/org.opensearch.jobscheduler.spi.JobSchedulerExtension b/plugins/command-manager/src/main/resources/META-INF/services/org.opensearch.jobscheduler.spi.JobSchedulerExtension new file mode 100644 index 00000000..69bad7f1 --- /dev/null +++ b/plugins/command-manager/src/main/resources/META-INF/services/org.opensearch.jobscheduler.spi.JobSchedulerExtension @@ -0,0 +1,6 @@ +# +# Copyright Wazuh Contributors +# SPDX-License-Identifier: Apache-2.0 +# + +com.wazuh.commandmanager.CommandManagerPlugin \ No newline at end of file diff --git a/plugins/command-manager/src/main/resources/index-template-commands.json b/plugins/command-manager/src/main/resources/index-template-commands.json index 3614c17b..6c834803 100644 --- a/plugins/command-manager/src/main/resources/index-template-commands.json +++ b/plugins/command-manager/src/main/resources/index-template-commands.json @@ -6,6 +6,9 @@ "date_detection": false, "dynamic": "strict", "properties": { + "@timestamp": { + "type": "date" + }, "agent": { "properties": { "groups": { @@ -83,6 +86,9 @@ "type": "keyword" } } + }, + "delivery_timestamp": { + "type": "date" } } }, diff --git a/plugins/command-manager/src/main/resources/index-template-scheduled-commands.json b/plugins/command-manager/src/main/resources/index-template-scheduled-commands.json new file mode 100644 index 00000000..232dbe73 --- /dev/null +++ b/plugins/command-manager/src/main/resources/index-template-scheduled-commands.json @@ -0,0 +1,51 @@ +{ + "index_patterns": [ + ".scheduled-commands" + ], + "mappings": { + "dynamic": "strict", + "properties": { + "name": { + "type": "keyword" + }, + "enabled": { + "type": "boolean" + }, + "schedule": { + "properties": { + "interval": { + "properties": { + "start_time": { + "type": "date", + "format": "epoch_millis" + }, + "period": { + "type": "integer" + }, + "unit": { + "type": "keyword" + } + } + } + } + }, + "enabled_time": { + "type": "date", + "format": "epoch_millis" + }, + "last_update_time": { + "type": "date", + "format": "epoch_millis" + } + } + }, + "order": 1, + "settings": { + "index": { + "hidden": true, + "number_of_replicas": "0", + "number_of_shards": "1", + "refresh_interval": "5s" + } + } +} diff --git a/plugins/command-manager/src/test/resources/wazuh-indexer.keystore.json b/plugins/command-manager/src/test/resources/wazuh-indexer.keystore.json deleted file mode 100644 index c93a5df3..00000000 --- a/plugins/command-manager/src/test/resources/wazuh-indexer.keystore.json +++ /dev/null @@ -1,11 +0,0 @@ -[ - { - "name" : "wazuh-api", - "properties" : { - "uri" : "http://localhost:9090", - "auth.type" : "basicauth", - "auth.username" : "admin", - "auth.password" : "type" - } - } -] \ No newline at end of file diff --git a/plugins/command-manager/src/yamlRestTest/resources/rest-api-spec/test/20_create.yml b/plugins/command-manager/src/yamlRestTest/resources/rest-api-spec/test/20_create.yml index f081ec51..ccc467c8 100644 --- a/plugins/command-manager/src/yamlRestTest/resources/rest-api-spec/test/20_create.yml +++ b/plugins/command-manager/src/yamlRestTest/resources/rest-api-spec/test/20_create.yml @@ -1,37 +1,46 @@ --- "Create command": - - do: - _plugins._command_manager: - body: - source: "Users/Services" - user: "user13" - target: { - id: "target4", - type: "agent" - } - action: { - name: "change_group", - args: [ "/path/to/executable/arg8" ], - version: "v4" - } - timeout: 100 + - do: + _plugins._command_manager: + body: + commands: + [ + { + source: "Users/Services", + user: "user13", + target: { id: "target4", type: "agent" }, + action: + { + name: "change_group", + args: ["/path/to/executable/arg8"], + version: "v4", + }, + timeout: 100, + }, - - set: { _id: document_id } - - match: { _index: .commands } + { + source: "Users/Services", + user: "user54", + target: { id: "target5", type: "agent" }, + action: + { + name: "stop", + args: ["/path/to/executable/arg7"], + version: "v4", + }, + timeout: 30, + }, + ] - - do: - get: - index: .commands - id: $document_id - - match: { _source.command.source: "Users/Services" } - - match: { _source.command.user: "user13" } - - match: { _source.command.target.type: "agent" } - - match: { _source.command.target.id: "target4" } - - match: { _source.command.action: - { - name: "change_group", - args: [ "/path/to/executable/arg8" ], - version: "v4" - } - } - - match: { _source.command.timeout: 100 } + - match: { _index: .commands } + - match: { result: "OK" } + + - do: + indices.refresh: + index: [.commands] + + - do: + count: + index: .commands + + - match: { count: 2 } diff --git a/plugins/setup/src/main/java/com/wazuh/setup/index/WazuhIndices.java b/plugins/setup/src/main/java/com/wazuh/setup/index/WazuhIndices.java index a0073efa..b7dd165d 100644 --- a/plugins/setup/src/main/java/com/wazuh/setup/index/WazuhIndices.java +++ b/plugins/setup/src/main/java/com/wazuh/setup/index/WazuhIndices.java @@ -50,6 +50,7 @@ public WazuhIndices(Client client, ClusterService clusterService) { this.indexTemplates.put("index-template-agent", ".agents"); this.indexTemplates.put("index-template-alerts", "wazuh-alerts-5.x-0001"); this.indexTemplates.put("index-template-commands", ".commands"); + this.indexTemplates.put("index-template-scheduled-commands", ".scheduled-commands"); this.indexTemplates.put("index-template-fim", "wazuh-states-fim"); this.indexTemplates.put("index-template-hardware", "wazuh-states-inventory-hardware"); this.indexTemplates.put("index-template-hotfixes", "wazuh-states-inventory-hotfixes"); diff --git a/plugins/setup/src/main/resources/index-template-alerts.json b/plugins/setup/src/main/resources/index-template-alerts.json index d50aba75..1358a430 100644 --- a/plugins/setup/src/main/resources/index-template-alerts.json +++ b/plugins/setup/src/main/resources/index-template-alerts.json @@ -4,7 +4,7 @@ ], "mappings": { "date_detection": false, - "dynamic": "strict", + "dynamic": true, "properties": { "@timestamp": { "type": "date" diff --git a/plugins/setup/src/main/resources/index-template-commands.json b/plugins/setup/src/main/resources/index-template-commands.json index 3614c17b..6c834803 100644 --- a/plugins/setup/src/main/resources/index-template-commands.json +++ b/plugins/setup/src/main/resources/index-template-commands.json @@ -6,6 +6,9 @@ "date_detection": false, "dynamic": "strict", "properties": { + "@timestamp": { + "type": "date" + }, "agent": { "properties": { "groups": { @@ -83,6 +86,9 @@ "type": "keyword" } } + }, + "delivery_timestamp": { + "type": "date" } } }, diff --git a/plugins/setup/src/main/resources/index-template-fim.json b/plugins/setup/src/main/resources/index-template-fim.json index f873565b..4bcd4650 100644 --- a/plugins/setup/src/main/resources/index-template-fim.json +++ b/plugins/setup/src/main/resources/index-template-fim.json @@ -322,208 +322,6 @@ } } }, - "host": { - "properties": { - "architecture": { - "ignore_above": 1024, - "type": "keyword" - }, - "boot": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "cpu": { - "properties": { - "usage": { - "type": "float" - } - } - }, - "disk": { - "properties": { - "read": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "write": { - "properties": { - "bytes": { - "type": "long" - } - } - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "geo": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "postal_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "timezone": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "network": { - "properties": { - "egress": { - "properties": { - "bytes": { - "type": "long" - }, - "packets": { - "type": "long" - } - } - }, - "ingress": { - "properties": { - "bytes": { - "type": "long" - }, - "packets": { - "type": "long" - } - } - } - } - }, - "os": { - "properties": { - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "full": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "pid_ns_ino": { - "ignore_above": 1024, - "type": "keyword" - }, - "risk": { - "properties": { - "calculated_level": { - "ignore_above": 1024, - "type": "keyword" - }, - "calculated_score": { - "type": "float" - }, - "calculated_score_norm": { - "type": "float" - }, - "static_level": { - "ignore_above": 1024, - "type": "keyword" - }, - "static_score": { - "type": "float" - }, - "static_score_norm": { - "type": "float" - } - } - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "uptime": { - "type": "long" - } - } - }, "registry": { "properties": { "key": { diff --git a/plugins/setup/src/main/resources/index-template-hardware.json b/plugins/setup/src/main/resources/index-template-hardware.json index 7d28fe8b..1ade0c83 100644 --- a/plugins/setup/src/main/resources/index-template-hardware.json +++ b/plugins/setup/src/main/resources/index-template-hardware.json @@ -277,18 +277,6 @@ }, "host": { "properties": { - "architecture": { - "ignore_above": 1024, - "type": "keyword" - }, - "boot": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, "cpu": { "properties": { "cores": { @@ -307,90 +295,6 @@ }, "type": "object" }, - "disk": { - "properties": { - "read": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "write": { - "properties": { - "bytes": { - "type": "long" - } - } - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "geo": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "postal_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "timezone": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, "memory": { "properties": { "free": { @@ -409,101 +313,6 @@ } }, "type": "object" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "network": { - "properties": { - "egress": { - "properties": { - "bytes": { - "type": "long" - }, - "packets": { - "type": "long" - } - } - }, - "ingress": { - "properties": { - "bytes": { - "type": "long" - }, - "packets": { - "type": "long" - } - } - } - } - }, - "os": { - "properties": { - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "full": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "pid_ns_ino": { - "ignore_above": 1024, - "type": "keyword" - }, - "risk": { - "properties": { - "calculated_level": { - "ignore_above": 1024, - "type": "keyword" - }, - "calculated_score": { - "type": "float" - }, - "calculated_score_norm": { - "type": "float" - }, - "static_level": { - "ignore_above": 1024, - "type": "keyword" - }, - "static_score": { - "type": "float" - }, - "static_score_norm": { - "type": "float" - } - } - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "uptime": { - "type": "long" } } }, diff --git a/plugins/setup/src/main/resources/index-template-hotfixes.json b/plugins/setup/src/main/resources/index-template-hotfixes.json index 654d8430..52d66fe9 100644 --- a/plugins/setup/src/main/resources/index-template-hotfixes.json +++ b/plugins/setup/src/main/resources/index-template-hotfixes.json @@ -245,208 +245,6 @@ } } }, - "host": { - "properties": { - "architecture": { - "ignore_above": 1024, - "type": "keyword" - }, - "boot": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "cpu": { - "properties": { - "usage": { - "type": "float" - } - } - }, - "disk": { - "properties": { - "read": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "write": { - "properties": { - "bytes": { - "type": "long" - } - } - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "geo": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "postal_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "timezone": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "network": { - "properties": { - "egress": { - "properties": { - "bytes": { - "type": "long" - }, - "packets": { - "type": "long" - } - } - }, - "ingress": { - "properties": { - "bytes": { - "type": "long" - }, - "packets": { - "type": "long" - } - } - } - } - }, - "os": { - "properties": { - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "full": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "pid_ns_ino": { - "ignore_above": 1024, - "type": "keyword" - }, - "risk": { - "properties": { - "calculated_level": { - "ignore_above": 1024, - "type": "keyword" - }, - "calculated_score": { - "type": "float" - }, - "calculated_score_norm": { - "type": "float" - }, - "static_level": { - "ignore_above": 1024, - "type": "keyword" - }, - "static_score": { - "type": "float" - }, - "static_score_norm": { - "type": "float" - } - } - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "uptime": { - "type": "long" - } - } - }, "package": { "properties": { "hotfix": { diff --git a/plugins/setup/src/main/resources/index-template-networks.json b/plugins/setup/src/main/resources/index-template-networks.json index e8d27e21..9c5c1886 100644 --- a/plugins/setup/src/main/resources/index-template-networks.json +++ b/plugins/setup/src/main/resources/index-template-networks.json @@ -471,6 +471,21 @@ } } }, + "interface": { + "properties": { + "mtu": { + "type": "long" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "network": { "properties": { "broadcast": { diff --git a/plugins/setup/src/main/resources/index-template-packages.json b/plugins/setup/src/main/resources/index-template-packages.json index 79657dbb..353de124 100644 --- a/plugins/setup/src/main/resources/index-template-packages.json +++ b/plugins/setup/src/main/resources/index-template-packages.json @@ -245,208 +245,6 @@ } } }, - "host": { - "properties": { - "architecture": { - "ignore_above": 1024, - "type": "keyword" - }, - "boot": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "cpu": { - "properties": { - "usage": { - "type": "float" - } - } - }, - "disk": { - "properties": { - "read": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "write": { - "properties": { - "bytes": { - "type": "long" - } - } - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "geo": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "postal_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "timezone": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "network": { - "properties": { - "egress": { - "properties": { - "bytes": { - "type": "long" - }, - "packets": { - "type": "long" - } - } - }, - "ingress": { - "properties": { - "bytes": { - "type": "long" - }, - "packets": { - "type": "long" - } - } - } - } - }, - "os": { - "properties": { - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "full": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "pid_ns_ino": { - "ignore_above": 1024, - "type": "keyword" - }, - "risk": { - "properties": { - "calculated_level": { - "ignore_above": 1024, - "type": "keyword" - }, - "calculated_score": { - "type": "float" - }, - "calculated_score_norm": { - "type": "float" - }, - "static_level": { - "ignore_above": 1024, - "type": "keyword" - }, - "static_score": { - "type": "float" - }, - "static_score_norm": { - "type": "float" - } - } - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "uptime": { - "type": "long" - } - } - }, "package": { "properties": { "architecture": { diff --git a/plugins/setup/src/main/resources/index-template-ports.json b/plugins/setup/src/main/resources/index-template-ports.json index 8d2598cb..e96eb56c 100644 --- a/plugins/setup/src/main/resources/index-template-ports.json +++ b/plugins/setup/src/main/resources/index-template-ports.json @@ -279,123 +279,10 @@ }, "host": { "properties": { - "architecture": { - "ignore_above": 1024, - "type": "keyword" - }, - "boot": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "cpu": { - "properties": { - "usage": { - "type": "float" - } - } - }, - "disk": { - "properties": { - "read": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "write": { - "properties": { - "bytes": { - "type": "long" - } - } - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "geo": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "postal_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "timezone": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, "network": { "properties": { "egress": { "properties": { - "bytes": { - "type": "long" - }, - "packets": { - "type": "long" - }, "queue": { "type": "long" } @@ -403,85 +290,20 @@ }, "ingress": { "properties": { - "bytes": { - "type": "long" - }, - "packets": { - "type": "long" - }, "queue": { "type": "long" } } } } - }, - "os": { - "properties": { - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "full": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "pid_ns_ino": { - "ignore_above": 1024, - "type": "keyword" - }, - "risk": { - "properties": { - "calculated_level": { - "ignore_above": 1024, - "type": "keyword" - }, - "calculated_score": { - "type": "float" - }, - "calculated_score_norm": { - "type": "float" - }, - "static_level": { - "ignore_above": 1024, - "type": "keyword" - }, - "static_score": { - "type": "float" - }, - "static_score_norm": { - "type": "float" - } - } - }, - "type": { + } + } + }, + "interface": { + "properties": { + "state": { "ignore_above": 1024, "type": "keyword" - }, - "uptime": { - "type": "long" } } }, diff --git a/plugins/setup/src/main/resources/index-template-processes.json b/plugins/setup/src/main/resources/index-template-processes.json index 74cfce49..c6bced0c 100644 --- a/plugins/setup/src/main/resources/index-template-processes.json +++ b/plugins/setup/src/main/resources/index-template-processes.json @@ -155,11 +155,6 @@ "type": "keyword" }, "full": { - "fields": { - "text": { - "type": "match_only_text" - } - }, "ignore_above": 1024, "type": "keyword" }, @@ -168,11 +163,6 @@ "type": "keyword" }, "name": { - "fields": { - "text": { - "type": "match_only_text" - } - }, "ignore_above": 1024, "type": "keyword" }, @@ -245,208 +235,6 @@ } } }, - "host": { - "properties": { - "architecture": { - "ignore_above": 1024, - "type": "keyword" - }, - "boot": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "cpu": { - "properties": { - "usage": { - "type": "float" - } - } - }, - "disk": { - "properties": { - "read": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "write": { - "properties": { - "bytes": { - "type": "long" - } - } - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "geo": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "postal_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "timezone": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "network": { - "properties": { - "egress": { - "properties": { - "bytes": { - "type": "long" - }, - "packets": { - "type": "long" - } - } - }, - "ingress": { - "properties": { - "bytes": { - "type": "long" - }, - "packets": { - "type": "long" - } - } - } - } - }, - "os": { - "properties": { - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "full": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "pid_ns_ino": { - "ignore_above": 1024, - "type": "keyword" - }, - "risk": { - "properties": { - "calculated_level": { - "ignore_above": 1024, - "type": "keyword" - }, - "calculated_score": { - "type": "float" - }, - "calculated_score_norm": { - "type": "float" - }, - "static_level": { - "ignore_above": 1024, - "type": "keyword" - }, - "static_score": { - "type": "float" - }, - "static_score_norm": { - "type": "float" - } - } - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "uptime": { - "type": "long" - } - } - }, "process": { "properties": { "args": { @@ -454,12 +242,7 @@ "type": "keyword" }, "command_line": { - "fields": { - "text": { - "type": "match_only_text" - } - }, - "type": "wildcard" + "type": "keyword" }, "group": { "properties": { @@ -470,11 +253,6 @@ } }, "name": { - "fields": { - "text": { - "type": "match_only_text" - } - }, "ignore_above": 1024, "type": "keyword" }, @@ -530,6 +308,18 @@ } } }, + "tty": { + "properties": { + "char_device": { + "properties": { + "major": { + "type": "long" + } + } + } + }, + "type": "object" + }, "user": { "properties": { "id": { diff --git a/plugins/setup/src/main/resources/index-template-scheduled-commands.json b/plugins/setup/src/main/resources/index-template-scheduled-commands.json new file mode 100644 index 00000000..232dbe73 --- /dev/null +++ b/plugins/setup/src/main/resources/index-template-scheduled-commands.json @@ -0,0 +1,51 @@ +{ + "index_patterns": [ + ".scheduled-commands" + ], + "mappings": { + "dynamic": "strict", + "properties": { + "name": { + "type": "keyword" + }, + "enabled": { + "type": "boolean" + }, + "schedule": { + "properties": { + "interval": { + "properties": { + "start_time": { + "type": "date", + "format": "epoch_millis" + }, + "period": { + "type": "integer" + }, + "unit": { + "type": "keyword" + } + } + } + } + }, + "enabled_time": { + "type": "date", + "format": "epoch_millis" + }, + "last_update_time": { + "type": "date", + "format": "epoch_millis" + } + } + }, + "order": 1, + "settings": { + "index": { + "hidden": true, + "number_of_replicas": "0", + "number_of_shards": "1", + "refresh_interval": "5s" + } + } +}