Skip to content

Commit

Permalink
update(userspace/falco): add libsinsp state metrics option
Browse files Browse the repository at this point in the history 
Signed-off-by: Melissa Kilby <[email protected]>
  • Loading branch information
@incertum
incertum committed Oct 21, 2023
1 parent 3ff2bb5 commit eadff28
Show file tree
Hide file tree
Showing 4 changed files with 100 additions and 57 deletions.
24 changes: 17 additions & 7 deletions falco.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -727,13 +727,22 @@ syscall_event_drops:
# number of CPUs to determine overall usage. Memory metrics are provided in raw
# units (`kb` for `RSS`, `PSS` and `VSZ` or `bytes` for `container_memory_used`)
# and can be uniformly converted to megabytes (MB) using the
# `convert_memory_to_mb` functionality. In environments such as Kubernetes, it
# is crucial to track Falco's container memory usage. To customize the path of
# the memory metric file, you can create an environment variable named
# `FALCO_CGROUP_MEM_PATH` and set it to the desired file path. By default, Falco
# uses the file `/sys/fs/cgroup/memory/memory.usage_in_bytes` to monitor
# container memory usage, which aligns with Kubernetes'
# `container_memory_working_set_bytes` metric.
# `convert_memory_to_mb` functionality. In environments such as Kubernetes when
# deployed as daemonset, it is crucial to track Falco's container memory usage.
# To customize the path of the memory metric file, you can create an environment
# variable named `FALCO_CGROUP_MEM_PATH` and set it to the desired file path. By
# default, Falco uses the file `/sys/fs/cgroup/memory/memory.usage_in_bytes` to
# monitor container memory usage, which aligns with Kubernetes'
# `container_memory_working_set_bytes` metric. Finally, we emit the overall host
# CPU and memory usages, along with the total number of processes and open file
# descriptors (fds) on the host, obtained from the proc file system unrelated to
# Falco's monitoring. These metrics help assess Falco's usage in relation to the
# server's workload intensity.
#
# `state_counters_enabled`: Emit counters related to Falco's state engine, including
# added, removed threads or file descriptors (fds), and failed lookup, store, or
# retrieve actions in relation to Falco's underlying process cache table (threadtable).
# We also log the number of currently cached containers if applicable.
#
# `kernel_event_counters_enabled`: Emit kernel side event and drop counters, as
# an alternative to `syscall_event_drops`, but with some differences. These
Expand Down Expand Up @@ -766,6 +775,7 @@ metrics:
output_rule: true
# output_file: /tmp/falco_stats.jsonl
resource_utilization_enabled: true
state_counters_enabled: true
kernel_event_counters_enabled: true
libbpf_stats_enabled: true
convert_memory_to_mb: true
Expand Down
2 changes: 2 additions & 0 deletions userspace/falco/configuration.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ falco_configuration::falco_configuration():
m_metrics_stats_rule_enabled(false),
m_metrics_output_file(""),
m_metrics_resource_utilization_enabled(true),
m_metrics_state_counters_enabled(true),
m_metrics_kernel_event_counters_enabled(true),
m_metrics_libbpf_stats_enabled(true),
m_metrics_convert_memory_to_mb(true),
Expand Down Expand Up @@ -380,6 +381,7 @@ void falco_configuration::load_yaml(const std::string& config_name, const yaml_h
m_metrics_stats_rule_enabled = config.get_scalar<bool>("metrics.output_rule", false);
m_metrics_output_file = config.get_scalar<std::string>("metrics.output_file", "");
m_metrics_resource_utilization_enabled = config.get_scalar<bool>("metrics.resource_utilization_enabled", true);
m_metrics_state_counters_enabled = config.get_scalar<bool>("metrics.state_counters_enabled", true);
m_metrics_kernel_event_counters_enabled = config.get_scalar<bool>("metrics.kernel_event_counters_enabled", true);
m_metrics_libbpf_stats_enabled = config.get_scalar<bool>("metrics.libbpf_stats_enabled", true);
m_metrics_convert_memory_to_mb = config.get_scalar<bool>("metrics.convert_memory_to_mb", true);
Expand Down
1 change: 1 addition & 0 deletions userspace/falco/configuration.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ class falco_configuration
bool m_metrics_stats_rule_enabled;
std::string m_metrics_output_file;
bool m_metrics_resource_utilization_enabled;
bool m_metrics_state_counters_enabled;
bool m_metrics_kernel_event_counters_enabled;
bool m_metrics_libbpf_stats_enabled;
bool m_metrics_convert_memory_to_mb;
Expand Down
130 changes: 80 additions & 50 deletions userspace/falco/stats_writer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -296,60 +296,86 @@ void stats_writer::collector::get_metrics_output_fields_additional(
const scap_machine_info* machine_info = inspector->get_machine_info();

#if !defined(MINIMAL_BUILD) and !defined(__EMSCRIPTEN__)
/* Resource utilization, CPU and memory usage etc. */
uint32_t nstats = 0;
int32_t rc = 0;
uint32_t flags = 0;

if (m_writer->m_config->m_metrics_resource_utilization_enabled)
{
const scap_stats_v2* utilization;
auto buffer = inspector->get_sinsp_stats_v2_buffer();
utilization = libsinsp::resource_utilization::get_resource_utilization(agent_info, buffer, &nstats, &rc);
if (utilization && rc == 0 && nstats > 0)
/* Resource utilization, CPU and memory usage etc. */
flags |= PPM_SCAP_STATS_RESOURCE_UTILIZATION;
}

if (m_writer->m_config->m_metrics_state_counters_enabled)
{
/* Counters related to sinsp state (threadtable including fds per thread as well as container cache) */
flags |= PPM_SCAP_STATS_STATE_COUNTERS;
}

auto buffer = inspector->get_sinsp_stats_v2_buffer();
sinsp_stats_v2 sinsp_stats_v2 = inspector->get_sinsp_stats_v2();
sinsp_thread_manager* thread_manager = inspector->m_thread_manager;
const scap_stats_v2* sinsp_stats_v2_snapshot = libsinsp::stats::get_sinsp_stats_v2(flags, agent_info, thread_manager, sinsp_stats_v2, buffer, &nstats, &rc);

if (sinsp_stats_v2_snapshot && rc == 0 && nstats > 0)
{
for(uint32_t stat = 0; stat < nstats; stat++)
{
for(uint32_t stat = 0; stat < nstats; stat++)
if (sinsp_stats_v2_snapshot[stat].name[0] == '\0')
{
break;
}
char metric_name[STATS_NAME_MAX] = "falco.";
strlcat(metric_name, sinsp_stats_v2_snapshot[stat].name, sizeof(metric_name));
switch(sinsp_stats_v2_snapshot[stat].type)
{
char metric_name[STATS_NAME_MAX] = "falco.";
strlcat(metric_name, utilization[stat].name, sizeof(metric_name));
switch(utilization[stat].type)
case STATS_VALUE_TYPE_U64:
if (sinsp_stats_v2_snapshot[stat].value.u64 == 0 && !m_writer->m_config->m_metrics_include_empty_values)
{
case STATS_VALUE_TYPE_U64:
if (utilization[stat].value.u64 == 0 && !m_writer->m_config->m_metrics_include_empty_values)
{
break;
}
if (m_writer->m_config->m_metrics_convert_memory_to_mb && strncmp(utilization[stat].name, "container_memory_used", 22) == 0) // exact str match
{
output_fields[metric_name] = (uint64_t)(utilization[stat].value.u64 / (double)1024 / (double)1024);
}
else
{
output_fields[metric_name] = utilization[stat].value.u64;
}
break;
case STATS_VALUE_TYPE_U32:
if (utilization[stat].value.u32 == 0 && !m_writer->m_config->m_metrics_include_empty_values)
{
break;
}
if (m_writer->m_config->m_metrics_convert_memory_to_mb && strncmp(utilization[stat].name, "memory_", 7) == 0) // prefix match
}
if (m_writer->m_config->m_metrics_convert_memory_to_mb)
{
if (strncmp(sinsp_stats_v2_snapshot[stat].name, "container_memory_used", 22) == 0) // exact str match
{
output_fields[metric_name] = (uint32_t)(utilization[stat].value.u32 / (double)1024);
}
else
output_fields[metric_name] = (uint64_t)(sinsp_stats_v2_snapshot[stat].value.u64 / (double)1024 / (double)1024);

} else if (strncmp(sinsp_stats_v2_snapshot[stat].name, "memory_", 7) == 0) // prefix match
{
output_fields[metric_name] = utilization[stat].value.u32;
}
break;
case STATS_VALUE_TYPE_D:
if (utilization[stat].value.d == 0 && !m_writer->m_config->m_metrics_include_empty_values)
output_fields[metric_name] = (uint64_t)(sinsp_stats_v2_snapshot[stat].value.u64 / (double)1024);
} else
{
break;
output_fields[metric_name] = sinsp_stats_v2_snapshot[stat].value.u64;
}
output_fields[metric_name] = utilization[stat].value.d;
}
else
{
output_fields[metric_name] = sinsp_stats_v2_snapshot[stat].value.u64;
}
break;
case STATS_VALUE_TYPE_U32:
if (sinsp_stats_v2_snapshot[stat].value.u32 == 0 && !m_writer->m_config->m_metrics_include_empty_values)
{
break;
default:
}
if (m_writer->m_config->m_metrics_convert_memory_to_mb && strncmp(sinsp_stats_v2_snapshot[stat].name, "memory_", 7) == 0) // prefix match
{
output_fields[metric_name] = (uint32_t)(sinsp_stats_v2_snapshot[stat].value.u32 / (double)1024);
}
else
{
output_fields[metric_name] = sinsp_stats_v2_snapshot[stat].value.u32;
}
break;
case STATS_VALUE_TYPE_D:
if (sinsp_stats_v2_snapshot[stat].value.d == 0 && !m_writer->m_config->m_metrics_include_empty_values)
{
break;
}
output_fields[metric_name] = sinsp_stats_v2_snapshot[stat].value.d;
break;
default:
break;
}
}
}
Expand All @@ -362,7 +388,7 @@ void stats_writer::collector::get_metrics_output_fields_additional(
/* Kernel side stats counters and libbpf stats if applicable. */
nstats = 0;
rc = 0;
uint32_t flags = 0;
flags = 0;

if (m_writer->m_config->m_metrics_kernel_event_counters_enabled)
{
Expand All @@ -372,8 +398,8 @@ void stats_writer::collector::get_metrics_output_fields_additional(
{
flags |= PPM_SCAP_STATS_LIBBPF_STATS;
}
const scap_stats_v2* stats_v2 = inspector->get_capture_stats_v2(flags, &nstats, &rc);
if (stats_v2 && nstats > 0 && rc == 0)
const scap_stats_v2* stats_v2_snapshot = inspector->get_capture_stats_v2(flags, &nstats, &rc);
if (stats_v2_snapshot && nstats > 0 && rc == 0)
{
/* Cache n_evts and n_drops to derive n_drops_perc. */
uint64_t n_evts = 0;
Expand All @@ -382,17 +408,21 @@ void stats_writer::collector::get_metrics_output_fields_additional(
uint64_t n_drops_delta = 0;
for(uint32_t stat = 0; stat < nstats; stat++)
{
if (stats_v2_snapshot[stat].name[0] == '\0')
{
break;
}
// todo: as we expand scap_stats_v2 prefix may be pushed to scap or we may need to expand
// functionality here for example if we add userspace syscall counters that should be prefixed w/ `falco.`
char metric_name[STATS_NAME_MAX] = "scap.";
strlcat(metric_name, stats_v2[stat].name, sizeof(metric_name));
switch(stats_v2[stat].type)
strlcat(metric_name, stats_v2_snapshot[stat].name, sizeof(metric_name));
switch(stats_v2_snapshot[stat].type)
{
case STATS_VALUE_TYPE_U64:
/* Always send high level n_evts related fields, even if zero. */
if (strncmp(stats_v2[stat].name, "n_evts", 7) == 0) // exact not prefix match here
if (strncmp(stats_v2_snapshot[stat].name, "n_evts", 7) == 0) // exact not prefix match here
{
n_evts = stats_v2[stat].value.u64;
n_evts = stats_v2_snapshot[stat].value.u64;
output_fields[metric_name] = n_evts;
output_fields["scap.n_evts_prev"] = m_last_n_evts;
n_evts_delta = n_evts - m_last_n_evts;
Expand All @@ -408,9 +438,9 @@ void stats_writer::collector::get_metrics_output_fields_additional(
m_last_n_evts = n_evts;
}
/* Always send high level n_drops related fields, even if zero. */
else if (strncmp(stats_v2[stat].name, "n_drops", 8) == 0) // exact not prefix match here
else if (strncmp(stats_v2_snapshot[stat].name, "n_drops", 8) == 0) // exact not prefix match here
{
n_drops = stats_v2[stat].value.u64;
n_drops = stats_v2_snapshot[stat].value.u64;
output_fields[metric_name] = n_drops;
output_fields["scap.n_drops_prev"] = m_last_n_drops;
n_drops_delta = n_drops - m_last_n_drops;
Expand All @@ -425,11 +455,11 @@ void stats_writer::collector::get_metrics_output_fields_additional(
}
m_last_n_drops = n_drops;
}
if (stats_v2[stat].value.u64 == 0 && !m_writer->m_config->m_metrics_include_empty_values)
if (stats_v2_snapshot[stat].value.u64 == 0 && !m_writer->m_config->m_metrics_include_empty_values)
{
break;
}
output_fields[metric_name] = stats_v2[stat].value.u64;
output_fields[metric_name] = stats_v2_snapshot[stat].value.u64;
break;
default:
break;
Expand Down

0 comments on commit eadff28

Please sign in to comment.