From b9053524954ff64dbf37fae8d6de48330b292c6d Mon Sep 17 00:00:00 2001 From: Marco Mariani Date: Tue, 21 Jun 2022 14:53:51 +0200 Subject: [PATCH 01/10] cscli setup --- cmd/crowdsec-cli/main.go | 2 +- cmd/crowdsec-cli/setup.go | 312 +++++ config/detect.yaml | 456 ++++++++ debian/postinst | 30 +- go.mod | 1 + go.sum | 2 + pkg/acquisition/acquisition.go | 94 +- pkg/acquisition/acquisition_test.go | 46 +- .../modules/cloudwatch/cloudwatch.go | 70 +- pkg/acquisition/modules/docker/docker.go | 46 +- pkg/acquisition/modules/file/file.go | 55 +- pkg/acquisition/modules/file/file_test.go | 2 +- .../modules/journalctl/journalctl.go | 38 +- pkg/acquisition/modules/kafka/kafka.go | 30 +- pkg/acquisition/modules/kinesis/kinesis.go | 30 +- pkg/acquisition/modules/syslog/syslog.go | 47 +- .../modules/wineventlog/wineventlog.go | 4 + .../wineventlog/wineventlog_windows.go | 38 +- pkg/cstest/utils.go | 19 + pkg/cwhub/cwhub.go | 2 + pkg/setup/README.md | 338 ++++++ pkg/setup/detect.go | 581 +++++++++ pkg/setup/detect_test.go | 997 ++++++++++++++++ pkg/setup/export_test.go | 9 + pkg/setup/install.go | 255 ++++ pkg/setup/units.go | 59 + pkg/setup/units_test.go | 32 + scripts/test_env.sh | 90 +- tests/ansible/requirements.yml | 3 + .../experimental/wizard-centos-8/Vagrantfile | 42 + .../experimental/wizard-centos-8/bootstrap | 5 + .../wizard-debian-bullseye/Vagrantfile | 42 + .../wizard-debian-bullseye/bootstrap | 5 + .../wizard-debian-buster/Vagrantfile | 42 + .../wizard-debian-buster/bootstrap | 5 + .../experimental/wizard-fedora-36/Vagrantfile | 42 + .../experimental/wizard-fedora-36/bootstrap | 5 + .../wizard-ubuntu-22.04/Vagrantfile | 42 + .../wizard-ubuntu-22.04/bootstrap | 5 + tests/bats-detect/WARNING.md | 8 + tests/bats-detect/apache2-deb.bats | 49 + tests/bats-detect/apache2-rpm.bats | 49 + tests/bats-detect/asterisk-deb.bats | 47 + tests/bats-detect/asterisk-rpm.bats | 50 + tests/bats-detect/caddy-deb.bats | 53 + tests/bats-detect/caddy-rpm.bats | 49 + tests/bats-detect/dovecot-deb.bats | 47 + tests/bats-detect/dovecot-rpm.bats | 47 + tests/bats-detect/emby-deb.bats | 52 + tests/bats-detect/emby-rpm.bats | 52 + tests/bats-detect/endlessh-deb.bats | 48 + tests/bats-detect/endlessh-rpm.bats | 48 + tests/bats-detect/gitea.bats | 46 + tests/bats-detect/haproxy-deb.bats | 47 + tests/bats-detect/haproxy-rpm.bats | 47 + tests/bats-detect/lemonldap-deb.bats | 47 + tests/bats-detect/lemonldap-rpm.bats | 50 + tests/bats-detect/lib/setup_file_detect.sh | 55 + tests/bats-detect/litespeed.bats | 47 + tests/bats-detect/mariadb-deb.bats | 47 + tests/bats-detect/mariadb-rpm.bats | 47 + tests/bats-detect/mysql-deb.bats | 64 + tests/bats-detect/mysql-rpm.bats | 48 + tests/bats-detect/nginx-deb.bats | 47 + tests/bats-detect/nginx-rpm.bats | 47 + tests/bats-detect/odoo-deb.bats | 52 + tests/bats-detect/odoo-rpm.bats | 49 + tests/bats-detect/ombi-deb.bats | 52 + tests/bats-detect/openresty-deb.bats | 57 + tests/bats-detect/openresty-rpm.bats | 54 + tests/bats-detect/pgsql-deb.bats | 61 + tests/bats-detect/pgsql-rpm.bats | 51 + tests/bats-detect/postfix-deb.bats | 49 + tests/bats-detect/postfix-rpm.bats | 47 + tests/bats-detect/proftpd-deb.bats | 47 + tests/bats-detect/proftpd-rpm.bats | 47 + tests/bats-detect/proxmox-deb.bats | 62 + tests/bats-detect/pureftpd-deb.bats | 47 + tests/bats-detect/pureftpd-rpm.bats | 47 + tests/bats-detect/smb-deb.bats | 50 + tests/bats-detect/smb-rpm.bats | 47 + tests/bats-detect/sshd-deb.bats | 49 + tests/bats-detect/sshd-rpm.bats | 49 + tests/bats-detect/suricata-deb.bats | 47 + tests/bats-detect/suricata-rpm.bats | 47 + .../testdata/enable_lst_debian_repo.sh | 65 + tests/bats-detect/vsftpd-deb.bats | 48 + tests/bats-detect/vsftpd-rpm.bats | 47 + tests/bats/07_setup.bats | 816 +++++++++++++ tests/bats/testdata/07_setup/detect.yaml | 88 ++ tests/lib/config/config-local | 3 + wizard.sh | 1042 ++++++++--------- 92 files changed, 7499 insertions(+), 782 deletions(-) create mode 100644 cmd/crowdsec-cli/setup.go create mode 100644 config/detect.yaml create mode 100644 pkg/setup/README.md create mode 100644 pkg/setup/detect.go create mode 100644 pkg/setup/detect_test.go create mode 100644 pkg/setup/export_test.go create mode 100644 pkg/setup/install.go create mode 100644 pkg/setup/units.go create mode 100644 pkg/setup/units_test.go create mode 100644 tests/ansible/vagrant/experimental/wizard-centos-8/Vagrantfile create mode 100755 tests/ansible/vagrant/experimental/wizard-centos-8/bootstrap create mode 100644 tests/ansible/vagrant/experimental/wizard-debian-bullseye/Vagrantfile create mode 100755 tests/ansible/vagrant/experimental/wizard-debian-bullseye/bootstrap create mode 100644 tests/ansible/vagrant/experimental/wizard-debian-buster/Vagrantfile create mode 100755 tests/ansible/vagrant/experimental/wizard-debian-buster/bootstrap create mode 100644 tests/ansible/vagrant/experimental/wizard-fedora-36/Vagrantfile create mode 100755 tests/ansible/vagrant/experimental/wizard-fedora-36/bootstrap create mode 100644 tests/ansible/vagrant/experimental/wizard-ubuntu-22.04/Vagrantfile create mode 100755 tests/ansible/vagrant/experimental/wizard-ubuntu-22.04/bootstrap create mode 100644 tests/bats-detect/WARNING.md create mode 100644 tests/bats-detect/apache2-deb.bats create mode 100644 tests/bats-detect/apache2-rpm.bats create mode 100644 tests/bats-detect/asterisk-deb.bats create mode 100644 tests/bats-detect/asterisk-rpm.bats create mode 100644 tests/bats-detect/caddy-deb.bats create mode 100644 tests/bats-detect/caddy-rpm.bats create mode 100644 tests/bats-detect/dovecot-deb.bats create mode 100644 tests/bats-detect/dovecot-rpm.bats create mode 100644 tests/bats-detect/emby-deb.bats create mode 100644 tests/bats-detect/emby-rpm.bats create mode 100644 tests/bats-detect/endlessh-deb.bats create mode 100644 tests/bats-detect/endlessh-rpm.bats create mode 100644 tests/bats-detect/gitea.bats create mode 100644 tests/bats-detect/haproxy-deb.bats create mode 100644 tests/bats-detect/haproxy-rpm.bats create mode 100644 tests/bats-detect/lemonldap-deb.bats create mode 100644 tests/bats-detect/lemonldap-rpm.bats create mode 100755 tests/bats-detect/lib/setup_file_detect.sh create mode 100644 tests/bats-detect/litespeed.bats create mode 100644 tests/bats-detect/mariadb-deb.bats create mode 100644 tests/bats-detect/mariadb-rpm.bats create mode 100644 tests/bats-detect/mysql-deb.bats create mode 100644 tests/bats-detect/mysql-rpm.bats create mode 100644 tests/bats-detect/nginx-deb.bats create mode 100644 tests/bats-detect/nginx-rpm.bats create mode 100644 tests/bats-detect/odoo-deb.bats create mode 100644 tests/bats-detect/odoo-rpm.bats create mode 100644 tests/bats-detect/ombi-deb.bats create mode 100644 tests/bats-detect/openresty-deb.bats create mode 100644 tests/bats-detect/openresty-rpm.bats create mode 100644 tests/bats-detect/pgsql-deb.bats create mode 100644 tests/bats-detect/pgsql-rpm.bats create mode 100644 tests/bats-detect/postfix-deb.bats create mode 100644 tests/bats-detect/postfix-rpm.bats create mode 100644 tests/bats-detect/proftpd-deb.bats create mode 100644 tests/bats-detect/proftpd-rpm.bats create mode 100644 tests/bats-detect/proxmox-deb.bats create mode 100644 tests/bats-detect/pureftpd-deb.bats create mode 100644 tests/bats-detect/pureftpd-rpm.bats create mode 100644 tests/bats-detect/smb-deb.bats create mode 100644 tests/bats-detect/smb-rpm.bats create mode 100644 tests/bats-detect/sshd-deb.bats create mode 100644 tests/bats-detect/sshd-rpm.bats create mode 100644 tests/bats-detect/suricata-deb.bats create mode 100644 tests/bats-detect/suricata-rpm.bats create mode 100755 tests/bats-detect/testdata/enable_lst_debian_repo.sh create mode 100644 tests/bats-detect/vsftpd-deb.bats create mode 100644 tests/bats-detect/vsftpd-rpm.bats create mode 100644 tests/bats/07_setup.bats create mode 100644 tests/bats/testdata/07_setup/detect.yaml diff --git a/cmd/crowdsec-cli/main.go b/cmd/crowdsec-cli/main.go index 0b2b8654879..569e910e572 100644 --- a/cmd/crowdsec-cli/main.go +++ b/cmd/crowdsec-cli/main.go @@ -226,7 +226,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall rootCmd.AddCommand(NewHubTestCmd()) rootCmd.AddCommand(NewNotificationsCmd()) rootCmd.AddCommand(NewSupportCmd()) - + rootCmd.AddCommand(NewSetupCmd()) if err := rootCmd.Execute(); err != nil { if bincoverTesting != "" { log.Debug("coverage report is enabled") diff --git a/cmd/crowdsec-cli/setup.go b/cmd/crowdsec-cli/setup.go new file mode 100644 index 00000000000..8bfa5b8c1d2 --- /dev/null +++ b/cmd/crowdsec-cli/setup.go @@ -0,0 +1,312 @@ +package main + +import ( + "bytes" + "fmt" + "os" + "os/exec" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "gopkg.in/yaml.v3" + kyaml "sigs.k8s.io/yaml" + + "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/crowdsecurity/crowdsec/pkg/setup" +) + +// NewSetupCmd defines the "cscli setup" command. +func NewSetupCmd() *cobra.Command { + cmdSetup := &cobra.Command{ + Use: "setup", + Short: "Tools to configure crowdsec", + Long: "Manage hub configuration and service detection", + Args: cobra.MinimumNArgs(0), + DisableAutoGenTag: true, + } + + // + // cscli setup detect + // + { + cmdSetupDetect := &cobra.Command{ + Use: "detect", + Short: "detect running services, generate a setup file", + DisableAutoGenTag: true, + RunE: runSetupDetect, + } + + defaultServiceDetect := csconfig.DefaultConfigPath("hub", "detect.yaml") + + flags := cmdSetupDetect.Flags() + flags.String("detect-config", defaultServiceDetect, "path to service detection configuration") + flags.Bool("list-supported-services", false, "do not detect; only print supported services") + flags.StringSlice("force-unit", nil, "force detection of a systemd unit (can be repeated)") + flags.StringSlice("force-process", nil, "force detection of a running process (can be repeated)") + flags.StringSlice("skip-service", nil, "ignore a service, don't recommend hub/datasources (can be repeated)") + flags.String("force-os-family", "", "override OS.Family: one of linux, freebsd, windows or darwin") + flags.String("force-os-id", "", "override OS.ID=[debian | ubuntu | , redhat...]") + flags.String("force-os-version", "", "override OS.RawVersion (of OS or Linux distribution)") + flags.Bool("snub-systemd", false, "don't use systemd, even if available") + flags.Bool("yaml", false, "output yaml, not json") + cmdSetup.AddCommand(cmdSetupDetect) + } + + // + // cscli setup install-hub + // + { + cmdSetupInstallHub := &cobra.Command{ + Use: "install-hub [setup_file] [flags]", + Short: "install items from a setup file", + Args: cobra.ExactArgs(1), + DisableAutoGenTag: true, + RunE: runSetupInstallHub, + } + + flags := cmdSetupInstallHub.Flags() + flags.Bool("dry-run", false, "don't install anything; print out what would have been") + cmdSetup.AddCommand(cmdSetupInstallHub) + } + + // + // cscli setup datasources + // + { + cmdSetupDataSources := &cobra.Command{ + Use: "datasources [setup_file] [flags]", + Short: "generate datasource (acquisition) configuration from a setup file", + Args: cobra.ExactArgs(1), + DisableAutoGenTag: true, + RunE: runSetupDataSources, + } + + flags := cmdSetupDataSources.Flags() + flags.String("to-dir", "", "write the configuration to a directory, in multiple files") + cmdSetup.AddCommand(cmdSetupDataSources) + } + + // + // cscli setup validate + // + { + cmdSetupValidate := &cobra.Command{ + Use: "validate [setup_file]", + Short: "validate a setup file", + Args: cobra.ExactArgs(1), + DisableAutoGenTag: true, + RunE: runSetupValidate, + } + + cmdSetup.AddCommand(cmdSetupValidate) + } + + return cmdSetup +} + +func runSetupDetect(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + detectConfigFile, err := flags.GetString("detect-config") + if err != nil { + return err + } + + listSupportedServices, err := flags.GetBool("list-supported-services") + if err != nil { + return err + } + + forcedUnits, err := flags.GetStringSlice("force-unit") + if err != nil { + return err + } + + forcedProcesses, err := flags.GetStringSlice("force-process") + if err != nil { + return err + } + + forcedOSFamily, err := flags.GetString("force-os-family") + if err != nil { + return err + } + + forcedOSID, err := flags.GetString("force-os-id") + if err != nil { + return err + } + + forcedOSVersion, err := flags.GetString("force-os-version") + if err != nil { + return err + } + + skipServices, err := flags.GetStringSlice("skip-service") + if err != nil { + return err + } + + snubSystemd, err := flags.GetBool("snub-systemd") + if err != nil { + return err + } + + if !snubSystemd { + _, err := exec.LookPath("systemctl") + if err != nil { + log.Debug("systemctl not available: snubbing systemd") + snubSystemd = true + } + } + + outYaml, err := flags.GetBool("yaml") + if err != nil { + return err + } + + if forcedOSFamily == "" && forcedOSID != "" { + log.Debug("force-os-id is set: force-os-family defaults to 'linux'") + forcedOSFamily = "linux" + } + + if listSupportedServices { + supported, err := setup.ListSupported(detectConfigFile) + if err != nil { + return err + } + + for _, svc := range supported { + fmt.Println(svc) + } + + return nil + } + + opts := setup.DetectOptions{ + ForcedUnits: forcedUnits, + ForcedProcesses: forcedProcesses, + ForcedOS: setup.ExprOS{ + Family: forcedOSFamily, + ID: forcedOSID, + RawVersion: forcedOSVersion, + }, + SkipServices: skipServices, + SnubSystemd: snubSystemd, + } + + hubSetup, err := setup.Detect(detectConfigFile, opts) + if err != nil { + return fmt.Errorf("detecting services: %w", err) + } + + setup, err := setupAsString(hubSetup, outYaml) + if err != nil { + return err + } + fmt.Println(setup) + + return nil +} + +func setupAsString(cs setup.Setup, outYaml bool) (string, error) { + var ( + ret []byte + err error + ) + + wrap := func(err error) error { + return fmt.Errorf("while marshaling setup: %w", err) + } + + indentLevel := 2 + buf := &bytes.Buffer{} + enc := yaml.NewEncoder(buf) + enc.SetIndent(indentLevel) + + if err = enc.Encode(cs); err != nil { + return "", wrap(err) + } + + if err = enc.Close(); err != nil { + return "", wrap(err) + } + + ret = buf.Bytes() + + if !outYaml { + // take a general approach to output json, so we avoid the + // double tags in the structures and can use go-yaml features + // missing from the json package + ret, err = kyaml.YAMLToJSON(ret) + if err != nil { + return "", wrap(err) + } + } + + return string(ret), nil +} + +func runSetupDataSources(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + fromFile := args[0] + + toDir, err := flags.GetString("to-dir") + if err != nil { + return err + } + + input, err := os.ReadFile(fromFile) + if err != nil { + return fmt.Errorf("while reading setup file: %w", err) + } + + output, err := setup.DataSources(input, toDir) + if err != nil { + return err + } + + if toDir == "" { + fmt.Println(output) + } + + return nil +} + +func runSetupInstallHub(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + fromFile := args[0] + + dryRun, err := flags.GetBool("dry-run") + if err != nil { + return err + } + + input, err := os.ReadFile(fromFile) + if err != nil { + return fmt.Errorf("while reading file %s: %w", fromFile, err) + } + + if err = setup.InstallHubItems(csConfig, input, dryRun); err != nil { + return err + } + + return nil +} + +func runSetupValidate(cmd *cobra.Command, args []string) error { + fromFile := args[0] + input, err := os.ReadFile(fromFile) + if err != nil { + return fmt.Errorf("while reading stdin: %w", err) + } + + if err = setup.Validate(input); err != nil { + fmt.Printf("%v\n", err) + return fmt.Errorf("invalid setup file") + } + + return nil +} diff --git a/config/detect.yaml b/config/detect.yaml new file mode 100644 index 00000000000..2ad0ab9e37d --- /dev/null +++ b/config/detect.yaml @@ -0,0 +1,456 @@ +--- +version: 1.0 + +detect: + + # + # crowdsecurity/apache2 + # + + # XXX some distro is using this path? + # - /var/log/*http*/*.log + + apache2-systemd-deb: + when: + - UnitFound("apache2.service") + - PathExists("/etc/debian_version") + install: + collections: + - crowdsecurity/apache2 + datasource: + filenames: + - /var/log/apache2/*.log + labels: + type: apache2 + + apache2-systemd-rpm: + when: + - UnitFound("httpd.service") + - PathExists("/etc/redhat-release") + install: + collections: + - crowdsecurity/apache2 + datasource: + filenames: + - /var/log/httpd/*.log + # XXX /var/log/*http*/*.log + labels: + type: apache2 + + # + # crowdsecurity/asterisk + # + + asterisk-systemd: + when: + - UnitFound("asterisk.service") + install: + collections: + - crowdsecurity/asterisk + datasource: + labels: + type: asterisk + filenames: + - /var/log/asterisk/*.log + + # + # crowdsecurity/caddy + # + + caddy-systemd: + when: + - UnitFound("caddy.service") + install: + collections: + - crowdsecurity/caddy + datasource: + labels: + type: caddy + filenames: + - /var/log/caddy/*.log + + # + # crowdsecurity/dovecot + # + + dovecot-systemd: + when: + - UnitFound("dovecot.service") + install: + collections: + - crowdsecurity/dovecot + datasource: + labels: + type: syslog + filenames: + - /var/log/mail.log + + # + # LePresidente/emby + # + + emby-systemd: + when: + - UnitFound("emby-server.service") + install: + collections: + - LePresidente/emby + datasource: + labels: + type: emby + filenames: + - /var/log/embyserver.txt + + # + # crowdsecurity/endlessh + # + + endlessh-systemd: + when: + - UnitFound("endlessh.service") + install: + collections: + - crowdsecurity/endlessh + datasource: + source: journalctl + labels: + type: syslog + # XXX this? or /var/log/syslog? + journalctl_filter: + - "_SYSTEMD_UNIT=endlessh.service" + + # + # crowdsecurity/gitea + # + + # XXX untested + + gitea-systemd: + when: + - UnitFound("gitea.service") + install: + collections: + - crowdsecurity/gitea + datasource: + labels: + type: gitea + filenames: + - /var/log/gitea.log + + # + # crowdsecurity/haproxy + # + + haproxy-systemd: + when: + - UnitFound("haproxy.service") + install: + collections: + - crowdsecurity/haproxy + datasource: + labels: + type: haproxy + filenames: + - /var/log/haproxy/*.log + + # + # firewallservices/lemonldap-ng + # + + lemonldap-ng-systemd: + when: + - UnitFound("lemonldap-ng-fastcgi-server.service") + install: + collections: + - firewallservices/lemonldap-ng + datasource: + # XXX todo where are the logs? + labels: + type: syslog + + # + # crowdsecurity/mariadb + # + + mariadb-systemd: + when: + - UnitFound("mariadb.service") + install: + collections: + - crowdsecurity/mariadb + datasource: + labels: + type: mysql + filenames: + - /var/log/mysql/error.log + + # + # crowdsecurity/mysql + # + + mysql-systemd: + when: + - UnitFound("mysql.service") + install: + collections: + - crowdsecurity/mysql + datasource: + labels: + type: mysql + filenames: + - /var/log/mysql/error.log + + # + # crowdsecurity/nginx + # + + nginx-systemd: + when: + - UnitFound("nginx.service") + install: + collections: + - crowdsecurity/nginx + datasource: + labels: + type: nginx + filenames: + - /var/log/nginx/*.log + + openresty-systemd: + when: + - UnitFound("openresty.service") + install: + collections: + - crowdsecurity/nginx + datasource: + labels: + type: nginx + filenames: + - /usr/local/openresty/nginx/logs/*.log + + # + # crowdsecurity/odoo + # + + odoo-systemd: + when: + - UnitFound("odoo.service") + install: + collections: + - crowdsecurity/odoo + datasource: + labels: + type: odoo + filenames: + - /var/log/odoo/*.log + + # + # LePresidente/ombi + # + + # This only works on deb-based systems. On other distributions, the + # application is run from the release tarball and the log location depends on + # the location it's run from. + + ombi-systemd: + when: + - UnitFound("ombi.service") + - PathExists("/etc/debian_version") + install: + collections: + - LePresidente/ombi + datasource: + labels: + type: ombi + filenames: + - /var/log/ombi/log-*.txt + + # + # crowdsecurity/pgsql + # + + pgsql-systemd-deb: + when: + - UnitFound("postgresql.service") + - PathExists("/etc/debian_version") + install: + collections: + - crowdsecurity/pgsql + datasource: + labels: + type: postgres + filenames: + - /var/log/postgresql/*.log + + pgsql-systemd-rpm: + when: + - UnitFound("postgresql.service") + - PathExists("/etc/redhat-release") + install: + collections: + - crowdsecurity/pgsql + datasource: + labels: + type: postgres + filenames: + - /var/lib/pgsql/data/log/*.log + + # + # crowdsecurity/postfix + # + + postfix-systemd: + when: + - UnitFound("postfix.service") + install: + collections: + - crowdsecurity/postfix + datasource: + labels: + type: syslog + filenames: + - /var/log/mail.log + + # + # crowdsecurity/proftpd + # + + proftpd-systemd: + when: + - UnitFound("proftpd.service") + install: + collections: + - crowdsecurity/proftpd + datasource: + labels: + type: proftpd + filenames: + - /var/log/proftpd/*.log + + # + # fulljackz/pureftpd + # + + pureftpd-systemd: + when: + - UnitFound("pure-ftpd.service") + install: + collections: + - fulljackz/pureftpd + # XXX ? + datasource: + labels: + type: syslog + filenames: + - /var/log/pure-ftpd/*.log + + # + # crowdsecurity/smb + # + + smb-systemd: + when: + # deb -> smbd.service + # rpm -> smb.service + - UnitFound("smbd.service") or UnitFound("smb.service") + install: + collections: + - crowdsecurity/smb + datasource: + labels: + type: smb + filenames: + - /var/log/samba*.log + + # + # crowdsecurity/sshd + # + + sshd-systemd: + when: + # deb -> ssh.service + # rpm -> sshd.service + - UnitFound("ssh.service") or UnitFound("sshd.service") or UnitFound("ssh.socket") or UnitFound("sshd.socket") + install: + collections: + - crowdsecurity/sshd + datasource: + labels: + type: syslog + filenames: + - /var/log/auth.log + - /var/log/sshd.log + - /var/log/secure + + # + # crowdsecurity/suricata + # + + suricata-systemd: + when: + - UnitFound("suricata.service") + install: + collections: + - crowdsecurity/suricata + datasource: + labels: + type: suricata-evelogs + filenames: + - /var/log/suricata/eve.json + + # + # crowdsecurity/vsftpd + # + + vsftpd-systemd: + when: + - UnitFound("vsftpd.service") + install: + collections: + - crowdsecurity/vsftpd + datasource: + labels: + type: vsftpd + filenames: + - /var/log/vsftpd/*.log + + # + # Operating Systems + # + + linux: + when: + - OS.Family == "linux" + install: + collections: + - crowdsecurity/linux + datasource: + labels: + type: syslog + filenames: + - /var/log/syslog + - /var/log/kern.log + - /var/log/messages + + freebsd: + when: + - OS.Family == "freebsd" + install: + collections: + - crowdsecurity/freebsd + + windows: + when: + - OS.Family == "windows" + install: + collections: + - crowdsecurity/windows + + # + # anti-lockout + # + + whitelists: + install: + parsers: + - crowdsecurity/whitelists diff --git a/debian/postinst b/debian/postinst index a862c88750d..3aabdd086a0 100644 --- a/debian/postinst +++ b/debian/postinst @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh COLLECTIONS=false set -e @@ -7,11 +7,11 @@ set -e . /usr/share/debconf/confmodule if [ "$1" = configure ]; then - if [[ ! -d /var/lib/crowdsec/data ]]; then + if [ ! -d /var/lib/crowdsec/data ]; then mkdir -p /var/lib/crowdsec/data fi - if [[ -d /var/lib/crowdsec/backup ]]; then + if [ -d /var/lib/crowdsec/backup ]; then cscli config restore /var/lib/crowdsec/backup/backup.config rm -rf /var/lib/crowdsec/backup /usr/bin/cscli hub update @@ -19,29 +19,27 @@ if [ "$1" = configure ]; then systemctl start crowdsec fi - . /usr/share/crowdsec/wizard.sh -n - if ! [[ -f /etc/crowdsec/acquis.yaml ]]; then - echo Creating /etc/crowdsec/acquis.yaml - set +e - SILENT=true detect_services - SILENT=true TMP_ACQUIS_FILE_SKIP=skip genacquisition - set -e - COLLECTIONS=true + if ! find /etc/crowdsec/acquis.d -maxdepth 1 -type f -name '*' 2>/dev/null | grep -q '.'; then + echo Creating /etc/crowdsec/acquis.d + mkdir -p /etc/crowdsec/acquis.d + cscli setup detect >/etc/crowdsec/.setup.yaml + cscli setup install-hub /etc/crowdsec/.setup.yaml + cscli setup datasources /etc/crowdsec/.setup.yaml --to-dir /etc/crowdsec/acquis.d fi - if [[ -f /etc/crowdsec/local_api_credentials.yaml ]] ; then + if [ -f /etc/crowdsec/local_api_credentials.yaml ] ; then chmod 600 /etc/crowdsec/local_api_credentials.yaml fi - if [[ -f /etc/crowdsec/online_api_credentials.yaml ]]; then + if [ -f /etc/crowdsec/online_api_credentials.yaml ]; then chmod 600 /etc/crowdsec/online_api_credentials.yaml fi - if [[ ! -f /etc/crowdsec/local_api_credentials.yaml ]] || [[ ! -f /etc/crowdsec/online_api_credentials.yaml ]]; then - if [[ ! -f /etc/crowdsec/local_api_credentials.yaml ]] ; then + if [ ! -f /etc/crowdsec/local_api_credentials.yaml ] || [ ! -f /etc/crowdsec/online_api_credentials.yaml ]; then + if [ ! -f /etc/crowdsec/local_api_credentials.yaml ] ; then install -m 600 /dev/null /etc/crowdsec/local_api_credentials.yaml fi - if [[ ! -f /etc/crowdsec/online_api_credentials.yaml ]] ; then + if [ ! -f /etc/crowdsec/online_api_credentials.yaml ] ; then install -m 600 /dev/null /etc/crowdsec/online_api_credentials.yaml fi diff --git a/go.mod b/go.mod index a1c4ed18821..991c092d0b9 100644 --- a/go.mod +++ b/go.mod @@ -128,6 +128,7 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/k0kubun/pp v3.0.1+incompatible // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/compress v1.15.7 // indirect github.com/leodido/go-urn v1.2.1 // indirect diff --git a/go.sum b/go.sum index 152b73d30ec..e65e7392a60 100644 --- a/go.sum +++ b/go.sum @@ -499,6 +499,8 @@ github.com/jszwec/csvutil v1.5.1 h1:c3GFBhj6DFMUl4dMK3+B6rz2+LWWS/e9VJiVJ9t9kfQ= github.com/jszwec/csvutil v1.5.1/go.mod h1:Rpu7Uu9giO9subDyMCIQfHVDuLrcaC36UA4YcJjGBkg= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/k0kubun/pp v3.0.1+incompatible h1:3tqvf7QgUnZ5tXO6pNAZlrvHgl6DvifjDrd9g2S9Z40= +github.com/k0kubun/pp v3.0.1+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go index 00c5325d861..b0e171e0e40 100644 --- a/pkg/acquisition/acquisition.go +++ b/pkg/acquisition/acquisition.go @@ -6,6 +6,11 @@ import ( "os" "strings" + "github.com/prometheus/client_golang/prometheus" + log "github.com/sirupsen/logrus" + tomb "gopkg.in/tomb.v2" + "gopkg.in/yaml.v2" + "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" cloudwatchacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/cloudwatch" dockeracquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/docker" @@ -17,19 +22,14 @@ import ( wineventlogacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/wineventlog" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" - - tomb "gopkg.in/tomb.v2" ) // The interface each datasource must implement type DataSource interface { GetMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module GetAggregMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module (aggregated mode, limits cardinality) - Configure([]byte, *log.Entry) error // Configure the datasource + UnmarshalConfig([]byte) error // Decode and pre-validate the YAML datasource - anything that can be checked before runtime + Configure([]byte, *log.Entry) error // Complete the YAML datasource configuration and perform runtime checks. ConfigureByDSN(string, map[string]string, *log.Entry) error // Configure the datasource GetMode() string // Get the mode (TAIL, CAT or SERVER) GetName() string // Get the name of the module @@ -39,66 +39,37 @@ type DataSource interface { Dump() interface{} } -var AcquisitionSources = []struct { - name string - iface func() DataSource -}{ - { - name: "file", - iface: func() DataSource { return &fileacquisition.FileSource{} }, - }, - { - name: "journalctl", - iface: func() DataSource { return &journalctlacquisition.JournalCtlSource{} }, - }, - { - name: "cloudwatch", - iface: func() DataSource { return &cloudwatchacquisition.CloudwatchSource{} }, - }, - { - name: "syslog", - iface: func() DataSource { return &syslogacquisition.SyslogSource{} }, - }, - { - name: "docker", - iface: func() DataSource { return &dockeracquisition.DockerSource{} }, - }, - { - name: "kinesis", - iface: func() DataSource { return &kinesisacquisition.KinesisSource{} }, - }, - { - name: "wineventlog", - iface: func() DataSource { return &wineventlogacquisition.WinEventLogSource{} }, - }, - { - name: "kafka", - iface: func() DataSource { return &kafkaacquisition.KafkaSource{} }, - }, +var AcquisitionSources = map[string]func() DataSource{ + "file": func() DataSource { return &fileacquisition.FileSource{} }, + "journalctl": func() DataSource { return &journalctlacquisition.JournalCtlSource{} }, + "cloudwatch": func() DataSource { return &cloudwatchacquisition.CloudwatchSource{} }, + "syslog": func() DataSource { return &syslogacquisition.SyslogSource{} }, + "docker": func() DataSource { return &dockeracquisition.DockerSource{} }, + "kinesis": func() DataSource { return &kinesisacquisition.KinesisSource{} }, + "wineventlog": func() DataSource { return &wineventlogacquisition.WinEventLogSource{} }, + "kafka": func() DataSource { return &kafkaacquisition.KafkaSource{} }, } func GetDataSourceIface(dataSourceType string) DataSource { - for _, source := range AcquisitionSources { - if source.name == dataSourceType { - return source.iface() - } + source := AcquisitionSources[dataSourceType] + if source == nil { + return nil } - return nil + return source() } func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataSource, error) { - - //we dump it back to []byte, because we want to decode the yaml blob twice : - //once to DataSourceCommonCfg, and then later to the dedicated type of the datasource + // we dump it back to []byte, because we want to decode the yaml blob twice: + // once to DataSourceCommonCfg, and then later to the dedicated type of the datasource yamlConfig, err := yaml.Marshal(commonConfig) if err != nil { - return nil, errors.Wrap(err, "unable to marshal back interface") + return nil, fmt.Errorf("unable to marshal back interface: %v", err) } if dataSrc := GetDataSourceIface(commonConfig.Source); dataSrc != nil { /* this logger will then be used by the datasource at runtime */ clog := log.New() if err := types.ConfigureLogger(clog); err != nil { - return nil, errors.Wrap(err, "while configuring datasource logger") + return nil, fmt.Errorf("while configuring datasource logger: %w", err) } if commonConfig.LogLevel != nil { clog.SetLevel(*commonConfig.LogLevel) @@ -112,11 +83,11 @@ func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataS subLogger := clog.WithFields(customLog) /* check eventual dependencies are satisfied (ie. journald will check journalctl availability) */ if err := dataSrc.CanRun(); err != nil { - return nil, errors.Wrapf(err, "datasource %s cannot be run", commonConfig.Source) + return nil, fmt.Errorf("datasource %s cannot be run: %w", commonConfig.Source, err) } /* configure the actual datasource */ if err := dataSrc.Configure(yamlConfig, subLogger); err != nil { - return nil, errors.Wrapf(err, "failed to configure datasource %s", commonConfig.Source) + return nil, fmt.Errorf("failed to configure datasource %s: %w", commonConfig.Source, err) } return &dataSrc, nil @@ -153,14 +124,14 @@ func LoadAcquisitionFromDSN(dsn string, labels map[string]string) ([]DataSource, /* this logger will then be used by the datasource at runtime */ clog := log.New() if err := types.ConfigureLogger(clog); err != nil { - return nil, errors.Wrap(err, "while configuring datasource logger") + return nil, fmt.Errorf("while configuring datasource logger: %w", err) } subLogger := clog.WithFields(log.Fields{ "type": dsn, }) err := dataSrc.ConfigureByDSN(dsn, labels, subLogger) if err != nil { - return nil, errors.Wrapf(err, "while configuration datasource for %s", dsn) + return nil, fmt.Errorf("while configuration datasource for %s: %w", dsn, err) } sources = append(sources, dataSrc) return sources, nil @@ -168,7 +139,6 @@ func LoadAcquisitionFromDSN(dsn string, labels map[string]string) ([]DataSource, // LoadAcquisitionFromFile unmarshals the configuration item and checks its availability func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource, error) { - var sources []DataSource for _, acquisFile := range config.AcquisitionFiles { @@ -188,7 +158,7 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource, log.Tracef("End of yaml file") break } - return nil, errors.Wrapf(err, "failed to yaml decode %s", acquisFile) + return nil, fmt.Errorf("failed to yaml decode %s: %w", acquisFile, err) } //for backward compat ('type' was not mandatory, detect it) @@ -212,7 +182,7 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource, } src, err := DataSourceConfigure(sub) if err != nil { - return nil, errors.Wrapf(err, "while configuring datasource of type %s from %s (position: %d)", sub.Source, acquisFile, idx) + return nil, fmt.Errorf("while configuring datasource of type %s from %s (position: %d): %w", sub.Source, acquisFile, idx, err) } sources = append(sources, *src) idx += 1 @@ -232,9 +202,9 @@ func GetMetrics(sources []DataSource, aggregated bool) error { for _, metric := range metrics { if err := prometheus.Register(metric); err != nil { if _, ok := err.(prometheus.AlreadyRegisteredError); !ok { - return errors.Wrapf(err, "could not register metrics for datasource %s", sources[i].GetName()) + return fmt.Errorf("could not register metrics for datasource %s: %w", sources[i].GetName(), err) } - //ignore the error + // ignore the error } } diff --git a/pkg/acquisition/acquisition_test.go b/pkg/acquisition/acquisition_test.go index a547970a862..99d7e5ae9b5 100644 --- a/pkg/acquisition/acquisition_test.go +++ b/pkg/acquisition/acquisition_test.go @@ -6,7 +6,6 @@ import ( "testing" "time" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" @@ -26,10 +25,19 @@ type MockSource struct { logger *log.Entry } +func (f *MockSource) UnmarshalConfig(cfg []byte) error { + err := yaml.UnmarshalStrict(cfg, &f) + if err != nil { + return err + } + + return nil +} + func (f *MockSource) Configure(cfg []byte, logger *log.Entry) error { f.logger = logger - if err := yaml.UnmarshalStrict(cfg, &f); err != nil { - return errors.Wrap(err, "while unmarshaling to reader specific config") + if err := f.UnmarshalConfig(cfg); err != nil { + return err } if f.Mode == "" { f.Mode = configuration.CAT_MODE @@ -65,24 +73,10 @@ func (f *MockSourceCantRun) GetName() string { return "mock_cant_run" } // appendMockSource is only used to add mock source for tests func appendMockSource() { if GetDataSourceIface("mock") == nil { - mock := struct { - name string - iface func() DataSource - }{ - name: "mock", - iface: func() DataSource { return &MockSource{} }, - } - AcquisitionSources = append(AcquisitionSources, mock) + AcquisitionSources["mock"] = func() DataSource { return &MockSource{} } } if GetDataSourceIface("mock_cant_run") == nil { - mock := struct { - name string - iface func() DataSource - }{ - name: "mock_cant_run", - iface: func() DataSource { return &MockSourceCantRun{} }, - } - AcquisitionSources = append(AcquisitionSources, mock) + AcquisitionSources["mock_cant_run"] = func() DataSource { return &MockSourceCantRun{} } } } @@ -313,6 +307,8 @@ func (f *MockCat) Configure(cfg []byte, logger *log.Entry) error { } return nil } + +func (f *MockCat) UnmarshalConfig(cfg []byte) error { return nil } func (f *MockCat) GetName() string { return "mock_cat" } func (f *MockCat) GetMode() string { return "cat" } func (f *MockCat) OneShotAcquisition(out chan types.Event, tomb *tomb.Tomb) error { @@ -351,6 +347,8 @@ func (f *MockTail) Configure(cfg []byte, logger *log.Entry) error { } return nil } + +func (f *MockTail) UnmarshalConfig(cfg []byte) error { return nil } func (f *MockTail) GetName() string { return "mock_tail" } func (f *MockTail) GetMode() string { return "tail" } func (f *MockTail) OneShotAcquisition(out chan types.Event, tomb *tomb.Tomb) error { @@ -482,6 +480,7 @@ type MockSourceByDSN struct { logger *log.Entry //nolint: unused } +func (f *MockSourceByDSN) UnmarshalConfig(cfg []byte) error { return nil } func (f *MockSourceByDSN) Configure(cfg []byte, logger *log.Entry) error { return nil } func (f *MockSourceByDSN) GetMode() string { return f.Mode } func (f *MockSourceByDSN) OneShotAcquisition(chan types.Event, *tomb.Tomb) error { return nil } @@ -524,14 +523,7 @@ func TestConfigureByDSN(t *testing.T) { } if GetDataSourceIface("mockdsn") == nil { - mock := struct { - name string - iface func() DataSource - }{ - name: "mockdsn", - iface: func() DataSource { return &MockSourceByDSN{} }, - } - AcquisitionSources = append(AcquisitionSources, mock) + AcquisitionSources["mockdsn"] = func() DataSource { return &MockSourceByDSN{} } } for _, tc := range tests { diff --git a/pkg/acquisition/modules/cloudwatch/cloudwatch.go b/pkg/acquisition/modules/cloudwatch/cloudwatch.go index 9a3a66d5e41..77375f5a287 100644 --- a/pkg/acquisition/modules/cloudwatch/cloudwatch.go +++ b/pkg/acquisition/modules/cloudwatch/cloudwatch.go @@ -101,61 +101,85 @@ var ( def_AwsConfigDir = "" ) -func (cw *CloudwatchSource) Configure(cfg []byte, logger *log.Entry) error { - cwConfig := CloudwatchSourceConfiguration{} - targetStream := "*" - if err := yaml.UnmarshalStrict(cfg, &cwConfig); err != nil { - return errors.Wrap(err, "Cannot parse CloudwatchSource configuration") +func (cw *CloudwatchSource) UnmarshalConfig(yamlConfig []byte) error { + cw.Config = CloudwatchSourceConfiguration{} + if err := yaml.UnmarshalStrict(yamlConfig, &cw.Config); err != nil { + return fmt.Errorf("cannot parse CloudwatchSource configuration: %w", err) } - cw.Config = cwConfig + if len(cw.Config.GroupName) == 0 { return fmt.Errorf("group_name is mandatory for CloudwatchSource") } - cw.logger = logger.WithField("group", cw.Config.GroupName) + if cw.Config.Mode == "" { cw.Config.Mode = configuration.TAIL_MODE } - logger.Debugf("Starting configuration for Cloudwatch group %s", cw.Config.GroupName) if cw.Config.DescribeLogStreamsLimit == nil { cw.Config.DescribeLogStreamsLimit = &def_DescribeLogStreamsLimit } - logger.Tracef("describelogstreams_limit set to %d", *cw.Config.DescribeLogStreamsLimit) + if cw.Config.PollNewStreamInterval == nil { cw.Config.PollNewStreamInterval = &def_PollNewStreamInterval } - logger.Tracef("poll_new_stream_interval set to %v", *cw.Config.PollNewStreamInterval) + if cw.Config.MaxStreamAge == nil { cw.Config.MaxStreamAge = &def_MaxStreamAge } - logger.Tracef("max_stream_age set to %v", *cw.Config.MaxStreamAge) + if cw.Config.PollStreamInterval == nil { cw.Config.PollStreamInterval = &def_PollStreamInterval } - logger.Tracef("poll_stream_interval set to %v", *cw.Config.PollStreamInterval) + if cw.Config.StreamReadTimeout == nil { cw.Config.StreamReadTimeout = &def_StreamReadTimeout } - logger.Tracef("stream_read_timeout set to %v", *cw.Config.StreamReadTimeout) + if cw.Config.GetLogEventsPagesLimit == nil { cw.Config.GetLogEventsPagesLimit = &def_GetLogEventsPagesLimit } - logger.Tracef("getlogeventspages_limit set to %v", *cw.Config.GetLogEventsPagesLimit) + if cw.Config.AwsApiCallTimeout == nil { cw.Config.AwsApiCallTimeout = &def_AwsApiCallTimeout } - logger.Tracef("aws_api_timeout set to %v", *cw.Config.AwsApiCallTimeout) - if *cw.Config.MaxStreamAge > *cw.Config.StreamReadTimeout { - logger.Warningf("max_stream_age > stream_read_timeout, stream might keep being opened/closed") - } + if cw.Config.AwsConfigDir == nil { cw.Config.AwsConfigDir = &def_AwsConfigDir } - logger.Tracef("aws_config_dir set to %s", *cw.Config.AwsConfigDir) + + return nil +} + + + + +func (cw *CloudwatchSource) Configure(yamlConfig []byte, logger *log.Entry) error { + err := cw.UnmarshalConfig(yamlConfig) + if err != nil { + return err + } + + cw.logger = logger.WithField("group", cw.Config.GroupName) + + // XXX the following must be logger or cw.logger? + cw.logger.Debugf("Starting configuration for Cloudwatch group %s", cw.Config.GroupName) + cw.logger.Tracef("describelogstreams_limit set to %d", *cw.Config.DescribeLogStreamsLimit) + cw.logger.Tracef("poll_new_stream_interval set to %v", *cw.Config.PollNewStreamInterval) + cw.logger.Tracef("max_stream_age set to %v", *cw.Config.MaxStreamAge) + cw.logger.Tracef("poll_stream_interval set to %v", *cw.Config.PollStreamInterval) + cw.logger.Tracef("stream_read_timeout set to %v", *cw.Config.StreamReadTimeout) + cw.logger.Tracef("getlogeventspages_limit set to %v", *cw.Config.GetLogEventsPagesLimit) + cw.logger.Tracef("aws_api_timeout set to %v", *cw.Config.AwsApiCallTimeout) + + if *cw.Config.MaxStreamAge > *cw.Config.StreamReadTimeout { + cw.logger.Warningf("max_stream_age > stream_read_timeout, stream might keep being opened/closed") + } + cw.logger.Tracef("aws_config_dir set to %s", *cw.Config.AwsConfigDir) + if *cw.Config.AwsConfigDir != "" { _, err := os.Stat(*cw.Config.AwsConfigDir) if err != nil { - logger.Errorf("can't read aws_config_dir '%s' got err %s", *cw.Config.AwsConfigDir, err) + cw.logger.Errorf("can't read aws_config_dir '%s' got err %s", *cw.Config.AwsConfigDir, err) return fmt.Errorf("can't read aws_config_dir %s got err %s ", *cw.Config.AwsConfigDir, err) } os.Setenv("AWS_SDK_LOAD_CONFIG", "1") @@ -164,7 +188,7 @@ func (cw *CloudwatchSource) Configure(cfg []byte, logger *log.Entry) error { os.Setenv("AWS_SHARED_CREDENTIALS_FILE", fmt.Sprintf("%s/credentials", *cw.Config.AwsConfigDir)) } else { if cw.Config.AwsRegion == nil { - logger.Errorf("aws_region is not specified, specify it or aws_config_dir") + cw.logger.Errorf("aws_region is not specified, specify it or aws_config_dir") return fmt.Errorf("aws_region is not specified, specify it or aws_config_dir") } os.Setenv("AWS_REGION", *cw.Config.AwsRegion) @@ -174,6 +198,8 @@ func (cw *CloudwatchSource) Configure(cfg []byte, logger *log.Entry) error { return err } cw.streamIndexes = make(map[string]string) + + targetStream := "*" if cw.Config.StreamRegexp != nil { if _, err := regexp.Compile(*cw.Config.StreamRegexp); err != nil { return errors.Wrapf(err, "error while compiling regexp '%s'", *cw.Config.StreamRegexp) @@ -183,7 +209,7 @@ func (cw *CloudwatchSource) Configure(cfg []byte, logger *log.Entry) error { targetStream = *cw.Config.StreamName } - logger.Infof("Adding cloudwatch group '%s' (stream:%s) to datasources", cw.Config.GroupName, targetStream) + cw.logger.Infof("Adding cloudwatch group '%s' (stream:%s) to datasources", cw.Config.GroupName, targetStream) return nil } diff --git a/pkg/acquisition/modules/docker/docker.go b/pkg/acquisition/modules/docker/docker.go index 117eadda229..3687731ddf9 100644 --- a/pkg/acquisition/modules/docker/docker.go +++ b/pkg/acquisition/modules/docker/docker.go @@ -67,24 +67,22 @@ type ContainerConfig struct { Tty bool } -func (d *DockerSource) Configure(Config []byte, logger *log.Entry) error { - var err error - +func (d *DockerSource) UnmarshalConfig(yamlConfig []byte) error { d.Config = DockerConfiguration{ FollowStdout: true, // default FollowStdErr: true, // default CheckInterval: "1s", // default } - d.logger = logger - - d.runningContainerState = make(map[string]*ContainerConfig) - err = yaml.UnmarshalStrict(Config, &d.Config) + err := yaml.UnmarshalStrict(yamlConfig, &d.Config) if err != nil { return errors.Wrap(err, "Cannot parse DockerAcquisition configuration") } - d.logger.Tracef("DockerAcquisition configuration: %+v", d.Config) + if d.logger != nil { + d.logger.Tracef("DockerAcquisition configuration: %+v", d.Config) + } + if len(d.Config.ContainerName) == 0 && len(d.Config.ContainerID) == 0 && len(d.Config.ContainerIDRegexp) == 0 && len(d.Config.ContainerNameRegexp) == 0 { return fmt.Errorf("no containers names or containers ID configuration provided") } @@ -100,7 +98,6 @@ func (d *DockerSource) Configure(Config []byte, logger *log.Entry) error { if d.Config.Mode != configuration.CAT_MODE && d.Config.Mode != configuration.TAIL_MODE { return fmt.Errorf("unsupported mode %s for docker datasource", d.Config.Mode) } - d.logger.Tracef("Actual DockerAcquisition configuration %+v", d.Config) for _, cont := range d.Config.ContainerNameRegexp { d.compiledContainerName = append(d.compiledContainerName, regexp.MustCompile(cont)) @@ -110,11 +107,6 @@ func (d *DockerSource) Configure(Config []byte, logger *log.Entry) error { d.compiledContainerID = append(d.compiledContainerID, regexp.MustCompile(cont)) } - dockerClient, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) - if err != nil { - return err - } - if d.Config.Since == "" { d.Config.Since = time.Now().UTC().Format(time.RFC3339) } @@ -130,17 +122,37 @@ func (d *DockerSource) Configure(Config []byte, logger *log.Entry) error { d.containerLogsOptions.Until = d.Config.Until } + return nil +} + +func (d *DockerSource) Configure(yamlConfig []byte, logger *log.Entry) error { + d.logger = logger + + err := d.UnmarshalConfig(yamlConfig) + if err != nil { + return err + } + + d.runningContainerState = make(map[string]*ContainerConfig) + + d.logger.Tracef("Actual DockerAcquisition configuration %+v", d.Config) + + dockerClient, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + if err != nil { + return err + } + if d.Config.DockerHost != "" { - if err := client.WithHost(d.Config.DockerHost)(dockerClient); err != nil { + err = client.WithHost(d.Config.DockerHost)(dockerClient) + if err != nil { return err } } d.Client = dockerClient _, err = d.Client.Info(context.Background()) - if err != nil { - return errors.Wrapf(err, "failed to configure docker datasource %s", d.Config.DockerHost) + return fmt.Errorf("failed to configure docker datasource %s: %w", d.Config.DockerHost, err) } return nil diff --git a/pkg/acquisition/modules/file/file.go b/pkg/acquisition/modules/file/file.go index b1b29c93056..7bf72f04555 100644 --- a/pkg/acquisition/modules/file/file.go +++ b/pkg/acquisition/modules/file/file.go @@ -50,41 +50,62 @@ type FileSource struct { exclude_regexps []*regexp.Regexp } -func (f *FileSource) Configure(Config []byte, logger *log.Entry) error { - fileConfig := FileConfiguration{} - f.logger = logger - f.watchedDirectories = make(map[string]bool) - f.tails = make(map[string]bool) - err := yaml.UnmarshalStrict(Config, &fileConfig) +func (f *FileSource) UnmarshalConfig(yamlConfig []byte) error { + f.config = FileConfiguration{} + err := yaml.UnmarshalStrict(yamlConfig, &f.config) if err != nil { - return errors.Wrap(err, "Cannot parse FileAcquisition configuration") + return fmt.Errorf("cannot parse FileAcquisition configuration: %w", err) } - f.logger.Tracef("FileAcquisition configuration: %+v", fileConfig) - if len(fileConfig.Filename) != 0 { - fileConfig.Filenames = append(fileConfig.Filenames, fileConfig.Filename) + + if f.logger != nil { + f.logger.Tracef("FileAcquisition configuration: %+v", f.config) } - if len(fileConfig.Filenames) == 0 { + + if len(f.config.Filename) != 0 { + f.config.Filenames = append(f.config.Filenames, f.config.Filename) + } + + if len(f.config.Filenames) == 0 { return fmt.Errorf("no filename or filenames configuration provided") } - f.config = fileConfig + if f.config.Mode == "" { f.config.Mode = configuration.TAIL_MODE } + if f.config.Mode != configuration.CAT_MODE && f.config.Mode != configuration.TAIL_MODE { return fmt.Errorf("unsupported mode %s for file source", f.config.Mode) } - f.watcher, err = fsnotify.NewWatcher() - if err != nil { - return errors.Wrapf(err, "Could not create fsnotify watcher") - } + for _, exclude := range f.config.ExcludeRegexps { re, err := regexp.Compile(exclude) if err != nil { - return errors.Wrapf(err, "Could not compile regexp %s", exclude) + return fmt.Errorf("could not compile regexp %s: %w", exclude, err) } f.exclude_regexps = append(f.exclude_regexps, re) } + + return nil +} + +func (f *FileSource) Configure(yamlConfig []byte, logger *log.Entry) error { + f.logger = logger + + err := f.UnmarshalConfig(yamlConfig) + if err != nil { + return err + } + + f.watchedDirectories = make(map[string]bool) + f.tails = make(map[string]bool) + + f.watcher, err = fsnotify.NewWatcher() + if err != nil { + return errors.Wrapf(err, "Could not create fsnotify watcher") + } + f.logger.Tracef("Actual FileAcquisition Configuration %+v", f.config) + for _, pattern := range f.config.Filenames { if f.config.ForceInotify { directory := filepath.Dir(pattern) diff --git a/pkg/acquisition/modules/file/file_test.go b/pkg/acquisition/modules/file/file_test.go index 06653fbe1a7..d01a2246821 100644 --- a/pkg/acquisition/modules/file/file_test.go +++ b/pkg/acquisition/modules/file/file_test.go @@ -42,7 +42,7 @@ func TestBadConfiguration(t *testing.T) { name: "bad exclude regexp", config: `filenames: ["asd.log"] exclude_regexps: ["as[a-$d"]`, - expectedErr: "Could not compile regexp as", + expectedErr: "could not compile regexp as", }, } diff --git a/pkg/acquisition/modules/journalctl/journalctl.go b/pkg/acquisition/modules/journalctl/journalctl.go index 6745637533b..72f8c7330f7 100644 --- a/pkg/acquisition/modules/journalctl/journalctl.go +++ b/pkg/acquisition/modules/journalctl/journalctl.go @@ -163,28 +163,42 @@ func (j *JournalCtlSource) GetAggregMetrics() []prometheus.Collector { return []prometheus.Collector{linesRead} } -func (j *JournalCtlSource) Configure(yamlConfig []byte, logger *log.Entry) error { - config := JournalCtlConfiguration{} - j.logger = logger - err := yaml.UnmarshalStrict(yamlConfig, &config) +func (j *JournalCtlSource) UnmarshalConfig(yamlConfig []byte) error { + j.config = JournalCtlConfiguration{} + err := yaml.UnmarshalStrict(yamlConfig, &j.config) if err != nil { - return errors.Wrap(err, "Cannot parse JournalCtlSource configuration") + return fmt.Errorf("cannot parse JournalCtlSource configuration: %w", err) } - if config.Mode == "" { - config.Mode = configuration.TAIL_MODE + + if j.config.Mode == "" { + j.config.Mode = configuration.TAIL_MODE } + var args []string - if config.Mode == configuration.TAIL_MODE { + if j.config.Mode == configuration.TAIL_MODE { args = journalctlArgstreaming } else { args = journalctlArgsOneShot } - if len(config.Filters) == 0 { + + if len(j.config.Filters) == 0 { return fmt.Errorf("journalctl_filter is required") } - j.args = append(args, config.Filters...) - j.src = fmt.Sprintf("journalctl-%s", strings.Join(config.Filters, ".")) - j.config = config + j.args = append(args, j.config.Filters...) + j.src = fmt.Sprintf("journalctl-%s", strings.Join(j.config.Filters, ".")) + + return nil +} + + +func (j *JournalCtlSource) Configure(yamlConfig []byte, logger *log.Entry) error { + j.logger = logger + + err := j.UnmarshalConfig(yamlConfig) + if err != nil { + return err + } + return nil } diff --git a/pkg/acquisition/modules/kafka/kafka.go b/pkg/acquisition/modules/kafka/kafka.go index b8b8937e54b..6f28917849f 100644 --- a/pkg/acquisition/modules/kafka/kafka.go +++ b/pkg/acquisition/modules/kafka/kafka.go @@ -54,35 +54,53 @@ type KafkaSource struct { Reader *kafka.Reader } -func (k *KafkaSource) Configure(Config []byte, logger *log.Entry) error { - var err error - +func (k* KafkaSource) UnmarshalConfig(yamlConfig []byte) error { k.Config = KafkaConfiguration{} - k.logger = logger - err = yaml.UnmarshalStrict(Config, &k.Config) + + err := yaml.UnmarshalStrict(yamlConfig, &k.Config) if err != nil { - return errors.Wrapf(err, "cannot parse %s datasource configuration", dataSourceName) + return fmt.Errorf("cannot parse %s datasource configuration: %w", dataSourceName, err) } + if len(k.Config.Brokers) == 0 { return fmt.Errorf("cannot create a %s reader with an empty list of broker addresses", dataSourceName) } + if k.Config.Topic == "" { return fmt.Errorf("cannot create a %s reader with am empty topic", dataSourceName) } + if k.Config.Mode == "" { k.Config.Mode = configuration.TAIL_MODE } + + return err +} + + + +func (k *KafkaSource) Configure(yamlConfig []byte, logger *log.Entry) error { + k.logger = logger + + err := k.UnmarshalConfig(yamlConfig) + if err != nil { + return err + } + dialer, err := k.Config.NewDialer() if err != nil { return errors.Wrapf(err, "cannot create %s dialer", dataSourceName) } + k.Reader, err = k.Config.NewReader(dialer) if err != nil { return errors.Wrapf(err, "cannote create %s reader", dataSourceName) } + if k.Reader == nil { return fmt.Errorf("cannot create %s reader", dataSourceName) } + return nil } diff --git a/pkg/acquisition/modules/kinesis/kinesis.go b/pkg/acquisition/modules/kinesis/kinesis.go index cce5ffbfce0..0445558adab 100644 --- a/pkg/acquisition/modules/kinesis/kinesis.go +++ b/pkg/acquisition/modules/kinesis/kinesis.go @@ -113,17 +113,18 @@ func (k *KinesisSource) GetAggregMetrics() []prometheus.Collector { return []prometheus.Collector{linesRead, linesReadShards} } -func (k *KinesisSource) Configure(yamlConfig []byte, logger *log.Entry) error { - config := KinesisConfiguration{} - k.logger = logger - err := yaml.UnmarshalStrict(yamlConfig, &config) +func (k *KinesisSource) UnmarshalConfig(yamlConfig []byte) error { + k.Config = KinesisConfiguration{} + + err := yaml.UnmarshalStrict(yamlConfig, &k.Config) if err != nil { return errors.Wrap(err, "Cannot parse kinesis datasource configuration") } - if config.Mode == "" { - config.Mode = configuration.TAIL_MODE + + if k.Config.Mode == "" { + k.Config.Mode = configuration.TAIL_MODE } - k.Config = config + if k.Config.StreamName == "" && !k.Config.UseEnhancedFanOut { return fmt.Errorf("stream_name is mandatory when use_enhanced_fanout is false") } @@ -139,10 +140,23 @@ func (k *KinesisSource) Configure(yamlConfig []byte, logger *log.Entry) error { if k.Config.MaxRetries <= 0 { k.Config.MaxRetries = 10 } + + return nil +} + +func (k *KinesisSource) Configure(yamlConfig []byte, logger *log.Entry) error { + k.logger = logger + + err := k.UnmarshalConfig(yamlConfig) + if err != nil { + return err + } + err = k.newClient() if err != nil { - return errors.Wrap(err, "Cannot create kinesis client") + return fmt.Errorf("cannot create kinesis client: %w", err) } + k.shardReaderTomb = &tomb.Tomb{} return nil } diff --git a/pkg/acquisition/modules/syslog/syslog.go b/pkg/acquisition/modules/syslog/syslog.go index 2cd0083b738..c50b05b4caf 100644 --- a/pkg/acquisition/modules/syslog/syslog.go +++ b/pkg/acquisition/modules/syslog/syslog.go @@ -89,31 +89,44 @@ func validateAddr(addr string) bool { return net.ParseIP(addr) != nil } -func (s *SyslogSource) Configure(yamlConfig []byte, logger *log.Entry) error { - s.logger = logger - s.logger.Infof("Starting syslog datasource configuration") - syslogConfig := SyslogConfiguration{} - syslogConfig.Mode = configuration.TAIL_MODE - err := yaml.UnmarshalStrict(yamlConfig, &syslogConfig) +func (s *SyslogSource) UnmarshalConfig(yamlConfig []byte) error { + s.config = SyslogConfiguration{} + s.config.Mode = configuration.TAIL_MODE + + err := yaml.UnmarshalStrict(yamlConfig, &s.config) if err != nil { return errors.Wrap(err, "Cannot parse syslog configuration") } - if syslogConfig.Addr == "" { - syslogConfig.Addr = "127.0.0.1" //do we want a usable or secure default ? + + if s.config.Addr == "" { + s.config.Addr = "127.0.0.1" //do we want a usable or secure default ? } - if syslogConfig.Port == 0 { - syslogConfig.Port = 514 + if s.config.Port == 0 { + s.config.Port = 514 } - if syslogConfig.MaxMessageLen == 0 { - syslogConfig.MaxMessageLen = 2048 + if s.config.MaxMessageLen == 0 { + s.config.MaxMessageLen = 2048 } - if !validatePort(syslogConfig.Port) { - return fmt.Errorf("invalid port %d", syslogConfig.Port) + if !validatePort(s.config.Port) { + return fmt.Errorf("invalid port %d", s.config.Port) } - if !validateAddr(syslogConfig.Addr) { - return fmt.Errorf("invalid listen IP %s", syslogConfig.Addr) + if !validateAddr(s.config.Addr) { + return fmt.Errorf("invalid listen IP %s", s.config.Addr) } - s.config = syslogConfig + + return nil +} + + +func (s *SyslogSource) Configure(yamlConfig []byte, logger *log.Entry) error { + s.logger = logger + s.logger.Infof("Starting syslog datasource configuration") + + err := s.UnmarshalConfig(yamlConfig) + if err != nil { + return err + } + return nil } diff --git a/pkg/acquisition/modules/wineventlog/wineventlog.go b/pkg/acquisition/modules/wineventlog/wineventlog.go index 92bbd7be4a2..5d3f3907e73 100644 --- a/pkg/acquisition/modules/wineventlog/wineventlog.go +++ b/pkg/acquisition/modules/wineventlog/wineventlog.go @@ -14,6 +14,10 @@ import ( type WinEventLogSource struct{} +func (w *WinEventLogSource) UnmarshalConfig(yamlConfig []byte) error { + return nil +} + func (w *WinEventLogSource) Configure(yamlConfig []byte, logger *log.Entry) error { return nil } diff --git a/pkg/acquisition/modules/wineventlog/wineventlog_windows.go b/pkg/acquisition/modules/wineventlog/wineventlog_windows.go index 7e7bb577897..77876555a37 100644 --- a/pkg/acquisition/modules/wineventlog/wineventlog_windows.go +++ b/pkg/acquisition/modules/wineventlog/wineventlog_windows.go @@ -228,29 +228,26 @@ func (w *WinEventLogSource) generateConfig(query string) (*winlog.SubscribeConfi return &config, nil } -func (w *WinEventLogSource) Configure(yamlConfig []byte, logger *log.Entry) error { - - config := WinEventLogConfiguration{} - w.logger = logger - err := yaml.UnmarshalStrict(yamlConfig, &config) +func (w *WinEventLogSource) UnmarshalConfig(yamlConfig []byte) error { + w.config = WinEventLogConfiguration{} + err := yaml.UnmarshalStrict(yamlConfig, &w.config) if err != nil { return fmt.Errorf("unable to parse configuration: %v", err) } - if config.EventChannel != "" && config.XPathQuery != "" { + if w.config.EventChannel != "" && w.config.XPathQuery != "" { return fmt.Errorf("event_channel and xpath_query are mutually exclusive") } - if config.EventChannel == "" && config.XPathQuery == "" { + if w.config.EventChannel == "" && w.config.XPathQuery == "" { return fmt.Errorf("event_channel or xpath_query must be set") } - config.Mode = configuration.TAIL_MODE - w.config = config + w.config.Mode = configuration.TAIL_MODE if config.XPathQuery != "" { - w.query = config.XPathQuery + w.query = w.config.XPathQuery } else { w.query, err = w.buildXpathQuery() if err != nil { @@ -258,15 +255,26 @@ func (w *WinEventLogSource) Configure(yamlConfig []byte, logger *log.Entry) erro } } - w.evtConfig, err = w.generateConfig(w.query) + if w.config.PrettyName != "" { + w.name = w.config.PrettyName + } else { + w.name = w.query + } + + return nil +} + +func (w *WinEventLogSource) Configure(yamlConfig []byte, logger *log.Entry) error { + w.logger = logger + + err := w.UnmarshalConfig(yamlConfig) if err != nil { return err } - if config.PrettyName != "" { - w.name = config.PrettyName - } else { - w.name = w.query + w.evtConfig, err = w.generateConfig(w.query) + if err != nil { + return err } return nil diff --git a/pkg/cstest/utils.go b/pkg/cstest/utils.go index 2c26be89e32..fb8300094a3 100644 --- a/pkg/cstest/utils.go +++ b/pkg/cstest/utils.go @@ -1,7 +1,9 @@ package cstest import ( + "strings" "testing" + "text/template" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -28,3 +30,20 @@ func RequireErrorContains(t *testing.T, err error, expectedErr string) { require.NoError(t, err) } + +// Interpolate fills a string template with the given values, can be map or struct. +// example: Interpolate("{{.Name}}", map[string]string{"Name": "JohnDoe"}) +func Interpolate(s string, data interface{}) (string, error) { + tmpl, err := template.New("").Parse(s) + if err != nil { + return "", err + } + + var b strings.Builder + err = tmpl.Execute(&b, data) + if err != nil { + return "", err + } + + return b.String(), nil +} diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index f48dc223f16..757d511358f 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -134,6 +134,7 @@ func GetItemMap(itemType string) map[string]Item { var m map[string]Item var ok bool + log.Tracef("hubIdx: %v", hubIdx) if m, ok = hubIdx[itemType]; !ok { return nil } @@ -178,6 +179,7 @@ func GetItemByPath(itemType string, itemPath string) (*Item, error) { } func GetItem(itemType string, itemName string) *Item { + log.Tracef("getting hub item %s: %s", itemType, itemName) if m, ok := GetItemMap(itemType)[itemName]; ok { return &m } diff --git a/pkg/setup/README.md b/pkg/setup/README.md new file mode 100644 index 00000000000..bd04d198a85 --- /dev/null +++ b/pkg/setup/README.md @@ -0,0 +1,338 @@ + +--- + +# cscli setup + +The "cscli setup" command can configure a crowdsec instance based on the services that are installed or running on the server. + +There are three main subcommands: + +- `cscli setup detect`: *detect* the services, the OS family, version or the Linux distribution +- `cscli setup install-hub`: *install* the recommended collections, parsers, etc. based on the detection result +- `cscli setup datasources`: *generate* the appropriate acquisition rules + +The setup command is used in the `wizard.sh` script, but can also be invoked by hand or customized via a configuration file +by adding new services, log locations and detection rules. + +Detection and installation are performed as separate steps, as you can see in the following diagram: + +``` + +-------------+ + | | + | detect.yaml | + | | + +-------------+ + | + v + setup detect + | + v + +--------------+ + | +---> setup install-hub +-----------------------+ + | setup.yaml | | | + | +---> setup datasources --->| etc/crowdsec/acquis.d | + +--------------+ | | + +-----------------------+ +``` + +You can inspect and customize the intermediary file (`setup.yaml`), which is useful +in case of many instances, deployment automation or unusual setups. + +A subcommand can be used to check your changes in this case: + +- `cscli setup validate`: *validate* or report errors on a setup file + +## Basic usage + +Identify the existing services and write out what was detected: + +```console +# cscli setup detect > setup.yaml +``` + +See what was found. + +```console +# cscli setup install-hub setup.yaml --dry-run +dry-run: would install collection crowdsecurity/apache2 +dry-run: would install collection crowdsecurity/linux +dry-run: would install collection crowdsecurity/pgsql +dry-run: would install parser crowdsecurity/whitelists +``` + +Install the objects (parsers, scenarios...) required to support the detected services: + +```console +# cscli setup install-hub setup.yaml +INFO[29-06-2022 03:16:14 PM] crowdsecurity/apache2-logs : OK +INFO[29-06-2022 03:16:14 PM] Enabled parsers : crowdsecurity/apache2-logs +INFO[29-06-2022 03:16:14 PM] crowdsecurity/http-logs : OK +[...] +INFO[29-06-2022 03:16:18 PM] Enabled crowdsecurity/linux +``` + +Generate the datasource configuration: + +```console +# cscli setup datasources setup.yaml --to-dir /etc/crowdsec/acquis.d +``` + +With the above command, each detected service gets a corresponding file in the +`acquis.d` directory. Running `cscli setup` again may add more services as they +are detected, but datasource files or hub items are never removed +automatically. + + +## The detect.yaml file + +A detect.yaml file is downloaded when you first install crowdsec, and is updated by the `cscli hub update` +command. + +> **_NOTE_**: XXX XXX - this is currently not the case, the file is distributed in the crowdsec repository, but it should change. + +You can see the default location with `cscli setup detect --help | grep detect-config` + +The YAML file contains a version number (always 1.0) and a list of sections, one per supported service. + +Each service defines its detection rules, the recommended hub items and +recommended datasources. The same software can be defined in multiple service +sections: for example, apache on debian and fedora have different detection +rules and different datasources so it requires two sections to support both platforms. + +The following are minimal `detect.yaml` examples just to show a few concepts. + +```yaml +version: 1.0 + +services: + + apache2: + when: + - ProcessRunning("apache2") + install: + collections: + - crowdsecurity/apache2 + datasources: + source: file + labels: + type: apache2 + filenames: + - /var/log/apache2/*.log + - /var/log/httpd/*.log +``` + + +- `ProcessRunning()` matches the process name of a running application. The +`when:` clause can contain any number of expressions, they are all evaluated +and must all return true for a service to be detected (implied *and* clause, no +short-circuit). A missing or empty `when:` section is evaluated as true. +The [expression +engine](https://github.com/antonmedv/expr/blob/master/docs/Language-Definition.md) +is the same one used by CrowdSec parser filters. You can force the detection of +a process by using the `cscli setup detect... --force-process ` +flag. It will always behave as if `` was running. + +The `install:` section can contain any number of collections, parsers, scenarios +and postoverflows. In practices, it's most often a single collection. + +The `datasource:` section is copied as-is in the acquisition file. + +> **_NOTE_**: XXX TODO - the current version does not validate the `datasource:` mapping. Bad content is written to acquis.d until crowdsec chokes on it. + +Detecting a running process may seem a good idea, but if a process manager like +systemd is available it's better to ask it for the information we want. + + +```yaml +version: 1.0 + +services: + + apache2-systemd: + when: + - UnitFound("apache2.service") + - OS.ID != "centos" + install: + collections: + - crowdsecurity/apache2 + datasource: + source: file + labels: + type: syslog + filenames: + - /var/log/apache2/*.log + + apache2-systemd-centos: + when: + - UnitFound("httpd.service") + - OS.ID == "centos" + install: + collections: + - crowdsecurity/apache2 + datasource: + source: file + labels: + type: syslog + filenames: + - /var/log/httpd/*.log +``` + +Here we see two more detection methods: + +- `UnitFound()` matches the name of systemd units, if the are in state enabled, + generated or static. You can see here that CentOS is using a different unit + name for Apache so it must have its own service section. You can force the + detection of a unit by using the `cscli setup detect... --force-unit ` flag. + +- OS.Family, OS.ID and OS.RawVersion are read from /etc/os-release in case of + Linux, and detected by other methods for FreeBSD and Windows. Under FreeBSD + and Windows, the value of OS.ID is the same as OS.Family. If OS detection + fails, it can be overridden with the flags `--force-os-family`, `--force-os-id` + and `--force-os-version`. + +If you want to ignore one or more services (i.e. not install anything and not +generate acquisition rules) you can specify it with `cscli setup detect... +--skip-service `. For example, `--skip-service apache2-systemd`. +If you want to disable systemd unit detection, use `cscli setup detect... --snub-systemd`. + +If you used the `--force-process` or `--force-unit` flags, but none of the +defined services is looking for them, you'll have an error like "detecting +services: process(es) forced but not supported". + +> **_NOTE_**: XXX XXX - having an error for this is maybe too much, but can tell that a configuration is outdated. Could this be a warning with optional flag to make it an error? + +We used the `OS.ID` value to check for the linux distribution, but since the same configuration +is required for CentOS and the other RedHat derivatives, it's better to check for the existence +of a file that is known to exist in all of them: + +```yaml +version: 1.0 + +services: + + apache2-systemd-deb: + when: + - UnitFound("apache2.service") + - PathExists("/etc/debian_version") + install: + # [...] + + apache2-systemd-rpm: + when: + - UnitFound("httpd.service") + - PathExists("/etc/redhat-release") + install: + # [...] +``` + +- `PathExists()` evaluates to true if a file, directory or link exists at the + given path. It does not check for broken links. + + + +Rules can be used to detect operating systems and environments: + +```yaml +version: 1.0 + +services: + + linux: + when: + - OS.Family == "linux" + install: + collections: + - crowdsecurity/linux + datasource: + type: file + labels: + type: syslog + log_files: + - /var/log/syslog + - /var/log/kern.log + - /var/log/messages + + freebsd: + when: + - OS.Family == "freebsd" + install: + collections: + - crowdsecurity/freebsd + + windows: + when: + - OS.Family == "windows" + install: + collections: + - crowdsecurity/windows +``` + +The OS object contains a methods to check for version numbers: +`OS.VersionCheck("")`. It uses the +[Masterminds/semver](https://github.com/Masterminds/semver) package and accepts +a variety of operators. + +Instead of: OS.RawVersion == "1.2.3" you should use `OS.VersionCheck("~1")`, +`OS.VersionCheck("~1.2")` depending if you want to match the major or the minor +version. It's unlikely that you need to match the exact patch level. + +Leading zeroes are permitted, to allow comparison of Ubuntu versions: strict semver rules would treat "22.04" as invalid. + + +# The `setup.yaml` file + +This file does not actually have a specific name, as it's usually written to standard output. + +For example, on a Debian system running Apache under systemd you can execute: + +```console +$ cscli setup detect --yaml +setup: + - detected_service: apache2-systemd-deb + install: + collections: + - crowdsecurity/apache2 + datasource: + filenames: + - /var/log/apache2/*.log + labels: + type: apache2 + - detected_service: linux + install: + collections: + - crowdsecurity/linux + datasource: + filenames: + - /var/log/syslog + - /var/log/kern.log + - /var/log/messages + labels: + type: syslog + - detected_service: whitelists + install: + parsers: + - crowdsecurity/whitelists +``` + +The default output format is JSON, which is compatible with YAML but less readable to humans. + + - `detected_service`: used to generate a name for the files written to `acquis.d` + - `install`: can contain collections, parsers, scenarios, postoverflows + - `datasource`: copied to `acquis.d` + + +```console +$ cscli setup datasources --help +generate datasource (acquisition) configuration from a setup file + +Usage: + cscli setup datasources [setup_file] [flags] + +Flags: + -h, --help help for datasources + --to-dir string write the configuration to a directory, in multiple files +[...] +``` + +If the `--to-dir` option is not specified, a single monolithic `acquis.yaml` is printed to the standard output. + diff --git a/pkg/setup/detect.go b/pkg/setup/detect.go new file mode 100644 index 00000000000..94208ac7234 --- /dev/null +++ b/pkg/setup/detect.go @@ -0,0 +1,581 @@ +package setup + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "sort" + + "github.com/Masterminds/semver" + "github.com/antonmedv/expr" + "github.com/blackfireio/osinfo" + "github.com/shirou/gopsutil/v3/process" + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" + // goccyyaml "github.com/goccy/go-yaml" + + // "github.com/k0kubun/pp" + + "github.com/crowdsecurity/crowdsec/pkg/acquisition" + "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" +) + +// ExecCommand can be replaced with a mock during tests. +var ExecCommand = exec.Command + +// HubItems contains the objects that are recommended to support a service. +type HubItems struct { + Collections []string `yaml:"collections,omitempty"` + Parsers []string `yaml:"parsers,omitempty"` + Scenarios []string `yaml:"scenarios,omitempty"` + PostOverflows []string `yaml:"postoverflows,omitempty"` +} + +type DataSourceItem map[string]interface{} + +// ServiceSetup describes the recommendations (hub objects and datasources) for a detected service. +type ServiceSetup struct { + DetectedService string `yaml:"detected_service"` + Install *HubItems `yaml:"install,omitempty"` + DataSource DataSourceItem `yaml:"datasource,omitempty"` +} + +// Setup is a container for a list of ServiceSetup objects, allowing for future extensions. +type Setup struct { + Setup []ServiceSetup `yaml:"setup"` +} + +func validateDataSource(opaqueDS DataSourceItem) error { + if len(opaqueDS) == 0 { + // empty datasource is valid + return nil + } + + + // formally validate YAML + + commonDS := configuration.DataSourceCommonCfg{} + body, err := yaml.Marshal(opaqueDS) + if err != nil { + return err + } + + err = yaml.Unmarshal(body, &commonDS) + if err != nil { + return err + } + + // source is mandatory // XXX unless it's not? + + if commonDS.Source == "" { + return fmt.Errorf("source is empty") + } + + + // source must be known + + ds := acquisition.GetDataSourceIface(commonDS.Source) + if ds == nil { + return fmt.Errorf("unknown source '%s'", commonDS.Source) + } + + // unmarshal and validate the rest with the specific implementation + + err = ds.UnmarshalConfig(body) + if err != nil { + return err + } + + // pp.Println(ds) + return nil +} + +func readDetectConfig(file string) (DetectConfig, error) { + var dc DetectConfig + + yamlBytes, err := os.ReadFile(file) + if err != nil { + return DetectConfig{}, fmt.Errorf("while reading file: %w", err) + } + + dec := yaml.NewDecoder(bytes.NewBuffer(yamlBytes)) + dec.KnownFields(true) + + if err = dec.Decode(&dc); err != nil { + return DetectConfig{}, fmt.Errorf("while parsing %s: %w", file, err) + } + + switch dc.Version { + case "": + return DetectConfig{}, fmt.Errorf("missing version tag (must be 1.0)") + case "1.0": + // all is well + default: + return DetectConfig{}, fmt.Errorf("unsupported version tag '%s' (must be 1.0)", dc.Version) + } + + for name, svc := range dc.Detect { + err = validateDataSource(svc.DataSource) + if err != nil { + return DetectConfig{}, fmt.Errorf("invalid datasource for %s: %w", name, err) + } + } + + return dc, nil +} + +// Service describes the rules for detecting a service and its recommended items. +type Service struct { + When []string `yaml:"when"` + Install *HubItems `yaml:"install,omitempty"` + DataSource DataSourceItem `yaml:"datasource,omitempty"` + // AcquisYAML []byte +} + +// DetectConfig is the container of all detection rules (detect.yaml). +type DetectConfig struct { + Version string `yaml:"version"` + Detect map[string]Service `yaml:"detect"` +} + +// ExprState keeps a global state for the duration of the service detection (cache etc.) +type ExprState struct { + unitsSearched map[string]bool + detectOptions DetectOptions + + // cache + installedUnits map[string]bool + // true if the list of running processes has already been retrieved, we can + // avoid getting it a second time. + processesSearched map[string]bool + // cache + runningProcesses map[string]bool +} + +// ExprServiceState keep a local state during the detection of a single service. It is reset before each service rules' evaluation. +type ExprServiceState struct { + detectedUnits []string +} + +// ExprOS contains the detected (or forced) OS fields available to the rule engine. +type ExprOS struct { + Family string + ID string + RawVersion string +} + +// This is not required with Masterminds/semver +/* +// normalizeVersion strips leading zeroes from each part, to allow comparison of ubuntu-like versions. +func normalizeVersion(version string) string { + // if it doesn't match a version string, return unchanged + if ok := regexp.MustCompile(`^(\d+)(\.\d+)?(\.\d+)?$`).MatchString(version); !ok { + // definitely not an ubuntu-like version, return unchanged + return version + } + + ret := []rune{} + + var cur rune + + trim := true + for _, next := range version + "." { + if trim && cur == '0' && next != '.' { + cur = next + + continue + } + + if cur != 0 { + ret = append(ret, cur) + } + + trim = (cur == '.' || cur == 0) + cur = next + } + + return string(ret) +} +*/ + +// VersionCheck returns true if the version of the OS matches the given constraint +func (os ExprOS) VersionCheck(constraint string) (bool, error) { + v, err := semver.NewVersion(os.RawVersion) + if err != nil { + return false, err + } + + c, err := semver.NewConstraint(constraint) + if err != nil { + return false, err + } + + return c.Check(v), nil +} + +// VersionAtLeast returns true if the version of the OS is at least the given version. +func (os ExprOS) VersionAtLeast(constraint string) (bool, error) { + return os.VersionCheck(">=" + constraint) +} + +// VersionIsLower returns true if the version of the OS is lower than the given version. +func (os ExprOS) VersionIsLower(version string) (bool, error) { + result, err := os.VersionAtLeast(version) + if err != nil { + return false, err + } + + return !result, nil +} + +// ExprEnvironment is used to expose functions and values to to the rule engine. +// It can cache the results of service detection commands, like systemctl etc. +type ExprEnvironment struct { + OS ExprOS + + _serviceState *ExprServiceState + _state *ExprState +} + +// NewExprEnvironment creates an environment object for the rule engine. +func NewExprEnvironment(opts DetectOptions, os ExprOS) ExprEnvironment { + return ExprEnvironment{ + _state: &ExprState{ + detectOptions: opts, + + unitsSearched: make(map[string]bool), + installedUnits: make(map[string]bool), + + processesSearched: make(map[string]bool), + runningProcesses: make(map[string]bool), + }, + _serviceState: &ExprServiceState{}, + OS: os, + } +} + +// PathExists returns true if the given path exists. +func (e ExprEnvironment) PathExists(path string) bool { + _, err := os.Stat(path) + + return err == nil +} + +// UnitFound returns true if the unit is listed in the systemctl output. +// Whether a disabled or failed unit is considered found or not, depends on the +// systemctl parameters used. +func (e ExprEnvironment) UnitFound(unitName string) (bool, error) { + // fill initial caches + if len(e._state.unitsSearched) == 0 { + if !e._state.detectOptions.SnubSystemd { + units, err := systemdUnitList() + if err != nil { + return false, err + } + + for _, name := range units { + e._state.installedUnits[name] = true + } + } + + for _, name := range e._state.detectOptions.ForcedUnits { + e._state.installedUnits[name] = true + } + } + + e._state.unitsSearched[unitName] = true + if e._state.installedUnits[unitName] { + e._serviceState.detectedUnits = append(e._serviceState.detectedUnits, unitName) + + return true, nil + } + + return false, nil +} + +// ProcessRunning returns true if there is a running process with the given name. +func (e ExprEnvironment) ProcessRunning(processName string) (bool, error) { + if len(e._state.processesSearched) == 0 { + procs, err := process.Processes() + if err != nil { + return false, fmt.Errorf("while looking up running processes: %w", err) + } + + for _, p := range procs { + name, err := p.Name() + if err != nil { + return false, fmt.Errorf("while looking up running processes: %w", err) + } + + e._state.runningProcesses[name] = true + } + + for _, name := range e._state.detectOptions.ForcedProcesses { + e._state.runningProcesses[name] = true + } + } + + e._state.processesSearched[processName] = true + + return e._state.runningProcesses[processName], nil +} + +// applyRules checks if the 'when' expressions are true and returns a Service struct, +// augmented with default values and anything that might be useful later on +// +// All expressions are evaluated (no short-circuit) because we want to know if there are errors. +func applyRules(svc Service, env ExprEnvironment) (Service, bool, error) { + newsvc := svc + svcok := true + env._serviceState = &ExprServiceState{} + + for _, rule := range svc.When { + out, err := expr.Eval(rule, env) + log.Tracef(" Rule '%s' -> %t, %v", rule, out, err) + + if err != nil { + return Service{}, false, fmt.Errorf("rule '%s': %w", rule, err) + } + + outbool, ok := out.(bool) + if !ok { + return Service{}, false, fmt.Errorf("rule '%s': type must be a boolean", rule) + } + + svcok = svcok && outbool + } + + // if newsvc.Acquis == nil || (newsvc.Acquis.LogFiles == nil && newsvc.Acquis.JournalCTLFilter == nil) { + // for _, unitName := range env._serviceState.detectedUnits { + // if newsvc.Acquis == nil { + // newsvc.Acquis = &AcquisItem{} + // } + // // if there is reference to more than one unit in the rules, we use the first one + // newsvc.Acquis.JournalCTLFilter = []string{fmt.Sprintf(`_SYSTEMD_UNIT=%s`, unitName)} + // break //nolint // we want to exit after one iteration + // } + // } + + return newsvc, svcok, nil +} + +// filterWithRules decorates a DetectConfig map by filtering according to the when: clauses, +// and applying default values or whatever useful to the Service items. +func filterWithRules(dc DetectConfig, env ExprEnvironment) (map[string]Service, error) { + ret := make(map[string]Service) + + for name := range dc.Detect { + // + // an empty list of when: clauses defaults to true, if we want + // to change this behavior, the place is here. + // if len(svc.When) == 0 { + // log.Warningf("empty 'when' clause: %+v", svc) + // } + // + log.Trace("Evaluating rules for: ", name) + + svc, ok, err := applyRules(dc.Detect[name], env) + if err != nil { + return nil, fmt.Errorf("while looking for service %s: %w", name, err) + } + + if !ok { + log.Tracef(" Skipping %s", name) + + continue + } + + log.Tracef(" Detected %s", name) + + ret[name] = svc + } + + return ret, nil +} + +// return units that have been forced but not searched yet. +func (e ExprEnvironment) unsearchedUnits() []string { + ret := []string{} + + for _, unit := range e._state.detectOptions.ForcedUnits { + if !e._state.unitsSearched[unit] { + ret = append(ret, unit) + } + } + + return ret +} + +// return processes that have been forced but not searched yet. +func (e ExprEnvironment) unsearchedProcesses() []string { + ret := []string{} + + for _, proc := range e._state.detectOptions.ForcedProcesses { + if !e._state.processesSearched[proc] { + ret = append(ret, proc) + } + } + + return ret +} + +// checkConsumedForcedItems checks if all the "forced" options (units or processes) have been evaluated during the service detection. +func checkConsumedForcedItems(e ExprEnvironment) error { + unconsumed := e.unsearchedUnits() + + unitMsg := "" + if len(unconsumed) > 0 { + unitMsg = fmt.Sprintf("unit(s) forced but not supported: %v", unconsumed) + } + + unconsumed = e.unsearchedProcesses() + + procsMsg := "" + if len(unconsumed) > 0 { + procsMsg = fmt.Sprintf("process(es) forced but not supported: %v", unconsumed) + } + + join := "" + if unitMsg != "" && procsMsg != "" { + join = "; " + } + + if unitMsg != "" || procsMsg != "" { + return fmt.Errorf("%s%s%s", unitMsg, join, procsMsg) + } + + return nil +} + +// DetectOptions contains parameters for the Detect function. +type DetectOptions struct { + // slice of unit names that we want to force-detect + ForcedUnits []string + // slice of process names that we want to force-detect + ForcedProcesses []string + ForcedOS ExprOS + SkipServices []string + SnubSystemd bool +} + +// Detect performs the service detection from a given configuration. +// It outputs a setup file that can be used as input to "cscli setup install-hub" +// or "cscli setup datasources". +func Detect(serviceDetectionFile string, opts DetectOptions) (Setup, error) { + ret := Setup{} + + // explicitly initialize to avoid json mashaling an empty slice as "null" + ret.Setup = make([]ServiceSetup, 0) + + log.Tracef("Reading detection rules: %s", serviceDetectionFile) + + sc, err := readDetectConfig(serviceDetectionFile) + if err != nil { + return ret, err + } + + // // generate acquis.yaml snippet for this service + // for key := range sc.Detect { + // svc := sc.Detect[key] + // if svc.Acquis != nil { + // svc.AcquisYAML, err = yaml.Marshal(svc.Acquis) + // if err != nil { + // return ret, err + // } + // sc.Detect[key] = svc + // } + // } + + var osfull *osinfo.OSInfo + + os := opts.ForcedOS + if os == (ExprOS{}) { + osfull, err = osinfo.GetOSInfo() + if err != nil { + return ret, fmt.Errorf("detecting OS: %w", err) + } + + log.Tracef("Detected OS - %+v", *osfull) + + os = ExprOS{ + Family: osfull.Family, + ID: osfull.ID, + RawVersion: osfull.Version, + } + } else { + log.Tracef("Forced OS - %+v", os) + } + + if len(opts.ForcedUnits) > 0 { + log.Tracef("Forced units - %v", opts.ForcedUnits) + } + + if len(opts.ForcedProcesses) > 0 { + log.Tracef("Forced processes - %v", opts.ForcedProcesses) + } + + env := NewExprEnvironment(opts, os) + + detected, err := filterWithRules(sc, env) + if err != nil { + return ret, err + } + + if err = checkConsumedForcedItems(env); err != nil { + return ret, err + } + + // remove services the user asked to ignore + for _, name := range opts.SkipServices { + delete(detected, name) + } + + // sort the keys (service names) to have them in a predictable + // order in the final output + + keys := make([]string, 0) + for k := range detected { + keys = append(keys, k) + } + + sort.Strings(keys) + + for _, name := range keys { + svc := detected[name] + // if svc.DataSource != nil { + // if svc.DataSource.Labels["type"] == "" { + // return Setup{}, fmt.Errorf("missing type label for service %s", name) + // } + // err = yaml.Unmarshal(svc.AcquisYAML, svc.DataSource) + // if err != nil { + // return Setup{}, fmt.Errorf("while unmarshaling datasource for service %s: %w", name, err) + // } + // } + + ret.Setup = append(ret.Setup, ServiceSetup{ + DetectedService: name, + Install: svc.Install, + DataSource: svc.DataSource, + }) + } + + return ret, nil +} + +// ListSupported parses the configuration file and outputs a list of the supported services. +func ListSupported(serviceDetectionFile string) ([]string, error) { + dc, err := readDetectConfig(serviceDetectionFile) + if err != nil { + return nil, err + } + + keys := make([]string, 0) + for k := range dc.Detect { + keys = append(keys, k) + } + + sort.Strings(keys) + + return keys, nil +} diff --git a/pkg/setup/detect_test.go b/pkg/setup/detect_test.go new file mode 100644 index 00000000000..90d71488df1 --- /dev/null +++ b/pkg/setup/detect_test.go @@ -0,0 +1,997 @@ +package setup_test + +import ( + "fmt" + "os" + "os/exec" + "runtime" + "testing" + + "github.com/lithammer/dedent" + "github.com/stretchr/testify/require" + + "github.com/crowdsecurity/crowdsec/pkg/cstest" + "github.com/crowdsecurity/crowdsec/pkg/setup" +) + +var fakeSystemctlOutput = `UNIT FILE STATE VENDOR PRESET +crowdsec-setup-detect.service enabled enabled +apache2.service enabled enabled +apparmor.service enabled enabled +apport.service enabled enabled +atop.service enabled enabled +atopacct.service enabled enabled +finalrd.service enabled enabled +fwupd-refresh.service enabled enabled +fwupd.service enabled enabled + +9 unit files listed.` + +func fakeExecCommandNotFound(command string, args ...string) *exec.Cmd { + cs := []string{"-test.run=TestSetupHelperProcess", "--", command} + cs = append(cs, args...) + cmd := exec.Command("this-command-does-not-exist", cs...) + cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"} + + return cmd +} + +func fakeExecCommand(command string, args ...string) *exec.Cmd { + cs := []string{"-test.run=TestSetupHelperProcess", "--", command} + cs = append(cs, args...) + //nolint:gosec + cmd := exec.Command(os.Args[0], cs...) + cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"} + + return cmd +} + +func TestSetupHelperProcess(t *testing.T) { + if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" { + return + } + + fmt.Fprint(os.Stdout, fakeSystemctlOutput) + os.Exit(0) +} + +func tempYAML(t *testing.T, content string) string { + t.Helper() + require := require.New(t) + file, err := os.CreateTemp("", "") + require.NoError(err) + + _, err = file.WriteString(dedent.Dedent(content)) + require.NoError(err) + + err = file.Close() + require.NoError(err) + + return file.Name() +} + +func TestPathExists(t *testing.T) { + t.Parallel() + + tests := []struct { + path string + expected bool + }{ + {"/boot", true}, + {"/tmp", true}, + {"/this-should-not-exist", false}, + } + + for _, tc := range tests { + tc := tc + env := setup.NewExprEnvironment(setup.DetectOptions{}, setup.ExprOS{}) + + t.Run(tc.path, func(t *testing.T) { + t.Parallel() + actual := env.PathExists(tc.path) + require.Equal(t, tc.expected, actual) + }) + } +} + +func TestVersionCheck(t *testing.T) { + t.Parallel() + + tests := []struct { + version string + constraint string + expected bool + expectedErr string + }{ + {"1", "=1", true, ""}, + {"1", "!=1", false, ""}, + {"1", "<=1", true, ""}, + {"1", ">1", false, ""}, + {"1", ">=1", true, ""}, + {"1.0", "<1.0", false, ""}, + {"1", "<1", true, ""}, // XXX why? + {"1.3.5", "1.3", false, ""}, // XXX ok? + {"1.0", "<1.0", false, ""}, + {"1.0", "<=1.0", true, ""}, + {"2", ">1, <3", true, ""}, + {"2", "<=2, >=2.2", false, ""}, + {"2.3", "~2", true, ""}, + {"2.3", "=2", true, ""}, + {"1.1.1", "=1.1", false, ""}, + {"1.1.1", "1.1", false, ""}, + {"1.1", "!=1.1.1", true, ""}, + {"1.1", "~1.1.1", false, ""}, + {"1.1.1", "~1.1", true, ""}, + {"1.1.3", "~1.1", true, ""}, + {"19.04", "<19.10", true, ""}, + {"19.04", ">=19.10", false, ""}, + {"19.04", "=19.4", true, ""}, + {"19.04", "~19.4", true, ""}, + {"1.2.3", "~1.2", true, ""}, + {"1.2.3", "!=1.2", true, ""}, + {"1.2.3", "1.1.1 - 1.3.4", true, ""}, + {"1.3.5", "1.1.1 - 1.3.4", false, ""}, + {"1.3.5", "=1", true, ""}, + {"1.3.5", "1", true, ""}, + } + + for _, tc := range tests { + tc := tc + e := setup.ExprOS{RawVersion: tc.version} + + t.Run(fmt.Sprintf("Check(%s,%s)", tc.version, tc.constraint), func(t *testing.T) { + t.Parallel() + actual, err := e.VersionCheck(tc.constraint) + cstest.RequireErrorContains(t, err, tc.expectedErr) + require.Equal(t, tc.expected, actual) + }) + } +} + +// This is not required for Masterminds/semver +/* +func TestNormalizeVersion(t *testing.T) { + t.Parallel() + + tests := []struct { + version string + expected string + }{ + {"0", "0"}, + {"2", "2"}, + {"3.14", "3.14"}, + {"1.0", "1.0"}, + {"18.04", "18.4"}, + {"0.0.0", "0.0.0"}, + {"18.04.0", "18.4.0"}, + {"18.0004.0", "18.4.0"}, + {"21.04.2", "21.4.2"}, + {"050", "50"}, + {"trololo", "trololo"}, + {"0001.002.03", "1.2.3"}, + {"0001.002.03-trololo", "0001.002.03-trololo"}, + } + + for _, tc := range tests { + tc := tc + t.Run(tc.version, func(t *testing.T) { + t.Parallel() + actual := setup.NormalizeVersion(tc.version) + require.Equal(t, tc.expected, actual) + }) + } +} +*/ + +func TestListSupported(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + yml string + expected []string + expectedErr string + }{ + { + "list configured services", + ` + version: 1.0 + detect: + foo: + bar: + baz: + `, + []string{"foo", "bar", "baz"}, + "", + }, + { + "invalid yaml: blah blah", + "blahblah", + nil, + "yaml: unmarshal errors:", + }, + { + "invalid yaml: tabs are not allowed", + ` + version: 1.0 + detect: + foos: + `, + nil, + "yaml: line 4: found character that cannot start any token", + }, + { + "invalid yaml: no version", + "{}", + nil, + "missing version tag (must be 1.0)", + }, + { + "invalid yaml: bad version", + "version: 2.0", + nil, + "unsupported version tag '2.0' (must be 1.0)", + }, + } + + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + f := tempYAML(t, tc.yml) + defer os.Remove(f) + supported, err := setup.ListSupported(f) + cstest.RequireErrorContains(t, err, tc.expectedErr) + require.ElementsMatch(t, tc.expected, supported) + }) + } +} + +func TestApplyRules(t *testing.T) { + t.Parallel() + require := require.New(t) + + tests := []struct { + name string + rules []string + expectedOk bool + expectedErr string + }{ + { + "empty list is always true", // XXX or false? + []string{}, + true, + "", + }, + { + "simple true expression", + []string{"1+1==2"}, + true, + "", + }, + { + "simple false expression", + []string{"2+2==5"}, + false, + "", + }, + { + "all expressions are true", + []string{"1+2==3", "1!=2"}, + true, + "", + }, + { + "all expressions must be true", + []string{"true", "1==3", "1!=2"}, + false, + "", + }, + { + "each expression must be a boolan", + []string{"true", "\"notabool\""}, + false, + "rule '\"notabool\"': type must be a boolean", + }, + { + // we keep evaluating expressions to ensure that the + // file is formally correct, even if it can some time. + "each expression must be a boolan (no short circuit)", + []string{"false", "3"}, + false, + "rule '3': type must be a boolean", + }, + { + "unknown variable", + []string{"false", "doesnotexist"}, + false, + "rule 'doesnotexist': cannot fetch doesnotexist from", + }, + { + "unknown expression", + []string{"false", "doesnotexist()"}, + false, + "rule 'doesnotexist()': cannot get \"doesnotexist\" from", + }, + } + + env := setup.ExprEnvironment{} + + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + svc := setup.Service{When: tc.rules} + _, actualOk, err := setup.ApplyRules(svc, env) //nolint:typecheck,nolintlint // exported only for tests + cstest.RequireErrorContains(t, err, tc.expectedErr) + require.Equal(tc.expectedOk, actualOk) + }) + } +} + +// XXX TODO: TestApplyRules with journalctl default + +func TestUnitFound(t *testing.T) { + require := require.New(t) + setup.ExecCommand = fakeExecCommand + + defer func() { setup.ExecCommand = exec.Command }() + + env := setup.NewExprEnvironment(setup.DetectOptions{}, setup.ExprOS{}) + + installed, err := env.UnitFound("crowdsec-setup-detect.service") + require.NoError(err) + + require.Equal(true, installed) +} + +// TODO apply rules to filter a list of Service structs +// func testFilterWithRules(t *testing.T) { +// } + +func TestDetectSimpleRule(t *testing.T) { + require := require.New(t) + setup.ExecCommand = fakeExecCommand + + f := tempYAML(t, ` + version: 1.0 + detect: + good: + when: + - true + bad: + when: + - false + ugly: + `) + defer os.Remove(f) + + detected, err := setup.Detect(f, setup.DetectOptions{}) + require.NoError(err) + + expected := []setup.ServiceSetup{ + {DetectedService: "good"}, + {DetectedService: "ugly"}, + } + + require.ElementsMatch(expected, detected.Setup) +} + +func TestDetectUnitError(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("skipping on windows") + } + + require := require.New(t) + setup.ExecCommand = fakeExecCommandNotFound + + defer func() { setup.ExecCommand = exec.Command }() + + tests := []struct { + name string + config string + expected setup.Setup + expectedErr string + }{ + { + "error is reported if systemctl does not exist", + ` +version: 1.0 +detect: + wizard: + when: + - UnitFound("crowdsec-setup-detect.service")`, + setup.Setup{[]setup.ServiceSetup{}}, + `while looking for service wizard: rule 'UnitFound("crowdsec-setup-detect.service")': ` + + `running systemctl: exec: "this-command-does-not-exist": executable file not found in $PATH`, + }, + } + + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + f := tempYAML(t, tc.config) + defer os.Remove(f) + + detected, err := setup.Detect(f, setup.DetectOptions{}) + cstest.RequireErrorContains(t, err, tc.expectedErr) + require.Equal(tc.expected, detected) + }) + } +} + +func TestDetectUnit(t *testing.T) { + require := require.New(t) + setup.ExecCommand = fakeExecCommand + + defer func() { setup.ExecCommand = exec.Command }() + + tests := []struct { + name string + config string + expected setup.Setup + expectedErr string + }{ + // { + // "detect a single unit, with default log filter", + // ` + // version: 1.0 + // detect: + // wizard: + // when: + // - UnitFound("crowdsec-setup-detect.service") + // datasource: + // labels: + // type: syslog + // sorcerer: + // when: + // - UnitFound("sorcerer.service")`, + // setup.Setup{ + // Setup: []setup.ServiceSetup{ + // { + // DetectedService: "wizard", + // DataSource: setup.DataSourceItem{ + // "Labels": map[string]string{"type": "syslog"}, + // "JournalCTLFilter": []string{"_SYSTEMD_UNIT=crowdsec-setup-detect.service"}, + // }, + // }, + // }, + // }, + // "", + // }, + // { + // "detect a single unit, but type label is missing", + // ` + // version: 1.0 + // detect: + // wizard: + // when: + // - UnitFound("crowdsec-setup-detect.service")`, + // setup.Setup{}, + // "missing type label for service wizard", + // }, + { + "detect unit and pick up acquisistion filter", + ` +version: 1.0 +detect: + wizard: + when: + - UnitFound("crowdsec-setup-detect.service") + datasource: + source: journalctl + labels: + type: syslog + journalctl_filter: + - _MY_CUSTOM_FILTER=something`, + setup.Setup{ + Setup: []setup.ServiceSetup{ + { + DetectedService: "wizard", + DataSource: setup.DataSourceItem{ + // XXX this should not be DataSourceItem ?? + "source": "journalctl", + "labels": setup.DataSourceItem{"type": "syslog"}, + "journalctl_filter": []interface{}{"_MY_CUSTOM_FILTER=something"}, + }, + }, + }, + }, + "", + }, + } + + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + f := tempYAML(t, tc.config) + defer os.Remove(f) + + detected, err := setup.Detect(f, setup.DetectOptions{}) + cstest.RequireErrorContains(t, err, tc.expectedErr) + require.Equal(tc.expected, detected) + }) + } +} + +func TestDetectForcedUnit(t *testing.T) { + require := require.New(t) + setup.ExecCommand = fakeExecCommand + + defer func() { setup.ExecCommand = exec.Command }() + + f := tempYAML(t, ` + version: 1.0 + detect: + wizard: + when: + - UnitFound("crowdsec-setup-forced.service") + datasource: + source: journalctl + labels: + type: syslog + journalctl_filter: + - _SYSTEMD_UNIT=crowdsec-setup-forced.service + `) + defer os.Remove(f) + + detected, err := setup.Detect(f, setup.DetectOptions{ForcedUnits: []string{"crowdsec-setup-forced.service"}}) + require.NoError(err) + + expected := setup.Setup{ + Setup: []setup.ServiceSetup{ + { + DetectedService: "wizard", + DataSource: setup.DataSourceItem{ + "source": "journalctl", + "labels": setup.DataSourceItem{"type": "syslog"}, + "journalctl_filter": []interface{}{"_SYSTEMD_UNIT=crowdsec-setup-forced.service"}, + }, + }, + }, + } + require.Equal(expected, detected) +} + +func TestDetectForcedProcess(t *testing.T) { + require := require.New(t) + setup.ExecCommand = fakeExecCommand + + defer func() { setup.ExecCommand = exec.Command }() + + f := tempYAML(t, ` + version: 1.0 + detect: + wizard: + when: + - ProcessRunning("foobar") + `) + defer os.Remove(f) + + detected, err := setup.Detect(f, setup.DetectOptions{ForcedProcesses: []string{"foobar"}}) + require.NoError(err) + + expected := setup.Setup{ + Setup: []setup.ServiceSetup{ + {DetectedService: "wizard"}, + }, + } + require.Equal(expected, detected) +} + +func TestDetectSkipService(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("skipping on windows") + } + + require := require.New(t) + setup.ExecCommand = fakeExecCommand + + defer func() { setup.ExecCommand = exec.Command }() + + f := tempYAML(t, ` + version: 1.0 + detect: + wizard: + when: + - ProcessRunning("foobar") + `) + defer os.Remove(f) + + detected, err := setup.Detect(f, setup.DetectOptions{ForcedProcesses: []string{"foobar"}, SkipServices: []string{"wizard"}}) + require.NoError(err) + + expected := setup.Setup{[]setup.ServiceSetup{}} + require.Equal(expected, detected) +} + +func TestDetectForcedOS(t *testing.T) { + require := require.New(t) + setup.ExecCommand = fakeExecCommand + + defer func() { setup.ExecCommand = exec.Command }() + + type test struct { + name string + config string + forced setup.ExprOS + expected setup.Setup + expectedErr string + } + + tests := []test{ + { + "detect OS - force linux", + ` + version: 1.0 + detect: + linux: + when: + - OS.Family == "linux"`, + setup.ExprOS{Family: "linux"}, + setup.Setup{ + Setup: []setup.ServiceSetup{ + {DetectedService: "linux"}, + }, + }, + "", + }, + { + "detect OS - force windows", + ` + version: 1.0 + detect: + windows: + when: + - OS.Family == "windows"`, + setup.ExprOS{Family: "windows"}, + setup.Setup{ + Setup: []setup.ServiceSetup{ + {DetectedService: "windows"}, + }, + }, + "", + }, + { + "detect OS - ubuntu (no match)", + ` + version: 1.0 + detect: + linux: + when: + - OS.Family == "linux" && OS.ID == "ubuntu"`, + setup.ExprOS{Family: "linux"}, + setup.Setup{[]setup.ServiceSetup{}}, + "", + }, + { + "detect OS - ubuntu (match)", + ` + version: 1.0 + detect: + linux: + when: + - OS.Family == "linux" && OS.ID == "ubuntu"`, + setup.ExprOS{Family: "linux", ID: "ubuntu"}, + setup.Setup{ + Setup: []setup.ServiceSetup{ + {DetectedService: "linux"}, + }, + }, + "", + }, + { + "detect OS - ubuntu (match with version)", + ` + version: 1.0 + detect: + linux: + when: + - OS.Family == "linux" && OS.ID == "ubuntu" && OS.VersionCheck("19.04")`, + setup.ExprOS{Family: "linux", ID: "ubuntu", RawVersion: "19.04"}, + setup.Setup{ + Setup: []setup.ServiceSetup{ + {DetectedService: "linux"}, + }, + }, + "", + }, + { + "detect OS - ubuntu >= 20.04 (no match: no version detected)", + ` + version: 1.0 + detect: + linux: + when: + - OS.ID == "ubuntu" && OS.VersionCheck(">=20.04")`, + setup.ExprOS{Family: "linux"}, + setup.Setup{[]setup.ServiceSetup{}}, + "", + }, + { + "detect OS - ubuntu >= 20.04 (no match: version is lower)", + ` + version: 1.0 + detect: + linux: + when: + - OS.ID == "ubuntu" && OS.VersionCheck(">=20.04")`, + setup.ExprOS{Family: "linux", ID: "ubuntu", RawVersion: "19.10"}, + setup.Setup{[]setup.ServiceSetup{}}, + "", + }, + { + "detect OS - ubuntu >= 20.04 (match: same version)", + ` + version: 1.0 + detect: + linux: + when: + - OS.ID == "ubuntu" && OS.VersionCheck(">=20.04")`, + setup.ExprOS{Family: "linux", ID: "ubuntu", RawVersion: "20.04"}, + setup.Setup{ + Setup: []setup.ServiceSetup{ + {DetectedService: "linux"}, + }, + }, + "", + }, + { + "detect OS - ubuntu >= 20.04 (match: version is higher)", + ` + version: 1.0 + detect: + linux: + when: + - OS.ID == "ubuntu" && OS.VersionCheck(">=20.04")`, + setup.ExprOS{Family: "linux", ID: "ubuntu", RawVersion: "22.04"}, + setup.Setup{ + Setup: []setup.ServiceSetup{ + {DetectedService: "linux"}, + }, + }, + "", + }, + + { + "detect OS - ubuntu < 20.04 (no match: no version detected)", + ` + version: 1.0 + detect: + linux: + when: + - OS.ID == "ubuntu" && OS.VersionCheck("<20.04")`, + setup.ExprOS{Family: "linux"}, + setup.Setup{[]setup.ServiceSetup{}}, + "", + }, + { + "detect OS - ubuntu < 20.04 (no match: version is higher)", + ` + version: 1.0 + detect: + linux: + when: + - OS.ID == "ubuntu" && OS.VersionCheck("<20.04")`, + setup.ExprOS{Family: "linux", ID: "ubuntu", RawVersion: "20.10"}, + setup.Setup{[]setup.ServiceSetup{}}, + "", + }, + { + "detect OS - ubuntu < 20.04 (no match: same version)", + ` + version: 1.0 + detect: + linux: + when: + - OS.ID == "ubuntu" && OS.VersionCheck("<20.04")`, + setup.ExprOS{Family: "linux", ID: "ubuntu", RawVersion: "20.04"}, + setup.Setup{[]setup.ServiceSetup{}}, + "", + }, + { + "detect OS - ubuntu < 20.04 (match: version is lower)", + ` + version: 1.0 + detect: + linux: + when: + - OS.ID == "ubuntu" + - OS.VersionCheck("<20.04")`, + setup.ExprOS{Family: "linux", ID: "ubuntu", RawVersion: "19.10"}, + setup.Setup{ + Setup: []setup.ServiceSetup{ + {DetectedService: "linux"}, + }, + }, + "", + }, + } + + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + f := tempYAML(t, tc.config) + defer os.Remove(f) + + detected, err := setup.Detect(f, setup.DetectOptions{ForcedOS: tc.forced}) + cstest.RequireErrorContains(t, err, tc.expectedErr) + require.Equal(tc.expected, detected) + }) + } +} + +func TestDetectDatasourceValidation(t *testing.T) { + // It could be a good idea to test UnmarshalConfig() separately in addition + // to Configure(), in each datasource. For now, we test these here. + + require := require.New(t) + setup.ExecCommand = fakeExecCommand + + defer func() { setup.ExecCommand = exec.Command }() + + type test struct { + name string + config string + expected setup.Setup + expectedErr string + } + + tests := []test{ + { + name: "source is empty", + config: ` + version: 1.0 + detect: + wizard: + datasource: + labels: + type: something`, + expected: setup.Setup{Setup:[]setup.ServiceSetup{}}, + expectedErr: "invalid datasource for wizard: source is empty", + }, { + name: "source is unknown", + config: ` + version: 1.0 + detect: + foobar: + datasource: + source: wombat`, + expected: setup.Setup{Setup:[]setup.ServiceSetup{}}, + expectedErr: "invalid datasource for foobar: unknown source 'wombat'", + }, { + name: "source is misplaced", + config: ` + version: 1.0 + detect: + foobar: + datasource: + source: file`, + expected: setup.Setup{Setup:[]setup.ServiceSetup{}}, + expectedErr: "while parsing {{.DetectYaml}}: yaml: unmarshal errors:\n line 6: field source not found in type setup.Service", + }, { + name: "source file: required fields", + config: ` + version: 1.0 + detect: + foobar: + datasource: + source: file`, + expected: setup.Setup{Setup:[]setup.ServiceSetup{}}, + expectedErr: "invalid datasource for foobar: no filename or filenames configuration provided", + }, { + name: "source journalctl: required fields", + config: ` + version: 1.0 + detect: + foobar: + datasource: + source: journalctl`, + expected: setup.Setup{Setup:[]setup.ServiceSetup{}}, + expectedErr: "invalid datasource for foobar: journalctl_filter is required", + }, { + name: "source cloudwatch: required fields", + config: ` + version: 1.0 + detect: + foobar: + datasource: + source: cloudwatch`, + expected: setup.Setup{Setup:[]setup.ServiceSetup{}}, + expectedErr: "invalid datasource for foobar: group_name is mandatory for CloudwatchSource", + }, { + name: "source syslog: all fields are optional", + config: ` + version: 1.0 + detect: + foobar: + datasource: + source: syslog`, + expected: setup.Setup{ + Setup: []setup.ServiceSetup{ + { + DetectedService:"foobar", + DataSource: setup.DataSourceItem{"source":"syslog"}, + }, + }, + }, + }, { + name: "source docker: required fields", + config: ` + version: 1.0 + detect: + foobar: + datasource: + source: docker`, + expected: setup.Setup{Setup:[]setup.ServiceSetup{}}, + expectedErr: "invalid datasource for foobar: no containers names or containers ID configuration provided", + }, { + name: "source kinesis: required fields (enhanced fanout=false)", + config: ` + version: 1.0 + detect: + foobar: + datasource: + source: kinesis`, + expected: setup.Setup{Setup:[]setup.ServiceSetup{}}, + expectedErr: "invalid datasource for foobar: stream_name is mandatory when use_enhanced_fanout is false", + }, { + name: "source kinesis: required fields (enhanced fanout=true)", + config: ` + version: 1.0 + detect: + foobar: + datasource: + source: kinesis + use_enhanced_fanout: true`, + expected: setup.Setup{Setup:[]setup.ServiceSetup{}}, + expectedErr: "invalid datasource for foobar: stream_arn is mandatory when use_enhanced_fanout is true", + }, { + name: "source kafka: required fields", + config: ` + version: 1.0 + detect: + foobar: + datasource: + source: kafka`, + expected: setup.Setup{Setup:[]setup.ServiceSetup{}}, + expectedErr: "invalid datasource for foobar: cannot create a kafka reader with an empty list of broker addresses", + }, + } + + if runtime.GOOS == "windows" { + tests = append(tests, test{ + name: "source wineventlog: required fields", + config: ` + version: 1.0 + detect: + foobar: + datasource: + source: wineventlog`, + expected: setup.Setup{Setup:[]setup.ServiceSetup{}}, + expectedErr: "", + }) + } + + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + detectYaml := tempYAML(t, tc.config) + defer os.Remove(detectYaml) + + data := map[string]string{ + "DetectYaml": detectYaml, + } + + expectedErr, err := cstest.Interpolate(tc.expectedErr, data) + require.NoError(err) + + detected, err := setup.Detect(detectYaml, setup.DetectOptions{}) + cstest.RequireErrorContains(t, err, expectedErr) + require.Equal(tc.expected, detected) + }) + } +} + + + diff --git a/pkg/setup/export_test.go b/pkg/setup/export_test.go new file mode 100644 index 00000000000..56ca0294546 --- /dev/null +++ b/pkg/setup/export_test.go @@ -0,0 +1,9 @@ +package setup + +var ( + SystemdUnitList = systemdUnitList + FilterWithRules = filterWithRules + ApplyRules = applyRules + +// NormalizeVersion = normalizeVersion +) diff --git a/pkg/setup/install.go b/pkg/setup/install.go new file mode 100644 index 00000000000..5d3bfdbc995 --- /dev/null +++ b/pkg/setup/install.go @@ -0,0 +1,255 @@ +package setup + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "strings" + + goccyyaml "github.com/goccy/go-yaml" + "gopkg.in/yaml.v3" + + "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +// AcquisDocument is created from a SetupItem. It represents a single YAML document, and can be part of a multi-document file. +type AcquisDocument struct { + AcquisFilename string + DataSource map[string]interface{} +} + +func decodeSetup(input []byte, fancyErrors bool) (Setup, error) { + ret := Setup{} + + // parse with goccy to have better error messages in many cases + dec := goccyyaml.NewDecoder(bytes.NewBuffer(input), goccyyaml.Strict()) + + if err := dec.Decode(&ret); err != nil { + if fancyErrors { + return ret, fmt.Errorf("%v", goccyyaml.FormatError(err, true, true)) + } + // XXX errors here are multiline, should we just print them to stderr instead of logging? + return ret, fmt.Errorf("%v", err) + } + + // parse again because goccy is not strict enough anyway + dec2 := yaml.NewDecoder(bytes.NewBuffer(input)) + dec2.KnownFields(true) + + if err := dec2.Decode(&ret); err != nil { + return ret, fmt.Errorf("while unmarshaling setup file: %w", err) + } + + return ret, nil +} + +// InstallHubItems installs the objects recommended in a setup file. +func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error { + setupEnvelope, err := decodeSetup(input, false) + if err != nil { + return err + } + + if err := csConfig.LoadHub(); err != nil { + return fmt.Errorf("loading hub: %w", err) + } + + if err := cwhub.SetHubBranch(); err != nil { + return fmt.Errorf("setting hub branch: %w", err) + } + + if err := cwhub.GetHubIdx(csConfig.Hub); err != nil { + return fmt.Errorf("getting hub index: %w", err) + } + + for _, setupItem := range setupEnvelope.Setup { + forceAction := false + downloadOnly := false + install := setupItem.Install + + if install == nil { + continue + } + + if len(install.Collections) > 0 { + for _, collection := range setupItem.Install.Collections { + if dryRun { + fmt.Println("dry-run: would install collection", collection) + + continue + } + + if err := cwhub.InstallItem(csConfig, collection, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil { + return fmt.Errorf("while installing collection %s: %w", collection, err) + } + } + } + + if len(install.Parsers) > 0 { + for _, parser := range setupItem.Install.Parsers { + if dryRun { + fmt.Println("dry-run: would install parser", parser) + + continue + } + + if err := cwhub.InstallItem(csConfig, parser, cwhub.PARSERS, forceAction, downloadOnly); err != nil { + return fmt.Errorf("while installing parser %s: %w", parser, err) + } + } + } + + if len(install.Scenarios) > 0 { + for _, scenario := range setupItem.Install.Scenarios { + if dryRun { + fmt.Println("dry-run: would install scenario", scenario) + + continue + } + + if err := cwhub.InstallItem(csConfig, scenario, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil { + return fmt.Errorf("while installing scenario %s: %w", scenario, err) + } + } + } + + if len(install.PostOverflows) > 0 { + for _, postoverflow := range setupItem.Install.PostOverflows { + if dryRun { + fmt.Println("dry-run: would install postoverflow", postoverflow) + + continue + } + + if err := cwhub.InstallItem(csConfig, postoverflow, cwhub.PARSERS_OVFLW, forceAction, downloadOnly); err != nil { + return fmt.Errorf("while installing postoverflow %s: %w", postoverflow, err) + } + } + } + } + + return nil +} + +// marshalAcquisDocuments creates the monolithic file, or itemized files (if a directory is provided) with the acquisition documents. +func marshalAcquisDocuments(ads []AcquisDocument, toDir string) (string, error) { + var sb strings.Builder + + dashTerminator := false + + disclaimer := ` +# +# This file was automatically generated by "cscli setup datasources". +# You can modify it by hand, but will be responsible for its maintenance. +# To add datasources or logfiles, you can instead write a new configuration +# in the directory defined by acquisition_dir. +# + +` + + if toDir == "" { + sb.WriteString(disclaimer) + } else { + _, err := os.Stat(toDir) + if os.IsNotExist(err) { + return "", fmt.Errorf("directory %s does not exist", toDir) + } + } + + for _, ad := range ads { + out, err := goccyyaml.MarshalWithOptions(ad.DataSource, goccyyaml.IndentSequence(true)) + if err != nil { + return "", fmt.Errorf("while encoding datasource: %w", err) + } + + if toDir != "" { + if ad.AcquisFilename == "" { + return "", fmt.Errorf("empty acquis filename") + } + + fname := filepath.Join(toDir, ad.AcquisFilename) + fmt.Println("creating", fname) + + f, err := os.Create(fname) + if err != nil { + return "", fmt.Errorf("creating acquisition file: %w", err) + } + defer f.Close() + + _, err = f.WriteString(disclaimer) + if err != nil { + return "", fmt.Errorf("while writing to %s: %w", ad.AcquisFilename, err) + } + + _, err = f.Write(out) + if err != nil { + return "", fmt.Errorf("while writing to %s: %w", ad.AcquisFilename, err) + } + + f.Sync() + + continue + } + + if dashTerminator { + sb.WriteString("---\n") + } + + sb.Write(out) + + dashTerminator = true + } + + return sb.String(), nil +} + +// Validate checks the validity of a setup file. +func Validate(input []byte) error { + _, err := decodeSetup(input, true) + if err != nil { + return err + } + + return nil +} + +// DataSources generates the acquisition documents from a setup file. +func DataSources(input []byte, toDir string) (string, error) { + setupEnvelope, err := decodeSetup(input, false) + if err != nil { + return "", err + } + + ads := make([]AcquisDocument, 0) + + filename := func(basename string, ext string) string { + if basename == "" { + return basename + } + + return basename + ext + } + + for _, setupItem := range setupEnvelope.Setup { + datasource := setupItem.DataSource + + basename := "" + if toDir != "" { + basename = "setup." + setupItem.DetectedService + } + + if datasource == nil { + continue + } + + ad := AcquisDocument{ + AcquisFilename: filename(basename, ".yaml"), + DataSource: datasource, + } + ads = append(ads, ad) + } + + return marshalAcquisDocuments(ads, toDir) +} diff --git a/pkg/setup/units.go b/pkg/setup/units.go new file mode 100644 index 00000000000..a0bccba4aac --- /dev/null +++ b/pkg/setup/units.go @@ -0,0 +1,59 @@ +package setup + +import ( + "bufio" + "fmt" + "strings" + + log "github.com/sirupsen/logrus" +) + +// systemdUnitList returns all enabled systemd units. +// It needs to parse the table because -o json does not work everywhere. +func systemdUnitList() ([]string, error) { + wrap := func(err error) error { + return fmt.Errorf("running systemctl: %w", err) + } + + ret := make([]string, 0) + cmd := ExecCommand("systemctl", "list-unit-files", "--state=enabled,generated,static") + + stdout, err := cmd.StdoutPipe() + if err != nil { + return ret, wrap(err) + } + + log.Debugf("Running systemctl...") + + if err := cmd.Start(); err != nil { + return ret, wrap(err) + } + + scanner := bufio.NewScanner(stdout) + header := true // skip the first line + + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 { + break // the rest of the output is footer + } + + if !header { + spaceIdx := strings.IndexRune(line, ' ') + if spaceIdx == -1 { + return ret, fmt.Errorf("can't parse systemctl output") + } + + line = line[:spaceIdx] + ret = append(ret, line) + } + + header = false + } + + if err := cmd.Wait(); err != nil { + return ret, wrap(err) + } + + return ret, nil +} diff --git a/pkg/setup/units_test.go b/pkg/setup/units_test.go new file mode 100644 index 00000000000..b1bfd8816bc --- /dev/null +++ b/pkg/setup/units_test.go @@ -0,0 +1,32 @@ +package setup_test + +import ( + "os/exec" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/crowdsecurity/crowdsec/pkg/setup" +) + +func TestSystemdUnitList(t *testing.T) { + require := require.New(t) + setup.ExecCommand = fakeExecCommand + + defer func() { setup.ExecCommand = exec.Command }() + + units, err := setup.SystemdUnitList() //nolint:typecheck,nolintlint // exported only for tests + require.NoError(err) + + require.Equal([]string{ + "crowdsec-setup-detect.service", + "apache2.service", + "apparmor.service", + "apport.service", + "atop.service", + "atopacct.service", + "finalrd.service", + "fwupd-refresh.service", + "fwupd.service", + }, units) +} diff --git a/scripts/test_env.sh b/scripts/test_env.sh index b203e7f3ef2..8402070ce6e 100755 --- a/scripts/test_env.sh +++ b/scripts/test_env.sh @@ -1,22 +1,66 @@ -#!/bin/bash +#!/bin/sh +set -e + +# XXX this can't be a good place to make the tree BASE="./tests" usage() { echo "Usage:" - echo " ./wizard.sh -h Display this help message." - echo " ./test_env.sh -d ./tests Create test environment in './tests' folder" + echo " $0 -h Display this help message." + echo " $0 -d ./tests Create test environment in './tests' folder" exit 0 } +set_colors() { + FG_BLACK="" + FG_RED="" + FG_GREEN="" + FG_YELLOW="" + FG_BLUE="" + FG_MAGENTA="" + FG_CYAN="" + FG_WHITE="" + BOLD="" + RESET="" + + #shellcheck disable=SC2034 + if tput sgr0 >/dev/null; then + FG_BLACK=$(tput setaf 0) + FG_RED=$(tput setaf 1) + FG_GREEN=$(tput setaf 2) + FG_YELLOW=$(tput setaf 3) + FG_BLUE=$(tput setaf 4) + FG_MAGENTA=$(tput setaf 5) + FG_CYAN=$(tput setaf 6) + FG_WHITE=$(tput setaf 7) + BOLD=$(tput bold) + RESET=$(tput sgr0) + fi +} + +log_info() { + msg=$1 + date=$(date +%x:%X) + echo "{FG_BLUE}INFO${RESET}[${date}] $msg" +} + +log_err() { + msg=$1 + date=$(date +%x:%X) + echo "${FG_RED}ERR${RESET}[${date}] $msg" >&2 +} + -while [[ $# -gt 0 ]] +set_colors() + +while [ $# -gt 0 ] do key="${1}" case ${key} in -d|--directory) - BASE=${2} - shift #past argument + shift + BASE=$1 shift ;; -h|--help) @@ -31,7 +75,7 @@ do esac done -BASE=$(realpath $BASE) +BASE=$(realpath "$BASE") DATA_DIR="$BASE/data" @@ -51,13 +95,8 @@ PLUGINS="http slack splunk email" PLUGINS_DIR="plugins" NOTIF_DIR="notifications" -log_info() { - msg=$1 - date=$(date +%x:%X) - echo -e "[$date][INFO] $msg" -} -create_arbo() { +create_tree() { mkdir -p "$BASE" mkdir -p "$DATA_DIR" mkdir -p "$LOG_DIR" @@ -83,38 +122,37 @@ copy_files() { cp "./config/acquis.yaml" "$CONFIG_DIR" touch "$CONFIG_DIR"/local_api_credentials.yaml touch "$CONFIG_DIR"/online_api_credentials.yaml - envsubst < "./config/dev.yaml" > $BASE/dev.yaml - for plugin in $PLUGINS - do - cp $PLUGINS_DIR/$NOTIF_DIR/$plugin/notification-$plugin $BASE/$PLUGINS_DIR/notification-$plugin - cp $PLUGINS_DIR/$NOTIF_DIR/$plugin/$plugin.yaml $CONFIG_DIR/$NOTIF_DIR/$plugin.yaml + envsubst < "./config/dev.yaml" > "$BASE/dev.yaml" + for plugin in $PLUGINS; do + cp "$PLUGINS_DIR/$NOTIF_DIR/$plugin/notification-$plugin" "$BASE/$PLUGINS_DIR/notification-$plugin" + cp "$PLUGINS_DIR/$NOTIF_DIR/$plugin/$plugin.yaml" "$CONFIG_DIR/$NOTIF_DIR/$plugin.yaml" done } setup() { - $BASE/cscli -c "$CONFIG_FILE" hub update - $BASE/cscli -c "$CONFIG_FILE" collections install crowdsecurity/linux + "$BASE/cscli" -c "$CONFIG_FILE" hub update + "$BASE/cscli" -c "$CONFIG_FILE" collections install crowdsecurity/linux } setup_api() { - $BASE/cscli -c "$CONFIG_FILE" machines add test -p testpassword -f $CONFIG_DIR/local_api_credentials.yaml --force + "$BASE/cscli" -c "$CONFIG_FILE" machines add test -p testpassword -f "$CONFIG_DIR/local_api_credentials.yaml" --force } main() { - log_info "Creating test arboresence in $BASE" - create_arbo - log_info "Arboresence created" + log_info "Creating directory tree in $BASE" + create_tree + log_info "Directory tree created" log_info "Copying needed files for tests environment" copy_files log_info "Files copied" log_info "Setting up configurations" CURRENT_PWD=$(pwd) - cd $BASE + cd "$BASE" setup_api setup - cd $CURRENT_PWD + cd "$CURRENT_PWD" log_info "Environment is ready in $BASE" } diff --git a/tests/ansible/requirements.yml b/tests/ansible/requirements.yml index ec0936423e7..b1a28b70a02 100644 --- a/tests/ansible/requirements.yml +++ b/tests/ansible/requirements.yml @@ -6,6 +6,9 @@ roles: - src: https://github.com/crowdsecurity/ansible-role-postgresql version: crowdsec name: geerlingguy.postgresql + # these should be included as dependencies of crowdsecurity.testing, but sometime are not + - src: geerlingguy.repo-epel + - src: gantsign.golang collections: - name: https://github.com/crowdsecurity/ansible-collection-crowdsecurity.testing.git diff --git a/tests/ansible/vagrant/experimental/wizard-centos-8/Vagrantfile b/tests/ansible/vagrant/experimental/wizard-centos-8/Vagrantfile new file mode 100644 index 00000000000..a3cbd3de5a0 --- /dev/null +++ b/tests/ansible/vagrant/experimental/wizard-centos-8/Vagrantfile @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +Vagrant.configure('2') do |config| + config.vm.box = 'centos/stream8' + config.vm.define 'wizard' + + config.vm.provision 'shell', path: 'bootstrap' + + config.vm.provider :libvirt do |libvirt| + libvirt.cpus = 4 + libvirt.memory = 4096 + end + + config.vm.synced_folder '.', '/vagrant', disabled: true + + # config.vm.provision 'ansible' do |ansible| + # ansible.config_file = '../../ansible.cfg' + # ansible.playbook = '../../run_all.yml' + # end + + # same as above, to run the steps separately + + config.vm.provision 'ansible' do |provdep| + provdep.config_file = '../../../ansible.cfg' + provdep.playbook = '../../../provision_dependencies.yml' + end + + config.vm.provision 'ansible' do |provtest| + provtest.config_file = '../../../ansible.cfg' + provtest.playbook = '../../../provision_test_suite.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../prepare_tests.yml' + end + + # config.vm.provision 'ansible' do |runtests| + # runtests.config_file = '../../../ansible.cfg' + # runtests.playbook = '../../../run_tests.yml' + # end +end diff --git a/tests/ansible/vagrant/experimental/wizard-centos-8/bootstrap b/tests/ansible/vagrant/experimental/wizard-centos-8/bootstrap new file mode 100755 index 00000000000..b33ad9c881a --- /dev/null +++ b/tests/ansible/vagrant/experimental/wizard-centos-8/bootstrap @@ -0,0 +1,5 @@ +#!/bin/sh +unset IFS +set -euf + +sudo dnf -y update diff --git a/tests/ansible/vagrant/experimental/wizard-debian-bullseye/Vagrantfile b/tests/ansible/vagrant/experimental/wizard-debian-bullseye/Vagrantfile new file mode 100644 index 00000000000..b2cde23f092 --- /dev/null +++ b/tests/ansible/vagrant/experimental/wizard-debian-bullseye/Vagrantfile @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +Vagrant.configure('2') do |config| + config.vm.box = 'debian/bullseye64' + config.vm.define 'wizard' + + config.vm.provision 'shell', path: 'bootstrap' + + config.vm.provider :libvirt do |libvirt| + libvirt.cpus = 4 + libvirt.memory = 4096 + end + + config.vm.synced_folder '.', '/vagrant', disabled: true + + # config.vm.provision 'ansible' do |ansible| + # ansible.config_file = '../../ansible.cfg' + # ansible.playbook = '../../run_all.yml' + # end + + # same as above, to run the steps separately + + config.vm.provision 'ansible' do |provdep| + provdep.config_file = '../../../ansible.cfg' + provdep.playbook = '../../../provision_dependencies.yml' + end + + config.vm.provision 'ansible' do |provtest| + provtest.config_file = '../../../ansible.cfg' + provtest.playbook = '../../../provision_test_suite.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../prepare_tests.yml' + end + + # config.vm.provision 'ansible' do |runtests| + # runtests.config_file = '../../../ansible.cfg' + # runtests.playbook = '../../../run_tests.yml' + # end +end diff --git a/tests/ansible/vagrant/experimental/wizard-debian-bullseye/bootstrap b/tests/ansible/vagrant/experimental/wizard-debian-bullseye/bootstrap new file mode 100755 index 00000000000..6a5df521a60 --- /dev/null +++ b/tests/ansible/vagrant/experimental/wizard-debian-bullseye/bootstrap @@ -0,0 +1,5 @@ +#!/bin/sh +unset IFS +set -euf + +sudo apt install -y aptitude diff --git a/tests/ansible/vagrant/experimental/wizard-debian-buster/Vagrantfile b/tests/ansible/vagrant/experimental/wizard-debian-buster/Vagrantfile new file mode 100644 index 00000000000..a1783249358 --- /dev/null +++ b/tests/ansible/vagrant/experimental/wizard-debian-buster/Vagrantfile @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +Vagrant.configure('2') do |config| + config.vm.box = 'debian/buster64' + config.vm.define 'wizard' + + config.vm.provision 'shell', path: 'bootstrap' + + config.vm.provider :libvirt do |libvirt| + libvirt.cpus = 4 + libvirt.memory = 4096 + end + + config.vm.synced_folder '.', '/vagrant', disabled: true + + # config.vm.provision 'ansible' do |ansible| + # ansible.config_file = '../../ansible.cfg' + # ansible.playbook = '../../run_all.yml' + # end + + # same as above, to run the steps separately + + config.vm.provision 'ansible' do |provdep| + provdep.config_file = '../../../ansible.cfg' + provdep.playbook = '../../../provision_dependencies.yml' + end + + config.vm.provision 'ansible' do |provtest| + provtest.config_file = '../../../ansible.cfg' + provtest.playbook = '../../../provision_test_suite.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../prepare_tests.yml' + end + + # config.vm.provision 'ansible' do |runtests| + # runtests.config_file = '../../../ansible.cfg' + # runtests.playbook = '../../../run_tests.yml' + # end +end diff --git a/tests/ansible/vagrant/experimental/wizard-debian-buster/bootstrap b/tests/ansible/vagrant/experimental/wizard-debian-buster/bootstrap new file mode 100755 index 00000000000..6a5df521a60 --- /dev/null +++ b/tests/ansible/vagrant/experimental/wizard-debian-buster/bootstrap @@ -0,0 +1,5 @@ +#!/bin/sh +unset IFS +set -euf + +sudo apt install -y aptitude diff --git a/tests/ansible/vagrant/experimental/wizard-fedora-36/Vagrantfile b/tests/ansible/vagrant/experimental/wizard-fedora-36/Vagrantfile new file mode 100644 index 00000000000..9ba19a3e38f --- /dev/null +++ b/tests/ansible/vagrant/experimental/wizard-fedora-36/Vagrantfile @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +Vagrant.configure('2') do |config| + config.vm.box = 'fedora/36-cloud-base' + config.vm.define 'wizard' + + config.vm.provision 'shell', path: 'bootstrap' + + config.vm.provider :libvirt do |libvirt| + libvirt.cpus = 4 + libvirt.memory = 4096 + end + + config.vm.synced_folder '.', '/vagrant', disabled: true + + # config.vm.provision 'ansible' do |ansible| + # ansible.config_file = '../../ansible.cfg' + # ansible.playbook = '../../run_all.yml' + # end + + # same as above, to run the steps separately + + config.vm.provision 'ansible' do |provdep| + provdep.config_file = '../../../ansible.cfg' + provdep.playbook = '../../../provision_dependencies.yml' + end + + config.vm.provision 'ansible' do |provtest| + provtest.config_file = '../../../ansible.cfg' + provtest.playbook = '../../../provision_test_suite.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../prepare_tests.yml' + end + + # config.vm.provision 'ansible' do |runtests| + # runtests.config_file = '../../../ansible.cfg' + # runtests.playbook = '../../../run_tests.yml' + # end +end diff --git a/tests/ansible/vagrant/experimental/wizard-fedora-36/bootstrap b/tests/ansible/vagrant/experimental/wizard-fedora-36/bootstrap new file mode 100755 index 00000000000..b33ad9c881a --- /dev/null +++ b/tests/ansible/vagrant/experimental/wizard-fedora-36/bootstrap @@ -0,0 +1,5 @@ +#!/bin/sh +unset IFS +set -euf + +sudo dnf -y update diff --git a/tests/ansible/vagrant/experimental/wizard-ubuntu-22.04/Vagrantfile b/tests/ansible/vagrant/experimental/wizard-ubuntu-22.04/Vagrantfile new file mode 100644 index 00000000000..0ca270d768e --- /dev/null +++ b/tests/ansible/vagrant/experimental/wizard-ubuntu-22.04/Vagrantfile @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +Vagrant.configure('2') do |config| + config.vm.box = 'generic/ubuntu2204' + config.vm.define 'wizard' + + config.vm.provision 'shell', path: 'bootstrap' + + config.vm.provider :libvirt do |libvirt| + libvirt.cpus = 4 + libvirt.memory = 4096 + end + + config.vm.synced_folder '.', '/vagrant', disabled: true + + # config.vm.provision 'ansible' do |ansible| + # ansible.config_file = '../../ansible.cfg' + # ansible.playbook = '../../run_all.yml' + # end + + # same as above, to run the steps separately + + config.vm.provision 'ansible' do |provdep| + provdep.config_file = '../../../ansible.cfg' + provdep.playbook = '../../../provision_dependencies.yml' + end + + config.vm.provision 'ansible' do |provtest| + provtest.config_file = '../../../ansible.cfg' + provtest.playbook = '../../../provision_test_suite.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../prepare_tests.yml' + end + + # config.vm.provision 'ansible' do |runtests| + # runtests.config_file = '../../../ansible.cfg' + # runtests.playbook = '../../../run_tests.yml' + # end +end diff --git a/tests/ansible/vagrant/experimental/wizard-ubuntu-22.04/bootstrap b/tests/ansible/vagrant/experimental/wizard-ubuntu-22.04/bootstrap new file mode 100755 index 00000000000..6a5df521a60 --- /dev/null +++ b/tests/ansible/vagrant/experimental/wizard-ubuntu-22.04/bootstrap @@ -0,0 +1,5 @@ +#!/bin/sh +unset IFS +set -euf + +sudo apt install -y aptitude diff --git a/tests/bats-detect/WARNING.md b/tests/bats-detect/WARNING.md new file mode 100644 index 00000000000..379261b460b --- /dev/null +++ b/tests/bats-detect/WARNING.md @@ -0,0 +1,8 @@ + +Running the tests in this directory WILL change the system configuration in +unpredictable ways, remove packages and data (with a peculiar appetite for +databases) and possibly bring the system to an unusable state. + +They are meant to be run, as root, on temporary VMs. They are only intended to +ease the development of configurations for "cscli setup detect". + diff --git a/tests/bats-detect/apache2-deb.bats b/tests/bats-detect/apache2-deb.bats new file mode 100644 index 00000000000..2c6e1deaf44 --- /dev/null +++ b/tests/bats-detect/apache2-deb.bats @@ -0,0 +1,49 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + deb-remove apache2 +} + +setup() { + if ! command -v dpkg >/dev/null; then + skip 'not a debian-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "apache2: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'apache2-systemd-deb' + refute_line 'apache2-systemd-rpm' +} + +@test "apache2: install" { + run -0 deb-install apache2 + run -0 sudo systemctl enable apache2.service +} + +@test "apache2: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'apache2-systemd-deb' + refute_line 'apache2-systemd-rpm' +} + +@test "apache2: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/apache2-rpm.bats b/tests/bats-detect/apache2-rpm.bats new file mode 100644 index 00000000000..9b0fda87d9f --- /dev/null +++ b/tests/bats-detect/apache2-rpm.bats @@ -0,0 +1,49 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + rpm-remove httpd +} + +setup() { + if ! command -v dnf >/dev/null; then + skip 'not a redhat-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "apache2: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'apache2-systemd-rpm' + refute_line 'apache2-systemd-deb' +} + +@test "apache2: install" { + run -0 rpm-install httpd + run -0 sudo systemctl enable httpd.service +} + +@test "apache2: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'apache2-systemd-rpm' + refute_line 'apache2-systemd-deb' +} + +@test "apache2: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/asterisk-deb.bats b/tests/bats-detect/asterisk-deb.bats new file mode 100644 index 00000000000..7087cd735d5 --- /dev/null +++ b/tests/bats-detect/asterisk-deb.bats @@ -0,0 +1,47 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + deb-remove asterisk +} + +setup() { + if ! command -v dpkg >/dev/null; then + skip 'not a debian-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "asterisk: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'asterisk-systemd' +} + +@test "asterisk: install" { + run -0 deb-install asterisk + run -0 sudo systemctl enable asterisk.service +} + +@test "asterisk: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'asterisk-systemd' +} + +@test "asterisk: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/asterisk-rpm.bats b/tests/bats-detect/asterisk-rpm.bats new file mode 100644 index 00000000000..081195f2ad5 --- /dev/null +++ b/tests/bats-detect/asterisk-rpm.bats @@ -0,0 +1,50 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + rpm-remove asterisk +} + +setup() { + if ! command -v dnf >/dev/null; then + skip 'not a redhat-like system' + fi + if ! dnf list | grep -q asterisk; then + skip 'asterisk package not available' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "asterisk: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'asterisk-systemd' +} + +@test "asterisk: install" { + run -0 rpm-install asterisk + run -0 sudo systemctl enable asterisk.service +} + +@test "asterisk: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'asterisk-systemd' +} + +@test "asterisk: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/caddy-deb.bats b/tests/bats-detect/caddy-deb.bats new file mode 100644 index 00000000000..b2b35496cc5 --- /dev/null +++ b/tests/bats-detect/caddy-deb.bats @@ -0,0 +1,53 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + deb-remove caddy +} + +setup() { + if ! command -v dpkg >/dev/null; then + skip 'not a debian-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "caddy: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'caddy-systemd' +} + +@test "caddy: install" { + run -0 deb-install debian-keyring debian-archive-keyring apt-transport-https + run -0 curl -1sSLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' + run -0 sudo gpg --yes --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg < <(output) + run -0 curl -1sSLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' + run -0 sudo tee /etc/apt/sources.list.d/caddy-stable.list < <(output) + run -0 deb-update + run -0 deb-install caddy + run -0 sudo systemctl enable caddy.service +} + +@test "caddy: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'caddy-systemd' +} + +@test "caddy: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/caddy-rpm.bats b/tests/bats-detect/caddy-rpm.bats new file mode 100644 index 00000000000..5ac225f2466 --- /dev/null +++ b/tests/bats-detect/caddy-rpm.bats @@ -0,0 +1,49 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + rpm-remove caddy +} + +setup() { + if ! command -v dnf >/dev/null; then + skip 'not a redhat-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "caddy: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'caddy-systemd' +} + +@test "caddy: install" { + run -0 rpm-install 'dnf-command(copr)' + run -0 sudo dnf -q -y copr enable @caddy/caddy + run -0 rpm-install caddy + run -0 sudo systemctl enable caddy.service +} + +@test "caddy: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'caddy-systemd' +} + +@test "caddy: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/dovecot-deb.bats b/tests/bats-detect/dovecot-deb.bats new file mode 100644 index 00000000000..bc14bd4d1c2 --- /dev/null +++ b/tests/bats-detect/dovecot-deb.bats @@ -0,0 +1,47 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + deb-remove dovecot-core +} + +setup() { + if ! command -v dpkg >/dev/null; then + skip 'not a debian-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "dovecot: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'dovecot-systemd' +} + +@test "dovecot: install" { + run -0 deb-install dovecot-core + run -0 sudo systemctl enable dovecot.service +} + +@test "dovecot: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'dovecot-systemd' +} + +@test "dovecot: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/dovecot-rpm.bats b/tests/bats-detect/dovecot-rpm.bats new file mode 100644 index 00000000000..5a17f11a57d --- /dev/null +++ b/tests/bats-detect/dovecot-rpm.bats @@ -0,0 +1,47 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + rpm-remove dovecot +} + +setup() { + if ! command -v dnf >/dev/null; then + skip 'not a redhat-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "dovecot: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'dovecot-systemd' +} + +@test "dovecot: install" { + run -0 rpm-install dovecot + run -0 sudo systemctl enable dovecot.service +} + +@test "dovecot: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'dovecot-systemd' +} + +@test "dovecot: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/emby-deb.bats b/tests/bats-detect/emby-deb.bats new file mode 100644 index 00000000000..9554af304c3 --- /dev/null +++ b/tests/bats-detect/emby-deb.bats @@ -0,0 +1,52 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + deb-remove emby-server +} + +setup() { + if ! command -v dpkg >/dev/null; then + skip 'not a debian-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "emby: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'emby-systemd' +} + +@test "emby: install" { + # https://emby.media/linux-server.html + version=4.7.6.0 + filename="emby-server-deb_${version}_amd64.deb" + # don't download twice + run -0 curl -1sSLf "https://github.com/MediaBrowser/Emby.Releases/releases/download/${version}/${filename}" -o "${CACHEDIR}/${filename}" + run -0 sudo dpkg --install "${CACHEDIR}/${filename}" + run -0 sudo systemctl enable emby-server.service +} + +@test "emby: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'emby-systemd' +} + +@test "emby: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/emby-rpm.bats b/tests/bats-detect/emby-rpm.bats new file mode 100644 index 00000000000..72c9a01cbcb --- /dev/null +++ b/tests/bats-detect/emby-rpm.bats @@ -0,0 +1,52 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + rpm-remove emby-server +} + +setup() { + if ! command -v dnf >/dev/null; then + skip 'not a redhat-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "emby: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'emby-systemd' +} + +@test "emby: install" { + # https://emby.media/linux-server.html + version=4.7.6.0 + filename="emby-server-rpm_${version}_x86_64.rpm" + # don't download twice + run -0 curl -1sSLf "https://github.com/MediaBrowser/Emby.Releases/releases/download/${version}/${filename}" -o "${CACHEDIR}/${filename}" + run -0 rpm-install "${CACHEDIR}/${filename}" + run -0 sudo systemctl enable emby-server.service +} + +@test "emby: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'emby-systemd' +} + +@test "emby: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/endlessh-deb.bats b/tests/bats-detect/endlessh-deb.bats new file mode 100644 index 00000000000..55a8da8ff3e --- /dev/null +++ b/tests/bats-detect/endlessh-deb.bats @@ -0,0 +1,48 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + deb-remove endlessh +} + +setup() { + if ! command -v dpkg >/dev/null; then + skip 'not a debian-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "endlessh: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'endlessh-systemd' +} + +@test "endlessh: install" { + # https://github.com/skeeto/endlessh + run -0 deb-install endlessh + run -0 sudo systemctl enable endlessh.service +} + +@test "endlessh: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'endlessh-systemd' +} + +@test "endlessh: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/endlessh-rpm.bats b/tests/bats-detect/endlessh-rpm.bats new file mode 100644 index 00000000000..812d627bbed --- /dev/null +++ b/tests/bats-detect/endlessh-rpm.bats @@ -0,0 +1,48 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + rpm-remove endlessh +} + +setup() { + if ! command -v dnf >/dev/null; then + skip 'not a redhat-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "endlessh: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'endlessh-systemd' +} + +@test "endlessh: install" { + # https://github.com/skeeto/endlessh + run -0 rpm-install endlessh + run -0 sudo systemctl enable endlessh.service +} + +@test "endlessh: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'endlessh-systemd' +} + +@test "endlessh: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/gitea.bats b/tests/bats-detect/gitea.bats new file mode 100644 index 00000000000..b2e094ed684 --- /dev/null +++ b/tests/bats-detect/gitea.bats @@ -0,0 +1,46 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + skip 'WIP' + ./instance-data load +} + +#---------- + +@test "gitea: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'gitea-systemd' +} + +@test "gitea: install" { + # https://docs.gitea.io/en-us/install-from-binary/#download + version=1.16.9 + # don't download twice + run -0 wget -nc --directory-prefix "$CACHEDIR" "https://dl.gitea.io/gitea/${version}/gitea-${version}-linux-amd64" +} + +@test "gitea: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'gitea-systemd' +} + +@test "gitea: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/haproxy-deb.bats b/tests/bats-detect/haproxy-deb.bats new file mode 100644 index 00000000000..173ff5fcb12 --- /dev/null +++ b/tests/bats-detect/haproxy-deb.bats @@ -0,0 +1,47 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + deb-remove haproxy +} + +setup() { + if ! command -v dpkg >/dev/null; then + skip 'not a debian-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "haproxy: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'haproxy-systemd' +} + +@test "haproxy: install" { + run -0 deb-install haproxy + run -0 sudo systemctl enable haproxy.service +} + +@test "haproxy: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'haproxy-systemd' +} + +@test "haproxy: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/haproxy-rpm.bats b/tests/bats-detect/haproxy-rpm.bats new file mode 100644 index 00000000000..d29aeb9df3b --- /dev/null +++ b/tests/bats-detect/haproxy-rpm.bats @@ -0,0 +1,47 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + rpm-remove haproxy +} + +setup() { + if ! command -v dnf >/dev/null; then + skip 'not a redhat-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "haproxy: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'haproxy-systemd' +} + +@test "haproxy: install" { + run -0 rpm-install haproxy + run -0 sudo systemctl enable haproxy.service +} + +@test "haproxy: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'haproxy-systemd' +} + +@test "haproxy: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/lemonldap-deb.bats b/tests/bats-detect/lemonldap-deb.bats new file mode 100644 index 00000000000..c77c0ae2054 --- /dev/null +++ b/tests/bats-detect/lemonldap-deb.bats @@ -0,0 +1,47 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + deb-remove lemonldap-ng +} + +setup() { + if ! command -v dpkg >/dev/null; then + skip 'not a debian-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "lemonldap: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'lemonldap-ng-systemd' +} + +@test "lemonldap: install" { + run -0 deb-install lemonldap-ng + run -0 sudo systemctl enable lemonldap-ng-fastcgi-server.service +} + +@test "lemonldap: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'lemonldap-ng-systemd' +} + +@test "lemonldap: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/lemonldap-rpm.bats b/tests/bats-detect/lemonldap-rpm.bats new file mode 100644 index 00000000000..319c7c55ebd --- /dev/null +++ b/tests/bats-detect/lemonldap-rpm.bats @@ -0,0 +1,50 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + rpm-remove lemonldap-ng +} + +setup() { + if ! command -v dnf >/dev/null; then + skip 'not a redhat-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load + skip 'WIP' +} + +#---------- + +@test "lemonldap: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'lemonldap-ng-systemd' +} + +@test "lemonldap: install" { + run -0 rpm-install 'dnf-command(copr)' + run -0 sudo dnf -q -y copr enable xavierb/lemonldap-ng + run -0 rpm-install lemonldap-ng + run -0 sudo systemctl enable lemonldap-ng-fastcgi-server.service +} + +@test "lemonldap: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'lemonldap-ng-systemd' +} + +@test "lemonldap: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/lib/setup_file_detect.sh b/tests/bats-detect/lib/setup_file_detect.sh new file mode 100755 index 00000000000..f644962a5c8 --- /dev/null +++ b/tests/bats-detect/lib/setup_file_detect.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +TESTDATA="${BATS_TEST_DIRNAME}/testdata" +export TESTDATA + +CACHEDIR="${TESTDATA}/.cache" +export CACHEDIR + +mkdir -p "${CACHEDIR}" + +DEBIAN_FRONTEND=noninteractive +export DEBIAN_FRONTEND + +# avoid warnings in stderr, especially from perl modules +LC_ALL=C +export LC_ALL + +deb-install() { + # use aptitude to reliably purge dependencies too + sudo aptitude install "$@" -yq >/dev/null + # this does not work well enough + # sudo apt-get -qq -y -o Dpkg:Use-Pty=0 install "$@" >/dev/null + # sudo apt-mark auto "$@" +} +export -f deb-install + +deb-update() { + sudo apt-get -qq -y -o Dpkg:Use-Pty=0 update +} +export -f deb-update + +deb-remove() { + for pkg in "$@"; do + if dpkg -s "${pkg}" >/dev/null 2>&1; then + # use aptitude to reliably purge dependencies too + sudo aptitude purge "${pkg}" -yq >/dev/null + # this does not work well enough + # sudo apt-get -qq -y purge --auto-remove "${pkg}" >/dev/null + fi + done +} +export -f deb-remove + +rpm-install() { + sudo dnf -q -y install "$@" +} +export -f rpm-install + +rpm-remove() { + # don't fail if dnf does not exist (teardown is called on deb distros too) + if command -v dnf >/dev/null; then + sudo dnf -q -y remove "$@" >/dev/null + fi +} +export -f rpm-remove diff --git a/tests/bats-detect/litespeed.bats b/tests/bats-detect/litespeed.bats new file mode 100644 index 00000000000..ee6ba205f2d --- /dev/null +++ b/tests/bats-detect/litespeed.bats @@ -0,0 +1,47 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + deb-remove openlitespeed +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + skip 'WIP' + ./instance-data load +} + +#---------- + +@test "openlitespeed: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'litespeed-systemd' +} + +@test "openlitespeed: install" { + run -0 sudo "${TESTDATA}/enable_lst_debian_repo.sh" + run -0 deb-update + run -0 deb-install openlitespeed + # run -0 sudo systemctl enable XXX TODO +} + +@test "litespeed: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'litespeed-systemd' +} + +@test "litespeed: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/mariadb-deb.bats b/tests/bats-detect/mariadb-deb.bats new file mode 100644 index 00000000000..1d3546f4edc --- /dev/null +++ b/tests/bats-detect/mariadb-deb.bats @@ -0,0 +1,47 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + deb-remove mariadb-server +} + +setup() { + if ! command -v dpkg >/dev/null; then + skip 'not a debian-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "mariadb: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'mariadb-systemd' +} + +@test "mariadb: install" { + run -0 deb-install mariadb-server + run -0 sudo systemctl enable mariadb.service +} + +@test "mariadb: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'mariadb-systemd' +} + +@test "mariadb: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/mariadb-rpm.bats b/tests/bats-detect/mariadb-rpm.bats new file mode 100644 index 00000000000..54365d179db --- /dev/null +++ b/tests/bats-detect/mariadb-rpm.bats @@ -0,0 +1,47 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + rpm-remove mariadb-server +} + +setup() { + if ! command -v dnf >/dev/null; then + skip 'not a redhat-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "mariadb: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'mariadb-systemd' +} + +@test "mariadb: install" { + run -0 rpm-install mariadb-server + run -0 sudo systemctl enable mariadb.service +} + +@test "mariadb: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'mariadb-systemd' +} + +@test "mariadb: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/mysql-deb.bats b/tests/bats-detect/mysql-deb.bats new file mode 100644 index 00000000000..a12ea025d2f --- /dev/null +++ b/tests/bats-detect/mysql-deb.bats @@ -0,0 +1,64 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + # debian: mysql-community-server + # ubuntu: mysql-server + deb-remove mysql-server mysql-community-server +} + +setup() { + if ! command -v dpkg >/dev/null; then + skip 'not a debian-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load + if apt-cache search --names-only "^mysql-server$"; then + skip "mysql-server package not available" + fi +} + +#---------- + +@test "mysql: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'mysql-systemd' +} + +@test "mysql: install" { + # ubuntu comes with mysql, debian does not + if apt-cache search --names-only "^mysql-server$"; then + # package not available, install the repo + filename="mysql-apt-config_0.8.23-1_all.deb" + run -0 curl -1sSLf "https://dev.mysql.com/get/${filename}" -o "${CACHEDIR}/${filename}" + # XXX md5 c2b410031867dc7c966ca5b1aa0c72aa + run -0 sudo dpkg --install "${CACHEDIR}/${filename}" + run -0 deb-update + # XXX this hangs + run -0 deb-install mysql-community-server + else + run -0 deb-install mysql-server + fi + run -0 sudo systemctl enable mysql.service +} + +@test "mysql: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'mysql-systemd' +} + +@test "mysql: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/mysql-rpm.bats b/tests/bats-detect/mysql-rpm.bats new file mode 100644 index 00000000000..0ab911010cb --- /dev/null +++ b/tests/bats-detect/mysql-rpm.bats @@ -0,0 +1,48 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + rpm-remove mysql-server +} + +setup() { + if ! command -v dnf >/dev/null; then + skip 'not a redhat-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + skip 'WIP' + ./instance-data load +} + +#---------- + +@test "mysql: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'mysql-systemd' +} + +@test "mysql: install" { + run -0 rpm-install mysql-server + run -0 sudo systemctl enable mysql.service +} + +@test "mysql: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'mysql-systemd' +} + +@test "mysql: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/nginx-deb.bats b/tests/bats-detect/nginx-deb.bats new file mode 100644 index 00000000000..b269a6bc35a --- /dev/null +++ b/tests/bats-detect/nginx-deb.bats @@ -0,0 +1,47 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + deb-remove nginx +} + +setup() { + if ! command -v dpkg >/dev/null; then + skip 'not a debian-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "nginx: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'nginx-systemd' +} + +@test "nginx: install" { + run -0 deb-install nginx + run -0 sudo systemctl enable nginx.service +} + +@test "nginx: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'nginx-systemd' +} + +@test "nginx: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/nginx-rpm.bats b/tests/bats-detect/nginx-rpm.bats new file mode 100644 index 00000000000..f2c93bb610f --- /dev/null +++ b/tests/bats-detect/nginx-rpm.bats @@ -0,0 +1,47 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + rpm-remove nginx +} + +setup() { + if ! command -v dnf >/dev/null; then + skip 'not a redhat-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "nginx: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'nginx-systemd' +} + +@test "nginx: install" { + run -0 rpm-install nginx + run -0 sudo systemctl enable nginx.service +} + +@test "nginx: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'nginx-systemd' +} + +@test "nginx: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/odoo-deb.bats b/tests/bats-detect/odoo-deb.bats new file mode 100644 index 00000000000..e57e53d8308 --- /dev/null +++ b/tests/bats-detect/odoo-deb.bats @@ -0,0 +1,52 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + deb-remove odoo +} + +setup() { + if ! command -v dpkg >/dev/null; then + skip 'not a debian-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "odoo: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'odoo-systemd' +} + +@test "odoo: install" { + run -0 deb-install debian-keyring debian-archive-keyring apt-transport-https + run -0 curl -1sSLf https://nightly.odoo.com/odoo.key + run -0 sudo gpg --yes --dearmor -o /usr/share/keyrings/odoo-keyring.gpg < <(output) + run -0 sudo tee <<< "deb [signed-by=/usr/share/keyrings/odoo-keyring.gpg] https://nightly.odoo.com/15.0/nightly/deb/ ./" /etc/apt/sources.list.d/odoo.list >/dev/null + run -0 deb-update + run -0 deb-install odoo +# run -0 sudo systemctl enable caddy.service +} + +@test "odoo: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'odoo-systemd' +} + +@test "odoo: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/odoo-rpm.bats b/tests/bats-detect/odoo-rpm.bats new file mode 100644 index 00000000000..e2577daeb35 --- /dev/null +++ b/tests/bats-detect/odoo-rpm.bats @@ -0,0 +1,49 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + deb-remove odoo +} + +setup() { + if ! command -v dnf >/dev/null; then + skip 'not a redhat-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + skip 'WIP (https://bytemeta.vip/repo/odoo/odoo/issues/95168)' + ./instance-data load +} + +#---------- + +@test "odoo: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'odoo-systemd' +} + +@test "odoo: install" { + run -0 sudo dnf config-manager --add-repo=https://nightly.odoo.com/15.0/nightly/rpm/odoo.repo + run -0 rpm-install odoo + run -0 sudo systemctl enable odoo +} + +@test "odoo: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'odoo-systemd' +} + +@test "odoo: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/ombi-deb.bats b/tests/bats-detect/ombi-deb.bats new file mode 100644 index 00000000000..d02c0d89ea4 --- /dev/null +++ b/tests/bats-detect/ombi-deb.bats @@ -0,0 +1,52 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + deb-remove ombi +} + +setup() { + if ! command -v dpkg >/dev/null; then + skip 'not a debian-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "ombi: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'ombi-systemd' +} + +@test "ombi: install" { + run -0 deb-install debian-keyring debian-archive-keyring apt-transport-https + run -0 curl -1sSLf https://apt.ombi.app/pub.key + run -0 sudo gpg --yes --dearmor -o /usr/share/keyrings/ombi-keyring.gpg < <(output) + run -0 sudo tee <<< "deb [signed-by=/usr/share/keyrings/ombi-keyring.gpg] https://apt.ombi.app/develop jessie main" /etc/apt/sources.list.d/ombi.list >/dev/null + run -0 deb-update + run -0 deb-install ombi + run -0 sudo systemctl enable ombi.service +} + +@test "ombi: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'ombi-systemd' +} + +@test "ombi: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/openresty-deb.bats b/tests/bats-detect/openresty-deb.bats new file mode 100644 index 00000000000..c1e91949df3 --- /dev/null +++ b/tests/bats-detect/openresty-deb.bats @@ -0,0 +1,57 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + deb-remove openresty +} + +setup() { + if ! command -v dpkg >/dev/null; then + skip 'not a debian-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "openresty: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'openresty-systemd' +} + +@test "openresty: install" { + run -0 deb-install debian-keyring debian-archive-keyring apt-transport-https + run -0 curl -1sSLf 'https://openresty.org/package/pubkey.gpg' + if [[ "$(lsb_release -is)" == "Ubuntu" ]]; then + run -0 sudo gpg --yes --dearmor -o /usr/share/keyrings/openresty.gpg < <(output) + run -0 sudo tee <<< "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/openresty.gpg] http://openresty.org/package/ubuntu $(lsb_release -sc) main" /etc/apt/sources.list.d/openresty.list + else + run -0 sudo apt-key add - < <(output) + run -0 sudo tee <<< "deb http://openresty.org/package/debian $(lsb_release -sc) openresty" /etc/apt/sources.list.d/openresty.list + fi + run -0 deb-update + run -0 deb-install openresty + run -0 sudo systemctl enable openresty.service +} + +@test "openresty: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'openresty-systemd' +} + +@test "openresty: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/openresty-rpm.bats b/tests/bats-detect/openresty-rpm.bats new file mode 100644 index 00000000000..6fc0a8a0975 --- /dev/null +++ b/tests/bats-detect/openresty-rpm.bats @@ -0,0 +1,54 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + rpm-remove openresty +} + +setup() { + if ! command -v dnf >/dev/null; then + skip 'not a redhat-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "openresty: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'openresty-systemd' +} + +@test "openresty: install" { + run -0 rpm-install redhat-lsb-core + if [[ "$(lsb_release -is)" == "Fedora" ]]; then + run -0 sudo curl -1sSLf "https://openresty.org/package/fedora/openresty.repo" -o "/etc/yum.repos.d/openresty.repo" + elif [[ "$(lsb_release -is)" == "CentOS" ]]; then + run -0 sudo curl -1sSLf "https://openresty.org/package/centos/openresty.repo" -o "/etc/yum.repos.d/openresty.repo" + fi + run -0 sudo dnf check-update + run -0 rpm-install openresty + run -0 sudo systemctl enable openresty.service +} + +@test "openresty: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'openresty-systemd' +} + +@test "openresty: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/pgsql-deb.bats b/tests/bats-detect/pgsql-deb.bats new file mode 100644 index 00000000000..a8781968ac2 --- /dev/null +++ b/tests/bats-detect/pgsql-deb.bats @@ -0,0 +1,61 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +cleanup() { + command -v dpkg >/dev/null || return 0 + # sudo systemctl stop postgresql.service || : + # remove the DB to avoid a prompt from postrm + if [[ -d /var/lib/postgresql ]]; then + # shellcheck disable=SC2045 + for cluster in $(ls /var/lib/postgresql 2>/dev/null); do + sudo pg_dropcluster --stop "${cluster}" main + done + fi + deb-remove postgresql $(dpkg -l | grep postgres | awk '{print $2}') +} + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" + cleanup +} + +teardown_file() { + load "../lib/teardown_file.sh" + cleanup +} + +setup() { + if ! command -v dpkg >/dev/null; then + skip 'not a debian-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "pgsql: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'pgsql-systemd-deb' +} + +@test "pgsql: install" { + run -0 deb-install postgresql + run -0 sudo systemctl enable postgresql.service +} + +@test "pgsql: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'pgsql-systemd-deb' +} + +@test "pgsql: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/pgsql-rpm.bats b/tests/bats-detect/pgsql-rpm.bats new file mode 100644 index 00000000000..b2fba8af47d --- /dev/null +++ b/tests/bats-detect/pgsql-rpm.bats @@ -0,0 +1,51 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + rpm-remove postgresql-server +} + +setup() { + if ! command -v dnf >/dev/null; then + skip 'not a redhat-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "pgsql: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'pgsql-systemd-rpm' +} + +@test "pgsql: install" { + run -0 rpm-install postgresql-server + # for centos 8, we need to create the cluster + if ! sudo bash -c 'stat /var/lib/pgsql/data/*'; then + sudo /usr/bin/postgresql-setup --initdb + fi + run -0 sudo systemctl enable postgresql.service +} + +@test "pgsql: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'pgsql-systemd-rpm' +} + +@test "pgsql: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/postfix-deb.bats b/tests/bats-detect/postfix-deb.bats new file mode 100644 index 00000000000..c67285d75a1 --- /dev/null +++ b/tests/bats-detect/postfix-deb.bats @@ -0,0 +1,49 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + deb-remove postfix +} + +setup() { + if ! command -v dpkg >/dev/null; then + skip 'not a debian-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "postfix: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'postfix-systemd' +} + +@test "postfix: install" { + run -0 sudo debconf-set-selections <<< "postfix postfix/mailname string hostname.example.com" + run -0 sudo debconf-set-selections <<< "postfix postfix/main_mailer_type string 'Internet Site'" + run -0 deb-install postfix + run -0 sudo systemctl enable postfix.service +} + +@test "postfix: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'postfix-systemd' +} + +@test "postfix: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/postfix-rpm.bats b/tests/bats-detect/postfix-rpm.bats new file mode 100644 index 00000000000..dc6b42a63a1 --- /dev/null +++ b/tests/bats-detect/postfix-rpm.bats @@ -0,0 +1,47 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + rpm-remove postfix +} + +setup() { + if ! command -v dnf >/dev/null; then + skip 'not a redhat-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "postfix: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'postfix-systemd' +} + +@test "postfix: install" { + run -0 rpm-install postfix + run -0 sudo systemctl enable postfix.service +} + +@test "postfix: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'postfix-systemd' +} + +@test "postfix: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/proftpd-deb.bats b/tests/bats-detect/proftpd-deb.bats new file mode 100644 index 00000000000..b21ea466d8d --- /dev/null +++ b/tests/bats-detect/proftpd-deb.bats @@ -0,0 +1,47 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + deb-remove proftpd +} + +setup() { + if ! command -v dpkg >/dev/null; then + skip 'not a debian-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "proftpd: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'proftpd-systemd' +} + +@test "proftpd: install" { + run -0 deb-install proftpd + run -0 sudo systemctl enable proftpd.service +} + +@test "proftpd: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'proftpd-systemd' +} + +@test "proftpd: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/proftpd-rpm.bats b/tests/bats-detect/proftpd-rpm.bats new file mode 100644 index 00000000000..2c9df2b5545 --- /dev/null +++ b/tests/bats-detect/proftpd-rpm.bats @@ -0,0 +1,47 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + rpm-remove proftpd +} + +setup() { + if ! command -v dnf >/dev/null; then + skip 'not a redhat-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "proftpd: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'proftpd-systemd' +} + +@test "proftpd: install" { + run -0 rpm-install proftpd + run -0 sudo systemctl enable proftpd.service +} + +@test "proftpd: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'proftpd-systemd' +} + +@test "proftpd: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/proxmox-deb.bats b/tests/bats-detect/proxmox-deb.bats new file mode 100644 index 00000000000..ae02375c381 --- /dev/null +++ b/tests/bats-detect/proxmox-deb.bats @@ -0,0 +1,62 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + deb-remove proxmox-ve +} + +setup() { + if ! command -v dpkg >/dev/null; then + skip 'not a debian-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load + + . /etc/os-release + case "$VERSION_CODENAME" in + bullseye | buster | jessie | squeeze | stretch | wheezy) + skip "the installation does not work" + ;; + *) + skip "unsupported distribution" + ;; + esac + export VERSION_CODENAME +} + +#---------- + +@test "proxmox: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'proxmox-systemd' +} + +@test "proxmox: install" { + run -0 deb-install debian-keyring debian-archive-keyring apt-transport-https + run -0 sudo curl -1sSLf http://download.proxmox.com/debian/proxmox-ve-release-6.x.gpg -o /etc/apt/trusted.gpg.d/proxmox-ve-release-6.x.gpg + run -0 sudo tee <<<"deb http://download.proxmox.com/debian/pve ${VERSION_CODENAME} pve-no-subscription" /etc/apt/sources.list.d/proxmox.list >/dev/null + run -0 deb-update + run -0 deb-install proxmox-ve + run -0 sudo systemctl enable proxmox.service +} + +@test "proxmox: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'proxmox-systemd' +} + +@test "proxmox: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/pureftpd-deb.bats b/tests/bats-detect/pureftpd-deb.bats new file mode 100644 index 00000000000..3b7aa68c9e4 --- /dev/null +++ b/tests/bats-detect/pureftpd-deb.bats @@ -0,0 +1,47 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + deb-remove pure-ftpd +} + +setup() { + if ! command -v dpkg >/dev/null; then + skip 'not a debian-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "pureftpd: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'pureftpd-systemd' +} + +@test "pureftpd: install" { + run -0 deb-install pure-ftpd + run -0 sudo systemctl enable pure-ftpd.service +} + +@test "pureftpd: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'pureftpd-systemd' +} + +@test "pureftpd: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/pureftpd-rpm.bats b/tests/bats-detect/pureftpd-rpm.bats new file mode 100644 index 00000000000..d4d0b7ad9f5 --- /dev/null +++ b/tests/bats-detect/pureftpd-rpm.bats @@ -0,0 +1,47 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + rpm-remove pure-ftpd +} + +setup() { + if ! command -v dnf >/dev/null; then + skip 'not a redhat-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "pureftpd: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'pureftpd-systemd' +} + +@test "pureftpd: install" { + run -0 rpm-install pure-ftpd + run -0 sudo systemctl enable pure-ftpd.service +} + +@test "pureftpd: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'pureftpd-systemd' +} + +@test "pureftpd: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/smb-deb.bats b/tests/bats-detect/smb-deb.bats new file mode 100644 index 00000000000..bb0ec5156e3 --- /dev/null +++ b/tests/bats-detect/smb-deb.bats @@ -0,0 +1,50 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + deb-remove samba +} + +setup() { + if ! command -v dpkg >/dev/null; then + skip 'not a debian-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "smb: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'smb-systemd' +} + +@test "smb: install" { + run -0 sudo debconf-set-selections <<< "samba-common samba-common/workgroup string WORKGROUP" + run -0 sudo debconf-set-selections <<< "samba-common samba-common/dhcp boolean true" + run -0 sudo debconf-set-selections <<< "samba-common samba-common/do_debconf boolean true" + run -0 deb-install samba + run -0 sudo systemctl enable smbd.service +} + +@test "smb: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'smb-systemd' +} + +@test "smb: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/smb-rpm.bats b/tests/bats-detect/smb-rpm.bats new file mode 100644 index 00000000000..1866f540cfe --- /dev/null +++ b/tests/bats-detect/smb-rpm.bats @@ -0,0 +1,47 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + rpm-remove samba +} + +setup() { + if ! command -v dnf >/dev/null; then + skip 'not a redhat-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "smb: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'smb-systemd' +} + +@test "smb: install" { + run -0 rpm-install samba + run -0 sudo systemctl enable smb.service +} + +@test "smb: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'smb-systemd' +} + +@test "smb: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/sshd-deb.bats b/tests/bats-detect/sshd-deb.bats new file mode 100644 index 00000000000..32a363d46c7 --- /dev/null +++ b/tests/bats-detect/sshd-deb.bats @@ -0,0 +1,49 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + # don't remove ssh here, we assume it's needed +} + +setup() { + if ! command -v dpkg >/dev/null; then + skip 'not a debian-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "sshd: detect unit (fail)" { + run -0 sudo systemctl mask ssh.service + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'sshd-systemd' +} + +@test "sshd: install" { +# run -0 deb-install openssh-server + run -0 sudo systemctl unmask ssh.service + run -0 sudo systemctl enable ssh.service +} + +@test "sshd: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'sshd-systemd' +} + +@test "sshd: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/sshd-rpm.bats b/tests/bats-detect/sshd-rpm.bats new file mode 100644 index 00000000000..f6e0d5be1d1 --- /dev/null +++ b/tests/bats-detect/sshd-rpm.bats @@ -0,0 +1,49 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + # don't remove ssh here, we assume it's needed +} + +setup() { + if ! command -v dnf >/dev/null; then + skip 'not a redhat-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "sshd: detect unit (fail)" { + run -0 sudo systemctl mask sshd.service + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'sshd-systemd' +} + +@test "sshd: install" { +# run -0 rpm-install openssh-server + run -0 sudo systemctl unmask sshd.service + run -0 sudo systemctl enable sshd.service +} + +@test "sshd: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'sshd-systemd' +} + +@test "sshd: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/suricata-deb.bats b/tests/bats-detect/suricata-deb.bats new file mode 100644 index 00000000000..13207b35696 --- /dev/null +++ b/tests/bats-detect/suricata-deb.bats @@ -0,0 +1,47 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + deb-remove suricata +} + +setup() { + if ! command -v dpkg >/dev/null; then + skip 'not a debian-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "suricata: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'suricata-systemd' +} + +@test "suricata: install" { + run -0 deb-install suricata + run -0 sudo systemctl enable suricata.service +} + +@test "suricata: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'suricata-systemd' +} + +@test "suricata: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/suricata-rpm.bats b/tests/bats-detect/suricata-rpm.bats new file mode 100644 index 00000000000..c3c48036f7e --- /dev/null +++ b/tests/bats-detect/suricata-rpm.bats @@ -0,0 +1,47 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + rpm-remove suricata +} + +setup() { + if ! command -v dnf >/dev/null; then + skip 'not a redhat-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "suricata: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'suricata-systemd' +} + +@test "suricata: install" { + run -0 rpm-install suricata + run -0 sudo systemctl enable suricata.service +} + +@test "suricata: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'suricata-systemd' +} + +@test "suricata: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/testdata/enable_lst_debian_repo.sh b/tests/bats-detect/testdata/enable_lst_debian_repo.sh new file mode 100755 index 00000000000..4731907b6f9 --- /dev/null +++ b/tests/bats-detect/testdata/enable_lst_debian_repo.sh @@ -0,0 +1,65 @@ +#!/bin/bash + + +if [ -r /etc/os-release ]; then + + echo " detecting OS type : " + + . /etc/os-release + + if [ $ID == "debian" ]; then + echo "detected OS: $ID - $VERSION_ID" + echo " now enable the LiteSpeed Debian Repo " + if [ $VERSION_ID == "11" ]; then + echo "deb http://rpms.litespeedtech.com/debian/ bullseye main" > /etc/apt/sources.list.d/lst_debian_repo.list + echo "#deb http://rpms.litespeedtech.com/edge/debian/ bullseye main" >> /etc/apt/sources.list.d/lst_debian_repo.list + elif [ $VERSION_ID == "10" ]; then + echo "deb http://rpms.litespeedtech.com/debian/ buster main" > /etc/apt/sources.list.d/lst_debian_repo.list + echo "#deb http://rpms.litespeedtech.com/edge/debian/ buster main" >> /etc/apt/sources.list.d/lst_debian_repo.list + elif [ $VERSION_ID == "9" ]; then + echo "deb http://rpms.litespeedtech.com/debian/ stretch main" > /etc/apt/sources.list.d/lst_debian_repo.list + echo "#deb http://rpms.litespeedtech.com/edge/debian/ stretch main" >> /etc/apt/sources.list.d/lst_debian_repo.list + elif [ $VERSION_ID == "8" ]; then + echo "deb http://rpms.litespeedtech.com/debian/ jessie main" > /etc/apt/sources.list.d/lst_debian_repo.list + echo "#deb http://rpms.litespeedtech.com/edge/debian/ jessie main" >> /etc/apt/sources.list.d/lst_debian_repo.list + fi + elif [ $ID == "ubuntu" ]; then + echo "detected OS: $ID - $VERSION_ID" + echo " now enable the LiteSpeed Debian Repo " + if [ `echo "$VERSION_ID" | cut -b-2 ` == "14" ]; then + echo "deb http://rpms.litespeedtech.com/debian/ trusty main" > /etc/apt/sources.list.d/lst_debian_repo.list + echo "#deb http://rpms.litespeedtech.com/edge/debian/ trusty main" >> /etc/apt/sources.list.d/lst_debian_repo.list + elif [ `echo "$VERSION_ID" | cut -b-2 ` == "12" ]; then + echo "deb http://rpms.litespeedtech.com/debian/ precise main" > /etc/apt/sources.list.d/lst_debian_repo.list + echo "#deb http://rpms.litespeedtech.com/edge/debian/ precise main" >> /etc/apt/sources.list.d/lst_debian_repo.list + elif [ `echo "$VERSION_ID" | cut -b-2 ` == "16" ]; then + echo "deb http://rpms.litespeedtech.com/debian/ xenial main" > /etc/apt/sources.list.d/lst_debian_repo.list + echo "#deb http://rpms.litespeedtech.com/edge/debian/ xenial main" >> /etc/apt/sources.list.d/lst_debian_repo.list + elif [ `echo "$VERSION_ID" | cut -b-2 ` == "18" ]; then + echo "deb http://rpms.litespeedtech.com/debian/ bionic main" > /etc/apt/sources.list.d/lst_debian_repo.list + echo "#deb http://rpms.litespeedtech.com/edge/debian/ bionic main" >> /etc/apt/sources.list.d/lst_debian_repo.list + elif [ `echo "$VERSION_ID" | cut -b-2 ` == "20" ]; then + echo "deb http://rpms.litespeedtech.com/debian/ focal main" > /etc/apt/sources.list.d/lst_debian_repo.list + echo "#deb http://rpms.litespeedtech.com/edge/debian/ focal main" >> /etc/apt/sources.list.d/lst_debian_repo.list + elif [ `echo "$VERSION_ID" | cut -b-2 ` == "22" ]; then + echo "deb http://rpms.litespeedtech.com/debian/ focal main" > /etc/apt/sources.list.d/lst_debian_repo.list + echo "#deb http://rpms.litespeedtech.com/edge/debian/ focal main" >> /etc/apt/sources.list.d/lst_debian_repo.list + fi + else + echo " This distribution is not currently supported by LST repo " + echo " If you really have the needs please contact LiteSpeed for support " + fi +else + echo " The /etc/os-release file doesn't exist " + echo " This script couldn't determine which distribution of the repo should be enabled " + echo " Please consult LiteSpeed Customer Support for further assistance " +fi + +echo " register LiteSpeed GPG key " +wget -O /etc/apt/trusted.gpg.d/lst_debian_repo.gpg http://rpms.litespeedtech.com/debian/lst_debian_repo.gpg +wget -O /etc/apt/trusted.gpg.d/lst_repo.gpg http://rpms.litespeedtech.com/debian/lst_repo.gpg + +echo " update the repo " +apt-get update + +echo " All done, congratulations and enjoy ! " diff --git a/tests/bats-detect/vsftpd-deb.bats b/tests/bats-detect/vsftpd-deb.bats new file mode 100644 index 00000000000..f0cd4f73371 --- /dev/null +++ b/tests/bats-detect/vsftpd-deb.bats @@ -0,0 +1,48 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + sudo systemctl stop vsftpd.service 2>/dev/null || : + deb-remove vsftpd +} + +setup() { + if ! command -v dpkg >/dev/null; then + skip 'not a debian-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "vsftpd: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'vsftpd-systemd' +} + +@test "vsftpd: install" { + run -0 deb-install vsftpd + run -0 sudo systemctl enable vsftpd.service +} + +@test "vsftpd: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'vsftpd-systemd' +} + +@test "vsftpd: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats-detect/vsftpd-rpm.bats b/tests/bats-detect/vsftpd-rpm.bats new file mode 100644 index 00000000000..54b1fd99a8c --- /dev/null +++ b/tests/bats-detect/vsftpd-rpm.bats @@ -0,0 +1,47 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + load "${BATS_TEST_DIRNAME}/lib/setup_file_detect.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" + rpm-remove vsftpd +} + +setup() { + if ! command -v dnf >/dev/null; then + skip 'not a redhat-like system' + fi + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load +} + +#---------- + +@test "vsftpd: detect unit (fail)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + refute_line 'vsftpd-systemd' +} + +@test "vsftpd: install" { + run -0 rpm-install vsftpd + run -0 sudo systemctl enable vsftpd.service +} + +@test "vsftpd: detect unit (succeed)" { + run -0 cscli setup detect + run -0 jq -r '.setup | .[].detected_service' <(output) + assert_line 'vsftpd-systemd' +} + +@test "vsftpd: install detected collection" { + run -0 cscli setup detect + run -0 cscli setup install-hub <(output) +} diff --git a/tests/bats/07_setup.bats b/tests/bats/07_setup.bats new file mode 100644 index 00000000000..e5524b042d5 --- /dev/null +++ b/tests/bats/07_setup.bats @@ -0,0 +1,816 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + ./instance-data load + HUB_DIR=$(config_get '.config_paths.hub_dir') + export HUB_DIR + DETECT_YAML="${HUB_DIR}/detect.yaml" + export DETECT_YAML + # shellcheck disable=SC2154 + TESTDATA="${BATS_TEST_DIRNAME}/testdata/07_setup" + export TESTDATA +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + load "../lib/bats-mock/load.bash" + ./instance-data load +} + +teardown() { + ./instance-crowdsec stop +} + +#---------- + +#shellcheck disable=SC2154 +@test "cscli setup" { + run -0 cscli help + assert_line --regexp '^ +setup +Tools to configure crowdsec$' + + run -0 cscli setup --help + assert_line 'Usage:' + assert_line ' cscli setup [command]' + assert_line 'Manage hub configuration and service detection' + assert_line --partial "detect detect running services, generate a setup file" + assert_line --partial "datasources generate datasource (acquisition) configuration from a setup file" + assert_line --partial "install-hub install items from a setup file" + assert_line --partial "validate validate a setup file" + + # cobra should return error for non-existing sub-subcommands, but doesn't + run -0 cscli setup blahblah + assert_line 'Usage:' +} + +@test "cscli setup detect --help; --detect-config" { + run -0 cscli setup detect --help + assert_line --regexp "detect running services, generate a setup file" + assert_line 'Usage:' + assert_line ' cscli setup detect [flags]' + assert_line --partial "--detect-config string path to service detection configuration (default \"${HUB_DIR}/detect.yaml\")" + assert_line --partial "--force-process strings force detection of a running process (can be repeated)" + assert_line --partial "--force-unit strings force detection of a systemd unit (can be repeated)" + assert_line --partial "--list-supported-services do not detect; only print supported services" + assert_line --partial "--force-os-family string override OS.Family: one of linux, freebsd, windows or darwin" + assert_line --partial "--force-os-id string override OS.ID=[debian | ubuntu | , redhat...]" + assert_line --partial "--force-os-version string override OS.RawVersion (of OS or Linux distribution)" + assert_line --partial "--skip-service strings ignore a service, don't recommend hub/datasources (can be repeated)" + + run -1 --separate-stderr cscli setup detect --detect-config /path/does/not/exist + assert_stderr --partial "detecting services: while reading file: open /path/does/not/exist: no such file or directory" + + # rm -f "${HUB_DIR}/detect.yaml" +} + +@test "cscli setup detect (linux), --skip-service" { + [[ ${OSTYPE} =~ linux.* ]] || skip + tempfile=$(TMPDIR="$BATS_TEST_TMPDIR" mktemp) + cat <<-EOT >"${tempfile}" + version: 1.0 + detect: + linux: + when: + - OS.Family == "linux" + install: + collections: + - crowdsecurity/linux + thewiz: + when: + - OS.Family != "linux" + foobarbaz: + EOT + + run -0 cscli setup detect --detect-config "$tempfile" + assert_json '{setup:[{detected_service:"foobarbaz"},{detected_service:"linux",install:{collections:["crowdsecurity/linux"]}}]}' + + run -0 cscli setup detect --detect-config "$tempfile" --skip-service linux + assert_json '{setup:[{detected_service:"foobarbaz"}]}' +} + +@test "cscli setup detect --force-os-*" { + run -0 cscli setup detect --force-os-family linux --detect-config "${TESTDATA}/detect.yaml" + run -0 jq -cS '.setup[] | select(.detected_service=="linux")' <(output) + assert_json '{detected_service:"linux",install:{collections:["crowdsecurity/linux"]},datasource:{source:"file",labels:{type:"syslog"},filenames:["/var/log/syslog","/var/log/kern.log","/var/log/messages"]}}' + + run -0 cscli setup detect --force-os-family freebsd --detect-config "${TESTDATA}/detect.yaml" + run -0 jq -cS '.setup[] | select(.detected_service=="freebsd")' <(output) + assert_json '{detected_service:"freebsd",install:{collections:["crowdsecurity/freebsd"]}}' + + run -0 cscli setup detect --force-os-family windows --detect-config "${TESTDATA}/detect.yaml" + run -0 jq -cS '.setup[] | select(.detected_service=="windows")' <(output) + assert_json '{detected_service:"windows",install:{collections:["crowdsecurity/windows"]}}' + + run -0 --separate-stderr cscli setup detect --force-os-family darwin --detect-config "${TESTDATA}/detect.yaml" + refute_stderr + # XXX do we want do disallow unknown family? + # assert_stderr --partial "detecting services: OS 'darwin' not supported" + + # XXX TODO force-os-id, force-os-version +} + +@test "cscli setup detect --list-supported-services" { + tempfile=$(TMPDIR="$BATS_TEST_TMPDIR" mktemp) + cat <<-EOT >"${tempfile}" + version: 1.0 + detect: + thewiz: + foobarbaz: + apache2: + EOT + + run -0 cscli setup detect --list-supported-services --detect-config "$tempfile" + # the service list is sorted + assert_output - <<-EOT + apache2 + foobarbaz + thewiz + EOT + + cat <<-EOT >"${tempfile}" + thisisajoke + EOT + + run -1 --separate-stderr cscli setup detect --list-supported-services --detect-config "$tempfile" + assert_stderr --partial "while parsing ${tempfile}: yaml: unmarshal errors:" + + rm -f "$tempfile" +} + +@test "cscli setup detect (systemctl)" { + cat <<-EOT >"${DETECT_YAML}" + version: 1.0 + detect: + apache2: + when: + - UnitFound("mock-apache2.service") + datasource: + source: file + filename: dummy.log + labels: + type: apache2 + EOT + + # transparently mock systemctl. It's easier if you can tell the application + # under test which executable to call (in which case just call $mock) but + # here we do the symlink and $PATH dance as an example + mocked_command="systemctl" + + # mock setup + mock="$(mock_create)" + mock_path="${mock%/*}" + mock_file="${mock##*/}" + ln -sf "${mock_path}/${mock_file}" "${mock_path}/${mocked_command}" + + #shellcheck disable=SC2030 + PATH="${mock_path}:${PATH}" + + mock_set_output "$mock" \ +'UNIT FILE STATE VENDOR PRESET +snap-bare-5.mount enabled enabled +snap-core-13308.mount enabled enabled +snap-firefox-1635.mount enabled enabled +snap-fx-158.mount enabled enabled +snap-gimp-393.mount enabled enabled +snap-gtk\x2dcommon\x2dthemes-1535.mount enabled enabled +snap-kubectl-2537.mount enabled enabled +snap-rustup-1027.mount enabled enabled +cups.path enabled enabled +console-setup.service enabled enabled +dmesg.service enabled enabled +getty@.service enabled enabled +grub-initrd-fallback.service enabled enabled +irqbalance.service enabled enabled +keyboard-setup.service enabled enabled +mock-apache2.service enabled enabled +networkd-dispatcher.service enabled enabled +ua-timer.timer enabled enabled +update-notifier-download.timer enabled enabled +update-notifier-motd.timer enabled enabled + +20 unit files listed.' + mock_set_status "$mock" 1 2 + + run -0 cscli setup detect + run -0 jq -c '.setup' <(output) + + # If a call to UnitFoundwas part of the expression and it returned true, + # there is a default journalctl_filter derived from the unit's name. + assert_json '[{datasource:{source:"file",filename:"dummy.log",labels:{type:"apache2"}},detected_service:"apache2"}]' + + # the command was called exactly once + [[ $(mock_get_call_num "$mock") -eq 1 ]] + + # the command was called with the expected parameters + [[ $(mock_get_call_args "$mock" 1) == "list-unit-files --state=enabled,generated,static" ]] + + run -1 systemctl + + # mock teardown + unlink "${mock_path}/${mocked_command}" + PATH="${PATH/${mock_path}:/}" +} + +# XXX this is the same boilerplate as the previous test, can be simplified +@test "cscli setup detect (snub systemd)" { + cat <<-EOT >"${DETECT_YAML}" + version: 1.0 + detect: + apache2: + when: + - UnitFound("mock-apache2.service") + datasource: + source: file + filename: dummy.log + labels: + type: apache2 + EOT + + # transparently mock systemctl. It's easier if you can tell the application + # under test which executable to call (in which case just call $mock) but + # here we do the symlink and $PATH dance as an example + mocked_command="systemctl" + + # mock setup + mock="$(mock_create)" + mock_path="${mock%/*}" + mock_file="${mock##*/}" + ln -sf "${mock_path}/${mock_file}" "${mock_path}/${mocked_command}" + + #shellcheck disable=SC2031 + PATH="${mock_path}:${PATH}" + + # we don't really care about the output, it's not used anyway + mock_set_output "$mock" "" + mock_set_status "$mock" 1 2 + + run -0 cscli setup detect --snub-systemd + + # setup must not be 'null', but an empty list + assert_json '{setup:[]}' + + # the command was never called + [[ $(mock_get_call_num "$mock") -eq 0 ]] + + run -0 systemctl + + # mock teardown + unlink "${mock_path}/${mocked_command}" + PATH="${PATH/${mock_path}:/}" +} + +@test "cscli setup detect --force-unit" { + cat <<-EOT >"${DETECT_YAML}" + version: 1.0 + detect: + apache2: + when: + - UnitFound("force-apache2") + datasource: + source: file + filename: dummy.log + labels: + type: apache2 + apache3: + when: + - UnitFound("force-apache3") + datasource: + source: file + filename: dummy.log + labels: + type: apache3 + EOT + + run -0 cscli setup detect --force-unit force-apache2 + run -0 jq -cS '.setup' <(output) + assert_json '[{datasource:{source:"file",filename:"dummy.log",labels:{"type":"apache2"}},detected_service:"apache2"}]' + + run -0 cscli setup detect --force-unit force-apache2,force-apache3 + run -0 jq -cS '.setup' <(output) + assert_json '[{datasource:{source:"file",filename:"dummy.log",labels:{type:"apache2"}},detected_service:"apache2"},{datasource:{source:"file",filename:"dummy.log",labels:{"type":"apache3"}},detected_service:"apache3"}]' + + # force-unit can be specified multiple times, the order does not matter + run -0 cscli setup detect --force-unit force-apache3 --force-unit force-apache2 + run -0 jq -cS '.setup' <(output) + assert_json '[{datasource:{source:"file",filename:"dummy.log",labels:{type:"apache2"}},detected_service:"apache2"},{datasource:{source:"file",filename:"dummy.log",labels:{type:"apache3"}},detected_service:"apache3"}]' + + run -1 --separate-stderr cscli setup detect --force-unit mock-doesnotexist + assert_stderr --partial "detecting services: unit(s) forced but not supported: [mock-doesnotexist]" +} + +@test "cscli setup detect (process)" { + # This is harder to mock, because gopsutil requires proc/ to be a mount + # point. So we pick a process that exists for sure. + expected_process=$(basename "$SHELL") + + cat <<-EOT >"${DETECT_YAML}" + version: 1.0 + detect: + apache2: + when: + - ProcessRunning("${expected_process}") + apache3: + when: + - ProcessRunning("this-does-not-exist") + EOT + + run -0 cscli setup detect + run -0 jq -cS '.setup' <(output) + assert_json '[{detected_service:"apache2"}]' +} + +@test "cscli setup detect --force-process" { + cat <<-EOT >"${DETECT_YAML}" + version: 1.0 + detect: + apache2: + when: + - ProcessRunning("force-apache2") + apache3: + when: + - ProcessRunning("this-does-not-exist") + EOT + + run -0 cscli setup detect --force-process force-apache2 + run -0 jq -cS '.setup' <(output) + assert_json '[{detected_service:"apache2"}]' +} + +@test "cscli setup detect (acquisition only, no hub items)" { + cat <<-EOT >"${DETECT_YAML}" + version: 1.0 + detect: + apache2: + when: + - UnitFound("force-apache2") + datasource: + source: file + filename: dummy.log + labels: + type: apache2 + EOT + + run -0 cscli setup detect --force-unit force-apache2 + run -0 jq -cS '.setup' <(output) + assert_json '[{datasource:{source:"file",filename:"dummy.log",labels:{type:"apache2"}},detected_service:"apache2"}]' + + run -0 cscli setup detect --force-unit force-apache2 --yaml + assert_output - <<-EOT + setup: + - detected_service: apache2 + datasource: + filename: dummy.log + labels: + type: apache2 + source: file + EOT +} + +@test "cscli setup detect (full acquisition section)" { + skip "not supported yet" + cat <<-EOT >"${DETECT_YAML}" + version: 1.0 + detect: + foobar: + datasource: + filenames: + - /path/to/log/*.log + exclude_regexps: + - ^/path/to/log/excludeme\.log$ + force_inotify: true + mode: tail + labels: + type: foolog + EOT + + run -0 cscli setup detect --yaml + assert_output - <<-EOT + setup: + - detected_service: foobar + datasource: + filenames: + - /path/to/log/*.log + exclude_regexps: + - ^/path/to/log/excludeme.log$ + force_inotify: true + mode: tail + labels: + type: foolog + EOT +} + +@test "cscli setup detect + acquis + install (no acquisition, no hub items)" { + # no-op edge case, to make sure we don't crash + cat <<-EOT >"${DETECT_YAML}" + version: 1.0 + detect: + always: + EOT + + run -0 cscli setup detect + assert_json '{setup:[{detected_service:"always"}]}' + setup=$output + run -0 cscli setup datasources /dev/stdin <<<"$setup" + run -0 cscli setup install-hub /dev/stdin <<<"$setup" +} + +@test "cscli setup detect (with collections)" { + cat <<-EOT >"${DETECT_YAML}" + version: 1.0 + detect: + foobar: + when: + - ProcessRunning("force-foobar") + install: + collections: + - crowdsecurity/foobar + qox: + when: + - ProcessRunning("test-qox") + install: + collections: + - crowdsecurity/foobar + apache2: + when: + - ProcessRunning("force-apache2") + install: + collections: + - crowdsecurity/apache2 + EOT + + run -0 cscli setup detect --force-process force-apache2,force-foobar + run -0 jq -Sc '.setup | sort' <(output) + assert_json '[{install:{collections:["crowdsecurity/apache2"]},detected_service:"apache2"},{install:{collections:["crowdsecurity/foobar"]},detected_service:"foobar"}]' +} + +@test "cscli setup detect (with acquisition)" { + cat <<-EOT >"${DETECT_YAML}" + version: 1.0 + detect: + foobar: + when: + - ProcessRunning("force-foobar") + datasource: + source: file + labels: + type: foobar + filenames: + - /var/log/apache2/*.log + - /var/log/*http*/*.log + EOT + + run -0 cscli setup detect --force-process force-foobar + run -0 yq -op '.setup | sort_keys(..)' <(output) + assert_output - <<-EOT + 0.datasource.filenames.0 = /var/log/apache2/*.log + 0.datasource.filenames.1 = /var/log/*http*/*.log + 0.datasource.labels.type = foobar + 0.datasource.source = file + 0.detected_service = foobar + EOT + + run -1 --separate-stderr cscli setup detect --force-process mock-doesnotexist + assert_stderr --partial "detecting services: process(es) forced but not supported: [mock-doesnotexist]" +} + +@test "cscli setup detect (datasource validation)" { + cat <<-EOT >"${DETECT_YAML}" + version: 1.0 + detect: + foobar: + datasource: + labels: + type: something + EOT + + run -1 --separate-stderr cscli setup detect + assert_stderr --partial "detecting services: invalid datasource for foobar: source is empty" + + # more datasource-specific tests are in detect_test.go +} + +@test "cscli setup install-hub (dry run)" { + # it's not installed + run -0 cscli collections list -o json + run -0 jq -r '.collections[].name' <(output) + refute_line "crowdsecurity/apache2" + + # we install it + run -0 --separate-stderr cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/apache2"]}}]}' + assert_output 'dry-run: would install collection crowdsecurity/apache2' + + # still not installed + run -0 cscli collections list -o json + run -0 jq -r '.collections[].name' <(output) + refute_line "crowdsecurity/apache2" +} + +@test "cscli setup install-hub (dry run: install multiple collections)" { + # it's not installed + run -0 cscli collections list -o json + run -0 jq -r '.collections[].name' <(output) + refute_line "crowdsecurity/apache2" + + # we install it + run -0 --separate-stderr cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/apache2"]}}]}' + assert_output 'dry-run: would install collection crowdsecurity/apache2' + + # still not installed + run -0 cscli collections list -o json + run -0 jq -r '.collections[].name' <(output) + refute_line "crowdsecurity/apache2" +} + +@test "cscli setup install-hub (dry run: install multiple collections, parsers, scenarios, postoverflows)" { + run -0 --separate-stderr cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/foo","johndoe/bar"],"parsers":["crowdsecurity/fooparser","johndoe/barparser"],"scenarios":["crowdsecurity/fooscenario","johndoe/barscenario"],"postoverflows":["crowdsecurity/foopo","johndoe/barpo"]}}]}' + assert_line 'dry-run: would install collection crowdsecurity/foo' + assert_line 'dry-run: would install collection johndoe/bar' + assert_line 'dry-run: would install parser crowdsecurity/fooparser' + assert_line 'dry-run: would install parser johndoe/barparser' + assert_line 'dry-run: would install scenario crowdsecurity/fooscenario' + assert_line 'dry-run: would install scenario johndoe/barscenario' + assert_line 'dry-run: would install postoverflow crowdsecurity/foopo' + assert_line 'dry-run: would install postoverflow johndoe/barpo' +} + +@test "cscli setup datasources" { + run -0 cscli setup datasources --help + assert_line --partial "--to-dir string write the configuration to a directory, in multiple files" + + # single item + + run -0 cscli setup datasources /dev/stdin <<-EOT + setup: + - datasource: + source: file + labels: + type: syslog + filenames: + - /var/log/apache2/*.log + - /var/log/*http*/*.log + - /var/log/httpd/*.log + EOT + + # remove diclaimer + run -0 yq '. head_comment=""' <(output) + assert_output - <<-EOT + filenames: + - /var/log/apache2/*.log + - /var/log/*http*/*.log + - /var/log/httpd/*.log + labels: + type: syslog + source: file + EOT + + # multiple items + + run -0 cscli setup datasources /dev/stdin <<-EOT + setup: + - datasource: + labels: + type: syslog + filenames: + - /var/log/apache2/*.log + - /var/log/*http*/*.log + - /var/log/httpd/*.log + - datasource: + labels: + type: foobar + filenames: + - /var/log/foobar/*.log + - datasource: + labels: + type: barbaz + filenames: + - /path/to/barbaz.log + EOT + + run -0 yq '. head_comment=""' <(output) + assert_output - <<-EOT + filenames: + - /var/log/apache2/*.log + - /var/log/*http*/*.log + - /var/log/httpd/*.log + labels: + type: syslog + --- + filenames: + - /var/log/foobar/*.log + labels: + type: foobar + --- + filenames: + - /path/to/barbaz.log + labels: + type: barbaz + EOT + + # multiple items, to a directory + + # avoid the BATS_TEST_TMPDIR variable, it can have a double // + acquisdir=$(TMPDIR="$BATS_FILE_TMPDIR" mktemp -u) + mkdir "$acquisdir" + + run -0 cscli setup datasources /dev/stdin --to-dir "$acquisdir" <<-EOT + setup: + - detected_service: apache2 + datasource: + labels: + type: syslog + filenames: + - /var/log/apache2/*.log + - /var/log/*http*/*.log + - /var/log/httpd/*.log + - detected_service: foobar + datasource: + labels: + type: foobar + filenames: + - /var/log/foobar/*.log + - detected_service: barbaz + datasource: + labels: + type: barbaz + filenames: + - /path/to/barbaz.log + EOT + + # XXX what if detected_service is missing? + + run -0 cat "${acquisdir}/setup.apache2.yaml" + run -0 yq '. head_comment=""' <(output) + assert_output - <<-EOT + filenames: + - /var/log/apache2/*.log + - /var/log/*http*/*.log + - /var/log/httpd/*.log + labels: + type: syslog + EOT + + run -0 cat "${acquisdir}/setup.foobar.yaml" + run -0 yq '. head_comment=""' <(output) + assert_output - <<-EOT + filenames: + - /var/log/foobar/*.log + labels: + type: foobar + EOT + + run -0 cat "${acquisdir}/setup.barbaz.yaml" + run -0 yq '. head_comment=""' <(output) + assert_output - <<-EOT + filenames: + - /path/to/barbaz.log + labels: + type: barbaz + EOT + + rm -rf -- "${acquisdir:?}" + mkdir "$acquisdir" + + # having both filenames and journalctl does not generate two files: the datasource is copied as-is, even if incorrect + + run -0 cscli setup datasources /dev/stdin --to-dir "$acquisdir" <<-EOT + setup: + - detected_service: apache2 + install: + collections: + - crowdsecurity/apache2 + datasource: + labels: + type: apache2 + filenames: + - /var/log/apache2/*.log + - /var/log/*http*/*.log + - /var/log/httpd/*.log + journalctl_filter: + - _SYSTEMD_UNIT=apache2.service + EOT + + run -0 cat "${acquisdir}/setup.apache2.yaml" + run -0 yq '. head_comment=""' <(output) + assert_output - <<-EOT + filenames: + - /var/log/apache2/*.log + - /var/log/*http*/*.log + - /var/log/httpd/*.log + journalctl_filter: + - _SYSTEMD_UNIT=apache2.service + labels: + type: apache2 + EOT + + # the directory must exist + run -1 --separate-stderr cscli setup datasources /dev/stdin --to-dir /path/does/not/exist <<< '{}' + assert_stderr --partial "directory /path/does/not/exist does not exist" + + # of course it must be a directory + + touch "${acquisdir}/notadir" + + run -1 --separate-stderr cscli setup datasources /dev/stdin --to-dir "${acquisdir}/notadir" <<-EOT + setup: + - detected_service: apache2 + datasource: + filenames: + - /var/log/apache2/*.log + EOT + assert_stderr --partial "open ${acquisdir}/notadir/setup.apache2.yaml: not a directory" + + rm -rf -- "${acquisdir:?}" +} + +@test "cscli setup datasources (disclaimer)" { + disclaimer="This file was automatically generated" + + run -0 cscli setup datasources /dev/stdin <<<"setup:" + run -0 yq 'head_comment' <(output) + assert_output --partial "$disclaimer" + + run -0 cscli setup datasources /dev/stdin <<-EOT + setup: + - detected_service: something + datasource: + labels: + type: syslog + filenames: + - /var/log/something.log + EOT + run -0 yq 'head_comment' <(output) + assert_output --partial "$disclaimer" +} + +@test "cscli setup (custom journalctl filter)" { + tempfile=$(TMPDIR="$BATS_TEST_TMPDIR" mktemp) + cat <<-EOT >"${tempfile}" + version: 1.0 + detect: + thewiz: + when: + - UnitFound("thewiz.service") + datasource: + source: journalctl + labels: + type: thewiz + journalctl_filter: + - "SYSLOG_IDENTIFIER=TheWiz" + EOT + + run -0 cscli setup detect --detect-config "$tempfile" --force-unit thewiz.service + run -0 jq -cS '.' <(output) + assert_json '{setup:[{datasource:{source:"journalctl",journalctl_filter:["SYSLOG_IDENTIFIER=TheWiz"],labels:{type:"thewiz"}},detected_service:"thewiz"}]}' + run -0 cscli setup datasources <(output) + run -0 yq '. head_comment=""' <(output) + assert_output - <<-EOT + journalctl_filter: + - SYSLOG_IDENTIFIER=TheWiz + labels: + type: thewiz + source: journalctl + EOT + + rm -f "$tempfile" +} + +@test "cscli setup validate" { + + # an empty file is not enough + run -1 --separate-stderr cscli setup validate /dev/null + assert_output "EOF" + assert_stderr --partial "invalid setup file" + + # this is ok; install nothing + run -0 --separate-stderr cscli setup validate /dev/stdin <<-EOT + setup: + EOT + refute_output + refute_stderr + + run -1 --separate-stderr cscli setup validate /dev/stdin <<-EOT + se tup: + EOT + assert_output - <<-EOT + [1:1] unknown field "se tup" + > 1 | se tup: + ^ + EOT + assert_stderr --partial "invalid setup file" + + run -1 --separate-stderr cscli setup validate /dev/stdin <<-EOT + setup: + alsdk al; sdf + EOT + assert_output "while unmarshaling setup file: yaml: line 2: could not find expected ':'" + assert_stderr --partial "invalid setup file" +} + diff --git a/tests/bats/testdata/07_setup/detect.yaml b/tests/bats/testdata/07_setup/detect.yaml new file mode 100644 index 00000000000..704dfea8a62 --- /dev/null +++ b/tests/bats/testdata/07_setup/detect.yaml @@ -0,0 +1,88 @@ +# TODO: windows, use_time_machine, event support (see https://hub.crowdsec.net/author/crowdsecurity/collections/iis) + +--- +version: 1.0 + +detect: + apache2: + when: + - ProcessRunning("apache2") + install: + collections: + - crowdsecurity/apache2 + datasource: + source: file + labels: + type: apache2 + filenames: + - /var/log/apache2/*.log + - /var/log/*http*/*.log + - /var/log/httpd/*.log + + apache2-systemd: + when: + - UnitFound("apache2.service") + - OS.ID != "centos" + install: + collections: + - crowdsecurity/apache2 + datasource: + source: journalctl + journalctl_filter: + - "_SYSTEMD_UNIT=mock-apache2.service" + labels: + type: apache2 + + apache2-systemd-centos: + when: + - UnitFound("httpd.service") + - OS.ID == "centos" + install: + collections: + - crowdsecurity/apache2 + datasource: + source: journalctl + journalctl_filter: + - "_SYSTEMD_UNIT=httpd.service" + + ssh-systemd: + when: + - UnitFound("ssh.service") or UnitFound("ssh.socket") + install: + collections: + - crowdsecurity/apache2 + datasource: + source: journalctl + journalctl_filter: + - "_SYSTEMD_UNIT=ssh.service" + labels: + type: syslog + + linux: + when: + - OS.Family == "linux" + install: + collections: + - crowdsecurity/linux + datasource: + source: file + labels: + type: syslog + filenames: + - /var/log/syslog + - /var/log/kern.log + - /var/log/messages + + freebsd: + when: + - OS.Family == "freebsd" + install: + collections: + - crowdsecurity/freebsd + + windows: + when: + - OS.Family == "windows" + install: + collections: + - crowdsecurity/windows diff --git a/tests/lib/config/config-local b/tests/lib/config/config-local index 7d84ac7dfba..93e6eb14dfd 100755 --- a/tests/lib/config/config-local +++ b/tests/lib/config/config-local @@ -61,6 +61,9 @@ config_generate() { ../config/online_api_credentials.yaml \ "${CONFIG_DIR}/" + cp ../config/detect.yaml \ + "${HUB_DIR}" + # the default acquis file contains files that are not readable by everyone touch "$LOG_DIR/empty.log" cat <<-EOT >"$CONFIG_DIR/acquis.yaml" diff --git a/wizard.sh b/wizard.sh index 7a314d86ae2..f7e8b692951 100755 --- a/wizard.sh +++ b/wizard.sh @@ -1,33 +1,36 @@ -#!/usr/bin/env bash +#!/bin/sh -set -o pipefail -#set -x +# allow calling functions in an "if" statement +#shellcheck disable=SC2310 -skip_tmp_acquis() { - [[ "${TMP_ACQUIS_FILE_SKIP}" == "skip" ]] -} +set -e +checkroot() { + #shellcheck disable=SC2312 + if [ "$(id -u)" -ne 0 ]; then + log_err "Please run the wizard as root or with sudo" + exit 1 + fi +} -RED='\033[0;31m' -BLUE='\033[0;34m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -ORANGE='\033[0;33m' -NC='\033[0m' +interactive() { + if [ ! -t 0 ] || [ "$SILENT" = "true" ]; then + return 1 + fi + return 0 +} SILENT="false" DOCKER_MODE="false" -CROWDSEC_RUN_DIR="/var/run" CROWDSEC_LIB_DIR="/var/lib/crowdsec" CROWDSEC_USR_DIR="/usr/local/lib/crowdsec" CROWDSEC_DATA_DIR="${CROWDSEC_LIB_DIR}/data" CROWDSEC_DB_PATH="${CROWDSEC_DATA_DIR}/crowdsec.db" CROWDSEC_PATH="/etc/crowdsec" -CROWDSEC_CONFIG_PATH="${CROWDSEC_PATH}" +CROWDSEC_CONFIG_PATH="$CROWDSEC_PATH" CROWDSEC_LOG_FILE="/var/log/crowdsec.log" LAPI_LOG_FILE="/var/log/crowdsec_api.log" -CROWDSEC_PLUGIN_DIR="${CROWDSEC_USR_DIR}/plugins" CROWDSEC_BIN="./cmd/crowdsec/crowdsec" CSCLI_BIN="./cmd/crowdsec-cli/cscli" @@ -35,19 +38,17 @@ CSCLI_BIN="./cmd/crowdsec-cli/cscli" CLIENT_SECRETS="local_api_credentials.yaml" LAPI_SECRETS="online_api_credentials.yaml" -CONSOLE_FILE="console.yaml" - BIN_INSTALL_PATH="/usr/local/bin" CROWDSEC_BIN_INSTALLED="${BIN_INSTALL_PATH}/crowdsec" -if [[ -f "/usr/bin/cscli" ]] ; then +if [ -f "/usr/bin/cscli" ]; then CSCLI_BIN_INSTALLED="/usr/bin/cscli" else CSCLI_BIN_INSTALLED="${BIN_INSTALL_PATH}/cscli" fi -ACQUIS_PATH="${CROWDSEC_CONFIG_PATH}" -ACQUIS_TARGET="${ACQUIS_PATH}/acquis.yaml" +ACQUIS_DIR="${CROWDSEC_CONFIG_PATH}/acquis.d" +ACQUIS_YAML="${CROWDSEC_CONFIG_PATH}/acquis.yaml" SYSTEMD_PATH_FILE="/etc/systemd/system/crowdsec.service" @@ -59,375 +60,204 @@ ACTION="" DEBUG_MODE="false" FORCE_MODE="false" -SUPPORTED_SERVICES='apache2 -httpd -nginx -sshd -mysql -telnet -smb -' - - -HTTP_PLUGIN_BINARY="./plugins/notifications/http/notification-http" -SLACK_PLUGIN_BINARY="./plugins/notifications/slack/notification-slack" -SPLUNK_PLUGIN_BINARY="./plugins/notifications/splunk/notification-splunk" -EMAIL_PLUGIN_BINARY="./plugins/notifications/email/notification-email" - -HTTP_PLUGIN_CONFIG="./plugins/notifications/http/http.yaml" -SLACK_PLUGIN_CONFIG="./plugins/notifications/slack/slack.yaml" -SPLUNK_PLUGIN_CONFIG="./plugins/notifications/splunk/splunk.yaml" -EMAIL_PLUGIN_CONFIG="./plugins/notifications/email/email.yaml" +PLUGIN_CONFIGURATION_SRC="./plugins/notifications" +PLUGIN_CONFIGURATION_DEST="/etc/crowdsec/notifications" +PLUGIN_BINARIES_SRC="./plugins/notifications" +PLUGIN_BINARIES_DEST="${CROWDSEC_USR_DIR}/plugins" +# XXX WTH should remove it later BACKUP_DIR=$(mktemp -d) rm -rf -- "$BACKUP_DIR" +set_colors() { + #shellcheck disable=SC2034 + if [ ! -t 0 ]; then + # terminal is not interactive; no colors + FG_RED="" + FG_GREEN="" + FG_YELLOW="" + FG_BLUE="" + FG_MAGENTA="" + FG_CYAN="" + FG_WHITE="" + BOLD="" + RESET="" + elif tput sgr0 >/dev/null; then + # terminfo + FG_RED=$(tput setaf 1) + FG_GREEN=$(tput setaf 2) + FG_YELLOW=$(tput setaf 3) + FG_BLUE=$(tput setaf 4) + FG_MAGENTA=$(tput setaf 5) + FG_CYAN=$(tput setaf 6) + FG_WHITE=$(tput setaf 7) + BOLD=$(tput bold) + RESET=$(tput sgr0) + else + FG_RED=$(printf '%b' '\033[31m') + FG_GREEN=$(printf '%b' '\033[32m') + FG_YELLOW=$(printf '%b' '\033[33m') + FG_BLUE=$(printf '%b' '\033[34m') + FG_MAGENTA=$(printf '%b' '\033[35m') + FG_CYAN=$(printf '%b' '\033[36m') + FG_WHITE=$(printf '%b' '\033[37m') + BOLD=$(printf '%b' '\033[1m') + RESET=$(printf '%b' '\033[0m') + fi +} + +#XXX logging is not consistent log_info() { msg=$1 date=$(date +%x:%X) - echo -e "${BLUE}INFO${NC}[${date}] crowdsec_wizard: ${msg}" + echo "${FG_BLUE}INFO${RESET}[${date}] crowdsec_wizard: ${msg}" } log_fatal() { msg=$1 date=$(date +%x:%X) - echo -e "${RED}FATA${NC}[${date}] crowdsec_wizard: ${msg}" 1>&2 + echo "${FG_RED}FATA${RESET}[${date}] crowdsec_wizard: ${msg}" >&2 exit 1 } log_warn() { msg=$1 date=$(date +%x:%X) - echo -e "${ORANGE}WARN${NC}[${date}] crowdsec_wizard: ${msg}" + echo "${FG_YELLOW}WARN${RESET}[${date}] crowdsec_wizard: ${msg}" } log_err() { msg=$1 date=$(date +%x:%X) - echo -e "${RED}ERR${NC}[${date}] crowdsec_wizard: ${msg}" 1>&2 + echo "${FG_RED}ERR${RESET}[${date}] crowdsec_wizard: ${msg}" >&2 } log_dbg() { - if [[ ${DEBUG_MODE} == "true" ]]; then + if [ "$DEBUG_MODE" = "true" ]; then msg=$1 date=$(date +%x:%X) - echo -e "[${date}][${YELLOW}DBG${NC}] crowdsec_wizard: ${msg}" 1>&2 - fi -} - -detect_services () { - DETECTED_SERVICES=() - HMENU=() - #list systemd services - SYSTEMD_SERVICES=`systemctl --state=enabled list-unit-files '*.service' | cut -d ' ' -f1` - #raw ps - PSAX=`ps ax -o comm=` - for SVC in ${SUPPORTED_SERVICES} ; do - log_dbg "Checking if service '${SVC}' is running (ps+systemd)" - for SRC in "${SYSTEMD_SERVICES}" "${PSAX}" ; do - echo ${SRC} | grep ${SVC} >/dev/null - if [ $? -eq 0 ]; then - #on centos, apache2 is named httpd - if [[ ${SVC} == "httpd" ]] ; then - SVC="apache2"; - fi - DETECTED_SERVICES+=(${SVC}) - HMENU+=(${SVC} "on") - log_dbg "Found '${SVC}' running" - break; - fi; - done; - done; - if [[ ${OSTYPE} == "linux-gnu" ]] || [[ ${OSTYPE} == "linux-gnueabihf" ]]; then - DETECTED_SERVICES+=("linux") - HMENU+=("linux" "on") - else - log_info "NOT A LINUX" - fi; - - if [[ ${SILENT} == "false" ]]; then - #we put whiptail results in an array, notice the dark magic fd redirection - DETECTED_SERVICES=($(whiptail --separate-output --noitem --ok-button Continue --title "Services to monitor" --checklist "Detected services, uncheck to ignore. Ignored services won't be monitored." 18 70 10 ${HMENU[@]} 3>&1 1>&2 2>&3)) - if [ $? -eq 1 ]; then - log_err "user bailed out at services selection" - exit 1; - fi; - log_dbg "Detected services (interactive) : ${DETECTED_SERVICES[@]}" - else - log_dbg "Detected services (unattended) : ${DETECTED_SERVICES[@]}" - fi; -} - -declare -A log_input_tags -log_input_tags[apache2]='type: apache2' -log_input_tags[nginx]='type: nginx' -log_input_tags[sshd]='type: syslog' -log_input_tags[rsyslog]='type: syslog' -log_input_tags[telnet]='type: telnet' -log_input_tags[mysql]='type: mysql' -log_input_tags[smb]='type: smb' -log_input_tags[linux]="type: syslog" - -declare -A log_locations -log_locations[apache2]='/var/log/apache2/*.log,/var/log/*httpd*.log,/var/log/httpd/*log' -log_locations[nginx]='/var/log/nginx/*.log,/usr/local/openresty/nginx/logs/*.log' -log_locations[sshd]='/var/log/auth.log,/var/log/sshd.log,/var/log/secure' -log_locations[rsyslog]='/var/log/syslog' -log_locations[telnet]='/var/log/telnetd*.log' -log_locations[mysql]='/var/log/mysql/error.log' -log_locations[smb]='/var/log/samba*.log' -log_locations[linux]='/var/log/syslog,/var/log/kern.log,/var/log/messages' - -#$1 is service name, such those in SUPPORTED_SERVICES -find_logs_for() { - ret="" - x=${1} - #we have trailing and starting quotes because of whiptail - SVC="${x%\"}" - SVC="${SVC#\"}" - DETECTED_LOGFILES=() - HMENU=() - #log_info "Searching logs for ${SVC} : ${log_locations[${SVC}]}" - - #split the line into an array with ',' separator - OIFS=${IFS} - IFS=',' read -r -a a <<< "${log_locations[${SVC}]}," - IFS=${OIFS} - #readarray -td, a <<<"${log_locations[${SVC}]},"; unset 'a[-1]'; - for poss_path in "${a[@]}"; do - #Split /var/log/nginx/*.log into '/var/log/nginx' and '*.log' so we can use find - path=${poss_path%/*} - fname=${poss_path##*/} - candidates=`find "${path}" -type f -mtime -5 -ctime -5 -name "$fname"` - #We have some candidates, add them - for final_file in ${candidates} ; do - log_dbg "Found logs file for '${SVC}': ${final_file}" - DETECTED_LOGFILES+=(${final_file}) - HMENU+=(${final_file} "on") - done; - done; - - if [[ ${SILENT} == "false" ]]; then - DETECTED_LOGFILES=($(whiptail --separate-output --noitem --ok-button Continue --title "Log files to process for ${SVC}" --checklist "Detected logfiles for ${SVC}, uncheck to ignore" 18 70 10 ${HMENU[@]} 3>&1 1>&2 2>&3)) - if [ $? -eq 1 ]; then - log_err "user bailed out at log file selection" - exit 1; - fi; - fi -} - -in_array() { - str=$1 - shift - array=("$@") - for element in "${array[@]}"; do - if [[ ${str} == crowdsecurity/${element} ]]; then - return 0 - fi - done - return 1 -} - -install_collection() { - HMENU=() - readarray -t AVAILABLE_COLLECTION < <(${CSCLI_BIN_INSTALLED} collections list -o raw -a) - COLLECTION_TO_INSTALL=() - for collect_info in "${AVAILABLE_COLLECTION[@]:1}"; do - collection="$(echo ${collect_info} | cut -d "," -f1)" - description="$(echo ${collect_info} | cut -d "," -f4)" - in_array $collection "${DETECTED_SERVICES[@]}" - if [[ $? == 0 ]]; then - HMENU+=("${collection}" "${description}" "ON") - #in case we're not in interactive mode, assume defaults - COLLECTION_TO_INSTALL+=(${collection}) - else - if [[ ${collection} == "linux" ]]; then - HMENU+=("${collection}" "${description}" "ON") - #in case we're not in interactive mode, assume defaults - COLLECTION_TO_INSTALL+=(${collection}) - else - HMENU+=("${collection}" "${description}" "OFF") - fi - fi - done + echo "[${date}][${FG_YELLOW}DBG${RESET}] crowdsec_wizard: ${msg}" >&2 + fi +} - if [[ ${SILENT} == "false" ]]; then - COLLECTION_TO_INSTALL=($(whiptail --separate-output --ok-button Continue --title "Crowdsec collections" --checklist "Available collections in crowdsec, try to pick one that fits your profile. Collections contains parsers and scenarios to protect your system." 20 120 10 "${HMENU[@]}" 3>&1 1>&2 2>&3)) - if [ $? -eq 1 ]; then - log_err "user bailed out at collection selection" - exit 1; - fi; - fi; - - for collection in "${COLLECTION_TO_INSTALL[@]}"; do - log_info "Installing collection '${collection}'" - ${CSCLI_BIN_INSTALLED} collections install "${collection}" > /dev/null 2>&1 || log_err "fail to install collection ${collection}" - done +crowdsec_service_stop() { + if command -v systemctl >/dev/null && systemctl is-active --quiet crowdsec; then + systemctl stop crowdsec.service + fi +} - ${CSCLI_BIN_INSTALLED} parsers install "crowdsecurity/whitelists" > /dev/null 2>&1 || log_err "fail to install collection crowdsec/whitelists" - if [[ ${SILENT} == "false" ]]; then - whiptail --msgbox "Out of safety, I installed a parser called 'crowdsecurity/whitelists'. This one will prevent private IP addresses from being banned, feel free to remove it any time." 20 50 +crowdsec_service_disable() { + if command -v systemctl >/dev/null && systemctl is-enabled --quiet crowdsec; then + systemctl disable crowdsec.service fi +} - if [[ ${SILENT} == "false" ]]; then - whiptail --msgbox "CrowdSec alone will not block any IP address. If you want to block them, you must use a bouncer. You can find them on https://hub.crowdsec.net/browse/#bouncers" 20 50 +crowdsec_service_restart() { + if command -v systemctl >/dev/null; then + systemctl restart crowdsec fi } -#$1 is the service name, $... is the list of candidate logs (from find_logs_for) -genyamllog() { - local service="${1}" - shift - local files=("${@}") - - echo "#Generated acquisition file - wizard.sh (service: ${service}) / files : ${files[@]}" >> ${TMP_ACQUIS_FILE} - - echo "filenames:" >> ${TMP_ACQUIS_FILE} - for fd in ${files[@]}; do - echo " - ${fd}" >> ${TMP_ACQUIS_FILE} - done - echo "labels:" >> ${TMP_ACQUIS_FILE} - echo " "${log_input_tags[${service}]} >> ${TMP_ACQUIS_FILE} - echo "---" >> ${TMP_ACQUIS_FILE} - log_dbg "${ACQUIS_FILE_MSG}" -} - -genyamljournal() { - local service="${1}" - shift - - echo "#Generated acquisition file - wizard.sh (service: ${service}) / files : ${files[@]}" >> ${TMP_ACQUIS_FILE} - - echo "journalctl_filter:" >> ${TMP_ACQUIS_FILE} - echo " - _SYSTEMD_UNIT="${service}".service" >> ${TMP_ACQUIS_FILE} - echo "labels:" >> ${TMP_ACQUIS_FILE} - echo " "${log_input_tags[${service}]} >> ${TMP_ACQUIS_FILE} - echo "---" >> ${TMP_ACQUIS_FILE} - log_dbg "${ACQUIS_FILE_MSG}" -} - -genacquisition() { - if skip_tmp_acquis; then - TMP_ACQUIS_FILE="${ACQUIS_TARGET}" - ACQUIS_FILE_MSG="acquisition file generated to: ${TMP_ACQUIS_FILE}" - else - TMP_ACQUIS_FILE="tmp-acquis.yaml" - ACQUIS_FILE_MSG="tmp acquisition file generated to: ${TMP_ACQUIS_FILE}" - fi - - log_dbg "Found following services : "${DETECTED_SERVICES[@]} - for PSVG in ${DETECTED_SERVICES[@]} ; do - find_logs_for ${PSVG} - if [[ ${#DETECTED_LOGFILES[@]} -gt 0 ]] ; then - log_info "service '${PSVG}': ${DETECTED_LOGFILES[*]}" - genyamllog ${PSVG} ${DETECTED_LOGFILES[@]} - elif [[ ${PSVG} != "linux" ]] ; then - log_info "using journald for '${PSVG}'" - genyamljournal ${PSVG} - fi; - done -} - -detect_cs_install () { - if [[ -f "$CROWDSEC_BIN_INSTALLED" ]]; then - log_warn "Crowdsec is already installed !" + +detect_cs_install() { + if [ -f "$CROWDSEC_BIN_INSTALLED" ]; then + log_warn "Crowdsec is already installed!" echo "" - echo "We recommend to upgrade : sudo ./wizard.sh --upgrade " + echo "We recommend to upgrade: sudo $0 --upgrade " echo "If you want to install it anyway, please use '--force'." echo "" - echo "Run : sudo ./wizard.sh -i --force" - if [[ ${FORCE_MODE} == "false" ]]; then + echo "Run: sudo $0 -i --force" + if [ "$FORCE_MODE" = "false" ]; then exit 1 fi fi } -check_cs_version () { +check_cs_version() { CURRENT_CS_VERSION=$(crowdsec -version 2>&1 | grep version | grep -Eio 'v[0-9]+.[0-9]+.[0-9]+' | cut -c 2-) - NEW_CS_VERSION=$($CROWDSEC_BIN -version 2>&1 | grep version | grep -Eio 'v[0-9]+.[0-9]+.[0-9]+' | cut -c 2-) - CURRENT_MAJOR_VERSION=$(echo $CURRENT_CS_VERSION | cut -d'.' -f1) - CURRENT_MINOR_VERSION=$(echo $CURRENT_CS_VERSION | cut -d'.' -f2) - CURRENT_PATCH_VERSION=$(echo $CURRENT_CS_VERSION | cut -d'.' -f3) - NEW_MAJOR_VERSION=$(echo $NEW_CS_VERSION | cut -d'.' -f1) - NEW_MINOR_VERSION=$(echo $NEW_CS_VERSION | cut -d'.' -f2) - NEW_PATCH_VERSION=$(echo $NEW_CS_VERSION | cut -d'.' -f3) - - if [[ $NEW_MAJOR_VERSION -gt $CURRENT_MAJOR_VERSION ]]; then - if [[ ${FORCE_MODE} == "false" ]]; then - log_warn "new version ($NEW_CS_VERSION) is a major, you should follow documentation to upgrade !" + NEW_CS_VERSION=$("$CROWDSEC_BIN" -version 2>&1 | grep version | grep -Eio 'v[0-9]+.[0-9]+.[0-9]+' | cut -c 2-) + CURRENT_MAJOR_VERSION=$(echo "$CURRENT_CS_VERSION" | cut -d'.' -f1) + CURRENT_MINOR_VERSION=$(echo "$CURRENT_CS_VERSION" | cut -d'.' -f2) + CURRENT_PATCH_VERSION=$(echo "$CURRENT_CS_VERSION" | cut -d'.' -f3) + NEW_MAJOR_VERSION=$(echo "$NEW_CS_VERSION" | cut -d'.' -f1) + NEW_MINOR_VERSION=$(echo "$NEW_CS_VERSION" | cut -d'.' -f2) + NEW_PATCH_VERSION=$(echo "$NEW_CS_VERSION" | cut -d'.' -f3) + + if [ "$NEW_MAJOR_VERSION" -gt "$CURRENT_MAJOR_VERSION" ]; then + if [ "$FORCE_MODE" = "false" ]; then + log_warn "new version (${NEW_CS_VERSION}) is a major, please follow the documentation to upgrade!" echo "" exit 1 fi - elif [[ $NEW_MINOR_VERSION -gt $CURRENT_MINOR_VERSION ]] ; then - log_warn "new version ($NEW_CS_VERSION) is a minor upgrade !" - if [[ $ACTION != "upgrade" ]] ; then - if [[ ${FORCE_MODE} == "false" ]]; then + elif [ "$NEW_MINOR_VERSION" -gt "$CURRENT_MINOR_VERSION" ]; then + log_warn "new version (${NEW_CS_VERSION}) is a minor upgrade!" + if [ "$ACTION" != "upgrade" ]; then + if [ "$FORCE_MODE" = "false" ]; then echo "" - echo "We recommend to upgrade with : sudo ./wizard.sh --upgrade " - echo "If you want to $ACTION anyway, please use '--force'." + echo "We recommend to upgrade with: sudo $0 --upgrade" + echo "If you want to ${ACTION} anyway, please use '--force'." echo "" - echo "Run : sudo ./wizard.sh --$ACTION --force" + echo "Run: sudo $0 --${ACTION} --force" exit 1 fi fi - elif [[ $NEW_PATCH_VERSION -gt $CURRENT_PATCH_VERSION ]] ; then - log_warn "new version ($NEW_CS_VERSION) is a patch !" - if [[ $ACTION != "binupgrade" ]] ; then - if [[ ${FORCE_MODE} == "false" ]]; then + elif [ "$NEW_PATCH_VERSION" -gt "$CURRENT_PATCH_VERSION" ]; then + log_warn "new version (${NEW_CS_VERSION}) is a patch !" + if [ "$ACTION" != "binupgrade" ]; then + if [ "$FORCE_MODE" = "false" ]; then echo "" - echo "We recommend to upgrade binaries only : sudo ./wizard.sh --binupgrade " - echo "If you want to $ACTION anyway, please use '--force'." + echo "We recommend to upgrade binaries only: sudo $0 --binupgrade" + echo "If you want to ${ACTION} anyway, please use '--force'." echo "" - echo "Run : sudo ./wizard.sh --$ACTION --force" + echo "Run: sudo $0 --${ACTION} --force" exit 1 fi fi - elif [[ $NEW_MINOR_VERSION -eq $CURRENT_MINOR_VERSION ]]; then - log_warn "new version ($NEW_CS_VERSION) is same as current version ($CURRENT_CS_VERSION) !" - if [[ ${FORCE_MODE} == "false" ]]; then + elif [ "$NEW_MINOR_VERSION" -eq "$CURRENT_MINOR_VERSION" ]; then + log_warn "new version (${NEW_CS_VERSION}) is same as current version (${CURRENT_CS_VERSION})!" + if [ "$FORCE_MODE" = "false" ]; then echo "" - echo "We recommend to $ACTION only if it's an higher version. " + echo "We recommend to ${ACTION} only if it's an higher version." echo "If it's an RC version (vX.X.X-rc) you can upgrade it using '--force'." echo "" - echo "Run : sudo ./wizard.sh --$ACTION --force" + echo "Run: sudo $0 --${ACTION} --force" exit 1 fi fi } -#install crowdsec and cscli install_crowdsec() { - mkdir -p "${CROWDSEC_DATA_DIR}" - (cd config && find patterns -type f -exec install -Dm 644 "{}" "${CROWDSEC_CONFIG_PATH}/{}" \; && cd ../) || exit - mkdir -p "${CROWDSEC_CONFIG_PATH}/scenarios" || exit - mkdir -p "${CROWDSEC_CONFIG_PATH}/postoverflows" || exit - mkdir -p "${CROWDSEC_CONFIG_PATH}/collections" || exit - mkdir -p "${CROWDSEC_CONFIG_PATH}/patterns" || exit - - #tmp - mkdir -p /tmp/data - mkdir -p /etc/crowdsec/hub/ - install -v -m 600 -D "./config/${CLIENT_SECRETS}" "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit - install -v -m 600 -D "./config/${LAPI_SECRETS}" "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit - - ## end tmp - - install -v -m 600 -D ./config/config.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit - install -v -m 644 -D ./config/dev.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit - install -v -m 644 -D ./config/user.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit - install -v -m 644 -D ./config/acquis.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit - install -v -m 644 -D ./config/profiles.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit - install -v -m 644 -D ./config/simulation.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit - install -v -m 644 -D ./config/"${CONSOLE_FILE}" "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit - - DATA=${CROWDSEC_DATA_DIR} CFG=${CROWDSEC_CONFIG_PATH} envsubst '$CFG $DATA' < ./config/user.yaml > ${CROWDSEC_CONFIG_PATH}"/user.yaml" || log_fatal "unable to generate user configuration file" - if [[ ${DOCKER_MODE} == "false" ]]; then - CFG=${CROWDSEC_CONFIG_PATH} BIN=${CROWDSEC_BIN_INSTALLED} envsubst '$CFG $BIN' < ./config/crowdsec.service > "${SYSTEMD_PATH_FILE}" || log_fatal "unable to crowdsec systemd file" + mkdir -p "$CROWDSEC_DATA_DIR" + mkdir -p "$CROWDSEC_CONFIG_PATH/collections" + mkdir -p "$CROWDSEC_CONFIG_PATH/parsers" + mkdir -p "$CROWDSEC_CONFIG_PATH/patterns" + mkdir -p "$CROWDSEC_CONFIG_PATH/postoverflows" + mkdir -p "$CROWDSEC_CONFIG_PATH/scenarios" + (cd config && find patterns -maxdepth 1 -type f -exec install -m 0644 "{}" "${CROWDSEC_CONFIG_PATH}/{}" \; && cd ../) + + install -m 0600 "./config/$CLIENT_SECRETS" "$CROWDSEC_CONFIG_PATH" + install -m 0600 "./config/$LAPI_SECRETS" "$CROWDSEC_CONFIG_PATH" + + install -m 0600 ./config/config.yaml "$CROWDSEC_CONFIG_PATH" + install -m 0644 ./config/dev.yaml "$CROWDSEC_CONFIG_PATH" + install -m 0644 ./config/user.yaml "$CROWDSEC_CONFIG_PATH" + install -m 0644 ./config/profiles.yaml "$CROWDSEC_CONFIG_PATH" + install -m 0644 ./config/simulation.yaml "$CROWDSEC_CONFIG_PATH" + install -m 0644 ./config/console.yaml "$CROWDSEC_CONFIG_PATH" + + mkdir -p "$CROWDSEC_CONFIG_PATH/hub" + install -m 0644 ./config/detect.yaml "${CROWDSEC_CONFIG_PATH}/hub" + + #shellcheck disable=SC2016 + DATA=${CROWDSEC_DATA_DIR} CFG=${CROWDSEC_CONFIG_PATH} envsubst '$CFG $DATA' <./config/user.yaml >"${CROWDSEC_CONFIG_PATH}/user.yaml" || log_fatal "unable to generate user configuration file" + if [ "$DOCKER_MODE" = "false" ]; then + #shellcheck disable=SC2016 + CFG=${CROWDSEC_CONFIG_PATH} BIN=${CROWDSEC_BIN_INSTALLED} envsubst '$CFG $BIN' <./config/crowdsec.service >"$SYSTEMD_PATH_FILE" || log_fatal "unable to generate systemd file" fi install_bins - if [[ ${DOCKER_MODE} == "false" ]]; then - systemctl daemon-reload + if [ "$DOCKER_MODE" = "false" ]; then + systemctl daemon-reload fi } @@ -440,315 +270,455 @@ update_bins() { } update_full() { - - if [[ ! -f "$CROWDSEC_BIN" ]]; then - log_err "Crowdsec binary '$CROWDSEC_BIN' not found. Please build it with 'make build'" && exit + if [ ! -f "$CROWDSEC_BIN" ]; then + log_err "Crowdsec binary '${CROWDSEC_BIN}' not found. Please build it with 'make build'" + exit fi - if [[ ! -f "$CSCLI_BIN" ]]; then - log_err "Cscli binary '$CSCLI_BIN' not found. Please build it with 'make build'" && exit + if [ ! -f "$CSCLI_BIN" ]; then + log_err "Cscli binary '${CSCLI_BIN}' not found. Please build it with 'make build'" + exit fi log_info "Backing up existing configuration" - ${CSCLI_BIN_INSTALLED} config backup ${BACKUP_DIR} + "$CSCLI_BIN_INSTALLED" config backup "$BACKUP_DIR" log_info "Saving default database content if exist" - if [[ -f "/var/lib/crowdsec/data/crowdsec.db" ]]; then - cp /var/lib/crowdsec/data/crowdsec.db ${BACKUP_DIR}/crowdsec.db + if [ -f "/var/lib/crowdsec/data/crowdsec.db" ]; then + cp /var/lib/crowdsec/data/crowdsec.db "${BACKUP_DIR}/crowdsec.db" fi log_info "Cleanup existing crowdsec configuration" uninstall_crowdsec log_info "Installing crowdsec" install_crowdsec log_info "Restoring configuration" - ${CSCLI_BIN_INSTALLED} hub update - ${CSCLI_BIN_INSTALLED} config restore ${BACKUP_DIR} + "$CSCLI_BIN_INSTALLED" hub update + "$CSCLI_BIN_INSTALLED" config restore "$BACKUP_DIR" log_info "Restoring saved database if exist" - if [[ -f "${BACKUP_DIR}/crowdsec.db" ]]; then - cp ${BACKUP_DIR}/crowdsec.db /var/lib/crowdsec/data/crowdsec.db + if [ -f "${BACKUP_DIR}/crowdsec.db" ]; then + cp "${BACKUP_DIR}/crowdsec.db" /var/lib/crowdsec/data/crowdsec.db fi log_info "Finished, restarting" - systemctl restart crowdsec || log_fatal "Failed to restart crowdsec" + crowdsec_service_restart || log_fatal "Failed to restart crowdsec" } install_bins() { log_dbg "Installing crowdsec binaries" - install -v -m 755 -D "${CROWDSEC_BIN}" "${CROWDSEC_BIN_INSTALLED}" 1> /dev/null || exit - install -v -m 755 -D "${CSCLI_BIN}" "${CSCLI_BIN_INSTALLED}" 1> /dev/null || exit - which systemctl && systemctl is-active --quiet crowdsec - if [ $? -eq 0 ]; then - systemctl stop crowdsec - fi + install -m 0755 "$CROWDSEC_BIN" "$CROWDSEC_BIN_INSTALLED" >/dev/null + install -m 0755 "$CSCLI_BIN" "$CSCLI_BIN_INSTALLED" >/dev/null + + crowdsec_service_stop install_plugins symlink_bins } symlink_bins() { - if grep -q "${BIN_INSTALL_PATH}" <<< $PATH; then + if echo "$PATH" | grep -q "$BIN_INSTALL_PATH"; then log_dbg "${BIN_INSTALL_PATH} found in PATH" else - ln -s "${CSCLI_BIN_INSTALLED}" /usr/bin/cscli - ln -s "${CROWDSEC_BIN_INSTALLED}" /usr/bin/crowdsec + ln -s "$CSCLI_BIN_INSTALLED" /usr/bin/cscli + ln -s "$CROWDSEC_BIN_INSTALLED" /usr/bin/crowdsec fi } delete_bins() { log_info "Removing crowdsec binaries" - rm -f ${CROWDSEC_BIN_INSTALLED} - rm -f ${CSCLI_BIN_INSTALLED} + rm -f -- "$CROWDSEC_BIN_INSTALLED" + rm -f -- "$CSCLI_BIN_INSTALLED" } delete_plugins() { - rm -rf ${CROWDSEC_PLUGIN_DIR} + rm -rf -- "$PLUGIN_BINARIES_DEST" +} + +detect_only() { + "$CSCLI_BIN_INSTALLED" setup detect --yaml +} + +edit_file() { + editor="$VISUAL" + if [ "$editor" = "" ]; then + #shellcheck disable=SC2153 + editor="$EDITOR" + fi + if [ "$editor" = "" ]; then + if command -v nano >/dev/null; then + editor="nano" + elif command -v nano-tiny >/dev/null; then + editor="nano-tiny" + elif command -v vi >/dev/null; then + editor="vi" + else + echo "No editor found" + exit 1 + fi + fi + "$editor" "$1" +} + +detect_edit_validate() { + setup_yaml_path="$1" + while true; do + cat <<-EOT >"$setup_yaml_path" + # + # XXX detection timestamp, how to edit + # blah blah blah + # + # Out of safety, we recommend installing the parser 'crowdsecurity/whitelists'. + # It will prevent private IP addresses from being banned. It's an anti-lockout measure, + # feel free to remove it any time. + # + + EOT + + echo + "$CSCLI_BIN_INSTALLED" setup detect --yaml | tee -a "$setup_yaml_path" + + # + # If the user asked for --unattended, or the script is not interactive, + # we use the detected setup without changes. + # + if ! interactive; then + return 0 + fi + + printf '%s ' "Crowdsec has detected these services. Do you want to edit the list now? (Y/n)" + read -r confirm + + if echo "$confirm" | grep -q '^[Nn]'; then + return 0 + fi + + while true; do + edit_file "$setup_yaml_path" + + if ! errors=$("$CSCLI_BIN_INSTALLED" setup validate "$setup_yaml_path" 2>/dev/null); then + echo + echo "The setup file has errors:" + echo + + if [ "$errors" = "EOF" ]; then + errors="The file is empty. A 'setup:' section is required, even if it has no items." + fi + + echo "$errors" + echo + printf '%s ' "[E]dit, [D]etect again, [Q]uit configuration? (E/d/q)" + + read -r confirm + + if echo "$confirm" | grep -q '^[Dd]'; then + break + fi + + if echo "$confirm" | grep -q '^[Qq]'; then + rm -f "$setup_yaml_path" + return 1 + fi + else + return 0 + fi + done + done +} + +# Pause until the user types +# unless the script is run in non-interactive mode. +ask_press_enter() { + if ! interactive; then + return 0 + fi + + printf "%s " "Press Enter to continue:" >&2 + read -r key } -install_plugins(){ - mkdir -p ${CROWDSEC_PLUGIN_DIR} - mkdir -p /etc/crowdsec/notifications +# Check if we can proceed with the automatic detection and hub + acquisition configuration. +# If the script is interactive, we ask the user for confirmation when it makes sense. +# +# arguments: none +# return: 0 if we can proceed with the configuration, 1 if we should skip it. +safe_to_configure() { + # if "wizard.sh" is in ACQUIS_YAML, never detect + if grep -q 'wizard.sh' "$ACQUIS_YAML" 2>/dev/null; then + cat <<-EOT >&2 + + A previous version of Crowdsec has detected the running services and put + datasource configuration in the file $ACQUIS_YAML. + + In this version, the same information goes in $ACQUIS_DIR, one + file per service. + + If you want to run the automated service detection again, please remove the + relevant sections from $ACQUIS_YAML or rename the file, and run "$0 --configure" + again. + + EOT + + ask_press_enter + return 1 + fi + + # if acquis.yaml exists but has no wizard.sh, ask for confirmation (if + # interactive) before detecting + if [ -f "$ACQUIS_YAML" ]; then + + if ! interactive; then + echo "Skipping automatic detection because $ACQUIS_YAML already exists." >&2 + echo "Run \"$0 --configure\" to detect the services again." >&2 + return 1 + fi + + cat <<-EOT >&2 + + A previous version of Crowdsec was already configured. + + If you run the automated service detection now, it will create new acquisition + directives in $ACQUIS_DIR, in addition to the ones already in $ACQUIS_YAML. + + When the configuration is done, please check the content of these files + to avoid duplicate log locations. + + EOT + + printf '%s ' "Do you want to run the service detection now? (y/N)" + read -r confirm + + if echo "$confirm" | grep -q '^[Nn]'; then + return 1 + fi + fi + + return 0 +} + + +detect_and_install_hub() { + if ! safe_to_configure; then + return 1 + fi + + tmp_dir=$(mktemp -d) + tmp_file="$tmp_dir/setup.yaml" + + if ! detect_edit_validate "$tmp_file"; then + echo + echo "Exiting crowdsec configuration, you can run it again with '$0 --configure'" >&2 + ask_press_enter - cp ${SLACK_PLUGIN_BINARY} ${CROWDSEC_PLUGIN_DIR} - cp ${SPLUNK_PLUGIN_BINARY} ${CROWDSEC_PLUGIN_DIR} - cp ${HTTP_PLUGIN_BINARY} ${CROWDSEC_PLUGIN_DIR} - cp ${EMAIL_PLUGIN_BINARY} ${CROWDSEC_PLUGIN_DIR} + rm -f "$tmp_file" + rmdir "$tmp_dir" + return 1 + fi + + echo "Installing hub objects...." + "$CSCLI_BIN_INSTALLED" setup install-hub "$tmp_file" + + mkdir -p "$ACQUIS_DIR" - if [[ ${DOCKER_MODE} == "false" ]]; then - cp -n ${SLACK_PLUGIN_CONFIG} /etc/crowdsec/notifications/ - cp -n ${SPLUNK_PLUGIN_CONFIG} /etc/crowdsec/notifications/ - cp -n ${HTTP_PLUGIN_CONFIG} /etc/crowdsec/notifications/ - cp -n ${EMAIL_PLUGIN_CONFIG} /etc/crowdsec/notifications/ + echo "Generating acquisition files..." + "$CSCLI_BIN_INSTALLED" setup datasources "$tmp_file" --to-dir "$ACQUIS_DIR" + + if [ ! -f "$ACQUIS_YAML" ]; then + cat <<-EOT >"$ACQUIS_YAML" + --- + # Your datasource configuration goes here. + EOT fi + + echo "Done" + + rm -f "$tmp_file" + rmdir "$tmp_dir" +} + +install_plugins() { + for plugin in email http slack splunk; do + mkdir -p "$PLUGIN_BINARIES_DEST" + install -m 0755 "$PLUGIN_BINARIES_SRC/$plugin/notification-$plugin" "$PLUGIN_BINARIES_DEST/" + + if [ "$DOCKER_MODE" = "false" ]; then + if [ -f "$PLUGIN_CONFIGURATION_DEST/$plugin/$plugin.yaml" ]; then + chmod 0600 "$PLUGIN_CONFIGURATION_DEST/$plugin/$plugin.yaml" + else + mkdir -p "$PLUGIN_CONFIGURATION_DEST/$plugin" + install -m 0600 "$PLUGIN_CONFIGURATION_SRC/$plugin/$plugin.yaml" "$PLUGIN_CONFIGURATION_DEST/$plugin/$plugin.yaml" + fi + fi + done } check_running_bouncers() { - #when uninstalling, check if user still has bouncers - BOUNCERS_COUNT=$(${CSCLI_BIN} bouncers list -o=raw | tail -n +2 | wc -l) - if [[ ${BOUNCERS_COUNT} -gt 0 ]] ; then - if [[ ${FORCE_MODE} == "false" ]]; then - echo "WARNING : You have at least one bouncer registered (cscli bouncers list)." - echo "WARNING : Uninstalling crowdsec with a running bouncer will let it in an unpredictable state." - echo "WARNING : If you want to uninstall crowdsec, you should first uninstall the bouncers." + # when uninstalling, check if the user still has bouncers + BOUNCERS_COUNT=$("$CSCLI_BIN" bouncers list -o=raw | tail -n +2 | wc -l) + if [ "$BOUNCERS_COUNT" -gt 0 ]; then + if [ "$FORCE_MODE" = "false" ]; then + echo "WARNING: You have at least one bouncer registered (cscli bouncers list)." + echo "WARNING: Uninstalling crowdsec with a running bouncer will leave it in an unpredictable state." + echo "WARNING: If you want to uninstall crowdsec, you should first uninstall the bouncers." echo "Specify --force to bypass this restriction." exit 1 - fi; + fi fi } # uninstall crowdsec and cscli uninstall_crowdsec() { - systemctl stop crowdsec.service 1>/dev/null - systemctl disable -q crowdsec.service 1>/dev/null - ${CSCLI_BIN} dashboard remove -f -y >/dev/null + crowdsec_service_stop + crowdsec_service_disable + # there is no way to know if the dashboard exists, so we have to ignore errors. + log_info "Removing dashboard..." + if "$CSCLI_BIN" dashboard remove -f -y; then + log_info "...done." + else + log_warn "...dashboard removal failed." + fi delete_bins - # tmp - rm -rf /tmp/data/ - ## end tmp - - find /etc/crowdsec -maxdepth 1 -mindepth 1 | grep -v "bouncer" | xargs rm -rf || echo "" - rm -f ${CROWDSEC_LOG_FILE} || echo "" - rm -f ${LAPI_LOG_FILE} || echo "" - rm -f ${CROWDSEC_DB_PATH} || echo "" - rm -rf ${CROWDSEC_LIB_DIR} || echo "" - rm -rf ${CROWDSEC_USR_DIR} || echo "" - rm -f ${SYSTEMD_PATH_FILE} || echo "" + rm -f -- "$CROWDSEC_LOG_FILE" "$LAPI_LOG_FILE" "$CROWDSEC_DB_PATH" "$SYSTEMD_PATH_FILE" + rm -rf -- "$CROWDSEC_LIB_DIR" "$CROWDSEC_USR_DIR" log_info "crowdsec successfully uninstalled" } - -function show_link { - echo "" - echo "Useful links to start with Crowdsec:" - echo "" - echo " - Documentation : https://doc.crowdsec.net/docs/getting_started/crowdsec_tour" - echo " - Crowdsec Hub : https://hub.crowdsec.net/ " - echo " - Open issues : https://github.com/crowdsecurity/crowdsec/issues" - echo "" - echo "Useful commands to start with Crowdsec:" - echo "" - echo " - sudo cscli metrics : https://doc.crowdsec.net/docs/observability/cscli" - echo " - sudo cscli decisions list : https://doc.crowdsec.net/docs/user_guides/decisions_mgmt" - echo " - sudo cscli hub list : https://doc.crowdsec.net/docs/user_guides/hub_mgmt" - echo "" - echo "Next step: visualize all your alerts and explore our community CTI : https://app.crowdsec.net" - echo "" +show_links() { + cat <<-EOT + + Useful links to start with Crowdsec: + + - Documentation : ${BOLD}https://doc.crowdsec.net/docs/getting_started/crowdsec_tour${RESET} + - Crowdsec Hub : ${BOLD}https://hub.crowdsec.net/${RESET} + - Open issues : https://github.com/crowdsecurity/crowdsec/issues + + Useful commands to start with Crowdsec: + + - sudo cscli metrics : https://doc.crowdsec.net/docs/observability/cscli + - sudo cscli decisions list : https://doc.crowdsec.net/docs/user_guides/decisions_mgmt + - sudo cscli hub list : https://doc.crowdsec.net/docs/user_guides/hub_mgmt + + Next step: visualize all your alerts and explore our community CTI - ${BOLD}https://app.crowdsec.net${RESET} + + CrowdSec alone will ${FG_YELLOW}${BOLD}not${RESET} block any IP address. If you want to block them, you must use a bouncer. + You can find them on ${BOLD}https://hub.crowdsec.net/browse/#bouncers${RESET} + + EOT } main() { - - if [ "$1" == "install" ] || [ "$1" == "configure" ] || [ "$1" == "detect" ]; then - if [ "${SILENT}" == "false" ]; then - which whiptail > /dev/null - if [ $? -ne 0 ]; then - log_fatal "whiptail binary is needed to use the wizard in interactive mode, exiting ..." - fi - fi - which envsubst > /dev/null - if [ $? -ne 0 ]; then - log_fatal "envsubst binary is needed to use do a full install with the wizard, exiting ..." + if [ "$1" = "install" ] || [ "$1" = "configure" ] || [ "$1" = "detect" ]; then + if ! command -v envsubst >/dev/null; then + log_fatal "envsubst binary is needed to use do a full install with the wizard, exiting..." fi fi - if [[ "$1" == "binupgrade" ]]; - then - if ! [ $(id -u) = 0 ]; then - log_err "Please run the wizard as root or with sudo" - exit 1 - fi + if [ "$1" = "binupgrade" ]; then + checkroot check_cs_version update_bins - return + return 0 fi - if [[ "$1" == "upgrade" ]]; - then - if ! [ $(id -u) = 0 ]; then - log_err "Please run the wizard as root or with sudo" - exit 1 - fi + if [ "$1" = "upgrade" ]; then + checkroot check_cs_version update_full - return + return 0 fi - if [[ "$1" == "configure" ]]; - then - if ! [ $(id -u) = 0 ]; then - log_err "Please run the wizard as root or with sudo" - exit 1 - fi - detect_services - ${CSCLI_BIN_INSTALLED} hub update - install_collection - genacquisition - if ! skip_tmp_acquis; then - mv "${TMP_ACQUIS_FILE}" "${ACQUIS_TARGET}" - fi - - return + if [ "$1" = "configure" ]; then + checkroot + "$CSCLI_BIN_INSTALLED" hub update + detect_and_install_hub + crowdsec_service_restart + show_links + return 0 fi - if [[ "$1" == "noop" ]]; - then - return + if [ "$1" = "noop" ]; then + return 0 fi - - if [[ "$1" == "uninstall" ]]; - then - if ! [ $(id -u) = 0 ]; then - log_err "Please run the wizard as root or with sudo" - exit 1 - fi + + if [ "$1" = "uninstall" ]; then + checkroot check_running_bouncers uninstall_crowdsec - return + return 0 fi - if [[ "$1" == "bininstall" ]]; - then - if ! [ $(id -u) = 0 ]; then - log_err "Please run the wizard as root or with sudo" - exit 1 - fi + if [ "$1" = "bininstall" ]; then + checkroot log_info "checking existing crowdsec install" detect_cs_install log_info "installing crowdsec" install_crowdsec - show_link - return + show_links + return 0 fi - if [[ "$1" == "install" ]]; - then - if ! [ $(id -u) = 0 ]; then - log_err "Please run the wizard as root or with sudo" - exit 1 - fi + if [ "$1" = "install" ]; then + checkroot log_info "checking if crowdsec is installed" detect_cs_install - ## Do make build before installing (as non--root) in order to have the binary and then install crowdsec as root + + # Run "make release" before installing (as non-root) in order to have the binary and then install crowdsec as root + log_info "installing crowdsec" install_crowdsec log_dbg "configuring ${CSCLI_BIN_INSTALLED}" - ${CSCLI_BIN_INSTALLED} hub update > /dev/null 2>&1 || (log_err "fail to update crowdsec hub. exiting" && exit 1) - # detect running services - detect_services - if ! [ ${#DETECTED_SERVICES[@]} -gt 0 ] ; then - log_err "No detected or selected services, stopping." + if ! "$CSCLI_BIN_INSTALLED" hub update >/dev/null 2>&1; then + log_err "fail to update crowdsec hub. exiting" exit 1 - fi; - - # Generate acquisition file and move it to the right folder - genacquisition - if ! skip_tmp_acquis; then - mv "${TMP_ACQUIS_FILE}" "${ACQUIS_TARGET}" fi - log_info "acquisition file path: ${ACQUIS_TARGET}" - # Install collections according to detected services - log_dbg "Installing needed collections ..." - install_collection + + "$CSCLI_BIN_INSTALLED" hub update # install patterns/ folder log_dbg "Installing patterns" - mkdir -p "${PATTERNS_PATH}" + mkdir -p "$PATTERNS_PATH" cp "./${PATTERNS_FOLDER}/"* "${PATTERNS_PATH}/" - # api register - ${CSCLI_BIN_INSTALLED} machines add --force "$(cat /etc/machine-id)" -a -f "${CROWDSEC_CONFIG_PATH}/${CLIENT_SECRETS}" || log_fatal "unable to add machine to the local API" - log_dbg "Crowdsec LAPI registered" - - ${CSCLI_BIN_INSTALLED} capi register || log_fatal "unable to register to the Central API" - log_dbg "Crowdsec CAPI registered" - + # register api + "$CSCLI_BIN_INSTALLED" machines add --force "$(cat /etc/machine-id)" -a -f "${CROWDSEC_CONFIG_PATH}/${CLIENT_SECRETS}" || log_fatal "unable to add machine to the local API" + log_dbg "Crowdsec LAPI registered" + + "$CSCLI_BIN_INSTALLED" capi register || log_fatal "unable to register to the Central API" + log_dbg "Crowdsec CAPI registered" + + detect_and_install_hub + systemctl enable -q crowdsec >/dev/null || log_fatal "unable to enable crowdsec" systemctl start crowdsec >/dev/null || log_fatal "unable to start crowdsec" log_info "enabling and starting crowdsec daemon" - show_link - return - fi - if [[ "$1" == "detect" ]]; - then - if ! skip_tmp_acquis; then - rm -f "${TMP_ACQUIS_FILE}" - fi - detect_services - if [[ ${DETECTED_SERVICES} == "" ]] ; then - log_err "No detected or selected services, stopping." - exit - fi; - log_info "Found ${#DETECTED_SERVICES[@]} supported services running:" - genacquisition - cat "${TMP_ACQUIS_FILE}" - if ! skip_tmp_acquis; then - rm "${TMP_ACQUIS_FILE}" - fi - return + show_links + return 0 fi + if [ "$1" = "detect" ]; then + detect_only + fi } usage() { - echo "Usage:" - echo " ./wizard.sh -h Display this help message." - echo " ./wizard.sh -d|--detect Detect running services and associated logs file" - echo " ./wizard.sh -i|--install Assisted installation of crowdsec/cscli and collections" - echo " ./wizard.sh --bininstall Install binaries and empty config, no wizard." - echo " ./wizard.sh --uninstall Uninstall crowdsec/cscli" - echo " ./wizard.sh --binupgrade Upgrade crowdsec/cscli binaries" - echo " ./wizard.sh --upgrade Perform a full upgrade and try to migrate configs" - echo " ./wizard.sh --unattended Install in unattended mode, no question will be asked and defaults will be followed" - echo " ./wizard.sh --docker-mode Will install crowdsec without systemd and generate random machine-id" - echo " ./wizard.sh -n|--noop Do nothing" - - exit 0 -} - -if [[ $# -eq 0 ]]; then -usage + echo "Usage:" + echo " ./wizard.sh -h Display this help message." + echo " ./wizard.sh -c|--configure Detect running services and install hub objects + acquis files" + echo " ./wizard.sh -d|--detect Detect running services and print the result" + echo " ./wizard.sh -i|--install Assisted installation of crowdsec/cscli and hub objects" + echo " ./wizard.sh --bininstall Install binaries and empty config, no wizard." + echo " ./wizard.sh --uninstall Uninstall crowdsec/cscli" + echo " ./wizard.sh --binupgrade Upgrade crowdsec/cscli binaries" + echo " ./wizard.sh --upgrade Perform a full upgrade and try to migrate configs" + echo " ./wizard.sh --unattended Install in unattended mode, no question will be asked and defaults will be followed" + echo " ./wizard.sh --docker-mode Will install crowdsec without systemd and generate random machine-id" + echo " ./wizard.sh -n|--noop Do nothing" +} + +if [ $# -eq 0 ]; then + usage + exit 0 fi -while [[ $# -gt 0 ]] -do +while [ $# -gt 0 ]; do key="${1}" case ${key} in --uninstall) @@ -763,7 +733,7 @@ do ACTION="upgrade" shift #past argument ;; - -i|--install) + -i | --install) ACTION="install" shift # past argument ;; @@ -776,15 +746,15 @@ do ACTION="bininstall" shift # past argument ;; - -c|--configure) + -c | --configure) ACTION="configure" shift # past argument ;; - -d|--detect) + -d | --detect) ACTION="detect" shift # past argument ;; - -n|--noop) + -n | --noop) ACTION="noop" shift # past argument ;; @@ -793,19 +763,19 @@ do ACTION="install" shift ;; - -f|--force) + -f | --force) FORCE_MODE="true" shift - ;; - -v|--verbose) + ;; + -v | --verbose) DEBUG_MODE="true" shift - ;; - -h|--help) + ;; + -h | --help) usage exit 0 ;; - *) # unknown option + *) # unknown option log_err "Unknown argument ${key}." usage exit 1 @@ -813,4 +783,6 @@ do esac done -main ${ACTION} +set_colors +main "$ACTION" +exit 0 From cbcd7e60499ed07491f7bc2f34489c7861993732 Mon Sep 17 00:00:00 2001 From: Marco Mariani Date: Thu, 20 Oct 2022 15:18:29 +0200 Subject: [PATCH 02/10] update go.mod, go.sum --- go.mod | 16 ++++++++++++++-- go.sum | 29 +++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 991c092d0b9..9c565c7509a 100644 --- a/go.mod +++ b/go.mod @@ -65,17 +65,22 @@ require ( ) require ( + github.com/Masterminds/semver v1.5.0 github.com/Masterminds/sprig/v3 v3.2.2 github.com/aquasecurity/table v1.8.0 github.com/beevik/etree v1.1.0 github.com/blackfireio/osinfo v1.0.3 + github.com/goccy/go-yaml v1.9.5 github.com/google/winops v0.0.0-20211216095627-f0e86eb1453b github.com/ivanpirog/coloredcobra v1.0.1 github.com/mattn/go-isatty v0.0.14 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/segmentio/kafka-go v0.4.34 + github.com/shirou/gopsutil/v3 v3.22.9 github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f + gopkg.in/yaml.v3 v3.0.1 + sigs.k8s.io/yaml v1.2.0 ) require ( @@ -96,6 +101,7 @@ require ( github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/go-units v0.4.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/analysis v0.19.16 // indirect github.com/go-openapi/inflect v0.19.0 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect @@ -111,7 +117,7 @@ require ( github.com/golang-jwt/jwt/v4 v4.2.0 // indirect github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/google/go-cmp v0.5.7 // indirect + github.com/google/go-cmp v0.5.9 // indirect github.com/gorilla/mux v1.7.3 // indirect github.com/hashicorp/hcl/v2 v2.13.0 // indirect github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect @@ -132,6 +138,8 @@ require ( github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/compress v1.15.7 // indirect github.com/leodido/go-urn v1.2.1 // indirect + github.com/lithammer/dedent v1.1.0 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mailru/easyjson v0.7.6 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect @@ -151,6 +159,7 @@ require ( github.com/opencontainers/image-spec v1.0.2-0.20211117181255-693428a734f5 // indirect github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/common v0.30.0 // indirect github.com/prometheus/procfs v0.7.3 // indirect github.com/rivo/uniseg v0.2.0 // indirect @@ -160,19 +169,22 @@ require ( github.com/spf13/cast v1.3.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/tidwall/gjson v1.13.0 // indirect + github.com/tklauser/go-sysconf v0.3.10 // indirect + github.com/tklauser/numcpus v0.4.0 // indirect github.com/ugorji/go/codec v1.2.6 // indirect github.com/vjeantet/grok v1.0.1 // indirect github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect + github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/zclconf/go-cty v1.10.0 // indirect go.mongodb.org/mongo-driver v1.9.0 // indirect golang.org/x/net v0.0.0-20220706163947-c90051bbdb60 // indirect golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 // indirect golang.org/x/text v0.3.7 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) replace golang.org/x/time/rate => github.com/crowdsecurity/crowdsec/pkg/time/rate v0.0.0 diff --git a/go.sum b/go.sum index e65e7392a60..c4dc39fd17a 100644 --- a/go.sum +++ b/go.sum @@ -48,6 +48,8 @@ github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8= @@ -168,6 +170,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -193,6 +196,8 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= @@ -318,6 +323,8 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= +github.com/goccy/go-yaml v1.9.5 h1:Eh/+3uk9kLxG4koCX6lRMAPS1OaMSAi+FJcya0INdB0= +github.com/goccy/go-yaml v1.9.5/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA= github.com/godbus/dbus v4.1.0+incompatible/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= @@ -372,6 +379,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -538,8 +547,12 @@ github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= +github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/lucasb-eyer/go-colorful v1.0.2/go.mod h1:0MS4r+7BZKSJ5mw4/S5MPN+qHFF1fYclkSPilDOKW0s= github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -553,6 +566,7 @@ github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcncea github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= @@ -637,6 +651,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= @@ -688,6 +704,8 @@ github.com/segmentio/kafka-go v0.4.34 h1:Dm6YlLMiVSiwwav20KY0AoY63s661FXevwJ3CVH github.com/segmentio/kafka-go v0.4.34/go.mod h1:GAjxBQJdQMB5zfNA21AhpaqOB2Mu+w3De4ni3Gbm8y0= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shirou/gopsutil/v3 v3.22.9 h1:yibtJhIVEMcdw+tCTbOPiF1VcsuDeTE4utJ8Dm4c5eA= +github.com/shirou/gopsutil/v3 v3.22.9/go.mod h1:bBYl1kjgEJpWpxeHmLI+dVHWtyAwfcmSBLDsp2TNT8A= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= @@ -734,6 +752,10 @@ github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JT github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw= +github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= +github.com/tklauser/numcpus v0.4.0 h1:E53Dm1HjH1/R2/aoCtXtPgzmElmn51aOkhCFSuZq//o= +github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn0= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= @@ -760,6 +782,8 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= +github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zclconf/go-cty v1.10.0 h1:mp9ZXQeIcN8kAwuqorjH+Q+njbJKjLrvB2yIh4q7U+0= github.com/zclconf/go-cty v1.10.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= @@ -924,6 +948,7 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -952,6 +977,7 @@ golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -963,6 +989,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1180,3 +1207,5 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= From 695ec92d3fcb07542dfb034e812ca44e4e248fa4 Mon Sep 17 00:00:00 2001 From: Marco Mariani Date: Thu, 20 Oct 2022 16:47:13 +0200 Subject: [PATCH 03/10] skip cache --- .github/workflows/ci_golangci-lint.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_golangci-lint.yml b/.github/workflows/ci_golangci-lint.yml index d45b42eb594..d815ea9a419 100644 --- a/.github/workflows/ci_golangci-lint.yml +++ b/.github/workflows/ci_golangci-lint.yml @@ -40,8 +40,8 @@ jobs: only-new-issues: false # Optional: if set to true then the all caching functionality will be complete disabled, # takes precedence over all other caching options. - skip-cache: false + skip-cache: true # Optional: if set to true then the action don't cache or restore ~/go/pkg. - skip-pkg-cache: false + skip-pkg-cache: true # Optional: if set to true then the action don't cache or restore ~/.cache/go-build. - skip-build-cache: false + skip-build-cache: true From 9ae8b7f75c0ba1e6fc2175c663234a152175c46b Mon Sep 17 00:00:00 2001 From: Marco Mariani Date: Fri, 21 Oct 2022 10:24:04 +0200 Subject: [PATCH 04/10] lint --- pkg/acquisition/modules/wineventlog/wineventlog_windows.go | 2 +- pkg/setup/detect.go | 2 +- pkg/setup/detect_test.go | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/acquisition/modules/wineventlog/wineventlog_windows.go b/pkg/acquisition/modules/wineventlog/wineventlog_windows.go index 77876555a37..2d93178199d 100644 --- a/pkg/acquisition/modules/wineventlog/wineventlog_windows.go +++ b/pkg/acquisition/modules/wineventlog/wineventlog_windows.go @@ -246,7 +246,7 @@ func (w *WinEventLogSource) UnmarshalConfig(yamlConfig []byte) error { w.config.Mode = configuration.TAIL_MODE - if config.XPathQuery != "" { + if w.config.XPathQuery != "" { w.query = w.config.XPathQuery } else { w.query, err = w.buildXpathQuery() diff --git a/pkg/setup/detect.go b/pkg/setup/detect.go index 94208ac7234..957f70a258d 100644 --- a/pkg/setup/detect.go +++ b/pkg/setup/detect.go @@ -229,7 +229,7 @@ func (os ExprOS) VersionIsLower(version string) (bool, error) { return !result, nil } -// ExprEnvironment is used to expose functions and values to to the rule engine. +// ExprEnvironment is used to expose functions and values to the rule engine. // It can cache the results of service detection commands, like systemctl etc. type ExprEnvironment struct { OS ExprOS diff --git a/pkg/setup/detect_test.go b/pkg/setup/detect_test.go index 90d71488df1..9dd3a7ce87d 100644 --- a/pkg/setup/detect_test.go +++ b/pkg/setup/detect_test.go @@ -14,6 +14,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/setup" ) +//nolint:dupword var fakeSystemctlOutput = `UNIT FILE STATE VENDOR PRESET crowdsec-setup-detect.service enabled enabled apache2.service enabled enabled @@ -205,7 +206,7 @@ func TestListSupported(t *testing.T) { "", }, { - "invalid yaml: blah blah", + "invalid yaml: blahblah", "blahblah", nil, "yaml: unmarshal errors:", From 8ab7f970382a8fdbcd019da2fe6d46447496b566 Mon Sep 17 00:00:00 2001 From: Marco Mariani Date: Fri, 21 Oct 2022 22:49:10 +0200 Subject: [PATCH 05/10] test fix --- pkg/setup/detect_test.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/pkg/setup/detect_test.go b/pkg/setup/detect_test.go index 9dd3a7ce87d..cf607e92b1f 100644 --- a/pkg/setup/detect_test.go +++ b/pkg/setup/detect_test.go @@ -74,15 +74,21 @@ func tempYAML(t *testing.T, content string) string { func TestPathExists(t *testing.T) { t.Parallel() - tests := []struct { + type test struct { path string expected bool - }{ - {"/boot", true}, - {"/tmp", true}, + } + + tests := []test{ {"/this-should-not-exist", false}, } + if runtime.GOOS == "windows" { + tests = append(tests, test{`C:\`, true}) + } else { + tests = append(tests, test{"/tmp", true}) + } + for _, tc := range tests { tc := tc env := setup.NewExprEnvironment(setup.DetectOptions{}, setup.ExprOS{}) @@ -970,7 +976,7 @@ func TestDetectDatasourceValidation(t *testing.T) { datasource: source: wineventlog`, expected: setup.Setup{Setup:[]setup.ServiceSetup{}}, - expectedErr: "", + expectedErr: "invalid datasource for foobar: event_channel or xpath_query must be set", }) } From 2324859ee3b2cafb6cf0ca5af4ac471fce48ecab Mon Sep 17 00:00:00 2001 From: Marco Mariani Date: Mon, 31 Oct 2022 10:56:47 +0100 Subject: [PATCH 06/10] don't skip cache --- .github/workflows/ci_golangci-lint.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_golangci-lint.yml b/.github/workflows/ci_golangci-lint.yml index d815ea9a419..d45b42eb594 100644 --- a/.github/workflows/ci_golangci-lint.yml +++ b/.github/workflows/ci_golangci-lint.yml @@ -40,8 +40,8 @@ jobs: only-new-issues: false # Optional: if set to true then the all caching functionality will be complete disabled, # takes precedence over all other caching options. - skip-cache: true + skip-cache: false # Optional: if set to true then the action don't cache or restore ~/go/pkg. - skip-pkg-cache: true + skip-pkg-cache: false # Optional: if set to true then the action don't cache or restore ~/.cache/go-build. - skip-build-cache: true + skip-build-cache: false From e60380a4b20f0e2999763bfc4c6b423398b519bb Mon Sep 17 00:00:00 2001 From: Marco Mariani Date: Mon, 31 Oct 2022 17:09:54 +0100 Subject: [PATCH 07/10] update config/detect.yaml to make it valid --- config/detect.yaml | 32 ++++++++++++++++++++++++++++---- pkg/setup/detect_test.go | 14 +++++++++++--- tests/bats/07_setup.bats | 1 - 3 files changed, 39 insertions(+), 8 deletions(-) diff --git a/config/detect.yaml b/config/detect.yaml index 2ad0ab9e37d..ded471254a1 100644 --- a/config/detect.yaml +++ b/config/detect.yaml @@ -18,6 +18,7 @@ detect: collections: - crowdsecurity/apache2 datasource: + source: file filenames: - /var/log/apache2/*.log labels: @@ -31,6 +32,7 @@ detect: collections: - crowdsecurity/apache2 datasource: + source: file filenames: - /var/log/httpd/*.log # XXX /var/log/*http*/*.log @@ -48,6 +50,7 @@ detect: collections: - crowdsecurity/asterisk datasource: + source: file labels: type: asterisk filenames: @@ -64,6 +67,7 @@ detect: collections: - crowdsecurity/caddy datasource: + source: file labels: type: caddy filenames: @@ -80,6 +84,7 @@ detect: collections: - crowdsecurity/dovecot datasource: + source: file labels: type: syslog filenames: @@ -96,6 +101,7 @@ detect: collections: - LePresidente/emby datasource: + source: file labels: type: emby filenames: @@ -132,6 +138,7 @@ detect: collections: - crowdsecurity/gitea datasource: + source: file labels: type: gitea filenames: @@ -148,6 +155,7 @@ detect: collections: - crowdsecurity/haproxy datasource: + source: file labels: type: haproxy filenames: @@ -163,10 +171,10 @@ detect: install: collections: - firewallservices/lemonldap-ng - datasource: - # XXX todo where are the logs? - labels: - type: syslog + #datasource: + # # XXX todo where are the logs? + # labels: + # type: syslog # # crowdsecurity/mariadb @@ -179,6 +187,7 @@ detect: collections: - crowdsecurity/mariadb datasource: + source: file labels: type: mysql filenames: @@ -195,6 +204,7 @@ detect: collections: - crowdsecurity/mysql datasource: + source: file labels: type: mysql filenames: @@ -211,6 +221,7 @@ detect: collections: - crowdsecurity/nginx datasource: + source: file labels: type: nginx filenames: @@ -223,6 +234,7 @@ detect: collections: - crowdsecurity/nginx datasource: + source: file labels: type: nginx filenames: @@ -239,6 +251,7 @@ detect: collections: - crowdsecurity/odoo datasource: + source: file labels: type: odoo filenames: @@ -260,6 +273,7 @@ detect: collections: - LePresidente/ombi datasource: + source: file labels: type: ombi filenames: @@ -277,6 +291,7 @@ detect: collections: - crowdsecurity/pgsql datasource: + source: file labels: type: postgres filenames: @@ -290,6 +305,7 @@ detect: collections: - crowdsecurity/pgsql datasource: + source: file labels: type: postgres filenames: @@ -306,6 +322,7 @@ detect: collections: - crowdsecurity/postfix datasource: + source: file labels: type: syslog filenames: @@ -322,6 +339,7 @@ detect: collections: - crowdsecurity/proftpd datasource: + source: file labels: type: proftpd filenames: @@ -339,6 +357,7 @@ detect: - fulljackz/pureftpd # XXX ? datasource: + source: file labels: type: syslog filenames: @@ -357,6 +376,7 @@ detect: collections: - crowdsecurity/smb datasource: + source: file labels: type: smb filenames: @@ -375,6 +395,7 @@ detect: collections: - crowdsecurity/sshd datasource: + source: file labels: type: syslog filenames: @@ -393,6 +414,7 @@ detect: collections: - crowdsecurity/suricata datasource: + source: file labels: type: suricata-evelogs filenames: @@ -409,6 +431,7 @@ detect: collections: - crowdsecurity/vsftpd datasource: + source: file labels: type: vsftpd filenames: @@ -425,6 +448,7 @@ detect: collections: - crowdsecurity/linux datasource: + source: file labels: type: syslog filenames: diff --git a/pkg/setup/detect_test.go b/pkg/setup/detect_test.go index cf607e92b1f..fa18d47217c 100644 --- a/pkg/setup/detect_test.go +++ b/pkg/setup/detect_test.go @@ -876,6 +876,17 @@ func TestDetectDatasourceValidation(t *testing.T) { source: file`, expected: setup.Setup{Setup:[]setup.ServiceSetup{}}, expectedErr: "while parsing {{.DetectYaml}}: yaml: unmarshal errors:\n line 6: field source not found in type setup.Service", + }, { + name: "source is mismatched", + config: ` + version: 1.0 + detect: + foobar: + datasource: + source: journalctl + filename: /path/to/file.log`, + expected: setup.Setup{Setup:[]setup.ServiceSetup{}}, + expectedErr: "invalid datasource for foobar: cannot parse JournalCtlSource configuration: yaml: unmarshal errors:\n line 1: field filename not found in type journalctlacquisition.JournalCtlConfiguration", }, { name: "source file: required fields", config: ` @@ -999,6 +1010,3 @@ func TestDetectDatasourceValidation(t *testing.T) { }) } } - - - diff --git a/tests/bats/07_setup.bats b/tests/bats/07_setup.bats index e5524b042d5..1d31c621e81 100644 --- a/tests/bats/07_setup.bats +++ b/tests/bats/07_setup.bats @@ -783,7 +783,6 @@ update-notifier-motd.timer enabled enabled } @test "cscli setup validate" { - # an empty file is not enough run -1 --separate-stderr cscli setup validate /dev/null assert_output "EOF" From a76d3f0e9a867784562fbd2d9e21d4e8d69832aa Mon Sep 17 00:00:00 2001 From: Marco Mariani Date: Wed, 2 Nov 2022 14:10:15 +0100 Subject: [PATCH 08/10] quote --- .golangci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.golangci.yml b/.golangci.yml index 3047402bffc..9ff0947f06d 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -181,6 +181,9 @@ linters: issues: + # “Look, that’s why there’s rules, understand? So that you think before you + # break ‘em.” ― Terry Pratchett + max-issues-per-linter: 0 max-same-issues: 10 exclude-rules: From 8496d1deffce1ef57e81a4b8d7e08a3c8480e582 Mon Sep 17 00:00:00 2001 From: Marco Mariani Date: Thu, 17 Nov 2022 12:31:29 +0100 Subject: [PATCH 09/10] don't check for reload-time log message which is sometimes on the old log file (?) --- tests/bats/01_crowdsec.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bats/01_crowdsec.bats b/tests/bats/01_crowdsec.bats index a60b576dd9b..3841cdbe508 100644 --- a/tests/bats/01_crowdsec.bats +++ b/tests/bats/01_crowdsec.bats @@ -125,7 +125,7 @@ teardown() { ls -la "$log_new" || true cat "$log_new" || true - assert_file_contains "$log_new" "CrowdSec Local API listening on 127.0.0.1:8080" + # assert_file_contains "$log_new" "CrowdSec Local API listening on 127.0.0.1:8080" assert_file_contains "$log_new" "Reload is finished" run -0 ./instance-crowdsec stop From 9f953f6a581de3eee9e295f0c265732f0dafa3dc Mon Sep 17 00:00:00 2001 From: Marco Mariani Date: Thu, 24 Nov 2022 12:27:04 +0100 Subject: [PATCH 10/10] Revert "don't check for reload-time log message which is sometimes on the old log file (?)" This reverts commit 8496d1deffce1ef57e81a4b8d7e08a3c8480e582. --- tests/bats/01_crowdsec.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bats/01_crowdsec.bats b/tests/bats/01_crowdsec.bats index 3841cdbe508..a60b576dd9b 100644 --- a/tests/bats/01_crowdsec.bats +++ b/tests/bats/01_crowdsec.bats @@ -125,7 +125,7 @@ teardown() { ls -la "$log_new" || true cat "$log_new" || true - # assert_file_contains "$log_new" "CrowdSec Local API listening on 127.0.0.1:8080" + assert_file_contains "$log_new" "CrowdSec Local API listening on 127.0.0.1:8080" assert_file_contains "$log_new" "Reload is finished" run -0 ./instance-crowdsec stop