Merge pull request #141 from rackspace-cookbooks/cleanup_martinb3

Clean up elkstack's workarounds, java, deps

.kitchen.yml

@@ -46,6 +46,9 @@ provisioner:
         number_of_replicas: 0
         # so the kitchen node doesn't have unallocated replica shards
         # and comes up green (healthy) instead of yellow
+      discovery:
+        zen:
+          minimum_master_nodes: 1 # since search returns more than one, but they are fake
     elkstack:
       config:
         kibana:
@@ -64,42 +67,20 @@ platforms:
 
 suites:
   - name: default # server
-    data_bags_path: "test/integration/default/data_bags"
-    encrypted_data_bag_secret_key_path: "test/integration/default/encrypted_data_bag_secret"
     run_list:
-      - recipe[elkstack::java]
-      - recipe[elkstack::cluster] # not testing single, it's practically the same
-    attributes:
-      elasticsearch:
-        discovery:
-          zen:
-            minimum_master_nodes: 1 # since search returns more than one, but they are fake
+      - recipe[java::default]
+      - recipe[elkstack::default]
 
-  - name: lumberjack # server with lumberjack disabled
-    data_bags_path: "test/integration/default/data_bags"
-    encrypted_data_bag_secret_key_path: "test/integration/default/encrypted_data_bag_secret"
-    run_list:
-      - recipe[elkstack::java]
-      - recipe[elkstack::cluster] # not testing single, it's practically the same
-    attributes:
-      elkstack:
-        config:
-          lumberjack_data_bag: false
-      elasticsearch:
-        discovery:
-          zen:
-            minimum_master_nodes: 1 # since search returns more than one, but they are fake
-
-  - name: agent
-    data_bags_path: "test/integration/agent/data_bags"
-    encrypted_data_bag_secret_key_path: "test/integration/agent/encrypted_data_bag_secret"
+  - name: agent # java agent with server
     run_list:
       - recipe[wrapper::logstash_override]
-      - recipe[elkstack::java]
+      - recipe[java::default]
       - recipe[elkstack::agent]
 
-  - name: forwarder
-    data_bags_path: "test/integration/agent/data_bags"
-    encrypted_data_bag_secret_key_path: "test/integration/agent/encrypted_data_bag_secret"
+  - name: forwarder # alternative golang agent with server
+    data_bags_path: "test/integration/forwarder/data_bags"
+    encrypted_data_bag_secret_key_path: "test/integration/forwarder/encrypted_data_bag_secret"
     run_list:
+      - recipe[java::default]
+      - recipe[elkstack::default]
       - recipe[elkstack::forwarder]

Berksfile

@@ -2,28 +2,13 @@ source "https://api.berkshelf.com"
 
 metadata
 
+cookbook 'java'
+
 # until https://github.com/elastic/cookbook-elasticsearch/pull/230
 cookbook 'elasticsearch', '~> 0.3', git:'[email protected]:racker/cookbook-elasticsearch.git'
 
-# until https://github.com/poise/python/pull/120
-cookbook 'python', git: '[email protected]:racker/python.git'
-
-# until https://github.com/lusis/chef-logstash/issues/394
-cookbook 'logstash', git: '[email protected]:lusis/chef-logstash.git'
-
-cookbook 'rackspace_iptables', git: '[email protected]:rackspace-cookbooks/rackspace_iptables.git'
-cookbook 'rackspacecloud', git: '[email protected]:rackspace-cookbooks/rackspacecloud.git'
-cookbook 'rackspace_gluster', git: '[email protected]:rackspace-cookbooks/rackspace_gluster.git'
-cookbook 'rackops_rolebook', git: '[email protected]:rackops/rackops_rolebook.git'
-
-# not published
-cookbook 'rackspace_cloudbackup', git:'[email protected]:rackspace-cookbooks/rackspace_cloudbackup.git'
-
 group :integration do
   cookbook 'wrapper', path: 'test/fixtures/cookbooks/wrapper'
   cookbook 'apt'
   cookbook 'yum'
 end
-
-# until https://github.com/opscode-cookbooks/openssl/pull/11
-cookbook 'openssl', git: '[email protected]:racker/openssl.git'

README.md

@@ -22,9 +22,9 @@ upstream attributes have been exposed/overriden for our needs.
 
 - This cookbook requires java. Because not everyone has the same desires for
 java versions, concurrently installed versions, or particular vendor versions,
-this cookbook simply assumes you have already satisfied this requirement. If you
-want just 'some java', feel free to use the `::java` recipe and it will include
-the community java cookbook with default values.
+this cookbook simply assumes you have already satisfied this requirement. This
+cookbook _does_ ship with default attributes to make the community cookbook use
+Java 7 over the default of Java 6.
 
 - You must update your Berksfile to use this cookbook. Due to the upstream
 changes constantly occuring, you should consult the `Berksfile` in this cookbook
@@ -115,12 +115,6 @@ CentOS 6.5
     <td>Default logstash instance name</td>
     <td><tt>server</tt></td>
   </tr>
-  <tr>
-    <td><tt>['elkstack']['config']['cluster']</tt></td>
-    <td>Boolean</td>
-    <td>Whether to search for and connect Elasticsearch to cluster nodes</td>
-    <td><tt>false</tt></td>
-  </tr>
   <tr>
     <td><tt>['elasticsearch']['discovery']['search_query']</tt></td>
     <td>String</td>
@@ -228,20 +222,10 @@ To override anything else, set the appropriate node hash (`logstash`, `kibana`,
 
 ### elkstack::default
 
-Default recipe, does not do anything.
-
-### elkstack::single
-
 A simple wrapper recipe that sets up Elasticsearch, Logstash, and Kibana. Also
 configures an rsyslog sink into logstash on the local box. Everything except
 Logstash and Kibana is locked down to listen only on localhost.
 
-### elkstack::cluster
-
-A simple wrapper recipe that sets up Elasticsearch, Logstash, and Kibana. Also
-configures an rsyslog sink into logstash on the local box. Sets the cluster flag
-so that the elasticsearch recipe builds it in a cluster-friendly way.
-
 ### elkstack::agent
 
 A simple wrapper recipe that sets up a logstash agent on the local box. Also
@@ -284,18 +268,11 @@ Leans on the upstream `lusis/chef-kibana` cookbook for most of its work. Sets up
 an nginx site for kibana by default. By default, it also does not pass through
 most of the http paths directly to elasticsearch (whitelist).
 
-### elkstack::java
-
-Wrapper for a java recipe. This is not included on the run list normally, so if
-you don't already, you must include this recipe or get another JVM installed
-before including anything else in this cookbook.
-
 ### elkstack::newrelic
 
 Validates if there is a newrelic license set and based on that, see if the node
-is tagged as 'elkstack' or 'elkstack_cluster' and creates a file with
-elasticsearch details. Installs python, pip and setuptools packages in order to
-support newrelic_meetme_plugin
+is tagged as 'elkstack' and creates a file with elasticsearch details. Installs
+python, pip and setuptools packages in order to support newrelic_meetme_plugin
 
 ## elkstack::acl
 

attributes/default.rb

@@ -4,8 +4,8 @@
 # the name for an agent logstash instance, affects initscript names and other things
 default['elkstack']['config']['logstash']['agent_name'] = 'agent'
 
-# default to not running the cluster search recipe
-default['elkstack']['config']['cluster'] = false
+# attempt to use lumberjack protocol for java agents?
+default['elkstack']['config']['agent_protocol'] = 'tcp_udp' # could also be lumberjack
 
 # attempt to use performance cloud data disk
 default['elkstack']['config']['data_disk']['disk_config_type'] = false

attributes/elasticsearch.rb

@@ -28,11 +28,14 @@
 default['elasticsearch']['network']['host'] = '_eth1:ipv4_'
 
 # rubocop:disable LineLength
-default['elasticsearch']['discovery']['search_query'] = "tags:elkstack_cluster AND chef_environment:#{node.chef_environment} AND elasticsearch_cluster_name:#{node['elasticsearch']['cluster']['name']} AND NOT name:#{node.name}"
+default['elasticsearch']['discovery']['search_query'] = "tags:elkstack AND chef_environment:#{node.chef_environment} AND elasticsearch_cluster_name:#{node['elasticsearch']['cluster']['name']} AND NOT name:#{node.name}"
 # rubocop:enable LineLength
 
 # by default, won't do multicast
 default['elasticsearch']['discovery']['zen']['ping']['multicast']['enabled'] = false
 
 # in order to use kibana, we must enable dynamic scripting
 default['elasticsearch']['custom_config']['script.disable_dynamic'] = false
+
+# get on a much newer java, required by ES
+default['java']['jdk_version']           = '7' # newer ES requires

attributes/forwarder.rb

@@ -3,10 +3,6 @@
 default['logstash_forwarder']['user'] = 'root'
 default['logstash_forwarder']['group'] = 'root'
 
-default['logstash_forwarder']['app_dir'] = '/opt/logstash-forwarder'
-default['logstash_forwarder']['git_repo'] = 'https://github.com/elastic/logstash-forwarder.git'
-default['logstash_forwarder']['git_revision'] = 'v0.3.1'
-
 default['logstash_forwarder']['config']['network']['servers'] = []
 default['logstash_forwarder']['config']['network']['ssl certificate'] = './logstash-forwarder.crt'
 default['logstash_forwarder']['config']['network']['ssl key'] = './logstash-forwarder.key'

attributes/kibana.rb

@@ -1,6 +1,3 @@
-
-default['elkstack']['kibana4_workaround'] = false
-
 default['kibana']['web_dir'] = '/opt/kibana/current'
 default['kibana']['webserver_port'] = 443
 default['kibana']['webserver_scheme'] = 'https://'

files/default/logstash-forwarder-init

@@ -9,7 +9,7 @@
 PATH=/sbin:/usr/sbin:/bin:/usr/bin
 DESC="log shipper"
 NAME=logstash-forwarder
-DAEMON=/opt/logstash-forwarder/logstash-forwarder
+DAEMON=/opt/logstash-forwarder/bin/logstash-forwarder
 DAEMON_ARGS="-config /etc/logstash-forwarder -spool-size 100"
 PIDFILE=/var/run/$NAME.pid
 SCRIPTNAME=/etc/init.d/$NAME

metadata.rb

@@ -13,16 +13,13 @@
 depends 'cron', '~> 1.4.3'
 depends 'elasticsearch', '~> 0.3'
 depends 'htpasswd'
-depends 'java'
-depends 'golang'
 depends 'kibana_lwrp'
 depends 'line'
 depends 'logstash'
 depends 'openssl'
 depends 'newrelic_meetme_plugin'
 depends 'nginx'
 depends 'platformstack'
-depends 'python'
 depends 'rsyslog'
 depends 'runit'
 depends 'stack_commons', '>= 0.0.39'

recipes/_secrets.rb → recipes/_lumberjack_secrets.rb

@@ -35,61 +35,16 @@
   end
 end
 
-# generate our own keypair since we don't seem to have one
-if lumberjack_secrets.nil?
-  Chef::Log.warn("Generating a new lumberjack keypair and data bag item #{lumberjack_data_bag}/secrets")
-  cert_file = "#{Chef::Config[:file_cache_path]}/lumberjack.crt"
-  key_file = "#{Chef::Config[:file_cache_path]}/lumberjack.key"
-  openssl_x509 cert_file do
-    common_name 'elkstack'
-    org 'elkstack'
-    org_unit 'elkstack'
-    country 'US'
-    key_file key_file
-    action :nothing
-  end.run_action(:create) # do it at compilation
-
-  ruby_block 'read generated keypair from disk' do
-    block do
-      key_file_contents = IO.read(key_file)
-      cert_file_contents = IO.read(cert_file)
-      node.run_state['lumberjack_decoded_key_tmp'] = Base64.encode64(key_file_contents).tr("\n", '')
-      node.run_state['lumberjack_decoded_certificate_tmp'] = Base64.encode64(cert_file_contents).tr("\n", '')
-    end
-    action :nothing
-  end.run_action(:run) # do it at compilation
-
-  key_contents = node.run_state['lumberjack_decoded_key_tmp']
-  certificate_contents = node.run_state['lumberjack_decoded_certificate_tmp']
-
-  # try to create a data bag and put a random keypair in it next
-  secrets = {
-    'id' => 'secrets',
-    'key' => key_contents,
-    'certificate' => certificate_contents
-  }
-
-  # unencrypted data bag if we just need a shared secret for ourselves
-  lumberjack_secrets_bag = Chef::DataBag.new
-  lumberjack_secrets_bag.name(lumberjack_data_bag)
-  lumberjack_secrets_bag.save
-
-  lumberjack_secrets = Chef::DataBagItem.new
-  lumberjack_secrets.data_bag(lumberjack_data_bag)
-  lumberjack_secrets.raw_data = secrets
-  lumberjack_secrets.save
-end
-
 # now try to use the data bag
 if !lumberjack_secrets.nil? && lumberjack_secrets['key'] && lumberjack_secrets['certificate']
   node.run_state['lumberjack_decoded_key'] = Base64.decode64(lumberjack_secrets['key'])
   node.run_state['lumberjack_decoded_certificate'] = Base64.decode64(lumberjack_secrets['certificate'])
 elsif !lumberjack_secrets.nil?
-  fail 'Found a data bag for lumberjack secrets, but it was missing \'key\' and \'certificate\' data bag items'
+  Chef::Log.warn('Found a data bag for lumberjack secrets, but it was missing \'key\' and \'certificate\' data bag items')
 elsif lumberjack_secrets.nil?
-  fail 'Could not find an encrypted or unencrypted data bag to use as a lumberjack keypair, and could not generate a keypair either'
+  Chef::Log.warn('Could not find an encrypted or unencrypted data bag to use as a lumberjack keypair')
 else
-  fail 'Unable to complete lumberjack keypair configuration'
+  Chef::Log.warn('Unable to complete lumberjack keypair configuration')
 end
 
 logstash_basedir = node.deep_fetch('logstash', 'instance_default', 'basedir')

recipes/acl.rb

@@ -11,14 +11,13 @@
 # main point of elkstack, open syslog and lumberjack ports
 add_iptables_rule('INPUT', '-p tcp --dport 5959 -j ACCEPT', 9997, 'allow syslog entries inbound')
 add_iptables_rule('INPUT', '-p tcp --dport 5960 -j ACCEPT', 9997, 'allow lumberjack protocol inbound')
+add_iptables_rule('INPUT', '-p tcp --dport 5961 -j ACCEPT', 9997, 'allow tcp protocol inbound')
+add_iptables_rule('INPUT', '-p tcp --dport 5962 -j ACCEPT', 9997, 'allow udp protocol inbound')
 
-should_cluster = node.deep_fetch('elkstack', 'config', 'cluster')
-if !should_cluster.nil? && should_cluster
-  include_recipe 'elasticsearch::search_discovery'
-  es_nodes = node['elasticsearch']['discovery']['zen']['ping']['unicast']['hosts']
-  es_nodes.split(',').each do |host|
-    add_iptables_rule('INPUT', "-p tcp -s #{host} --dport 9300 -j ACCEPT", 9996, "allow ES host #{host} to connect")
-  end
+include_recipe 'elasticsearch::search_discovery'
+es_nodes = node['elasticsearch']['discovery']['zen']['ping']['unicast']['hosts']
+es_nodes.split(',').each do |host|
+  add_iptables_rule('INPUT', "-p tcp -s #{host} --dport 9300 -j ACCEPT", 9996, "allow ES host #{host} to connect")
 end
 
 # allow web clients to hit kibana on port 80 and 443

recipes/agent.rb

@@ -61,8 +61,24 @@
   chef_environment: node.chef_environment
 }
 
-include_recipe 'elkstack::_secrets'
-unless node.run_state['lumberjack_decoded_certificate'].nil? || node.run_state['lumberjack_decoded_certificate'].nil?
+# preload any lumberjack key or cert that might be available
+include_recipe 'elkstack::_lumberjack_secrets'
+lumberjack_keypair = node.run_state['lumberjack_decoded_key'] && node.run_state['lumberjack_decoded_certificate']
+
+# default is 'tcp_udp'
+if node['elkstack']['config']['agent_protocol'] == 'tcp_udp'
+  # TODO: udp and tcp senders
+
+  my_templates['output_tcp'] = 'logstash/output_tcp.conf.erb'
+  my_templates['output_udp'] = 'logstash/output_udp.conf.erb'
+
+  template_variables[:output_tcp_host] = elk_nodes.split(',').first
+  template_variables[:output_tcp_port] = 5961
+  template_variables[:output_udp_host] = elk_nodes.split(',').first
+  template_variables[:output_udp_port] = 5962
+
+# if flag is set *and* key & cert are available
+elsif node['elkstack']['config']['agent_protocol'] == 'lumberjack' && lumberjack_keypair
   my_templates['output_lumberjack'] = 'logstash/output_lumberjack.conf.erb'
   template_variables['output_lumberjack_ssl_certificate'] = "#{node['logstash']['instance_default']['basedir']}/lumberjack.crt"
   # template_variables['output_lumberjack_ssl_key'] = "#{node['logstash']['instance_default']['basedir']}/lumberjack.key"

recipes/default.rb

@@ -1,7 +1,18 @@
 # Encoding: utf-8
 #
 # Cookbook Name:: elkstack
-# Recipe:: default
+# Recipe:: single
 #
 # Copyright 2014, Rackspace
 #
+
+# base stack requirements for an all-in-one node
+include_recipe 'elkstack::_server'
+
+# include components
+include_recipe 'elkstack::elasticsearch'
+include_recipe 'elkstack::logstash'
+include_recipe 'elkstack::kibana'
+
+# see attributes, will forward to logstash on localhost
+include_recipe 'rsyslog::client'