Graylog: separate hosts groups by scope

This commit reviews the entire Graylog playbook (and role) so that it
will be to totally separate the hosts by their scope.
For each scope comes a specific group:

- graylog_mongodb: hosts deploying MongoDB cluster;
- graylog_elastic: hosts deploying Elasticsearch cluster;
- graylog_server: hosts deploying Graylog server;
- graylog_frontend: hosts deployinh Graylog VIP (NGinx/Keepalived);

All the hosts will be grouped under a specific macro group declared as
follows:

[graylog:children]
graylog_mongodb
graylog_elastic
graylog_server
graylog_frontend

The main playbook will configure the various roles on the different
groups so that it is now possible to split the entire Graylog setup
between single purpose machine groups.

Author: rascasoft

graylog.yml

@@ -13,15 +13,23 @@
     - import_tasks: 'roles/graylog-server/tasks/common.yml'
 
 # Configurations for backend servers
-- hosts: graylog_backend
+- hosts: graylog_mongodb
   become: true
   tasks:
     - import_role:
         name: 'mongodb-cluster'
 
+# Configurations for elastic servers
+- hosts: graylog_elastic
+  become: true
+  tasks:
     - import_role:
         name: 'ansible-elasticsearch'
 
+# Configurations for graylog servers
+- hosts: graylog_server
+  become: true
+  tasks:
     - import_role:
         name: 'graylog-server'
 

inventory/lab/group_vars/graylog.yml

@@ -0,0 +1,32 @@
+---
+
+# Configurations for ElasticSearch cluster
+es_repo_name: "6.x"
+es_version: "6.8.10"
+es_cluster_name: "graylog-es-cluster"
+es_instance_name: "graylog-es-cluster1"
+es_scripts: false
+es_templates: false
+es_version_lock: false
+es_heap_size: "512m"
+es_data_dirs:
+  - "/store/elasticsearch/data"
+es_log_dir: "/store/elasticsearch/log"
+es_work_dir: "/store/elasticsearch/tmp"
+es_bulk_queue_size: -1
+es_config:
+  cluster.name: "graylog-es-cluster"
+  network.host:
+    - "{{ ansible_eth0.ipv4.address }}"
+    - "_local_"
+  transport.host: "{{ graylog_elastic_addr }}"
+  discovery.zen.ping.unicast.hosts:
+    - 192.168.122.21
+    - 192.168.122.22
+    - 192.168.122.23
+  http.port: 9200
+  transport.tcp.port: 9300
+  node.data: true
+  node.master: true
+  node.name: "{{ ansible_hostname }}"
+  bootstrap.memory_lock: true

inventory/lab/group_vars/graylog_backend.yml

@@ -7,7 +7,7 @@ keepalived_vrrp_vips:
   - '192.168.122.25/24'
 
 # Nginx configuration
-nginx_backend_servers: "{{ groups['graylog_backend'] }}"
+nginx_backend_servers: "{{ groups['graylog_server'] }}"
 nginx_backend_port: 9000
 nginx_http_name: logmaster
 nginx_http_port: 80

inventory/lab/group_vars/graylog_elastic.yml

@@ -0,0 +1,19 @@
+---
+
+# Configurations for MongoDB cluster
+mongodb_master_node: "graylog-1" 
+mongodb_nodes: "graylog-1,graylog-2,graylog-3"
+# Config Server
+mongodb_cs_bind_ip: "{{ graylog_mongodb_addr }}"
+mongodb_cs_logpath: "/store/mongodb/cs/log"
+mongodb_cs_dbpath: "/store/mongodb/cs/data"
+mongodb_cs_replica_set_name: "configReplSet"
+# Query Router
+mongodb_qr_bind_ip: "{{ graylog_mongodb_addr }}"
+mongodb_qr_logpath: "/store/mongodb/qr/log"
+mongodb_qr_rs_nodes: "graylog-1:27019,graylog-2:27019,graylog-3:27019"
+# Shard Node
+mongodb_sn_bind_ip: "{{ graylog_mongodb_addr }}"
+mongodb_sn_logpath: "/store/mongodb/sn/log"
+mongodb_sn_dbpath: "/store/mongodb/sn/data"
+mongodb_sn_replica_set_name: "graylog-shard-rs0"

inventory/lab/group_vars/graylog_frontend.yml

@@ -0,0 +1,20 @@
+---
+
+# Configurations for Graylog Server
+graylog_server_password_secret: "2jueVqZpwLLjaWxV"
+graylog_server_admin_password: "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918"
+graylog_server_inputbuffer_size: 524288
+graylog_server_rest_listen_ip: "{{ ansible_eth0.ipv4.address }}"
+graylog_server_web_listen_ip: "{{ ansible_eth0.ipv4.address }}"
+graylog_server_timezone: 'Europe/Rome'
+graylog_server_elasticsearch_cluster_name: "graylog-es-cluster"
+graylog_server_elasticsearch_hosts: "http://graylog-1:9200,http://graylog-2:9200,http://graylog-3:9200"
+graylog_server_mongodb_uri: "mongodb://graylog-1:27017,graylog-2:27017,graylog-3:27017/graylog"
+graylog_server_message_journal_dir: '/store/graylog/journal'
+#graylog_server_processbuffer_processors: 32
+#graylog_server_outputbuffer_processors: 156
+#graylog_server_outputbuffer_processor_keep_alive_time: 5000
+#graylog_server_outputbuffer_processor_threads_core_pool_size: 36
+#graylog_server_outputbuffer_processor_threads_max_pool_size: 156
+#graylog_server_min_heap: '2048m'
+#graylog_server_max_heap: '2048m'

inventory/lab/group_vars/graylog_mongodb.yml

@@ -1,7 +1,7 @@
 ---
 
-interconnect_name: 'graylog-1-int'
-interconnect_addr: '192.168.122.21'
+graylog_elastic_addr: '192.168.122.21'
+graylog_mongodb_addr: '192.168.122.21'
 server_weight: 1
 graylog_server_inputbuffer_size: 65536
 graylog_server_is_master: true

inventory/lab/group_vars/graylog_server.yml

@@ -1,7 +1,7 @@
 ---
 
-interconnect_name: 'graylog-2-int'
-interconnect_addr: '192.168.122.22'
+graylog_elastic_addr: '192.168.122.22'
+graylog_mongodb_addr: '192.168.122.22'
 server_weight: 3
 graylog_server_inputbuffer_size: 65536
 keepalived_vrrp_priority: 100

inventory/lab/host_vars/graylog-1.yml

@@ -1,7 +1,7 @@
 ---
 
-interconnect_name: 'graylog-3-int'
-interconnect_addr: '192.168.122.23'
+graylog_elastic_addr: '192.168.122.23'
+graylog_mongodb_addr: '192.168.122.23'
 server_weight: 3
 graylog_server_inputbuffer_size: 65536
 keepalived_vrrp_priority: 120

inventory/lab/host_vars/graylog-2.yml

@@ -1,16 +1,29 @@
-# New Graylog Environment
-[graylog]
+# Graylog hosts
+[graylog:children]
+graylog_mongodb
+graylog_elastic
+graylog_server
+graylog_frontend
+
+# Graylog Mongodb servers
+[graylog_mongodb]
+graylog-1
+graylog-2
+graylog-3
+
+# Graylog Elastic servers
+[graylog_elastic]
 graylog-1
 graylog-2
 graylog-3
 
-# Backend servers (ElasticSearch + Graylog Server + MongoDB)
-[graylog_backend]
+# Graylog servers
+[graylog_server]
 graylog-1
 graylog-2
 graylog-3
 
-# Frontend servers (VIP + Graylog Web + ClusterControl)
+# Frontend servers (NGinx/Keepalived)
 [graylog_frontend]
 graylog-1
 graylog-2

inventory/lab/host_vars/graylog-3.yml

@@ -16,10 +16,10 @@
     msg: 'Variable graylog_server_elasticsearch_cluster_name required'
   when: graylog_server_elasticsearch_cluster_name is not defined
 
-- name: Check graylog_server_elasticsearch_discovery_zen_ping_unicast_hosts variable
+- name: Check graylog_server_elasticsearch_hosts variable
   fail:
-    msg: 'Variable graylog_server_elasticsearch_discovery_zen_ping_unicast_hosts required'
-  when: graylog_server_elasticsearch_discovery_zen_ping_unicast_hosts is not defined
+    msg: 'Variable graylog_server_elasticsearch_hosts required'
+  when: graylog_server_elasticsearch_hosts is not defined
 
 - name: Check graylog_server_mongodb_uri variable
   fail:

inventory/lab/hosts

@@ -47,11 +47,3 @@
     state: present
   with_items:
     - "{{ groups['graylog'] }}"
-
-- name: "Build hosts file for backend interconnect graylog group"
-  lineinfile:
-    dest: /etc/hosts
-    regexp: "^{{ hostvars[item].interconnect_addr }} {{ hostvars[item].interconnect_name }}$"
-    line: "{{ hostvars[item].interconnect_addr }} {{ hostvars[item].interconnect_name }}"
-    state: present
-  with_items: "{{ groups['graylog_backend'] }}"

roles/graylog-server/tasks/check_variables.yml

@@ -15,8 +15,7 @@ http_bind_address = {{ graylog_server_http_bind_address_ip }}:{{ graylog_server_
 elasticsearch_shards = {{ graylog_server_elasticsearch_shards }}
 elasticsearch_replicas = {{ graylog_server_elasticsearch_replicas }}
 elasticsearch_cluster_name = {{ graylog_server_elasticsearch_cluster_name }}
-elasticsearch_discovery_zen_ping_unicast_hosts = {{ graylog_es_cluster_interconnect_nodes }}
-elasticsearch_network_host = {{ interconnect_addr }}
+elasticsearch_hosts = {{ graylog_server_elasticsearch_hosts }}
 
 # Buffer
 processbuffer_processors = {{graylog_server_processbuffer_processors}}

roles/graylog-server/tasks/common.yml

@@ -94,9 +94,9 @@
         src: 'mongo-check-rs.js'
 
     # Mongo CS - Config Server
-    - name: "Check for port {{ mongodb_cs_port }} on {{ graylog_mongo_master_node }} to be accessible"
+    - name: "Check for port {{ mongodb_cs_port }} on {{ mongodb_master_node }} to be accessible"
       wait_for:
-        host: "{{ graylog_mongo_master_node }}"
+        host: "{{ mongodb_master_node }}"
         port: "{{ mongodb_cs_port }}"
         state: started
         delay: 5
@@ -108,19 +108,19 @@
 
     - name: Check if Mongo Config Server is already active
       shell: |
-        mongo --host {{ graylog_mongo_master_node }} --port {{ mongodb_cs_port }} --quiet mongo-check-rs.js
+        mongo --host {{ mongodb_master_node }} --port {{ mongodb_cs_port }} --quiet mongo-check-rs.js
       register: mongo_initial_cs_status
       changed_when: false
       failed_when: false
 
     - block:
         - name: Create Config Server in Mongo
           shell: |
-            mongo --host {{ graylog_mongo_master_node }} --port {{ mongodb_cs_port }} < cs.js
+            mongo --host {{ mongodb_master_node }} --port {{ mongodb_cs_port }} < cs.js
 
         - name: Check Mongo Config Server status
           shell: |
-            mongo --host {{ graylog_mongo_master_node }} --port {{ mongodb_cs_port }} --quiet mongo-check-rs.js
+            mongo --host {{ mongodb_master_node }} --port {{ mongodb_cs_port }} --quiet mongo-check-rs.js
           register: mongo_cs_status
           changed_when: false
           until: mongo_cs_status.stdout == 'OK'
@@ -130,9 +130,9 @@
         - mongo_initial_cs_status.stdout != 'OK'
 
     # Mongo SN - Shard Node
-    - name: "Check for port {{ mongodb_sn_port }} on {{ graylog_mongo_master_node }} to be accessible"
+    - name: "Check for port {{ mongodb_sn_port }} on {{ mongodb_master_node }} to be accessible"
       wait_for:
-        host: "{{ graylog_mongo_master_node }}"
+        host: "{{ mongodb_master_node }}"
         port: "{{ mongodb_sn_port }}"
         state: started
         delay: 5
@@ -144,19 +144,19 @@
 
     - name: Check if Mongo Shard Node is already active
       shell: |
-        mongo --host {{ graylog_mongo_master_node }} --port {{ mongodb_sn_port }} --quiet mongo-check-rs.js
+        mongo --host {{ mongodb_master_node }} --port {{ mongodb_sn_port }} --quiet mongo-check-rs.js
       register: mongo_initial_sn_status
       changed_when: false
       failed_when: false
 
     - block:
         - name: Create Shard Node in Mongo
           shell: |
-            mongo --host {{ graylog_mongo_master_node }} --port {{ mongodb_sn_port }} < sn.js
+            mongo --host {{ mongodb_master_node }} --port {{ mongodb_sn_port }} < sn.js
 
         - name: Check Mongo Shard Node status
           shell: |
-            mongo --host {{ graylog_mongo_master_node }} --port {{ mongodb_sn_port }} --quiet mongo-check-rs.js
+            mongo --host {{ mongodb_master_node }} --port {{ mongodb_sn_port }} --quiet mongo-check-rs.js
           register: mongo_sn_status
           changed_when: false
           until: mongo_sn_status.stdout == 'OK'
@@ -166,9 +166,9 @@
         - mongo_initial_sn_status.stdout != 'OK'
 
     # Mongo QR - Query Router
-    - name: "Check for port {{ mongodb_qr_port }} on {{ graylog_mongo_master_node }} to be accessible"
+    - name: "Check for port {{ mongodb_qr_port }} on {{ mongodb_master_node }} to be accessible"
       wait_for:
-        host: "{{ graylog_mongo_master_node }}"
+        host: "{{ mongodb_master_node }}"
         port: "{{ mongodb_qr_port }}"
         state: started
         delay: 5
@@ -180,7 +180,7 @@
 
     - name: Create Mongo QR - Query Router
       shell: |
-        mongo --host {{ graylog_mongo_master_node }} --port {{ mongodb_qr_port }} --quiet qr.js
+        mongo --host {{ mongodb_master_node }} --port {{ mongodb_qr_port }} --quiet qr.js
   when:
     - graylog_server_is_master is defined
     - graylog_server_is_master|bool

roles/graylog-server/templates/server.conf.j2

@@ -1,7 +1,7 @@
-rs.initiate( { _id: "configReplSet", configsvr: true, version: 1, members: [ { _id: 0, host: "{{ graylog_mongo_master_node }}:{{ mongodb_cs_port }}", priority: 1, votes: 1 } ] } )
+rs.initiate( { _id: "configReplSet", configsvr: true, version: 1, members: [ { _id: 0, host: "{{ mongodb_master_node }}:{{ mongodb_cs_port }}", priority: 1, votes: 1 } ] } )
 
-{% for node in es_cluster_interconnect_nodes.split(',') -%}
-{% if node != graylog_mongo_master_node -%}
+{% for node in mongodb_nodes.split(',') -%}
+{% if node != mongodb_master_node -%}
 rs.add( { host: "{{ node.strip() }}:{{ mongodb_cs_port }}", priority: 1, votes: 1 } )
 {% endif -%}
 {% endfor %}

roles/mongodb-cluster/tasks/main.yml

@@ -1,2 +1,2 @@
-sh.addShard("graylog-shard-rs0/{% for node in es_cluster_interconnect_nodes.split(',') -%}{{ node.strip() }}:{{ mongodb_sn_port }}{%- if not loop.last -%},{%- endif -%}{%- endfor -%}")
+sh.addShard("graylog-shard-rs0/{% for node in mongodb_nodes.split(',') -%}{{ node.strip() }}:{{ mongodb_sn_port }}{%- if not loop.last -%},{%- endif -%}{%- endfor -%}")
 sh.enableSharding("graylog")

templates/mongo/cs.js.j2

@@ -1,5 +1,5 @@
-rs.initiate( { _id: "graylog-shard-rs0", members: [ { _id: 0, host: "{{ graylog_mongo_master_node }}:{{ mongodb_sn_port }}" } ] })
+rs.initiate( { _id: "graylog-shard-rs0", members: [ { _id: 0, host: "{{ mongodb_master_node }}:{{ mongodb_sn_port }}" } ] })
 
-{% for node in es_cluster_interconnect_nodes.split(',') -%}
+{% for node in mongodb_nodes.split(',') -%}
 rs.add("{{ node.strip() }}:{{ mongodb_sn_port }}")
 {% endfor %}