Merge pull request #219 from wazuh/3.9.4_7.2.0_issue_216

Add Filebeat module to the installation tasks

Pipfile

@@ -5,8 +5,8 @@ name = "pypi"
 
 [packages]
 docker-py = "*"
-ansible = "==2.7.11"
+ansible = "==2.7.13"
-molecule = "*"
+molecule = "2.20"
 
 [dev-packages]
 
@@ -14,6 +14,7 @@ molecule = "*"
 python_version = "2.7"
 
 [scripts]
+destroy ="molecule destroy"
 test ="molecule test"
 agent ="molecule test -s wazuh-agent"
 elasticsearch ="molecule test -s elasticsearch"

molecule/default/molecule.yml

@@ -8,18 +8,37 @@ lint:
   enabled: false
 platforms:
   - name: bionic
-    image: ubuntu:bionic
+    image: solita/ubuntu-systemd:bionic
+    command: /sbin/init
+    ulimits:
+      - nofile:262144:262144
+    privileged: true
+    memory_reservation: 2048m
   - name: xenial
     image: solita/ubuntu-systemd:xenial
     privileged: true
+    memory_reservation: 2048m
     command: /sbin/init
-  - name: trusty
+    ulimits:
-    image: ubuntu:trusty
+      - nofile:262144:262144
-  - name: centos6
+#  - name: trusty
-    image: centos:6
+#    image: ubuntu:trusty
+#    privileged: true
+#    memory_reservation: 2048m
+#    ulimits:
+#      - nofile:262144:262144
+#  - name: centos6
+#    image: centos:6
+#    privileged: true
+#    memory_reservation: 2048m
+#    ulimits:
+#      - nofile:262144:262144
   - name: centos7
     image: milcom/centos7-systemd
+    memory_reservation: 2048m
     privileged: true
+    ulimits:
+      - nofile:262144:262144
 provisioner:
   name: ansible
   env:

molecule/default/playbook.yml

@@ -4,7 +4,3 @@
   roles:
     - role: wazuh/ansible-wazuh-manager
 
-# - {role: wazuh/ansible-filebeat} #, filebeat_output_elasticsearch_hosts: 'your elastic stack server IP'
-# Elasticsearch requires too much memory to test multiple containers concurrently - To Fix
-# - {role: elastic-stack/ansible-elasticsearch, elasticsearch_network_host: 'localhost'}
-# - {role: elastic-stack/ansible-kibana, elasticsearch_network_host: 'localhost'}

molecule/default/tests/test_default.py

@@ -74,7 +74,7 @@ def test_open_ports(host):
     distribution = host.system_info.distribution.lower()
     if distribution == 'ubuntu':
         assert host.socket("tcp://0.0.0.0:1515").is_listening
-        assert not host.socket("tcp://0.0.0.0:1514").is_listening
+        assert host.socket("tcp://0.0.0.0:1514").is_listening
     elif distribution == 'centos':
-        assert host.socket("tcp://:::1515").is_listening
+        assert host.socket("tcp://127.0.0.1:1515").is_listening
-        assert not host.socket("tcp://:::1514").is_listening
+        assert host.socket("tcp://127.0.0.1:1514").is_listening

molecule/elasticsearch/molecule.yml

@@ -23,18 +23,18 @@ platforms:
     command: /sbin/init
     ulimits:
       - nofile:262144:262144
-        #- name: trusty
+  #- name: trusty
-        #image: ubuntu:trusty
+  #image: ubuntu:trusty
-        #privileged: true
+  #privileged: true
-        #memory_reservation: 2048m
+  #memory_reservation: 2048m
-        #ulimits:
+  #ulimits:
-        #- nofile:262144:262144
+      #- nofile:262144:262144
-  - name: centos6
+  #- name: centos6
-    image: centos:6
+  #  image: centos:6
-    privileged: true
+  #  privileged: true
-    memory_reservation: 2048m
+  #  memory_reservation: 2048m
-    ulimits:
+  #  ulimits:
-      - nofile:262144:262144
+  #    - nofile:262144:262144
   - name: centos7
     image: milcom/centos7-systemd
     memory_reservation: 2048m

molecule/filebeat/molecule.yml

@@ -9,8 +9,8 @@ lint:
     config-data:
       ignore: .virtualenv
 platforms:
-  - name: trusty
+ # - name: trusty
-    image: ubuntu:trusty
+ #   image: ubuntu:trusty
   - name: bionic
     image: solita/ubuntu-systemd:bionic
     command: /sbin/init
@@ -19,12 +19,12 @@ platforms:
     image: solita/ubuntu-systemd:xenial
     privileged: true
     command: /sbin/init
-  - name: centos6
+ #- name: centos6
-    image: geerlingguy/docker-centos6-ansible
+ #  image: geerlingguy/docker-centos6-ansible
-    privileged: true
+ #  privileged: true
-    command: /sbin/init
+ #  command: /sbin/init
-    volumes:
+ #  volumes:
-      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+ #    - /sys/fs/cgroup:/sys/fs/cgroup:ro
   - name: centos7
     image: milcom/centos7-systemd
     privileged: true

molecule/filebeat/prepare.yml

@@ -7,6 +7,7 @@
     - name: "Install Python packages for Trusty to solve trust issues"
       package:
         name:
+          - python-apt
           - python-setuptools
           - python-pip
         state: latest

molecule/kibana/molecule.yml

@@ -28,12 +28,12 @@ platforms:
 #    memory_reservation: 1024m
 #    ulimits:
 #      - nofile:262144:262144
-  - name: centos6
+#  - name: centos6
-    image: centos:6
+#    image: centos:6
-    privileged: true
+#    privileged: true
-    memory_reservation: 1024m
+#    memory_reservation: 1024m
-    ulimits:
+#    ulimits:
-      - nofile:262144:262144
+#      - nofile:262144:262144
   - name: centos7
     image: milcom/centos7-systemd
     memory_reservation: 1024m
@@ -55,7 +55,6 @@ provisioner:
     group_vars:
       all:
         elasticsearch_jvm_xms: 256
-        kibana_plugin_install_ignore_error: true
 verifier:
   name: testinfra
   lint:

molecule/kibana/playbook.yml

@@ -2,4 +2,6 @@
 - name: Converge
   hosts: all
   roles:
+  
     - role: elastic-stack/ansible-kibana
+    

playbooks/wazuh-elastic.yml

@@ -1,4 +1,5 @@
 ---
-- hosts: <your elasticsearch host>
+- hosts: <YOUR_ELASTICSEARCH_IP>
   roles:
-    - {role: /etc/ansible/roles/wazuh-ansible/roles/elastic-stack/ansible-elasticsearch, elasticsearch_network_host: 'your elasticsearch IP'}
+    - role: /etc/ansible/roles/wazuh-ansible/roles/elastic-stack/ansible-elasticsearch 
+      elasticsearch_network_host: '<YOUR_ELASTICSEARCH_IP>'

playbooks/wazuh-kibana.yml

@@ -1,10 +1,6 @@
 ---
-- hosts: 172.16.0.162
+- hosts: <KIBANA_HOST>
   roles:
     - role: ../roles/elastic-stack/ansible-kibana
-      kibana_xpack_security: true
+      elasticsearch_network_host: <YOUR_ELASTICSEARCH_IP>
-      kibana_user: elastic
+
-      kibana_password: elastic_pass
-      kibana_node_name: node-2
-      elasticsearch_network_host: 172.16.0.161
-      node_certs_generator: false

playbooks/wazuh-manager.yml

@@ -1,10 +1,8 @@
 ---
-- hosts: 172.16.0.161
+- hosts: <WAZUH_MANAGER_HOST>
   roles:
     - role: ../roles/wazuh/ansible-wazuh-manager
     - role: ../roles/wazuh/ansible-filebeat
-      filebeat_output_elasticsearch_hosts: 172.16.0.161:9200
+      filebeat_output_elasticsearch_hosts: <YOUR_ELASTICSEARCH_IP>:9200
-      filebeat_xpack_security: true
+
-      filebeat_node_name: node-1
-      node_certs_generator: true
 

roles/elastic-stack/ansible-elasticsearch/defaults/main.yml

@@ -3,10 +3,9 @@ elasticsearch_cluster_name: wazuh
 elasticsearch_node_name: node-1
 elasticsearch_http_port: 9200
 elasticsearch_network_host: 127.0.0.1
-elasticsearch_host: 127.0.0.1
 elasticsearch_jvm_xms: null
 elastic_stack_version: 7.2.0
-single_node: false
+single_node: true
 elasticsearch_bootstrap_node: false
 elasticsearch_master_candidate: false
 elasticsearch_cluster_nodes:

roles/elastic-stack/ansible-elasticsearch/tasks/Debian.yml

@@ -8,12 +8,6 @@
   register: elasticsearch_ca_packages_installed
   until: elasticsearch_ca_packages_installed is succeeded
 
-- name: "Install Java Repo for Trusty"
-  apt_repository: repo='ppa:openjdk-r/ppa'
-  when:
-    - ansible_distribution == "Ubuntu"
-    - ansible_distribution_major_version | int == 14
-
 - name: Update and upgrade apt packages
   become: true
   apt:
@@ -24,14 +18,6 @@
     - ansible_distribution == "Ubuntu"
     - ansible_distribution_major_version | int == 14
 
-- name: Install Oracle Java 8
-  become: true
-  apt: name=openjdk-8-jdk
-
-  when:
-    - ansible_distribution == "Ubuntu"
-    - ansible_distribution_major_version | int == 14
-
 - name: Update and upgrade apt packages
   become: true
   apt:

roles/elastic-stack/ansible-elasticsearch/tasks/RedHat.yml

@@ -9,13 +9,6 @@
     gpgcheck: true
   changed_when: false
 
-- name: CentOS x.x => x.x < 7.0 | Installing Java
-  yum:
-    name: java-1.8.0-openjdk.x86_64
-    state: present
-  when:
-    - ansible_distribution in ['CentOS', 'RedHat'] and ansible_distribution_major_version|int < 7
-
 - name: RedHat/CentOS/Fedora | Install Elasticsarch
   package: name=elasticsearch-{{ elastic_stack_version }} state=present
   tags: install

roles/elastic-stack/ansible-elasticsearch/tasks/main.yml

@@ -245,13 +245,6 @@
     - not (ansible_distribution == "Debian" and ansible_distribution_version is version('8', '<'))
     - not (ansible_os_family == "RedHat" and ansible_distribution_version is version('7', '<'))
 
-- name: Distribution is centos 6.* | Enable Elasticsearch
-  service: name=elasticsearch enabled=yes
-
-- name: Distribution is centos 6.* | Start Elasticsearch
-  service: name=elasticsearch state=started
-  ignore_errors: true
-
 - name: Ensure Elasticsearch started and enabled
   service:
     name: elasticsearch
@@ -259,7 +252,7 @@
     state: started
 
 - name: Make sure Elasticsearch is running before proceeding
-  wait_for: host={{ elasticsearch_host }} port={{ elasticsearch_http_port }} delay=3 timeout=400
+  wait_for: host={{ elasticsearch_network_host }} port={{ elasticsearch_http_port }} delay=3 timeout=400
   tags:
     - configure
     - init

roles/elastic-stack/ansible-kibana/defaults/main.yml

@@ -23,3 +23,5 @@ node_certs_destination: /etc/kibana/certs
 rsync_path: /usr/bin/rsync
 rsync_user: vagrant
 rsync_extra_parameters: -avg -e 'ssh -o StrictHostKeyChecking=no' --rsync-path='sudo rsync'
+
+kibana_plugin_install_ignore_error: true

roles/elastic-stack/ansible-kibana/tasks/main.yml

@@ -118,14 +118,15 @@
   tags: install
 
 - name: Install Wazuh-APP (can take a while)
-  shell: "/usr/share/kibana/bin/kibana-plugin install  https://packages.wazuh.com/wazuhapp/wazuhapp-{{ wazuh_version }}_{{ elastic_stack_version }}.zip"
+  shell: "/usr/share/kibana/bin/kibana-plugin install https://packages.wazuh.com/wazuhapp/wazuhapp-{{ wazuh_version }}_{{ elastic_stack_version }}.zip"
   environment:
     NODE_OPTIONS: "--max-old-space-size=3072"
   args:
     executable: /bin/bash
     creates: /usr/share/kibana/plugins/wazuh/package.json
   notify: restart kibana
-  ignore_errors: "{{ kibana_plugin_install_ignore_error }}"
+  become: yes
+  become_user: kibana
   tags:
     - install
     - skip_ansible_lint

roles/wazuh/ansible-filebeat/defaults/main.yml

@@ -43,3 +43,8 @@ node_certs_destination: /etc/filebeat/certs
 rsync_path: /usr/bin/rsync
 rsync_user: vagrant
 rsync_extra_parameters: -avg -e 'ssh -o StrictHostKeyChecking=no' --rsync-path='sudo rsync'
+
+filebeat_module_package_name: wazuh-filebeat-0.1.tar.gz
+filebeat_module_package_path: /root/
+filebeat_module_destination: /usr/share/filebeat/module
+filebeat_module_folder: /usr/share/filebeat/module/wazuh

roles/wazuh/ansible-filebeat/tasks/main.yml

@@ -88,6 +88,41 @@
     - filebeat_xpack_security
   tags: xpack-security
 
+- name: Checking if Filebeat Module folder file exists
+  stat:
+    path: "{{ filebeat_module_folder }}"
+  register: filebeat_module_folder
+
+
+- name: Download Filebeat module package
+  get_url:
+    url: https://packages.wazuh.com/3.x/filebeat/{{ filebeat_module_package_name }}
+    dest: "{{ filebeat_module_package_path }}"
+  when: not filebeat_module_folder.stat.exists
+
+- name: Unpakcing Filebeat module package
+  unarchive:
+    src: "{{ filebeat_module_package_path }}/{{ filebeat_module_package_name }}"
+    dest: "{{ filebeat_module_destination }}"
+    remote_src: yes
+  when: not filebeat_module_folder.stat.exists
+
+- name: Setting 0755 permission for Filebeat module folder
+  file: dest={{ filebeat_module_folder }} mode=u=rwX,g=rwX,o=rwX recurse=yes
+  when: not filebeat_module_folder.stat.exists
+
+- name: Checking if Filebeat Module package file exists
+  stat:
+    path: "{{ filebeat_module_package_path }}/{{ filebeat_module_package_name }}"
+  register: filebeat_module_package
+  when: filebeat_module_package is not defined
+
+- name: Delete Filebeat module package file
+  file:
+    state: absent
+    path: "{{ filebeat_module_package_path }}/{{ filebeat_module_package_name }}"
+  when: filebeat_module_package.stat.exists
+
 - import_tasks: config.yml
   when: filebeat_create_config
   notify: restart filebeat