Merge pull request #443 from wazuh/opendistro-demo

Update Opendistro tasks

playbooks/wazuh-opendistro.yml

@@ -2,3 +2,16 @@
 - hosts: es_cluster
   roles:
     - role: ../roles/opendistro/opendistro-elasticsearch
+
+  vars:
+    instances:           # A certificate will be generated for every node using the name as CN.
+      node1:
+        name: node-1
+        ip: <node-1 IP>
+      node2:
+        name: node-2
+        ip: <node-2 IP>
+      node3:
+        name: node-3
+        ip: <node-3 IP>
+

roles/opendistro/opendistro-elasticsearch/defaults/main.yml

@@ -1,15 +1,26 @@
 ---
-# The OpenDistro version
+# Cluster Settings
+es_version: "7.3.2"
+es_major_version: "7.x"
+
 opendistro_version: 1.8.0
+
 elasticsearch_cluster_name: wazuh-cluster
+single_node: true
+opendistro_cluster_name: wazuh
+elasticsearch_node_data: true
+elasticsearch_node_ingest: true
+elasticsearch_lower_disk_requirements: false
+elasticsearch_cluster_nodes:
+  - 127.0.0.1
+elasticsearch_discovery_nodes:
+  - 127.0.0.1
+
+local_certs_path: ./opendistro/certificates
 
 # Minimum master nodes in cluster, 2 for 3 nodes elasticsearch cluster
 minimum_master_nodes: 2
 
-# Elasticsearch version
-es_version: "7.3.2"
-es_major_version: "7.x"
-
 # Configure hostnames for Elasticsearch nodes
 # Example es1.example.com, es2.example.com
 domain_name: wazuh.com
@@ -34,12 +45,16 @@ es_nodes: |-
 
 # Security password
 opendistro_security_password: admin
+opendistro_custom_user: ""
+opendistro_cusom_user_role: "admin"
+
 # Set JVM memory limits
 opendistro_jvm_xms: null
 
 opendistro_http_port: 9200
 
 certs_gen_tool_version: 1.7
+
 # Url of Search Guard certificates generator tool
 certs_gen_tool_url: "https://releases.floragunn.com/search-guard-tlstool/{{ certs_gen_tool_version }}/search-guard-tlstool-{{ certs_gen_tool_version }}.zip"
 
@@ -51,8 +66,7 @@ elasticrepo:
 
 opendistro_admin_password: changeme
 opendistro_kibana_password: changeme
-# Cluster Settings
-single_node: true
-opendistro_cluster_name: wazuh
 
-local_certs_path: /tmp/opendistro-nodecerts
+# Deployment settings
+generate_certs: true
+perform_installation: true

roles/opendistro/opendistro-elasticsearch/tasks/RedHat.yml

@@ -25,6 +25,21 @@
     yum:
       name: java-11-openjdk-devel
       state: present
+    when:
+      - ansible_distribution != 'Amazon'
+
+  - name: Amazon Linux | Install OpenJDK 11
+    block:
+      - name: Install Amazon extras
+        yum:
+          name: amazon-linux-extras
+          state: present
+
+      - name: Install OpenJDK 11
+        shell: amazon-linux-extras install java-openjdk11 -y
+
+    when:
+      - ansible_distribution == 'Amazon'
 
   - name: RedHat/CentOS/Fedora | Install OpenDistro dependencies
     yum:

roles/opendistro/opendistro-elasticsearch/tasks/local_actions.yml

@@ -1,4 +1,13 @@
 ---
+- name: Check if certificates already exists
+  stat:
+    path: "{{ local_certs_path }}"
+  register: certificates_folder
+  delegate_to: localhost
+  tags:
+    - generate-certs
+
+
 - block:
 
   - name: Local action | Create local temporary directory for certificates generation
@@ -70,3 +79,5 @@
   delegate_to: localhost
   tags:
     - generate-certs
+  when:
+    - not certificates_folder.stat.exists

roles/opendistro/opendistro-elasticsearch/tasks/main.yml

@@ -1,10 +1,15 @@
 ---
 
 - import_tasks: local_actions.yml
+  when:
+    - generate_certs
+
+- block:
 
   - import_tasks: RedHat.yml
     when: ansible_os_family == 'RedHat'
 
+
   - name: Install OpenDistro
     package:
       name: opendistroforelasticsearch-{{ opendistro_version }}
@@ -51,7 +56,7 @@
 
   - name: Wait for Elasticsearch API
     uri:
-    url: "https://{{ es_nodes.split(',')[0].split('\"')[0] }}:9200/_cluster/health/"
+      url: "https://{{ inventory_hostname }}:{{ opendistro_http_port }}/_cluster/health/"
       user: "admin" # Default OpenDistro user is always "admin"
       password: "{{ opendistro_admin_password }}"
       validate_certs: no
@@ -63,6 +68,27 @@
     retries: 24
     delay: 5
     tags: debug
+    when:
+      - hostvars[inventory_hostname]['private_ip'] is not defined or not hostvars[inventory_hostname]['private_ip']
+
+  - name: Wait for Elasticsearch API (Private IP)
+    uri:
+      url: "https://{{ hostvars[inventory_hostname]['private_ip'] }}:{{ opendistro_http_port }}/_cluster/health/"
+      user: "admin" # Default OpenDistro user is always "admin"
+      password: "{{ opendistro_admin_password }}"
+      validate_certs: no
+      status_code: 200,401
+      return_content: yes
+      timeout: 4
+    register: _result
+    until: (  _result.json is defined) and (_result.json.status == "green")
+    retries: 24
+    delay: 5
+    tags: debug
+    when:
+      - hostvars[inventory_hostname]['private_ip'] is defined and  hostvars[inventory_hostname]['private_ip']
 
   - import_tasks: "RMRedHat.yml"
     when: ansible_os_family == "RedHat"
+
+  when: perform_installation

roles/opendistro/opendistro-elasticsearch/tasks/security_actions.yml

@@ -9,6 +9,40 @@
       - "{{ opendistro_conf_path }}/esnode.pem"
       - "{{ opendistro_conf_path }}/esnode-key.pem"
 
+
+  - name: Configure node name
+    block:
+      - name: Setting node name (Elasticsearch)
+        set_fact:
+          od_node_name: "{{ elasticsearch_node_name }}"
+        when:
+          elasticsearch_node_name is defined and kibana_node_name is not defined
+
+      - name: Setting node name (Kibana)
+        set_fact:
+          od_node_name: "{{ kibana_node_name }}"
+        when:
+          kibana_node_name is defined
+
+      - name: Setting node name (Filebeat)
+        set_fact:
+          od_node_name: "{{ kibana_node_name }}"
+        when:
+          filebeat_node_name is defined
+
+  - name: Configure IP (Private address)
+    set_fact:
+      target_address: "{{ hostvars[inventory_hostname]['private_ip'] }}"
+    when:
+      - hostvars[inventory_hostname]['private_ip'] is defined
+
+  - name: Configure IP (Public address)
+    set_fact:
+      target_address: "{{ inventory_hostname }}"
+    when:
+      - hostvars[inventory_hostname]['private_ip'] is not defined
+
+
   - name: Copy the node & admin certificates to Elasticsearch cluster
     copy:
       src: "{{ local_certs_path }}/certs/{{ item }}"
@@ -17,17 +51,17 @@
     with_items:
       - root-ca.pem
       - root-ca.key
-      - "{{ inventory_hostname }}.key"
+      - "{{ od_node_name }}.key"
-      - "{{ inventory_hostname }}.pem"
+      - "{{ od_node_name }}.pem"
-      - "{{ inventory_hostname }}_http.key"
+      - "{{ od_node_name }}_http.key"
-      - "{{ inventory_hostname }}_http.pem"
+      - "{{ od_node_name }}_http.pem"
-      - "{{ inventory_hostname }}_elasticsearch_config_snippet.yml"
+      - "{{ od_node_name }}_elasticsearch_config_snippet.yml"
       - admin.key
       - admin.pem
 
   - name: Copy the OpenDistro security configuration file to cluster
     blockinfile:
-      block: "{{ lookup('file', '{{ local_certs_path }}/certs/{{ inventory_hostname }}_elasticsearch_config_snippet.yml') }}"
+      block: "{{ lookup('file', '{{ local_certs_path }}/certs/{{ od_node_name }}_elasticsearch_config_snippet.yml') }}"
       dest: "{{ opendistro_conf_path }}/elasticsearch.yml"
       insertafter: EOF
       marker: "## {mark} Opendistro Security Node & Admin certificates configuration ##"
@@ -51,10 +85,30 @@
       mode: 0644
     run_once: true
 
+  - name: Hashing the custom admin password
+    command: "{{ opendistro_sec_plugin_tools_path }}/hash.sh -p {{ opendistro_admin_password }}"
+    register: opendistro_admin_password_hashed
+    run_once: true
+
+  - name: Filtering hash result in case java path is not defined
+    set_fact:
+      opendistro_admin_password_hashed_filtered: "{{ opendistro_admin_password_hashed.stdout_lines[1] }}"
+    when:
+      - opendistro_admin_password_hashed.stdout_lines[1] is defined
+    run_once: true
+
+  - name: Setting admin hash result
+    set_fact:
+      opendistro_admin_password_hashed_filtered: "{{ opendistro_admin_password_hashed.stdout_lines[0] }}"
+    when:
+      - opendistro_admin_password_hashed.stdout_lines[1] is not defined
+    run_once: true
+
   - name: Set the Admin user password
-    shell: >
+    replace:
-      sed -i 's,{{ opendistro_admin_password }},'$(sh {{ opendistro_sec_plugin_tools_path }}/hash.sh -p {{ opendistro_admin_password }} | tail -1)','
+      path: "{{ opendistro_sec_plugin_conf_path }}/internal_users.yml"
-      {{ opendistro_sec_plugin_conf_path }}/internal_users.yml
+      regexp: '(?<=admin:\n  hash: )(.*)(?=)'
+      replace: "\"{{ opendistro_admin_password_hashed_filtered }}\""
     run_once: true
 
   - name: Set the kibanaserver role/user pasword
@@ -71,9 +125,28 @@
       -key {{ opendistro_conf_path }}/admin.key
       -cd {{ opendistro_sec_plugin_conf_path }}/
       -nhnv -icl
-      -h {{ hostvars[inventory_hostname]['ip'] }}
+      -h {{ target_address }}
     run_once: true
 
+  - name: Create custom user
+    uri:
+      url: "https://{{ target_address }}:{{ opendistro_http_port }}/_opendistro/_security/api/internalusers/{{ opendistro_custom_user }}"
+      method: PUT
+      user: "admin" # Default OpenDistro user is always "admin"
+      password: "{{ opendistro_admin_password }}"
+      body: |
+        {
+          "password": "{{ opendistro_admin_password }}",
+          "backend_roles": ["{{ opendistro_custom_user_role }}"]
+        }
+      body_format: json
+      validate_certs: no
+      status_code: 200,201,401
+      return_content: yes
+      timeout: 4
+    when:
+      - opendistro_custom_user is defined
+
   tags:
     - security
   when: install.changed

roles/opendistro/opendistro-elasticsearch/templates/elasticsearch.yml.j2

@@ -1,18 +1,36 @@
-cluster.name: "{{ opendistro_cluster_name }}"
+cluster.name: {{ elasticsearch_cluster_name }}
-
+node.name: {{ elasticsearch_node_name }}
-node.name: "{{ inventory_hostname }}"
-
 path.data: /var/lib/elasticsearch
-
 path.logs: /var/log/elasticsearch
+network.host: {{ elasticsearch_network_host }}
 
-network.host: "{{ hostvars[inventory_hostname]['ip'] }}"
+node.master: {{ elasticsearch_node_master|lower }}
 
-http.port: "{{ opendistro_http_port }}"
+cluster.initial_master_nodes:
+{% for item in elasticsearch_cluster_nodes %}
+  - {{ item }}
+{% endfor %}
 
-discovery.seed_hosts: ["{{ es_nodes }}"]
+discovery.seed_hosts:
+{% for item in elasticsearch_discovery_nodes %}
+  - {{ item }}
+{% endfor %}
 
-cluster.initial_master_nodes: ["{{ es_nodes }}"]
+{% if elasticsearch_node_data|lower == 'false' %}
+node.data: false
+{% endif %}
+
+{% if elasticsearch_node_ingest|lower == 'false' %}
+node.ingest: false
+{% endif %}
+
+
+{% if elasticsearch_lower_disk_requirements %}
+cluster.routing.allocation.disk.threshold_enabled: true
+cluster.routing.allocation.disk.watermark.flood_stage: 200mb
+cluster.routing.allocation.disk.watermark.low: 500mb
+cluster.routing.allocation.disk.watermark.high: 300mb
+{% endif %}
 
 discovery.zen.minimum_master_nodes: "{{ minimum_master_nodes }}"
 opendistro_security.allow_default_init_securityindex: true

roles/opendistro/opendistro-elasticsearch/templates/tlsconfig.yml.j2

@@ -24,28 +24,14 @@ defaults:
 # Specify the nodes of your ES cluster here
 #
 nodes:
-{% for item in groups['es_cluster'] %}
+{% for (key,value) in instances.items() %}
-  - name: {{ item }}
+{% if (value.ip is defined and value.ip | length > 0) %}
-    dn: CN={{ item }}.{{ domain_name }},OU=Ops,O={{ domain_name }}\, Inc.,DC={{ domain_name }}
+  - name: {{ value.name }}
-    dns: {{ item }}.{{ domain_name }}
+    dn: CN={{ value.name }}.{{ domain_name }},OU=Ops,O={{ domain_name }}\, Inc.,DC={{ domain_name }}
-    ip: {{ hostvars[item]['ip'] }}
+    dns: {{ value.name }}.{{ domain_name }}
-{% endfor %}
+    ip: {{ value.ip }}
-{% if groups['kibana'] is defined and groups['kibana']|length > 0 %}
-{% for item in groups['kibana'] %}
-  - name: {{ item }}
-    dn: CN={{ item }}.{{ domain_name }},OU=Ops,O={{ domain_name }}\, Inc.,DC={{ domain_name }}
-    dns: {{ item }}.{{ domain_name }}
-    ip: {{ hostvars[item]['ip'] }}
-{% endfor %}
 {% endif %}
-{% if groups['managers'] is defined and groups['managers']|length > 0 %}
-{% for item in groups['managers'] %}
-  - name: {{ item }}
-    dn: CN={{ item }}.{{ domain_name }},OU=Ops,O={{ domain_name }}\, Inc.,DC={{ domain_name }}
-    dns: {{ item }}.{{ domain_name }}
-    ip: {{ hostvars[item]['ip'] }}
 {% endfor %}
-{% endif %}
 ###
 ### Clients
 ###

roles/opendistro/opendistro-kibana/defaults/main.yml

@@ -1,5 +1,6 @@
 ---
 
+# Kibana configuration
 elasticsearch_http_port: 9200
 elasticsearch_nodes: |-
         {% for item in groups['es_cluster'] -%}
@@ -8,6 +9,7 @@ elasticsearch_nodes: |-
 elasticsearch_network_host: 172.16.0.161
 elastic_api_protocol: https
 kibana_conf_path: /etc/kibana
+kibana_node_name: node-1
 kibana_server_host: "0.0.0.0"
 kibana_server_port: "5601"
 kibana_server_name: "kibana"
@@ -17,6 +19,9 @@ wazuh_version: 3.13.0
 wazuh_app_url: https://packages.wazuh.com/wazuhapp/wazuhapp
 
 # The OpenDistro package repository
+kibana_opendistro_version: -1.8.0-1 # Version includes the - for RedHat family compatibility, replace with = for Debian hosts
+kibana_opendistro_package: "opendistroforelasticsearch-kibana{{ kibana_opendistro_version }}"
+
 package_repos:
   yum:
     opendistro:
@@ -45,7 +50,7 @@ opendistro_security_user: elastic
 opendistro_admin_password: changeme
 opendistro_kibana_user: kibanaserver
 opendistro_kibana_password: changeme
-local_certs_path: /tmp/opendistro-nodecerts
+local_certs_path: ./opendistro/certificates
 
 # Nodejs
 nodejs:

roles/opendistro/opendistro-kibana/tasks/main.yml

@@ -23,7 +23,7 @@
 
 - name: Install Kibana
   package:
-    name: opendistroforelasticsearch-kibana
+    name: "{{ kibana_opendistro_package }}"
     state: present
   register: install
   tags: install

roles/opendistro/opendistro-kibana/tasks/security_actions.yml

@@ -6,8 +6,9 @@
       dest: /usr/share/kibana
       mode: 0644
     with_items:
-      - "{{ inventory_hostname }}_http.key"
+      - "root-ca.pem"
-      - "{{ inventory_hostname }}_http.pem"
+      - "{{ kibana_node_name }}_http.key"
+      - "{{ kibana_node_name }}_http.pem"
   tags:
   - security
   when: install.changed

roles/opendistro/opendistro-kibana/templates/opendistro_kibana.yml.j2

@@ -10,27 +10,25 @@ server.host: {{ kibana_server_host }}
 
 
 {% if kibana_opendistro_security %}
+
 elasticsearch.hosts: "https://{{ elasticsearch_network_host }}:{{ elasticsearch_http_port }}"
+elasticsearch.username: {{ opendistro_kibana_user }}
+elasticsearch.password: {{ opendistro_kibana_password }}
+server.ssl.enabled: true
+server.ssl.certificate: "/usr/share/kibana/{{ kibana_node_name }}_http.pem"
+server.ssl.key: "/usr/share/kibana/{{ kibana_node_name }}_http.key"
+elasticsearch.ssl.certificateAuthorities: ["/usr/share/kibana/root-ca.pem"]
+elasticsearch.ssl.verificationMode: full
+
 {% else %}
 elasticsearch.hosts: "http://{{ elasticsearch_network_host }}:{{ elasticsearch_http_port }}"
 {% endif %}
 
-elasticsearch.username: {{ opendistro_kibana_user }}
-elasticsearch.password: {{ opendistro_kibana_password }}
-elasticsearch.ssl.verificationMode: none
-
 elasticsearch.requestHeadersWhitelist: ["securitytenant","Authorization"]
 opendistro_security.multitenancy.enabled: false # FIXME: should be enabled starting with Wazuh App v3.13
 opendistro_security.multitenancy.tenants.preferred: ["Private", "Global"]
 opendistro_security.readonly_mode.roles: ["kibana_read_only"]
 
-# OpenDistro Security
-{% if kibana_opendistro_security %}
-server.ssl.enabled: true
-server.ssl.certificate: "/usr/share/kibana/{{ inventory_hostname }}_http.pem"
-server.ssl.key: "/usr/share/kibana//{{ inventory_hostname }}_http.key"
-{% endif %}
-
 newsfeed.enabled: {{ kibana_newsfeed_enabled }}
 telemetry.optIn: {{ kibana_telemetry_optin }}
 telemetry.enabled: {{ kibana_telemetry_enabled }}

roles/wazuh/ansible-filebeat-oss/defaults/main.yml

@@ -23,7 +23,7 @@ filebeat_security_password: changeme
 filebeat_ssl_dir: /etc/pki/filebeat
 
 # Local path to store the generated certificates (OpenDistro security plugin)
-local_certs_path: /tmp/opendistro-nodecerts
+local_certs_path: ./opendistro/certificates
 
 elasticrepo:
   apt: 'https://artifacts.elastic.co/packages/oss-7.x/apt'

roles/wazuh/ansible-filebeat-oss/tasks/security_actions.yml

@@ -11,8 +11,8 @@
       dest: "{{ filebeat_ssl_dir }}"
       mode: 0644
     with_items:
-      - "{{ inventory_hostname }}.key"
+      - "{{ filebeat_node_name }}.key"
-      - "{{ inventory_hostname }}.pem"
+      - "{{ filebeat_node_name }}.pem"
       - "root-ca.pem"
 
   - name: Ensuring folder & certs permissions

roles/wazuh/ansible-filebeat-oss/templates/filebeat.yml.j2

@@ -24,8 +24,8 @@ output.elasticsearch:
   protocol: https
   ssl.certificate_authorities:
     - {{ filebeat_ssl_dir }}/root-ca.pem
-  ssl.certificate: "{{ filebeat_ssl_dir }}/{{ inventory_hostname }}.pem"
+  ssl.certificate: "{{ filebeat_ssl_dir }}/{{ filebeat_node_name }}.pem"
-  ssl.key: "{{ filebeat_ssl_dir }}/{{ inventory_hostname }}.key"
+  ssl.key: "{{ filebeat_ssl_dir }}/{{ filebeat_node_name }}.key"
 {% endif %}
 
 # Optional. Send events to Logstash instead of Elasticsearch