Merge pull request #417 from wazuh/feature-opendistro

Open Distro for Elasticsearch deployment

playbooks/wazuh-opendistro.yml

@@ -0,0 +1,4 @@
+---
+- hosts: es-cluster
+  roles:
+    - role: ../roles/opendistro/opendistro-elasticsearch

roles/opendistro/opendistro-elasticsearch/defaults/main.yml

@@ -0,0 +1,58 @@
+---
+# The OpenDistro version
+opendistro_version: 1.6.0
+elasticsearch_cluster_name: wazuh-cluster
+
+# Minimum master nodes in cluster, 2 for 3 nodes elasticsearch cluster
+minimum_master_nodes: 2
+
+# Elasticsearch version
+es_version: "7.3.2"
+es_major_version: "7.x"
+
+# Configure hostnames for Elasticsearch nodes
+# Example es1.example.com, es2.example.com
+domain_name: wazuh.com
+
+# The OpenDistro package repository
+package_repos:
+  yum: 
+    opendistro: 
+      baseurl: 'https://d3g5vo6xdbdb9a.cloudfront.net/yum/noarch/'
+      gpg: 'https://d3g5vo6xdbdb9a.cloudfront.net/GPG-KEY-opendistroforelasticsearch'
+    elasticsearch_oss:
+      baseurl: 'https://artifacts.elastic.co/packages/oss-7.x/yum'
+      gpg: 'https://artifacts.elastic.co/GPG-KEY-elasticsearch'
+
+opendistro_sec_plugin_conf_path: /usr/share/elasticsearch/plugins/opendistro_security/securityconfig
+opendistro_sec_plugin_tools_path: /usr/share/elasticsearch/plugins/opendistro_security/tools
+opendistro_conf_path: /etc/elasticsearch/
+es_nodes: |-
+        {% for item in groups['es-cluster'] -%}
+          {{ hostvars[item]['ip'] }}{% if not loop.last %}","{% endif %}
+        {%- endfor %}
+
+# Security password
+opendistro_security_password: admin
+# Set JVM memory limits
+opendistro_jvm_xms: null
+
+opendistro_http_port: 9200
+
+certs_gen_tool_version: 1.7
+# Url of Search Guard certificates generator tool
+certs_gen_tool_url: "https://releases.floragunn.com/search-guard-tlstool/{{ certs_gen_tool_version }}/search-guard-tlstool-{{ certs_gen_tool_version }}.zip"
+
+elasticrepo:
+  apt: 'https://artifacts.elastic.co/packages/7.x/apt'
+  yum: 'https://artifacts.elastic.co/packages/7.x/yum'
+  gpg: 'https://artifacts.elastic.co/GPG-KEY-opendistro'
+  key_id: '46095ACC8548582C1A2699A9D27D666CD88E42B4'
+
+opendistro_admin_password: changeme
+opendistro_kibana_password: changeme
+# Cluster Settings
+single_node: true
+opendistro_cluster_name: wazuh
+
+local_certs_path: /tmp/opendistro-nodecerts

roles/opendistro/opendistro-elasticsearch/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: restart elasticsearch
+  service: 
+    name: elasticsearch
+    state: restarted

roles/opendistro/opendistro-elasticsearch/meta/main.yml

@@ -0,0 +1,24 @@
+---
+galaxy_info:
+  author: Wazuh
+  description: Installing and maintaining Opendistro server.
+  company: wazuh.com
+  license: license (GPLv3)
+  min_ansible_version: 2.0
+  platforms:
+    - name: EL
+      versions:
+        - all
+    - name: Ubuntu
+      versions:
+        - all
+    - name: Debian
+      versions:
+        - all
+    - name: Fedora
+      versions:
+        - all
+  galaxy_tags:
+    - web
+    - system
+    - monitoring

roles/opendistro/opendistro-elasticsearch/tasks/RMRedHat.yml

@@ -0,0 +1,6 @@
+---
+- name: RedHat/CentOS/Fedora | Remove Elasticsearch repository (and clean up left-over metadata)
+  yum_repository:
+    name: elastic_repo_7
+    state: absent
+  changed_when: false

roles/opendistro/opendistro-elasticsearch/tasks/RedHat.yml

@@ -0,0 +1,38 @@
+---
+- block:
+
+  - name: RedHat/CentOS/Fedora | Add OpenDistro repo
+    yum_repository:
+      file: opendistro
+      name: opendistro_repo
+      description: Opendistro yum repository
+      baseurl: "{{ package_repos.yum.opendistro.baseurl }}"
+      gpgkey: "{{ package_repos.yum.opendistro.gpg }}"
+      gpgcheck: true
+    changed_when: false
+
+  - name: RedHat/CentOS/Fedora | Add Elasticsearch-oss repo
+    yum_repository:
+      file: opendistro
+      name: elasticsearch_oss_repo
+      description: Elasticsearch-oss yum repository
+      baseurl: "{{ package_repos.yum.elasticsearch_oss.baseurl }}"
+      gpgkey: "{{ package_repos.yum.elasticsearch_oss.gpg }}"
+      gpgcheck: true
+    changed_when: false
+
+  - name: RedHat/CentOS/Fedora | Install OpenJDK 11
+    yum:
+      name: java-11-openjdk-devel 
+      state: present
+
+  - name: RedHat/CentOS/Fedora | Install OpenDistro dependencies
+    yum:
+      name: "{{ packages }}"
+    vars:
+      packages:
+      - wget
+      - unzip
+
+  tags:
+  - install

roles/opendistro/opendistro-elasticsearch/tasks/local_actions.yml

@@ -0,0 +1,51 @@
+---
+- block:
+
+  - name: Local action | Create local temporary directory for certificates generation
+    local_action:
+      module: file
+      path: "{{ local_certs_path }}"
+      state: directory
+    run_once: true
+    
+  - name: Local action | Download certificates generation tool
+    local_action:
+      module: get_url
+      url: "{{ certs_gen_tool_url }}"
+      dest: "{{ local_certs_path }}/search-guard-tlstool-{{ certs_gen_tool_version }}.zip"
+    run_once: true
+
+  - name: Local action | Extract the certificates generation tool
+    local_action:
+      module: unarchive
+      src: "{{ local_certs_path }}/search-guard-tlstool-1.7.zip"
+      dest: "{{ local_certs_path }}/"
+
+  - name: Local action | Add the execution bit to the binary
+    local_action:
+      module: file
+      dest: "{{ local_certs_path }}/tools/sgtlstool.sh"
+      mode: a+x
+    run_once: true
+
+  - name: Local action | Prepare the certificates generation template file
+    local_action:
+      module: template
+      src: "templates/tlsconfig.yml.j2"
+      dest: "{{ local_certs_path }}/config/tlsconfig.yml"
+    run_once: true
+
+  - name: Local action | Check if root CA file exists
+    local_action:
+      module: stat 
+      path: "{{ local_certs_path }}/config/root-ca.key"
+    register: root_ca_file
+
+  - name: Local action | Generate the node & admin certificates in local
+    local_action:
+      module: command {{ local_certs_path }}/tools/sgtlstool.sh -c {{ local_certs_path }}/config/tlsconfig.yml -ca -crt -t {{ local_certs_path }}/config/ -f -o
+    run_once: true
+    when: root_ca_file.stat.exists == False
+
+  tags:
+  - generate-certs

roles/opendistro/opendistro-elasticsearch/tasks/main.yml

@@ -0,0 +1,68 @@
+---
+
+- import_tasks: local_actions.yml
+
+- import_tasks: RedHat.yml
+  when: ansible_os_family == 'RedHat'
+
+- name: Install OpenDistro
+  package:
+    name: opendistroforelasticsearch-{{ opendistro_version }} 
+    state: present
+  register: install
+  tags: install
+
+- name: Remove elasticsearch configuration file
+  file:
+    path: "{{ opendistro_conf_path }}/elasticsearch.yml"
+    state: absent
+  when: install.changed
+  tags: install
+  
+- name: Copy Configuration File
+  blockinfile:
+    block: "{{ lookup('template', 'elasticsearch.yml.j2') }}"
+    dest: "{{ opendistro_conf_path }}/elasticsearch.yml"
+    create: true
+    group: elasticsearch
+    mode: 0640
+    marker: "## {mark} Opendistro general settings ##"
+  when: install.changed
+  tags: install
+
+- import_tasks: security_actions.yml
+
+- name: Configure OpenDistro Elasticsearch JVM memmory.
+  template:
+    src: "templates/jvm.options.j2"
+    dest: /etc/elasticsearch/jvm.options
+    owner: root
+    group: elasticsearch
+    mode: 0644
+    force: yes
+  notify: restart elasticsearch
+  tags: install
+
+- name: Ensure Elasticsearch started and enabled
+  service:
+    name: elasticsearch
+    enabled: true
+    state: started
+
+- name: Wait for Elasticsearch API
+  uri:
+    url: "https://{{ es_nodes.split(',')[0].split('\"')[0] }}:9200/_cluster/health/"
+    user: "admin" # Default OpenDistro user is always "admin"
+    password: "{{ opendistro_admin_password }}"
+    validate_certs: no
+    status_code: 200,401
+    return_content: yes
+    timeout: 4
+  register: _result
+  until: (  _result.json is defined) and (_result.json.status == "green")
+  retries: 24
+  delay: 5
+  tags: debug
+
+- import_tasks: "RMRedHat.yml"
+  when: ansible_os_family == "RedHat"

roles/opendistro/opendistro-elasticsearch/tasks/security_actions.yml

@@ -0,0 +1,80 @@
+- block:
+
+  - name: Remove demo certs
+    file:
+      path: "{{ item }}"
+      state: absent
+    with_items:
+      - "{{ opendistro_conf_path }}/kirk.pem"
+      - "{{ opendistro_conf_path }}/kirk-key.pem"
+      - "{{ opendistro_conf_path }}/esnode.pem"
+      - "{{ opendistro_conf_path }}/esnode-key.pem"
+
+  - name: Copy the node & admin certificates to Elasticsearch cluster
+    copy:
+      src: "{{ local_certs_path }}/config/{{ item }}"
+      dest: /etc/elasticsearch/
+      mode: 0644
+    with_items:
+      - root-ca.pem
+      - root-ca.key
+      - "{{ inventory_hostname }}.key"
+      - "{{ inventory_hostname }}.pem"
+      - "{{ inventory_hostname }}_http.key"
+      - "{{ inventory_hostname }}_http.pem"
+      - "{{ inventory_hostname }}_elasticsearch_config_snippet.yml"
+      - admin.key
+      - admin.pem
+
+  - name: Copy the OpenDistro security configuration file to cluster
+    blockinfile:
+      block: "{{ lookup('file', '{{ local_certs_path }}/config/{{ inventory_hostname }}_elasticsearch_config_snippet.yml') }}"
+      dest: "{{ opendistro_conf_path }}/elasticsearch.yml"
+      insertafter: EOF
+      marker: "## {mark} Opendistro Security Node & Admin certificates configuration ##"
+
+  - name: Prepare the OpenDistro security configuration file
+    replace:
+      path: "{{ opendistro_conf_path }}/elasticsearch.yml"
+      regexp: 'searchguard'
+      replace: 'opendistro_security'
+    tags: local
+
+  - name: Restart elasticsearch with security configuration
+    systemd:
+      name: elasticsearch
+      state: restarted
+
+  - name: Copy the OpenDistro security internal users template
+    template:
+      src: "templates/internal_users.yml.j2"
+      dest: "{{ opendistro_sec_plugin_conf_path }}/internal_users.yml"
+      mode: 0644
+    run_once: true
+
+  - name: Set the Admin user password
+    shell: >
+      sed -i 's,{{ opendistro_admin_password }},'$(sh {{ opendistro_sec_plugin_tools_path }}/hash.sh -p {{ opendistro_admin_password }} | tail -1)','
+      {{ opendistro_sec_plugin_conf_path }}/internal_users.yml
+    run_once: true
+
+  - name: Set the kibanaserver role/user pasword
+    shell: >
+      sed -i 's,{{ opendistro_kibana_password }},'$(sh {{ opendistro_sec_plugin_tools_path }}/hash.sh -p {{ opendistro_kibana_password }} | tail -1)','
+      {{ opendistro_sec_plugin_conf_path }}/internal_users.yml
+    run_once: true
+
+  - name: Initialize the OpenDistro security index in elasticsearch
+    command: >
+      {{ opendistro_sec_plugin_tools_path }}/securityadmin.sh
+      -cacert {{ opendistro_conf_path }}/root-ca.pem
+      -cert {{ opendistro_conf_path }}/admin.pem
+      -key {{ opendistro_conf_path }}/admin.key
+      -cd {{ opendistro_sec_plugin_conf_path }}/
+      -nhnv -icl
+      -h {{ hostvars[inventory_hostname]['ip'] }}
+    run_once: true
+
+  tags:
+    - production_ready
+  when: install.changed

roles/opendistro/opendistro-elasticsearch/templates/elasticsearch.yml.j2

@@ -0,0 +1,22 @@
+cluster.name: "{{ opendistro_cluster_name }}"
+
+node.name: "{{ inventory_hostname }}"
+
+path.data: /var/lib/elasticsearch
+
+path.logs: /var/log/elasticsearch
+
+network.host: "{{ hostvars[inventory_hostname]['ip'] }}"
+
+http.port: "{{ opendistro_http_port }}"
+
+discovery.seed_hosts: ["{{ es_nodes }}"]
+
+cluster.initial_master_nodes: ["{{ es_nodes }}"]
+
+discovery.zen.minimum_master_nodes: "{{ minimum_master_nodes }}"
+opendistro_security.allow_default_init_securityindex: true
+opendistro_security.audit.type: internal_elasticsearch
+opendistro_security.enable_snapshot_restore_privilege: true
+opendistro_security.check_snapshot_restore_write_privileges: true
+opendistro_security.restapi.roles_enabled: ["all_access", "security_rest_api_access"]

roles/opendistro/opendistro-elasticsearch/templates/internal_users.yml.j2

@@ -0,0 +1,21 @@
+---
+# This is the internal user database
+# The hash value is a bcrypt hash and can be generated with plugin/tools/hash.sh
+
+_meta:
+  type: "internalusers"
+  config_version: 2
+
+# Define your internal users here
+
+admin:
+  hash: "{{ opendistro_admin_password }}"
+  reserved: true
+  backend_roles:
+  - "admin"
+  description: "admin user"
+
+kibanaserver:
+  hash: "{{ opendistro_kibana_password }}"
+  reserved: true
+  description: "kibanaserver user"

roles/opendistro/opendistro-elasticsearch/templates/jvm.options.j2

@@ -0,0 +1,117 @@
+#jinja2: trim_blocks:False
+# {{ ansible_managed }}
+## JVM configuration
+
+################################################################
+## IMPORTANT: JVM heap size
+################################################################
+##
+## You should always set the min and max JVM heap
+## size to the same value. For example, to set
+## the heap to 4 GB, set:
+##
+## -Xms4g
+## -Xmx4g
+##
+## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html
+## for more information
+##
+################################################################
+
+# Xms represents the initial size of total heap space
+# Xmx represents the maximum size of total heap space
+
+# Xms represents the initial size of total heap space
+# Xmx represents the maximum size of total heap space
+{% if opendistro_jvm_xms is not none %}
+{% if opendistro_jvm_xms < 32000 %}
+-Xms{{ opendistro_jvm_xms }}m
+-Xmx{{ opendistro_jvm_xms }}m
+{% else %}
+-Xms32000m
+-Xmx32000m
+{% endif %}
+{% else %}
+-Xms{% if ansible_memtotal_mb < 64000 %}{{ ((ansible_memtotal_mb|int)/2)|int }}m{% else %}32000m{% endif %}
+-Xmx{% if ansible_memtotal_mb < 64000 %}{{ ((ansible_memtotal_mb|int)/2)|int }}m{% else %}32000m{% endif %}
+{% endif %}
+
+################################################################
+## Expert settings
+################################################################
+##
+## All settings below this section are considered
+## expert settings. Don't tamper with them unless
+## you understand what you are doing
+##
+################################################################
+
+## GC configuration
+-XX:+UseConcMarkSweepGC
+-XX:CMSInitiatingOccupancyFraction=75
+-XX:+UseCMSInitiatingOccupancyOnly
+
+## optimizations
+
+# pre-touch memory pages used by the JVM during initialization
+-XX:+AlwaysPreTouch
+
+## basic
+
+# force the server VM
+-server
+
+# explicitly set the stack size
+-Xss1m
+
+# set to headless, just in case
+-Djava.awt.headless=true
+
+# ensure UTF-8 encoding by default (e.g. filenames)
+-Dfile.encoding=UTF-8
+
+# use our provided JNA always versus the system one
+-Djna.nosys=true
+
+# turn off a JDK optimization that throws away stack traces for common
+# exceptions because stack traces are important for debugging
+-XX:-OmitStackTraceInFastThrow
+
+# flags to configure Netty
+-Dio.netty.noUnsafe=true
+-Dio.netty.noKeySetOptimization=true
+-Dio.netty.recycler.maxCapacityPerThread=0
+
+# log4j 2
+-Dlog4j.shutdownHookEnabled=false
+-Dlog4j2.disable.jmx=true
+
+## heap dumps
+
+# generate a heap dump when an allocation from the Java heap fails
+# heap dumps are created in the working directory of the JVM
+-XX:+HeapDumpOnOutOfMemoryError
+
+# specify an alternative path for heap dumps
+# ensure the directory exists and has sufficient space
+-XX:HeapDumpPath=/var/lib/elasticsearch
+
+## GC logging
+
+#-XX:+PrintGCDetails
+#-XX:+PrintGCTimeStamps
+#-XX:+PrintGCDateStamps
+#-XX:+PrintClassHistogram
+#-XX:+PrintTenuringDistribution
+#-XX:+PrintGCApplicationStoppedTime
+
+# log GC status to a file with time stamps
+# ensure the directory exists
+#-Xloggc:${loggc}
+
+# By default, the GC log file will not rotate.
+# By uncommenting the lines below, the GC log file
+# will be rotated every 128MB at most 32 times.
+#-XX:+UseGCLogFileRotation
+#-XX:NumberOfGCLogFiles=32
+#-XX:GCLogFileSize=128M

roles/opendistro/opendistro-elasticsearch/templates/tlsconfig.yml.j2

@@ -0,0 +1,47 @@
+ca:
+   root:
+      dn: CN=root.ca.{{ domain_name }},OU=CA,O={{ domain_name }}\, Inc.,DC={{ domain_name }}
+      keysize: 2048
+      validityDays: 730
+      pkPassword: none
+      file: root-ca.pem
+
+### Default values and global settings
+defaults:
+      validityDays: 730
+      pkPassword: none
+      # Set this to true in order to generate config and certificates for
+      # the HTTP interface of nodes
+      httpsEnabled: true
+      reuseTransportCertificatesForHttp: false
+      verifyHostnames: false
+      resolveHostnames: false
+
+
+###
+### Nodes
+###
+#
+# Specify the nodes of your ES cluster here
+#
+nodes:
+{% for item in groups['es-cluster'] %}
+  - name: {{ item }}
+    dn: CN={{ item }}.{{ domain_name }},OU=Ops,O={{ domain_name }}\, Inc.,DC={{ domain_name }}
+    dns: {{ item }}.{{ domain_name }}
+    ip: {{ hostvars[item]['ip'] }}
+{% endfor %}
+
+###
+### Clients
+###
+#
+# Specify the clients that shall access your ES cluster with certificate authentication here
+#
+# At least one client must be an admin user (i.e., a super-user). Admin users can
+# be specified with the attribute admin: true
+#
+clients:
+  - name: admin
+    dn: CN=admin.{{ domain_name }},OU=Ops,O={{ domain_name }}\, Inc.,DC={{ domain_name }}
+    admin: true