feat: manage index templates

CHANGELOG.md

@@ -8,6 +8,7 @@ Which is based on [Keep A Changelog](http://keepachangelog.com/)
 ### Added
 
 - new option to set heap size
+- manage index templates
 
 ## v1.0.0 - 2019-09-05
 

README.md

@@ -20,6 +20,23 @@ Install and configure Elasticsearch
   path.logs: /var/log/elasticsearch
 ```
 
+* `elasticsearch_index_templates` - hash with the index templates configuration
+
+```
+  logstash:
+    index_patterns:
+      - 'logstash-*'
+    settings:
+      index:
+        number_of_replicas: 3
+    mappings:
+      metric:
+        type: short
+      date:
+        type: date
+        format: YYYY-MM-dd
+```
+
 ## How to use
 
 ```

defaults/main.yml

@@ -5,3 +5,4 @@ elasticsearch_default_config:
   path.data: /var/lib/elasticsearch
   path.logs: /var/log/elasticsearch
 elasticsearch_full_config: '{{ elasticsearch_default_config|combine(elasticsearch_config) }}'
+elasticsearch_index_templates: {}

library/elasticsearch_template.py

@@ -0,0 +1,112 @@
+#!/usr/bin/python
+
+from ansible.module_utils.basic import *
+from ansible.module_utils.elasticsearch_api import *
+
+class ElasticsearchTemplate:
+  def __init__(self, api, name):
+    self.api   = api
+    self.name  = name
+    self.exist = False
+    self.data  = {}
+
+  def get_data(self):
+    status_code, data = self.api.get('_template/{}'.format(self.name))
+    if status_code == 200:
+      self.exist = True
+      self.data  = data[self.name]
+
+  def dict_has_changed(self, new_data, data):
+    for option, value in new_data.items():
+      if not option in data:
+        if value:
+          return True
+
+      elif type(value) is dict:
+        if not type(data[option]) is dict:
+          return True
+        for k, v in value.items():
+          if not k in data[option]:
+            return True
+          if str(data[option][k]) != str(v):
+            return True
+
+      elif type(value) is list:
+        if not type(data[option]) is list:
+          return True
+        if data[option].sort() != value.sort():
+          return True
+
+      elif data[option] != value:
+        return True
+
+    return False
+
+  def has_changed(self, options):
+    if options['index_patterns'].sort() != self.data['index_patterns'].sort():
+      return True
+
+    if options['settings']:
+      if self.dict_has_changed(options['settings'], self.data['settings']):
+        return True
+    elif self.data['settings']:
+      return True
+
+    if options['mappings']['properties']:
+      if not self.data['mappings']:
+        return True
+      elif self.dict_has_changed(options['mappings']['properties'], self.data['mappings']['properties']):
+        return True
+    else:
+      if self.data['mappings']:
+        return True
+
+    return False
+
+  def create(self, options):
+    self.api.put(
+      '_template/{}'.format(self.name),
+      options
+    )
+    
+def main():
+  fields = {
+    'name':           { 'type': 'str',  'required': True },
+    'index_patterns': { 'type': 'list', 'default': [] },
+    'settings':       { 'type': 'dict', 'default': {} },
+    'mappings':       { 'type': 'dict', 'default': {} },
+    'api_url':        { 'type': 'str',  'default': 'http://127.0.0.1:9200' },
+    'api_user':       { 'type': 'str',  'default': None },
+    'api_password':   { 'type': 'str',  'default': None },
+  }
+  module = AnsibleModule(argument_spec=fields)
+  changed = False
+
+  options = {
+    'index_patterns': module.params['index_patterns'],
+    'settings':       module.params['settings'],
+    'mappings': {
+      'properties':   module.params['mappings'],
+    },
+  }
+
+  api = ElasticsearchApi(
+    module.params['api_url'],
+    module.params['api_user'],
+    module.params['api_password']
+  )
+
+  template = ElasticsearchTemplate(
+    api,
+    module.params['name'],
+  )
+  template.get_data()
+
+  if not template.exist or template.has_changed(options):
+    template.create(options)
+    changed = True
+
+  module.exit_json(changed=changed)
+
+if __name__ == '__main__':
+  main()

module_utils/elasticsearch_api.py

@@ -0,0 +1,38 @@
+#!/usr/bin/python
+import requests
+import base64
+
+class ElasticsearchApi:
+  def __init__(self, url, user, password):
+    self.url     = url
+    self.headers = {}
+    if user and password:
+      token = base64.b64encode('{}:{}',)
+      self.headers = { 'Authorization': 'Basic ' + base64.b64encode({},) }
+
+  def get(self, path):
+    r = requests.get(
+      '{}/{}'.format(self.url, path),
+      headers=self.headers
+    )
+
+    if r.status_code == 500:
+      raise Exception('Server return 500 error: {}'.format(r.text))
+    elif r.status_code == 401:
+      raise Exception('Authentification has failed')
+      
+    return r.status_code, r.json()
+
+  def put(self, path, data):
+    r = requests.put(
+      '{}/{}'.format(self.url, path),
+      headers=self.headers,
+      json=data
+    )
+    
+    if r.status_code == 500:
+      raise Exception('Server return 500 error: {}'.format(r.text))
+    elif r.status_code == 401:
+      raise Exception('Authentification has failed')
+    elif r.status_code != 200:
+      raise Exception('Server return an unknown error: {}'.format(r.text))

tasks/main.yml

@@ -1,7 +1,11 @@
 ---
 - name: install dependencies packages
   apt:
-    name: apt-transport-https
+    name: '{{ packages }}'
+  vars:
+    packages:
+      - apt-transport-https
+      - python-requests
   retries: 2
   register: result
   until: result is succeeded
@@ -55,3 +59,20 @@
     state: started
     enabled: true
   tags: elasticsearch
+
+- name: wait for api is available
+  wait_for:
+    port: 9200
+    timeout: 10
+  tags: elasticsearch
+
+- name: copy index templates
+  elasticsearch_template:
+    name: '{{ item.key }}'
+    index_patterns: '{{ item.value.index_patterns }}'
+    settings: '{{ item.value.settings|default({}) }}'
+    mappings: '{{ item.value.mappings|default({}) }}'
+  no_log: true
+  loop: '{{ elasticsearch_index_templates|dict2items }}'
+  run_once: true
+  tags: elasticsearch

test/integration/default/default.yml

@@ -3,6 +3,16 @@
   connection: local
   vars:
     elasticsearch_heap_size: 512m
+    elasticsearch_index_templates:
+      test:
+        index_patterns:
+          - 'hello*'
+        settings:
+          index:
+            number_of_replicas: 1
+        mappings:
+          metrics:
+            type: short
 
   roles:
     - ansible-role-elasticsearch

test/integration/default/serverspec/default_spec.rb

@@ -33,3 +33,8 @@ end
 describe command('ps faux | grep elasticsearch') do
   its(:stdout) { should contain('-Xms512m -Xmx512m') }
 end
+
+describe command('curl -v http://127.0.0.1:9200/_template/test') do
+  its(:exit_status) { should eq 0 }
+  its(:stdout) { should contain('"number_of_replicas":"1"') }
+end