release: version 1.1.0

.gitignore

@@ -1 +1,2 @@
 .kitchen/*
+*.pyc

.kitchen.yml

@@ -1,27 +0,0 @@
----
-driver:
-  name: docker_cli
-
-transport:
-  name: docker_cli
-
-provisioner:
-  name: ansible_playbook
-  hosts: localhost
-  require_ansible_repo: false
-  require_ansible_omnibus: false
-  require_chef_for_busser: true
-  ansible_verbose: false
-  ansible_inventory: ./test/integration/inventory
-
-platforms:
-  - name: debian-9
-    driver_config:
-      image: "nishiki/debian9:ansible-<%= ENV['ANSIBLE_VERSION'] ? ENV['ANSIBLE_VERSION'] : '2.7' %>"
-      command: /bin/systemd
-      volume:
-        - /sys/fs/cgroup:/sys/fs/cgroup:ro
-      security_opt: seccomp=unconfined
-
-suites:
-  - name: default

.yamllint

@@ -4,6 +4,7 @@ extends: default
 ignore: |
   .kitchen/*
   vendor/
+  molecule/default/converge.yml
 
 rules:
   line-length:

CHANGELOG.md

@@ -3,12 +3,28 @@
 This project adheres to [Semantic Versioning](http://semver.org/).
 Which is based on [Keep A Changelog](http://keepachangelog.com/)
 
-## [Unreleased]
+## Unreleased
+
+## v1.1.0 - 2021-08-24
+
+### Added
+
+- feat: add logstash patterns for grok
+- test: add support debian 11
+
+### Changed
+
+- change logstash_inputs variable: dict to array
+- test: replace kitchen to molecule
+- chore: use FQCN for module name
+
+### Removed
+
+- test: remove support debian 9
 
 ## v1.0.0 - 2019-09-29
 
 ### Added
 
-- install kibana package
 - copy configuration
 - use the logstash config test to validate the configuration

Gemfile

@@ -1,8 +0,0 @@
-source 'https://rubygems.org'
-
-group :development do
-  gem 'kitchen-ansible'
-  gem 'kitchen-docker_cli'
-  gem 'rubocop', '0.50.0'
-  gem 'test-kitchen'
-end

Gemfile.lock

@@ -1,130 +0,0 @@
-GEM
-  remote: https://rubygems.org/
-  specs:
-    ast (2.4.0)
-    bcrypt_pbkdf (1.0.1)
-    builder (3.2.3)
-    ed25519 (1.2.4)
-    equatable (0.6.1)
-    erubis (2.7.0)
-    ffi (1.11.1)
-    gssapi (1.3.0)
-      ffi (>= 1.0.1)
-    gyoku (1.3.1)
-      builder (>= 2.1.2)
-    httpclient (2.8.3)
-    kitchen-ansible (0.50.1)
-      net-ssh (>= 3)
-      test-kitchen (>= 1.4)
-    kitchen-docker_cli (0.19.0)
-      test-kitchen (>= 1.3)
-    license-acceptance (1.0.13)
-      pastel (~> 0.7)
-      tomlrb (~> 1.2)
-      tty-box (~> 0.3)
-      tty-prompt (~> 0.18)
-    little-plugger (1.1.4)
-    logging (2.2.2)
-      little-plugger (~> 1.1)
-      multi_json (~> 1.10)
-    mixlib-install (3.11.21)
-      mixlib-shellout
-      mixlib-versioning
-      thor
-    mixlib-shellout (3.0.7)
-    mixlib-versioning (1.2.7)
-    multi_json (1.13.1)
-    necromancer (0.5.0)
-    net-scp (2.0.0)
-      net-ssh (>= 2.6.5, < 6.0.0)
-    net-ssh (5.2.0)
-    net-ssh-gateway (2.0.0)
-      net-ssh (>= 4.0.0)
-    nori (2.6.0)
-    parallel (1.17.0)
-    parser (2.6.4.0)
-      ast (~> 2.4.0)
-    pastel (0.7.3)
-      equatable (~> 0.6)
-      tty-color (~> 0.5)
-    powerpack (0.1.2)
-    rainbow (2.2.2)
-      rake
-    rake (12.3.3)
-    rubocop (0.50.0)
-      parallel (~> 1.10)
-      parser (>= 2.3.3.1, < 3.0)
-      powerpack (~> 0.1)
-      rainbow (>= 2.2.2, < 3.0)
-      ruby-progressbar (~> 1.7)
-      unicode-display_width (~> 1.0, >= 1.0.1)
-    ruby-progressbar (1.10.1)
-    rubyntlm (0.6.2)
-    rubyzip (1.2.3)
-    strings (0.1.6)
-      strings-ansi (~> 0.1)
-      unicode-display_width (~> 1.5)
-      unicode_utils (~> 1.4)
-    strings-ansi (0.1.0)
-    test-kitchen (2.3.2)
-      bcrypt_pbkdf (~> 1.0)
-      ed25519 (~> 1.2)
-      license-acceptance (~> 1.0, >= 1.0.11)
-      mixlib-install (~> 3.6)
-      mixlib-shellout (>= 1.2, < 4.0)
-      net-scp (>= 1.1, < 3.0)
-      net-ssh (>= 2.9, < 6.0)
-      net-ssh-gateway (>= 1.2, < 3.0)
-      thor (~> 0.19)
-      winrm (~> 2.0)
-      winrm-elevated (~> 1.0)
-      winrm-fs (~> 1.1)
-    thor (0.20.3)
-    tomlrb (1.2.8)
-    tty-box (0.4.1)
-      pastel (~> 0.7.2)
-      strings (~> 0.1.6)
-      tty-cursor (~> 0.7)
-    tty-color (0.5.0)
-    tty-cursor (0.7.0)
-    tty-prompt (0.19.0)
-      necromancer (~> 0.5.0)
-      pastel (~> 0.7.0)
-      tty-reader (~> 0.6.0)
-    tty-reader (0.6.0)
-      tty-cursor (~> 0.7)
-      tty-screen (~> 0.7)
-      wisper (~> 2.0.0)
-    tty-screen (0.7.0)
-    unicode-display_width (1.6.0)
-    unicode_utils (1.4.0)
-    winrm (2.3.2)
-      builder (>= 2.1.2)
-      erubis (~> 2.7)
-      gssapi (~> 1.2)
-      gyoku (~> 1.0)
-      httpclient (~> 2.2, >= 2.2.0.2)
-      logging (>= 1.6.1, < 3.0)
-      nori (~> 2.0)
-      rubyntlm (~> 0.6.0, >= 0.6.1)
-    winrm-elevated (1.1.1)
-      winrm (~> 2.0)
-      winrm-fs (~> 1.0)
-    winrm-fs (1.3.2)
-      erubis (~> 2.7)
-      logging (>= 1.6.1, < 3.0)
-      rubyzip (~> 1.1)
-      winrm (~> 2.0)
-    wisper (2.0.0)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  kitchen-ansible
-  kitchen-docker_cli
-  rubocop (= 0.50.0)
-  test-kitchen
-
-BUNDLED WITH
-   1.16.6

README.md

@@ -1,14 +1,16 @@
 # Ansible role: Logstash
 
-[![Version](https://img.shields.io/badge/latest_version-1.0.0-green.svg)](https://git.yaegashi.fr/nishiki/ansible-role-logstash/releases)
+[![Version](https://img.shields.io/badge/latest_version-1.1.0-green.svg)](https://git.yaegashi.fr/nishiki/ansible-role-logstash/releases)
 [![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](https://git.yaegashi.fr/nishiki/ansible-role-logstash/src/branch/master/LICENSE)
 
 Install and configure logstash
 
 ## Requirements
 
-* Ansible >= 2.7
+* Ansible >= 2.9
-* Debian Stretch and Buster
+* Debian
+  * Buster
+  * Bullseye
 
 ## Role variables
 
@@ -55,6 +57,16 @@ Install and configure logstash
     }
 ```
 
+- `logstash_pattern` hash with grok patterns
+
+```
+  postfix: |
+    # Syslog stuff
+    PROCESS ([\w._\/%-]+)
+    COMPID postfix\/%{PROCESS:process}(?:\[%{NUMBER:pid}\])?
+    POSTFIX (?:%{SYSLOGTIMESTAMP:timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{COMPID}:
+```
+
 ## How to use
 
 ```
@@ -65,25 +77,12 @@ Install and configure logstash
 
 ## Development
 
-### Test syntax with yamllint
+### Test with molecule and docker
-
-* install `python` and `python-pip`
-* install yamllint `pip install yamllint`
-* run `yamllint .`
-
-### Test syntax with ansible-lint
-
-* install `python` and `python-pip`
-* install yamllint `pip install ansible-lint`
-* run `ansible-lint .`
-
-### Tests with docker
 
 * install [docker](https://docs.docker.com/engine/installation/)
-* install ruby
+* install `python3` and `python3-pip`
-* install bundler `gem install bundler`
+* install molecule and dependencies `pip3 install molecule molecule-docker docker ansible-lint pytest-testinfra yamllint`
-* install dependencies `bundle install`
+* run `molecule test`
-* run the tests `kitchen test`
 
 ## License
 

defaults/main.yml

@@ -6,6 +6,7 @@ logstash_default_config:
   path.data: /var/lib/logstash
   path.logs: /var/log/logstash
 logstash_full_config: '{{ logstash_default_config|combine(logstash_config, recursive=True) }}'
-logstash_inputs: {}
+logstash_inputs: []
 logstash_outputs: []
 logstash_filters: []
+logstash_patterns: {}

handlers/main.yml

@@ -1,5 +1,5 @@
 ---
 - name: restart logstash
-  systemd:
+  ansible.builtin.service:
     name: logstash
     state: restarted

meta/main.yml

@@ -5,13 +5,13 @@ galaxy_info:
   company: Adrien Waksberg
   description: Install and configure logstash
   license: Apache2
-  min_ansible_version: 2.7
+  min_ansible_version: 2.9
 
   platforms:
     - name: Debian
       versions:
-        - stretch
         - buster
+        - bullseye
 
   galaxy_tags:
     - elk

molecule/default/converge.yml

@@ -0,0 +1,30 @@
+---
+- name: Converge
+  hosts: all
+  roles:
+    - ansible-role-logstash
+  vars:
+    logstash_plugins:
+      - name: logstash-output-influxdb
+    logstash_patterns:
+      postfix: |
+        # Syslog stuff
+        PROCESS ([\w._\/%-]+)
+        COMPID postfix\/%{PROCESS:process}(?:\[%{NUMBER:pid}\])?
+        POSTFIX (?:%{SYSLOGTIMESTAMP:timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{COMPID}:
+    logstash_inputs:
+      - >
+        file {
+          path => "/var/log/syslog"
+        }
+    logstash_outputs:
+      - >
+        file {
+          path => "/tmp/logstash.log"
+          codec => "json"
+        }
+  pre_tasks:
+    - name: update apt cache
+      ansible.builtin.apt:
+        update_cache: true
+        cache_valid_time: 3600

molecule/default/molecule.yml

@@ -0,0 +1,26 @@
+---
+driver:
+  name: docker
+platforms:
+  - name: debian10
+    image: nishiki/debian10:molecule
+    privileged: true
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+    command: /bin/systemd
+    capabilities:
+      - SYS_ADMIN
+  - name: debian11
+    image: nishiki/debian11:molecule
+    privileged: true
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+    command: /bin/systemd
+    capabilities:
+      - SYS_ADMIN
+lint: |
+  set -e
+  yamllint .
+  ansible-lint .
+verifier:
+  name: testinfra

molecule/default/tests/test_default.py

@@ -0,0 +1,50 @@
+import testinfra.utils.ansible_runner
+import time
+
+def test_packages(host):
+  package = host.package('logstash')
+  assert package.is_installed
+
+def test_config_file(host):
+  path = host.file('/etc/logstash/logstash.yml')
+  assert path.exists
+  assert path.is_file
+  assert path.user == 'root'
+  assert path.group == 'root'
+  assert path.mode == 0o644
+
+def test_input_file(host):
+  path = host.file('/etc/logstash/conf.d/input.conf')
+  assert path.exists
+  assert path.is_file
+  assert path.user == 'root'
+  assert path.group == 'root'
+  assert path.mode == 0o644
+  assert path.contains('path => "/var/log/syslog"')
+
+def test_patterns_file(host):
+  path = host.file('/etc/logstash/patterns/postfix.conf')
+  assert path.exists
+  assert path.is_file
+  assert path.user == 'root'
+  assert path.group == 'root'
+  assert path.mode == 0o644
+  assert path.contains('# Syslog stuff')
+
+def test_service(host):
+  service = host.service('logstash')
+  assert service.is_running
+  assert service.is_enabled
+
+def test_output(host):
+  time.sleep(20)
+  host.run('echo test >> /var/log/syslog')
+  time.sleep(10)
+  path = host.file('/tmp/logstash.log')
+  assert path.exists
+  assert path.is_file
+  assert path.contains('test')
+
+def test_socket(host):
+  socket = host.socket('tcp://127.0.0.1:9600')
+  assert socket.is_listening

tasks/main.yml

@@ -1,18 +1,17 @@
 ---
 - name: install dependencies packages
-  package:
+  ansible.builtin.package:
-    name: '{{ packages }}'
+    name:
-  vars:
-    packages:
-      - default-jre
       - apt-transport-https
+      - default-jre
+      - gpg
   retries: 2
   register: result
   until: result is succeeded
   tags: logstash
 
 - name: add repository key
-  apt_key:
+  ansible.builtin.apt_key:
     url: https://artifacts.elastic.co/GPG-KEY-elasticsearch
   retries: 2
   register: result
@@ -20,20 +19,21 @@
   tags: logstash
 
 - name: add repository
-  apt_repository:
+  ansible.builtin.apt_repository:
     repo: 'deb https://artifacts.elastic.co/packages/{{ logstash_major_version }}.x/apt stable main'
   tags: logstash
 
 - name: install package
-  apt:
+  ansible.builtin.package:
-    name: logstash
+    name:
+      - logstash
   retries: 2
   register: result
   until: result is succeeded
   tags: logstash
 
 - name: copy general config file
-  copy:
+  ansible.builtin.copy:
     content: '{{ logstash_full_config|to_yaml }}'
     dest: /etc/logstash/logstash.yml
     owner: root
@@ -42,8 +42,29 @@
   notify: restart logstash
   tags: logstash
 
+- name: create patterns directory
+  ansible.builtin.file:
+    path: /etc/logstash/patterns
+    owner: root
+    group: root
+    mode: 0755
+    state: directory
+  tags: logstash
+
+- name: copy patterns files
+  ansible.builtin.copy:
+    content: '{{ item.value }}'
+    dest: '/etc/logstash/patterns/{{ item.key }}.conf'
+    owner: root
+    group: root
+    mode: 0644
+  loop: '{{ logstash_patterns|dict2items }}'
+  loop_control:
+    label: '{{ item.key }}'
+  tags: logstash
+
 - name: copy input, output and filter files
-  template:
+  ansible.builtin.template:
     src: '{{ item }}.conf.j2'
     dest: '/etc/logstash/conf.d/{{ item }}.conf'
     owner: root
@@ -58,14 +79,14 @@
   tags: logstash
 
 - name: install plugins
-  logstash_plugin:
+  community.general.logstash_plugin:
     name: '{{ item.name }}'
     state: '{{ item.state|default("present") }}'
   loop: '{{ logstash_plugins }}'
   tags: logstash
 
 - name: enable and start service
-  systemd:
+  ansible.builtin.service:
     name: logstash
     state: started
     enabled: true

templates/input.conf.j2

@@ -1,21 +1,7 @@
 # {{ ansible_managed }}
 
 input {
-{% for name, options in logstash_inputs.iteritems() %}
+{% for input in logstash_inputs %}
-  {{ name }} {
+{{ input }}
-{% for option, value in options.iteritems() %}
-{% if value is string %}
-    {{ option }} => "{{ value }}"
-{% elif value is number %}
-    {{ option }} => {{ value }}
-{% elif value is sameas True %}
-    {{ option }} => true
-{% elif value is sameas False %}
-    {{ option }} => false
-{% elif value is iterable %}
-    {{ option }} => ["{{ value|join('", ') }}"]
-{% endif %}
-{% endfor %}
-  }
 {% endfor %}
 }

test/integration/default/default.yml

@@ -1,18 +0,0 @@
----
-- hosts: localhost
-  connection: local
-  vars:
-    logstash_plugins:
-      - name: logstash-output-influxdb
-    logstash_inputs:
-      file:
-        path: /var/log/syslog
-    logstash_outputs:
-      - >
-        file {
-          path => "/var/log/syslog"
-          codec => "json"
-        }
-
-  roles:
-    - ansible-role-logstash

test/integration/default/serverspec/default_spec.rb

@@ -1,48 +0,0 @@
-require 'serverspec'
-
-set :backend, :exec
-
-puts
-puts '================================'
-puts %x(ansible --version)
-puts '================================'
-
-describe package('logstash') do
-  it { should be_installed }
-end
-
-describe file('/etc/logstash/logstash.yml') do
-  it { should be_file }
-  it { should be_mode 644 }
-  it { should be_owned_by 'root' }
-  it { should be_grouped_into 'root' }
-end
-
-describe file('/etc/logstash/conf.d/input.conf') do
-  it { should be_file }
-  it { should be_mode 644 }
-  it { should be_owned_by 'root' }
-  it { should be_grouped_into 'root' }
-  it { should contain 'path => "/var/log/syslog"' }
-end
-
-describe service('logstash') do
-  it { should be_enabled }
-  it { should be_running.under('systemd') }
-end
-
-sleep 20
-
-open('/tmp/logstash.log', 'a') do |f|
-  f << "test\n"
-end
-
-sleep 2
-
-describe port(9600) do
-  it { should be_listening }
-end
-
-describe file('/tmp/logstash.log') do
-  it { should be_file }
-end

test/integration/inventory

@@ -1 +0,0 @@
-localhost