Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/check_eol.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ on:

permissions:
contents: read
issues: write

jobs:
check-eol:
Expand Down Expand Up @@ -61,14 +62,13 @@ jobs:
}

echo "=== Checking OS distributions ==="
check_eol "rocky-linux" "8" "Rocky Linux 8"
check_eol "rocky-linux" "9" "Rocky Linux 9"
check_eol "rocky-linux" "10" "Rocky Linux 10"
check_eol "debian" "11" "Debian 11 (Bullseye)"
check_eol "debian" "12" "Debian 12 (Bookworm)"
check_eol "debian" "13" "Debian 13 (Trixie)"
check_eol "ubuntu" "22.04" "Ubuntu 22.04 (Jammy)"
check_eol "ubuntu" "24.04" "Ubuntu 24.04 (Noble)"
check_eol "ubuntu" "26.04" "Ubuntu 26.04 (Resolute Raccoon)"

echo ""
echo "=== Checking Python versions ==="
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/test_elasticsearch_custom_certs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,5 +42,5 @@ jobs:
uses: ./.github/workflows/molecule.yml
with:
scenarios: '["elasticsearch_custom_certs","elasticsearch_custom_certs_minimal"]'
distros: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '["rockylinux10","debian13"]' || '["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","debian12","debian13"]' }}
distros: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '["rockylinux10","debian13"]' || '["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","ubuntu2604","debian12","debian13"]' }}
secrets: inherit
2 changes: 1 addition & 1 deletion .github/workflows/test_elasticsearch_modules.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,6 @@ jobs:
uses: ./.github/workflows/molecule.yml
with:
scenarios: '["elasticsearch_test_modules"]'
distros: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '["rockylinux10","debian13"]' || '["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","debian12","debian13"]' }}
distros: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '["rockylinux10","debian13"]' || '["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","ubuntu2604","debian12","debian13"]' }}
timeout: 20
secrets: inherit
6 changes: 5 additions & 1 deletion .github/workflows/test_full_stack.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@ on:
- info
- warning
- debug
distros:
description: 'JSON array of distros (empty = all)'
required: false
type: string
pull_request:
merge_group:
schedule:
Expand Down Expand Up @@ -80,7 +84,7 @@ jobs:
strategy:
fail-fast: false
matrix:
distro: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && fromJSON('["rockylinux9","debian13"]') || fromJSON('["rockylinux9","ubuntu2204","ubuntu2404","debian12","debian13"]') }}
distro: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && fromJSON('["rockylinux9","debian13"]') || (inputs.distros != '' && fromJSON(inputs.distros)) || fromJSON('["rockylinux9","ubuntu2204","ubuntu2404","ubuntu2604","debian12","debian13"]') }}
scenario:
- elasticstack_default
- es_kibana
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/test_role_beats.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,5 +42,5 @@ jobs:
uses: ./.github/workflows/molecule.yml
with:
scenarios: '["beats_default","beats_peculiar","beats_advanced","beats_security"]'
distros: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '["rockylinux10","debian13"]' || '["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","debian12","debian13"]' }}
distros: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '["rockylinux10","debian13"]' || '["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","ubuntu2604","debian12","debian13"]' }}
secrets: inherit
2 changes: 1 addition & 1 deletion .github/workflows/test_role_elasticsearch.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ jobs:
uses: ./.github/workflows/molecule.yml
with:
scenarios: '["elasticsearch_default","elasticsearch_roles_calculation","elasticsearch_custom","elasticsearch_no-security","elasticsearch_cert_content"]'
distros: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '["rockylinux10","debian13"]' || '["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","debian12","debian13"]' }}
distros: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '["rockylinux10","debian13"]' || '["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","ubuntu2604","debian12","debian13"]' }}
secrets: inherit

molecule_elasticsearch_diagnostics:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/test_role_kibana.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,5 +42,5 @@ jobs:
uses: ./.github/workflows/molecule.yml
with:
scenarios: '["kibana_default","kibana_custom","kibana_custom_certs"]'
distros: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '["rockylinux10","debian13"]' || '["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","debian12","debian13"]' }}
distros: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '["rockylinux10","debian13"]' || '["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","ubuntu2604","debian12","debian13"]' }}
secrets: inherit
2 changes: 1 addition & 1 deletion .github/workflows/test_role_logstash.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,5 +42,5 @@ jobs:
uses: ./.github/workflows/molecule.yml
with:
scenarios: '["logstash_default","logstash_ssl","logstash_custom_pipeline","logstash_advanced","logstash_standalone_certs","logstash_centralized_pipelines"]'
distros: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '["rockylinux10","debian13"]' || '["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","debian12","debian13"]' }}
distros: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '["rockylinux10","debian13"]' || '["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","ubuntu2604","debian12","debian13"]' }}
secrets: inherit
2 changes: 1 addition & 1 deletion .github/workflows/test_role_repos.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,6 @@ jobs:
uses: ./.github/workflows/molecule.yml
with:
scenarios: '["repos_default"]'
distros: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '["rockylinux10","debian13"]' || '["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","debian12","debian13"]' }}
distros: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '["rockylinux10","debian13"]' || '["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","ubuntu2604","debian12","debian13"]' }}
timeout: 30
secrets: inherit
52 changes: 52 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
# Changelog

## 0.1.0 (unreleased)

First release of the `oddly.elasticstack` collection, forked from
[NETWAYS/ansible-collection-elasticstack](https://github.com/NETWAYS/ansible-collection-elasticstack).

### Highlights

- Full Elastic Stack 9.x support alongside 8.x, with version-aware templates
that switch configuration syntax automatically.
- Rolling upgrades from 8.x to 9.x with upgrade path validation, per-node
orchestration, and shard allocation management.
- Complete TLS PKI: CA generation, per-node certificates, automatic renewal,
and support for external certificates (PEM and PKCS12) with format
auto-detection.
- Inline PEM content variables for Vault / secrets manager integration
(Elasticsearch, Kibana, Beats).
- Separate transport and HTTP layer certificates on Elasticsearch.
- Certificate expiry warnings and tag-driven renewal (`--tags certificates`,
`--tags renew_es_cert`, `--tags renew_ca`).
- Logstash standalone certificate mode for independent deployments.
- Beats Filebeat `filestream` input type for 9.x (replacing deprecated `log`).
- Logstash `elastic_agent` input plugin support.
- Elasticsearch `cluster_settings` for runtime cluster configuration via API.
- LogsDB support with automatic enablement on fresh 9.x installs.
- Elasticsearch filesystem snapshot repository configuration.
- Cgroup-aware JVM heap auto-calculation for containerised deployments.
- Container workarounds: systemd `Type=exec` override, lenient disk watermarks.
- `elasticsearch_extra_config`, `kibana_extra_config`, `logstash_extra_config`
for arbitrary YAML settings without dedicated variables.
- `elasticstack_cert_pass` shortcut to set all TLS key passphrases at once.
- MkDocs Material documentation site with task flow diagrams, TLS guide, and
recipes.

### Supported platforms

- Debian 12 (bookworm), 13 (trixie)
- Ubuntu 22.04 (jammy), 24.04 (noble), 26.04 (resolute)
- Rocky Linux / RHEL 9, 10
- Elastic Stack 8.x and 9.x
- Ansible 2.18+

### Breaking changes from netways.elasticstack

- Namespace changed from `netways` to `oddly`.
- Minimum Ansible version raised to 2.18 (was 2.9).
- Debian 10/11 and Ubuntu 20.04 dropped from supported platforms.
- `elasticsearch_security: false` no longer permitted on ES 8.x+ (Elastic
upstream requirement).
- Logstash `logstash_beats_tls` renamed to `logstash_input_beats_ssl` (old name
still accepted for backwards compatibility).
6 changes: 3 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -99,9 +99,9 @@ See the **[getting started guide](https://oddly.github.io/elasticstack/getting-s

| | Versions |
|-|----------|
| Debian | 11, 12, 13 |
| Ubuntu | 22.04, 24.04 |
| Rocky Linux / RHEL | 8, 9, 10 |
| Debian | 12, 13 |
| Ubuntu | 22.04, 24.04, 26.04 |
| Rocky Linux / RHEL | 9, 10 |
| Elastic Stack | 8.x, 9.x |
| Ansible | 2.18+ |

Expand Down
2 changes: 1 addition & 1 deletion docs/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ ansible-galaxy collection install oddly.elasticstack
- **Automatic TLS** — generates and distributes certificates for inter-node and HTTP encryption
- **Security bootstrapping** — creates users, roles, and passwords on first run
- **Rolling upgrades** — 8.x to 9.x with zero-downtime node-by-node upgrades
- **Multi-platform** — Debian 11-13, Ubuntu 22.04/24.04, Rocky Linux/RHEL 8-10
- **Multi-platform** — Debian 12-13, Ubuntu 22.04/24.04/26.04, Rocky Linux/RHEL 9-10

<div class="grid cards" markdown>

Expand Down
2 changes: 1 addition & 1 deletion docs/introduction/getting-started.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
## Prerequisites

- Ansible 2.18 or later
- Target hosts running Debian 11/12/13, Ubuntu 22.04/24.04, or Rocky Linux/RHEL 8/9/10
- Target hosts running Debian 12/13, Ubuntu 22.04/24.04/26.04, or Rocky Linux/RHEL 9/10
- SSH access to target hosts with root or sudo privileges
- Minimum 4 GB RAM per Elasticsearch node (8 GB recommended)
- Python 3 on all target hosts (with `python3-apt` on Debian/Ubuntu)
Expand Down
1 change: 1 addition & 0 deletions molecule/shared/create.yml
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
rockylinux10: "molecule-rockylinux-10"
ubuntu2204: "molecule-ubuntu-22.04"
ubuntu2404: "molecule-ubuntu-24.04"
ubuntu2604: "molecule-ubuntu-26.04"

tasks:
- name: Delete any existing containers with same names
Expand Down
3 changes: 0 additions & 3 deletions roles/beats/defaults/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -200,9 +200,6 @@ beats_metricbeat_loadbalance: true
beats_cert_validity_period: 1095
# @var beats_cert_expiration_buffer:description: Days before certificate expiry to trigger renewal
beats_cert_expiration_buffer: 30
# @var beats_cert_will_expire_soon:description: Internal flag set when certificates are within the expiration buffer. Do not set manually
beats_cert_will_expire_soon: false

# @var beats_cert_source:description: Where to get TLS certificates. 'elasticsearch_ca' (default) auto-generates from the built-in CA; 'external' uses user-provided files
beats_cert_source: elasticsearch_ca

Expand Down
2 changes: 1 addition & 1 deletion roles/beats/meta/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ galaxy_info:
platforms:
- name: EL
versions:
- "8"
- "9"
- "10"
- name: Debian
Expand All @@ -21,6 +20,7 @@ galaxy_info:
versions:
- jammy
- noble
- resolute
galaxy_tags:
- elk
- beats
Expand Down
46 changes: 12 additions & 34 deletions roles/beats/tasks/beats-security.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,42 +10,20 @@

# -- Detect content mode vs file mode --
- name: beats-security | Detect content mode for Beats
ansible.builtin.set_fact:
_beats_content_mode: "{{ beats_tls_certificate_content | default('', true) | length > 0 }}"

- name: beats-security | Validate certificate is provided
ansible.builtin.assert:
that:
- (beats_tls_certificate_file | length > 0) or
(beats_tls_certificate_content | default('', true) | length > 0)
fail_msg: >-
External cert mode requires at minimum:
beats_tls_certificate_file or beats_tls_certificate_content

# -- Content mode: set format to PEM --
- name: beats-security | Set format facts for content mode
ansible.builtin.set_fact:
_beats_cert_format: pem
_beats_ca_extracted: false
when: _beats_content_mode | bool
ansible.builtin.include_tasks:
file: "{{ role_path }}/../elasticstack/tasks/certs/cert_detect_content_mode.yml"
vars:
_detect_cert_content_var: "{{ beats_tls_certificate_content | default('', true) }}"
_detect_key_content_var: "{{ beats_tls_key_content | default('', true) }}"
_detect_cert_file_var: "{{ beats_tls_certificate_file }}"
_detect_service_name: Beats
_detect_key_var_name: beats_tls_key_content

- name: beats-security | Check for CA chain in certificate content
- name: beats-security | Map detection results to Beats facts
ansible.builtin.set_fact:
_beats_ca_extracted: true
when:
- _beats_content_mode | bool
- >-
beats_tls_certificate_content | default('')
| regex_findall('-----BEGIN CERTIFICATE-----') | length > 1

- name: beats-security | Validate key content is provided (content mode)
ansible.builtin.assert:
that:
- beats_tls_key_content | default('', true) | length > 0
fail_msg: >-
Content mode requires both certificate and key content.
Set beats_tls_key_content.
when: _beats_content_mode | bool
_beats_content_mode: "{{ _detect_content_mode }}"
_beats_cert_format: "{{ _detect_cert_format | default(omit) }}"
_beats_ca_extracted: "{{ _detect_ca_extracted | default(false) }}"

# -- File mode: validate and detect format --
- name: beats-security | Validate Beats certificate (file mode)
Expand Down
9 changes: 9 additions & 0 deletions roles/beats/templates/_beats_logging.j2
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{% if beats_logging == "file" %}
logging.level: {{ beats_loglevel }}
logging.to_files: true
logging.files:
path: {{ beats_logpath }}
name: {{ _beat_name }}
keepfiles: {{ beats_logging_keepfiles }}
permissions: {{ beats_logging_permissions }}
{% endif %}
38 changes: 38 additions & 0 deletions roles/beats/templates/_beats_output.j2
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
{% if _beat_output == "elasticsearch" %}
output.elasticsearch:
{% if beats_security | bool %}
{% if elasticstack_full_stack | bool %}
hosts: [ {% for host in groups[elasticstack_elasticsearch_group_name] %}"https://{{ host }}:{{ elasticstack_elasticsearch_http_port }}"{% if not loop.last %},{% endif %}{% endfor %}]
{% else %}
hosts: [ {% for host in beats_target_hosts %}"https://{{ host }}:{{ elasticstack_elasticsearch_http_port }}"{% if not loop.last %},{% endif %}{% endfor %}]
{% endif %}
username: "elastic"
password: "{{ beats_writer_password.stdout }}"
ssl.enabled: true
ssl.verification_mode: {{ beats_ssl_verification_mode }}
ssl.certificate_authorities: ["/etc/beats/certs/ca.crt"]
{% else %}
{% if elasticstack_full_stack | bool %}
hosts: [ {% for host in groups[elasticstack_elasticsearch_group_name] %}"http://{{ host }}:{{ elasticstack_elasticsearch_http_port }}"{% if not loop.last %},{% endif %}{% endfor %}]
{% else %}
hosts: [ {% for host in beats_target_hosts %}"http://{{ host }}:{{ elasticstack_elasticsearch_http_port }}"{% if not loop.last %},{% endif %}{% endfor %}]
{% endif %}
{% endif %}
{% endif %}
{% if _beat_output == "logstash" %}
output.logstash:
{% if elasticstack_full_stack | bool %}
hosts: [ {% for host in groups[elasticstack_logstash_group_name] %}"{{ host }}:{{ elasticstack_beats_port }}"{% if not loop.last %},{% endif %}{% endfor %}]
{% else %}
hosts: [ {% for host in beats_target_hosts %}"{{ host }}:{{ elasticstack_beats_port }}"{% if not loop.last %},{% endif %}{% endfor %}]
{% endif %}
loadbalance: {{ _beat_loadbalance }}
{% if beats_security | bool %}
ssl.enabled: true
ssl.certificate_authorities: ["{{ beats_tls_cacert }}"]
ssl.verification_mode: full
ssl.key: {{ beats_tls_key }}
ssl.key_passphrase: {{ beats_tls_key_passphrase }}
ssl.certificate: {{ beats_tls_cert }}
{% endif %}
{% endif %}
52 changes: 5 additions & 47 deletions roles/beats/templates/auditbeat.yml.j2
Original file line number Diff line number Diff line change
Expand Up @@ -34,54 +34,12 @@ setup.kibana:
{% if elasticstack_full_stack | default(false) | bool and groups[elasticstack_kibana_group_name] | default([]) | length > 0 %}
host: "http://{{ groups[elasticstack_kibana_group_name][0] }}:{{ elasticstack_kibana_port }}"
{% endif %}
{% if beats_auditbeat_output == "elasticsearch" %}
output.elasticsearch:
{% if beats_security | bool %}
{% if elasticstack_full_stack | bool %}
hosts: [ {% for host in groups[elasticstack_elasticsearch_group_name] %}"https://{{ host }}:{{ elasticstack_elasticsearch_http_port }}"{% if not loop.last %},{% endif %}{% endfor %}]
{% else %}
hosts: [ {% for host in beats_target_hosts %}"https://{{ host }}:{{ elasticstack_elasticsearch_http_port }}"{% if not loop.last %},{% endif %}{% endfor %}]
{% endif %}
username: "elastic"
password: "{{ beats_writer_password.stdout }}"
ssl.enabled: true
ssl.verification_mode: {{ beats_ssl_verification_mode }}
ssl.certificate_authorities: ["/etc/beats/certs/ca.crt"]
{% else %}
{% if elasticstack_full_stack | bool %}
hosts: [ {% for host in groups[elasticstack_elasticsearch_group_name] %}"http://{{ host }}:{{ elasticstack_elasticsearch_http_port }}"{% if not loop.last %},{% endif %}{% endfor %}]
{% else %}
hosts: [ {% for host in beats_target_hosts %}"http://{{ host }}:{{ elasticstack_elasticsearch_http_port }}"{% if not loop.last %},{% endif %}{% endfor %}]
{% endif %}
{% endif %}
{% endif %}
{% if beats_auditbeat_output == "logstash" %}
output.logstash:
{% if elasticstack_full_stack | bool %}
hosts: [ {% for host in groups[elasticstack_logstash_group_name] %}"{{ host }}:{{ elasticstack_beats_port }}"{% if not loop.last %},{% endif %}{% endfor %}]
{% else %}
hosts: [ {% for host in beats_target_hosts %}"{{ host }}:{{ elasticstack_beats_port }}"{% if not loop.last %},{% endif %}{% endfor %}]
{% endif %}
loadbalance: {{ beats_auditbeat_loadbalance }}
{% if beats_security | bool %}
ssl.enabled: true
ssl.certificate_authorities: ["{{ beats_tls_cacert }}"]
ssl.verification_mode: full
ssl.key: {{ beats_tls_key }}
ssl.key_passphrase: {{ beats_tls_key_passphrase }}
ssl.certificate: {{ beats_tls_cert }}
{% endif %}
{% endif %}
{% set _beat_output = beats_auditbeat_output %}
{% set _beat_loadbalance = beats_auditbeat_loadbalance %}
{% set _beat_name = "auditbeat" %}
{% include '_beats_output.j2' %}

{% if beats_logging == "file" %}
logging.level: {{ beats_loglevel }}
logging.to_files: true
logging.files:
path: {{ beats_logpath }}
name: auditbeat
keepfiles: {{ beats_logging_keepfiles }}
permissions: {{ beats_logging_permissions }}
{% endif %}
{% include '_beats_logging.j2' %}

processors:
- add_host_metadata: ~
Expand Down
Loading
Loading