Send host logs off the machine

---
# host-log-shipping / linux / t0 "Quick start"
#
# Forwards everything rsyslog sees to a collector, with a queue on disk so
# nothing is lost while the collector is down.
#
# Change the two lines under vars, then run:
#   ansible-playbook -i inventory.ini playbook.yml
#
# Verify:
#   ansible all -i inventory.ini -a "logger -t iac-test hello from $(hostname)"
#   Then search your collector for iac-test. It should be there in seconds.
#
# This sends over plain TCP. Use it on a network you control. The Standard
# version adds TLS and is the one to use for anything crossing a boundary.

- name: Forward system logs to a collector
  hosts: all
  become: true
  gather_facts: true

  vars:
    collector_host: logs.example.internal     # change me
    collector_port: 514                       # change me

  tasks:
    - name: Make sure rsyslog is installed
      ansible.builtin.package:
        name: rsyslog
        state: present

    - name: Write the forwarding rule
      ansible.builtin.copy:
        dest: /etc/rsyslog.d/10-forward.conf
        owner: root
        group: root
        mode: "0644"
        content: |
          # Managed by Ansible. Local edits are overwritten on the next run.

          # Hold messages on disk when the collector is unreachable, then send
          # them when it comes back. Without this a collector restart loses
          # every message sent during the outage.
          $ActionQueueType LinkedList
          $ActionQueueFileName fwd_main
          $ActionQueueMaxDiskSpace 512m
          $ActionQueueSaveOnShutdown on
          $ActionResumeRetryCount -1

          *.* @@{{ collector_host }}:{{ collector_port }}
      notify: Restart rsyslog

    - name: Make sure rsyslog is enabled and running
      ansible.builtin.service:
        name: rsyslog
        enabled: true
        state: started

  handlers:
    - name: Restart rsyslog
      ansible.builtin.service:
        name: rsyslog
        state: restarted

---
# Where the logs go. Set these per environment in group_vars.
log_collector_host: ""                # required, the role stops without it
log_collector_port: 6514              # 6514 is the registered port for syslog over TLS

# Encryption. Turning this off is a decision worth writing down somewhere,
# because syslog carries usernames, hostnames, and often command lines.
log_tls_enabled: true
log_tls_ca_file: /etc/ssl/certs/ca-certificates.crt
log_tls_auth_mode: x509/name          # x509/name | x509/certvalid | anon
log_tls_permitted_peer: ""            # defaults to the collector hostname

# The disk queue. With it, a collector outage becomes a delay. Messages wait on
# disk and are sent when the collector returns.
log_queue_max_disk: 1g
log_queue_spool_dir: /var/spool/rsyslog

# Rate limiting. systemd-journald drops messages above its own limit before
# rsyslog ever sees them, so this raises that ceiling too.
log_rate_limit_interval: 0            # 0 turns rsyslog's own limiter off
log_journald_rate_limit_burst: 20000

# What to send. Everything by default. If volume forces you to narrow it, write
# down which facilities you dropped and why, next to the change.
log_selector: "*.*"

# Send a message at the end of the run and confirm rsyslog accepted it.
log_verify_after_run: true

---
- name: Restart rsyslog
  ansible.builtin.service:
    name: rsyslog
    state: restarted

- name: Restart journald
  ansible.builtin.service:
    name: systemd-journald
    state: restarted

---
- name: Stop if no collector was set
  ansible.builtin.assert:
    that:
      - log_collector_host | length > 0
    fail_msg: "Set log_collector_host in group_vars. There is no sensible default for this."

- name: Install rsyslog
  ansible.builtin.package:
    name: rsyslog
    state: present

# The TLS driver is a separate package on most distributions, and leaving it out
# is the most common reason a TLS config silently falls back or fails to start.
- name: Install the TLS driver
  ansible.builtin.package:
    name: rsyslog-gnutls
    state: present
  when: log_tls_enabled | bool

- name: Check the CA file is actually there
  ansible.builtin.stat:
    path: "{{ log_tls_ca_file }}"
  register: log_ca
  when: log_tls_enabled | bool

- name: Stop if the CA file is missing
  ansible.builtin.assert:
    that:
      - log_ca.stat.exists
    fail_msg: >-
      {{ log_tls_ca_file }} does not exist on {{ inventory_hostname }}. Distribute
      the CA first, or point log_tls_ca_file at the right path.
  when: log_tls_enabled | bool

- name: Make sure the queue directory exists
  ansible.builtin.file:
    path: "{{ log_queue_spool_dir }}"
    state: directory
    owner: root
    group: root
    mode: "0700"

- name: Raise the journald rate limit so it stops dropping before rsyslog sees it
  ansible.builtin.lineinfile:
    path: /etc/systemd/journald.conf
    regexp: '^#?RateLimitBurst='
    line: "RateLimitBurst={{ log_journald_rate_limit_burst }}"
    state: present
  notify: Restart journald

- name: Write the forwarding rule
  ansible.builtin.template:
    src: forward.conf.j2
    dest: /etc/rsyslog.d/10-forward.conf
    owner: root
    group: root
    mode: "0644"
    validate: rsyslogd -N1 -f %s
  notify: Restart rsyslog

- name: Make sure rsyslog is enabled and running
  ansible.builtin.service:
    name: rsyslog
    enabled: true
    state: started

- name: Flush handlers so the check below runs against the new config
  ansible.builtin.meta: flush_handlers

- name: Send a line the collector can be searched for
  ansible.builtin.command: >-
    logger -t iac-verify "log forwarding check from {{ inventory_hostname }}"
  changed_when: false
  when: log_verify_after_run | bool

- name: Confirm rsyslog started without complaining
  ansible.builtin.command: rsyslogd -N1
  changed_when: false
  register: log_config_check
  failed_when: log_config_check.rc != 0
  when: log_verify_after_run | bool

# Managed by Ansible, role log_forwarding. Local edits are overwritten.

{% if log_tls_enabled %}
global(
  DefaultNetstreamDriver="gtls"
  DefaultNetstreamDriverCAFile="{{ log_tls_ca_file }}"
)
{% endif %}

# Hold messages on disk when the collector is unreachable, then send them when
# it comes back. Without this, a collector restart becomes a gap in the record.
action(
  type="omfwd"
  target="{{ log_collector_host }}"
  port="{{ log_collector_port }}"
  protocol="tcp"
{% if log_tls_enabled %}
  StreamDriver="gtls"
  StreamDriverMode="1"
  StreamDriverAuthMode="{{ log_tls_auth_mode }}"
{% if log_tls_auth_mode != 'anon' %}
  StreamDriverPermittedPeers="{{ log_tls_permitted_peer or log_collector_host }}"
{% endif %}
{% endif %}
  queue.type="LinkedList"
  queue.filename="fwd_main"
  queue.spoolDirectory="{{ log_queue_spool_dir }}"
  queue.maxdiskspace="{{ log_queue_max_disk }}"
  queue.saveOnShutdown="on"
  action.resumeRetryCount="-1"
)

{% if log_rate_limit_interval == 0 %}
# Rate limiting off. A burst of messages during an incident is exactly the part
# you want to keep.
$SystemLogRateLimitInterval 0
{% endif %}

---
# host-log-shipping / linux / t1 "Standard"
#
# Same result as the quick start, over TLS, as a role you can point at a
# different collector per environment.
#
#   group_vars/prod.yml
#     log_collector_host: logs.prod.internal
#     log_tls_ca_file: /etc/ssl/certs/internal-ca.pem
#
# Run:
#   ansible-playbook -i inventory.ini site.yml --check --diff
#   ansible-playbook -i inventory.ini site.yml

- name: Forward system logs
  hosts: all
  become: true
  gather_facts: true
  roles:
    - role: log_forwarding
      tags: [logging]