Initial Ansible automation for homelab infrastructure

ansible/roles/monitoring/defaults/main.yml

@@ -0,0 +1,14 @@
+---
+prometheus_scrape_targets:
+  - job: node
+    targets:
+      - host: monitoring
+        address: "localhost:9100"
+      - host: gitea
+        address: "10.10.1.125:9100"
+  - job: loki
+    targets:
+      - host: monitoring
+        address: "localhost:3100"
+
+grafana_admin_password: admin

ansible/roles/monitoring/handlers/main.yml

@@ -0,0 +1,20 @@
+---
+- name: Restart prometheus
+  systemd:
+    name: prometheus
+    state: restarted
+
+- name: Restart loki
+  systemd:
+    name: loki
+    state: restarted
+
+- name: Restart promtail
+  systemd:
+    name: promtail
+    state: restarted
+
+- name: Restart grafana
+  systemd:
+    name: grafana
+    state: restarted

ansible/roles/monitoring/tasks/main.yml

@@ -0,0 +1,103 @@
+---
+- name: Install monitoring stack
+  community.general.pacman:
+    name:
+      - prometheus
+      - prometheus-node-exporter
+      - grafana
+      - loki
+      - promtail
+      - alertmanager
+    state: present
+
+- name: Create Loki data directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: loki
+    group: loki
+    mode: "0750"
+  loop:
+    - /var/lib/loki
+    - /var/lib/loki/chunks
+    - /var/lib/loki/rules
+
+- name: Create Prometheus rules directory
+  file:
+    path: /etc/prometheus/rules
+    state: directory
+    owner: prometheus
+    group: prometheus
+    mode: "0750"
+
+- name: Create promtail data directory
+  file:
+    path: /var/lib/promtail
+    state: directory
+    owner: promtail
+    group: promtail
+    mode: "0750"
+
+- name: Create Grafana provisioning directory
+  file:
+    path: /etc/grafana/provisioning/datasources
+    state: directory
+    owner: grafana
+    group: grafana
+    mode: "0750"
+
+- name: Configure Prometheus
+  template:
+    src: prometheus.yml.j2
+    dest: /etc/prometheus/prometheus.yml
+    owner: prometheus
+    group: prometheus
+    mode: "0640"
+  notify: Restart prometheus
+
+- name: Configure Loki
+  template:
+    src: loki.yaml.j2
+    dest: /etc/loki/loki.yaml
+    owner: loki
+    group: loki
+    mode: "0640"
+  notify: Restart loki
+
+- name: Configure Promtail
+  template:
+    src: promtail.yaml.j2
+    dest: /etc/loki/promtail.yaml
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Restart promtail
+
+- name: Configure Grafana datasources
+  template:
+    src: grafana-datasources.yaml.j2
+    dest: /etc/grafana/provisioning/datasources/homelab.yaml
+    owner: grafana
+    group: grafana
+    mode: "0640"
+  notify: Restart grafana
+
+- name: Set Grafana provisioning path
+  lineinfile:
+    path: /etc/grafana.ini
+    regexp: '^;?provisioning\s*='
+    line: "provisioning = /etc/grafana/provisioning"
+  notify: Restart grafana
+
+- name: Enable and start all monitoring services
+  systemd:
+    name: "{{ item }}"
+    enabled: true
+    state: started
+  loop:
+    - prometheus
+    - prometheus-node-exporter
+    - grafana
+    - loki
+    - promtail
+    - alertmanager

ansible/roles/monitoring/templates/grafana-datasources.yaml.j2

@@ -0,0 +1,14 @@
+apiVersion: 1
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://localhost:9090
+    isDefault: true
+    editable: true
+
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://localhost:3100
+    editable: true

ansible/roles/monitoring/templates/loki.yaml.j2

@@ -0,0 +1,44 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+  log_level: info
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /var/lib/loki
+  storage:
+    filesystem:
+      chunks_directory: /var/lib/loki/chunks
+      rules_directory: /var/lib/loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+limits_config:
+  metric_aggregation_enabled: true
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+ruler:
+  alertmanager_url: http://localhost:9093
+
+analytics:
+  reporting_enabled: false

ansible/roles/monitoring/templates/prometheus.yml.j2

@@ -0,0 +1,29 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets: ['localhost:9093']
+
+rule_files:
+  - /etc/prometheus/rules/*.yml
+
+scrape_configs:
+  - job_name: prometheus
+    static_configs:
+      - targets: ['localhost:9090']
+        labels:
+          instance: monitoring
+
+{% for job in prometheus_scrape_targets %}
+  - job_name: {{ job.job }}
+    static_configs:
+{% for target in job.targets %}
+      - targets: ['{{ target.address }}']
+        labels:
+          instance: {{ target.host }}
+{% endfor %}
+
+{% endfor %}

ansible/roles/monitoring/templates/promtail.yaml.j2

@@ -0,0 +1,30 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /var/lib/promtail/positions.yaml
+
+clients:
+  - url: http://localhost:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: journal
+    journal:
+      max_age: 12h
+      labels:
+        job: systemd-journal
+        host: monitoring
+    relabel_configs:
+      - source_labels: [__journal__systemd_unit]
+        target_label: unit
+      - source_labels: [__journal__hostname]
+        target_label: hostname
+
+  - job_name: varlogs
+    static_configs:
+      - targets: [localhost]
+        labels:
+          job: varlogs
+          host: monitoring
+          __path__: /var/log/*.log