Initial Ansible automation for homelab infrastructure

ansible/ansible.cfg

@@ -0,0 +1,13 @@
+[defaults]
+inventory          = inventory/hosts.yml
+remote_user        = adyrem
+private_key_file   = ../keys/homelab_ed25519
+roles_path         = roles
+host_key_checking  = False
+stdout_callback    = yaml
+diff_mode          = True
+
+[privilege_escalation]
+become        = True
+become_method = sudo
+become_user   = root

ansible/group_vars/all.yml

@@ -0,0 +1,5 @@
+timezone: Europe/Zurich
+admin_user: adyrem
+admin_ssh_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILdW4rhQFv561bYs8w0/VgR6AFgOLvVsAh6pcjZ/CO2R homelab"
+
+loki_url: "http://10.10.1.137:3100"

ansible/group_vars/infra.yml

@@ -0,0 +1 @@
+ansible_user: adyrem

ansible/inventory/hosts.yml

@@ -0,0 +1,19 @@
+all:
+  vars:
+    ansible_python_interpreter: /usr/bin/python3
+
+  children:
+    infra:
+      hosts:
+        gitea:
+          ansible_host: 10.10.1.125
+        monitoring:
+          ansible_host: 10.10.1.137
+
+    monitoring_servers:
+      hosts:
+        monitoring:
+
+    gitea_servers:
+      hosts:
+        gitea:

ansible/playbooks/gitea.yml

@@ -0,0 +1,7 @@
+---
+- name: Deploy Gitea
+  hosts: gitea_servers
+  roles:
+    - node_exporter
+    - promtail
+    - gitea

ansible/playbooks/monitoring.yml

@@ -0,0 +1,7 @@
+---
+- name: Deploy monitoring stack
+  hosts: monitoring_servers
+  roles:
+    - node_exporter
+    - promtail
+    - monitoring

ansible/playbooks/site.yml

@@ -0,0 +1,17 @@
+---
+- name: Common setup for all VMs
+  hosts: all
+  roles:
+    - common
+    - node_exporter
+    - promtail
+
+- name: Monitoring stack
+  hosts: monitoring_servers
+  roles:
+    - monitoring
+
+- name: Gitea
+  hosts: gitea_servers
+  roles:
+    - gitea

ansible/roles/common/tasks/main.yml

@@ -0,0 +1,28 @@
+---
+- name: Set timezone
+  community.general.timezone:
+    name: "{{ timezone }}"
+
+- name: Configure passwordless sudo for wheel
+  copy:
+    dest: /etc/sudoers.d/wheel
+    content: "%wheel ALL=(ALL) NOPASSWD: ALL\n"
+    mode: "0440"
+    validate: visudo -cf %s
+
+- name: Ensure admin SSH key is authorized
+  authorized_key:
+    user: "{{ admin_user }}"
+    key: "{{ admin_ssh_key }}"
+    state: present
+
+- name: Install base packages
+  community.general.pacman:
+    name:
+      - vim
+      - git
+      - htop
+      - curl
+      - base-devel
+    state: present
+    update_cache: true

ansible/roles/gitea/defaults/main.yml

@@ -0,0 +1,12 @@
+---
+gitea_http_port: 3000
+gitea_domain: "{{ ansible_host }}"
+gitea_db_name: gitea
+gitea_db_user: gitea
+gitea_db_password: gitea
+gitea_secret_key: "4XlGCDRMd6ojyxs2RT7t2gHt9zP1vyMdzw3ci8svaTt2JemYpxW9Lk5Ptp3YpeBz"
+gitea_internal_token: "opPpoCIIb3bzUtejXGTSDT1HnSjuP5SeUzdteUSp2XjI0ZvywO5Ahr3PBuhVteUM"
+
+gitea_admin_user: adyrem
+gitea_admin_password: ""      # set in vault or at runtime
+gitea_admin_email: ""         # set in group_vars or vault

ansible/roles/gitea/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: Restart gitea
+  systemd:
+    name: gitea
+    state: restarted

ansible/roles/gitea/tasks/main.yml

@@ -0,0 +1,89 @@
+---
+- name: Install gitea and postgresql
+  community.general.pacman:
+    name:
+      - gitea
+      - postgresql
+    state: present
+
+- name: Initialize PostgreSQL cluster
+  command: sudo -u postgres initdb --locale=en_US.UTF-8 -D /var/lib/postgres/data
+  args:
+    creates: /var/lib/postgres/data/PG_VERSION
+
+- name: Enable and start PostgreSQL
+  systemd:
+    name: postgresql
+    enabled: true
+    state: started
+
+- name: Create gitea database user
+  community.postgresql.postgresql_user:
+    name: "{{ gitea_db_user }}"
+    password: "{{ gitea_db_password }}"
+    state: present
+  become_user: postgres
+
+- name: Create gitea database
+  community.postgresql.postgresql_db:
+    name: "{{ gitea_db_name }}"
+    owner: "{{ gitea_db_user }}"
+    state: present
+  become_user: postgres
+
+- name: Create gitea data directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: gitea
+    group: gitea
+    mode: "0750"
+  loop:
+    - /var/lib/gitea/data/repositories
+    - /var/log/gitea
+
+- name: Configure Gitea
+  template:
+    src: app.ini.j2
+    dest: /etc/gitea/app.ini
+    owner: gitea
+    group: gitea
+    mode: "0640"
+  notify: Restart gitea
+
+- name: Fix gitea account expiry
+  command: chage -E -1 gitea
+
+- name: Enable and start Gitea
+  systemd:
+    name: gitea
+    enabled: true
+    state: started
+
+- name: Wait for Gitea to be ready
+  uri:
+    url: "http://localhost:{{ gitea_http_port }}/api/v1/version"
+    status_code: 200
+  register: gitea_ready
+  until: gitea_ready.status == 200
+  retries: 10
+  delay: 3
+
+- name: Check if admin user exists
+  command: gitea admin user list --config /etc/gitea/app.ini
+  become_user: gitea
+  register: gitea_user_list
+  changed_when: false
+
+- name: Create Gitea admin user
+  command: >
+    gitea admin user create
+    --username {{ gitea_admin_user }}
+    --password {{ gitea_admin_password }}
+    --email {{ gitea_admin_email }}
+    --admin
+    --config /etc/gitea/app.ini
+  become_user: gitea
+  when:
+    - gitea_admin_password != ""
+    - gitea_admin_user not in gitea_user_list.stdout

ansible/roles/gitea/templates/app.ini.j2

@@ -0,0 +1,50 @@
+[DEFAULT]
+RUN_USER = gitea
+RUN_MODE = prod
+
+[server]
+DOMAIN           = {{ gitea_domain }}
+HTTP_ADDR        = 0.0.0.0
+HTTP_PORT        = {{ gitea_http_port }}
+ROOT_URL         = http://{{ gitea_domain }}:{{ gitea_http_port }}/
+SSH_DOMAIN       = {{ gitea_domain }}
+SSH_PORT         = 22
+START_SSH_SERVER = false
+DISABLE_SSH      = false
+
+[database]
+DB_TYPE  = postgres
+HOST     = 127.0.0.1:5432
+NAME     = {{ gitea_db_name }}
+USER     = {{ gitea_db_user }}
+PASSWD   = {{ gitea_db_password }}
+SSL_MODE = disable
+
+[repository]
+ROOT = /var/lib/gitea/data/repositories
+
+[security]
+SECRET_KEY         = {{ gitea_secret_key }}
+INTERNAL_TOKEN     = {{ gitea_internal_token }}
+INSTALL_LOCK       = true
+PASSWORD_HASH_ALGO = argon2
+
+[service]
+DISABLE_REGISTRATION       = false
+REQUIRE_SIGNIN_VIEW        = false
+DEFAULT_KEEP_EMAIL_PRIVATE = true
+
+[log]
+MODE      = console
+LEVEL     = info
+ROOT_PATH = /var/log/gitea
+
+[session]
+PROVIDER = db
+
+[cache]
+ADAPTER = memory
+
+[indexer]
+ISSUE_INDEXER_TYPE   = bleve
+REPO_INDEXER_ENABLED = true

ansible/roles/monitoring/defaults/main.yml

@@ -0,0 +1,14 @@
+---
+prometheus_scrape_targets:
+  - job: node
+    targets:
+      - host: monitoring
+        address: "localhost:9100"
+      - host: gitea
+        address: "10.10.1.125:9100"
+  - job: loki
+    targets:
+      - host: monitoring
+        address: "localhost:3100"
+
+grafana_admin_password: admin

ansible/roles/monitoring/handlers/main.yml

@@ -0,0 +1,20 @@
+---
+- name: Restart prometheus
+  systemd:
+    name: prometheus
+    state: restarted
+
+- name: Restart loki
+  systemd:
+    name: loki
+    state: restarted
+
+- name: Restart promtail
+  systemd:
+    name: promtail
+    state: restarted
+
+- name: Restart grafana
+  systemd:
+    name: grafana
+    state: restarted

ansible/roles/monitoring/tasks/main.yml

@@ -0,0 +1,103 @@
+---
+- name: Install monitoring stack
+  community.general.pacman:
+    name:
+      - prometheus
+      - prometheus-node-exporter
+      - grafana
+      - loki
+      - promtail
+      - alertmanager
+    state: present
+
+- name: Create Loki data directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: loki
+    group: loki
+    mode: "0750"
+  loop:
+    - /var/lib/loki
+    - /var/lib/loki/chunks
+    - /var/lib/loki/rules
+
+- name: Create Prometheus rules directory
+  file:
+    path: /etc/prometheus/rules
+    state: directory
+    owner: prometheus
+    group: prometheus
+    mode: "0750"
+
+- name: Create promtail data directory
+  file:
+    path: /var/lib/promtail
+    state: directory
+    owner: promtail
+    group: promtail
+    mode: "0750"
+
+- name: Create Grafana provisioning directory
+  file:
+    path: /etc/grafana/provisioning/datasources
+    state: directory
+    owner: grafana
+    group: grafana
+    mode: "0750"
+
+- name: Configure Prometheus
+  template:
+    src: prometheus.yml.j2
+    dest: /etc/prometheus/prometheus.yml
+    owner: prometheus
+    group: prometheus
+    mode: "0640"
+  notify: Restart prometheus
+
+- name: Configure Loki
+  template:
+    src: loki.yaml.j2
+    dest: /etc/loki/loki.yaml
+    owner: loki
+    group: loki
+    mode: "0640"
+  notify: Restart loki
+
+- name: Configure Promtail
+  template:
+    src: promtail.yaml.j2
+    dest: /etc/loki/promtail.yaml
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Restart promtail
+
+- name: Configure Grafana datasources
+  template:
+    src: grafana-datasources.yaml.j2
+    dest: /etc/grafana/provisioning/datasources/homelab.yaml
+    owner: grafana
+    group: grafana
+    mode: "0640"
+  notify: Restart grafana
+
+- name: Set Grafana provisioning path
+  lineinfile:
+    path: /etc/grafana.ini
+    regexp: '^;?provisioning\s*='
+    line: "provisioning = /etc/grafana/provisioning"
+  notify: Restart grafana
+
+- name: Enable and start all monitoring services
+  systemd:
+    name: "{{ item }}"
+    enabled: true
+    state: started
+  loop:
+    - prometheus
+    - prometheus-node-exporter
+    - grafana
+    - loki
+    - promtail
+    - alertmanager

ansible/roles/monitoring/templates/grafana-datasources.yaml.j2

@@ -0,0 +1,14 @@
+apiVersion: 1
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://localhost:9090
+    isDefault: true
+    editable: true
+
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://localhost:3100
+    editable: true

ansible/roles/monitoring/templates/loki.yaml.j2

@@ -0,0 +1,44 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+  log_level: info
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /var/lib/loki
+  storage:
+    filesystem:
+      chunks_directory: /var/lib/loki/chunks
+      rules_directory: /var/lib/loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+limits_config:
+  metric_aggregation_enabled: true
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+ruler:
+  alertmanager_url: http://localhost:9093
+
+analytics:
+  reporting_enabled: false

ansible/roles/monitoring/templates/prometheus.yml.j2

@@ -0,0 +1,29 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets: ['localhost:9093']
+
+rule_files:
+  - /etc/prometheus/rules/*.yml
+
+scrape_configs:
+  - job_name: prometheus
+    static_configs:
+      - targets: ['localhost:9090']
+        labels:
+          instance: monitoring
+
+{% for job in prometheus_scrape_targets %}
+  - job_name: {{ job.job }}
+    static_configs:
+{% for target in job.targets %}
+      - targets: ['{{ target.address }}']
+        labels:
+          instance: {{ target.host }}
+{% endfor %}
+
+{% endfor %}

ansible/roles/monitoring/templates/promtail.yaml.j2

@@ -0,0 +1,30 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /var/lib/promtail/positions.yaml
+
+clients:
+  - url: http://localhost:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: journal
+    journal:
+      max_age: 12h
+      labels:
+        job: systemd-journal
+        host: monitoring
+    relabel_configs:
+      - source_labels: [__journal__systemd_unit]
+        target_label: unit
+      - source_labels: [__journal__hostname]
+        target_label: hostname
+
+  - job_name: varlogs
+    static_configs:
+      - targets: [localhost]
+        labels:
+          job: varlogs
+          host: monitoring
+          __path__: /var/log/*.log

ansible/roles/node_exporter/tasks/main.yml

@@ -0,0 +1,11 @@
+---
+- name: Install prometheus-node-exporter
+  community.general.pacman:
+    name: prometheus-node-exporter
+    state: present
+
+- name: Enable and start node_exporter
+  systemd:
+    name: prometheus-node-exporter
+    enabled: true
+    state: started

ansible/roles/promtail/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+promtail_host_label: "{{ inventory_hostname }}"

ansible/roles/promtail/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: Restart promtail
+  systemd:
+    name: promtail
+    state: restarted

ansible/roles/promtail/tasks/main.yml

@@ -0,0 +1,28 @@
+---
+- name: Install promtail
+  community.general.pacman:
+    name: promtail
+    state: present
+
+- name: Create promtail data directory
+  file:
+    path: /var/lib/promtail
+    state: directory
+    owner: promtail
+    group: promtail
+    mode: "0750"
+
+- name: Configure promtail
+  template:
+    src: promtail.yaml.j2
+    dest: /etc/loki/promtail.yaml
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Restart promtail
+
+- name: Enable and start promtail
+  systemd:
+    name: promtail
+    enabled: true
+    state: started

ansible/roles/promtail/templates/promtail.yaml.j2

@@ -0,0 +1,30 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /var/lib/promtail/positions.yaml
+
+clients:
+  - url: {{ loki_url }}/loki/api/v1/push
+
+scrape_configs:
+  - job_name: journal
+    journal:
+      max_age: 12h
+      labels:
+        job: systemd-journal
+        host: {{ promtail_host_label }}
+    relabel_configs:
+      - source_labels: [__journal__systemd_unit]
+        target_label: unit
+      - source_labels: [__journal__hostname]
+        target_label: hostname
+
+  - job_name: varlogs
+    static_configs:
+      - targets: [localhost]
+        labels:
+          job: varlogs
+          host: {{ promtail_host_label }}
+          __path__: /var/log/*.log