602 lines
15 KiB
Markdown
602 lines
15 KiB
Markdown
Based on your requirements (self-hosting everything, Debian as daily driver, Talos Linux for cattle/pets), here's a comprehensive guide:
|
|
|
|
## **Complete Self-Hosted Infrastructure Management with Ansible**
|
|
|
|
### **1. Project Structure**
|
|
```
|
|
infrastructure/
|
|
├── ansible.cfg
|
|
├── requirements.yml
|
|
├── Makefile
|
|
├── vault-pass.txt # Git-ignored, use env var in CI
|
|
├── inventory/
|
|
│ ├── production/
|
|
│ │ ├── group_vars/
|
|
│ │ │ ├── all.yml
|
|
│ │ │ ├── talos.yml
|
|
│ │ │ ├── debian.yml
|
|
│ │ │ └── k8s.yml
|
|
│ │ ├── host_vars/
|
|
│ │ │ ├── talos-control-01.yml
|
|
│ │ │ └── debian-workstation.yml
|
|
│ │ └── hosts.yml
|
|
│ └── staging/
|
|
├── playbooks/
|
|
│ ├── 01-provision-talos.yml
|
|
│ ├── 02-bootstrap-talos.yml
|
|
│ ├── 03-deploy-k8s-apps.yml
|
|
│ ├── 04-debian-setup.yml
|
|
│ └── 05-monitoring-stack.yml
|
|
├── roles/
|
|
│ ├── common/
|
|
│ │ ├── defaults/main.yml
|
|
│ │ ├── tasks/main.yml
|
|
│ │ ├── templates/
|
|
│ │ ├── files/
|
|
│ │ └── vars/main.yml
|
|
│ ├── talos-provision/
|
|
│ │ ├── tasks/
|
|
│ │ │ ├── download-talosctl.yml
|
|
│ │ │ ├── generate-configs.yml
|
|
│ │ │ └── apply-config.yml
|
|
│ │ └── templates/
|
|
│ │ └── talos-config.yaml.j2
|
|
│ ├── kubernetes-core/
|
|
│ │ ├── tasks/
|
|
│ │ │ ├── setup-kubeconfig.yml
|
|
│ │ │ ├── deploy-cni.yml
|
|
│ │ │ └── deploy-csi.yml
|
|
│ │ └── templates/
|
|
│ │ └── cni-values.yaml.j2
|
|
│ ├── selfhosted-apps/
|
|
│ │ ├── defaults/main.yml
|
|
│ │ ├── tasks/
|
|
│ │ │ ├── gitea.yml
|
|
│ │ │ ├── vaultwarden.yml
|
|
│ │ │ └── monitoring.yml
|
|
│ │ └── templates/
|
|
│ └── debian-workstation/
|
|
│ ├── tasks/
|
|
│ │ ├── packages.yml
|
|
│ │ ├── dotfiles.yml
|
|
│ │ └── development.yml
|
|
│ └── templates/
|
|
├── collections/requirements.yml
|
|
├── molecule/
|
|
│ └── default/
|
|
│ ├── molecule.yml
|
|
│ ├── converge.yml
|
|
│ ├── verify.yml
|
|
│ └── destroy.yml
|
|
├── .gitlab-ci.yml
|
|
├── .github/
|
|
│ └── workflows/
|
|
│ └── ansible-test.yml
|
|
└── vault/
|
|
├── secrets.yml
|
|
└── talos-secrets.yml
|
|
```
|
|
|
|
### **2. Detailed Configuration Examples**
|
|
|
|
#### **ansible.cfg**
|
|
```ini
|
|
[defaults]
|
|
inventory = inventory/
|
|
roles_path = roles
|
|
collections_path = collections
|
|
retry_files_enabled = False
|
|
host_key_checking = False
|
|
ansible_managed = Ansible managed: {file} modified on %Y-%m-%d %H:%M:%S by {uid}
|
|
stdout_callback = yaml
|
|
callback_whitelist = profile_tasks
|
|
|
|
[privilege_escalation]
|
|
become = True
|
|
become_method = sudo
|
|
become_user = root
|
|
become_ask_pass = False
|
|
```
|
|
|
|
#### **collections/requirements.yml**
|
|
```yaml
|
|
collections:
|
|
- name: kubernetes.core
|
|
version: ">=2.4.0"
|
|
- name: community.docker
|
|
version: ">=3.6.0"
|
|
- name: community.general
|
|
version: ">=7.3.0"
|
|
- name: ansible.posix
|
|
version: ">=1.5.0"
|
|
```
|
|
|
|
#### **inventory/production/hosts.yml**
|
|
```yaml
|
|
all:
|
|
children:
|
|
talos_cluster:
|
|
hosts:
|
|
talos-control-01:
|
|
ansible_host: 192.168.1.10
|
|
talos_role: controlplane
|
|
talos_endpoint: true
|
|
talos-control-02:
|
|
ansible_host: 192.168.1.11
|
|
talos_role: controlplane
|
|
talos-control-03:
|
|
ansible_host: 192.168.1.12
|
|
talos_role: controlplane
|
|
talos-worker-01:
|
|
ansible_host: 192.168.1.20
|
|
talos_role: worker
|
|
talos-worker-02:
|
|
ansible_host: 192.168.1.21
|
|
talos_role: worker
|
|
|
|
k8s_cluster:
|
|
hosts:
|
|
talos-control-01:
|
|
talos-control-02:
|
|
talos-control-03:
|
|
|
|
debian_hosts:
|
|
hosts:
|
|
debian-workstation:
|
|
ansible_host: 192.168.1.100
|
|
ansible_user: "{{ vault_debian_user }}"
|
|
ansible_become: true
|
|
debian-gateway:
|
|
ansible_host: 192.168.1.1
|
|
|
|
selfhosted_services:
|
|
hosts:
|
|
talos-worker-01:
|
|
talos-worker-02:
|
|
```
|
|
|
|
#### **Jinja2 Template Example: roles/talos-provision/templates/talos-config.yaml.j2**
|
|
```yaml
|
|
version: v1alpha1
|
|
machine:
|
|
type: {{ talos_role }}
|
|
install:
|
|
disk: /dev/sda
|
|
image: ghcr.io/siderolabs/installer:{{ talos_version }}
|
|
network:
|
|
hostname: {{ inventory_hostname }}
|
|
interfaces:
|
|
- interface: eth0
|
|
dhcp: true
|
|
kubelet:
|
|
extraArgs:
|
|
node-labels: "role={{ talos_role }},env=production"
|
|
{% if talos_role == 'controlplane' %}
|
|
certSANs:
|
|
- {{ ansible_host }}
|
|
- k8s-api.example.com
|
|
{% endif %}
|
|
cluster:
|
|
controlPlane:
|
|
endpoint: https://{{ groups['talos_cluster'] | selectattr('talos_endpoint') | map(attribute='ansible_host') | first }}:6443
|
|
clusterName: {{ cluster_name }}
|
|
```
|
|
|
|
#### **Ansible Vault Integration**
|
|
|
|
Create encrypted secrets:
|
|
```bash
|
|
# Create vault file
|
|
ansible-vault create vault/secrets.yml
|
|
|
|
# Edit existing
|
|
ansible-vault edit vault/secrets.yml
|
|
|
|
# View
|
|
ansible-vault view vault/secrets.yml
|
|
```
|
|
|
|
**vault/secrets.yml content:**
|
|
```yaml
|
|
---
|
|
vault_debian_user: "your_username"
|
|
vault_ssh_private_key: |
|
|
-----BEGIN OPENSSH PRIVATE KEY-----
|
|
[encrypted content]
|
|
-----END OPENSSH PRIVATE KEY-----
|
|
vault_gitea_admin_password: "secure_password"
|
|
vault_talos_ca_cert: |
|
|
-----BEGIN CERTIFICATE-----
|
|
[encrypted]
|
|
-----END CERTIFICATE-----
|
|
vault_cloudflare_api_token: "your_token"
|
|
```
|
|
|
|
#### **Playbook Example: playbooks/01-provision-talos.yml**
|
|
```yaml
|
|
---
|
|
- name: Provision Talos Linux Cluster
|
|
hosts: talos_cluster
|
|
gather_facts: false
|
|
vars_files:
|
|
- "{{ playbook_dir }}/../vault/secrets.yml"
|
|
|
|
tasks:
|
|
- name: Download talosctl
|
|
include_role:
|
|
name: talos-provision
|
|
tasks_from: download-talosctl.yml
|
|
delegate_to: localhost
|
|
|
|
- name: Generate Talos configuration
|
|
include_role:
|
|
name: talos-provision
|
|
tasks_from: generate-configs.yml
|
|
delegate_to: localhost
|
|
vars:
|
|
node_configs_dir: "/tmp/talos-configs"
|
|
|
|
- name: Apply configuration to nodes
|
|
include_role:
|
|
name: talos-provision
|
|
tasks_from: apply-config.yml
|
|
delegate_to: localhost
|
|
```
|
|
|
|
#### **Role Task Example: roles/kubernetes-core/tasks/deploy-cni.yml**
|
|
```yaml
|
|
---
|
|
- name: Create kubeconfig directory
|
|
ansible.builtin.file:
|
|
path: "~/.kube"
|
|
state: directory
|
|
mode: '0700'
|
|
|
|
- name: Fetch kubeconfig from Talos
|
|
kubernetes.core.k8s:
|
|
kubeconfig: "/tmp/kubeconfig"
|
|
state: present
|
|
definition:
|
|
apiVersion: v1
|
|
kind: Config
|
|
clusters:
|
|
- name: talos-cluster
|
|
cluster:
|
|
certificate-authority-data: "{{ vault_k8s_ca_cert | b64encode }}"
|
|
server: https://{{ talos_endpoint_host }}:6443
|
|
contexts:
|
|
- name: default
|
|
context:
|
|
cluster: talos-cluster
|
|
user: admin
|
|
current-context: default
|
|
users:
|
|
- name: admin
|
|
user:
|
|
client-certificate-data: "{{ vault_k8s_client_cert | b64encode }}"
|
|
client-key-data: "{{ vault_k8s_client_key | b64encode }}"
|
|
|
|
- name: Deploy Cilium CNI
|
|
kubernetes.core.helm:
|
|
kubeconfig: "/tmp/kubeconfig"
|
|
name: cilium
|
|
namespace: kube-system
|
|
chart_ref: cilium/cilium
|
|
release_values:
|
|
ipam:
|
|
mode: kubernetes
|
|
kubeProxyReplacement: strict
|
|
k8sServiceHost: "{{ talos_endpoint_host }}"
|
|
k8sServicePort: 6443
|
|
```
|
|
|
|
#### **Molecule Testing Setup**
|
|
|
|
**molecule/default/molecule.yml:**
|
|
```yaml
|
|
---
|
|
dependency:
|
|
name: galaxy
|
|
driver:
|
|
name: docker
|
|
platforms:
|
|
- name: talos-test
|
|
image: quay.io/siderolabs/talos:latest
|
|
command: /sbin/init
|
|
privileged: true
|
|
- name: debian-test
|
|
image: debian:bullseye
|
|
command: /sbin/init
|
|
privileged: true
|
|
provisioner:
|
|
name: ansible
|
|
playbooks:
|
|
converge: converge.yml
|
|
verify: verify.yml
|
|
verifier:
|
|
name: ansible
|
|
```
|
|
|
|
**molecule/default/verify.yml:**
|
|
```yaml
|
|
---
|
|
- name: Verify Talos configuration
|
|
hosts: talos-test
|
|
tasks:
|
|
- name: Check Talos version
|
|
command: talosctl version --client
|
|
register: talos_version
|
|
changed_when: false
|
|
|
|
- name: Validate configuration
|
|
assert:
|
|
that:
|
|
- "'Talosctl' in talos_version.stdout"
|
|
```
|
|
|
|
#### **GitLab CI Pipeline (.gitlab-ci.yml)**
|
|
```yaml
|
|
stages:
|
|
- test
|
|
- deploy
|
|
|
|
variables:
|
|
ANSIBLE_FORCE_COLOR: "1"
|
|
ANSIBLE_VAULT_PASSWORD_FILE: ".vault-pass"
|
|
|
|
before_script:
|
|
- apt-get update && apt-get install -y python3-pip
|
|
- pip3 install ansible ansible-lint molecule molecule-docker yamllint
|
|
- ansible-galaxy collection install -r collections/requirements.yml
|
|
- echo "$ANSIBLE_VAULT_PASSWORD" > .vault-pass
|
|
- chmod 600 .vault-pass
|
|
|
|
ansible-lint:
|
|
stage: test
|
|
script:
|
|
- ansible-lint playbooks/ roles/
|
|
rules:
|
|
- changes:
|
|
- playbooks/**/*
|
|
- roles/**/*
|
|
|
|
molecule-test:
|
|
stage: test
|
|
script:
|
|
- cd roles/common && molecule test
|
|
- cd roles/talos-provision && molecule test
|
|
rules:
|
|
- changes:
|
|
- roles/**/*
|
|
|
|
deploy-staging:
|
|
stage: deploy
|
|
script:
|
|
- ansible-playbook -i inventory/staging playbooks/*.yml --vault-password-file .vault-pass --check
|
|
environment:
|
|
name: staging
|
|
only:
|
|
- main
|
|
|
|
deploy-production:
|
|
stage: deploy
|
|
script:
|
|
- ansible-playbook -i inventory/production playbooks/*.yml --vault-password-file .vault-pass
|
|
environment:
|
|
name: production
|
|
when: manual
|
|
only:
|
|
- main
|
|
```
|
|
|
|
#### **GitHub Actions Alternative (.github/workflows/ansible-test.yml)**
|
|
```yaml
|
|
name: Ansible Testing
|
|
|
|
on: [push, pull_request]
|
|
|
|
jobs:
|
|
test:
|
|
runs-on: ubuntu-latest
|
|
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v4
|
|
with:
|
|
python-version: '3.10'
|
|
|
|
- name: Install dependencies
|
|
run: |
|
|
pip install ansible ansible-lint molecule molecule-docker
|
|
ansible-galaxy collection install -r collections/requirements.yml
|
|
|
|
- name: Run ansible-lint
|
|
run: ansible-lint playbooks/ roles/
|
|
|
|
- name: Run molecule tests
|
|
run: |
|
|
cd roles/common && molecule test
|
|
cd ../talos-provision && molecule test
|
|
|
|
- name: Dry-run deployment
|
|
env:
|
|
ANSIBLE_VAULT_PASSWORD: ${{ secrets.ANSIBLE_VAULT_PASSWORD }}
|
|
run: |
|
|
echo "$ANSIBLE_VAULT_PASSWORD" > .vault-pass
|
|
ansible-playbook -i inventory/staging playbooks/*.yml --vault-password-file .vault-pass --check
|
|
```
|
|
|
|
### **3. Self-Hosted Services Playbook Examples**
|
|
|
|
#### **Deploying Self-Hosted Apps on Talos/K8s**
|
|
```yaml
|
|
# playbooks/03-deploy-k8s-apps.yml
|
|
---
|
|
- name: Deploy Self-Hosted Applications
|
|
hosts: k8s_cluster
|
|
gather_facts: false
|
|
|
|
tasks:
|
|
- name: Create namespaces
|
|
kubernetes.core.k8s:
|
|
kubeconfig: "{{ kubeconfig_path }}"
|
|
state: present
|
|
definition:
|
|
apiVersion: v1
|
|
kind: Namespace
|
|
metadata:
|
|
name: "{{ item }}"
|
|
loop:
|
|
- gitea
|
|
- vaultwarden
|
|
- monitoring
|
|
- networking
|
|
|
|
- name: Deploy Longhorn for storage
|
|
kubernetes.core.helm:
|
|
kubeconfig: "{{ kubeconfig_path }}"
|
|
name: longhorn
|
|
namespace: longhorn-system
|
|
chart_ref: longhorn/longhorn
|
|
create_namespace: true
|
|
|
|
- name: Deploy Gitea
|
|
include_role:
|
|
name: selfhosted-apps
|
|
tasks_from: gitea.yml
|
|
|
|
- name: Deploy Vaultwarden
|
|
include_role:
|
|
name: selfhosted-apps
|
|
tasks_from: vaultwarden.yml
|
|
vars:
|
|
vaultwarden_admin_token: "{{ vault_vaultwarden_admin_token }}"
|
|
```
|
|
|
|
#### **Debian Daily Driver Setup**
|
|
```yaml
|
|
# playbooks/04-debian-setup.yml
|
|
---
|
|
- name: Configure Debian Workstation
|
|
hosts: debian_hosts
|
|
vars_files:
|
|
- "{{ playbook_dir }}/../vault/secrets.yml"
|
|
|
|
tasks:
|
|
- name: Install essential packages
|
|
apt:
|
|
name:
|
|
- git
|
|
- vim
|
|
- htop
|
|
- tmux
|
|
- curl
|
|
- wget
|
|
- build-essential
|
|
- python3-pip
|
|
- docker.io
|
|
- docker-compose
|
|
- tailscale
|
|
state: present
|
|
update_cache: true
|
|
|
|
- name: Configure Docker for non-root user
|
|
community.docker.docker_user:
|
|
user: "{{ vault_debian_user }}"
|
|
groups: docker
|
|
state: present
|
|
|
|
- name: Install development tools
|
|
include_role:
|
|
name: debian-workstation
|
|
tasks_from: development.yml
|
|
vars:
|
|
go_version: "1.21"
|
|
node_version: "18"
|
|
|
|
- name: Deploy dotfiles
|
|
include_role:
|
|
name: debian-workstation
|
|
tasks_from: dotfiles.yml
|
|
|
|
- name: Configure firewall
|
|
community.general.ufw:
|
|
policy: deny
|
|
logging: on
|
|
rule:
|
|
- from: "192.168.1.0/24"
|
|
port: "22"
|
|
proto: tcp
|
|
action: allow
|
|
- from: "192.168.1.0/24"
|
|
port: "80,443"
|
|
proto: tcp
|
|
action: allow
|
|
```
|
|
|
|
### **4. Secrets Management Strategy**
|
|
|
|
```yaml
|
|
# Best practices for secrets:
|
|
# 1. Separate vault files by environment
|
|
vault/
|
|
├── production/
|
|
│ ├── secrets.yml
|
|
│ ├── talos.yml
|
|
│ └── k8s.yml
|
|
├── staging/
|
|
└── development/
|
|
|
|
# 2. Use ansible-vault encrypt_string for individual secrets
|
|
# 3. Store vault password in CI/CD secrets, not in repo
|
|
# 4. Use different vault passwords per environment
|
|
```
|
|
|
|
### **5. Makefile for Common Operations**
|
|
```makefile
|
|
.PHONY: help install test deploy
|
|
|
|
help:
|
|
@echo "Available commands:"
|
|
@echo " make install Install collections and roles"
|
|
@echo " make lint Run ansible-lint"
|
|
@echo " make test Run molecule tests"
|
|
@echo " make deploy-dev Deploy to development"
|
|
@echo " make deploy-prod Deploy to production"
|
|
|
|
install:
|
|
ansible-galaxy collection install -r collections/requirements.yml
|
|
ansible-galaxy role install -r roles/requirements.yml
|
|
|
|
lint:
|
|
ansible-lint playbooks/ roles/ --exclude roles/external/
|
|
|
|
test:
|
|
cd roles/common && molecule test
|
|
cd roles/talos-provision && molecule test
|
|
|
|
deploy-dev:
|
|
ansible-playbook -i inventory/development site.yml --vault-password-file .vault-pass
|
|
|
|
deploy-prod:
|
|
ansible-playbook -i inventory/production site.yml --vault-password-file .vault-pass --limit production
|
|
|
|
vault-edit:
|
|
ansible-vault edit vault/production/secrets.yml
|
|
|
|
vault-view:
|
|
ansible-vault view vault/production/secrets.yml
|
|
```
|
|
|
|
### **Key Recommendations:**
|
|
|
|
1. **Version Control**: Keep everything in Git, including encrypted vault files
|
|
2. **Immutable Infrastructure**: Talos nodes should be treated as cattle - replace, don't repair
|
|
3. **Backup Strategy**: Use Velero for K8s backups, Borg/Restic for Debian
|
|
4. **Monitoring**: Deploy Prometheus, Grafana, and Loki on Talos cluster
|
|
5. **GitOps**: Consider ArgoCD for application deployment after initial setup
|
|
6. **Network**: Use Tailscale/WireGuard for secure access between nodes
|
|
|
|
This setup gives you a complete, self-hosted infrastructure managed entirely through Ansible with proper testing and CI/CD integration. |