Add tech_docs/Complete Self-Hosted.Infrastructure.with.Ansible.md
This commit is contained in:
602
tech_docs/Complete Self-Hosted.Infrastructure.with.Ansible.md
Normal file
602
tech_docs/Complete Self-Hosted.Infrastructure.with.Ansible.md
Normal file
@@ -0,0 +1,602 @@
|
||||
Based on your requirements (self-hosting everything, Debian as daily driver, Talos Linux for cattle/pets), here's a comprehensive guide:
|
||||
|
||||
## **Complete Self-Hosted Infrastructure Management with Ansible**
|
||||
|
||||
### **1. Project Structure**
|
||||
```
|
||||
infrastructure/
|
||||
├── ansible.cfg
|
||||
├── requirements.yml
|
||||
├── Makefile
|
||||
├── vault-pass.txt # Git-ignored, use env var in CI
|
||||
├── inventory/
|
||||
│ ├── production/
|
||||
│ │ ├── group_vars/
|
||||
│ │ │ ├── all.yml
|
||||
│ │ │ ├── talos.yml
|
||||
│ │ │ ├── debian.yml
|
||||
│ │ │ └── k8s.yml
|
||||
│ │ ├── host_vars/
|
||||
│ │ │ ├── talos-control-01.yml
|
||||
│ │ │ └── debian-workstation.yml
|
||||
│ │ └── hosts.yml
|
||||
│ └── staging/
|
||||
├── playbooks/
|
||||
│ ├── 01-provision-talos.yml
|
||||
│ ├── 02-bootstrap-talos.yml
|
||||
│ ├── 03-deploy-k8s-apps.yml
|
||||
│ ├── 04-debian-setup.yml
|
||||
│ └── 05-monitoring-stack.yml
|
||||
├── roles/
|
||||
│ ├── common/
|
||||
│ │ ├── defaults/main.yml
|
||||
│ │ ├── tasks/main.yml
|
||||
│ │ ├── templates/
|
||||
│ │ ├── files/
|
||||
│ │ └── vars/main.yml
|
||||
│ ├── talos-provision/
|
||||
│ │ ├── tasks/
|
||||
│ │ │ ├── download-talosctl.yml
|
||||
│ │ │ ├── generate-configs.yml
|
||||
│ │ │ └── apply-config.yml
|
||||
│ │ └── templates/
|
||||
│ │ └── talos-config.yaml.j2
|
||||
│ ├── kubernetes-core/
|
||||
│ │ ├── tasks/
|
||||
│ │ │ ├── setup-kubeconfig.yml
|
||||
│ │ │ ├── deploy-cni.yml
|
||||
│ │ │ └── deploy-csi.yml
|
||||
│ │ └── templates/
|
||||
│ │ └── cni-values.yaml.j2
|
||||
│ ├── selfhosted-apps/
|
||||
│ │ ├── defaults/main.yml
|
||||
│ │ ├── tasks/
|
||||
│ │ │ ├── gitea.yml
|
||||
│ │ │ ├── vaultwarden.yml
|
||||
│ │ │ └── monitoring.yml
|
||||
│ │ └── templates/
|
||||
│ └── debian-workstation/
|
||||
│ ├── tasks/
|
||||
│ │ ├── packages.yml
|
||||
│ │ ├── dotfiles.yml
|
||||
│ │ └── development.yml
|
||||
│ └── templates/
|
||||
├── collections/requirements.yml
|
||||
├── molecule/
|
||||
│ └── default/
|
||||
│ ├── molecule.yml
|
||||
│ ├── converge.yml
|
||||
│ ├── verify.yml
|
||||
│ └── destroy.yml
|
||||
├── .gitlab-ci.yml
|
||||
├── .github/
|
||||
│ └── workflows/
|
||||
│ └── ansible-test.yml
|
||||
└── vault/
|
||||
├── secrets.yml
|
||||
└── talos-secrets.yml
|
||||
```
|
||||
|
||||
### **2. Detailed Configuration Examples**
|
||||
|
||||
#### **ansible.cfg**
|
||||
```ini
|
||||
[defaults]
|
||||
inventory = inventory/
|
||||
roles_path = roles
|
||||
collections_path = collections
|
||||
retry_files_enabled = False
|
||||
host_key_checking = False
|
||||
ansible_managed = Ansible managed: {file} modified on %Y-%m-%d %H:%M:%S by {uid}
|
||||
stdout_callback = yaml
|
||||
callback_whitelist = profile_tasks
|
||||
|
||||
[privilege_escalation]
|
||||
become = True
|
||||
become_method = sudo
|
||||
become_user = root
|
||||
become_ask_pass = False
|
||||
```
|
||||
|
||||
#### **collections/requirements.yml**
|
||||
```yaml
|
||||
collections:
|
||||
- name: kubernetes.core
|
||||
version: ">=2.4.0"
|
||||
- name: community.docker
|
||||
version: ">=3.6.0"
|
||||
- name: community.general
|
||||
version: ">=7.3.0"
|
||||
- name: ansible.posix
|
||||
version: ">=1.5.0"
|
||||
```
|
||||
|
||||
#### **inventory/production/hosts.yml**
|
||||
```yaml
|
||||
all:
|
||||
children:
|
||||
talos_cluster:
|
||||
hosts:
|
||||
talos-control-01:
|
||||
ansible_host: 192.168.1.10
|
||||
talos_role: controlplane
|
||||
talos_endpoint: true
|
||||
talos-control-02:
|
||||
ansible_host: 192.168.1.11
|
||||
talos_role: controlplane
|
||||
talos-control-03:
|
||||
ansible_host: 192.168.1.12
|
||||
talos_role: controlplane
|
||||
talos-worker-01:
|
||||
ansible_host: 192.168.1.20
|
||||
talos_role: worker
|
||||
talos-worker-02:
|
||||
ansible_host: 192.168.1.21
|
||||
talos_role: worker
|
||||
|
||||
k8s_cluster:
|
||||
hosts:
|
||||
talos-control-01:
|
||||
talos-control-02:
|
||||
talos-control-03:
|
||||
|
||||
debian_hosts:
|
||||
hosts:
|
||||
debian-workstation:
|
||||
ansible_host: 192.168.1.100
|
||||
ansible_user: "{{ vault_debian_user }}"
|
||||
ansible_become: true
|
||||
debian-gateway:
|
||||
ansible_host: 192.168.1.1
|
||||
|
||||
selfhosted_services:
|
||||
hosts:
|
||||
talos-worker-01:
|
||||
talos-worker-02:
|
||||
```
|
||||
|
||||
#### **Jinja2 Template Example: roles/talos-provision/templates/talos-config.yaml.j2**
|
||||
```yaml
|
||||
version: v1alpha1
|
||||
machine:
|
||||
type: {{ talos_role }}
|
||||
install:
|
||||
disk: /dev/sda
|
||||
image: ghcr.io/siderolabs/installer:{{ talos_version }}
|
||||
network:
|
||||
hostname: {{ inventory_hostname }}
|
||||
interfaces:
|
||||
- interface: eth0
|
||||
dhcp: true
|
||||
kubelet:
|
||||
extraArgs:
|
||||
node-labels: "role={{ talos_role }},env=production"
|
||||
{% if talos_role == 'controlplane' %}
|
||||
certSANs:
|
||||
- {{ ansible_host }}
|
||||
- k8s-api.example.com
|
||||
{% endif %}
|
||||
cluster:
|
||||
controlPlane:
|
||||
endpoint: https://{{ groups['talos_cluster'] | selectattr('talos_endpoint') | map(attribute='ansible_host') | first }}:6443
|
||||
clusterName: {{ cluster_name }}
|
||||
```
|
||||
|
||||
#### **Ansible Vault Integration**
|
||||
|
||||
Create encrypted secrets:
|
||||
```bash
|
||||
# Create vault file
|
||||
ansible-vault create vault/secrets.yml
|
||||
|
||||
# Edit existing
|
||||
ansible-vault edit vault/secrets.yml
|
||||
|
||||
# View
|
||||
ansible-vault view vault/secrets.yml
|
||||
```
|
||||
|
||||
**vault/secrets.yml content:**
|
||||
```yaml
|
||||
---
|
||||
vault_debian_user: "your_username"
|
||||
vault_ssh_private_key: |
|
||||
-----BEGIN OPENSSH PRIVATE KEY-----
|
||||
[encrypted content]
|
||||
-----END OPENSSH PRIVATE KEY-----
|
||||
vault_gitea_admin_password: "secure_password"
|
||||
vault_talos_ca_cert: |
|
||||
-----BEGIN CERTIFICATE-----
|
||||
[encrypted]
|
||||
-----END CERTIFICATE-----
|
||||
vault_cloudflare_api_token: "your_token"
|
||||
```
|
||||
|
||||
#### **Playbook Example: playbooks/01-provision-talos.yml**
|
||||
```yaml
|
||||
---
|
||||
- name: Provision Talos Linux Cluster
|
||||
hosts: talos_cluster
|
||||
gather_facts: false
|
||||
vars_files:
|
||||
- "{{ playbook_dir }}/../vault/secrets.yml"
|
||||
|
||||
tasks:
|
||||
- name: Download talosctl
|
||||
include_role:
|
||||
name: talos-provision
|
||||
tasks_from: download-talosctl.yml
|
||||
delegate_to: localhost
|
||||
|
||||
- name: Generate Talos configuration
|
||||
include_role:
|
||||
name: talos-provision
|
||||
tasks_from: generate-configs.yml
|
||||
delegate_to: localhost
|
||||
vars:
|
||||
node_configs_dir: "/tmp/talos-configs"
|
||||
|
||||
- name: Apply configuration to nodes
|
||||
include_role:
|
||||
name: talos-provision
|
||||
tasks_from: apply-config.yml
|
||||
delegate_to: localhost
|
||||
```
|
||||
|
||||
#### **Role Task Example: roles/kubernetes-core/tasks/deploy-cni.yml**
|
||||
```yaml
|
||||
---
|
||||
- name: Create kubeconfig directory
|
||||
ansible.builtin.file:
|
||||
path: "~/.kube"
|
||||
state: directory
|
||||
mode: '0700'
|
||||
|
||||
- name: Fetch kubeconfig from Talos
|
||||
kubernetes.core.k8s:
|
||||
kubeconfig: "/tmp/kubeconfig"
|
||||
state: present
|
||||
definition:
|
||||
apiVersion: v1
|
||||
kind: Config
|
||||
clusters:
|
||||
- name: talos-cluster
|
||||
cluster:
|
||||
certificate-authority-data: "{{ vault_k8s_ca_cert | b64encode }}"
|
||||
server: https://{{ talos_endpoint_host }}:6443
|
||||
contexts:
|
||||
- name: default
|
||||
context:
|
||||
cluster: talos-cluster
|
||||
user: admin
|
||||
current-context: default
|
||||
users:
|
||||
- name: admin
|
||||
user:
|
||||
client-certificate-data: "{{ vault_k8s_client_cert | b64encode }}"
|
||||
client-key-data: "{{ vault_k8s_client_key | b64encode }}"
|
||||
|
||||
- name: Deploy Cilium CNI
|
||||
kubernetes.core.helm:
|
||||
kubeconfig: "/tmp/kubeconfig"
|
||||
name: cilium
|
||||
namespace: kube-system
|
||||
chart_ref: cilium/cilium
|
||||
release_values:
|
||||
ipam:
|
||||
mode: kubernetes
|
||||
kubeProxyReplacement: strict
|
||||
k8sServiceHost: "{{ talos_endpoint_host }}"
|
||||
k8sServicePort: 6443
|
||||
```
|
||||
|
||||
#### **Molecule Testing Setup**
|
||||
|
||||
**molecule/default/molecule.yml:**
|
||||
```yaml
|
||||
---
|
||||
dependency:
|
||||
name: galaxy
|
||||
driver:
|
||||
name: docker
|
||||
platforms:
|
||||
- name: talos-test
|
||||
image: quay.io/siderolabs/talos:latest
|
||||
command: /sbin/init
|
||||
privileged: true
|
||||
- name: debian-test
|
||||
image: debian:bullseye
|
||||
command: /sbin/init
|
||||
privileged: true
|
||||
provisioner:
|
||||
name: ansible
|
||||
playbooks:
|
||||
converge: converge.yml
|
||||
verify: verify.yml
|
||||
verifier:
|
||||
name: ansible
|
||||
```
|
||||
|
||||
**molecule/default/verify.yml:**
|
||||
```yaml
|
||||
---
|
||||
- name: Verify Talos configuration
|
||||
hosts: talos-test
|
||||
tasks:
|
||||
- name: Check Talos version
|
||||
command: talosctl version --client
|
||||
register: talos_version
|
||||
changed_when: false
|
||||
|
||||
- name: Validate configuration
|
||||
assert:
|
||||
that:
|
||||
- "'Talosctl' in talos_version.stdout"
|
||||
```
|
||||
|
||||
#### **GitLab CI Pipeline (.gitlab-ci.yml)**
|
||||
```yaml
|
||||
stages:
|
||||
- test
|
||||
- deploy
|
||||
|
||||
variables:
|
||||
ANSIBLE_FORCE_COLOR: "1"
|
||||
ANSIBLE_VAULT_PASSWORD_FILE: ".vault-pass"
|
||||
|
||||
before_script:
|
||||
- apt-get update && apt-get install -y python3-pip
|
||||
- pip3 install ansible ansible-lint molecule molecule-docker yamllint
|
||||
- ansible-galaxy collection install -r collections/requirements.yml
|
||||
- echo "$ANSIBLE_VAULT_PASSWORD" > .vault-pass
|
||||
- chmod 600 .vault-pass
|
||||
|
||||
ansible-lint:
|
||||
stage: test
|
||||
script:
|
||||
- ansible-lint playbooks/ roles/
|
||||
rules:
|
||||
- changes:
|
||||
- playbooks/**/*
|
||||
- roles/**/*
|
||||
|
||||
molecule-test:
|
||||
stage: test
|
||||
script:
|
||||
- cd roles/common && molecule test
|
||||
- cd roles/talos-provision && molecule test
|
||||
rules:
|
||||
- changes:
|
||||
- roles/**/*
|
||||
|
||||
deploy-staging:
|
||||
stage: deploy
|
||||
script:
|
||||
- ansible-playbook -i inventory/staging playbooks/*.yml --vault-password-file .vault-pass --check
|
||||
environment:
|
||||
name: staging
|
||||
only:
|
||||
- main
|
||||
|
||||
deploy-production:
|
||||
stage: deploy
|
||||
script:
|
||||
- ansible-playbook -i inventory/production playbooks/*.yml --vault-password-file .vault-pass
|
||||
environment:
|
||||
name: production
|
||||
when: manual
|
||||
only:
|
||||
- main
|
||||
```
|
||||
|
||||
#### **GitHub Actions Alternative (.github/workflows/ansible-test.yml)**
|
||||
```yaml
|
||||
name: Ansible Testing
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: '3.10'
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
pip install ansible ansible-lint molecule molecule-docker
|
||||
ansible-galaxy collection install -r collections/requirements.yml
|
||||
|
||||
- name: Run ansible-lint
|
||||
run: ansible-lint playbooks/ roles/
|
||||
|
||||
- name: Run molecule tests
|
||||
run: |
|
||||
cd roles/common && molecule test
|
||||
cd ../talos-provision && molecule test
|
||||
|
||||
- name: Dry-run deployment
|
||||
env:
|
||||
ANSIBLE_VAULT_PASSWORD: ${{ secrets.ANSIBLE_VAULT_PASSWORD }}
|
||||
run: |
|
||||
echo "$ANSIBLE_VAULT_PASSWORD" > .vault-pass
|
||||
ansible-playbook -i inventory/staging playbooks/*.yml --vault-password-file .vault-pass --check
|
||||
```
|
||||
|
||||
### **3. Self-Hosted Services Playbook Examples**
|
||||
|
||||
#### **Deploying Self-Hosted Apps on Talos/K8s**
|
||||
```yaml
|
||||
# playbooks/03-deploy-k8s-apps.yml
|
||||
---
|
||||
- name: Deploy Self-Hosted Applications
|
||||
hosts: k8s_cluster
|
||||
gather_facts: false
|
||||
|
||||
tasks:
|
||||
- name: Create namespaces
|
||||
kubernetes.core.k8s:
|
||||
kubeconfig: "{{ kubeconfig_path }}"
|
||||
state: present
|
||||
definition:
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: "{{ item }}"
|
||||
loop:
|
||||
- gitea
|
||||
- vaultwarden
|
||||
- monitoring
|
||||
- networking
|
||||
|
||||
- name: Deploy Longhorn for storage
|
||||
kubernetes.core.helm:
|
||||
kubeconfig: "{{ kubeconfig_path }}"
|
||||
name: longhorn
|
||||
namespace: longhorn-system
|
||||
chart_ref: longhorn/longhorn
|
||||
create_namespace: true
|
||||
|
||||
- name: Deploy Gitea
|
||||
include_role:
|
||||
name: selfhosted-apps
|
||||
tasks_from: gitea.yml
|
||||
|
||||
- name: Deploy Vaultwarden
|
||||
include_role:
|
||||
name: selfhosted-apps
|
||||
tasks_from: vaultwarden.yml
|
||||
vars:
|
||||
vaultwarden_admin_token: "{{ vault_vaultwarden_admin_token }}"
|
||||
```
|
||||
|
||||
#### **Debian Daily Driver Setup**
|
||||
```yaml
|
||||
# playbooks/04-debian-setup.yml
|
||||
---
|
||||
- name: Configure Debian Workstation
|
||||
hosts: debian_hosts
|
||||
vars_files:
|
||||
- "{{ playbook_dir }}/../vault/secrets.yml"
|
||||
|
||||
tasks:
|
||||
- name: Install essential packages
|
||||
apt:
|
||||
name:
|
||||
- git
|
||||
- vim
|
||||
- htop
|
||||
- tmux
|
||||
- curl
|
||||
- wget
|
||||
- build-essential
|
||||
- python3-pip
|
||||
- docker.io
|
||||
- docker-compose
|
||||
- tailscale
|
||||
state: present
|
||||
update_cache: true
|
||||
|
||||
- name: Configure Docker for non-root user
|
||||
community.docker.docker_user:
|
||||
user: "{{ vault_debian_user }}"
|
||||
groups: docker
|
||||
state: present
|
||||
|
||||
- name: Install development tools
|
||||
include_role:
|
||||
name: debian-workstation
|
||||
tasks_from: development.yml
|
||||
vars:
|
||||
go_version: "1.21"
|
||||
node_version: "18"
|
||||
|
||||
- name: Deploy dotfiles
|
||||
include_role:
|
||||
name: debian-workstation
|
||||
tasks_from: dotfiles.yml
|
||||
|
||||
- name: Configure firewall
|
||||
community.general.ufw:
|
||||
policy: deny
|
||||
logging: on
|
||||
rule:
|
||||
- from: "192.168.1.0/24"
|
||||
port: "22"
|
||||
proto: tcp
|
||||
action: allow
|
||||
- from: "192.168.1.0/24"
|
||||
port: "80,443"
|
||||
proto: tcp
|
||||
action: allow
|
||||
```
|
||||
|
||||
### **4. Secrets Management Strategy**
|
||||
|
||||
```yaml
|
||||
# Best practices for secrets:
|
||||
# 1. Separate vault files by environment
|
||||
vault/
|
||||
├── production/
|
||||
│ ├── secrets.yml
|
||||
│ ├── talos.yml
|
||||
│ └── k8s.yml
|
||||
├── staging/
|
||||
└── development/
|
||||
|
||||
# 2. Use ansible-vault encrypt_string for individual secrets
|
||||
# 3. Store vault password in CI/CD secrets, not in repo
|
||||
# 4. Use different vault passwords per environment
|
||||
```
|
||||
|
||||
### **5. Makefile for Common Operations**
|
||||
```makefile
|
||||
.PHONY: help install test deploy
|
||||
|
||||
help:
|
||||
@echo "Available commands:"
|
||||
@echo " make install Install collections and roles"
|
||||
@echo " make lint Run ansible-lint"
|
||||
@echo " make test Run molecule tests"
|
||||
@echo " make deploy-dev Deploy to development"
|
||||
@echo " make deploy-prod Deploy to production"
|
||||
|
||||
install:
|
||||
ansible-galaxy collection install -r collections/requirements.yml
|
||||
ansible-galaxy role install -r roles/requirements.yml
|
||||
|
||||
lint:
|
||||
ansible-lint playbooks/ roles/ --exclude roles/external/
|
||||
|
||||
test:
|
||||
cd roles/common && molecule test
|
||||
cd roles/talos-provision && molecule test
|
||||
|
||||
deploy-dev:
|
||||
ansible-playbook -i inventory/development site.yml --vault-password-file .vault-pass
|
||||
|
||||
deploy-prod:
|
||||
ansible-playbook -i inventory/production site.yml --vault-password-file .vault-pass --limit production
|
||||
|
||||
vault-edit:
|
||||
ansible-vault edit vault/production/secrets.yml
|
||||
|
||||
vault-view:
|
||||
ansible-vault view vault/production/secrets.yml
|
||||
```
|
||||
|
||||
### **Key Recommendations:**
|
||||
|
||||
1. **Version Control**: Keep everything in Git, including encrypted vault files
|
||||
2. **Immutable Infrastructure**: Talos nodes should be treated as cattle - replace, don't repair
|
||||
3. **Backup Strategy**: Use Velero for K8s backups, Borg/Restic for Debian
|
||||
4. **Monitoring**: Deploy Prometheus, Grafana, and Loki on Talos cluster
|
||||
5. **GitOps**: Consider ArgoCD for application deployment after initial setup
|
||||
6. **Network**: Use Tailscale/WireGuard for secure access between nodes
|
||||
|
||||
This setup gives you a complete, self-hosted infrastructure managed entirely through Ansible with proper testing and CI/CD integration.
|
||||
Reference in New Issue
Block a user