403 lines
9.3 KiB
HCL
403 lines
9.3 KiB
HCL
# Main application deployment with LiteFS for distributed SQLite
|
|
#
|
|
# Architecture:
|
|
# - StatefulSet for stable pod identities (abaci-app-0, abaci-app-1, etc.)
|
|
# - Pod-0 is always the primary (handles writes)
|
|
# - Other pods are replicas (receive replicated data, forward writes to primary)
|
|
# - Headless service for pod-to-pod DNS resolution
|
|
# - LiteFS handles SQLite replication transparently
|
|
|
|
resource "kubernetes_secret" "app_env" {
|
|
metadata {
|
|
name = "app-env"
|
|
namespace = kubernetes_namespace.abaci.metadata[0].name
|
|
}
|
|
|
|
data = {
|
|
AUTH_SECRET = var.auth_secret
|
|
}
|
|
}
|
|
|
|
resource "kubernetes_config_map" "app_config" {
|
|
metadata {
|
|
name = "app-config"
|
|
namespace = kubernetes_namespace.abaci.metadata[0].name
|
|
}
|
|
|
|
data = {
|
|
NODE_ENV = "production"
|
|
PORT = "3000"
|
|
# Note: Don't set HOSTNAME here - it conflicts with LiteFS which needs the pod hostname
|
|
# Next.js will use 0.0.0.0 by default if HOSTNAME is not set
|
|
NEXT_TELEMETRY_DISABLED = "1"
|
|
REDIS_URL = "redis://redis:6379"
|
|
# LiteFS mounts the database at /litefs
|
|
DATABASE_URL = "/litefs/sqlite.db"
|
|
}
|
|
}
|
|
|
|
# Headless service for StatefulSet pod-to-pod communication
|
|
resource "kubernetes_service" "app_headless" {
|
|
metadata {
|
|
name = "abaci-app-headless"
|
|
namespace = kubernetes_namespace.abaci.metadata[0].name
|
|
}
|
|
|
|
spec {
|
|
selector = {
|
|
app = "abaci-app"
|
|
}
|
|
|
|
# Headless service - no cluster IP
|
|
cluster_ip = "None"
|
|
|
|
port {
|
|
name = "litefs"
|
|
port = 20202
|
|
target_port = 20202
|
|
}
|
|
|
|
port {
|
|
name = "proxy"
|
|
port = 8080
|
|
target_port = 8080
|
|
}
|
|
}
|
|
}
|
|
|
|
# StatefulSet for stable pod identities (required for LiteFS primary election)
|
|
resource "kubernetes_stateful_set" "app" {
|
|
metadata {
|
|
name = "abaci-app"
|
|
namespace = kubernetes_namespace.abaci.metadata[0].name
|
|
labels = {
|
|
app = "abaci-app"
|
|
}
|
|
}
|
|
|
|
spec {
|
|
service_name = kubernetes_service.app_headless.metadata[0].name
|
|
replicas = var.app_replicas
|
|
|
|
selector {
|
|
match_labels = {
|
|
app = "abaci-app"
|
|
}
|
|
}
|
|
|
|
# Parallel pod management for faster scaling
|
|
pod_management_policy = "Parallel"
|
|
|
|
# Rolling update strategy
|
|
update_strategy {
|
|
type = "RollingUpdate"
|
|
}
|
|
|
|
template {
|
|
metadata {
|
|
labels = {
|
|
app = "abaci-app"
|
|
}
|
|
}
|
|
|
|
spec {
|
|
# LiteFS requires root for FUSE mount
|
|
# The app itself runs as non-root via litefs exec
|
|
security_context {
|
|
fs_group = 1001
|
|
}
|
|
|
|
# Init container to determine if this pod is the primary candidate
|
|
init_container {
|
|
name = "init-litefs-candidate"
|
|
image = "busybox:1.36"
|
|
|
|
command = ["/bin/sh", "-c"]
|
|
args = [<<-EOT
|
|
# Extract pod ordinal from hostname (e.g., abaci-app-0 -> 0)
|
|
ORDINAL=$(echo $HOSTNAME | rev | cut -d'-' -f1 | rev)
|
|
# Pod-0 is the primary candidate
|
|
if [ "$ORDINAL" = "0" ]; then
|
|
echo "true" > /config/litefs-candidate
|
|
else
|
|
echo "false" > /config/litefs-candidate
|
|
fi
|
|
echo "Pod $HOSTNAME: LITEFS_CANDIDATE=$(cat /config/litefs-candidate)"
|
|
EOT
|
|
]
|
|
|
|
volume_mount {
|
|
name = "config"
|
|
mount_path = "/config"
|
|
}
|
|
}
|
|
|
|
container {
|
|
name = "app"
|
|
image = var.app_image
|
|
|
|
# Override to use LiteFS
|
|
command = ["/bin/sh", "-c"]
|
|
args = [<<-EOT
|
|
export LITEFS_CANDIDATE=$(cat /config/litefs-candidate)
|
|
exec litefs mount
|
|
EOT
|
|
]
|
|
|
|
# Run as root for FUSE mount (app runs as non-root via litefs exec)
|
|
security_context {
|
|
run_as_user = 0
|
|
run_as_group = 0
|
|
privileged = true # Required for FUSE
|
|
}
|
|
|
|
port {
|
|
name = "proxy"
|
|
container_port = 8080
|
|
}
|
|
|
|
port {
|
|
name = "app"
|
|
container_port = 3000
|
|
}
|
|
|
|
port {
|
|
name = "litefs"
|
|
container_port = 20202
|
|
}
|
|
|
|
env_from {
|
|
config_map_ref {
|
|
name = kubernetes_config_map.app_config.metadata[0].name
|
|
}
|
|
}
|
|
|
|
env_from {
|
|
secret_ref {
|
|
name = kubernetes_secret.app_env.metadata[0].name
|
|
}
|
|
}
|
|
|
|
resources {
|
|
requests = {
|
|
memory = "256Mi"
|
|
cpu = "100m"
|
|
}
|
|
limits = {
|
|
memory = "512Mi"
|
|
cpu = "1000m"
|
|
}
|
|
}
|
|
|
|
# Health checks hit the LiteFS proxy
|
|
liveness_probe {
|
|
http_get {
|
|
path = "/api/health"
|
|
port = 8080
|
|
}
|
|
initial_delay_seconds = 30
|
|
period_seconds = 10
|
|
timeout_seconds = 5
|
|
failure_threshold = 3
|
|
}
|
|
|
|
readiness_probe {
|
|
http_get {
|
|
path = "/api/health"
|
|
port = 8080
|
|
}
|
|
initial_delay_seconds = 5
|
|
period_seconds = 5
|
|
timeout_seconds = 3
|
|
failure_threshold = 3
|
|
}
|
|
|
|
volume_mount {
|
|
name = "litefs-data"
|
|
mount_path = "/var/lib/litefs"
|
|
}
|
|
|
|
volume_mount {
|
|
name = "litefs-fuse"
|
|
mount_path = "/litefs"
|
|
# mount_propagation is needed for FUSE
|
|
}
|
|
|
|
volume_mount {
|
|
name = "config"
|
|
mount_path = "/config"
|
|
}
|
|
}
|
|
|
|
volume {
|
|
name = "litefs-fuse"
|
|
empty_dir {}
|
|
}
|
|
|
|
volume {
|
|
name = "config"
|
|
empty_dir {}
|
|
}
|
|
}
|
|
}
|
|
|
|
# Persistent volume for LiteFS data (transaction files)
|
|
volume_claim_template {
|
|
metadata {
|
|
name = "litefs-data"
|
|
}
|
|
|
|
spec {
|
|
access_modes = ["ReadWriteOnce"]
|
|
storage_class_name = "local-path"
|
|
|
|
resources {
|
|
requests = {
|
|
storage = "5Gi"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
depends_on = [kubernetes_deployment.redis]
|
|
}
|
|
|
|
# Main service for external access (load balances across all pods)
|
|
resource "kubernetes_service" "app" {
|
|
metadata {
|
|
name = "abaci-app"
|
|
namespace = kubernetes_namespace.abaci.metadata[0].name
|
|
}
|
|
|
|
spec {
|
|
selector = {
|
|
app = "abaci-app"
|
|
}
|
|
|
|
port {
|
|
port = 80
|
|
target_port = 8080 # LiteFS proxy port
|
|
}
|
|
|
|
type = "ClusterIP"
|
|
}
|
|
}
|
|
|
|
# Ingress with SSL via cert-manager
|
|
resource "kubernetes_ingress_v1" "app" {
|
|
metadata {
|
|
name = "abaci-app"
|
|
namespace = kubernetes_namespace.abaci.metadata[0].name
|
|
annotations = {
|
|
"cert-manager.io/cluster-issuer" = var.use_staging_certs ? "letsencrypt-staging" : "letsencrypt-prod"
|
|
"traefik.ingress.kubernetes.io/router.entrypoints" = "websecure"
|
|
"traefik.ingress.kubernetes.io/router.middlewares" = "${kubernetes_namespace.abaci.metadata[0].name}-hsts@kubernetescrd"
|
|
}
|
|
}
|
|
|
|
spec {
|
|
ingress_class_name = "traefik"
|
|
|
|
tls {
|
|
hosts = [var.app_domain]
|
|
secret_name = "abaci-tls"
|
|
}
|
|
|
|
rule {
|
|
host = var.app_domain
|
|
|
|
http {
|
|
path {
|
|
path = "/"
|
|
path_type = "Prefix"
|
|
|
|
backend {
|
|
service {
|
|
name = kubernetes_service.app.metadata[0].name
|
|
port {
|
|
number = 80
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
depends_on = [null_resource.cert_manager_issuers]
|
|
}
|
|
|
|
# HSTS middleware
|
|
resource "kubernetes_manifest" "hsts_middleware" {
|
|
manifest = {
|
|
apiVersion = "traefik.io/v1alpha1"
|
|
kind = "Middleware"
|
|
metadata = {
|
|
name = "hsts"
|
|
namespace = kubernetes_namespace.abaci.metadata[0].name
|
|
}
|
|
spec = {
|
|
headers = {
|
|
stsSeconds = 63072000
|
|
stsIncludeSubdomains = true
|
|
stsPreload = true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
# HTTP to HTTPS redirect
|
|
resource "kubernetes_ingress_v1" "app_http_redirect" {
|
|
metadata {
|
|
name = "abaci-app-http-redirect"
|
|
namespace = kubernetes_namespace.abaci.metadata[0].name
|
|
annotations = {
|
|
"traefik.ingress.kubernetes.io/router.entrypoints" = "web"
|
|
"traefik.ingress.kubernetes.io/router.middlewares" = "${kubernetes_namespace.abaci.metadata[0].name}-redirect-https@kubernetescrd"
|
|
}
|
|
}
|
|
|
|
spec {
|
|
ingress_class_name = "traefik"
|
|
|
|
rule {
|
|
host = var.app_domain
|
|
|
|
http {
|
|
path {
|
|
path = "/"
|
|
path_type = "Prefix"
|
|
|
|
backend {
|
|
service {
|
|
name = kubernetes_service.app.metadata[0].name
|
|
port {
|
|
number = 80
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
# Redirect middleware
|
|
resource "kubernetes_manifest" "redirect_https_middleware" {
|
|
manifest = {
|
|
apiVersion = "traefik.io/v1alpha1"
|
|
kind = "Middleware"
|
|
metadata = {
|
|
name = "redirect-https"
|
|
namespace = kubernetes_namespace.abaci.metadata[0].name
|
|
}
|
|
spec = {
|
|
redirectScheme = {
|
|
scheme = "https"
|
|
permanent = true
|
|
}
|
|
}
|
|
}
|
|
}
|