From 8999a868ef046228ddde4dd4f8977adc441b5afb Mon Sep 17 00:00:00 2001 From: admin Date: Sun, 4 Jan 2026 11:58:01 +0100 Subject: [PATCH] feat: add Loki + Promtail logging stack --- apps/loki/application.yaml | 22 +++++++ apps/loki/configmap.yaml | 82 ++++++++++++++++++++++++ apps/loki/namespace.yaml | 7 ++ apps/loki/promtail-configmap.yaml | 72 +++++++++++++++++++++ apps/loki/promtail-daemonset.yaml | 103 ++++++++++++++++++++++++++++++ apps/loki/promtail-ingress.yaml | 26 ++++++++ apps/loki/promtail-rbac.yaml | 38 +++++++++++ apps/loki/promtail-service.yaml | 17 +++++ apps/loki/pvc.yaml | 14 ++++ apps/loki/service.yaml | 38 +++++++++++ apps/loki/servicemonitor.yaml | 41 ++++++++++++ apps/loki/statefulset.yaml | 85 ++++++++++++++++++++++++ 12 files changed, 545 insertions(+) create mode 100644 apps/loki/application.yaml create mode 100644 apps/loki/configmap.yaml create mode 100644 apps/loki/namespace.yaml create mode 100644 apps/loki/promtail-configmap.yaml create mode 100644 apps/loki/promtail-daemonset.yaml create mode 100644 apps/loki/promtail-ingress.yaml create mode 100644 apps/loki/promtail-rbac.yaml create mode 100644 apps/loki/promtail-service.yaml create mode 100644 apps/loki/pvc.yaml create mode 100644 apps/loki/service.yaml create mode 100644 apps/loki/servicemonitor.yaml create mode 100644 apps/loki/statefulset.yaml diff --git a/apps/loki/application.yaml b/apps/loki/application.yaml new file mode 100644 index 0000000..7e776a7 --- /dev/null +++ b/apps/loki/application.yaml @@ -0,0 +1,22 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: loki + namespace: argocd + labels: + argocd.argoproj.io/instance: loki +spec: + project: default + source: + repoURL: http://gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops + targetRevision: HEAD + path: apps/loki + destination: + server: https://kubernetes.default.svc + namespace: loki + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true diff --git a/apps/loki/configmap.yaml b/apps/loki/configmap.yaml new file mode 100644 index 0000000..6000a14 --- /dev/null +++ b/apps/loki/configmap.yaml @@ -0,0 +1,82 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: loki-config + namespace: loki + labels: + app.kubernetes.io/name: loki +data: + loki.yaml: | + auth_enabled: false + + server: + http_listen_port: 3100 + grpc_listen_port: 9096 + log_level: info + + common: + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + instance_addr: 127.0.0.1 + kvstore: + store: inmemory + + query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + + schema_config: + configs: + - from: 2020-10-24 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + + ruler: + alertmanager_url: http://k8s-monitoring-kube-promet-alertmanager.monitoring:9093 + + ingester: + wal: + enabled: true + dir: /loki/wal + lifecycler: + ring: + replication_factor: 1 + chunk_idle_period: 1h + max_chunk_age: 1h + chunk_target_size: 1048576 + chunk_retain_period: 30s + + limits_config: + retention_period: 168h + enforce_metric_name: false + reject_old_samples: true + reject_old_samples_max_age: 168h + max_cache_freshness_per_query: 10m + split_queries_by_interval: 15m + ingestion_rate_mb: 10 + ingestion_burst_size_mb: 20 + per_stream_rate_limit: 5MB + per_stream_rate_limit_burst: 15MB + + compactor: + working_directory: /loki/compactor + compaction_interval: 10m + retention_enabled: true + retention_delete_delay: 2h + retention_delete_worker_count: 150 + delete_request_store: filesystem + + analytics: + reporting_enabled: false diff --git a/apps/loki/namespace.yaml b/apps/loki/namespace.yaml new file mode 100644 index 0000000..6d40148 --- /dev/null +++ b/apps/loki/namespace.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: loki + labels: + app.kubernetes.io/name: loki + app.kubernetes.io/part-of: logging diff --git a/apps/loki/promtail-configmap.yaml b/apps/loki/promtail-configmap.yaml new file mode 100644 index 0000000..6039c00 --- /dev/null +++ b/apps/loki/promtail-configmap.yaml @@ -0,0 +1,72 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: promtail-config + namespace: loki + labels: + app.kubernetes.io/name: promtail +data: + promtail.yaml: | + server: + http_listen_port: 3101 + grpc_listen_port: 0 + log_level: info + + positions: + filename: /run/promtail/positions.yaml + + clients: + - url: http://loki:3100/loki/api/v1/push + tenant_id: "" + batchwait: 1s + batchsize: 1048576 + timeout: 10s + + scrape_configs: + - job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + pipeline_stages: + - cri: {} + - labeldrop: + - filename + - stream + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_promtail_io_scrape] + action: drop + regex: "false" + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: pod + - source_labels: [__meta_kubernetes_pod_container_name] + action: replace + target_label: container + - source_labels: [__meta_kubernetes_pod_node_name] + action: replace + target_label: node + - source_labels: [__meta_kubernetes_pod_label_app] + action: replace + target_label: app + - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name] + action: replace + target_label: app + - replacement: /var/log/pods/*$1/*.log + separator: / + source_labels: + - __meta_kubernetes_pod_uid + - __meta_kubernetes_pod_container_name + target_label: __path__ + + - job_name: journal + journal: + max_age: 12h + labels: + job: systemd-journal + relabel_configs: + - source_labels: ["__journal__systemd_unit"] + target_label: unit + - source_labels: ["__journal__hostname"] + target_label: node diff --git a/apps/loki/promtail-daemonset.yaml b/apps/loki/promtail-daemonset.yaml new file mode 100644 index 0000000..122c0fa --- /dev/null +++ b/apps/loki/promtail-daemonset.yaml @@ -0,0 +1,103 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: promtail + namespace: loki + labels: + app.kubernetes.io/name: promtail +spec: + selector: + matchLabels: + app.kubernetes.io/name: promtail + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + app.kubernetes.io/name: promtail + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "3101" + prometheus.io/path: "/metrics" + spec: + serviceAccountName: promtail + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + containers: + - name: promtail + image: grafana/promtail:3.3.2 + imagePullPolicy: IfNotPresent + args: + - -config.file=/etc/promtail/promtail.yaml + ports: + - name: http-metrics + containerPort: 3101 + protocol: TCP + env: + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 5 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + volumeMounts: + - name: config + mountPath: /etc/promtail + - name: run + mountPath: /run/promtail + - name: containers + mountPath: /var/lib/docker/containers + readOnly: true + - name: pods + mountPath: /var/log/pods + readOnly: true + - name: journal + mountPath: /var/log/journal + readOnly: true + - name: machine-id + mountPath: /etc/machine-id + readOnly: true + volumes: + - name: config + configMap: + name: promtail-config + - name: run + emptyDir: {} + - name: containers + hostPath: + path: /var/lib/docker/containers + - name: pods + hostPath: + path: /var/log/pods + - name: journal + hostPath: + path: /var/log/journal + - name: machine-id + hostPath: + path: /etc/machine-id diff --git a/apps/loki/promtail-ingress.yaml b/apps/loki/promtail-ingress.yaml new file mode 100644 index 0000000..d8e9a11 --- /dev/null +++ b/apps/loki/promtail-ingress.yaml @@ -0,0 +1,26 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: promtail + namespace: loki + labels: + app.kubernetes.io/name: promtail + annotations: + kubernetes.io/ingress.class: traefik + cert-manager.io/cluster-issuer: letsencrypt-http +spec: + rules: + - host: promtail.thedevops.dev + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: promtail + port: + number: 3101 + tls: + - hosts: + - promtail.thedevops.dev + secretName: promtail-tls diff --git a/apps/loki/promtail-rbac.yaml b/apps/loki/promtail-rbac.yaml new file mode 100644 index 0000000..d24a255 --- /dev/null +++ b/apps/loki/promtail-rbac.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: promtail + namespace: loki + labels: + app.kubernetes.io/name: promtail +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: promtail + labels: + app.kubernetes.io/name: promtail +rules: + - apiGroups: [""] + resources: + - nodes + - nodes/proxy + - services + - endpoints + - pods + verbs: ["get", "watch", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: promtail + labels: + app.kubernetes.io/name: promtail +subjects: + - kind: ServiceAccount + name: promtail + namespace: loki +roleRef: + kind: ClusterRole + name: promtail + apiGroup: rbac.authorization.k8s.io diff --git a/apps/loki/promtail-service.yaml b/apps/loki/promtail-service.yaml new file mode 100644 index 0000000..646a3a6 --- /dev/null +++ b/apps/loki/promtail-service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: promtail + namespace: loki + labels: + app.kubernetes.io/name: promtail +spec: + type: ClusterIP + clusterIP: None + ports: + - name: http-metrics + port: 3101 + targetPort: http-metrics + protocol: TCP + selector: + app.kubernetes.io/name: promtail diff --git a/apps/loki/pvc.yaml b/apps/loki/pvc.yaml new file mode 100644 index 0000000..cfd3d46 --- /dev/null +++ b/apps/loki/pvc.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: loki-data + namespace: loki + labels: + app.kubernetes.io/name: loki +spec: + accessModes: + - ReadWriteOnce + storageClassName: longhorn + resources: + requests: + storage: 10Gi diff --git a/apps/loki/service.yaml b/apps/loki/service.yaml new file mode 100644 index 0000000..6898672 --- /dev/null +++ b/apps/loki/service.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: Service +metadata: + name: loki + namespace: loki + labels: + app.kubernetes.io/name: loki +spec: + type: ClusterIP + ports: + - name: http + port: 3100 + targetPort: http + protocol: TCP + - name: grpc + port: 9096 + targetPort: grpc + protocol: TCP + selector: + app.kubernetes.io/name: loki +--- +apiVersion: v1 +kind: Service +metadata: + name: loki-headless + namespace: loki + labels: + app.kubernetes.io/name: loki +spec: + type: ClusterIP + clusterIP: None + ports: + - name: http + port: 3100 + targetPort: http + protocol: TCP + selector: + app.kubernetes.io/name: loki diff --git a/apps/loki/servicemonitor.yaml b/apps/loki/servicemonitor.yaml new file mode 100644 index 0000000..d0cd08e --- /dev/null +++ b/apps/loki/servicemonitor.yaml @@ -0,0 +1,41 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: loki + namespace: loki + labels: + app.kubernetes.io/name: loki + release: k8s-monitoring +spec: + selector: + matchLabels: + app.kubernetes.io/name: loki + namespaceSelector: + matchNames: + - loki + endpoints: + - port: http + interval: 30s + scrapeTimeout: 10s + path: /metrics +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: promtail + namespace: loki + labels: + app.kubernetes.io/name: promtail + release: k8s-monitoring +spec: + selector: + matchLabels: + app.kubernetes.io/name: promtail + namespaceSelector: + matchNames: + - loki + endpoints: + - port: http-metrics + interval: 30s + scrapeTimeout: 10s + path: /metrics diff --git a/apps/loki/statefulset.yaml b/apps/loki/statefulset.yaml new file mode 100644 index 0000000..0fdfad4 --- /dev/null +++ b/apps/loki/statefulset.yaml @@ -0,0 +1,85 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: loki + namespace: loki + labels: + app.kubernetes.io/name: loki +spec: + replicas: 1 + serviceName: loki-headless + selector: + matchLabels: + app.kubernetes.io/name: loki + template: + metadata: + labels: + app.kubernetes.io/name: loki + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "3100" + prometheus.io/path: "/metrics" + spec: + securityContext: + fsGroup: 10001 + runAsGroup: 10001 + runAsNonRoot: true + runAsUser: 10001 + containers: + - name: loki + image: grafana/loki:3.3.2 + imagePullPolicy: IfNotPresent + args: + - -config.file=/etc/loki/loki.yaml + ports: + - name: http + containerPort: 3100 + protocol: TCP + - name: grpc + containerPort: 9096 + protocol: TCP + livenessProbe: + httpGet: + path: /ready + port: http + initialDelaySeconds: 45 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /ready + port: http + initialDelaySeconds: 45 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + volumeMounts: + - name: config + mountPath: /etc/loki + - name: data + mountPath: /loki + - name: tmp + mountPath: /tmp + volumes: + - name: config + configMap: + name: loki-config + - name: data + persistentVolumeClaim: + claimName: loki-data + - name: tmp + emptyDir: {}