WP-30/31/32: backup targets, strategies and execution

SMB (smbclient) and FTP/FTPS (curl with netrc) targets with encrypted
credentials and connection test; strategies with cron schedule, retention,
optional openssl AES-256 encryption; sources: PVC hostpath tar, pg_dumpall
in the Postgres pod, namespace manifests, host directory. Backup job
collects, encrypts, uploads, verifies size, records sha256 and applies
retention on the target; scheduler starts due strategies. Backups page
with target/strategy forms, run now and history. Restore guide in docs.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

backend/crates/api/src/backups.rs

@@ -0,0 +1,350 @@
+//! /api/backups: targets, strategies, records, manual runs.
+use application::backup_service::StrategyStatus;
+use axum::extract::{Path, State};
+use axum::http::StatusCode;
+use axum::routing::{get, post};
+use axum::{Json, Router};
+use domain::backup::{BackupRecord, BackupSource, BackupStrategy, BackupTarget, StorageKind};
+use domain::jobs::JobKind;
+use serde::{Deserialize, Serialize};
+use utoipa::ToSchema;
+use uuid::Uuid;
+
+use crate::error::ApiError;
+use crate::extract::{AdminUser, AuthUser};
+use crate::AppState;
+
+pub fn router() -> Router<AppState> {
+    Router::new()
+        .route("/targets", get(list_targets).post(create_target))
+        .route(
+            "/targets/{id}",
+            get(get_target).put(update_target).delete(delete_target),
+        )
+        .route("/targets/{id}/test", post(test_target))
+        .route("/strategies", get(list_strategies).post(create_strategy))
+        .route(
+            "/strategies/{id}",
+            get(get_strategy)
+                .put(update_strategy)
+                .delete(delete_strategy),
+        )
+        .route("/strategies/{id}/run", post(run_strategy))
+        .route("/strategies/{id}/records", get(records))
+}
+
+// ---- targets ----
+
+#[derive(Serialize, ToSchema)]
+pub struct TargetView {
+    pub id: Uuid,
+    pub name: String,
+    #[schema(value_type = String)]
+    pub kind: StorageKind,
+    pub host: String,
+    pub port: Option<u16>,
+    pub share: String,
+    pub path: String,
+    pub username: String,
+    pub tls: bool,
+    pub password_set: bool,
+}
+
+impl From<BackupTarget> for TargetView {
+    fn from(t: BackupTarget) -> Self {
+        Self {
+            id: t.id,
+            name: t.name,
+            kind: t.kind,
+            host: t.host,
+            port: t.port,
+            share: t.share,
+            path: t.path,
+            username: t.username,
+            tls: t.tls,
+            password_set: !t.password.is_empty(),
+        }
+    }
+}
+
+#[derive(Deserialize, ToSchema)]
+pub struct TargetRequest {
+    pub name: String,
+    #[schema(value_type = String)]
+    pub kind: StorageKind,
+    pub host: String,
+    pub port: Option<u16>,
+    #[serde(default)]
+    pub share: String,
+    #[serde(default)]
+    pub path: String,
+    #[serde(default)]
+    pub username: String,
+    /// Empty keeps the stored password on update.
+    #[serde(default)]
+    pub password: String,
+    #[serde(default)]
+    pub tls: bool,
+}
+
+impl TargetRequest {
+    fn into_target(self, id: Uuid) -> BackupTarget {
+        BackupTarget {
+            id,
+            name: self.name.trim().into(),
+            kind: self.kind,
+            host: self.host.trim().into(),
+            port: self.port,
+            share: self.share.trim().into(),
+            path: self.path.trim().into(),
+            username: self.username,
+            password: self.password,
+            tls: self.tls,
+        }
+    }
+}
+
+#[utoipa::path(get, path = "/api/backups/targets", tag = "backups", security(("bearer" = [])), responses((status = 200, body = Vec<TargetView>)))]
+async fn list_targets(
+    State(s): State<AppState>,
+    _: AuthUser,
+) -> Result<Json<Vec<TargetView>>, ApiError> {
+    Ok(Json(
+        s.backups
+            .list_targets()
+            .await?
+            .into_iter()
+            .map(Into::into)
+            .collect(),
+    ))
+}
+
+#[utoipa::path(get, path = "/api/backups/targets/{id}", tag = "backups", security(("bearer" = [])), responses((status = 200, body = TargetView), (status = 404)))]
+async fn get_target(
+    State(s): State<AppState>,
+    _: AuthUser,
+    Path(id): Path<Uuid>,
+) -> Result<Json<TargetView>, ApiError> {
+    Ok(Json(s.backups.get_target(id).await?.into()))
+}
+
+#[utoipa::path(post, path = "/api/backups/targets", tag = "backups", security(("bearer" = [])), request_body = TargetRequest, responses((status = 201, body = TargetView), (status = 422)))]
+async fn create_target(
+    State(s): State<AppState>,
+    _: AdminUser,
+    Json(req): Json<TargetRequest>,
+) -> Result<(StatusCode, Json<TargetView>), ApiError> {
+    Ok((
+        StatusCode::CREATED,
+        Json(
+            s.backups
+                .create_target(req.into_target(Uuid::nil()))
+                .await?
+                .into(),
+        ),
+    ))
+}
+
+#[utoipa::path(put, path = "/api/backups/targets/{id}", tag = "backups", security(("bearer" = [])), request_body = TargetRequest, responses((status = 200, body = TargetView), (status = 404), (status = 422)))]
+async fn update_target(
+    State(s): State<AppState>,
+    _: AdminUser,
+    Path(id): Path<Uuid>,
+    Json(req): Json<TargetRequest>,
+) -> Result<Json<TargetView>, ApiError> {
+    Ok(Json(
+        s.backups.update_target(req.into_target(id)).await?.into(),
+    ))
+}
+
+#[utoipa::path(delete, path = "/api/backups/targets/{id}", tag = "backups", security(("bearer" = [])), responses((status = 204), (status = 404), (status = 409)))]
+async fn delete_target(
+    State(s): State<AppState>,
+    _: AdminUser,
+    Path(id): Path<Uuid>,
+) -> Result<StatusCode, ApiError> {
+    s.backups.delete_target(id).await?;
+    Ok(StatusCode::NO_CONTENT)
+}
+
+#[utoipa::path(post, path = "/api/backups/targets/{id}/test", tag = "backups", security(("bearer" = [])), responses((status = 204), (status = 404), (status = 502)))]
+async fn test_target(
+    State(s): State<AppState>,
+    _: AdminUser,
+    Path(id): Path<Uuid>,
+) -> Result<StatusCode, ApiError> {
+    s.backups.test_target(id).await?;
+    Ok(StatusCode::NO_CONTENT)
+}
+
+// ---- strategies ----
+
+#[derive(Serialize, ToSchema)]
+pub struct StrategyView {
+    pub id: Uuid,
+    pub name: String,
+    #[schema(value_type = Object)]
+    pub source: BackupSource,
+    pub schedule: String,
+    pub target_id: Uuid,
+    pub retention: u32,
+    pub encrypted: bool,
+    pub enabled: bool,
+}
+
+impl From<BackupStrategy> for StrategyView {
+    fn from(s: BackupStrategy) -> Self {
+        Self {
+            id: s.id,
+            name: s.name,
+            source: s.source,
+            schedule: s.schedule,
+            target_id: s.target_id,
+            retention: s.retention,
+            encrypted: s.passphrase.is_some(),
+            enabled: s.enabled,
+        }
+    }
+}
+
+#[derive(Serialize, ToSchema)]
+pub struct StrategyStatusView {
+    #[serde(flatten)]
+    pub strategy: StrategyView,
+    pub target_name: String,
+    #[schema(value_type = Option<Object>)]
+    pub last_backup: Option<BackupRecord>,
+}
+
+impl From<StrategyStatus> for StrategyStatusView {
+    fn from(s: StrategyStatus) -> Self {
+        Self {
+            strategy: s.strategy.into(),
+            target_name: s.target_name,
+            last_backup: s.last_backup,
+        }
+    }
+}
+
+#[derive(Deserialize, ToSchema)]
+pub struct StrategyRequest {
+    pub name: String,
+    #[schema(value_type = Object)]
+    pub source: BackupSource,
+    pub schedule: String,
+    pub target_id: Uuid,
+    pub retention: u32,
+    /// `null` = no encryption; empty string keeps the stored passphrase on update.
+    #[serde(default)]
+    pub passphrase: Option<String>,
+    #[serde(default = "default_true")]
+    pub enabled: bool,
+}
+
+fn default_true() -> bool {
+    true
+}
+
+impl StrategyRequest {
+    fn into_strategy(self, id: Uuid) -> BackupStrategy {
+        BackupStrategy {
+            id,
+            name: self.name.trim().into(),
+            source: self.source,
+            schedule: self.schedule.trim().into(),
+            target_id: self.target_id,
+            retention: self.retention,
+            passphrase: self.passphrase,
+            enabled: self.enabled,
+        }
+    }
+}
+
+#[utoipa::path(get, path = "/api/backups/strategies", tag = "backups", security(("bearer" = [])), responses((status = 200, body = Vec<StrategyStatusView>)))]
+async fn list_strategies(
+    State(s): State<AppState>,
+    _: AuthUser,
+) -> Result<Json<Vec<StrategyStatusView>>, ApiError> {
+    Ok(Json(
+        s.backups
+            .list_strategies()
+            .await?
+            .into_iter()
+            .map(Into::into)
+            .collect(),
+    ))
+}
+
+#[utoipa::path(get, path = "/api/backups/strategies/{id}", tag = "backups", security(("bearer" = [])), responses((status = 200, body = StrategyView), (status = 404)))]
+async fn get_strategy(
+    State(s): State<AppState>,
+    _: AuthUser,
+    Path(id): Path<Uuid>,
+) -> Result<Json<StrategyView>, ApiError> {
+    Ok(Json(s.backups.get_strategy(id).await?.into()))
+}
+
+#[utoipa::path(post, path = "/api/backups/strategies", tag = "backups", security(("bearer" = [])), request_body = StrategyRequest, responses((status = 201, body = StrategyView), (status = 404), (status = 422)))]
+async fn create_strategy(
+    State(s): State<AppState>,
+    _: AdminUser,
+    Json(req): Json<StrategyRequest>,
+) -> Result<(StatusCode, Json<StrategyView>), ApiError> {
+    Ok((
+        StatusCode::CREATED,
+        Json(
+            s.backups
+                .create_strategy(req.into_strategy(Uuid::nil()))
+                .await?
+                .into(),
+        ),
+    ))
+}
+
+#[utoipa::path(put, path = "/api/backups/strategies/{id}", tag = "backups", security(("bearer" = [])), request_body = StrategyRequest, responses((status = 200, body = StrategyView), (status = 404), (status = 422)))]
+async fn update_strategy(
+    State(s): State<AppState>,
+    _: AdminUser,
+    Path(id): Path<Uuid>,
+    Json(req): Json<StrategyRequest>,
+) -> Result<Json<StrategyView>, ApiError> {
+    Ok(Json(
+        s.backups
+            .update_strategy(req.into_strategy(id))
+            .await?
+            .into(),
+    ))
+}
+
+#[utoipa::path(delete, path = "/api/backups/strategies/{id}", tag = "backups", security(("bearer" = [])), responses((status = 204), (status = 404)))]
+async fn delete_strategy(
+    State(s): State<AppState>,
+    _: AdminUser,
+    Path(id): Path<Uuid>,
+) -> Result<StatusCode, ApiError> {
+    s.backups.delete_strategy(id).await?;
+    Ok(StatusCode::NO_CONTENT)
+}
+
+#[utoipa::path(post, path = "/api/backups/strategies/{id}/run", tag = "backups", security(("bearer" = [])), responses((status = 202, body = crate::jobs::JobRunDto), (status = 404), (status = 409)))]
+async fn run_strategy(
+    State(s): State<AppState>,
+    AdminUser(admin): AdminUser,
+    Path(id): Path<Uuid>,
+) -> Result<(StatusCode, Json<crate::jobs::JobRunDto>), ApiError> {
+    s.backups.get_strategy(id).await?;
+    let run = s
+        .jobs
+        .start(JobKind::Backup, Some(id.to_string()), &admin.email)
+        .await?;
+    Ok((StatusCode::ACCEPTED, Json(run.into())))
+}
+
+#[utoipa::path(get, path = "/api/backups/strategies/{id}/records", tag = "backups", security(("bearer" = [])), responses((status = 200, body = Vec<Object>)))]
+async fn records(
+    State(s): State<AppState>,
+    _: AuthUser,
+    Path(id): Path<Uuid>,
+) -> Result<Json<Vec<BackupRecord>>, ApiError> {
+    Ok(Json(s.backups.records(id).await?))
+}

backend/crates/api/src/config.rs

@@ -11,6 +11,10 @@ pub struct Config {
     pub fake_host: bool,
     /// Path to a kubeconfig; None infers. Defaults to the microk8s client config if present.
     pub kubeconfig: Option<String>,
+    /// kubectl invocation for backups, e.g. ["/snap/bin/microk8s", "kubectl"].
+    pub kubectl: Vec<String>,
+    /// Scratch directory for backup archives.
+    pub work_dir: std::path::PathBuf,
     pub bind: SocketAddr,
     pub bootstrap_admin: Option<(String, String)>,
     pub cookie_secure: bool,
@@ -34,6 +38,18 @@ impl Config {
             jwt_secret,
             master_key,
             fake_host: env("FAKE_HOST").is_some_and(|v| v == "true" || v == "1"),
+            kubectl: env("KUBECTL")
+                .map(|v| v.split_whitespace().map(String::from).collect())
+                .unwrap_or_else(|| {
+                    if std::path::Path::new("/snap/bin/microk8s").exists() {
+                        vec!["/snap/bin/microk8s".into(), "kubectl".into()]
+                    } else {
+                        vec!["kubectl".into()]
+                    }
+                }),
+            work_dir: env("WORK_DIR")
+                .map(Into::into)
+                .unwrap_or_else(|| "data/work".into()),
             kubeconfig: env("KUBECONFIG").or_else(|| {
                 let microk8s = "/var/snap/microk8s/current/credentials/client.config";
                 std::path::Path::new(microk8s)

backend/crates/api/src/lib.rs

@@ -1,5 +1,6 @@
 //! HTTP API layer (axum). `build_app` is used by both the binary and the integration tests.
 pub mod auth;
+pub mod backups;
 pub mod cluster;
 pub mod config;
 pub mod error;
@@ -17,18 +18,24 @@ use std::sync::Arc;
 
 use application::scheduler::Scheduler;
 use application::{
-    AuthService, ClusterService, InventoryService, JobRunner, PackageRefreshJob, PackageUpgradeJob,
-    SettingsService, UserService, VulnerabilityScanJob, VulnerabilityService,
+    AuthService, BackupDeps, BackupJob, BackupService, ClusterService, InventoryService, JobRunner,
+    PackageRefreshJob, PackageUpgradeJob, SettingsService, UserService, VulnerabilityScanJob,
+    VulnerabilityService,
 };
 use axum::{routing::get, Json, Router};
 use domain::jobs::JobKind;
 use domain::ports::Mailer;
-use domain::ports::{ClusterGateway, HostInspector, HostUpdater, VulnerabilityScanner};
+use domain::ports::{
+    BackupCollector, BackupStorage, ClusterGateway, FileEncryptor, HostInspector, HostUpdater,
+    VulnerabilityScanner,
+};
 use infrastructure::{
-    AesGcmCipher, Argon2Hasher, DbPool, DebianInspector, DebianUpdater, FakeClusterGateway,
-    FakeHostInspector, FakeHostUpdater, FakeScanner, JwtIssuer, KubeGateway, LettreMailer,
-    SqliteAuditLog, SqliteFindings, SqliteInventory, SqliteJobRuns, SqliteRefreshTokens,
-    SqliteSettings, SqliteUsers, SystemCommandRunner, TrivyScanner,
+    AesGcmCipher, Argon2Hasher, CommandBackupStorage, DbPool, DebianInspector, DebianUpdater,
+    DirBackupStorage, FakeBackupCollector, FakeClusterGateway, FakeHostInspector, FakeHostUpdater,
+    FakeScanner, JwtIssuer, KubeBackupCollector, KubeGateway, LettreMailer, OpensslEncryptor,
+    SqliteAuditLog, SqliteBackupRecords, SqliteBackupStrategies, SqliteBackupTargets,
+    SqliteFindings, SqliteInventory, SqliteJobRuns, SqliteRefreshTokens, SqliteSettings,
+    SqliteUsers, SystemCommandRunner, TrivyScanner,
 };
 use tower_http::services::{ServeDir, ServeFile};
 use tower_http::trace::TraceLayer;
@@ -42,6 +49,9 @@ pub struct Adapters {
     pub updater: Arc<dyn HostUpdater>,
     pub cluster: Arc<dyn ClusterGateway>,
     pub scanner: Arc<dyn VulnerabilityScanner>,
+    pub storage: Arc<dyn BackupStorage>,
+    pub collector: Arc<dyn BackupCollector>,
+    pub encryptor: Arc<dyn FileEncryptor>,
 }
 
 #[derive(Clone)]
@@ -54,6 +64,7 @@ pub struct AppState {
     pub inventory: Arc<InventoryService>,
     pub cluster: Arc<ClusterService>,
     pub vulns: Arc<VulnerabilityService>,
+    pub backups: Arc<BackupService>,
     pub login_limiter: Arc<rate_limit::RateLimiter>,
 }
 
@@ -68,6 +79,9 @@ impl AppState {
                 updater: Arc::new(FakeHostUpdater),
                 cluster: Arc::new(FakeClusterGateway),
                 scanner: Arc::new(FakeScanner),
+                storage: Arc::new(DirBackupStorage::new(cfg.work_dir.join("fake-remote"))),
+                collector: Arc::new(FakeBackupCollector),
+                encryptor: Arc::new(OpensslEncryptor::new(runner.clone())),
             }
         } else {
             Adapters {
@@ -75,7 +89,13 @@ impl AppState {
                 inspector: Arc::new(DebianInspector::new(runner.clone())),
                 updater: Arc::new(DebianUpdater::new(runner.clone())),
                 cluster: Arc::new(KubeGateway::new(cfg.kubeconfig.clone())),
-                scanner: Arc::new(TrivyScanner::new(runner)),
+                scanner: Arc::new(TrivyScanner::new(runner.clone())),
+                storage: Arc::new(CommandBackupStorage::new(runner.clone())),
+                collector: Arc::new(KubeBackupCollector::new(
+                    runner.clone(),
+                    cfg.kubectl.clone(),
+                )),
+                encryptor: Arc::new(OpensslEncryptor::new(runner)),
             }
         };
         Self::with_adapters(cfg, pool, adapters, |r| r)
@@ -94,6 +114,9 @@ impl AppState {
             updater,
             cluster,
             scanner,
+            storage,
+            collector,
+            encryptor,
         } = adapters;
         let pool_for_findings = pool.clone();
         let cluster_gateway = cluster.clone();
@@ -107,6 +130,16 @@ impl AppState {
             Arc::new(JwtIssuer::new(&cfg.jwt_secret)),
         );
         let cipher = Arc::new(AesGcmCipher::from_hex(&cfg.master_key)?);
+        let backups = Arc::new(BackupService::new(BackupDeps {
+            targets: Arc::new(SqliteBackupTargets(pool.clone())),
+            strategies: Arc::new(SqliteBackupStrategies(pool.clone())),
+            records: Arc::new(SqliteBackupRecords(pool.clone())),
+            storage,
+            collector,
+            encryptor,
+            cipher: cipher.clone(),
+            work_dir: cfg.work_dir.clone(),
+        }));
         let settings = Arc::new(SettingsService::new(
             Arc::new(SqliteSettings(pool.clone())),
             cipher,
@@ -135,10 +168,14 @@ impl AppState {
             cluster_gateway,
             settings.clone(),
         ));
-        let jobs = Arc::new(register_jobs(runner.register(
-            JobKind::VulnerabilityScan,
-            Arc::new(VulnerabilityScanJob(vulns.clone())),
-        )));
+        let jobs = Arc::new(register_jobs(
+            runner
+                .register(
+                    JobKind::VulnerabilityScan,
+                    Arc::new(VulnerabilityScanJob(vulns.clone())),
+                )
+                .register(JobKind::Backup, Arc::new(BackupJob(backups.clone()))),
+        ));
         Ok(Self {
             cfg,
             auth: Arc::new(auth),
@@ -148,6 +185,7 @@ impl AppState {
             inventory,
             cluster,
             vulns,
+            backups,
             login_limiter: Arc::new(rate_limit::RateLimiter::new(
                 10,
                 std::time::Duration::from_secs(60),
@@ -157,7 +195,11 @@ impl AppState {
 
     /// Spawn the cron scheduler on the current runtime.
     pub fn start_scheduler(&self) {
-        tokio::spawn(Scheduler::new(self.jobs.clone(), self.settings.clone()).run());
+        tokio::spawn(
+            Scheduler::new(self.jobs.clone(), self.settings.clone())
+                .with_backups(self.backups.clone())
+                .run(),
+        );
     }
 
     pub async fn bootstrap(&self) -> anyhow::Result<()> {
@@ -187,6 +229,7 @@ pub fn build_app(state: AppState) -> Router {
         .nest("/api/system", system::router())
         .nest("/api/cluster", cluster::router())
         .nest("/api/vulnerabilities", vulnerabilities::router())
+        .nest("/api/backups", backups::router())
         .fallback_service(spa)
         .layer(TraceLayer::new_for_http())
         .with_state(state)

backend/crates/api/src/main.rs

@@ -16,6 +16,7 @@ async fn main() -> anyhow::Result<()> {
     {
         std::fs::create_dir_all(dir)?;
     }
+    std::fs::create_dir_all(&cfg.work_dir)?;
     let pool = infrastructure::connect(&cfg.database_url).await?;
     let state = AppState::new(cfg.clone(), pool)?;
     state.bootstrap().await?;

backend/crates/api/src/openapi.rs

@@ -30,6 +30,10 @@ impl Modify for BearerAuth {
         crate::cluster::overview, crate::cluster::restart, crate::cluster::scale, crate::cluster::set_image,
         crate::vulnerabilities::list, crate::vulnerabilities::summary, crate::vulnerabilities::targets, crate::vulnerabilities::set_status,
         crate::settings::get_notifications, crate::settings::put_notifications,
+        crate::backups::list_targets, crate::backups::get_target, crate::backups::create_target, crate::backups::update_target,
+        crate::backups::delete_target, crate::backups::test_target, crate::backups::list_strategies, crate::backups::get_strategy,
+        crate::backups::create_strategy, crate::backups::update_strategy, crate::backups::delete_strategy,
+        crate::backups::run_strategy, crate::backups::records,
     ),
     modifiers(&BearerAuth)
 )]

backend/crates/api/src/test_support.rs

@@ -17,6 +17,8 @@ pub fn test_config() -> Config {
         master_key: "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f".into(),
         fake_host: true,
         kubeconfig: None,
+        kubectl: vec!["kubectl".into()],
+        work_dir: std::env::temp_dir().join(format!("monitoring-test-{}", uuid::Uuid::new_v4())),
         bind: "127.0.0.1:0".parse().unwrap(),
         bootstrap_admin: None,
         cookie_secure: false,
@@ -86,6 +88,13 @@ async fn build_test_app_with(cfg: Config) -> Router {
         updater: Arc::new(infrastructure::FakeHostUpdater),
         cluster: Arc::new(infrastructure::FakeClusterGateway),
         scanner: Arc::new(infrastructure::FakeScanner),
+        storage: Arc::new(infrastructure::DirBackupStorage::new(
+            cfg.work_dir.join("remote"),
+        )),
+        collector: Arc::new(infrastructure::FakeBackupCollector),
+        encryptor: Arc::new(infrastructure::OpensslEncryptor::new(Arc::new(
+            infrastructure::SystemCommandRunner,
+        ))),
     };
     let state = AppState::with_adapters(cfg, pool, adapters, register_test_jobs).expect("state");
     state.bootstrap().await.expect("bootstrap");