fix(file)!: keep image holes, map the input bmap ranges, rename bmap options

Cargo.toml

@@ -7,7 +7,7 @@ license = "MIT OR Apache-2.0"
 name = "omnect-cli"
 readme = "README.md"
 repository = "https://github.com/omnect/omnect-cli"
-version = "0.28.5"
+version = "0.29.0"
 
 [dependencies]
 actix-web = { version = "4.17", default-features = false, features = ["macros"] }
@@ -31,7 +31,6 @@ hmac = { version = "0.13", default-features = false }
 mbrman = { version = "0.6", default-features = false }
 omnect-crypto = { git = "https://github.com/omnect/omnect-crypto.git", tag = "0.5.0" }
 keyring-core = { version = "1.0", default-features = false }
-libfs = { version = "0.9", default-features = false }
 log = { version = "0.4", default-features = false }
 num_cpus = { version = "1.17", default-features = false }
 oauth2 = { version = "5.0", default-features = false, features = [
@@ -44,6 +43,8 @@ reqwest = { version = "0.13", default-features = false, features = [
     "json",
     "native-tls",
 ] }
+roxmltree = { version = "0.21", default-features = false, features = ["std"] }
+rustix = { version = "1.1", default-features = false, features = ["fs", "std"] }
 serde = { version = "1.0", default-features = false, features = ["derive"] }
 serde_json = { version = "1.0", default-features = false }
 serde_path_to_error = { version = "0.1", default-features = false }
@@ -81,5 +82,5 @@ tar = "0.4"
 
 # metadata for building with cargo-deb (https://crates.io/crates/cargo-deb)
 [package.metadata.deb]
-depends = "bmap-tools, e2tools, keychain, libc6 (>= 2.39), libmagic1, libssl3 (>= 3.0.0), mtools"
+depends = "e2tools, keychain, libc6 (>= 2.39), libmagic1, libssl3 (>= 3.0.0), mtools"
 revision = ""

Dockerfile

@@ -13,7 +13,6 @@ COPY ${debian_dir}/omnect-cli_${omnect_cli_version}_amd64.deb omnect-cli_${omnec
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
-    bmap-tools \
     ca-certificates \
     e2tools \
     fdisk \
@@ -34,14 +33,11 @@ RUN <<EOT
     mkdir -p /copy/status.d /copy/usr/lib
 
     executables=(
-        /usr/bin/dd \
         /usr/bin/e2cp \
         /usr/bin/e2mkdir \
-        /usr/bin/fallocate \
         /usr/bin/mcopy \
         /usr/bin/omnect-cli \
         /usr/bin/ssh-keygen \
-        /usr/bin/sync \
         /usr/sbin/fdisk \
     )
 

README.md

@@ -48,9 +48,8 @@ docker run --rm -it \
   omnect/omnect-cli:latest file copy-to-image --files /source/my-source-file,boot:/my-dest-file -i /source/my-image.wic
   ```
 
-  **Note1**: `-b` option to create bmap file is not supported by omnect-cli docker image.<br>
-  **Note2**: The ssh tunnel option requires some additional settings. See [here](Usage-with-docker) for more details.<br>
-  **Note3**: The docker inject command is not supported by omnect-cli docker image.<br>.
+  **Note1**: The ssh tunnel option requires some additional settings. See [here](Usage-with-docker) for more details.<br>
+  **Note2**: The docker inject command is not supported by omnect-cli docker image.<br>.
 
 # Build from sources
 
@@ -184,6 +183,20 @@ omnect-cli file copy-to-image --help
 - File permissions: inject `systemd-tmpfiles.d`
 - Wifi: inject `wpa_supplicant-wlan0.conf`
 
+## Output image
+
+A command that changes an image writes the result to the input image. For a compressed image it writes the uncompressed image next to it, e.g. `my-image.wic` for `my-image.wic.xz`. With `-p` (`--pack-image`) it adds the extension of the compression (`.xz`, `.bz2` or `.gz`), e.g. `my-image.wic.xz` for `my-image.wic`. An image that already has an extension of that compression keeps its name. `-o` (`--output-image`) writes the result to the given path instead, and the input image stays unchanged. Its name must have the extension of the `-p` compression, or no compression extension without `-p`.
+
+The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. The output image is written to a new file in its directory and then renamed, so the directory needs write access and space for one more image. The new file belongs to the user who runs `omnect-cli`. `copy-from-image` only reads the image and writes nothing.
+
+## bmap files
+
+Commands that change an image create a bmap file with `-b` (`--output-bmap`). It is written next to the output image as `<output image>.bmap`, e.g. `my-image.wic.bmap` for `my-image.wic` and `my-image.wic.xz.bmap` for `my-image.wic.xz`, and replaces a file with this name. `bmaptool` finds it there without `--bmap`. The command fails if this file is the bmap of an image that stays, e.g. `my-image.wic.bmap` when `-i my-image.wic.xz` writes `my-image.wic`, also with `--input-bmap` or `--no-input-bmap`. Choose another output with `-o` then.
+
+The bmap of the input image is the base for it: `omnect-cli` takes the file given with `--input-bmap`, else the bmap next to the image that `bmaptool` would use: `<image>.bmap`, else the same with one extension of the image removed at a time, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it. The input bmap must have the size of the image and sha256 range checksums that match it, otherwise the command fails. A command without `-b` fails if such a bmap exists for the output image (e.g. `my-image.wic.bmap` for `my-image.wic.xz`), because `bmaptool` would use it and it would not match the image. So when you run several commands on one image, use `-b` on each of them. `--no-input-bmap` skips a bmap next to the image that does not match, and the created bmap then maps the whole image. The created bmap maps every range of the input bmap, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). It also maps every block that the image file stores as data, which includes the blocks `omnect-cli` changed. The image file itself stays sparse.
+
+Without an input bmap, the created bmap maps the whole image, and a flash writes every block.
+
 ## ssh tunnel
 
 ### Inject ssh tunnel credentials

project-context.md

@@ -50,15 +50,17 @@
 - `src/docker.rs` — `docker pull --platform` + `docker save` for multi-arch images
 - `src/image.rs` — firmware image architecture detection (ARM32/ARM64/x86_64)
 - `src/file/mod.rs` — high-level image operations: identity config, certs, hostname patching
-- `src/file/functions.rs` — partition read/modify/write via dd + e2cp/mcopy
+- `src/file/functions.rs` — sparse partition read, modify via e2cp/mcopy, write back of changed blocks only
+- `src/file/bmap.rs` — input bmap lookup and checksum check; writes the output bmap: input ranges (or the whole image) plus data blocks
+- `src/file/sparse.rs` — data/hole ranges via `SEEK_DATA`/`SEEK_HOLE`, sparse range copy
 - `src/file/partition.rs` — GPT/MBR partition table parsing
 - `src/file/compression.rs` — xz/bzip2/gzip compress/decompress with auto-detection
 - `src/validators/` — validation for identity config (TOML), device-update config (JSON), SSH keys
 - `conf/*.template` — 11 config templates (identity, device-update, WiFi)
 
 ## 4. Repository-Specific Constraints
 
-- External tools `e2cp`, `e2mkdir`, `mcopy`, `mmd`, `dd`, `fallocate`, `ssh-keygen`, `fdisk` must be available at runtime (Dockerfile copies them explicitly).
+- External tools `e2cp`, `e2mkdir`, `mcopy`, `mmd`, `ssh-keygen`, `fdisk` must be available at runtime (Dockerfile copies them explicitly).
 - Partition enum maps partition names to numbers differently for GPT vs MBR — see `file/functions.rs`.
 - OAuth2 callback binds to `127.0.0.1:4000` and `[::1]:4000`; container mode overrides to `0.0.0.0`.
 - `conf/` directory uses `.gitignore` to track only `*.template` files — actual configs are generated, never committed.

src/cli.rs

@@ -2,12 +2,35 @@ use crate::file::{
     compression::Compression,
     functions::{FileCopyFromParams, FileCopyToParams, Partition},
 };
-use clap::Parser;
+use clap::{Args, Parser};
 use std::path::PathBuf;
 use url::Url;
 
 const COPYRIGHT: &str = "Copyright © 2021 by conplement AG";
 
+#[derive(Args, Debug)]
+pub struct ImageOutput {
+    /// optional: create a bmap of the output image
+    #[arg(short = 'b', long = "output-bmap")]
+    pub output_bmap: bool,
+    /// optional: bmap of the input image; default: the bmap next to the image that bmaptool would use
+    #[arg(long = "input-bmap", requires = "output_bmap")]
+    pub input_bmap: Option<PathBuf>,
+    /// optional: do not use a bmap next to the image, so the bmap maps the whole image
+    #[arg(
+        long = "no-input-bmap",
+        requires = "output_bmap",
+        conflicts_with = "input_bmap"
+    )]
+    pub no_input_bmap: bool,
+    /// optional: write the output image to this path, with the --pack-image extension or no compression extension; default: the input image, without the compression extension, plus the --pack-image extension
+    #[arg(short = 'o', long = "output-image")]
+    pub output_image: Option<PathBuf>,
+    /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=')
+    #[arg(short = 'p', long = "pack-image", value_enum)]
+    pub compress_image: Option<Compression>,
+}
+
 // ToDo: command completion
 #[derive(Parser, Debug)]
 #[command(after_help = COPYRIGHT)]
@@ -34,12 +57,8 @@ pub enum Docker {
         /// destination path of the docker image in the firmware image (must end in ".tar.gz")
         #[clap(short = 'e', long = "dest")]
         dest: PathBuf,
-        /// optional: generate bmap file (currently not working in docker image)
-        #[arg(short = 'b', long = "generate-bmap-file")]
-        generate_bmap: bool,
-        /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=')
-        #[arg(short = 'p', long = "pack-image", value_enum)]
-        compress_image: Option<Compression>,
+        #[command(flatten)]
+        output: ImageOutput,
     },
 }
 
@@ -55,12 +74,8 @@ pub enum File {
         /// path to wic image file (optionally compressed with xz, bzip2 or gzip)
         #[arg(short = 'i', long = "image")]
         image: PathBuf,
-        /// optional: generate bmap file (currently not working in docker image)
-        #[arg(short = 'b', long = "generate-bmap-file")]
-        generate_bmap: bool,
-        /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=')
-        #[arg(short = 'p', long = "pack-image", value_enum)]
-        compress_image: Option<Compression>,
+        #[command(flatten)]
+        output: ImageOutput,
     },
     /// copy files from image
     CopyFromImage {
@@ -88,12 +103,8 @@ pub enum IdentityConfig {
         /// path to wic image file (optionally compressed with xz, bzip2 or gzip)
         #[arg(short = 'i', long = "image")]
         image: PathBuf,
-        /// optional: generate bmap file (currently not working in docker image)
-        #[arg(short = 'b', long = "generate-bmap-file")]
-        generate_bmap: bool,
-        /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=')
-        #[arg(short = 'p', long = "pack-image", value_enum)]
-        compress_image: Option<Compression>,
+        #[command(flatten)]
+        output: ImageOutput,
     },
     /// EXPERIMENTAL: set transparent gateway config.toml file and additional certificates and keys
     SetIotedgeGatewayConfig {
@@ -112,12 +123,8 @@ pub enum IdentityConfig {
         /// path to device identity certificate key file
         #[arg(short = 'k', long = "device_identity_key")]
         device_identity_key: PathBuf,
-        /// optional: generate bmap file (currently not working in docker image)
-        #[arg(short = 'b', long = "generate-bmap-file")]
-        generate_bmap: bool,
-        /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=')
-        #[arg(short = 'p', long = "pack-image", value_enum)]
-        compress_image: Option<Compression>,
+        #[command(flatten)]
+        output: ImageOutput,
     },
     /// EXPERIMENTAL: set leaf device config.toml file and additional certificate
     SetIotLeafSasConfig {
@@ -130,12 +137,8 @@ pub enum IdentityConfig {
         /// path to root ca certificate file
         #[arg(short = 'r', long = "root_ca")]
         root_ca: PathBuf,
-        /// optional: generate bmap file (currently not working in docker image)
-        #[arg(short = 'b', long = "generate-bmap-file")]
-        generate_bmap: bool,
-        /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=')
-        #[arg(short = 'p', long = "pack-image", value_enum)]
-        compress_image: Option<Compression>,
+        #[command(flatten)]
+        output: ImageOutput,
     },
     /// set certificates in order to support X.509 based DPS provisioning and certificate renewal via EST
     SetDeviceCertificate {
@@ -154,12 +157,8 @@ pub enum IdentityConfig {
         /// period of validity in days
         #[arg(short = 'D', long = "days")]
         days: u32,
-        /// optional: generate bmap file (currently not working in docker image)
-        #[arg(short = 'b', long = "generate-bmap-file")]
-        generate_bmap: bool,
-        /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=')
-        #[arg(short = 'p', long = "pack-image", value_enum)]
-        compress_image: Option<Compression>,
+        #[command(flatten)]
+        output: ImageOutput,
     },
     /// set certificates in order to support X.509 based DPS provisioning WITHOUT certificate renewal via EST
     SetDeviceCertificateNoEst {
@@ -172,12 +171,8 @@ pub enum IdentityConfig {
         /// path to wic image file (optionally compressed with xz, bzip2 or gzip)
         #[arg(short = 'i', long = "image")]
         image: PathBuf,
-        /// optional: generate bmap file (currently not working in docker image)
-        #[arg(short = 'b', long = "generate-bmap-file")]
-        generate_bmap: bool,
-        /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=')
-        #[arg(short = 'p', long = "pack-image", value_enum)]
-        compress_image: Option<Compression>,
+        #[command(flatten)]
+        output: ImageOutput,
     },
     /// generate and set bootstrap certificate for edge ca issuance/renewal.
     SetEdgeCaCertificate {
@@ -196,12 +191,8 @@ pub enum IdentityConfig {
         /// period of validity in days
         #[arg(short = 'D', long = "days")]
         days: u32,
-        /// optional: generate bmap file (currently not working in docker image)
-        #[arg(short = 'b', long = "generate-bmap-file")]
-        generate_bmap: bool,
-        /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=')
-        #[arg(short = 'p', long = "pack-image", value_enum)]
-        compress_image: Option<Compression>,
+        #[command(flatten)]
+        output: ImageOutput,
     },
 }
 
@@ -217,12 +208,8 @@ pub enum IotHubDeviceUpdate {
         /// path to wic image file (optionally compressed with xz, bzip2 or gzip)
         #[arg(short = 'i', long = "image")]
         image: PathBuf,
-        /// optional: generate bmap file (currently not working in docker image)
-        #[arg(short = 'b', long = "generate-bmap-file")]
-        generate_bmap: bool,
-        /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=')
-        #[arg(short = 'p', long = "pack-image", value_enum)]
-        compress_image: Option<Compression>,
+        #[command(flatten)]
+        output: ImageOutput,
     },
     /// import update to azure iot-hub
     ImportUpdate {
@@ -336,12 +323,8 @@ pub enum SshConfig {
         /// path to public key of the ssh root ca
         #[arg(short = 'r', long = "root_ca")]
         root_ca: PathBuf,
-        /// optional: generate bmap file (currently not working in docker image)
-        #[arg(short = 'b', long = "generate-bmap-file")]
-        generate_bmap: bool,
-        /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=')
-        #[arg(short = 'p', long = "pack-image", value_enum)]
-        compress_image: Option<Compression>,
+        #[command(flatten)]
+        output: ImageOutput,
     },
 
     /// set ssh connection parameters (see